diff --git a/.github/workflows/integration-tests-macrobenchmark.yml b/.github/workflows/integration-tests-macrobenchmark.yml new file mode 100644 index 00000000000..0ce486d83df --- /dev/null +++ b/.github/workflows/integration-tests-macrobenchmark.yml @@ -0,0 +1,71 @@ +name: 'Integration Tests - Macrobenchmark' +# Runs the sentry-uitest-android-macrobenchmark cold-start benchmark on a Sauce Labs real +# device and recovers timeToInitialDisplay from the device log. +# +on: + workflow_dispatch: + # Temporary scaffolding: workflow_dispatch cannot target a workflow that does not exist on + # the default branch yet, so trigger on pushes to this branch while we validate the flow. + # Remove before merge. + push: + branches: + - no/macrobenchmark-sauce-results + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + macrobenchmark: + name: Macrobenchmark + runs-on: ubuntu-latest + + # we copy the secret to the env variable in order to access it in the workflow + env: + SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} + + steps: + - name: Git checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: 'Set up Java: 17' + uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5 + with: + distribution: 'temurin' + java-version: '17' + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6.3.0 + + - name: Assemble target app and Macrobenchmark apk + if: env.SAUCE_USERNAME != null + run: ./gradlew :sentry-samples:sentry-samples-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android-macrobenchmark:assembleBenchmark + + - name: Run Macrobenchmark in SauceLab + uses: saucelabs/saucectl-run-action@283660aa934c02723c497efa151d582a3acc5801 # pin@v3 + if: env.SAUCE_USERNAME != null + env: + GITHUB_TOKEN: ${{ github.token }} + with: + sauce-username: ${{ secrets.SAUCE_USERNAME }} + sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} + config-file: .sauce/sentry-uitest-android-macrobenchmark.yml + + # Runs even when the suite fails: a failed benchmark still logs whatever it managed to + # measure, and the parser's own error explains what was missing. + - name: Recover benchmark results from the device log + if: always() && env.SAUCE_USERNAME != null + run: | + # Without pipefail the step passes on `tee`'s exit code, so a failed recovery + # would report success while silently producing no results. + set -o pipefail + python3 scripts/parse-macrobenchmark-log.py ./artifacts \ + --json-out ./artifacts/benchmarkData.json | tee -a "$GITHUB_STEP_SUMMARY" + + - name: Upload Sauce artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: always() + with: + name: macrobenchmark-results + path: ./artifacts/ + if-no-files-found: warn diff --git a/.sauce/sentry-uitest-android-macrobenchmark.yml b/.sauce/sentry-uitest-android-macrobenchmark.yml new file mode 100644 index 00000000000..4e3d5cfbb3d --- /dev/null +++ b/.sauce/sentry-uitest-android-macrobenchmark.yml @@ -0,0 +1,35 @@ +apiVersion: v1alpha +kind: espresso +sauce: + region: us-west-1 + concurrency: 1 + metadata: + build: sentry-uitest-android-macrobenchmark-$GITHUB_REF-$GITHUB_SHA + tags: + - benchmarks + - android + - macrobenchmark + +defaults: + timeout: 40m + +espresso: + app: ./sentry-samples/sentry-samples-android/build/outputs/apk/release/sentry-samples-android-release.apk + testApp: ./sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/build/outputs/apk/benchmark/sentry-uitest-android-macrobenchmark-benchmark.apk + +suites: + + - name: "Macrobenchmark startup (api 35)" + # No test orchestrator and no clearPackageData: Macrobenchmark manages its own process + # restarts and AOT compilation, and StartupMode.COLD intentionally keeps app data and + # permissions (it force-stops rather than `pm clear`). + devices: + - id: Google_Pixel_9_Pro_XL_15_real_sjc1 # Google Pixel 9 Pro XL - api 35 (15) - high end + +artifacts: + download: + when: always + match: + - junit.xml + - "*.log" + directory: ./artifacts/ diff --git a/scripts/parse-macrobenchmark-log.py b/scripts/parse-macrobenchmark-log.py new file mode 100755 index 00000000000..280970dbea8 --- /dev/null +++ b/scripts/parse-macrobenchmark-log.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Recover Macrobenchmark results from a Sauce Labs device log. + +Sauce Labs cannot pull arbitrary files off a real device, so +SentryStartupBenchmark echoes its `-benchmarkData.json` into logcat as +numbered chunks. This reassembles those chunks and prints a Markdown summary. + +Usage: + parse-macrobenchmark-log.py [--json-out benchmarkData.json] +""" + +import argparse +import json +import re +import sys +from pathlib import Path + +# Must match SentryStartupBenchmark.LOG_TAG and its "[index/total]" chunk prefix. +CHUNK_RE = re.compile(r"SentryBenchmarkData\s*:\s*\[(\d+)/(\d+)\](.*)$") + + +def log_messages(log_file): + """Yields the message text of every log entry. + + Sauce hands back device.log as JSON lines -- {"tag", "message", "level", ...} -- which + means the payload arrives with its quotes escaped, so it has to be decoded rather than + regexed out of the raw line. Plain-text lines are passed through unchanged so the same + parser works on `adb logcat` output from a local run. + """ + # Sauce device logs occasionally carry undecodable bytes; don't die on them. + for line in log_file.read_text(errors="replace").splitlines(): + line = line.strip() + if line.startswith("{"): + try: + yield json.loads(line).get("message", "") + continue + except json.JSONDecodeError: + pass + yield line + + +def collect_chunks(log_file): + """Returns one log's chunk texts keyed by index, plus the expected total.""" + chunks, total = {}, None + for message in log_messages(log_file): + match = CHUNK_RE.search(message) + if match: + index, total = int(match.group(1)), int(match.group(2)) + chunks[index] = match.group(3) + return chunks, total + + +def reassemble(chunks, total): + missing = [i for i in range(1, total + 1) if i not in chunks] + if missing: + sys.exit(f"Incomplete benchmark data: missing chunk(s) {missing} of {total}") + return "".join(chunks[i] for i in range(1, total + 1)) + + +def format_summary(data): + context = data["context"] + build = context["build"] + lines = [ + "## Macrobenchmark results", + "", + f"**Device:** {build['brand']} {build['model']} " + f"(api {build['version']['sdk']}, {context['cpuCoreCount']} cores) · " + f"**compilation:** {context['compilationMode']} · " + f"**CPU clocks locked:** {context['cpuLocked']}", + "", + ] + + if not context["cpuLocked"]: + lines += [ + "> CPU clocks are unlocked on this device, so run-to-run spread is wide. " + "Treat these numbers as a trend, not a regression gate.", + "", + ] + + table = [ + "| Benchmark | Metric | min | median | max | CoV | iterations |", + "|---|---|--:|--:|--:|--:|--:|", + ] + details = [] + for benchmark in data["benchmarks"]: + name = f"{benchmark['className'].rsplit('.', 1)[-1]}.{benchmark['name']}" + for metric, result in sorted(benchmark["metrics"].items()): + table.append( + f"| `{name}` | {metric} " + f"| {result['minimum']:.1f} | {result['median']:.1f} | {result['maximum']:.1f} " + f"| {result['coefficientOfVariation'] * 100:.1f}% | {len(result['runs'])} |" + ) + runs = ", ".join(f"{run:.1f}" for run in result["runs"]) + details += [ + "", + f"
{metric} per iteration", + "", + runs, + "", + "
", + ] + + return "\n".join(lines + table + details) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("artifacts_dir", type=Path, help="directory of downloaded Sauce artifacts") + parser.add_argument("--json-out", type=Path, help="where to write the recovered benchmarkData.json") + args = parser.parse_args() + + log_files = sorted(args.artifacts_dir.rglob("*.log")) + if not log_files: + sys.exit(f"No *.log files under {args.artifacts_dir}") + + # Keyed by file so chunks from two devices can never be merged into one bogus document. + per_log = {log: collect_chunks(log) for log in log_files} + with_chunks = {log: result for log, (result, total) in per_log.items() if total} + if not with_chunks: + sys.exit( + "No SentryBenchmarkData chunks in the device log. The benchmark most likely " + "failed before reporting — check junit.xml and the log for Macrobenchmark errors." + ) + if len(with_chunks) > 1: + sys.exit( + "Chunks from more than one run: " + + ", ".join(str(log) for log in with_chunks) + + ". This parser reports a single device." + ) + log_file = next(iter(with_chunks)) + chunks, total = per_log[log_file] + + data = json.loads(reassemble(chunks, total)) + + if args.json_out: + args.json_out.write_text(json.dumps(data, indent=2)) + + print(format_summary(data)) + + +if __name__ == "__main__": + main() diff --git a/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/README.md b/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/README.md index 4b89b7b6105..9914825584d 100644 --- a/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/README.md +++ b/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/README.md @@ -34,6 +34,34 @@ Connect a device, then: Results print to the console and are written to `build/outputs/connected_android_test_additional_output/.../*-benchmarkData.json`. +## Running on Sauce Labs + +The `Integration Tests - Macrobenchmark` workflow (manual trigger) runs the same benchmark on a +Sauce Labs real device. It reports numbers only — it is not a PR gate, because cloud devices run +with unlocked CPU clocks and the run-to-run spread swamps most SDK-init changes. + +Getting the numbers *back off* the device is the awkward part, so if you are changing this, know +what has already been ruled out: + +- **`artifacts.download.match` in `.sauce/*.yml` cannot reach the device.** It filters a + hardcoded list of assets Sauce hosts for the job (`device.log`, `junit.xml`, `video.mp4`, + `network.har`, `crash.json`, `screenshots.zip`); nothing enumerates device storage. A + `*-benchmarkData.json` pattern there matches nothing. +- **Macrobenchmark's own reporting channels don't survive.** The readable summary goes into the + instrumentation status bundle, which only Studio and AGP consume, and `benchmarkData.json` is + written to the app's external media dir, which Sauce never pulls. +- **The Real Device Access API can pull device files, but not on our account.** It offers + `pullFile` and `executeShellCommand`, and `GET /rdc/v2/devices/status` even lists the whole + public fleet — but `POST /sessions` answers `deviceClasses=[PRIVATE_DEVICE]` and there is no + parameter to request a public device. It would need leased private devices. That route would + also return the per-iteration perfetto traces, so it is worth revisiting if we ever get them. + +So `SentryStartupBenchmark` echoes its own `benchmarkData.json` into logcat in chunks, which +reaches CI inside `device.log`, and `scripts/parse-macrobenchmark-log.py` reassembles it and +writes a `timeToInitialDisplay` table to the job summary. Note this recovers the metrics only — +the perfetto traces are megabytes each and cannot go through logcat, so sub-millisecond work +still needs a local device. + ### Device hygiene (do this for trustworthy numbers) - **Wake and unlock the device first** — the launch check fails with "Unable to confirm activity diff --git a/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt b/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt index ee49fe8beff..824a2a0628f 100644 --- a/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt +++ b/sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt @@ -1,5 +1,6 @@ package io.sentry.uitest.android.macrobenchmark +import android.util.Log import androidx.benchmark.macro.CompilationMode import androidx.benchmark.macro.ExperimentalMetricApi import androidx.benchmark.macro.StartupMode @@ -7,6 +8,9 @@ import androidx.benchmark.macro.StartupTimingMetric import androidx.benchmark.macro.TraceSectionMetric import androidx.benchmark.macro.junit4.MacrobenchmarkRule import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import java.io.File +import org.junit.AfterClass import org.junit.Rule import org.junit.Test import org.junit.runner.RunWith @@ -23,10 +27,6 @@ import org.junit.runner.RunWith * [android.os.Trace] section the SDK emits, isolating SDK-init cost from the rest of the start. * * [CompilationMode.Full] pins ART AOT compilation so dexopt state does not drift between runs. - * Iterations are capped at 12: on an unthrottled Pixel 3, back-to-back cold starts hit thermal - * throttling after ~14 iterations, which inflates the tail of longer runs. This is NOT a CI test; - * it requires a connected device. To A/B an SDK change, see README.md (build the app twice, once - * per SDK variant, in interleaved rounds). */ @OptIn(ExperimentalMetricApi::class) @RunWith(AndroidJUnit4::class) @@ -47,10 +47,59 @@ class SentryStartupBenchmark { startActivityAndWait() } - private companion object { - const val TARGET_PACKAGE = "io.sentry.samples.android" + // Not private: @AfterClass needs a public static method. + companion object { + private const val TARGET_PACKAGE = "io.sentry.samples.android" // Matches the android.os.Trace section name in SentryAndroid.init. - const val INIT_TRACE_SECTION = "SentryAndroid.init" + private const val INIT_TRACE_SECTION = "SentryAndroid.init" + + private const val BENCHMARK_DATA_SUFFIX = "-benchmarkData.json" + + /** Kept in sync with `scripts/parse-macrobenchmark-log.py`. */ + private const val LOG_TAG = "SentryBenchmarkData" + + /** Well under logcat's ~4 KB per-message cap, so a chunk is never silently truncated. */ + private const val CHUNK_LENGTH = 2000 + + /** + * Echoes the benchmark results into logcat so CI can recover them. + * + * Macrobenchmark reports its numbers two ways, and on Sauce Labs neither one arrives: the + * human-readable summary goes into the instrumentation status bundle (which only Studio and AGP + * read), and `-benchmarkData.json` is written to the app's external media dir, which Sauce + * cannot pull — it only returns assets it produces itself, and logcat is one of them. + * + * Safe to run at this point because `ResultWriter` writes the file synchronously as each result + * is appended; only its *reporting* is deferred to the end of the run. Locally this is just + * extra logcat noise — Gradle still copies the real file into the build directory. + */ + @JvmStatic + @AfterClass + fun logBenchmarkDataToLogcat() { + val benchmarkData = findBenchmarkData() + if (benchmarkData == null) { + Log.w(LOG_TAG, "No *$BENCHMARK_DATA_SUFFIX found, cannot report results to CI") + return + } + + // Dropping the indentation makes the JSON compact enough to survive as a few logcat + // messages. Only structural whitespace is affected: JSON forbids raw newlines inside + // strings, so no value can span lines or start with the indentation being trimmed. + val compactJson = benchmarkData.readText().lineSequence().joinToString("") { it.trimStart() } + val chunks = compactJson.chunked(CHUNK_LENGTH) + chunks.forEachIndexed { index, chunk -> + Log.i(LOG_TAG, "[${index + 1}/${chunks.size}]$chunk") + } + } + + private fun findBenchmarkData(): File? { + val context = InstrumentationRegistry.getInstrumentation().targetContext + // This is where Macrobenchmark writes to for some reason. + @Suppress("DEPRECATION") val mediaDirs = context.externalMediaDirs.toList() + return (mediaDirs + context.externalCacheDir).filterNotNull().firstNotNullOfOrNull { dir -> + dir.listFiles()?.firstOrNull { it.name.endsWith(BENCHMARK_DATA_SUFFIX) } + } + } } }