ci(macrobenchmark): Run the startup benchmark on Sauce Labs and recover its results

.github/workflows/integration-tests-macrobenchmark.yml

@@ -0,0 +1,71 @@
+name: 'Integration Tests - Macrobenchmark'
+# Runs the sentry-uitest-android-macrobenchmark cold-start benchmark on a Sauce Labs real
+# device and recovers timeToInitialDisplay from the device log.
+#
+on:
+  workflow_dispatch:
+  # Temporary scaffolding: workflow_dispatch cannot target a workflow that does not exist on
+  # the default branch yet, so trigger on pushes to this branch while we validate the flow.
+  # Remove before merge.
+  push:
+    branches:
+      - no/macrobenchmark-sauce-results
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  macrobenchmark:
+    name: Macrobenchmark
+    runs-on: ubuntu-latest
+
+    # we copy the secret to the env variable in order to access it in the workflow
+    env:
+      SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
+
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: 'Set up Java: 17'
+        uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@9c971963bec38e04b3d30dcc455b5382be2fdbfb # v6.3.0
+
+      - name: Assemble target app and Macrobenchmark apk
+        if: env.SAUCE_USERNAME != null
+        run: ./gradlew :sentry-samples:sentry-samples-android:assembleRelease :sentry-android-integration-tests:sentry-uitest-android-macrobenchmark:assembleBenchmark
+
+      - name: Run Macrobenchmark in SauceLab
+        uses: saucelabs/saucectl-run-action@283660aa934c02723c497efa151d582a3acc5801 # pin@v3
+        if: env.SAUCE_USERNAME != null
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          sauce-username: ${{ secrets.SAUCE_USERNAME }}
+          sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }}
+          config-file: .sauce/sentry-uitest-android-macrobenchmark.yml
+
+      # Runs even when the suite fails: a failed benchmark still logs whatever it managed to
+      # measure, and the parser's own error explains what was missing.
+      - name: Recover benchmark results from the device log
+        if: always() && env.SAUCE_USERNAME != null
+        run: |
+          # Without pipefail the step passes on `tee`'s exit code, so a failed recovery
+          # would report success while silently producing no results.
+          set -o pipefail
+          python3 scripts/parse-macrobenchmark-log.py ./artifacts \
+            --json-out ./artifacts/benchmarkData.json | tee -a "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload Sauce artifacts
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: macrobenchmark-results
+          path: ./artifacts/
+          if-no-files-found: warn

.sauce/sentry-uitest-android-macrobenchmark.yml

@@ -0,0 +1,35 @@
+apiVersion: v1alpha
+kind: espresso
+sauce:
+  region: us-west-1
+  concurrency: 1
+  metadata:
+    build: sentry-uitest-android-macrobenchmark-$GITHUB_REF-$GITHUB_SHA
+    tags:
+      - benchmarks
+      - android
+      - macrobenchmark
+
+defaults:
+  timeout: 40m
+
+espresso:
+  app: ./sentry-samples/sentry-samples-android/build/outputs/apk/release/sentry-samples-android-release.apk
+  testApp: ./sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/build/outputs/apk/benchmark/sentry-uitest-android-macrobenchmark-benchmark.apk
+
+suites:
+
+  - name: "Macrobenchmark startup (api 35)"
+    # No test orchestrator and no clearPackageData: Macrobenchmark manages its own process
+    # restarts and AOT compilation, and StartupMode.COLD intentionally keeps app data and
+    # permissions (it force-stops rather than `pm clear`).
+    devices:
+      - id: Google_Pixel_9_Pro_XL_15_real_sjc1 # Google Pixel 9 Pro XL - api 35 (15) - high end
+
+artifacts:
+  download:
+    when: always
+    match:
+      - junit.xml
+      - "*.log"
+    directory: ./artifacts/

scripts/parse-macrobenchmark-log.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+"""Recover Macrobenchmark results from a Sauce Labs device log.
+
+Sauce Labs cannot pull arbitrary files off a real device, so
+SentryStartupBenchmark echoes its `<pkg>-benchmarkData.json` into logcat as
+numbered chunks. This reassembles those chunks and prints a Markdown summary.
+
+Usage:
+    parse-macrobenchmark-log.py <artifacts-dir> [--json-out benchmarkData.json]
+"""
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+# Must match SentryStartupBenchmark.LOG_TAG and its "[index/total]" chunk prefix.
+CHUNK_RE = re.compile(r"SentryBenchmarkData\s*:\s*\[(\d+)/(\d+)\](.*)$")
+
+
+def log_messages(log_file):
+    """Yields the message text of every log entry.
+
+    Sauce hands back device.log as JSON lines -- {"tag", "message", "level", ...} -- which
+    means the payload arrives with its quotes escaped, so it has to be decoded rather than
+    regexed out of the raw line. Plain-text lines are passed through unchanged so the same
+    parser works on `adb logcat` output from a local run.
+    """
+    # Sauce device logs occasionally carry undecodable bytes; don't die on them.
+    for line in log_file.read_text(errors="replace").splitlines():
+        line = line.strip()
+        if line.startswith("{"):
+            try:
+                yield json.loads(line).get("message", "")
+                continue
+            except json.JSONDecodeError:
+                pass
+        yield line
+
+
+def collect_chunks(log_file):
+    """Returns one log's chunk texts keyed by index, plus the expected total."""
+    chunks, total = {}, None
+    for message in log_messages(log_file):
+        match = CHUNK_RE.search(message)
+        if match:
+            index, total = int(match.group(1)), int(match.group(2))
+            chunks[index] = match.group(3)
+    return chunks, total
+
+
+def reassemble(chunks, total):
+    missing = [i for i in range(1, total + 1) if i not in chunks]
+    if missing:
+        sys.exit(f"Incomplete benchmark data: missing chunk(s) {missing} of {total}")
+    return "".join(chunks[i] for i in range(1, total + 1))
+
+
+def format_summary(data):
+    context = data["context"]
+    build = context["build"]
+    lines = [
+        "## Macrobenchmark results",
+        "",
+        f"**Device:** {build['brand']} {build['model']} "
+        f"(api {build['version']['sdk']}, {context['cpuCoreCount']} cores) &middot; "
+        f"**compilation:** {context['compilationMode']} &middot; "
+        f"**CPU clocks locked:** {context['cpuLocked']}",
+        "",
+    ]
+
+    if not context["cpuLocked"]:
+        lines += [
+            "> CPU clocks are unlocked on this device, so run-to-run spread is wide. "
+            "Treat these numbers as a trend, not a regression gate.",
+            "",
+        ]
+
+    table = [
+        "| Benchmark | Metric | min | median | max | CoV | iterations |",
+        "|---|---|--:|--:|--:|--:|--:|",
+    ]
+    details = []
+    for benchmark in data["benchmarks"]:
+        name = f"{benchmark['className'].rsplit('.', 1)[-1]}.{benchmark['name']}"
+        for metric, result in sorted(benchmark["metrics"].items()):
+            table.append(
+                f"| `{name}` | {metric} "
+                f"| {result['minimum']:.1f} | {result['median']:.1f} | {result['maximum']:.1f} "
+                f"| {result['coefficientOfVariation'] * 100:.1f}% | {len(result['runs'])} |"
+            )
+            runs = ", ".join(f"{run:.1f}" for run in result["runs"])
+            details += [
+                "",
+                f"<details><summary>{metric} per iteration</summary>",
+                "",
+                runs,
+                "",
+                "</details>",
+            ]
+
+    return "\n".join(lines + table + details)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("artifacts_dir", type=Path, help="directory of downloaded Sauce artifacts")
+    parser.add_argument("--json-out", type=Path, help="where to write the recovered benchmarkData.json")
+    args = parser.parse_args()
+
+    log_files = sorted(args.artifacts_dir.rglob("*.log"))
+    if not log_files:
+        sys.exit(f"No *.log files under {args.artifacts_dir}")
+
+    # Keyed by file so chunks from two devices can never be merged into one bogus document.
+    per_log = {log: collect_chunks(log) for log in log_files}
+    with_chunks = {log: result for log, (result, total) in per_log.items() if total}
+    if not with_chunks:
+        sys.exit(
+            "No SentryBenchmarkData chunks in the device log. The benchmark most likely "
+            "failed before reporting — check junit.xml and the log for Macrobenchmark errors."
+        )
+    if len(with_chunks) > 1:
+        sys.exit(
+            "Chunks from more than one run: "
+            + ", ".join(str(log) for log in with_chunks)
+            + ". This parser reports a single device."
+        )
+    log_file = next(iter(with_chunks))
+    chunks, total = per_log[log_file]
+
+    data = json.loads(reassemble(chunks, total))
+
+    if args.json_out:
+        args.json_out.write_text(json.dumps(data, indent=2))
+
+    print(format_summary(data))
+
+
+if __name__ == "__main__":
+    main()

sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/README.md

@@ -34,6 +34,34 @@ Connect a device, then:
 Results print to the console and are written to
 `build/outputs/connected_android_test_additional_output/.../*-benchmarkData.json`.
 
+## Running on Sauce Labs
+
+The `Integration Tests - Macrobenchmark` workflow (manual trigger) runs the same benchmark on a
+Sauce Labs real device. It reports numbers only — it is not a PR gate, because cloud devices run
+with unlocked CPU clocks and the run-to-run spread swamps most SDK-init changes.
+
+Getting the numbers *back off* the device is the awkward part, so if you are changing this, know
+what has already been ruled out:
+
+- **`artifacts.download.match` in `.sauce/*.yml` cannot reach the device.** It filters a
+  hardcoded list of assets Sauce hosts for the job (`device.log`, `junit.xml`, `video.mp4`,
+  `network.har`, `crash.json`, `screenshots.zip`); nothing enumerates device storage. A
+  `*-benchmarkData.json` pattern there matches nothing.
+- **Macrobenchmark's own reporting channels don't survive.** The readable summary goes into the
+  instrumentation status bundle, which only Studio and AGP consume, and `benchmarkData.json` is
+  written to the app's external media dir, which Sauce never pulls.
+- **The Real Device Access API can pull device files, but not on our account.** It offers
+  `pullFile` and `executeShellCommand`, and `GET /rdc/v2/devices/status` even lists the whole
+  public fleet — but `POST /sessions` answers `deviceClasses=[PRIVATE_DEVICE]` and there is no
+  parameter to request a public device. It would need leased private devices. That route would
+  also return the per-iteration perfetto traces, so it is worth revisiting if we ever get them.
+
+So `SentryStartupBenchmark` echoes its own `benchmarkData.json` into logcat in chunks, which
+reaches CI inside `device.log`, and `scripts/parse-macrobenchmark-log.py` reassembles it and
+writes a `timeToInitialDisplay` table to the job summary. Note this recovers the metrics only —
+the perfetto traces are megabytes each and cannot go through logcat, so sub-millisecond work
+still needs a local device.
+
 ### Device hygiene (do this for trustworthy numbers)
 
 - **Wake and unlock the device first** — the launch check fails with "Unable to confirm activity

sentry-android-integration-tests/sentry-uitest-android-macrobenchmark/src/main/java/io/sentry/uitest/android/macrobenchmark/SentryStartupBenchmark.kt

@@ -1,12 +1,16 @@
 package io.sentry.uitest.android.macrobenchmark
 
+import android.util.Log
 import androidx.benchmark.macro.CompilationMode
 import androidx.benchmark.macro.ExperimentalMetricApi
 import androidx.benchmark.macro.StartupMode
 import androidx.benchmark.macro.StartupTimingMetric
 import androidx.benchmark.macro.TraceSectionMetric
 import androidx.benchmark.macro.junit4.MacrobenchmarkRule
 import androidx.test.ext.junit.runners.AndroidJUnit4
+import androidx.test.platform.app.InstrumentationRegistry
+import java.io.File
+import org.junit.AfterClass
 import org.junit.Rule
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -23,10 +27,6 @@ import org.junit.runner.RunWith
  *   [android.os.Trace] section the SDK emits, isolating SDK-init cost from the rest of the start.
  *
  * [CompilationMode.Full] pins ART AOT compilation so dexopt state does not drift between runs.
- * Iterations are capped at 12: on an unthrottled Pixel 3, back-to-back cold starts hit thermal
- * throttling after ~14 iterations, which inflates the tail of longer runs. This is NOT a CI test;
- * it requires a connected device. To A/B an SDK change, see README.md (build the app twice, once
- * per SDK variant, in interleaved rounds).
  */
 @OptIn(ExperimentalMetricApi::class)
 @RunWith(AndroidJUnit4::class)
@@ -47,10 +47,59 @@ class SentryStartupBenchmark {
       startActivityAndWait()
     }
 
-  private companion object {
-    const val TARGET_PACKAGE = "io.sentry.samples.android"
+  // Not private: @AfterClass needs a public static method.
+  companion object {
+    private const val TARGET_PACKAGE = "io.sentry.samples.android"
 
     // Matches the android.os.Trace section name in SentryAndroid.init.
-    const val INIT_TRACE_SECTION = "SentryAndroid.init"
+    private const val INIT_TRACE_SECTION = "SentryAndroid.init"
+
+    private const val BENCHMARK_DATA_SUFFIX = "-benchmarkData.json"
+
+    /** Kept in sync with `scripts/parse-macrobenchmark-log.py`. */
+    private const val LOG_TAG = "SentryBenchmarkData"
+
+    /** Well under logcat's ~4 KB per-message cap, so a chunk is never silently truncated. */
+    private const val CHUNK_LENGTH = 2000
+
+    /**
+     * Echoes the benchmark results into logcat so CI can recover them.
+     *
+     * Macrobenchmark reports its numbers two ways, and on Sauce Labs neither one arrives: the
+     * human-readable summary goes into the instrumentation status bundle (which only Studio and AGP
+     * read), and `<pkg>-benchmarkData.json` is written to the app's external media dir, which Sauce
+     * cannot pull — it only returns assets it produces itself, and logcat is one of them.
+     *
+     * Safe to run at this point because `ResultWriter` writes the file synchronously as each result
+     * is appended; only its *reporting* is deferred to the end of the run. Locally this is just
+     * extra logcat noise — Gradle still copies the real file into the build directory.
+     */
+    @JvmStatic
+    @AfterClass
+    fun logBenchmarkDataToLogcat() {
+      val benchmarkData = findBenchmarkData()
+      if (benchmarkData == null) {
+        Log.w(LOG_TAG, "No *$BENCHMARK_DATA_SUFFIX found, cannot report results to CI")
+        return
+      }
+
+      // Dropping the indentation makes the JSON compact enough to survive as a few logcat
+      // messages. Only structural whitespace is affected: JSON forbids raw newlines inside
+      // strings, so no value can span lines or start with the indentation being trimmed.
+      val compactJson = benchmarkData.readText().lineSequence().joinToString("") { it.trimStart() }
+      val chunks = compactJson.chunked(CHUNK_LENGTH)
+      chunks.forEachIndexed { index, chunk ->
+        Log.i(LOG_TAG, "[${index + 1}/${chunks.size}]$chunk")
+      }
+    }
+
+    private fun findBenchmarkData(): File? {
+      val context = InstrumentationRegistry.getInstrumentation().targetContext
+      // This is where Macrobenchmark writes to for some reason.
+      @Suppress("DEPRECATION") val mediaDirs = context.externalMediaDirs.toList()
+      return (mediaDirs + context.externalCacheDir).filterNotNull().firstNotNullOfOrNull { dir ->
+        dir.listFiles()?.firstOrNull { it.name.endsWith(BENCHMARK_DATA_SUFFIX) }
+      }
+    }
   }
 }