Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 38 additions & 7 deletions .github/workflows/kotlin.yml
Original file line number Diff line number Diff line change
Expand Up @@ -610,32 +610,62 @@
report_paths: '**/build/test-results/*[tT]est/TEST-*.xml'
annotate_only: true # no write check perms on this workflow right now.

performance-tests:
name: Performance tests
macrobenchmarks:
name: Macrobenchmarks
runs-on: workflow-kotlin-test-runner-ubuntu-4core
timeout-minutes: 45
strategy:
# Allow tests to continue on other devices if they fail on one device.
fail-fast: false
matrix:
api-level:
- 31
steps:
- name: Checkout
uses: actions/checkout@v6

- name: Instrumented tests
uses: ./.github/actions/gradle-tasks-with-emulator
with:
tests-name: perf-tests-results
tests-name: macrobenchmarks-results
api-level: ${{ matrix.api-level }}
prepare-task: :benchmarks:performance-poetry:complex-poetry:prepareDebugAndroidTestArtifacts
test-task: :benchmarks:performance-poetry:complex-poetry:connectedCheck --continue
restore-cache-key: androidTest-build-artifacts
failure-path-upload: '**/build/reports/androidTests/connected'
failure-upload-name: 'perf-tests-report'
failure-upload-name: 'macrobenchmarks-report'

microbenchmarks:
name: Microbenchmarks
runs-on: workflow-kotlin-test-runner-ubuntu-4core
timeout-minutes: 45
strategy:
# Allow tests to continue on other devices if they fail on one device.
fail-fast: false
matrix:
api-level:
- 31
steps:
- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

- name: Instrumented tests
uses: ./.github/actions/gradle-tasks-with-emulator
with:
tests-name: microbenchmarks-results
api-level: ${{ matrix.api-level }}
# Microbenchmarks only run as release builds so we don't use the debug task like most
# builds.
prepare-task: :benchmarks:runtime-microbenchmark:prepareReleaseAndroidTestArtifacts -Pandroidx.benchmark.dryRunMode.enable=true -Pkotlin.native.ignoreDisabledTargets=true
# Run in dry run mode so they only run once, without warmup, since we don't care about measurements.
# See https://developer.android.com/topic/performance/benchmarking/microbenchmark-instrumentation-args#dryrunmode-enable
# Ignore iOS and other native targets being disabled since this is specifically and Android task.
test-task: :benchmarks:runtime-microbenchmark:connectedReleaseAndroidTest -Pandroidx.benchmark.dryRunMode.enable=true -Pkotlin.native.ignoreDisabledTargets=true --continue
restore-cache-key: androidTest-build-artifacts
failure-path-upload: 'benchmarks/runtime-microbenchmark/build/reports/androidTests/connected'
failure-upload-name: 'microbenchmarks-report'

instrumentation-tests:

Check warning

Code scanning / CodeQL

Workflow does not contain permissions Medium

Actions job or workflow does not limit the permissions of the GITHUB_TOKEN. Consider setting an explicit permissions block, using the following as a minimal starting point: {contents: read}
name: Instrumentation tests
runs-on: workflow-kotlin-test-runner-ubuntu-4core
timeout-minutes: 60
Expand All @@ -659,7 +689,7 @@
api-level: ${{ matrix.api-level }}
prepare-task: prepareConnectedCheckShard${{matrix.shardNum}}
# -x excludes tasks
test-task: connectedCheckShard${{matrix.shardNum}} -x :benchmarks:dungeon-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-poetry:connectedAndroidTest
test-task: connectedCheckShard${{matrix.shardNum}} -x :benchmarks:dungeon-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-poetry:connectedAndroidTest -x :benchmarks:runtime-microbenchmark:connectedAndroidTest
write-cache-key: androidTest-build-artifacts-${{matrix.shardNum}}
restore-cache-key: main-build-artifacts
failure-path-upload: '**/build/reports/androidTests/connected'
Expand Down Expand Up @@ -690,7 +720,7 @@
api-level: ${{ matrix.api-level }}
prepare-task: prepareConnectedCheckShard${{matrix.shardNum}} -Pworkflow.runtime=${{matrix.runtime}}
# -x excludes tasks
test-task: connectedCheckShard${{matrix.shardNum}} -Pworkflow.runtime=${{matrix.runtime}} -x :benchmarks:dungeon-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-poetry:connectedAndroidTest
test-task: connectedCheckShard${{matrix.shardNum}} -Pworkflow.runtime=${{matrix.runtime}} -x :benchmarks:dungeon-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-benchmark:connectedAndroidTest -x :benchmarks:performance-poetry:complex-poetry:connectedAndroidTest -x :benchmarks:runtime-microbenchmark:connectedAndroidTest
write-cache-key: androidTest-build-artifacts-${{matrix.shardNum}}-${{matrix.runtime}}
restore-cache-key: main-build-artifacts
failure-path-upload: '**/build/reports/androidTests/connected'
Expand Down Expand Up @@ -724,7 +754,8 @@
- unit-conflate-partial-drainExclusive-runtime-tests
- unit-all-runtime-tests
- ktlint
- performance-tests
- macrobenchmarks
- microbenchmarks
- runtime-instrumentation-tests
- shards-and-version
- tutorials
Expand Down
35 changes: 30 additions & 5 deletions benchmarks/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,18 @@ deterministic tests - such as those for render passes - can be run on any device
otherwise), but the benchmarks should be run on physical devices, or the closest approximation
to physical devices you can get.

## Baseline Profiles
There are two types of benchmarks:

- **Macro benchmarks** measure a particular flow in an entire app and are used to generate baseline
profiles. They're analogous to integration tests.
- **Micro benchmarks** measure a specific operation or set of small operations but running them
many times and collecting aggregate statistics. They're used to compare detailed performance of
competing implementations, detect specific performance regressions, and inform optimizations.
They're analogous to unit tests.

## Macro benchmarks

### Baseline Profiles

The sample apps can be used to extract 'baseline profiles' to improve code loading time after first
install. See [baseline profiles](https://developer.android.com/studio/profile/baselineprofiles).
Expand Down Expand Up @@ -36,7 +47,7 @@ This will create an output file separated by module and then also by package as
profile for each module can be added into its /src/main directory as `baseline-prof.txt`. Then on a
release build this will be included with the resulting APK/binary.

## dungeon-benchmark
### dungeon-benchmark

These are benchmarks for the [../samples/dungeon] app. Please instead use performance-poetry where
possible.
Expand All @@ -57,11 +68,11 @@ same scenario with and without forcing the use of the profiles. To force the use
`libs.androidx.profileinstaller` dependency is included into the app under profile (dungeon in this
case) for side-loading the profiles.

## performance-poetry
### performance-poetry

Module of code for performance testing related to poetry applications.

### complex-poetry
#### complex-poetry

This application is a modified version of the samples/containers/app-poetry app which also uses the
common components in samples/containers/common and samples/containers/poetry. It modifies this
Expand Down Expand Up @@ -92,9 +103,23 @@ This module also includes a [robots package](performance-poetry/complex-poetry/s
that provides some utility helper 'robots' for the UiAutomator [androidx.test.uiautomator.UiDevice]
as well as scenarios specific to this Complex Poetry application.

### complex-benchmark
#### complex-benchmark

This is an Android Test module which hosts an application that can run androidx.macrobenchmarks.
See the kdoc on [ComplexPoetryBenchmarks.](performance-poetry/complex-benchmark/src/main/java/com/squareup/benchmarks/performance/complex/poetry/benchmark/ComplexPoetryBenchmarks.kt)

The results for this are stored in the same folder at [ComplexPoetryResults.txt.](performance-poetry/complex-benchmark/src/main/java/com/squareup/benchmarks/performance/complex/poetry/benchmark/ComplexPoetryResults.txt)

## Micro benchmarks

### runtime-microbenchmark

This module contains benchmarks that measure the performance of the raw workflow runtime. To get
accurate results, these benchmarks must be ran on real physical devices, not emulators.

You can run them from the IDE like any other test (recommended), or with this command:
```bash
./gradlew :benchmarks:runtime-microbenchmark:connectedReleaseAndroidTest
```

We do not run benchmarks in CI, but we do validate that they compile by running them in "dry mode".
2 changes: 1 addition & 1 deletion benchmarks/dungeon-benchmark/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ kotlin {
}

dependencies {
implementation(libs.androidx.macro.benchmark)
implementation(libs.androidx.benchmark.macro)
implementation(libs.androidx.test.espresso.core)
implementation(libs.androidx.test.junit)
implementation(libs.androidx.test.uiautomator)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ kotlin {
}

dependencies {
implementation(libs.androidx.macro.benchmark)
implementation(libs.androidx.benchmark.macro)
implementation(libs.androidx.test.espresso.core)
implementation(libs.androidx.test.junit)
implementation(libs.androidx.test.uiautomator)
Expand Down
37 changes: 37 additions & 0 deletions benchmarks/runtime-microbenchmark/benchmark-proguard-rules.pro
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Add project specific ProGuard rules here.
# You can control the set of applied configuration files using the
# proguardFiles setting in build.gradle.
#
# For more details, see
# http://developer.android.com/guide/developing/tools/proguard.html

# If your project uses WebView with JS, uncomment the following
# and specify the fully qualified class name to the JavaScript interface
# class:
#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
# public *;
#}

# Uncomment this to preserve the line number information for
# debugging stack traces.
#-keepattributes SourceFile,LineNumberTable

# If you keep the line number information, uncomment this to
# hide the original source file name.
#-renamesourcefileattribute SourceFile

-dontobfuscate

-ignorewarnings

-keepattributes *Annotation*

-dontnote junit.framework.**
-dontnote junit.runner.**

-dontwarn androidx.test.**
-dontwarn org.junit.**
-dontwarn org.hamcrest.**
-dontwarn com.squareup.javawriter.JavaWriter

-keepclasseswithmembers @org.junit.runner.RunWith public class *
77 changes: 77 additions & 0 deletions benchmarks/runtime-microbenchmark/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
import com.rickbusarow.kgx.libsCatalog
import com.rickbusarow.kgx.version
import com.squareup.workflow1.buildsrc.internal.javaTarget
import com.squareup.workflow1.buildsrc.internal.javaTargetVersion

plugins {
// Must be applied before kotlin-android so the convention can detect that this is a benchmark.
alias(libs.plugins.androidx.benchmark)
id("com.android.library")
id("kotlin-android")
id("app.cash.burst")
}

// Note: We are not including our defaults from .buildscript as we do not need the base Workflow
// dependencies that those include.

android {
compileSdk = libsCatalog.version("compileSdk").toInt()

compileOptions {
sourceCompatibility = javaTargetVersion
targetCompatibility = javaTargetVersion
}

kotlinOptions {
jvmTarget = javaTarget
freeCompilerArgs += "-opt-in=kotlin.RequiresOptIn"
}

defaultConfig {
minSdk = 28
targetSdk = libsCatalog.version("targetSdk").toInt()

testInstrumentationRunner = "androidx.benchmark.junit4.AndroidBenchmarkRunner"
Comment thread
zach-klippenstein marked this conversation as resolved.

// This flag is supposed to enable dry run in the test runner, and it does so locally, but for
// some reason it doesn't seem to be working in CI.
Comment thread
zach-klippenstein marked this conversation as resolved.
val benchmarkDryRunEnabled = project.findProperty("androidx.benchmark.dryRunMode.enable")
if (benchmarkDryRunEnabled == "true") {
println("Running benchmarks in dry mode: emulator allowed, no measurements taken, no warmup.")
testInstrumentationRunnerArguments["androidx.benchmark.dryRunMode.enable"] = "true"
} else {
// must be one of: 'None', 'StackSampling', or 'MethodTracing'
testInstrumentationRunnerArguments["androidx.benchmark.profiling.mode"] = "MethodTracing"
testInstrumentationRunnerArguments["androidx.benchmark.output.enable"] = "true"
}
}

testBuildType = "release"
buildTypes {
debug {
// Since isDebuggable can"t be modified by gradle for library modules,
// it must be done in a manifest - see src/androidTest/AndroidManifest.xml
isMinifyEnabled = true
proguardFiles(
getDefaultProguardFile("proguard-android-optimize.txt"),
"benchmark-proguard-rules.pro"
)
}
release {
isDefault = true
}
}

namespace = "com.squareup.benchmark.runtime.benchmark"
testNamespace = "$namespace.test"
}

dependencies {
androidTestImplementation(project(":workflow-runtime"))
androidTestImplementation(libs.androidx.benchmark)
androidTestImplementation(libs.androidx.test.espresso.core)
androidTestImplementation(libs.androidx.test.junit)
androidTestImplementation(libs.androidx.test.uiautomator)
androidTestImplementation(libs.kotlin.test.jdk)
androidTestImplementation(libs.kotlinx.coroutines.test)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:tools="http://schemas.android.com/tools">

<!--
Important: disable debugging for accurate performance results

In a com.android.library project, this flag must be disabled from this
manifest, as it is not possible to override this flag from Gradle.
-->
<application
android:debuggable="false"
tools:ignore="HardcodedDebugMode"
tools:replace="android:debuggable">
<profileable android:shell="true"/>
</application>
</manifest>
Loading
Loading