diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4d4babc77e6..1f465c930f4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -690,6 +690,30 @@ jobs: for i in ${change_labels}; do echo "${i}=true" | tee -a $GITHUB_OUTPUT done + coveralls_done: + if: (always()) && (needs.classify_changes.outputs.no_code != 'true') + name: Coveralls Done + needs: + - test_python_linux_x86_64_0 + - test_python_linux_x86_64_1 + - test_python_linux_x86_64_2 + - test_python_linux_x86_64_3 + - test_python_linux_x86_64_4 + - test_python_linux_x86_64_5 + - test_python_linux_x86_64_6 + - test_python_linux_x86_64_7 + - test_python_linux_x86_64_8 + - test_python_linux_x86_64_9 + - test_python_linux_arm64 + - test_python_macos13_x86_64 + - classify_changes + runs-on: + - ubuntu-22.04 + steps: + - uses: coverallsapp/github-action@v2 + with: + fail-on-error: false + parallel-finished: true lint_python: if: (github.repository_owner == 'pantsbuild') && (needs.classify_changes.outputs.no_code != 'true') name: Lint Python and Shell @@ -793,6 +817,7 @@ jobs: - build_wheels_macos14_arm64 - check_release_notes - classify_changes + - coveralls_done - lint_python - test_python_linux_arm64 - test_python_linux_x86_64_0 @@ -873,6 +898,17 @@ jobs: name: logs-python-test-Linux-ARM64 overwrite: 'true' path: .pants.d/workdir/*.log + - continue-on-error: true + if: always() + name: Report coverage to coveralls.io + uses: coverallsapp/github-action@v2 + with: + allow-empty: true + fail-on-error: false + file: dist/coverage/python/coverage.xml + flag-name: test_python_linux_arm64 + format: cobertura + parallel: true timeout-minutes: 90 test_python_linux_x86_64_0: env: @@ -974,6 +1010,17 @@ jobs: name: logs-python-test-0_10-Linux-x86_64 overwrite: 'true' path: .pants.d/workdir/*.log + - continue-on-error: true + if: always() + name: Report coverage to coveralls.io + uses: coverallsapp/github-action@v2 + with: + allow-empty: true + fail-on-error: false + file: dist/coverage/python/coverage.xml + flag-name: test_python_linux_x86_64_0/10 + format: cobertura + parallel: true timeout-minutes: 90 test_python_linux_x86_64_1: env: @@ -1075,6 +1122,17 @@ jobs: name: logs-python-test-1_10-Linux-x86_64 overwrite: 'true' path: .pants.d/workdir/*.log + - continue-on-error: true + if: always() + name: Report coverage to coveralls.io + uses: coverallsapp/github-action@v2 + with: + allow-empty: true + fail-on-error: false + file: dist/coverage/python/coverage.xml + flag-name: test_python_linux_x86_64_1/10 + format: cobertura + parallel: true timeout-minutes: 90 test_python_linux_x86_64_2: env: @@ -1176,6 +1234,17 @@ jobs: name: logs-python-test-2_10-Linux-x86_64 overwrite: 'true' path: .pants.d/workdir/*.log + - continue-on-error: true + if: always() + name: Report coverage to coveralls.io + uses: coverallsapp/github-action@v2 + with: + allow-empty: true + fail-on-error: false + file: dist/coverage/python/coverage.xml + flag-name: test_python_linux_x86_64_2/10 + format: cobertura + parallel: true timeout-minutes: 90 test_python_linux_x86_64_3: env: @@ -1277,6 +1346,17 @@ jobs: name: logs-python-test-3_10-Linux-x86_64 overwrite: 'true' path: .pants.d/workdir/*.log + - continue-on-error: true + if: always() + name: Report coverage to coveralls.io + uses: coverallsapp/github-action@v2 + with: + allow-empty: true + fail-on-error: false + file: dist/coverage/python/coverage.xml + flag-name: test_python_linux_x86_64_3/10 + format: cobertura + parallel: true timeout-minutes: 90 test_python_linux_x86_64_4: env: @@ -1378,6 +1458,17 @@ jobs: name: logs-python-test-4_10-Linux-x86_64 overwrite: 'true' path: .pants.d/workdir/*.log + - continue-on-error: true + if: always() + name: Report coverage to coveralls.io + uses: coverallsapp/github-action@v2 + with: + allow-empty: true + fail-on-error: false + file: dist/coverage/python/coverage.xml + flag-name: test_python_linux_x86_64_4/10 + format: cobertura + parallel: true timeout-minutes: 90 test_python_linux_x86_64_5: env: @@ -1479,6 +1570,17 @@ jobs: name: logs-python-test-5_10-Linux-x86_64 overwrite: 'true' path: .pants.d/workdir/*.log + - continue-on-error: true + if: always() + name: Report coverage to coveralls.io + uses: coverallsapp/github-action@v2 + with: + allow-empty: true + fail-on-error: false + file: dist/coverage/python/coverage.xml + flag-name: test_python_linux_x86_64_5/10 + format: cobertura + parallel: true timeout-minutes: 90 test_python_linux_x86_64_6: env: @@ -1580,6 +1682,17 @@ jobs: name: logs-python-test-6_10-Linux-x86_64 overwrite: 'true' path: .pants.d/workdir/*.log + - continue-on-error: true + if: always() + name: Report coverage to coveralls.io + uses: coverallsapp/github-action@v2 + with: + allow-empty: true + fail-on-error: false + file: dist/coverage/python/coverage.xml + flag-name: test_python_linux_x86_64_6/10 + format: cobertura + parallel: true timeout-minutes: 90 test_python_linux_x86_64_7: env: @@ -1681,6 +1794,17 @@ jobs: name: logs-python-test-7_10-Linux-x86_64 overwrite: 'true' path: .pants.d/workdir/*.log + - continue-on-error: true + if: always() + name: Report coverage to coveralls.io + uses: coverallsapp/github-action@v2 + with: + allow-empty: true + fail-on-error: false + file: dist/coverage/python/coverage.xml + flag-name: test_python_linux_x86_64_7/10 + format: cobertura + parallel: true timeout-minutes: 90 test_python_linux_x86_64_8: env: @@ -1782,6 +1906,17 @@ jobs: name: logs-python-test-8_10-Linux-x86_64 overwrite: 'true' path: .pants.d/workdir/*.log + - continue-on-error: true + if: always() + name: Report coverage to coveralls.io + uses: coverallsapp/github-action@v2 + with: + allow-empty: true + fail-on-error: false + file: dist/coverage/python/coverage.xml + flag-name: test_python_linux_x86_64_8/10 + format: cobertura + parallel: true timeout-minutes: 90 test_python_linux_x86_64_9: env: @@ -1883,6 +2018,17 @@ jobs: name: logs-python-test-9_10-Linux-x86_64 overwrite: 'true' path: .pants.d/workdir/*.log + - continue-on-error: true + if: always() + name: Report coverage to coveralls.io + uses: coverallsapp/github-action@v2 + with: + allow-empty: true + fail-on-error: false + file: dist/coverage/python/coverage.xml + flag-name: test_python_linux_x86_64_9/10 + format: cobertura + parallel: true timeout-minutes: 90 test_python_macos13_x86_64: env: @@ -1953,6 +2099,17 @@ jobs: name: logs-python-test-macOS13-x86_64 overwrite: 'true' path: .pants.d/workdir/*.log + - continue-on-error: true + if: always() + name: Report coverage to coveralls.io + uses: coverallsapp/github-action@v2 + with: + allow-empty: true + fail-on-error: false + file: dist/coverage/python/coverage.xml + flag-name: test_python_macos13_x86_64 + format: cobertura + parallel: true timeout-minutes: 90 name: Pull Request CI 'on': diff --git a/pants.ci.toml b/pants.ci.toml index fe9ea624516..5bea72b5c4c 100644 --- a/pants.ci.toml +++ b/pants.ci.toml @@ -7,10 +7,14 @@ sandboxer = true [test] report = true attempts_default = 3 +use_coverage = true [pytest] args = ["--no-header", "--noskip", "-vv"] +[coverage-py] +report = ["raw", "xml"] + [subprocess-environment] env_vars.add = [ # Works around bad `-arch arm64` flag embedded in Xcode 12.x Python interpreters on intel diff --git a/pyproject.toml b/pyproject.toml index e56475355c5..f45b4ebf4fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,17 @@ known_first_party = ["internal_plugins", "pants", "pants_test"] asyncio_mode = "auto" markers = ["platform_specific_behavior", "call_by_type"] +[tool.coverage.run] +omit = [ + # pants/engine/internals tests + "preludes/bad.py", + # from testprojects/src/python/vcs_test/ + "tables.py", + # While BUILD files are Python, during testing we create/inject/munge and do + # all sorts of things with them that is hard to track + "BUILD", +] + [tool.coverage.report] # We inject this file at test time (see src/python/pants/conftest.py), and so # coverage will gather stats for it, but it doesn't correspond to a real source file, diff --git a/src/python/pants_release/generate_github_workflows.py b/src/python/pants_release/generate_github_workflows.py index 91d07cb4138..cfa480e0c14 100644 --- a/src/python/pants_release/generate_github_workflows.py +++ b/src/python/pants_release/generate_github_workflows.py @@ -25,6 +25,7 @@ def action(name: str) -> str: "attest-build-provenance": "actions/attest-build-provenance@v2", "cache": "actions/cache@v4", "checkout": "actions/checkout@v4", + "coverallsapp": "coverallsapp/github-action@v2", "download-artifact": "actions/download-artifact@v4", "github-action-required-labels": "mheap/github-action-required-labels@v4.0.0", "rust-cache": "benjyw/rust-cache@5ed697a6894712d2854c80635bb00a2496ea307a", @@ -53,6 +54,8 @@ def action(name: str) -> str: """ ) +TEST_PYTHON_JOB_PREFIX = "test_python" + Step = dict[str, Any] Jobs = dict[str, Any] @@ -430,8 +433,11 @@ def platform_name(self) -> str: def job_name_suffix(self) -> str: return self.platform_name().lower().replace("-", "_") - def job_name(self, prefix: str) -> str: - return f"{prefix}_{self.job_name_suffix()}" + def job_name(self, prefix: str, shard: str | None = None) -> str: + name = f"{prefix}_{self.job_name_suffix()}" + if shard: + name += f"_{shard}" + return name def runs_on(self) -> list[str]: # GHA strongly recommends targeting the self-hosted label as well as @@ -647,6 +653,22 @@ def upload_test_reports(self) -> Step: }, } + def coveralls_report(self, flag: str) -> Step: + return { + "name": "Report coverage to coveralls.io", + "uses": action("coverallsapp"), + "if": "always()", + "continue-on-error": True, + "with": { + "flag-name": flag, + "parallel": True, + "file": "dist/coverage/python/coverage.xml", + "format": "cobertura", + "allow-empty": True, + "fail-on-error": False, + }, + } + class RustTesting(Enum): NONE = "NONE" @@ -780,6 +802,7 @@ def test_jobs( }, helper.upload_test_reports(), helper.upload_log_artifacts(name=log_name), + helper.coveralls_report(flag=helper.job_name(TEST_PYTHON_JOB_PREFIX, shard)), ], } @@ -790,21 +813,20 @@ def linux_x86_64_test_jobs() -> Jobs: def test_python_linux(shard: str) -> dict[str, Any]: return test_jobs(helper, shard, platform_specific=False, with_remote_caching=True) - shard_name_prefix = helper.job_name("test_python") jobs = { helper.job_name("bootstrap_pants"): bootstrap_jobs( helper, validate_ci_config=True, rust_testing=RustTesting.ALL ), - f"{shard_name_prefix}_0": test_python_linux("0/10"), - f"{shard_name_prefix}_1": test_python_linux("1/10"), - f"{shard_name_prefix}_2": test_python_linux("2/10"), - f"{shard_name_prefix}_3": test_python_linux("3/10"), - f"{shard_name_prefix}_4": test_python_linux("4/10"), - f"{shard_name_prefix}_5": test_python_linux("5/10"), - f"{shard_name_prefix}_6": test_python_linux("6/10"), - f"{shard_name_prefix}_7": test_python_linux("7/10"), - f"{shard_name_prefix}_8": test_python_linux("8/10"), - f"{shard_name_prefix}_9": test_python_linux("9/10"), + helper.job_name(TEST_PYTHON_JOB_PREFIX, "0"): test_python_linux("0/10"), + helper.job_name(TEST_PYTHON_JOB_PREFIX, "1"): test_python_linux("1/10"), + helper.job_name(TEST_PYTHON_JOB_PREFIX, "2"): test_python_linux("2/10"), + helper.job_name(TEST_PYTHON_JOB_PREFIX, "3"): test_python_linux("3/10"), + helper.job_name(TEST_PYTHON_JOB_PREFIX, "4"): test_python_linux("4/10"), + helper.job_name(TEST_PYTHON_JOB_PREFIX, "5"): test_python_linux("5/10"), + helper.job_name(TEST_PYTHON_JOB_PREFIX, "6"): test_python_linux("6/10"), + helper.job_name(TEST_PYTHON_JOB_PREFIX, "7"): test_python_linux("7/10"), + helper.job_name(TEST_PYTHON_JOB_PREFIX, "8"): test_python_linux("8/10"), + helper.job_name(TEST_PYTHON_JOB_PREFIX, "9"): test_python_linux("9/10"), } return jobs @@ -819,7 +841,7 @@ def linux_arm64_test_jobs() -> Jobs: ), # We run these on a dedicated host with ample local cache, so remote caching # just adds cost but little value. - helper.job_name("test_python"): test_jobs( + helper.job_name(TEST_PYTHON_JOB_PREFIX): test_jobs( helper, shard=None, platform_specific=True, with_remote_caching=False ), } @@ -836,7 +858,7 @@ def macos13_x86_64_test_jobs() -> Jobs: ), # We run these on a dedicated host with ample local cache, so remote caching # just adds cost but little value. - helper.job_name("test_python"): test_jobs( + helper.job_name(TEST_PYTHON_JOB_PREFIX): test_jobs( helper, shard=None, platform_specific=True, with_remote_caching=False ), } @@ -1080,6 +1102,7 @@ def test_workflow_jobs() -> Jobs: }, } ) + jobs.update(coveralls_done([key for key in jobs.keys() if key.startswith("test_")])) return jobs @@ -1821,6 +1844,26 @@ def merge_ok(pr_jobs: list[str]) -> Jobs: } +def coveralls_done(test_job_keys: list[str]) -> Jobs: + return { + "coveralls_done": { + "name": "Coveralls Done", + "runs-on": Helper(Platform.LINUX_X86_64).runs_on(), + "if": "always()", + "needs": test_job_keys, + "steps": [ + { + "uses": action("coverallsapp"), + "with": { + "parallel-finished": True, + "fail-on-error": False, + }, + } + ], + } + } + + def generate() -> dict[Path, str]: """Generate all YAML configs with repo-relative paths."""