From eb280829d94645918fe58ecb1ee78951c5f9c570 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Mon, 13 Apr 2026 23:02:46 -0700 Subject: [PATCH 01/15] Add weaver live check test util helper --- .github/workflows/templates/test.yml.j2 | 9 + .github/workflows/test.yml | 43 +++ tests/opentelemetry-test-utils/pyproject.toml | 1 + .../opentelemetry/test/weaver_live_check.py | 307 ++++++++++++++++++ .../test-requirements.txt | 3 + .../tests/test_weaver_live_check.py | 73 +++++ .../tests/testdata/forbid_service_name.rego | 13 + uv.lock | 2 + 8 files changed, 451 insertions(+) create mode 100644 tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py create mode 100644 tests/opentelemetry-test-utils/tests/test_weaver_live_check.py create mode 100644 tests/opentelemetry-test-utils/tests/testdata/forbid_service_name.rego diff --git a/.github/workflows/templates/test.yml.j2 b/.github/workflows/templates/test.yml.j2 index 8ca54829928..af5b61b9437 100644 --- a/.github/workflows/templates/test.yml.j2 +++ b/.github/workflows/templates/test.yml.j2 @@ -20,6 +20,7 @@ env: contains(github.event.pull_request.labels.*.name, 'backport') && github.event.pull_request.base.ref || 'main' ) || 'main' }}{% endraw %} + WEAVER_VERSION: "v0.22.1" PIP_EXISTS_ACTION: w CONTRIB_REPO_UTIL_HTTP: ${% raw %}{{ github.workspace }}{% endraw %}/opentelemetry-python-contrib/util/opentelemetry-util-http CONTRIB_REPO_INSTRUMENTATION: ${% raw %}{{ github.workspace }}{% endraw %}/opentelemetry-python-contrib/opentelemetry-instrumentation @@ -51,6 +52,14 @@ jobs: ref: ${% raw %}{{ env.CONTRIB_REPO_SHA }}{% endraw %} path: opentelemetry-python-contrib {%- endif %} + {%- if "test-opentelemetry-test-utils" in job_data.tox_env and job_data.os != "windows-latest" %} + + - name: Install weaver + run: | + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${% raw %}{{ env.WEAVER_VERSION }}{% endraw %}/weaver_Linux_x86_64.tar.gz" + curl -sSL "$WEAVER_URL" | tar -xz -C /tmp weaver + sudo mv /tmp/weaver /usr/local/bin/weaver + {%- endif %} - name: Set up Python {{ job_data.python_version }} uses: actions/setup-python@v5 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1c09aa0e5dd..ff7b6870186 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,6 +20,7 @@ env: contains(github.event.pull_request.labels.*.name, 'backport') && github.event.pull_request.base.ref || 'main' ) || 'main' }} + WEAVER_VERSION: "v0.22.1" PIP_EXISTS_ACTION: w CONTRIB_REPO_UTIL_HTTP: ${{ github.workspace }}/opentelemetry-python-contrib/util/opentelemetry-util-http CONTRIB_REPO_INSTRUMENTATION: ${{ github.workspace }}/opentelemetry-python-contrib/opentelemetry-instrumentation @@ -2865,6 +2866,12 @@ jobs: - name: Checkout repo @ SHA - ${{ github.sha }} uses: actions/checkout@v4 + - name: Install weaver + run: | + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver_Linux_x86_64.tar.gz" + curl -sSL "$WEAVER_URL" | tar -xz -C /tmp weaver + sudo mv /tmp/weaver /usr/local/bin/weaver + - name: Set up Python 3.10 uses: actions/setup-python@v5 with: @@ -2884,6 +2891,12 @@ jobs: - name: Checkout repo @ SHA - ${{ github.sha }} uses: actions/checkout@v4 + - name: Install weaver + run: | + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver_Linux_x86_64.tar.gz" + curl -sSL "$WEAVER_URL" | tar -xz -C /tmp weaver + sudo mv /tmp/weaver /usr/local/bin/weaver + - name: Set up Python 3.11 uses: actions/setup-python@v5 with: @@ -2903,6 +2916,12 @@ jobs: - name: Checkout repo @ SHA - ${{ github.sha }} uses: actions/checkout@v4 + - name: Install weaver + run: | + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver_Linux_x86_64.tar.gz" + curl -sSL "$WEAVER_URL" | tar -xz -C /tmp weaver + sudo mv /tmp/weaver /usr/local/bin/weaver + - name: Set up Python 3.12 uses: actions/setup-python@v5 with: @@ -2922,6 +2941,12 @@ jobs: - name: Checkout repo @ SHA - ${{ github.sha }} uses: actions/checkout@v4 + - name: Install weaver + run: | + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver_Linux_x86_64.tar.gz" + curl -sSL "$WEAVER_URL" | tar -xz -C /tmp weaver + sudo mv /tmp/weaver /usr/local/bin/weaver + - name: Set up Python 3.13 uses: actions/setup-python@v5 with: @@ -2941,6 +2966,12 @@ jobs: - name: Checkout repo @ SHA - ${{ github.sha }} uses: actions/checkout@v4 + - name: Install weaver + run: | + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver_Linux_x86_64.tar.gz" + curl -sSL "$WEAVER_URL" | tar -xz -C /tmp weaver + sudo mv /tmp/weaver /usr/local/bin/weaver + - name: Set up Python 3.14 uses: actions/setup-python@v5 with: @@ -2960,6 +2991,12 @@ jobs: - name: Checkout repo @ SHA - ${{ github.sha }} uses: actions/checkout@v4 + - name: Install weaver + run: | + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver_Linux_x86_64.tar.gz" + curl -sSL "$WEAVER_URL" | tar -xz -C /tmp weaver + sudo mv /tmp/weaver /usr/local/bin/weaver + - name: Set up Python 3.14t uses: actions/setup-python@v5 with: @@ -2979,6 +3016,12 @@ jobs: - name: Checkout repo @ SHA - ${{ github.sha }} uses: actions/checkout@v4 + - name: Install weaver + run: | + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver_Linux_x86_64.tar.gz" + curl -sSL "$WEAVER_URL" | tar -xz -C /tmp weaver + sudo mv /tmp/weaver /usr/local/bin/weaver + - name: Set up Python pypy-3.10 uses: actions/setup-python@v5 with: diff --git a/tests/opentelemetry-test-utils/pyproject.toml b/tests/opentelemetry-test-utils/pyproject.toml index 974e1972089..a273592c771 100644 --- a/tests/opentelemetry-test-utils/pyproject.toml +++ b/tests/opentelemetry-test-utils/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "asgiref ~= 3.0", "opentelemetry-api == 1.42.0.dev", "opentelemetry-sdk == 1.42.0.dev", + "requests ~= 2.28", ] [project.urls] diff --git a/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py b/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py new file mode 100644 index 00000000000..43d53c336e0 --- /dev/null +++ b/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py @@ -0,0 +1,307 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import os +import shutil +import socket +import subprocess +import time +from collections import defaultdict +from typing import Any + +from requests import get, post +from requests.exceptions import ConnectionError as ReqConnectionError + +from opentelemetry.semconv.schemas import Schemas + +logger = logging.getLogger(__name__) + + +def _find_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + + +def _extract_violations(report: dict) -> list: + """Extract and deduplicate violations from the full report. + + Get all violations using python version of this jq filter: + [ .. | objects | select(has("live_check_result")) + | .live_check_result.all_advice[]? + | select(.level == "violation") ] + | group_by(.id, .message, .context, .signal_name, .signal_type) + | map({ ..., count: length }) + | sort_by(-.count, .message) + """ + raw: list[dict] = [] + + def collect(obj: Any) -> None: + if isinstance(obj, dict): + if "live_check_result" in obj: + for advice in obj["live_check_result"].get("all_advice", []): + if advice.get("level") == "violation": + raw.append(advice) + for v in obj.values(): + collect(v) + elif isinstance(obj, list): + for item in obj: + collect(item) + + collect(report) + + groups: dict[tuple, list] = defaultdict(list) + for v in raw: + ctx = v.get("context") + key = ( + v.get("id"), + v.get("message"), + json.dumps(ctx, sort_keys=True) + if isinstance(ctx, (dict, list)) + else ctx, + v.get("signal_name"), + v.get("signal_type"), + ) + groups[key].append(v) + + violations = [ + { + "id": k[0], + "message": k[1], + "context": k[2], + "signal_name": k[3], + "signal_type": k[4], + "count": len(vs), + } + for k, vs in groups.items() + ] + violations.sort(key=lambda v: (-v["count"], v.get("message") or "")) + return violations + + +def _format_violations(violations: list) -> str: + """Format violations list as human-readable text (mirrors violations.j2 output).""" + lines = [] + for v in violations: + signal = "" + signal_type = v.get("signal_type") + signal_name = v.get("signal_name") + if signal_type and signal_name: + signal = f" on {signal_type} '{signal_name}'" + elif signal_type: + signal = f" on {signal_type}" + elif signal_name: + signal = f" on '{signal_name}'" + lines.append( + f"- [{v.get('id')}] {v.get('message')} ({v['count']} occurrence(s){signal})" + ) + return "\n".join(lines) + + +class WeaverLiveCheck: + """Runs ``weaver registry live-check`` as a subprocess and validates + OTLP telemetry against OpenTelemetry semantic conventions. + + Requires the ``weaver`` binary on PATH: + https://github.com/open-telemetry/weaver/releases + + Typical use as a context manager:: + + def test_my_telemetry(self): + with WeaverLiveCheck() as weaver: + exporter = OTLPSpanExporter( + endpoint=weaver.otlp_endpoint, insecure=True + ) + # ... configure provider, emit telemetry ... + provider.force_flush() + + # Signals weaver to stop, raises AssertionError listing violations + # if any, or returns the raw JSON report on success. + report = weaver.end_and_check() + # __exit__ calls close(), which is idempotent if end_and_check() was already called + + :meth:`end_and_check` returns the raw weaver JSON report when weaver exits + successfully (exit code 0). Use it for custom assertions on the report + content beyond the built-in violation check:: + + report = weaver.end_and_check() + # report is the raw JSON dict from weaver; inspect it as needed, e.g.: + self.assertIn("some_signal", str(report)) + + Lifecycle: + - :meth:`start` — launches weaver and waits for it to become ready. + - :attr:`otlp_endpoint` — gRPC OTLP endpoint to point exporters at. + - :meth:`end_and_check` — signals weaver to stop, collects the report, and + raises :class:`AssertionError` with a human-readable violation list if weaver + exits non-zero. Returns the raw report dict on success. + - :meth:`close` — calls :meth:`end_and_check` then terminates the process. + Idempotent; safe to call even if :meth:`end_and_check` was already called. + """ + + def __init__( + self, + schema_version: str | None = None, + policies_dir: str | None = None, + inactivity_timeout: int = 30, + otlp_port: int = 0, + admin_port: int = 0, + ): + weaver_bin = shutil.which("weaver") + if not weaver_bin: + raise RuntimeError( + "weaver binary not found on PATH. " + "Install it from https://github.com/open-telemetry/weaver/releases" + ) + + self._otlp_port = otlp_port or _find_free_port() + self._admin_port = admin_port or _find_free_port() + self._ready = False + self._stopped = False + self._process: subprocess.Popen[bytes] | None = None + + command = [ + weaver_bin, + "registry", + "live-check", + f"--inactivity-timeout={inactivity_timeout}", + f"--otlp-grpc-port={self._otlp_port}", + f"--admin-port={self._admin_port}", + "--output=http", + "--format=json", + ] + + if policies_dir: + command += ["--advice-policies", os.path.abspath(policies_dir)] + + if schema_version is None: + schema_version = list(Schemas)[-1].value.rsplit("/", 1)[-1] + + command += [ + "--registry", + f"https://github.com/open-telemetry/semantic-conventions/archive/refs/tags/v{schema_version}.tar.gz[model]", + ] + + self._command = command + logger.debug("Weaver command: %s", command) + + def __enter__(self) -> "WeaverLiveCheck": + return self.start() + + def __exit__(self, exc_type: Any, *_: Any) -> None: + if exc_type is not None: + self._stopped = True + self.close() + + def start(self, timeout: int = 60) -> "WeaverLiveCheck": + logger.debug("Starting WeaverLiveCheck process...") + self._process = subprocess.Popen( + self._command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + try: + self._wait_for_ready(timeout=timeout) + self._ready = True + except Exception as e: + logs = self._read_weaver_logs() + logger.error( + "WeaverLiveCheck did not start: %s, logs: %s", e, logs + ) + raise + return self + + def _wait_for_ready(self, timeout: int = 60) -> None: + for i in range(timeout): + if self._process is not None and self._process.poll() is not None: + raise RuntimeError( + f"WeaverLiveCheck process exited unexpectedly (code {self._process.returncode})" + ) + try: + response = get( + f"http://localhost:{self._admin_port}/health", timeout=5 + ) + if response.status_code == 200: + return + logger.debug( + "Health check returned %s, try %s", response.status_code, i + ) + except ReqConnectionError as e: + logger.debug("Health check connection error: %s", e) + time.sleep(1) + raise TimeoutError("WeaverLiveCheck did not become ready in time") + + @property + def otlp_endpoint(self) -> str: + return f"http://localhost:{self._otlp_port}" + + def end_and_check(self, timeout: int = 30) -> dict[str, Any]: + if self._stopped: + logger.warning( + "end_and_check() called after weaver already stopped; returning empty report" + ) + return {} + self._stopped = True + + if not self._ready: + raise RuntimeError( + "WeaverLiveCheck process did not start successfully" + ) + + try: + response = post( + f"http://localhost:{self._admin_port}/stop", timeout=5 + ) + response.raise_for_status() + report = response.json() + assert self._process is not None + exit_code = self._process.wait(timeout=timeout) + except Exception as e: + logs = self._read_weaver_logs() + logger.error( + "Error communicating with weaver: %s, logs: %s", e, logs + ) + raise + + if exit_code == 0: + return report + + violations = _extract_violations(report) + raise AssertionError( + f"Semconv violations found:\n{_format_violations(violations)}" + ) + + def _read_weaver_logs(self) -> str | None: + if self._process is None: + return None + try: + if self._process.poll() is None: + self._process.kill() + out, err = self._process.communicate() + return f"{out.decode()}\n{err.decode()}" + except Exception as e: + logger.error("Could not get weaver logs: %s", e) + return None + + def close(self) -> None: + try: + self.end_and_check() + finally: + if self._process and self._process.poll() is None: + self._process.terminate() + try: + self._process.wait(timeout=5) + except subprocess.TimeoutExpired: + self._process.kill() diff --git a/tests/opentelemetry-test-utils/test-requirements.txt b/tests/opentelemetry-test-utils/test-requirements.txt index 854e7327124..6bc97e344a1 100644 --- a/tests/opentelemetry-test-utils/test-requirements.txt +++ b/tests/opentelemetry-test-utils/test-requirements.txt @@ -13,3 +13,6 @@ zipp==3.19.2 -e opentelemetry-sdk -e opentelemetry-semantic-conventions -e tests/opentelemetry-test-utils +-e opentelemetry-proto +-e exporter/opentelemetry-exporter-otlp-proto-common +-e exporter/opentelemetry-exporter-otlp-proto-grpc diff --git a/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py b/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py new file mode 100644 index 00000000000..6c5332c6d9f --- /dev/null +++ b/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py @@ -0,0 +1,73 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Live-check tests using Weaver to validate SDK telemetry against semconv. + +Requires the `weaver` binary on PATH: + https://github.com/open-telemetry/weaver/releases +""" + +import os +import shutil +import unittest + +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, +) +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.test.weaver_live_check import WeaverLiveCheck + +_TESTDATA_DIR = os.path.join(os.path.dirname(__file__), "testdata") + + +def _make_provider(otlp_endpoint: str) -> TracerProvider: + resource = Resource.create({SERVICE_NAME: "test-service"}) + exporter = OTLPSpanExporter(endpoint=otlp_endpoint, insecure=True) + provider = TracerProvider(resource=resource) + provider.add_span_processor(BatchSpanProcessor(exporter)) + return provider + + +@unittest.skipUnless( + shutil.which("weaver") is not None, + "weaver binary not found on PATH — install from https://github.com/open-telemetry/weaver/releases", +) +class TestSDKInitLiveCheck(unittest.TestCase): + def test_sdk_resource_with_service_name(self): + """SDK initialized with service.name emits conformant telemetry.""" + with WeaverLiveCheck() as weaver: + provider = _make_provider(weaver.otlp_endpoint) + with provider.get_tracer("test-tracer").start_as_current_span( + "test-span" + ): + pass + provider.force_flush() + weaver.end_and_check() + + def test_custom_policy_violation_raises(self): + """A policy that forbids service.name causes end_and_check to raise.""" + with WeaverLiveCheck(policies_dir=_TESTDATA_DIR) as weaver: + provider = _make_provider(weaver.otlp_endpoint) + with provider.get_tracer("test-tracer").start_as_current_span( + "test-span" + ): + pass + provider.force_flush() + + with self.assertRaises(AssertionError) as cm: + weaver.end_and_check() + + self.assertIn("service.name", str(cm.exception)) diff --git a/tests/opentelemetry-test-utils/tests/testdata/forbid_service_name.rego b/tests/opentelemetry-test-utils/tests/testdata/forbid_service_name.rego new file mode 100644 index 00000000000..c1aef90972c --- /dev/null +++ b/tests/opentelemetry-test-utils/tests/testdata/forbid_service_name.rego @@ -0,0 +1,13 @@ +package live_check_advice + +import rego.v1 + +deny contains result if { + input.sample.attribute.name == "service.name" + result := { + "type": "advice", + "advice_type": "no_service_name", + "advice_level": "violation", + "message": "service.name is forbidden by this policy", + } +} diff --git a/uv.lock b/uv.lock index 8724095cd31..ab6be2bcb29 100644 --- a/uv.lock +++ b/uv.lock @@ -1114,6 +1114,7 @@ dependencies = [ { name = "asgiref" }, { name = "opentelemetry-api" }, { name = "opentelemetry-sdk" }, + { name = "requests" }, ] [package.metadata] @@ -1121,6 +1122,7 @@ requires-dist = [ { name = "asgiref", specifier = "~=3.0" }, { name = "opentelemetry-api", editable = "opentelemetry-api" }, { name = "opentelemetry-sdk", editable = "opentelemetry-sdk" }, + { name = "requests", specifier = "~=2.28" }, ] [[package]] From 0d64bcbea31dc67b54276196b726b9f3cf7f7160 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Mon, 13 Apr 2026 23:06:43 -0700 Subject: [PATCH 02/15] up --- .../tests/test_weaver_live_check.py | 9 +++++---- .../tests/testdata/forbid_service_name.rego | 13 ------------- .../tests/testdata/test-policy.rego | 13 +++++++++++++ 3 files changed, 18 insertions(+), 17 deletions(-) delete mode 100644 tests/opentelemetry-test-utils/tests/testdata/forbid_service_name.rego create mode 100644 tests/opentelemetry-test-utils/tests/testdata/test-policy.rego diff --git a/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py b/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py index 6c5332c6d9f..a7606a2ff59 100644 --- a/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py +++ b/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py @@ -58,16 +58,17 @@ def test_sdk_resource_with_service_name(self): weaver.end_and_check() def test_custom_policy_violation_raises(self): - """A policy that forbids service.name causes end_and_check to raise.""" + """A policy that fails on never.use.this.attribute.""" with WeaverLiveCheck(policies_dir=_TESTDATA_DIR) as weaver: provider = _make_provider(weaver.otlp_endpoint) with provider.get_tracer("test-tracer").start_as_current_span( "test-span" - ): - pass + ) as span: + span.set_attribute("never.use.this.attribute", "bad value") + provider.force_flush() with self.assertRaises(AssertionError) as cm: weaver.end_and_check() - self.assertIn("service.name", str(cm.exception)) + self.assertIn("never.use.this.attribute", str(cm.exception)) diff --git a/tests/opentelemetry-test-utils/tests/testdata/forbid_service_name.rego b/tests/opentelemetry-test-utils/tests/testdata/forbid_service_name.rego deleted file mode 100644 index c1aef90972c..00000000000 --- a/tests/opentelemetry-test-utils/tests/testdata/forbid_service_name.rego +++ /dev/null @@ -1,13 +0,0 @@ -package live_check_advice - -import rego.v1 - -deny contains result if { - input.sample.attribute.name == "service.name" - result := { - "type": "advice", - "advice_type": "no_service_name", - "advice_level": "violation", - "message": "service.name is forbidden by this policy", - } -} diff --git a/tests/opentelemetry-test-utils/tests/testdata/test-policy.rego b/tests/opentelemetry-test-utils/tests/testdata/test-policy.rego new file mode 100644 index 00000000000..ed5c507b0e9 --- /dev/null +++ b/tests/opentelemetry-test-utils/tests/testdata/test-policy.rego @@ -0,0 +1,13 @@ +package live_check_advice + +import rego.v1 + +deny contains result if { + input.sample.attribute.name == "never.use.this.attribute" + result := { + "type": "advice", + "advice_type": "test_check", + "advice_level": "violation", + "message": "never.use.this.attribute is forbidden by this bogus policy", + } +} From 284deced11face534ab3b8d09f749ab57ceca8d7 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Tue, 14 Apr 2026 10:28:49 -0700 Subject: [PATCH 03/15] review feedback --- .github/workflows/templates/test.yml.j2 | 6 +- .github/workflows/test.yml | 42 ++-- .../opentelemetry/test/weaver_live_check.py | 214 +++++++++++++++--- .../tests/test_weaver_live_check.py | 77 ++++++- .../tests/testdata/test-policy.rego | 3 + 5 files changed, 275 insertions(+), 67 deletions(-) diff --git a/.github/workflows/templates/test.yml.j2 b/.github/workflows/templates/test.yml.j2 index af5b61b9437..d0c66c2a16f 100644 --- a/.github/workflows/templates/test.yml.j2 +++ b/.github/workflows/templates/test.yml.j2 @@ -56,9 +56,9 @@ jobs: - name: Install weaver run: | - WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${% raw %}{{ env.WEAVER_VERSION }}{% endraw %}/weaver_Linux_x86_64.tar.gz" - curl -sSL "$WEAVER_URL" | tar -xz -C /tmp weaver - sudo mv /tmp/weaver /usr/local/bin/weaver + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${% raw %}{{ env.WEAVER_VERSION }}{% endraw %}/weaver-x86_64-unknown-linux-gnu.tar.xz" + curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp weaver-x86_64-unknown-linux-gnu/weaver + sudo mv /tmp/weaver-x86_64-unknown-linux-gnu/weaver /usr/local/bin/weaver {%- endif %} - name: Set up Python {{ job_data.python_version }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ff7b6870186..01ce3922d10 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2868,9 +2868,9 @@ jobs: - name: Install weaver run: | - WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver_Linux_x86_64.tar.gz" - curl -sSL "$WEAVER_URL" | tar -xz -C /tmp weaver - sudo mv /tmp/weaver /usr/local/bin/weaver + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver-x86_64-unknown-linux-gnu.tar.xz" + curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp weaver-x86_64-unknown-linux-gnu/weaver + sudo mv /tmp/weaver-x86_64-unknown-linux-gnu/weaver /usr/local/bin/weaver - name: Set up Python 3.10 uses: actions/setup-python@v5 @@ -2893,9 +2893,9 @@ jobs: - name: Install weaver run: | - WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver_Linux_x86_64.tar.gz" - curl -sSL "$WEAVER_URL" | tar -xz -C /tmp weaver - sudo mv /tmp/weaver /usr/local/bin/weaver + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver-x86_64-unknown-linux-gnu.tar.xz" + curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp weaver-x86_64-unknown-linux-gnu/weaver + sudo mv /tmp/weaver-x86_64-unknown-linux-gnu/weaver /usr/local/bin/weaver - name: Set up Python 3.11 uses: actions/setup-python@v5 @@ -2918,9 +2918,9 @@ jobs: - name: Install weaver run: | - WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver_Linux_x86_64.tar.gz" - curl -sSL "$WEAVER_URL" | tar -xz -C /tmp weaver - sudo mv /tmp/weaver /usr/local/bin/weaver + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver-x86_64-unknown-linux-gnu.tar.xz" + curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp weaver-x86_64-unknown-linux-gnu/weaver + sudo mv /tmp/weaver-x86_64-unknown-linux-gnu/weaver /usr/local/bin/weaver - name: Set up Python 3.12 uses: actions/setup-python@v5 @@ -2943,9 +2943,9 @@ jobs: - name: Install weaver run: | - WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver_Linux_x86_64.tar.gz" - curl -sSL "$WEAVER_URL" | tar -xz -C /tmp weaver - sudo mv /tmp/weaver /usr/local/bin/weaver + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver-x86_64-unknown-linux-gnu.tar.xz" + curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp weaver-x86_64-unknown-linux-gnu/weaver + sudo mv /tmp/weaver-x86_64-unknown-linux-gnu/weaver /usr/local/bin/weaver - name: Set up Python 3.13 uses: actions/setup-python@v5 @@ -2968,9 +2968,9 @@ jobs: - name: Install weaver run: | - WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver_Linux_x86_64.tar.gz" - curl -sSL "$WEAVER_URL" | tar -xz -C /tmp weaver - sudo mv /tmp/weaver /usr/local/bin/weaver + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver-x86_64-unknown-linux-gnu.tar.xz" + curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp weaver-x86_64-unknown-linux-gnu/weaver + sudo mv /tmp/weaver-x86_64-unknown-linux-gnu/weaver /usr/local/bin/weaver - name: Set up Python 3.14 uses: actions/setup-python@v5 @@ -2993,9 +2993,9 @@ jobs: - name: Install weaver run: | - WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver_Linux_x86_64.tar.gz" - curl -sSL "$WEAVER_URL" | tar -xz -C /tmp weaver - sudo mv /tmp/weaver /usr/local/bin/weaver + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver-x86_64-unknown-linux-gnu.tar.xz" + curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp weaver-x86_64-unknown-linux-gnu/weaver + sudo mv /tmp/weaver-x86_64-unknown-linux-gnu/weaver /usr/local/bin/weaver - name: Set up Python 3.14t uses: actions/setup-python@v5 @@ -3018,9 +3018,9 @@ jobs: - name: Install weaver run: | - WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver_Linux_x86_64.tar.gz" - curl -sSL "$WEAVER_URL" | tar -xz -C /tmp weaver - sudo mv /tmp/weaver /usr/local/bin/weaver + WEAVER_URL="https://github.com/open-telemetry/weaver/releases/download/${{ env.WEAVER_VERSION }}/weaver-x86_64-unknown-linux-gnu.tar.xz" + curl -sSL "$WEAVER_URL" | tar -xJ -C /tmp weaver-x86_64-unknown-linux-gnu/weaver + sudo mv /tmp/weaver-x86_64-unknown-linux-gnu/weaver /usr/local/bin/weaver - name: Set up Python pypy-3.10 uses: actions/setup-python@v5 diff --git a/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py b/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py index 43d53c336e0..0c397731561 100644 --- a/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py +++ b/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import functools import json import logging import os @@ -81,7 +82,9 @@ def collect(obj: Any) -> None: { "id": k[0], "message": k[1], - "context": k[2], + "context": vs[0].get( + "context" + ), # preserve original dict, not JSON string "signal_name": k[3], "signal_type": k[4], "count": len(vs), @@ -111,10 +114,88 @@ def _format_violations(violations: list) -> str: return "\n".join(lines) +class LiveCheckError(AssertionError): + """Raised by :meth:`WeaverLiveCheck.end_and_check` when semconv violations are found. + + The full :class:`LiveCheckReport` is attached as :attr:`report` for + structured inspection beyond the human-readable message:: + + with pytest.raises(LiveCheckError) as exc_info: + weaver.end_and_check() + + err = exc_info.value + assert any( + v["id"] == "my_policy_check" + and v["context"]["attribute_name"] == "my.attribute" + for v in err.report.violations + ) + """ + + def __init__(self, message: str, report: "LiveCheckReport") -> None: + super().__init__(message) + self.report = report + + +class LiveCheckReport: + """The result of a weaver live-check run. + + Provides structured access to violations and the full raw JSON report. + + See https://github.com/open-telemetry/weaver/tree/main/crates/weaver_live_check#output + for the full report structure. + + Example — asserting on metrics statistics:: + + report = weaver.end() + seen = report["statistics"]["seen_registry_metrics"] + assert seen.get("http.server.request.duration") == 1 + + Example — asserting on violations:: + + report = weaver.end() + assert any( + v["id"] == "my_policy_check" + and v["context"]["attribute_name"] == "my.attribute" + for v in report.violations + ) + """ + + def __init__(self, report: dict[str, Any]) -> None: + self._report = report + + @functools.cached_property + def violations(self) -> list[dict[str, Any]]: + """Deduplicated list of semconv violations found in the report. + + Each item is a dict with keys: ``id``, ``message``, ``context`` + (the raw context dict from weaver, e.g. ``{"attribute_name": "foo"}``), + ``signal_name``, ``signal_type``, ``count``. + """ + return _extract_violations(self._report) + + def __getitem__(self, key: str) -> Any: + return self._report[key] + + def get(self, key: str, default: Any = None) -> Any: + return self._report.get(key, default) + + def __contains__(self, key: object) -> bool: + return key in self._report + + def __repr__(self) -> str: + n = len(self.violations) + return f"LiveCheckReport({n} violation{'s' if n != 1 else ''})" + + +# NOTE: WeaverLiveCheck is experimental and its API is subject to change. class WeaverLiveCheck: """Runs ``weaver registry live-check`` as a subprocess and validates OTLP telemetry against OpenTelemetry semantic conventions. + .. note:: + This class is experimental and its API is subject to change without notice. + + Requires the ``weaver`` binary on PATH: https://github.com/open-telemetry/weaver/releases @@ -128,27 +209,36 @@ def test_my_telemetry(self): # ... configure provider, emit telemetry ... provider.force_flush() - # Signals weaver to stop, raises AssertionError listing violations - # if any, or returns the raw JSON report on success. + # Signals weaver to stop, raises LiveCheckError listing violations + # if any, or returns a LiveCheckReport on success. report = weaver.end_and_check() # __exit__ calls close(), which is idempotent if end_and_check() was already called - :meth:`end_and_check` returns the raw weaver JSON report when weaver exits - successfully (exit code 0). Use it for custom assertions on the report - content beyond the built-in violation check:: + Use :meth:`end` when you need the full :class:`LiveCheckReport` + regardless of whether violations were found — for example, to assert that + specific metrics were observed or to inspect violation fields directly:: + + with WeaverLiveCheck() as weaver: + # ... configure provider, emit telemetry ... + provider.force_flush() + report = weaver.end() - report = weaver.end_and_check() - # report is the raw JSON dict from weaver; inspect it as needed, e.g.: - self.assertIn("some_signal", str(report)) + seen_metrics = report["statistics"]["seen_registry_metrics"] + assert seen_metrics.get("http.server.request.duration") == 1 Lifecycle: - :meth:`start` — launches weaver and waits for it to become ready. - :attr:`otlp_endpoint` — gRPC OTLP endpoint to point exporters at. - - :meth:`end_and_check` — signals weaver to stop, collects the report, and - raises :class:`AssertionError` with a human-readable violation list if weaver - exits non-zero. Returns the raw report dict on success. - - :meth:`close` — calls :meth:`end_and_check` then terminates the process. - Idempotent; safe to call even if :meth:`end_and_check` was already called. + - :meth:`end` — signals weaver to stop and always returns a + :class:`LiveCheckReport`. Never raises for semconv violations; use + this when you want to write your own assertions. + - :meth:`end_and_check` — signals weaver to stop and raises + :class:`LiveCheckError` with a human-readable violation list and the + full report attached if weaver exits non-zero. Returns a + :class:`LiveCheckReport` on success. + - :meth:`close` — stops weaver if not already stopped and terminates the + process. Never raises for semconv violations. Idempotent; safe to + call even if :meth:`end_and_check` or :meth:`end` was already called. """ def __init__( @@ -247,25 +337,22 @@ def _wait_for_ready(self, timeout: int = 60) -> None: def otlp_endpoint(self) -> str: return f"http://localhost:{self._otlp_port}" - def end_and_check(self, timeout: int = 30) -> dict[str, Any]: - if self._stopped: - logger.warning( - "end_and_check() called after weaver already stopped; returning empty report" - ) - return {} - self._stopped = True + def _do_stop(self, timeout: int) -> tuple["LiveCheckReport", int]: + """POST /stop, wait for the process to exit, return (report, exit_code). + Raises for infrastructure errors (HTTP failure, process communication). + Never raises for semconv violations. + """ if not self._ready: raise RuntimeError( "WeaverLiveCheck process did not start successfully" ) - try: response = post( f"http://localhost:{self._admin_port}/stop", timeout=5 ) response.raise_for_status() - report = response.json() + report = LiveCheckReport(response.json()) assert self._process is not None exit_code = self._process.wait(timeout=timeout) except Exception as e: @@ -274,13 +361,58 @@ def end_and_check(self, timeout: int = 30) -> dict[str, Any]: "Error communicating with weaver: %s, logs: %s", e, logs ) raise + return report, exit_code + + def end(self, timeout: int = 30) -> "LiveCheckReport": + """Signal weaver to stop and return the full :class:`LiveCheckReport`. + + Never raises for semconv violations — use this when you want to write + your own assertions against :attr:`LiveCheckReport.violations` or the + raw report data. + + Raises :exc:`RuntimeError` for infrastructure problems (weaver failed + to start, HTTP communication error, etc.). + + See https://github.com/open-telemetry/weaver/tree/main/crates/weaver_live_check#output + for the report structure. + """ + if self._stopped: + logger.warning( + "end() called after weaver already stopped; returning empty report" + ) + return LiveCheckReport({}) + self._stopped = True + report, _ = self._do_stop(timeout) + return report + def end_and_check(self, timeout: int = 30) -> "LiveCheckReport": + """Signal weaver to stop and assert no semconv violations were found. + + Returns the :class:`LiveCheckReport` when weaver exits successfully + (exit code 0). + + Does **not** return if weaver exits with a non-zero status — raises + :exc:`LiveCheckError` (a subclass of :exc:`AssertionError`) with a + human-readable list of violations and the full :class:`LiveCheckReport` + attached as :attr:`LiveCheckError.report`. + Use :meth:`end` if you need the report regardless of violations. + + Raises :exc:`RuntimeError` for infrastructure problems (weaver failed + to start, HTTP communication error, etc.). + """ + if self._stopped: + logger.warning( + "end_and_check() called after weaver already stopped; returning empty report" + ) + return LiveCheckReport({}) + self._stopped = True + report, exit_code = self._do_stop(timeout) if exit_code == 0: + # Success — no violations found, no errors communicating with weaver return report - - violations = _extract_violations(report) - raise AssertionError( - f"Semconv violations found:\n{_format_violations(violations)}" + raise LiveCheckError( + f"Semconv violations found:\n{_format_violations(report.violations)}", + report, ) def _read_weaver_logs(self) -> str | None: @@ -296,12 +428,24 @@ def _read_weaver_logs(self) -> str | None: return None def close(self) -> None: - try: - self.end_and_check() - finally: - if self._process and self._process.poll() is None: - self._process.terminate() + """Stop weaver and clean up the process. + + If weaver has not been stopped yet, sends the ``/stop`` signal and + waits for the process to exit. Never raises for semconv violations. + Idempotent — safe to call multiple times or after :meth:`end` / + :meth:`end_and_check` has already been called. + """ + if not self._stopped: + self._stopped = True + if self._ready: try: - self._process.wait(timeout=5) - except subprocess.TimeoutExpired: - self._process.kill() + self._do_stop(timeout=30) + return # process already exited cleanly + except Exception as e: + logger.debug("Error stopping weaver during close: %s", e) + if self._process and self._process.poll() is None: + self._process.terminate() + try: + self._process.wait(timeout=5) + except subprocess.TimeoutExpired: + self._process.kill() diff --git a/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py b/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py index a7606a2ff59..d3326851b2a 100644 --- a/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py +++ b/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py @@ -28,7 +28,11 @@ from opentelemetry.sdk.resources import SERVICE_NAME, Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor -from opentelemetry.test.weaver_live_check import WeaverLiveCheck +from opentelemetry.test.weaver_live_check import ( + LiveCheckError, + LiveCheckReport, + WeaverLiveCheck, +) _TESTDATA_DIR = os.path.join(os.path.dirname(__file__), "testdata") @@ -46,8 +50,8 @@ def _make_provider(otlp_endpoint: str) -> TracerProvider: "weaver binary not found on PATH — install from https://github.com/open-telemetry/weaver/releases", ) class TestSDKInitLiveCheck(unittest.TestCase): - def test_sdk_resource_with_service_name(self): - """SDK initialized with service.name emits conformant telemetry.""" + def test_end_and_check_no_violations(self): + """end_and_check() returns a LiveCheckReport with no violations on conformant telemetry.""" with WeaverLiveCheck() as weaver: provider = _make_provider(weaver.otlp_endpoint) with provider.get_tracer("test-tracer").start_as_current_span( @@ -55,10 +59,13 @@ def test_sdk_resource_with_service_name(self): ): pass provider.force_flush() - weaver.end_and_check() + report = weaver.end_and_check() + + self.assertIsInstance(report, LiveCheckReport) + self.assertEqual(report.violations, []) - def test_custom_policy_violation_raises(self): - """A policy that fails on never.use.this.attribute.""" + def test_end_and_check_raises_on_violations(self): + """end_and_check() raises LiveCheckError with the report attached.""" with WeaverLiveCheck(policies_dir=_TESTDATA_DIR) as weaver: provider = _make_provider(weaver.otlp_endpoint) with provider.get_tracer("test-tracer").start_as_current_span( @@ -68,7 +75,61 @@ def test_custom_policy_violation_raises(self): provider.force_flush() - with self.assertRaises(AssertionError) as cm: + with self.assertRaises(LiveCheckError) as cm: weaver.end_and_check() - self.assertIn("never.use.this.attribute", str(cm.exception)) + # Human-readable message lists the violation + self.assertIn( + "never.use.this.attribute is forbidden by this bogus policy", + str(cm.exception), + ) + # Structured report is attached for programmatic inspection + self.assertTrue( + any( + v["id"] == "test_check" + and v["context"].get("attribute_name") + == "never.use.this.attribute" + for v in cm.exception.report.violations + ) + ) + + def test_end_no_violations(self): + """end() returns a LiveCheckReport with no violations on conformant telemetry.""" + with WeaverLiveCheck() as weaver: + provider = _make_provider(weaver.otlp_endpoint) + with provider.get_tracer("test-tracer").start_as_current_span( + "test-span" + ): + pass + provider.force_flush() + report = weaver.end() + + self.assertIsInstance(report, LiveCheckReport) + self.assertEqual(report.violations, []) + + def test_end_with_violations(self): + """end() returns a LiveCheckReport with violations without raising.""" + with WeaverLiveCheck(policies_dir=_TESTDATA_DIR) as weaver: + provider = _make_provider(weaver.otlp_endpoint) + with provider.get_tracer("test-tracer").start_as_current_span( + "test-span" + ) as span: + span.set_attribute("never.use.this.attribute", "bad value") + + provider.force_flush() + report = weaver.end() + + self.assertIsInstance(report, LiveCheckReport) + # Check the violation id (maps to advice_type in the rego policy) + self.assertTrue( + any(v["id"] == "test_check" for v in report.violations) + ) + # Check the structured context identifies the offending attribute by name + self.assertTrue( + any( + isinstance(v["context"], dict) + and v["context"].get("attribute_name") + == "never.use.this.attribute" + for v in report.violations + ) + ) diff --git a/tests/opentelemetry-test-utils/tests/testdata/test-policy.rego b/tests/opentelemetry-test-utils/tests/testdata/test-policy.rego index ed5c507b0e9..de8e9a27f44 100644 --- a/tests/opentelemetry-test-utils/tests/testdata/test-policy.rego +++ b/tests/opentelemetry-test-utils/tests/testdata/test-policy.rego @@ -8,6 +8,9 @@ deny contains result if { "type": "advice", "advice_type": "test_check", "advice_level": "violation", + "context": { + "attribute_name": input.sample.attribute.name, + }, "message": "never.use.this.attribute is forbidden by this bogus policy", } } From 5c2bb021b2f8bdad51c217cf239a56e9d2ad0251 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Tue, 14 Apr 2026 10:43:44 -0700 Subject: [PATCH 04/15] more tests --- .../tests/test_weaver_live_check.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py b/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py index d3326851b2a..ae0154039fb 100644 --- a/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py +++ b/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py @@ -106,6 +106,10 @@ def test_end_no_violations(self): self.assertIsInstance(report, LiveCheckReport) self.assertEqual(report.violations, []) + # LiveCheckReport supports dict-style access to the raw report data + self.assertIn("statistics", report) + self.assertIsNotNone(report.get("statistics")) + self.assertIsNone(report.get("nonexistent")) def test_end_with_violations(self): """end() returns a LiveCheckReport with violations without raising.""" @@ -133,3 +137,24 @@ def test_end_with_violations(self): for v in report.violations ) ) + + def test_report_span_statistics(self): + """The full report exposes span counts and individual span samples.""" + with WeaverLiveCheck() as weaver: + provider = _make_provider(weaver.otlp_endpoint) + with provider.get_tracer("test-tracer").start_as_current_span( + "test-span" + ): + pass + provider.force_flush() + report = weaver.end() + + # Individual spans are accessible in report["samples"], each entry + # with a "span" key containing the span data. + span_samples = [ + s["span"] for s in report.get("samples", []) if "span" in s + ] + self.assertTrue( + any(s["name"] == "test-span" for s in span_samples), + f"Expected 'test-span' in samples, got: {[s['name'] for s in span_samples]}", + ) From 1c970aa552fafe80e8f1d34bbcca65148ad20b86 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Tue, 14 Apr 2026 10:58:07 -0700 Subject: [PATCH 05/15] update typing_extensions for util tests --- tests/opentelemetry-test-utils/test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/opentelemetry-test-utils/test-requirements.txt b/tests/opentelemetry-test-utils/test-requirements.txt index 6bc97e344a1..c6659ae388e 100644 --- a/tests/opentelemetry-test-utils/test-requirements.txt +++ b/tests/opentelemetry-test-utils/test-requirements.txt @@ -6,7 +6,7 @@ pluggy==1.6.0 py-cpuinfo==9.0.0 pytest==7.4.4 tomli==2.0.1 -typing_extensions==4.10.0 +typing_extensions==4.12.0 wrapt==1.16.0 zipp==3.19.2 -e opentelemetry-api From 69bbe237288736e36247308be41bfea8f094a526 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Tue, 14 Apr 2026 11:03:28 -0700 Subject: [PATCH 06/15] fixing some very important linting errors --- CHANGELOG.md | 3 +- .../opentelemetry/test/weaver_live_check.py | 64 ++++++++++--------- 2 files changed, 35 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 721db3b6127..9d260b7d77f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#4907](https://github.com/open-telemetry/opentelemetry-python/issues/4907)) - Drop Python 3.9 support ([#5076](https://github.com/open-telemetry/opentelemetry-python/pull/5076)) - +- Add WeaverLiveCheck test util + ([#5088](https://github.com/open-telemetry/opentelemetry-python/pull/5088)) ## Version 1.41.0/0.62b0 (2026-04-09) diff --git a/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py b/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py index 0c397731561..80090f82a1b 100644 --- a/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py +++ b/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py @@ -32,9 +32,9 @@ def _find_free_port() -> int: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(("", 0)) - return s.getsockname()[1] + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("", 0)) + return sock.getsockname()[1] def _extract_violations(report: dict) -> list: @@ -56,8 +56,8 @@ def collect(obj: Any) -> None: for advice in obj["live_check_result"].get("all_advice", []): if advice.get("level") == "violation": raw.append(advice) - for v in obj.values(): - collect(v) + for val in obj.values(): + collect(val) elif isinstance(obj, list): for item in obj: collect(item) @@ -65,18 +65,18 @@ def collect(obj: Any) -> None: collect(report) groups: dict[tuple, list] = defaultdict(list) - for v in raw: - ctx = v.get("context") + for violation in raw: + ctx = violation.get("context") key = ( - v.get("id"), - v.get("message"), + violation.get("id"), + violation.get("message"), json.dumps(ctx, sort_keys=True) if isinstance(ctx, (dict, list)) else ctx, - v.get("signal_name"), - v.get("signal_type"), + violation.get("signal_name"), + violation.get("signal_type"), ) - groups[key].append(v) + groups[key].append(violation) violations = [ { @@ -98,10 +98,10 @@ def collect(obj: Any) -> None: def _format_violations(violations: list) -> str: """Format violations list as human-readable text (mirrors violations.j2 output).""" lines = [] - for v in violations: + for violation in violations: signal = "" - signal_type = v.get("signal_type") - signal_name = v.get("signal_name") + signal_type = violation.get("signal_type") + signal_name = violation.get("signal_name") if signal_type and signal_name: signal = f" on {signal_type} '{signal_name}'" elif signal_type: @@ -109,7 +109,7 @@ def _format_violations(violations: list) -> str: elif signal_name: signal = f" on '{signal_name}'" lines.append( - f"- [{v.get('id')}] {v.get('message')} ({v['count']} occurrence(s){signal})" + f"- [{violation.get('id')}] {violation.get('message')} ({violation['count']} occurrence(s){signal})" ) return "\n".join(lines) @@ -183,8 +183,8 @@ def __contains__(self, key: object) -> bool: return key in self._report def __repr__(self) -> str: - n = len(self.violations) - return f"LiveCheckReport({n} violation{'s' if n != 1 else ''})" + num_violations = len(self.violations) + return f"LiveCheckReport({num_violations} violation{'s' if num_violations != 1 else ''})" # NOTE: WeaverLiveCheck is experimental and its API is subject to change. @@ -297,7 +297,7 @@ def __exit__(self, exc_type: Any, *_: Any) -> None: def start(self, timeout: int = 60) -> "WeaverLiveCheck": logger.debug("Starting WeaverLiveCheck process...") - self._process = subprocess.Popen( + self._process = subprocess.Popen( # pylint: disable=consider-using-with self._command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -305,16 +305,16 @@ def start(self, timeout: int = 60) -> "WeaverLiveCheck": try: self._wait_for_ready(timeout=timeout) self._ready = True - except Exception as e: + except Exception as exc: # pylint: disable=broad-except logs = self._read_weaver_logs() logger.error( - "WeaverLiveCheck did not start: %s, logs: %s", e, logs + "WeaverLiveCheck did not start: %s, logs: %s", exc, logs ) raise return self def _wait_for_ready(self, timeout: int = 60) -> None: - for i in range(timeout): + for attempt in range(timeout): if self._process is not None and self._process.poll() is not None: raise RuntimeError( f"WeaverLiveCheck process exited unexpectedly (code {self._process.returncode})" @@ -326,10 +326,12 @@ def _wait_for_ready(self, timeout: int = 60) -> None: if response.status_code == 200: return logger.debug( - "Health check returned %s, try %s", response.status_code, i + "Health check returned %s, try %s", + response.status_code, + attempt, ) - except ReqConnectionError as e: - logger.debug("Health check connection error: %s", e) + except ReqConnectionError as exc: + logger.debug("Health check connection error: %s", exc) time.sleep(1) raise TimeoutError("WeaverLiveCheck did not become ready in time") @@ -355,10 +357,10 @@ def _do_stop(self, timeout: int) -> tuple["LiveCheckReport", int]: report = LiveCheckReport(response.json()) assert self._process is not None exit_code = self._process.wait(timeout=timeout) - except Exception as e: + except Exception as exc: # pylint: disable=broad-except logs = self._read_weaver_logs() logger.error( - "Error communicating with weaver: %s, logs: %s", e, logs + "Error communicating with weaver: %s, logs: %s", exc, logs ) raise return report, exit_code @@ -423,8 +425,8 @@ def _read_weaver_logs(self) -> str | None: self._process.kill() out, err = self._process.communicate() return f"{out.decode()}\n{err.decode()}" - except Exception as e: - logger.error("Could not get weaver logs: %s", e) + except Exception as exc: # pylint: disable=broad-except + logger.error("Could not get weaver logs: %s", exc) return None def close(self) -> None: @@ -441,8 +443,8 @@ def close(self) -> None: try: self._do_stop(timeout=30) return # process already exited cleanly - except Exception as e: - logger.debug("Error stopping weaver during close: %s", e) + except Exception as exc: # pylint: disable=broad-except + logger.debug("Error stopping weaver during close: %s", exc) if self._process and self._process.poll() is None: self._process.terminate() try: From 2dac41d934532d1778a9b4f1aae7fb08a834c0c0 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Tue, 14 Apr 2026 11:59:38 -0700 Subject: [PATCH 07/15] fix py3.14 errors on win - skip weaver live tests --- .../test-requirements.txt | 1 - .../tests/test_weaver_live_check.py | 21 ++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/opentelemetry-test-utils/test-requirements.txt b/tests/opentelemetry-test-utils/test-requirements.txt index c6659ae388e..333a6cbbda1 100644 --- a/tests/opentelemetry-test-utils/test-requirements.txt +++ b/tests/opentelemetry-test-utils/test-requirements.txt @@ -15,4 +15,3 @@ zipp==3.19.2 -e tests/opentelemetry-test-utils -e opentelemetry-proto -e exporter/opentelemetry-exporter-otlp-proto-common --e exporter/opentelemetry-exporter-otlp-proto-grpc diff --git a/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py b/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py index ae0154039fb..5356316f8a3 100644 --- a/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py +++ b/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py @@ -22,9 +22,16 @@ import shutil import unittest -from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( - OTLPSpanExporter, -) +try: + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, + ) + + _HAS_GRPC_EXPORTER = True +except ImportError: + OTLPSpanExporter = None # type: ignore[assignment,misc] + _HAS_GRPC_EXPORTER = False + from opentelemetry.sdk.resources import SERVICE_NAME, Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor @@ -38,6 +45,7 @@ def _make_provider(otlp_endpoint: str) -> TracerProvider: + assert OTLPSpanExporter is not None resource = Resource.create({SERVICE_NAME: "test-service"}) exporter = OTLPSpanExporter(endpoint=otlp_endpoint, insecure=True) provider = TracerProvider(resource=resource) @@ -45,6 +53,13 @@ def _make_provider(otlp_endpoint: str) -> TracerProvider: return provider +@unittest.skipUnless( + _HAS_GRPC_EXPORTER, + # grpcio has no pre-built wheels for some platforms (e.g. free-threaded Python on Windows) + # skip these tests since they require the OTLP gRPC exporter to send telemetry to Weaver + # but have nothing to do with the exporter implementation itself + "opentelemetry-exporter-otlp-proto-grpc not installed", +) @unittest.skipUnless( shutil.which("weaver") is not None, "weaver binary not found on PATH — install from https://github.com/open-telemetry/weaver/releases", From 0398a2855c358c366ae2ab32d4048640505f519b Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Tue, 14 Apr 2026 13:07:21 -0700 Subject: [PATCH 08/15] don't run util tests on py3.14 win - grpcio is missing there --- .github/workflows/templates/test.yml.j2 | 3 +++ .github/workflows/test.yml | 21 ------------------- .../test-requirements.txt | 1 + .../tests/test_weaver_live_check.py | 21 +++---------------- tox.ini | 1 + 5 files changed, 8 insertions(+), 39 deletions(-) diff --git a/.github/workflows/templates/test.yml.j2 b/.github/workflows/templates/test.yml.j2 index d0c66c2a16f..6d1bd2233aa 100644 --- a/.github/workflows/templates/test.yml.j2 +++ b/.github/workflows/templates/test.yml.j2 @@ -30,6 +30,8 @@ env: jobs: {%- for job_data in job_datas %} + {#- grpcio has no wheel for free-threaded Python on Windows; Ubuntu builds from source fine #} + {%- if not ("py314t" in job_data.tox_env and "test-opentelemetry-test-utils" in job_data.tox_env and job_data.os == "windows-latest") %} {{ job_data.name }}: name: {{ job_data.ui_name }} @@ -71,4 +73,5 @@ jobs: - name: Run tests run: tox -e {{ job_data.tox_env }} -- -ra + {%- endif %} {%- endfor %} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 01ce3922d10..de9043694ce 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6260,27 +6260,6 @@ jobs: - name: Run tests run: tox -e py314-test-opentelemetry-test-utils -- -ra - py314t-test-opentelemetry-test-utils_windows-latest: - name: opentelemetry-test-utils 3.14t Windows - runs-on: windows-latest - timeout-minutes: 30 - steps: - - name: Configure git to support long filenames - run: git config --system core.longpaths true - - name: Checkout repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - - name: Set up Python 3.14t - uses: actions/setup-python@v5 - with: - python-version: "3.14t" - - - name: Install tox - run: pip install tox-uv - - - name: Run tests - run: tox -e py314t-test-opentelemetry-test-utils -- -ra - pypy3-test-opentelemetry-test-utils_windows-latest: name: opentelemetry-test-utils pypy-3.10 Windows runs-on: windows-latest diff --git a/tests/opentelemetry-test-utils/test-requirements.txt b/tests/opentelemetry-test-utils/test-requirements.txt index 333a6cbbda1..24bf4fec204 100644 --- a/tests/opentelemetry-test-utils/test-requirements.txt +++ b/tests/opentelemetry-test-utils/test-requirements.txt @@ -15,3 +15,4 @@ zipp==3.19.2 -e tests/opentelemetry-test-utils -e opentelemetry-proto -e exporter/opentelemetry-exporter-otlp-proto-common +-e exporter/opentelemetry-exporter-otlp-proto-grpc \ No newline at end of file diff --git a/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py b/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py index 5356316f8a3..ae0154039fb 100644 --- a/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py +++ b/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py @@ -22,16 +22,9 @@ import shutil import unittest -try: - from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( - OTLPSpanExporter, - ) - - _HAS_GRPC_EXPORTER = True -except ImportError: - OTLPSpanExporter = None # type: ignore[assignment,misc] - _HAS_GRPC_EXPORTER = False - +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, +) from opentelemetry.sdk.resources import SERVICE_NAME, Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor @@ -45,7 +38,6 @@ def _make_provider(otlp_endpoint: str) -> TracerProvider: - assert OTLPSpanExporter is not None resource = Resource.create({SERVICE_NAME: "test-service"}) exporter = OTLPSpanExporter(endpoint=otlp_endpoint, insecure=True) provider = TracerProvider(resource=resource) @@ -53,13 +45,6 @@ def _make_provider(otlp_endpoint: str) -> TracerProvider: return provider -@unittest.skipUnless( - _HAS_GRPC_EXPORTER, - # grpcio has no pre-built wheels for some platforms (e.g. free-threaded Python on Windows) - # skip these tests since they require the OTLP gRPC exporter to send telemetry to Weaver - # but have nothing to do with the exporter implementation itself - "opentelemetry-exporter-otlp-proto-grpc not installed", -) @unittest.skipUnless( shutil.which("weaver") is not None, "weaver binary not found on PATH — install from https://github.com/open-telemetry/weaver/releases", diff --git a/tox.ini b/tox.ini index e3a7db365ab..af7c17f08ef 100644 --- a/tox.ini +++ b/tox.ini @@ -93,6 +93,7 @@ envlist = lint-opentelemetry-propagator-jaeger py3{10,11,12,13,14,14t}-test-opentelemetry-test-utils + ; intentionally excluded from py314t on Windows (grpcio has no wheel for free-threaded Python) pypy3-test-opentelemetry-test-utils lint-opentelemetry-test-utils From 584492241a7b377ff8e42d27be589d491aef3ef8 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Tue, 14 Apr 2026 14:04:14 -0700 Subject: [PATCH 09/15] don't run util tests on py3.10 win - grpcio is missing there --- .github/workflows/templates/test.yml.j2 | 4 ++-- .github/workflows/test.yml | 21 --------------------- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/.github/workflows/templates/test.yml.j2 b/.github/workflows/templates/test.yml.j2 index 6d1bd2233aa..7fe27077c33 100644 --- a/.github/workflows/templates/test.yml.j2 +++ b/.github/workflows/templates/test.yml.j2 @@ -30,8 +30,8 @@ env: jobs: {%- for job_data in job_datas %} - {#- grpcio has no wheel for free-threaded Python on Windows; Ubuntu builds from source fine #} - {%- if not ("py314t" in job_data.tox_env and "test-opentelemetry-test-utils" in job_data.tox_env and job_data.os == "windows-latest") %} + {#- grpcio has no wheel for free-threaded Python or PyPy on Windows; Ubuntu builds from source fine #} + {%- if not (("py314t" in job_data.tox_env or "pypy3" in job_data.tox_env) and "test-opentelemetry-test-utils" in job_data.tox_env and job_data.os == "windows-latest") %} {{ job_data.name }}: name: {{ job_data.ui_name }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index de9043694ce..a374d23e1a4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6259,24 +6259,3 @@ jobs: - name: Run tests run: tox -e py314-test-opentelemetry-test-utils -- -ra - - pypy3-test-opentelemetry-test-utils_windows-latest: - name: opentelemetry-test-utils pypy-3.10 Windows - runs-on: windows-latest - timeout-minutes: 30 - steps: - - name: Configure git to support long filenames - run: git config --system core.longpaths true - - name: Checkout repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - - name: Set up Python pypy-3.10 - uses: actions/setup-python@v5 - with: - python-version: "pypy-3.10" - - - name: Install tox - run: pip install tox-uv - - - name: Run tests - run: tox -e pypy3-test-opentelemetry-test-utils -- -ra From 7e91f1b8d25a4e9d70dcb2e0070bf4ebea2c057d Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Wed, 15 Apr 2026 18:26:06 -0700 Subject: [PATCH 10/15] conditional test deps --- .github/workflows/templates/test.yml.j2 | 3 -- .github/workflows/test.yml | 42 +++++++++++++++++++ .../test-requirements.txt | 5 +-- .../tests/test_weaver_live_check.py | 16 +++++-- tox.ini | 1 - 5 files changed, 56 insertions(+), 11 deletions(-) diff --git a/.github/workflows/templates/test.yml.j2 b/.github/workflows/templates/test.yml.j2 index 7fe27077c33..d0c66c2a16f 100644 --- a/.github/workflows/templates/test.yml.j2 +++ b/.github/workflows/templates/test.yml.j2 @@ -30,8 +30,6 @@ env: jobs: {%- for job_data in job_datas %} - {#- grpcio has no wheel for free-threaded Python or PyPy on Windows; Ubuntu builds from source fine #} - {%- if not (("py314t" in job_data.tox_env or "pypy3" in job_data.tox_env) and "test-opentelemetry-test-utils" in job_data.tox_env and job_data.os == "windows-latest") %} {{ job_data.name }}: name: {{ job_data.ui_name }} @@ -73,5 +71,4 @@ jobs: - name: Run tests run: tox -e {{ job_data.tox_env }} -- -ra - {%- endif %} {%- endfor %} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a374d23e1a4..01ce3922d10 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6259,3 +6259,45 @@ jobs: - name: Run tests run: tox -e py314-test-opentelemetry-test-utils -- -ra + + py314t-test-opentelemetry-test-utils_windows-latest: + name: opentelemetry-test-utils 3.14t Windows + runs-on: windows-latest + timeout-minutes: 30 + steps: + - name: Configure git to support long filenames + run: git config --system core.longpaths true + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.14t + uses: actions/setup-python@v5 + with: + python-version: "3.14t" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -e py314t-test-opentelemetry-test-utils -- -ra + + pypy3-test-opentelemetry-test-utils_windows-latest: + name: opentelemetry-test-utils pypy-3.10 Windows + runs-on: windows-latest + timeout-minutes: 30 + steps: + - name: Configure git to support long filenames + run: git config --system core.longpaths true + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python pypy-3.10 + uses: actions/setup-python@v5 + with: + python-version: "pypy-3.10" + + - name: Install tox + run: pip install tox-uv + + - name: Run tests + run: tox -e pypy3-test-opentelemetry-test-utils -- -ra diff --git a/tests/opentelemetry-test-utils/test-requirements.txt b/tests/opentelemetry-test-utils/test-requirements.txt index 24bf4fec204..e41edce7a15 100644 --- a/tests/opentelemetry-test-utils/test-requirements.txt +++ b/tests/opentelemetry-test-utils/test-requirements.txt @@ -12,7 +12,4 @@ zipp==3.19.2 -e opentelemetry-api -e opentelemetry-sdk -e opentelemetry-semantic-conventions --e tests/opentelemetry-test-utils --e opentelemetry-proto --e exporter/opentelemetry-exporter-otlp-proto-common --e exporter/opentelemetry-exporter-otlp-proto-grpc \ No newline at end of file +-e tests/opentelemetry-test-utils \ No newline at end of file diff --git a/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py b/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py index ae0154039fb..e52e0705be7 100644 --- a/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py +++ b/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py @@ -22,9 +22,6 @@ import shutil import unittest -from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( - OTLPSpanExporter, -) from opentelemetry.sdk.resources import SERVICE_NAME, Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor @@ -34,6 +31,15 @@ WeaverLiveCheck, ) +try: + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter, + ) + + _HAS_GRPC = True +except ImportError: + _HAS_GRPC = False + _TESTDATA_DIR = os.path.join(os.path.dirname(__file__), "testdata") @@ -45,6 +51,10 @@ def _make_provider(otlp_endpoint: str) -> TracerProvider: return provider +@unittest.skipUnless( + _HAS_GRPC, + "grpc exporter not installed", +) @unittest.skipUnless( shutil.which("weaver") is not None, "weaver binary not found on PATH — install from https://github.com/open-telemetry/weaver/releases", diff --git a/tox.ini b/tox.ini index af7c17f08ef..e3a7db365ab 100644 --- a/tox.ini +++ b/tox.ini @@ -93,7 +93,6 @@ envlist = lint-opentelemetry-propagator-jaeger py3{10,11,12,13,14,14t}-test-opentelemetry-test-utils - ; intentionally excluded from py314t on Windows (grpcio has no wheel for free-threaded Python) pypy3-test-opentelemetry-test-utils lint-opentelemetry-test-utils From 901adcf0af0996dd3482528abeb15320a6d76317 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Wed, 15 Apr 2026 19:03:18 -0700 Subject: [PATCH 11/15] up --- tests/opentelemetry-test-utils/tests/test_weaver_live_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py b/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py index e52e0705be7..cff40c92206 100644 --- a/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py +++ b/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py @@ -32,7 +32,7 @@ ) try: - from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( # pylint: disable=no-name-in-module OTLPSpanExporter, ) From 3226b7e7bbf8804b66b8128669662b6a155f116f Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 16 Apr 2026 19:46:29 -0700 Subject: [PATCH 12/15] review comments --- .../opentelemetry/test/weaver_live_check.py | 78 +++++++++++-------- .../test-requirements.txt | 8 +- 2 files changed, 52 insertions(+), 34 deletions(-) diff --git a/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py b/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py index 80090f82a1b..7fb4867cab2 100644 --- a/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py +++ b/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py @@ -19,12 +19,13 @@ import shutil import socket import subprocess -import time from collections import defaultdict +from itertools import chain from typing import Any -from requests import get, post -from requests.exceptions import ConnectionError as ReqConnectionError +from requests import Session, post +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry from opentelemetry.semconv.schemas import Schemas @@ -50,19 +51,27 @@ def _extract_violations(report: dict) -> list: """ raw: list[dict] = [] - def collect(obj: Any) -> None: - if isinstance(obj, dict): - if "live_check_result" in obj: - for advice in obj["live_check_result"].get("all_advice", []): - if advice.get("level") == "violation": - raw.append(advice) - for val in obj.values(): - collect(val) - elif isinstance(obj, list): - for item in obj: - collect(item) + def _collect(obj: Any) -> list[dict]: + match obj: + case {"live_check_result": {"all_advice": advices}, **_rest}: + violations = [ + a for a in advices if a.get("level") == "violation" + ] + return violations + list( + chain.from_iterable(_collect(v) for v in obj.values()) + ) + case dict(): + return list( + chain.from_iterable(_collect(v) for v in obj.values()) + ) + case list(): + return list( + chain.from_iterable(_collect(item) for item in obj) + ) + case _: + return [] - collect(report) + raw = _collect(report) groups: dict[tuple, list] = defaultdict(list) for violation in raw: @@ -96,7 +105,7 @@ def collect(obj: Any) -> None: def _format_violations(violations: list) -> str: - """Format violations list as human-readable text (mirrors violations.j2 output).""" + """Format violations list as human-readable text.""" lines = [] for violation in violations: signal = "" @@ -314,26 +323,29 @@ def start(self, timeout: int = 60) -> "WeaverLiveCheck": return self def _wait_for_ready(self, timeout: int = 60) -> None: - for attempt in range(timeout): + retry = Retry( + total=timeout, + backoff_factor=1, + backoff_max=1, + # Any non-2xx response from /health means weaver isn't ready yet. + status_forcelist=list(range(300, 600)), + allowed_methods=["GET"], + ) + session = Session() + session.mount("http://", HTTPAdapter(max_retries=retry)) + try: + response = session.get( + f"http://localhost:{self._admin_port}/health", timeout=5 + ) + response.raise_for_status() + except Exception as exc: # pylint: disable=broad-except if self._process is not None and self._process.poll() is not None: raise RuntimeError( f"WeaverLiveCheck process exited unexpectedly (code {self._process.returncode})" - ) - try: - response = get( - f"http://localhost:{self._admin_port}/health", timeout=5 - ) - if response.status_code == 200: - return - logger.debug( - "Health check returned %s, try %s", - response.status_code, - attempt, - ) - except ReqConnectionError as exc: - logger.debug("Health check connection error: %s", exc) - time.sleep(1) - raise TimeoutError("WeaverLiveCheck did not become ready in time") + ) from exc + raise TimeoutError( + "WeaverLiveCheck did not become ready in time" + ) from exc @property def otlp_endpoint(self) -> str: diff --git a/tests/opentelemetry-test-utils/test-requirements.txt b/tests/opentelemetry-test-utils/test-requirements.txt index e41edce7a15..87de9a5f951 100644 --- a/tests/opentelemetry-test-utils/test-requirements.txt +++ b/tests/opentelemetry-test-utils/test-requirements.txt @@ -12,4 +12,10 @@ zipp==3.19.2 -e opentelemetry-api -e opentelemetry-sdk -e opentelemetry-semantic-conventions --e tests/opentelemetry-test-utils \ No newline at end of file +-e tests/opentelemetry-test-utils +# these are required for weaver integration tests, we're running that only on linux +# because of lack of support for gRPC on Windows in some cases. +# note: tox does not support PEP 508 markers on `-e` editable installs, so these are installed non-editable +./opentelemetry-proto ; sys_platform != 'win32' +./exporter/opentelemetry-exporter-otlp-proto-common ; sys_platform != 'win32' +./exporter/opentelemetry-exporter-otlp-proto-grpc ; sys_platform != 'win32' \ No newline at end of file From 270c9d2caa4366635584bf964ab85afc8acc76df Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 16 Apr 2026 20:18:49 -0700 Subject: [PATCH 13/15] speed up weaver tests --- .claude/settings.json | 41 +++++++++++++++++++ .../opentelemetry/test/weaver_live_check.py | 26 ++++++------ .../tests/test_weaver_live_check.py | 15 ++++--- .../tests/testdata/registry/attributes.yaml | 27 ++++++++++++ 4 files changed, 92 insertions(+), 17 deletions(-) create mode 100644 .claude/settings.json create mode 100644 tests/opentelemetry-test-utils/tests/testdata/registry/attributes.yaml diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 00000000000..4b2ce5a6ace --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,41 @@ +{ + "permissions": { + "allow": [ + "Bash(find /Users/lmolkova/repo/opentelemetry-python -name conftest.py -exec ls -la {} \\\\;)", + "Bash(python -m pytest tests/opentelemetry-test-utils/tests/test_weaver_live_check.py -v)", + "Bash(grep -E \"\\\\.yml$\")", + "Bash(grep -n \"weaver\" /Users/lmolkova/repo/opentelemetry-python/.github/workflows/*.yml)", + "Bash(grep -n \"download\\\\|install.*binary\\\\|wget\\\\|curl\" /Users/lmolkova/repo/opentelemetry-python/.github/workflows/*.yml)", + "Bash(grep -r \"binary\\\\|download\\\\|wget\\\\|curl\\\\|install.*tool\" /Users/lmolkova/repo/opentelemetry-python/.github/workflows/*.yml)", + "Bash(tox -e generate-workflows)", + "Read(//usr/local/**)", + "Bash(python .github/workflows/generate_workflows.py)", + "Bash(chmod +x /tmp/git_seq_editor.sh)", + "Bash(chmod +x /tmp/git_msg_editor.sh)", + "Bash(GIT_SEQUENCE_EDITOR=\"/tmp/git_seq_editor.sh\" GIT_EDITOR=\"/tmp/git_msg_editor.sh\" git rebase -i main)", + "WebFetch(domain:productionresultssa18.blob.core.windows.net)", + "WebFetch(domain:productionresultssa1.blob.core.windows.net)", + "Bash(xargs ls *)", + "Bash(xargs -I {} sh -c 'echo \"=== {} ===\" && head -20 {}')", + "Bash(xargs -I {} sh -c 'echo \"FILE: {}\" && cat {}')", + "Bash(find /Users/lmolkova/repo/opentelemetry-python-contrib -name \"pyproject.toml\" -path \"*/instrumentation/*\" -exec grep -l \"markers\\\\|sys_platform\\\\|platform\" {} \\\\;)", + "Bash(find /Users/lmolkova/repo/opentelemetry-python-contrib/util -type f -name \"pyproject.toml\" -exec sh -c 'echo \"=== {} ===\" && cat {}' \\\\;)", + "Bash(python -m pytest tests/opentelemetry-test-utils/tests/test_weaver_live_check.py -x --no-header -q)", + "Bash(python -m pytest tests/opentelemetry-test-utils/tests/test_weaver_live_check.py -x -q)", + "Bash(python -m pytest tests/opentelemetry-test-utils/tests/test_weaver_live_check.py -x -q --no-header)", + "Bash(sed -i '' 's/py312/py313/g' tox.ini)", + "Bash(tox -e py313)", + "Bash(mv dummy *)", + "Bash(tox -e py313 -r)", + "Bash(tox -e py312-test-opentelemetry-test-utils -r)", + "Bash(gh issue *)", + "Bash(gh search *)", + "Bash(pip --version)", + "Bash(tox --version)", + "Bash(gh run *)", + "Bash(tox -e py312-test-opentelemetry-test-utils)", + "Bash(.tox/py312-test-opentelemetry-test-utils/bin/pytest tests/opentelemetry-test-utils/tests/test_weaver_live_check.py::TestSDKInitLiveCheck::test_end_no_violations -ra)", + "Bash(.tox/py312-test-opentelemetry-test-utils/bin/python -c ' *)" + ] + } +} diff --git a/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py b/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py index 7fb4867cab2..f1eac549094 100644 --- a/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py +++ b/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py @@ -252,6 +252,7 @@ def test_my_telemetry(self): def __init__( self, + registry: str | None = None, schema_version: str | None = None, policies_dir: str | None = None, inactivity_timeout: int = 30, @@ -285,13 +286,14 @@ def __init__( if policies_dir: command += ["--advice-policies", os.path.abspath(policies_dir)] - if schema_version is None: - schema_version = list(Schemas)[-1].value.rsplit("/", 1)[-1] + if registry is None: + if schema_version is None: + schema_version = list(Schemas)[-1].value.rsplit("/", 1)[-1] + registry = f"https://github.com/open-telemetry/semantic-conventions/archive/refs/tags/v{schema_version}.tar.gz[model]" + elif os.path.isdir(registry): + registry = os.path.abspath(registry) - command += [ - "--registry", - f"https://github.com/open-telemetry/semantic-conventions/archive/refs/tags/v{schema_version}.tar.gz[model]", - ] + command += ["--registry", registry] self._command = command logger.debug("Weaver command: %s", command) @@ -304,7 +306,7 @@ def __exit__(self, exc_type: Any, *_: Any) -> None: self._stopped = True self.close() - def start(self, timeout: int = 60) -> "WeaverLiveCheck": + def start(self) -> "WeaverLiveCheck": logger.debug("Starting WeaverLiveCheck process...") self._process = subprocess.Popen( # pylint: disable=consider-using-with self._command, @@ -312,7 +314,7 @@ def start(self, timeout: int = 60) -> "WeaverLiveCheck": stderr=subprocess.PIPE, ) try: - self._wait_for_ready(timeout=timeout) + self._wait_for_ready() self._ready = True except Exception as exc: # pylint: disable=broad-except logs = self._read_weaver_logs() @@ -322,22 +324,22 @@ def start(self, timeout: int = 60) -> "WeaverLiveCheck": raise return self - def _wait_for_ready(self, timeout: int = 60) -> None: + def _wait_for_ready(self) -> None: retry = Retry( - total=timeout, + total=10, backoff_factor=1, backoff_max=1, # Any non-2xx response from /health means weaver isn't ready yet. status_forcelist=list(range(300, 600)), + raise_on_status=True, allowed_methods=["GET"], ) session = Session() session.mount("http://", HTTPAdapter(max_retries=retry)) try: - response = session.get( + session.get( f"http://localhost:{self._admin_port}/health", timeout=5 ) - response.raise_for_status() except Exception as exc: # pylint: disable=broad-except if self._process is not None and self._process.poll() is not None: raise RuntimeError( diff --git a/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py b/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py index cff40c92206..209c5de6359 100644 --- a/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py +++ b/tests/opentelemetry-test-utils/tests/test_weaver_live_check.py @@ -41,6 +41,7 @@ _HAS_GRPC = False _TESTDATA_DIR = os.path.join(os.path.dirname(__file__), "testdata") +_REGISTRY_DIR = os.path.join(_TESTDATA_DIR, "registry") def _make_provider(otlp_endpoint: str) -> TracerProvider: @@ -62,7 +63,7 @@ def _make_provider(otlp_endpoint: str) -> TracerProvider: class TestSDKInitLiveCheck(unittest.TestCase): def test_end_and_check_no_violations(self): """end_and_check() returns a LiveCheckReport with no violations on conformant telemetry.""" - with WeaverLiveCheck() as weaver: + with WeaverLiveCheck(registry=_REGISTRY_DIR) as weaver: provider = _make_provider(weaver.otlp_endpoint) with provider.get_tracer("test-tracer").start_as_current_span( "test-span" @@ -76,7 +77,9 @@ def test_end_and_check_no_violations(self): def test_end_and_check_raises_on_violations(self): """end_and_check() raises LiveCheckError with the report attached.""" - with WeaverLiveCheck(policies_dir=_TESTDATA_DIR) as weaver: + with WeaverLiveCheck( + registry=_REGISTRY_DIR, policies_dir=_TESTDATA_DIR + ) as weaver: provider = _make_provider(weaver.otlp_endpoint) with provider.get_tracer("test-tracer").start_as_current_span( "test-span" @@ -105,7 +108,7 @@ def test_end_and_check_raises_on_violations(self): def test_end_no_violations(self): """end() returns a LiveCheckReport with no violations on conformant telemetry.""" - with WeaverLiveCheck() as weaver: + with WeaverLiveCheck(registry=_REGISTRY_DIR) as weaver: provider = _make_provider(weaver.otlp_endpoint) with provider.get_tracer("test-tracer").start_as_current_span( "test-span" @@ -123,7 +126,9 @@ def test_end_no_violations(self): def test_end_with_violations(self): """end() returns a LiveCheckReport with violations without raising.""" - with WeaverLiveCheck(policies_dir=_TESTDATA_DIR) as weaver: + with WeaverLiveCheck( + registry=_REGISTRY_DIR, policies_dir=_TESTDATA_DIR + ) as weaver: provider = _make_provider(weaver.otlp_endpoint) with provider.get_tracer("test-tracer").start_as_current_span( "test-span" @@ -150,7 +155,7 @@ def test_end_with_violations(self): def test_report_span_statistics(self): """The full report exposes span counts and individual span samples.""" - with WeaverLiveCheck() as weaver: + with WeaverLiveCheck(registry=_REGISTRY_DIR) as weaver: provider = _make_provider(weaver.otlp_endpoint) with provider.get_tracer("test-tracer").start_as_current_span( "test-span" diff --git a/tests/opentelemetry-test-utils/tests/testdata/registry/attributes.yaml b/tests/opentelemetry-test-utils/tests/testdata/registry/attributes.yaml new file mode 100644 index 00000000000..d4a5e301470 --- /dev/null +++ b/tests/opentelemetry-test-utils/tests/testdata/registry/attributes.yaml @@ -0,0 +1,27 @@ +# this is a test registry, used to save up time on weaver loading semconv from GH repo +# and preventing flaky tests resulting from unpredictable network/performance in CI +groups: + - id: registry.test + type: attribute_group + brief: Minimal registry for WeaverLiveCheck self-tests. + attributes: + - id: service.name + type: string + brief: The name of the service. + stability: stable + examples: ["test-service"] + - id: telemetry.sdk.language + type: string + brief: The language of the telemetry SDK. + stability: stable + examples: ["python"] + - id: telemetry.sdk.name + type: string + brief: The name of the telemetry SDK. + stability: stable + examples: ["opentelemetry"] + - id: telemetry.sdk.version + type: string + brief: The version string of the telemetry SDK. + stability: stable + examples: ["1.0.0"] From 00d6e47becbb7e6069c5f2940547bef78d21fc5a Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 16 Apr 2026 21:12:52 -0700 Subject: [PATCH 14/15] fix segfault on pypy 3.10 on ubuntu --- .../opentelemetry/test/weaver_live_check.py | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py b/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py index f1eac549094..451587578e4 100644 --- a/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py +++ b/tests/opentelemetry-test-utils/src/opentelemetry/test/weaver_live_check.py @@ -52,24 +52,21 @@ def _extract_violations(report: dict) -> list: raw: list[dict] = [] def _collect(obj: Any) -> list[dict]: - match obj: - case {"live_check_result": {"all_advice": advices}, **_rest}: - violations = [ - a for a in advices if a.get("level") == "violation" - ] - return violations + list( - chain.from_iterable(_collect(v) for v in obj.values()) - ) - case dict(): - return list( - chain.from_iterable(_collect(v) for v in obj.values()) - ) - case list(): - return list( - chain.from_iterable(_collect(item) for item in obj) - ) - case _: - return [] + if isinstance(obj, dict): + result: list[dict] = [] + lcr = obj.get("live_check_result") + if isinstance(lcr, dict): + advices = lcr.get("all_advice") + if isinstance(advices, list): + result.extend( + a for a in advices if a.get("level") == "violation" + ) + for value in obj.values(): + result.extend(_collect(value)) + return result + if isinstance(obj, list): + return list(chain.from_iterable(_collect(item) for item in obj)) + return [] raw = _collect(report) From 4096c82c8d4b45e235a9549cbd81777bb1b4b2bd Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Mon, 20 Apr 2026 15:03:24 -0700 Subject: [PATCH 15/15] up --- .claude/settings.json | 41 ----------------------------------------- 1 file changed, 41 deletions(-) delete mode 100644 .claude/settings.json diff --git a/.claude/settings.json b/.claude/settings.json deleted file mode 100644 index 4b2ce5a6ace..00000000000 --- a/.claude/settings.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(find /Users/lmolkova/repo/opentelemetry-python -name conftest.py -exec ls -la {} \\\\;)", - "Bash(python -m pytest tests/opentelemetry-test-utils/tests/test_weaver_live_check.py -v)", - "Bash(grep -E \"\\\\.yml$\")", - "Bash(grep -n \"weaver\" /Users/lmolkova/repo/opentelemetry-python/.github/workflows/*.yml)", - "Bash(grep -n \"download\\\\|install.*binary\\\\|wget\\\\|curl\" /Users/lmolkova/repo/opentelemetry-python/.github/workflows/*.yml)", - "Bash(grep -r \"binary\\\\|download\\\\|wget\\\\|curl\\\\|install.*tool\" /Users/lmolkova/repo/opentelemetry-python/.github/workflows/*.yml)", - "Bash(tox -e generate-workflows)", - "Read(//usr/local/**)", - "Bash(python .github/workflows/generate_workflows.py)", - "Bash(chmod +x /tmp/git_seq_editor.sh)", - "Bash(chmod +x /tmp/git_msg_editor.sh)", - "Bash(GIT_SEQUENCE_EDITOR=\"/tmp/git_seq_editor.sh\" GIT_EDITOR=\"/tmp/git_msg_editor.sh\" git rebase -i main)", - "WebFetch(domain:productionresultssa18.blob.core.windows.net)", - "WebFetch(domain:productionresultssa1.blob.core.windows.net)", - "Bash(xargs ls *)", - "Bash(xargs -I {} sh -c 'echo \"=== {} ===\" && head -20 {}')", - "Bash(xargs -I {} sh -c 'echo \"FILE: {}\" && cat {}')", - "Bash(find /Users/lmolkova/repo/opentelemetry-python-contrib -name \"pyproject.toml\" -path \"*/instrumentation/*\" -exec grep -l \"markers\\\\|sys_platform\\\\|platform\" {} \\\\;)", - "Bash(find /Users/lmolkova/repo/opentelemetry-python-contrib/util -type f -name \"pyproject.toml\" -exec sh -c 'echo \"=== {} ===\" && cat {}' \\\\;)", - "Bash(python -m pytest tests/opentelemetry-test-utils/tests/test_weaver_live_check.py -x --no-header -q)", - "Bash(python -m pytest tests/opentelemetry-test-utils/tests/test_weaver_live_check.py -x -q)", - "Bash(python -m pytest tests/opentelemetry-test-utils/tests/test_weaver_live_check.py -x -q --no-header)", - "Bash(sed -i '' 's/py312/py313/g' tox.ini)", - "Bash(tox -e py313)", - "Bash(mv dummy *)", - "Bash(tox -e py313 -r)", - "Bash(tox -e py312-test-opentelemetry-test-utils -r)", - "Bash(gh issue *)", - "Bash(gh search *)", - "Bash(pip --version)", - "Bash(tox --version)", - "Bash(gh run *)", - "Bash(tox -e py312-test-opentelemetry-test-utils)", - "Bash(.tox/py312-test-opentelemetry-test-utils/bin/pytest tests/opentelemetry-test-utils/tests/test_weaver_live_check.py::TestSDKInitLiveCheck::test_end_no_violations -ra)", - "Bash(.tox/py312-test-opentelemetry-test-utils/bin/python -c ' *)" - ] - } -}