From e8acd586bccf1e03ec4e3e71e1f86651e9511c72 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 18:42:45 -0700 Subject: [PATCH 01/70] feat(deploy): implement Step 1 PR Extractor for release notes tool - Add ComponentConfig and SourceRule multi-repository mapping rules in config.py - Add PullRequest, ComponentVersionInfo, and ReleaseInfoManifest models in models.py - Add PRExtractor with gcloud container image tag lookup and single-call gh pr list date search in pr_extractor.py - Add unit and integration tests in deploy/generate_release_notes/tests/ - Add generate-release-notes dependency group to pyproject.toml --- deploy/generate_release_notes/__init__.py | 3 + deploy/generate_release_notes/config.py | 124 +++++++ deploy/generate_release_notes/models.py | 79 ++++ deploy/generate_release_notes/pr_extractor.py | 340 ++++++++++++++++++ .../generate_release_notes/tests/__init__.py | 1 + .../tests/test_pr_extractor.py | 171 +++++++++ pyproject.toml | 15 +- uv.lock | 121 ++++++- 8 files changed, 834 insertions(+), 20 deletions(-) create mode 100644 deploy/generate_release_notes/__init__.py create mode 100644 deploy/generate_release_notes/config.py create mode 100644 deploy/generate_release_notes/models.py create mode 100644 deploy/generate_release_notes/pr_extractor.py create mode 100644 deploy/generate_release_notes/tests/__init__.py create mode 100644 deploy/generate_release_notes/tests/test_pr_extractor.py diff --git a/deploy/generate_release_notes/__init__.py b/deploy/generate_release_notes/__init__.py new file mode 100644 index 00000000..af11b22a --- /dev/null +++ b/deploy/generate_release_notes/__init__.py @@ -0,0 +1,3 @@ +"""Data Commons Platform (DCP) Release Notes Generator Package.""" + +__version__ = "0.1.0" diff --git a/deploy/generate_release_notes/config.py b/deploy/generate_release_notes/config.py new file mode 100644 index 00000000..6f7a6628 --- /dev/null +++ b/deploy/generate_release_notes/config.py @@ -0,0 +1,124 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Configuration and multi-repository component mappings for DCP release notes generation.""" + +from dataclasses import dataclass, field +from typing import Dict, List, Optional + +DEFAULT_GITHUB_ORG = "datacommonsorg" + + +@dataclass +class SourceRule: + """Source repository rule with optional file path filter.""" + + repo: str + path_filter: Optional[str] = ( + None # If set, only PRs modifying this path match this component + ) + + +@dataclass +class ComponentConfig: + """Configuration for a tracked repository/component in DCP releases.""" + + id: str # Unique key (e.g. 'dcp', 'preprocessing', 'services') + name: str # Human-readable name for release notes + artifact_type: ( + str # 'dcp_platform', 'docker_services', 'docker_data', etc. + ) + image_uri: Optional[str] = None # Primary container image URI + default_tag_prefix: str = "v" # Tag prefix (e.g. 'v' for 'v1.1.2') + sources: List[SourceRule] = field( + default_factory=list + ) # Multi-repo contributing sources + + +# Master registry of all tracked components across Data Commons repositories +COMPONENTS: Dict[str, ComponentConfig] = { + "dcp": ComponentConfig( + id="dcp", + name="DCP Monorepo & Infra (CLI, Admin, DB, Terraform)", + artifact_type="dcp_platform", + image_uri=None, + default_tag_prefix="v", + sources=[ + SourceRule(repo="datacommonsorg/datacommons"), + ], + ), + "services": ComponentConfig( + id="services", + name="Core Services (Website, Mixer, MCP)", + artifact_type="docker_services", + image_uri="gcr.io/datcom-ci/datacommons-services", + default_tag_prefix="v", + sources=[ + SourceRule(repo="datacommonsorg/website"), # All non-cdc_data website PRs + SourceRule(repo="datacommonsorg/mixer"), # All mixer PRs + SourceRule(repo="datacommonsorg/agent-toolkit"), # All MCP PRs + ], + ), + "preprocessing": ComponentConfig( + id="preprocessing", + name="Data Preprocessor (datacommons-data)", + artifact_type="docker_data", + image_uri="gcr.io/datcom-ci/datacommons-data", + default_tag_prefix="v", + sources=[ + SourceRule(repo="datacommonsorg/import", path_filter="simple/"), + SourceRule( + repo="datacommonsorg/website", path_filter="build/cdc_data/" + ), + ], + ), + "dataflow_worker": ComponentConfig( + id="dataflow_worker", + name="Dataflow Ingestion Worker", + artifact_type="dataflow_template", + image_uri="us-docker.pkg.dev/datcom-ci/gcr.io/dataflow-templates/ingestion", + default_tag_prefix="v", + sources=[ + SourceRule( + repo="datacommonsorg/import", path_filter="pipeline/ingestion/" + ), + ], + ), + "ingestion_helper": ComponentConfig( + id="ingestion_helper", + name="Ingestion Helper Service", + artifact_type="docker_helper", + image_uri="gcr.io/datcom-ci/datacommons-ingestion-helper", + default_tag_prefix="v", + sources=[ + SourceRule( + repo="datacommonsorg/import", + path_filter="pipeline/workflow/ingestion-helper/", + ), + ], + ), + "postprocessing": ComponentConfig( + id="postprocessing", + name="Postprocessing Aggregation Helper Service", + artifact_type="docker_helper", + image_uri="gcr.io/datcom-ci/datacommons-aggregation-helper", + default_tag_prefix="v", + sources=[ + SourceRule( + repo="datacommonsorg/import", + path_filter="pipeline/workflow/aggregation-helper/", + ), + ], + ), +} diff --git a/deploy/generate_release_notes/models.py b/deploy/generate_release_notes/models.py new file mode 100644 index 00000000..1a16e4b1 --- /dev/null +++ b/deploy/generate_release_notes/models.py @@ -0,0 +1,79 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Data models for Data Commons Platform (DCP) release notes generation.""" + +from dataclasses import dataclass, field +from typing import Dict, List, Optional + + +@dataclass +class PullRequest: + """Represents a single merged GitHub Pull Request.""" + + number: int + title: str + body: str + author: str + url: str + merged_at: str + repo_name: str + labels: List[str] = field(default_factory=list) + files_changed: List[str] = field(default_factory=list) + commit_shas: List[str] = field(default_factory=list) + target_components: List[str] = field(default_factory=list) + + +@dataclass +class ComponentVersionInfo: + """Version, SHA, and timestamp details for a single component/image.""" + + component_id: str + component_name: str + repo_name: str + previous_version: str + new_version: str + previous_sha: Optional[str] = None + new_sha: Optional[str] = None + prev_timestamp: Optional[str] = None + new_timestamp: Optional[str] = None + image_uri: Optional[str] = None + + +@dataclass +class FeatureUpdate: + """Represents a synthesized feature update combining one or more PRs.""" + + id: str + title: str + description: str + category: str # e.g. "Spanner Graph & APIs", "Ingestion & Safety", "Search & Website", "Infra & Tooling" + target_components: List[str] = field(default_factory=list) + included_prs: List[int] = field(default_factory=list) + is_dcp_relevant: bool = True + breaking_changes: Optional[str] = None + + +@dataclass +class ReleaseInfoManifest: + """Container for all raw sourced information and mapped PRs for a release.""" + + previous_version: str + new_version: str + components: Dict[str, ComponentVersionInfo] = field(default_factory=dict) + pull_requests_by_component: Dict[str, List[PullRequest]] = field( + default_factory=dict + ) + all_pull_requests: List[PullRequest] = field(default_factory=list) + additional_instructions: Optional[str] = None diff --git a/deploy/generate_release_notes/pr_extractor.py b/deploy/generate_release_notes/pr_extractor.py new file mode 100644 index 00000000..e757d5ed --- /dev/null +++ b/deploy/generate_release_notes/pr_extractor.py @@ -0,0 +1,340 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Step 1: PR Extractor for Data Commons Platform (DCP) release notes generation. + +Extracts Pull Requests and image mappings across Data Commons repositories using +gcloud container image tags and GitHub CLI (gh pr list). +""" + +from datetime import datetime +import json +import logging +import subprocess +from typing import Dict, List, Optional, Set, Tuple + +from deploy.generate_release_notes.config import ( + COMPONENTS, + ComponentConfig, + SourceRule, +) +from deploy.generate_release_notes.models import ( + ComponentVersionInfo, + PullRequest, + ReleaseInfoManifest, +) + +logger = logging.getLogger(__name__) + + +def normalize_version(version: str) -> str: + """Strip leading 'v' if present to normalize version string (e.g. 'v1.1.2' -> '1.1.2').""" + return version[1:] if version.startswith("v") else version + + +def format_version_tag(version: str) -> str: + """Ensure leading 'v' is present for Git tags (e.g. '1.1.2' -> 'v1.1.2').""" + return version if version.startswith("v") else f"v{version}" + + +class PRExtractor: + """Extracts all PRs and image mappings between two release versions using gcloud and gh pr list.""" + + def __init__(self, use_cache: bool = True): + self.use_cache = use_cache + + def resolve_image_tag_info( + self, image_uri: str, version: str + ) -> Optional[Dict]: + """Resolves container image tag in Artifact Registry/GCR via gcloud container images list-tags. + + Returns dict with 'digest', 'tags', and 'timestamp' if found, else None. + """ + raw_version = normalize_version(version) + cmd = [ + "gcloud", + "container", + "images", + "list-tags", + image_uri, + f"--filter=tags={raw_version}", + "--format=json", + ] + try: + res = subprocess.run( + cmd, capture_output=True, text=True, check=True + ) + data = json.loads(res.stdout) + if data and isinstance(data, list) and len(data) > 0: + return data[0] + except Exception as e: + logger.warning( + f"Could not resolve tag '{raw_version}' for image '{image_uri}': {e}" + ) + return None + + def get_git_tag_timestamp( + self, repo: str, version: str + ) -> Optional[Tuple[str, str]]: + """Gets Git commit SHA and ISO timestamp for a git tag via gh api. + + Returns (commit_sha, iso_timestamp) or None. + """ + tag_name = format_version_tag(version) + cmd = [ + "gh", + "api", + f"repos/{repo}/git/matching-refs/tags/{tag_name}", + "--jq", + ".[0].object.sha", + ] + try: + res = subprocess.run( + cmd, capture_output=True, text=True, check=True + ) + sha = res.stdout.strip() + if not sha: + return None + + # Fetch commit details to get timestamp + commit_cmd = [ + "gh", + "api", + f"repos/{repo}/commits/{sha}", + "--jq", + ".commit.committer.date", + ] + commit_res = subprocess.run( + commit_cmd, capture_output=True, text=True, check=True + ) + timestamp = commit_res.stdout.strip() + return sha, timestamp + except Exception as e: + logger.warning( + f"Could not resolve git tag '{tag_name}' for repo '{repo}': {e}" + ) + return None + + def resolve_component_version( + self, comp: ComponentConfig, prev_version: str, new_version: str + ) -> ComponentVersionInfo: + """Resolves version info (SHAs, timestamps, image URI) for a component.""" + info = ComponentVersionInfo( + component_id=comp.id, + component_name=comp.name, + repo_name=comp.sources[0].repo if comp.sources else "", + previous_version=prev_version, + new_version=new_version, + image_uri=comp.image_uri, + ) + + # 1. Primary image resolution via gcloud container images list-tags + if comp.image_uri: + prev_data = self.resolve_image_tag_info( + comp.image_uri, prev_version + ) + new_data = self.resolve_image_tag_info(comp.image_uri, new_version) + + if prev_data and "timestamp" in prev_data: + info.prev_timestamp = prev_data["timestamp"].get("datetime") + info.previous_sha = prev_data.get("digest") + if new_data and "timestamp" in new_data: + info.new_timestamp = new_data["timestamp"].get("datetime") + info.new_sha = new_data.get("digest") + + # 2. Fallback to Git tag resolution for monorepo or missing image tags + if not info.prev_timestamp and comp.sources: + git_prev = self.get_git_tag_timestamp( + comp.sources[0].repo, prev_version + ) + if git_prev: + info.previous_sha, info.prev_timestamp = git_prev + + if not info.new_timestamp and comp.sources: + git_new = self.get_git_tag_timestamp( + comp.sources[0].repo, new_version + ) + if git_new: + info.new_sha, info.new_timestamp = git_new + + return info + + def fetch_prs_for_date_range( + self, repo: str, prev_timestamp: str, new_timestamp: str + ) -> List[PullRequest]: + """Fetches all merged PRs for a repository between prev_timestamp and new_timestamp in 1 single call. + + Executes `gh pr list --search 'merged:T_prev..T_new base:main'`. + """ + # Format timestamps for GitHub search API (YYYY-MM-DDTHH:MM:SSZ) + t_prev = ( + prev_timestamp.split(".")[0].replace(" ", "T") + if prev_timestamp + else "" + ) + t_new = ( + new_timestamp.split(".")[0].replace(" ", "T") + if new_timestamp + else "" + ) + + search_query = f"merged:{t_prev}..{t_new} base:main" + cmd = [ + "gh", + "pr", + "list", + "--repo", + repo, + "--state", + "merged", + "--search", + search_query, + "--json", + "number,title,body,author,url,labels,files,mergedAt", + "--limit", + "200", + ] + + logger.info(f"Fetching PRs for {repo} with query '{search_query}'...") + try: + res = subprocess.run( + cmd, capture_output=True, text=True, check=True + ) + raw_prs = json.loads(res.stdout) + prs: List[PullRequest] = [] + for item in raw_prs: + author_login = ( + item.get("author", {}).get("login", "unknown") + if isinstance(item.get("author"), dict) + else "unknown" + ) + labels = [ + l.get("name", "") + for l in item.get("labels", []) + if isinstance(l, dict) + ] + files = [ + f.get("path", "") + for f in item.get("files", []) + if isinstance(f, dict) + ] + + pr = PullRequest( + number=item["number"], + title=item.get("title", ""), + body=item.get("body", ""), + author=author_login, + url=item.get("url", ""), + merged_at=item.get("mergedAt", ""), + repo_name=repo, + labels=labels, + files_changed=files, + ) + prs.append(pr) + return prs + except Exception as e: + logger.error(f"Failed to fetch PRs for {repo}: {e}") + return [] + + def is_pr_matching_rule(self, pr: PullRequest, rule: SourceRule) -> bool: + """Checks if a PR matches a component's SourceRule (repo name and optional path filter).""" + if pr.repo_name != rule.repo: + return False + + # If no path filter, match all PRs in that repo + if not rule.path_filter: + return True + + # If path filter specified, check if any changed file matches + path = rule.path_filter.rstrip("/") + for f in pr.files_changed: + if f.startswith(path) or f.startswith(f"{path}/"): + return True + return False + + def extract( + self, + prev_version: str, + new_version: str, + additional_instructions: Optional[str] = None, + ) -> ReleaseInfoManifest: + """Main entry point: orchestrates tag resolution, date-range PR fetching, and manifest assembly.""" + logger.info( + f"Starting PR extraction for release range {prev_version} -> {new_version}..." + ) + + manifest = ReleaseInfoManifest( + previous_version=prev_version, + new_version=new_version, + additional_instructions=additional_instructions, + ) + + # 1. Resolve version info and timestamps across all components + repo_timestamps: Dict[str, List[str]] = {} + for comp_id, comp in COMPONENTS.items(): + comp_info = self.resolve_component_version( + comp, prev_version, new_version + ) + manifest.components[comp_id] = comp_info + + # Track timestamps per repo to compute widest range + for rule in comp.sources: + if rule.repo not in repo_timestamps: + repo_timestamps[rule.repo] = [] + if comp_info.prev_timestamp: + repo_timestamps[rule.repo].append(comp_info.prev_timestamp) + if comp_info.new_timestamp: + repo_timestamps[rule.repo].append(comp_info.new_timestamp) + + # 2. For each repository, compute min & max timestamps and fetch PRs in 1 single call + raw_prs_by_repo: Dict[str, List[PullRequest]] = {} + all_prs_set: Dict[Tuple[str, int], PullRequest] = {} + + for repo, ts_list in repo_timestamps.items(): + if not ts_list: + # Default to fallback timestamp if image tags missing + t_min = "2026-01-01T00:00:00Z" + t_max = datetime.utcnow().isoformat() + "Z" + else: + sorted_ts = sorted(ts_list) + t_min = sorted_ts[0] + t_max = sorted_ts[-1] + + prs = self.fetch_prs_for_date_range(repo, t_min, t_max) + raw_prs_by_repo[repo] = prs + for pr in prs: + all_prs_set[(pr.repo_name, pr.number)] = pr + + # 3. Map PRs to components based on SourceRules + for comp_id, comp in COMPONENTS.items(): + comp_prs: List[PullRequest] = [] + comp_info = manifest.components.get(comp_id) + + for rule in comp.sources: + prs_for_repo = raw_prs_by_repo.get(rule.repo, []) + for pr in prs_for_repo: + if self.is_pr_matching_rule(pr, rule): + # Add target component tag to PR + if comp_id not in pr.target_components: + pr.target_components.append(comp_id) + if pr not in comp_prs: + comp_prs.append(pr) + + manifest.pull_requests_by_component[comp_id] = comp_prs + + manifest.all_pull_requests = list(all_prs_set.values()) + logger.info( + f"Successfully extracted {len(manifest.all_pull_requests)} unique PRs across {len(manifest.components)} components." + ) + return manifest diff --git a/deploy/generate_release_notes/tests/__init__.py b/deploy/generate_release_notes/tests/__init__.py new file mode 100644 index 00000000..d6f7fe06 --- /dev/null +++ b/deploy/generate_release_notes/tests/__init__.py @@ -0,0 +1 @@ +"""Tests package for deploy/generate_release_notes.""" diff --git a/deploy/generate_release_notes/tests/test_pr_extractor.py b/deploy/generate_release_notes/tests/test_pr_extractor.py new file mode 100644 index 00000000..1750019b --- /dev/null +++ b/deploy/generate_release_notes/tests/test_pr_extractor.py @@ -0,0 +1,171 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit and Integration tests for PR Extractor (deploy/generate_release_notes/pr_extractor.py).""" + +import json +import os +from unittest.mock import MagicMock, patch +import pytest + +from deploy.generate_release_notes.config import COMPONENTS, SourceRule +from deploy.generate_release_notes.models import PullRequest, ReleaseInfoManifest +from deploy.generate_release_notes.pr_extractor import PRExtractor + + +class TestPRExtractorUnit: + """Unit tests for PRExtractor helper functions and SourceRule matching.""" + + def test_source_rule_path_filtering(self): + """Test multi-source path filtering rules across website and import repos.""" + extractor = PRExtractor() + + # 1. Website PR modifying build/cdc_data/ -> Preprocessor + pr_cdc_data = PullRequest( + number=101, + title="Update cdc_data Dockerfile", + body="", + author="testuser", + url="https://github.com/datacommonsorg/website/pull/101", + merged_at="2026-07-20T10:00:00Z", + repo_name="datacommonsorg/website", + files_changed=["build/cdc_data/Dockerfile", "build/cdc_data/run.sh"], + ) + + # 2. Website PR modifying server/ -> Services + pr_website_server = PullRequest( + number=102, + title="Update Flask routes", + body="", + author="testuser", + url="https://github.com/datacommonsorg/website/pull/102", + merged_at="2026-07-21T10:00:00Z", + repo_name="datacommonsorg/website", + files_changed=["server/routes.py", "static/js/app.js"], + ) + + # 3. Import PR modifying simple/ -> Preprocessor + pr_import_simple = PullRequest( + number=201, + title="Update CSV parser in simple importer", + body="", + author="testuser", + url="https://github.com/datacommonsorg/import/pull/201", + merged_at="2026-07-22T10:00:00Z", + repo_name="datacommonsorg/import", + files_changed=["simple/parser.py", "simple/main.go"], + ) + + # 4. Import PR modifying pipeline/workflow/ingestion-helper/ -> Ingestion Helper + pr_import_ingestion = PullRequest( + number=202, + title="Fix ingestion helper Spanner query", + body="", + author="testuser", + url="https://github.com/datacommonsorg/import/pull/202", + merged_at="2026-07-23T10:00:00Z", + repo_name="datacommonsorg/import", + files_changed=["pipeline/workflow/ingestion-helper/main.go"], + ) + + # Rules + rule_prep_website = SourceRule(repo="datacommonsorg/website", path_filter="build/cdc_data/") + rule_services_website = SourceRule(repo="datacommonsorg/website", path_filter=None) + rule_prep_import = SourceRule(repo="datacommonsorg/import", path_filter="simple/") + rule_ingestion_helper = SourceRule(repo="datacommonsorg/import", path_filter="pipeline/workflow/ingestion-helper/") + + # Assertions + assert extractor.is_pr_matching_rule(pr_cdc_data, rule_prep_website) is True + assert extractor.is_pr_matching_rule(pr_website_server, rule_prep_website) is False + assert extractor.is_pr_matching_rule(pr_website_server, rule_services_website) is True + + assert extractor.is_pr_matching_rule(pr_import_simple, rule_prep_import) is True + assert extractor.is_pr_matching_rule(pr_import_ingestion, rule_prep_import) is False + assert extractor.is_pr_matching_rule(pr_import_ingestion, rule_ingestion_helper) is True + + @patch("deploy.generate_release_notes.pr_extractor.subprocess.run") + def test_gcloud_image_tag_resolution(self, mock_run): + """Test resolving container image tags via gcloud list-tags mock.""" + mock_output = [ + { + "digest": "sha256:1234567890abcdef", + "tags": ["1.1.1", "latest"], + "timestamp": {"datetime": "2026-07-15 12:00:00-07:00"}, + } + ] + mock_res = MagicMock() + mock_res.stdout = json.dumps(mock_output) + mock_run.return_value = mock_res + + extractor = PRExtractor() + info = extractor.resolve_image_tag_info("gcr.io/datcom-ci/datacommons-services", "1.1.1") + + assert info is not None + assert info["digest"] == "sha256:1234567890abcdef" + assert info["timestamp"]["datetime"] == "2026-07-15 12:00:00-07:00" + + +class TestPRExtractorIntegration: + """Integration test executing PRExtractor against real GitHub repositories.""" + + @pytest.mark.integration + def test_real_pr_extraction_v1_1_0_to_v1_1_1(self): + """Extracts PRs between v1.1.0 and v1.1.1 across public Data Commons repos.""" + extractor = PRExtractor() + manifest = extractor.extract( + prev_version="v1.1.0", + new_version="v1.1.1", + additional_instructions="Integration test run for v1.1.0 -> v1.1.1", + ) + + assert isinstance(manifest, ReleaseInfoManifest) + assert manifest.previous_version == "v1.1.0" + assert manifest.new_version == "v1.1.1" + assert len(manifest.components) > 0 + + # Dump manifest to /tmp for schema inspection + output_file = "/tmp/test_manifest_v1.1.1.json" + manifest_dict = { + "previous_version": manifest.previous_version, + "new_version": manifest.new_version, + "total_prs_extracted": len(manifest.all_pull_requests), + "components": { + k: { + "id": v.component_id, + "name": v.component_name, + "prev_timestamp": v.prev_timestamp, + "new_timestamp": v.new_timestamp, + } + for k, v in manifest.components.items() + }, + "pull_requests_count_by_component": { + k: len(v) for k, v in manifest.pull_requests_by_component.items() + }, + "sample_prs": [ + { + "number": pr.number, + "title": pr.title, + "author": pr.author, + "repo": pr.repo_name, + "target_components": pr.target_components, + } + for pr in manifest.all_pull_requests[:10] + ], + } + + with open(output_file, "w") as f: + json.dump(manifest_dict, f, indent=2) + + print(f"\nSaved integration test manifest summary to {output_file}") + assert os.path.exists(output_file) diff --git a/pyproject.toml b/pyproject.toml index 309df685..47da7b9f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,13 @@ dependencies = [ [dependency-groups] dev = [ {include-group = "lint"}, - {include-group = "test"} + {include-group = "test"}, + {include-group = "generate-release-notes"} +] +generate-release-notes = [ + "google-genai>=0.1.0", + "click>=8.1.7", + "jinja2>=3.1.0", ] lint = [ "pre-commit>=4.5.1", @@ -145,8 +151,9 @@ build-backend = "setuptools.build_meta" # Tell setuptools this is a meta-package and to stop looking for code packages = [] +[tool.pytest.ini_options] +pythonpath = ["."] +testpaths = ["packages/*/tests", "deploy/generate_release_notes/tests"] + [tool.setuptools.dynamic] version = {file = "VERSION"} - -[tool.pytest.ini_options] -testpaths = ["packages"] diff --git a/uv.lock b/uv.lock index fdb58b1b..62377e6c 100644 --- a/uv.lock +++ b/uv.lock @@ -425,14 +425,22 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "click" }, { name = "datacommons-admin" }, { name = "datacommons-api" }, { name = "datacommons-db" }, { name = "datacommons-schema" }, + { name = "google-genai" }, + { name = "jinja2" }, { name = "pre-commit" }, { name = "pytest" }, { name = "ruff" }, ] +generate-release-notes = [ + { name = "click" }, + { name = "google-genai" }, + { name = "jinja2" }, +] lint = [ { name = "pre-commit" }, { name = "ruff" }, @@ -450,14 +458,22 @@ requires-dist = [{ name = "datacommons-cli", editable = "packages/datacommons-cl [package.metadata.requires-dev] dev = [ + { name = "click", specifier = ">=8.1.7" }, { name = "datacommons-admin", editable = "packages/datacommons-admin" }, { name = "datacommons-api", editable = "packages/datacommons-api" }, { name = "datacommons-db", editable = "packages/datacommons-db" }, { name = "datacommons-schema", editable = "packages/datacommons-schema" }, + { name = "google-genai", specifier = ">=0.1.0" }, + { name = "jinja2", specifier = ">=3.1.0" }, { name = "pre-commit", specifier = ">=4.5.1" }, { name = "pytest", specifier = ">=9.0.3" }, { name = "ruff", specifier = ">=0.15.0" }, ] +generate-release-notes = [ + { name = "click", specifier = ">=8.1.7" }, + { name = "google-genai", specifier = ">=0.1.0" }, + { name = "jinja2", specifier = ">=3.1.0" }, +] lint = [ { name = "pre-commit", specifier = ">=4.5.1" }, { name = "ruff", specifier = ">=0.15.0" }, @@ -505,6 +521,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, ] +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, +] + [[package]] name = "fastapi" version = "0.128.6" @@ -554,16 +579,20 @@ grpc = [ [[package]] name = "google-auth" -version = "2.48.0" +version = "2.56.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography" }, { name = "pyasn1-modules" }, - { name = "rsa" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0c/41/242044323fbd746615884b1c16639749e73665b718209946ebad7ba8a813/google_auth-2.48.0.tar.gz", hash = "sha256:4f7e706b0cd3208a3d940a19a822c37a476ddba5450156c3e6624a71f7c841ce", size = 326522, upload-time = "2026-01-26T19:22:47.157Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/33/dbc946a407401b975f0719658f18e664ece2109f79ffd1ff3bf226c205f4/google_auth-2.56.2.tar.gz", hash = "sha256:e28f103ca8091fb7012b99c44243d7366c29863713b8e34a220c3322b7a07051", size = 365820, upload-time = "2026-07-21T21:53:28.188Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/83/1d/d6466de3a5249d35e832a52834115ca9d1d0de6abc22065f049707516d47/google_auth-2.48.0-py3-none-any.whl", hash = "sha256:2e2a537873d449434252a9632c28bfc268b0adb1e53f9fb62afc5333a975903f", size = 236499, upload-time = "2026-01-26T19:22:45.099Z" }, + { url = "https://files.pythonhosted.org/packages/88/63/50636aae68c9bf17c891c7eb18b49baa9bd6b31d2a97b8de4813a9fc8d1c/google_auth-2.56.2-py3-none-any.whl", hash = "sha256:c8270ea95b2697b74e3d8438ae9c5b898e38b623b915c7b5c5635921e7de68a6", size = 258588, upload-time = "2026-07-21T21:53:26.399Z" }, +] + +[package.optional-dependencies] +requests = [ + { name = "requests" }, ] [[package]] @@ -668,6 +697,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9c/97/7d75fe37a7a6ed171a2cf17117177e7aab7e6e0d115858741b41e9dd4254/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c", size = 28800, upload-time = "2025-12-16T00:40:30.322Z" }, ] +[[package]] +name = "google-genai" +version = "2.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "google-auth", extra = ["requests"] }, + { name = "httpx" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "sniffio" }, + { name = "tenacity" }, + { name = "typing-extensions" }, + { name = "websockets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/53/b2c9b0a74b817a393d388a2303ec4da8bda27ea744b23914480d1b024d84/google_genai-2.15.0.tar.gz", hash = "sha256:ef71bdb79ce9931bca1cf0a393c8cfb606e1075b6100fcdde02b7b467db8235d", size = 640674, upload-time = "2026-07-29T17:43:21.853Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/86/2ef6d992955307bf525f305b44cb3e2066eb01636549607082a0407f7047/google_genai-2.15.0-py3-none-any.whl", hash = "sha256:f322a94c3c1ddb1b1cc536086708f5f8e13101347d062f63dcd7947b598c1d16", size = 1030459, upload-time = "2026-07-29T17:43:20.286Z" }, +] + [[package]] name = "google-resumable-media" version = "2.9.0" @@ -706,6 +756,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/e8/2e1462c8fdbe0f210feb5ac7ad2d9029af8be3bf45bd9fa39765f821642f/greenlet-3.3.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:5fd23b9bc6d37b563211c6abbb1b3cab27db385a4449af5c32e932f93017080c", size = 274974, upload-time = "2026-01-23T15:31:02.891Z" }, { url = "https://files.pythonhosted.org/packages/7e/a8/530a401419a6b302af59f67aaf0b9ba1015855ea7e56c036b5928793c5bd/greenlet-3.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09f51496a0bfbaa9d74d36a52d2580d1ef5ed4fdfcff0a73730abfbbbe1403dd", size = 577175, upload-time = "2026-01-23T16:00:56.213Z" }, { url = "https://files.pythonhosted.org/packages/8e/89/7e812bb9c05e1aaef9b597ac1d0962b9021d2c6269354966451e885c4e6b/greenlet-3.3.1-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb0feb07fe6e6a74615ee62a880007d976cf739b6669cce95daa7373d4fc69c5", size = 590401, upload-time = "2026-01-23T16:05:26.365Z" }, + { url = "https://files.pythonhosted.org/packages/70/ae/e2d5f0e59b94a2269b68a629173263fa40b63da32f5c231307c349315871/greenlet-3.3.1-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:67ea3fc73c8cd92f42467a72b75e8f05ed51a0e9b1d15398c913416f2dafd49f", size = 601161, upload-time = "2026-01-23T16:15:53.456Z" }, { url = "https://files.pythonhosted.org/packages/5c/ae/8d472e1f5ac5efe55c563f3eabb38c98a44b832602e12910750a7c025802/greenlet-3.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:39eda9ba259cc9801da05351eaa8576e9aa83eb9411e8f0c299e05d712a210f2", size = 590272, upload-time = "2026-01-23T15:32:49.411Z" }, { url = "https://files.pythonhosted.org/packages/a8/51/0fde34bebfcadc833550717eade64e35ec8738e6b097d5d248274a01258b/greenlet-3.3.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e2e7e882f83149f0a71ac822ebf156d902e7a5d22c9045e3e0d1daf59cee2cc9", size = 1550729, upload-time = "2026-01-23T16:04:20.867Z" }, { url = "https://files.pythonhosted.org/packages/16/c9/2fb47bee83b25b119d5a35d580807bb8b92480a54b68fef009a02945629f/greenlet-3.3.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:80aa4d79eb5564f2e0a6144fcc744b5a37c56c4a92d60920720e99210d88db0f", size = 1615552, upload-time = "2026-01-23T15:33:45.743Z" }, @@ -714,6 +765,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/c8/9d76a66421d1ae24340dfae7e79c313957f6e3195c144d2c73333b5bfe34/greenlet-3.3.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:7e806ca53acf6d15a888405880766ec84721aa4181261cd11a457dfe9a7a4975", size = 276443, upload-time = "2026-01-23T15:30:10.066Z" }, { url = "https://files.pythonhosted.org/packages/81/99/401ff34bb3c032d1f10477d199724f5e5f6fbfb59816ad1455c79c1eb8e7/greenlet-3.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d842c94b9155f1c9b3058036c24ffb8ff78b428414a19792b2380be9cecf4f36", size = 597359, upload-time = "2026-01-23T16:00:57.394Z" }, { url = "https://files.pythonhosted.org/packages/2b/bc/4dcc0871ed557792d304f50be0f7487a14e017952ec689effe2180a6ff35/greenlet-3.3.1-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:20fedaadd422fa02695f82093f9a98bad3dab5fcda793c658b945fcde2ab27ba", size = 607805, upload-time = "2026-01-23T16:05:28.068Z" }, + { url = "https://files.pythonhosted.org/packages/3b/cd/7a7ca57588dac3389e97f7c9521cb6641fd8b6602faf1eaa4188384757df/greenlet-3.3.1-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c620051669fd04ac6b60ebc70478210119c56e2d5d5df848baec4312e260e4ca", size = 622363, upload-time = "2026-01-23T16:15:54.754Z" }, { url = "https://files.pythonhosted.org/packages/cf/05/821587cf19e2ce1f2b24945d890b164401e5085f9d09cbd969b0c193cd20/greenlet-3.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14194f5f4305800ff329cbf02c5fcc88f01886cadd29941b807668a45f0d2336", size = 609947, upload-time = "2026-01-23T15:32:51.004Z" }, { url = "https://files.pythonhosted.org/packages/a4/52/ee8c46ed9f8babaa93a19e577f26e3d28a519feac6350ed6f25f1afee7e9/greenlet-3.3.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7b2fe4150a0cf59f847a67db8c155ac36aed89080a6a639e9f16df5d6c6096f1", size = 1567487, upload-time = "2026-01-23T16:04:22.125Z" }, { url = "https://files.pythonhosted.org/packages/8f/7c/456a74f07029597626f3a6db71b273a3632aecb9afafeeca452cfa633197/greenlet-3.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:49f4ad195d45f4a66a0eb9c1ba4832bb380570d361912fa3554746830d332149", size = 1636087, upload-time = "2026-01-23T15:33:47.486Z" }, @@ -722,6 +774,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/ab/d26750f2b7242c2b90ea2ad71de70cfcd73a948a49513188a0fc0d6fc15a/greenlet-3.3.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:7ab327905cabb0622adca5971e488064e35115430cec2c35a50fd36e72a315b3", size = 275205, upload-time = "2026-01-23T15:30:24.556Z" }, { url = "https://files.pythonhosted.org/packages/10/d3/be7d19e8fad7c5a78eeefb2d896a08cd4643e1e90c605c4be3b46264998f/greenlet-3.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:65be2f026ca6a176f88fb935ee23c18333ccea97048076aef4db1ef5bc0713ac", size = 599284, upload-time = "2026-01-23T16:00:58.584Z" }, { url = "https://files.pythonhosted.org/packages/ae/21/fe703aaa056fdb0f17e5afd4b5c80195bbdab701208918938bd15b00d39b/greenlet-3.3.1-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7a3ae05b3d225b4155bda56b072ceb09d05e974bc74be6c3fc15463cf69f33fd", size = 610274, upload-time = "2026-01-23T16:05:29.312Z" }, + { url = "https://files.pythonhosted.org/packages/06/00/95df0b6a935103c0452dad2203f5be8377e551b8466a29650c4c5a5af6cc/greenlet-3.3.1-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:12184c61e5d64268a160226fb4818af4df02cfead8379d7f8b99a56c3a54ff3e", size = 624375, upload-time = "2026-01-23T16:15:55.915Z" }, { url = "https://files.pythonhosted.org/packages/cb/86/5c6ab23bb3c28c21ed6bebad006515cfe08b04613eb105ca0041fecca852/greenlet-3.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6423481193bbbe871313de5fd06a082f2649e7ce6e08015d2a76c1e9186ca5b3", size = 612904, upload-time = "2026-01-23T15:32:52.317Z" }, { url = "https://files.pythonhosted.org/packages/c2/f3/7949994264e22639e40718c2daf6f6df5169bf48fb038c008a489ec53a50/greenlet-3.3.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:33a956fe78bbbda82bfc95e128d61129b32d66bcf0a20a1f0c08aa4839ffa951", size = 1567316, upload-time = "2026-01-23T16:04:23.316Z" }, { url = "https://files.pythonhosted.org/packages/8d/6e/d73c94d13b6465e9f7cd6231c68abde838bb22408596c05d9059830b7872/greenlet-3.3.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4b065d3284be43728dd280f6f9a13990b56470b81be20375a207cdc814a983f2", size = 1636549, upload-time = "2026-01-23T15:33:48.643Z" }, @@ -730,6 +783,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/fb/011c7c717213182caf78084a9bea51c8590b0afda98001f69d9f853a495b/greenlet-3.3.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:bd59acd8529b372775cd0fcbc5f420ae20681c5b045ce25bd453ed8455ab99b5", size = 275737, upload-time = "2026-01-23T15:32:16.889Z" }, { url = "https://files.pythonhosted.org/packages/41/2e/a3a417d620363fdbb08a48b1dd582956a46a61bf8fd27ee8164f9dfe87c2/greenlet-3.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b31c05dd84ef6871dd47120386aed35323c944d86c3d91a17c4b8d23df62f15b", size = 646422, upload-time = "2026-01-23T16:01:00.354Z" }, { url = "https://files.pythonhosted.org/packages/b4/09/c6c4a0db47defafd2d6bab8ddfe47ad19963b4e30f5bed84d75328059f8c/greenlet-3.3.1-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:02925a0bfffc41e542c70aa14c7eda3593e4d7e274bfcccca1827e6c0875902e", size = 658219, upload-time = "2026-01-23T16:05:30.956Z" }, + { url = "https://files.pythonhosted.org/packages/e2/89/b95f2ddcc5f3c2bc09c8ee8d77be312df7f9e7175703ab780f2014a0e781/greenlet-3.3.1-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3e0f3878ca3a3ff63ab4ea478585942b53df66ddde327b59ecb191b19dbbd62d", size = 671455, upload-time = "2026-01-23T16:15:57.232Z" }, { url = "https://files.pythonhosted.org/packages/80/38/9d42d60dffb04b45f03dbab9430898352dba277758640751dc5cc316c521/greenlet-3.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34a729e2e4e4ffe9ae2408d5ecaf12f944853f40ad724929b7585bca808a9d6f", size = 660237, upload-time = "2026-01-23T15:32:53.967Z" }, { url = "https://files.pythonhosted.org/packages/96/61/373c30b7197f9e756e4c81ae90a8d55dc3598c17673f91f4d31c3c689c3f/greenlet-3.3.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:aec9ab04e82918e623415947921dea15851b152b822661cce3f8e4393c3df683", size = 1615261, upload-time = "2026-01-23T16:04:25.066Z" }, { url = "https://files.pythonhosted.org/packages/fd/d3/ca534310343f5945316f9451e953dcd89b36fe7a19de652a1dc5a0eeef3f/greenlet-3.3.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:71c767cf281a80d02b6c1bdc41c9468e1f5a494fb11bc8688c360524e273d7b1", size = 1683719, upload-time = "2026-01-23T15:33:50.61Z" }, @@ -738,6 +792,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/28/24/cbbec49bacdcc9ec652a81d3efef7b59f326697e7edf6ed775a5e08e54c2/greenlet-3.3.1-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:3e63252943c921b90abb035ebe9de832c436401d9c45f262d80e2d06cc659242", size = 282706, upload-time = "2026-01-23T15:33:05.525Z" }, { url = "https://files.pythonhosted.org/packages/86/2e/4f2b9323c144c4fe8842a4e0d92121465485c3c2c5b9e9b30a52e80f523f/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:76e39058e68eb125de10c92524573924e827927df5d3891fbc97bd55764a8774", size = 651209, upload-time = "2026-01-23T16:01:01.517Z" }, { url = "https://files.pythonhosted.org/packages/d9/87/50ca60e515f5bb55a2fbc5f0c9b5b156de7d2fc51a0a69abc9d23914a237/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9f9d5e7a9310b7a2f416dd13d2e3fd8b42d803968ea580b7c0f322ccb389b97", size = 654300, upload-time = "2026-01-23T16:05:32.199Z" }, + { url = "https://files.pythonhosted.org/packages/7c/25/c51a63f3f463171e09cb586eb64db0861eb06667ab01a7968371a24c4f3b/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b9721549a95db96689458a1e0ae32412ca18776ed004463df3a9299c1b257ab", size = 662574, upload-time = "2026-01-23T16:15:58.364Z" }, { url = "https://files.pythonhosted.org/packages/1d/94/74310866dfa2b73dd08659a3d18762f83985ad3281901ba0ee9a815194fb/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92497c78adf3ac703b57f1e3813c2d874f27f71a178f9ea5887855da413cd6d2", size = 653842, upload-time = "2026-01-23T15:32:55.671Z" }, { url = "https://files.pythonhosted.org/packages/97/43/8bf0ffa3d498eeee4c58c212a3905dd6146c01c8dc0b0a046481ca29b18c/greenlet-3.3.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ed6b402bc74d6557a705e197d47f9063733091ed6357b3de33619d8a8d93ac53", size = 1614917, upload-time = "2026-01-23T16:04:26.276Z" }, { url = "https://files.pythonhosted.org/packages/89/90/a3be7a5f378fc6e84abe4dcfb2ba32b07786861172e502388b4c90000d1b/greenlet-3.3.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:59913f1e5ada20fde795ba906916aea25d442abcc0593fba7e26c92b7ad76249", size = 1676092, upload-time = "2026-01-23T15:33:52.176Z" }, @@ -853,6 +908,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7d/c9/f6e1e8567660bc5b0aba281f2b0017b2a7665fcad6bf3ed67286a0c72cd4/html5rdf-1.2.1-py2.py3-none-any.whl", hash = "sha256:1f519121bc366af3e485310dc8041d2e86e5173c1a320fac3dc9d2604069b83e", size = 109765, upload-time = "2024-10-30T05:06:52.507Z" }, ] +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + [[package]] name = "httptools" version = "0.7.1" @@ -889,6 +957,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/53/cf/878f3b91e4e6e011eff6d1fa9ca39f7eb17d19c9d7971b04873734112f30/httptools-0.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:cfabda2a5bb85aa2a904ce06d974a3f30fb36cc63d7feaddec05d2050acede96", size = 88205, upload-time = "2025-10-10T03:55:00.389Z" }, ] +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + [[package]] name = "identify" version = "2.6.16" @@ -1581,18 +1664,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/56/5d/c814546c2333ceea4ba42262d8c4d55763003e767fa169adc693bd524478/requests-2.33.0-py3-none-any.whl", hash = "sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b", size = 65017, upload-time = "2026-03-25T15:10:40.382Z" }, ] -[[package]] -name = "rsa" -version = "4.9.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyasn1" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/da/8a/22b7beea3ee0d44b1916c0c1cb0ee3af23b700b6da9f04991899d0c555d4/rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75", size = 29034, upload-time = "2025-04-16T09:51:18.218Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", size = 34696, upload-time = "2025-04-16T09:51:17.142Z" }, -] - [[package]] name = "ruff" version = "0.15.0" @@ -1627,6 +1698,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/23/63/5517029d6696ddf2bd378d46f63f479be001c31b462303170a1da57650cb/setuptools-80.0.0-py3-none-any.whl", hash = "sha256:a38f898dcd6e5380f4da4381a87ec90bd0a7eec23d204a5552e80ee3cab6bd27", size = 1240907, upload-time = "2025-04-27T17:21:09.175Z" }, ] +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + [[package]] name = "sqlalchemy" version = "2.0.46" @@ -1712,6 +1792,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/0d/13d1d239a25cbfb19e740db83143e95c772a1fe10202dda4b76792b114dd/starlette-0.52.1-py3-none-any.whl", hash = "sha256:0029d43eb3d273bc4f83a08720b4912ea4b071087a3b48db01b7c839f7954d74", size = 74272, upload-time = "2026-01-18T13:34:09.188Z" }, ] +[[package]] +name = "tenacity" +version = "9.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" From 81bdb1282e5568e7a07f87c6a6c0435f7336e845 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 18:47:44 -0700 Subject: [PATCH 02/70] fix(deploy): enforce strict timestamp and tag resolution in PRExtractor to exclude post-release PRs --- deploy/generate_release_notes/pr_extractor.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/deploy/generate_release_notes/pr_extractor.py b/deploy/generate_release_notes/pr_extractor.py index e757d5ed..7de3096d 100644 --- a/deploy/generate_release_notes/pr_extractor.py +++ b/deploy/generate_release_notes/pr_extractor.py @@ -214,6 +214,15 @@ def fetch_prs_for_date_range( raw_prs = json.loads(res.stdout) prs: List[PullRequest] = [] for item in raw_prs: + merged_at = item.get("mergedAt", "") + # Strict timestamp check: omit PRs merged after new_timestamp if new_timestamp is set + if new_timestamp and merged_at: + if merged_at > t_new: + continue + if prev_timestamp and merged_at: + if merged_at < t_prev: + continue + author_login = ( item.get("author", {}).get("login", "unknown") if isinstance(item.get("author"), dict) @@ -236,7 +245,7 @@ def fetch_prs_for_date_range( body=item.get("body", ""), author=author_login, url=item.get("url", ""), - merged_at=item.get("mergedAt", ""), + merged_at=merged_at, repo_name=repo, labels=labels, files_changed=files, From 40fbc807623ee267f2cbc70fceecc40b4a55ef17 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 18:50:33 -0700 Subject: [PATCH 03/70] fix(deploy): fail explicitly when required container image tag or git tag is missing --- deploy/generate_release_notes/pr_extractor.py | 50 ++++++++++++------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/deploy/generate_release_notes/pr_extractor.py b/deploy/generate_release_notes/pr_extractor.py index 7de3096d..cdbb744c 100644 --- a/deploy/generate_release_notes/pr_extractor.py +++ b/deploy/generate_release_notes/pr_extractor.py @@ -139,34 +139,48 @@ def resolve_component_version( image_uri=comp.image_uri, ) - # 1. Primary image resolution via gcloud container images list-tags + # 1. Image-based component resolution (must find tag in Container Registry) if comp.image_uri: prev_data = self.resolve_image_tag_info( comp.image_uri, prev_version ) new_data = self.resolve_image_tag_info(comp.image_uri, new_version) - if prev_data and "timestamp" in prev_data: - info.prev_timestamp = prev_data["timestamp"].get("datetime") + if not prev_data: + logger.warning( + f"Container image tag '{prev_version}' not found for {comp.name} at {comp.image_uri}" + ) + else: + info.prev_timestamp = prev_data.get("timestamp", {}).get("datetime") info.previous_sha = prev_data.get("digest") - if new_data and "timestamp" in new_data: - info.new_timestamp = new_data["timestamp"].get("datetime") + + if not new_data: + raise ValueError( + f"Required container image tag '{new_version}' not found for '{comp.name}' at {comp.image_uri}. " + f"Ensure the release container image has been built and tagged before generating release notes." + ) + else: + info.new_timestamp = new_data.get("timestamp", {}).get("datetime") info.new_sha = new_data.get("digest") - # 2. Fallback to Git tag resolution for monorepo or missing image tags - if not info.prev_timestamp and comp.sources: - git_prev = self.get_git_tag_timestamp( - comp.sources[0].repo, prev_version - ) - if git_prev: - info.previous_sha, info.prev_timestamp = git_prev + # 2. Non-image component resolution (e.g. monorepo packages, infra) via Git tags + else: + if comp.sources: + git_prev = self.get_git_tag_timestamp( + comp.sources[0].repo, prev_version + ) + if git_prev: + info.previous_sha, info.prev_timestamp = git_prev - if not info.new_timestamp and comp.sources: - git_new = self.get_git_tag_timestamp( - comp.sources[0].repo, new_version - ) - if git_new: - info.new_sha, info.new_timestamp = git_new + git_new = self.get_git_tag_timestamp( + comp.sources[0].repo, new_version + ) + if git_new: + info.new_sha, info.new_timestamp = git_new + else: + raise ValueError( + f"Required Git tag '{new_version}' not found for '{comp.name}' in repo '{comp.sources[0].repo}'." + ) return info From b02d3b71d2b85a77bbd24df954c29c3c96256853 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 18:51:55 -0700 Subject: [PATCH 04/70] feat(deploy): add skip_missing_images flag to PRExtractor to allow bypassing missing image errors --- deploy/generate_release_notes/pr_extractor.py | 17 ++++++++++++----- .../tests/test_pr_extractor.py | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/deploy/generate_release_notes/pr_extractor.py b/deploy/generate_release_notes/pr_extractor.py index cdbb744c..3281a458 100644 --- a/deploy/generate_release_notes/pr_extractor.py +++ b/deploy/generate_release_notes/pr_extractor.py @@ -51,8 +51,9 @@ def format_version_tag(version: str) -> str: class PRExtractor: """Extracts all PRs and image mappings between two release versions using gcloud and gh pr list.""" - def __init__(self, use_cache: bool = True): + def __init__(self, use_cache: bool = True, skip_missing_images: bool = False): self.use_cache = use_cache + self.skip_missing_images = skip_missing_images def resolve_image_tag_info( self, image_uri: str, version: str @@ -155,10 +156,16 @@ def resolve_component_version( info.previous_sha = prev_data.get("digest") if not new_data: - raise ValueError( - f"Required container image tag '{new_version}' not found for '{comp.name}' at {comp.image_uri}. " - f"Ensure the release container image has been built and tagged before generating release notes." - ) + if self.skip_missing_images: + logger.warning( + f"Skipping missing image component '{comp.name}' ({comp.image_uri}:{new_version}) because skip_missing_images is set." + ) + else: + raise ValueError( + f"Required container image tag '{new_version}' not found for '{comp.name}' at {comp.image_uri}. " + f"Ensure the release container image has been built and tagged before generating release notes, " + f"or pass --skip-missing-images / skip_missing_images=True to bypass." + ) else: info.new_timestamp = new_data.get("timestamp", {}).get("datetime") info.new_sha = new_data.get("digest") diff --git a/deploy/generate_release_notes/tests/test_pr_extractor.py b/deploy/generate_release_notes/tests/test_pr_extractor.py index 1750019b..9b67ed17 100644 --- a/deploy/generate_release_notes/tests/test_pr_extractor.py +++ b/deploy/generate_release_notes/tests/test_pr_extractor.py @@ -115,6 +115,22 @@ def test_gcloud_image_tag_resolution(self, mock_run): assert info["digest"] == "sha256:1234567890abcdef" assert info["timestamp"]["datetime"] == "2026-07-15 12:00:00-07:00" + @patch("deploy.generate_release_notes.pr_extractor.PRExtractor.resolve_image_tag_info") + def test_missing_image_tag_handling(self, mock_resolve): + """Test error raising and bypass flag when container image tag is missing.""" + mock_resolve.return_value = None # Image tag not found + comp = COMPONENTS["services"] + + # 1. Default mode: should raise ValueError + extractor_strict = PRExtractor(skip_missing_images=False) + with pytest.raises(ValueError, match="Required container image tag '9.9.9' not found"): + extractor_strict.resolve_component_version(comp, "1.1.0", "9.9.9") + + # 2. Skip mode: should log warning and not raise error + extractor_permissive = PRExtractor(skip_missing_images=True) + info = extractor_permissive.resolve_component_version(comp, "1.1.0", "9.9.9") + assert info.new_timestamp is None + class TestPRExtractorIntegration: """Integration test executing PRExtractor against real GitHub repositories.""" From 909db08a04d3fb18e5981de35e5bbd066de76ced Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 18:57:01 -0700 Subject: [PATCH 05/70] feat(deploy): implement Step 2 Feature Extractor with Two-Stage Gemini pipeline (Flash + Pro) - Add Two-Stage Gemini LLM Pipeline in feature_extractor.py (Stage 1 Flash noise filter + Stage 2 Pro synthesis and SOP classification) - Add unit tests with mocked Gemini Client in test_feature_extractor.py --- .../feature_extractor.py | 260 ++++++++++++++++++ .../tests/test_feature_extractor.py | 221 +++++++++++++++ 2 files changed, 481 insertions(+) create mode 100644 deploy/generate_release_notes/feature_extractor.py create mode 100644 deploy/generate_release_notes/tests/test_feature_extractor.py diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py new file mode 100644 index 00000000..5501510e --- /dev/null +++ b/deploy/generate_release_notes/feature_extractor.py @@ -0,0 +1,260 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Step 2: Feature Extractor for Data Commons Platform (DCP) release notes generation. + +Executes a Two-Stage Gemini LLM Pipeline (Flash + Pro) for noise filtering, +SOP classification, feature grouping, and technical release notes synthesis. +""" + +import json +import logging +import os +from typing import Dict, List, Optional, Any + +from google import genai +from google.genai import types + +from deploy.generate_release_notes.models import ( + FeatureUpdate, + PullRequest, + ReleaseInfoManifest, +) + +logger = logging.getLogger(__name__) + +DEFAULT_FILTER_MODEL = "gemini-2.5-flash" +DEFAULT_SYNTHESIS_MODEL = "gemini-2.5-pro" + + +class FeatureExtractor: + """Two-Stage Gemini LLM Pipeline for filtering, classifying, and synthesizing DCP release features.""" + + def __init__( + self, + api_key: Optional[str] = None, + filter_model: str = DEFAULT_FILTER_MODEL, + synthesis_model: str = DEFAULT_SYNTHESIS_MODEL, + ): + key = api_key or os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY") + if not key: + logger.warning( + "Neither GEMINI_API_KEY nor GOOGLE_API_KEY set in environment. Gemini API calls will fail if not authenticated via GCP default credentials." + ) + self.client = genai.Client() + else: + self.client = genai.Client(api_key=key) + + self.filter_model = filter_model + self.synthesis_model = synthesis_model + + def filter_prs_with_flash(self, manifest: ReleaseInfoManifest) -> List[PullRequest]: + """Stage 1 (Flash Model): Rapidly triages all raw PRs to weed out bot bumps, typo fixes, and non-informative noise.""" + if not manifest.all_pull_requests: + logger.warning("No PRs provided in manifest for Stage 1 filtering.") + return [] + + logger.info( + f"Stage 1: Filtering {len(manifest.all_pull_requests)} raw PRs using {self.filter_model}..." + ) + + # Build compact representation for Flash model + pr_summaries = [] + for pr in manifest.all_pull_requests: + pr_summaries.append( + { + "number": pr.number, + "title": pr.title, + "repo": pr.repo_name, + "author": pr.author, + "files_changed_count": len(pr.files_changed), + "sample_files": pr.files_changed[:3], + } + ) + + prompt = f"""You are a Senior Technical Release Engineer for Data Commons Platform (DCP). +Analyze the following list of merged Pull Requests for release range {manifest.previous_version} -> {manifest.new_version}. + +Goal: Identify all SUBSTANTIVE, meaningful Pull Requests that represent feature additions, bug fixes, infrastructure changes, or configuration updates for the Data Commons Platform. + +Filter OUT: +- Automated bot version bumps (e.g., 'chore: bump version to 1.1.1', dependabot, renovate). +- Trivial formatting, linting, or typo fixes in documentation/README (e.g., 'fix typo in README'). +- Internal test-only refactors with zero functional impact. + +Here is the list of PRs: +{json.dumps(pr_summaries, indent=2)} + +Respond ONLY with a JSON object containing a single key "relevant_pr_numbers" with an array of integer PR numbers that should be included for release notes synthesis. +Example: {{"relevant_pr_numbers": [101, 105, 112]}} +""" + + try: + config = types.GenerateContentConfig( + response_mime_type="application/json", + temperature=0.1, + ) + res = self.client.models.generate_content( + model=self.filter_model, + contents=prompt, + config=config, + ) + data = json.loads(res.text) + relevant_numbers = set(data.get("relevant_pr_numbers", [])) + + candidate_prs = [ + pr for pr in manifest.all_pull_requests if pr.number in relevant_numbers + ] + logger.info( + f"Stage 1 Complete: Retained {len(candidate_prs)} / {len(manifest.all_pull_requests)} substantive PRs." + ) + return candidate_prs + except Exception as e: + logger.error(f"Stage 1 Flash filtering failed: {e}. Falling back to all non-bot PRs.") + # Basic fallback for error resilience + return [ + pr for pr in manifest.all_pull_requests + if "bump version" not in pr.title.lower() and pr.author != "datacommons-robot-author" + ] + + def synthesize_features_with_pro( + self, + manifest: ReleaseInfoManifest, + candidate_prs: List[PullRequest], + additional_instructions: Optional[str] = None, + ) -> List[FeatureUpdate]: + """Stage 2 (Pro Model): Performs deep semantic classification, feature grouping, override resolution, and SOP drafting.""" + if not candidate_prs: + logger.warning("No candidate PRs provided for Stage 2 synthesis.") + return [] + + logger.info( + f"Stage 2: Synthesizing features from {len(candidate_prs)} candidate PRs using {self.synthesis_model}..." + ) + + # Build detailed PR context for Pro model + detailed_prs = [] + for pr in candidate_prs: + detailed_prs.append( + { + "number": pr.number, + "title": pr.title, + "author": pr.author, + "repo": pr.repo_name, + "url": pr.url, + "target_components": pr.target_components, + "files_changed": pr.files_changed, + "body_summary": pr.body[:500] if pr.body else "", + } + ) + + instructions_context = "" + if additional_instructions or manifest.additional_instructions: + instructions_text = additional_instructions or manifest.additional_instructions + instructions_context = f"\n### Additional User Context & Highlights:\n{instructions_text}\n" + + prompt = f"""You are an expert Technical Release Manager drafting official release notes for Data Commons Platform (DCP) release {manifest.new_version} (previous version: {manifest.previous_version}). + +### Task Instructions: +1. **Semantic Classification**: Categorize each feature into EXACTLY ONE of the 4 standard DCP SOP categories: + - "Spanner Graph & APIs" (SDMX 3.0 REST API, `/v2/observation` StatVars, MCP server/tools, Spanner gRPC serving/protos) + - "Ingestion & Safety" (Ingestion Helper, Aggregation Helper, Dataflow Java worker, timestamp bounds, safety checks, Spanner loading) + - "Search & Website" (Spanner vector embeddings / `NodeEmbedding`, private instance `detect-and-fulfill`, Website UI, Nginx/Envoy) + - "Infra & Tooling" (DCP Terraform modules, `datacommons admin`/`cli` PyPI packages, monorepo packages, Cloud Build release pipelines) + +2. **Feature Grouping & Deduplication**: + - Combine related PRs (e.g., an initial feature PR + follow-up bug fixes + test PRs) into a SINGLE cohesive `FeatureUpdate`. + - List all included PR numbers in `included_prs` (e.g. `[188, 189]`). + +3. **Supersede Resolution**: + - If a PR was superseded or modified by a later PR in this release, describe only the FINAL state at {manifest.new_version}. + +4. **Technical Writing**: + - Write clear, concise, engineer-style titles and descriptions. Avoid marketing fluff or non-technical summaries. + - Set `is_dcp_relevant: true` for all platform-relevant features, or `false` for base-only features. + +{instructions_context} + +### Substantive Candidate PRs: +{json.dumps(detailed_prs, indent=2)} + +Respond ONLY with a JSON array of FeatureUpdate objects with the following schema: +[ + {{ + "id": "short_unique_snake_case_id", + "title": "Clear Technical Feature Title", + "description": "2-3 sentence technical description of the feature, changes, and impact.", + "category": "Spanner Graph & APIs | Ingestion & Safety | Search & Website | Infra & Tooling", + "target_components": ["dcp", "services", "preprocessing", "dataflow_worker", "ingestion_helper", "postprocessing"], + "included_prs": [188, 189], + "is_dcp_relevant": true, + "breaking_changes": "Optional string describing breaking change if any, else null" + }} +] +""" + + try: + config = types.GenerateContentConfig( + response_mime_type="application/json", + temperature=0.2, + ) + res = self.client.models.generate_content( + model=self.synthesis_model, + contents=prompt, + config=config, + ) + raw_features = json.loads(res.text) + features: List[FeatureUpdate] = [] + for item in raw_features: + feature = FeatureUpdate( + id=item.get("id", f"feature_{len(features)+1}"), + title=item.get("title", "Untitled Feature"), + description=item.get("description", ""), + category=item.get("category", "Infra & Tooling"), + target_components=item.get("target_components", []), + included_prs=item.get("included_prs", []), + is_dcp_relevant=item.get("is_dcp_relevant", True), + breaking_changes=item.get("breaking_changes"), + ) + features.append(feature) + + logger.info( + f"Stage 2 Complete: Synthesized {len(features)} structured FeatureUpdate objects across categories." + ) + return features + except Exception as e: + logger.error(f"Stage 2 Pro synthesis failed: {e}") + raise RuntimeError(f"Failed to synthesize release features with Gemini Pro: {e}") + + def extract_features( + self, + manifest: ReleaseInfoManifest, + additional_instructions: Optional[str] = None, + ) -> List[FeatureUpdate]: + """Main entry point: orchestrates Stage 1 Flash filtering -> Stage 2 Pro synthesis -> List[FeatureUpdate].""" + logger.info( + f"Starting Step 2 Feature Extraction for release {manifest.previous_version} -> {manifest.new_version}..." + ) + + # Stage 1: Fast Flash Noise Filter + candidate_prs = self.filter_prs_with_flash(manifest) + + # Stage 2: Deep Pro Synthesis & SOP Classification + features = self.synthesize_features_with_pro( + manifest=manifest, + candidate_prs=candidate_prs, + additional_instructions=additional_instructions, + ) + + return features diff --git a/deploy/generate_release_notes/tests/test_feature_extractor.py b/deploy/generate_release_notes/tests/test_feature_extractor.py new file mode 100644 index 00000000..a0527f5d --- /dev/null +++ b/deploy/generate_release_notes/tests/test_feature_extractor.py @@ -0,0 +1,221 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit and Integration tests for Feature Extractor (deploy/generate_release_notes/feature_extractor.py).""" + +import json +import os +from unittest.mock import MagicMock, patch +import pytest + +from deploy.generate_release_notes.feature_extractor import FeatureExtractor +from deploy.generate_release_notes.models import ( + FeatureUpdate, + PullRequest, + ReleaseInfoManifest, +) + + +class TestFeatureExtractorUnit: + """Unit tests for FeatureExtractor with mocked Gemini API calls.""" + + @patch("google.genai.Client") + def test_filter_prs_with_flash(self, mock_client_cls): + """Test Stage 1 Flash model filtering out bot version bumps and noise.""" + mock_client = MagicMock() + mock_client_cls.return_value = mock_client + + # Mock Flash model response returning PRs #188 and #189 + mock_res = MagicMock() + mock_res.text = json.dumps({"relevant_pr_numbers": [188, 189]}) + mock_client.models.generate_content.return_value = mock_res + + pr188 = PullRequest( + number=188, + title="[DCP Ingestion] Remove status set to Success at end of dataflow stage", + body="Fixes status race condition", + author="gmechali", + url="https://github.com/datacommonsorg/datacommons/pull/188", + merged_at="2026-07-24T18:05:13Z", + repo_name="datacommonsorg/datacommons", + files_changed=["infra/dcp/dataflow_job.tf"], + ) + pr189 = PullRequest( + number=189, + title="[DCP Ingestion] Allow dataflow to scale workers based on Terraform Variables", + body="Adds max_workers variable", + author="gmechali", + url="https://github.com/datacommonsorg/datacommons/pull/189", + merged_at="2026-07-24T19:13:17Z", + repo_name="datacommonsorg/datacommons", + files_changed=["infra/dcp/variables.tf"], + ) + pr195_bot = PullRequest( + number=195, + title="chore: bump version to 1.1.1", + body="Automated release bump", + author="datacommons-robot-author", + url="https://github.com/datacommonsorg/datacommons/pull/195", + merged_at="2026-07-29T00:37:18Z", + repo_name="datacommonsorg/datacommons", + files_changed=["VERSION"], + ) + + manifest = ReleaseInfoManifest( + previous_version="v1.1.0", + new_version="v1.1.1", + all_pull_requests=[pr188, pr189, pr195_bot], + ) + + extractor = FeatureExtractor(api_key="mock_key") + candidates = extractor.filter_prs_with_flash(manifest) + + assert len(candidates) == 2 + candidate_numbers = [pr.number for pr in candidates] + assert 188 in candidate_numbers + assert 189 in candidate_numbers + assert 195 not in candidate_numbers + + @patch("google.genai.Client") + def test_synthesize_features_with_pro(self, mock_client_cls): + """Test Stage 2 Pro model feature grouping and SOP classification.""" + mock_client = MagicMock() + mock_client_cls.return_value = mock_client + + mock_synthesis_output = [ + { + "id": "ingestion_dataflow_scaling", + "title": "Dataflow Worker Auto-Scaling & Pipeline Safety", + "description": "Configured Dataflow worker auto-scaling via Terraform variables and resolved premature success status marking.", + "category": "Ingestion & Safety", + "target_components": ["dcp", "dataflow_worker"], + "included_prs": [188, 189], + "is_dcp_relevant": True, + "breaking_changes": None, + } + ] + mock_res = MagicMock() + mock_res.text = json.dumps(mock_synthesis_output) + mock_client.models.generate_content.return_value = mock_res + + pr188 = PullRequest( + number=188, + title="[DCP Ingestion] Remove status set to Success at end of dataflow stage", + body="Fixes status race condition", + author="gmechali", + url="https://github.com/datacommonsorg/datacommons/pull/188", + merged_at="2026-07-24T18:05:13Z", + repo_name="datacommonsorg/datacommons", + files_changed=["infra/dcp/dataflow_job.tf"], + ) + pr189 = PullRequest( + number=189, + title="[DCP Ingestion] Allow dataflow to scale workers based on Terraform Variables", + body="Adds max_workers variable", + author="gmechali", + url="https://github.com/datacommonsorg/datacommons/pull/189", + merged_at="2026-07-24T19:13:17Z", + repo_name="datacommonsorg/datacommons", + files_changed=["infra/dcp/variables.tf"], + ) + + manifest = ReleaseInfoManifest( + previous_version="v1.1.0", + new_version="v1.1.1", + all_pull_requests=[pr188, pr189], + ) + + extractor = FeatureExtractor(api_key="mock_key") + features = extractor.synthesize_features_with_pro( + manifest=manifest, + candidate_prs=[pr188, pr189], + additional_instructions="Focus on Dataflow scaling", + ) + + assert len(features) == 1 + feature = features[0] + assert isinstance(feature, FeatureUpdate) + assert feature.id == "ingestion_dataflow_scaling" + assert feature.category == "Ingestion & Safety" + assert feature.included_prs == [188, 189] + assert feature.is_dcp_relevant is True + + +class TestFeatureExtractorIntegration: + """Integration test running FeatureExtractor against real manifest from Step 1 with Gemini API.""" + + @pytest.mark.integration + def test_real_feature_extraction_v1_1_0_to_v1_1_1(self): + """Runs 2-stage Gemini pipeline against real /tmp/test_manifest_v1.1.1.json.""" + if not os.getenv("GEMINI_API_KEY") and not os.getenv("GOOGLE_API_KEY"): + pytest.skip("GEMINI_API_KEY or GOOGLE_API_KEY not set in environment. Skipping real Gemini API integration test.") + + manifest_file = "/tmp/test_manifest_v1.1.1.json" + if not os.path.exists(manifest_file): + pytest.skip(f"Manifest file {manifest_file} not found. Run Step 1 test first.") + + # Re-construct ReleaseInfoManifest from test manifest summary + with open(manifest_file, "r") as f: + manifest_dict = json.load(f) + + prs = [ + PullRequest( + number=item["number"], + title=item["title"], + body="", + author=item.get("author", "unknown"), + url=f"https://github.com/{item['repo']}/pull/{item['number']}", + merged_at="2026-07-25T00:00:00Z", + repo_name=item["repo"], + target_components=item.get("target_components", []), + ) + for item in manifest_dict.get("sample_prs", []) + ] + + manifest = ReleaseInfoManifest( + previous_version=manifest_dict["previous_version"], + new_version=manifest_dict["new_version"], + all_pull_requests=prs, + ) + + extractor = FeatureExtractor() + features = extractor.extract_features( + manifest=manifest, + additional_instructions="Integrate Maps API and Dataflow scaling highlights.", + ) + + assert len(features) > 0 + print(f"\nSuccessfully extracted {len(features)} feature updates using Gemini pipeline:") + for feat in features: + print(f" - [{feat.category}] {feat.title} (PRs: {feat.included_prs})") + + # Save features to /tmp for inspection + output_file = "/tmp/test_features_v1.1.1.json" + features_dict = [ + { + "id": f.id, + "title": f.title, + "description": f.description, + "category": f.category, + "target_components": f.target_components, + "included_prs": f.included_prs, + "is_dcp_relevant": f.is_dcp_relevant, + } + for f in features + ] + with open(output_file, "w") as f: + json.dump(features_dict, f, indent=2) + + print(f"Saved synthesized features summary to {output_file}") + assert os.path.exists(output_file) From 8a1ae42aa5506127b87f2a0650cc64436395afca Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:01:25 -0700 Subject: [PATCH 06/70] feat(deploy): apply design fixes to Step 2 Feature Extractor (qualified PR IDs, SOPCategory enum, merged_at timestamps) --- .../feature_extractor.py | 70 +++++++++++++------ deploy/generate_release_notes/models.py | 20 +++++- .../tests/test_feature_extractor.py | 16 ++--- 3 files changed, 76 insertions(+), 30 deletions(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index 5501510e..1de238b3 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -30,6 +30,7 @@ FeatureUpdate, PullRequest, ReleaseInfoManifest, + SOPCategory, ) logger = logging.getLogger(__name__) @@ -37,6 +38,8 @@ DEFAULT_FILTER_MODEL = "gemini-2.5-flash" DEFAULT_SYNTHESIS_MODEL = "gemini-2.5-pro" +VALID_SOP_CATEGORIES = {cat.value for cat in SOPCategory} + class FeatureExtractor: """Two-Stage Gemini LLM Pipeline for filtering, classifying, and synthesizing DCP release features.""" @@ -69,35 +72,37 @@ def filter_prs_with_flash(self, manifest: ReleaseInfoManifest) -> List[PullReque f"Stage 1: Filtering {len(manifest.all_pull_requests)} raw PRs using {self.filter_model}..." ) - # Build compact representation for Flash model + # Build compact representation for Flash model using qualified PR IDs (e.g. 'datacommons#188') pr_summaries = [] for pr in manifest.all_pull_requests: pr_summaries.append( { - "number": pr.number, + "id": pr.qualified_id, "title": pr.title, "repo": pr.repo_name, "author": pr.author, "files_changed_count": len(pr.files_changed), - "sample_files": pr.files_changed[:3], + "sample_files": pr.files_changed[:10], } ) prompt = f"""You are a Senior Technical Release Engineer for Data Commons Platform (DCP). Analyze the following list of merged Pull Requests for release range {manifest.previous_version} -> {manifest.new_version}. -Goal: Identify all SUBSTANTIVE, meaningful Pull Requests that represent feature additions, bug fixes, infrastructure changes, or configuration updates for the Data Commons Platform. +Goal: Identify all SUBSTANTIVE, meaningful Pull Requests that represent feature additions, bug fixes, infrastructure changes, configuration updates, or Data Commons Platform/Base capabilities. -Filter OUT: +Filter OUT ONLY non-informative noise: - Automated bot version bumps (e.g., 'chore: bump version to 1.1.1', dependabot, renovate). - Trivial formatting, linting, or typo fixes in documentation/README (e.g., 'fix typo in README'). -- Internal test-only refactors with zero functional impact. +- Trivial internal test-only refactors with zero functional impact. + +DO NOT filter out base Data Commons features or infrastructure PRs — keep all substantive PRs! Here is the list of PRs: {json.dumps(pr_summaries, indent=2)} -Respond ONLY with a JSON object containing a single key "relevant_pr_numbers" with an array of integer PR numbers that should be included for release notes synthesis. -Example: {{"relevant_pr_numbers": [101, 105, 112]}} +Respond ONLY with a JSON object containing a single key "relevant_pr_ids" with an array of qualified PR ID strings (e.g., ["datacommons#188", "import#42"]). +Example: {{"relevant_pr_ids": ["datacommons#188", "import#42"]}} """ try: @@ -111,10 +116,10 @@ def filter_prs_with_flash(self, manifest: ReleaseInfoManifest) -> List[PullReque config=config, ) data = json.loads(res.text) - relevant_numbers = set(data.get("relevant_pr_numbers", [])) + relevant_ids = set(data.get("relevant_pr_ids", [])) candidate_prs = [ - pr for pr in manifest.all_pull_requests if pr.number in relevant_numbers + pr for pr in manifest.all_pull_requests if pr.qualified_id in relevant_ids ] logger.info( f"Stage 1 Complete: Retained {len(candidate_prs)} / {len(manifest.all_pull_requests)} substantive PRs." @@ -143,31 +148,43 @@ def synthesize_features_with_pro( f"Stage 2: Synthesizing features from {len(candidate_prs)} candidate PRs using {self.synthesis_model}..." ) - # Build detailed PR context for Pro model + # Build detailed PR context for Pro model with merged_at timestamps and qualified IDs detailed_prs = [] for pr in candidate_prs: + # Preserve "BREAKING CHANGE:" sections if present in body + body_text = pr.body or "" + body_snippet = body_text[:500] + if "BREAKING CHANGE" in body_text and "BREAKING CHANGE" not in body_snippet: + bc_start = body_text.find("BREAKING CHANGE") + body_snippet += "\n...\n" + body_text[bc_start : bc_start + 300] + detailed_prs.append( { - "number": pr.number, + "id": pr.qualified_id, "title": pr.title, "author": pr.author, "repo": pr.repo_name, "url": pr.url, + "merged_at": pr.merged_at, "target_components": pr.target_components, - "files_changed": pr.files_changed, - "body_summary": pr.body[:500] if pr.body else "", + "files_changed": pr.files_changed[:10], + "body_summary": body_snippet, } ) instructions_context = "" if additional_instructions or manifest.additional_instructions: instructions_text = additional_instructions or manifest.additional_instructions - instructions_context = f"\n### Additional User Context & Highlights:\n{instructions_text}\n" + instructions_context = ( + f"\n### Additional User Context & High-Priority Highlights:\n" + f"Note: User instructions have top priority and override default classification or filtering where applicable:\n" + f"{instructions_text}\n" + ) prompt = f"""You are an expert Technical Release Manager drafting official release notes for Data Commons Platform (DCP) release {manifest.new_version} (previous version: {manifest.previous_version}). ### Task Instructions: -1. **Semantic Classification**: Categorize each feature into EXACTLY ONE of the 4 standard DCP SOP categories: +1. **Semantic Classification**: Categorize each feature into EXACTLY ONE of the 4 standard DCP SOP categories (use exact category names): - "Spanner Graph & APIs" (SDMX 3.0 REST API, `/v2/observation` StatVars, MCP server/tools, Spanner gRPC serving/protos) - "Ingestion & Safety" (Ingestion Helper, Aggregation Helper, Dataflow Java worker, timestamp bounds, safety checks, Spanner loading) - "Search & Website" (Spanner vector embeddings / `NodeEmbedding`, private instance `detect-and-fulfill`, Website UI, Nginx/Envoy) @@ -175,9 +192,10 @@ def synthesize_features_with_pro( 2. **Feature Grouping & Deduplication**: - Combine related PRs (e.g., an initial feature PR + follow-up bug fixes + test PRs) into a SINGLE cohesive `FeatureUpdate`. - - List all included PR numbers in `included_prs` (e.g. `[188, 189]`). + - List all included PR qualified IDs in `included_prs` (e.g. `["datacommons#188", "datacommons#189"]`). -3. **Supersede Resolution**: +3. **Supersede Resolution & Chronology**: + - Use the `merged_at` timestamps to understand commit order. - If a PR was superseded or modified by a later PR in this release, describe only the FINAL state at {manifest.new_version}. 4. **Technical Writing**: @@ -197,7 +215,7 @@ def synthesize_features_with_pro( "description": "2-3 sentence technical description of the feature, changes, and impact.", "category": "Spanner Graph & APIs | Ingestion & Safety | Search & Website | Infra & Tooling", "target_components": ["dcp", "services", "preprocessing", "dataflow_worker", "ingestion_helper", "postprocessing"], - "included_prs": [188, 189], + "included_prs": ["datacommons#188", "datacommons#189"], "is_dcp_relevant": true, "breaking_changes": "Optional string describing breaking change if any, else null" }} @@ -217,11 +235,23 @@ def synthesize_features_with_pro( raw_features = json.loads(res.text) features: List[FeatureUpdate] = [] for item in raw_features: + cat = item.get("category", "Infra & Tooling") + if cat not in VALID_SOP_CATEGORIES: + # Fuzzy match or fallback to Infra & Tooling + matched = False + for valid_cat in VALID_SOP_CATEGORIES: + if valid_cat.lower() in cat.lower() or cat.lower() in valid_cat.lower(): + cat = valid_cat + matched = True + break + if not matched: + cat = SOPCategory.INFRA_TOOLING.value + feature = FeatureUpdate( id=item.get("id", f"feature_{len(features)+1}"), title=item.get("title", "Untitled Feature"), description=item.get("description", ""), - category=item.get("category", "Infra & Tooling"), + category=cat, target_components=item.get("target_components", []), included_prs=item.get("included_prs", []), is_dcp_relevant=item.get("is_dcp_relevant", True), diff --git a/deploy/generate_release_notes/models.py b/deploy/generate_release_notes/models.py index 1a16e4b1..6d77e6f2 100644 --- a/deploy/generate_release_notes/models.py +++ b/deploy/generate_release_notes/models.py @@ -15,9 +15,19 @@ """Data models for Data Commons Platform (DCP) release notes generation.""" from dataclasses import dataclass, field +from enum import Enum from typing import Dict, List, Optional +class SOPCategory(str, Enum): + """Standard SOP Categories for Data Commons Platform Release Notes.""" + + SPANNER_APIS = "Spanner Graph & APIs" + INGESTION_SAFETY = "Ingestion & Safety" + SEARCH_WEBSITE = "Search & Website" + INFRA_TOOLING = "Infra & Tooling" + + @dataclass class PullRequest: """Represents a single merged GitHub Pull Request.""" @@ -34,6 +44,12 @@ class PullRequest: commit_shas: List[str] = field(default_factory=list) target_components: List[str] = field(default_factory=list) + @property + def qualified_id(self) -> str: + """Returns a qualified repo#number identifier (e.g. 'datacommons#188' or 'import#42').""" + repo_short = self.repo_name.split("/")[-1] + return f"{repo_short}#{self.number}" + @dataclass class ComponentVersionInfo: @@ -58,9 +74,9 @@ class FeatureUpdate: id: str title: str description: str - category: str # e.g. "Spanner Graph & APIs", "Ingestion & Safety", "Search & Website", "Infra & Tooling" + category: str # Must match one of SOPCategory values target_components: List[str] = field(default_factory=list) - included_prs: List[int] = field(default_factory=list) + included_prs: List[str] = field(default_factory=list) # Qualified PR IDs e.g. ["datacommons#188"] is_dcp_relevant: bool = True breaking_changes: Optional[str] = None diff --git a/deploy/generate_release_notes/tests/test_feature_extractor.py b/deploy/generate_release_notes/tests/test_feature_extractor.py index a0527f5d..905dca9b 100644 --- a/deploy/generate_release_notes/tests/test_feature_extractor.py +++ b/deploy/generate_release_notes/tests/test_feature_extractor.py @@ -36,9 +36,9 @@ def test_filter_prs_with_flash(self, mock_client_cls): mock_client = MagicMock() mock_client_cls.return_value = mock_client - # Mock Flash model response returning PRs #188 and #189 + # Mock Flash model response returning PRs datacommons#188 and datacommons#189 mock_res = MagicMock() - mock_res.text = json.dumps({"relevant_pr_numbers": [188, 189]}) + mock_res.text = json.dumps({"relevant_pr_ids": ["datacommons#188", "datacommons#189"]}) mock_client.models.generate_content.return_value = mock_res pr188 = PullRequest( @@ -82,10 +82,10 @@ def test_filter_prs_with_flash(self, mock_client_cls): candidates = extractor.filter_prs_with_flash(manifest) assert len(candidates) == 2 - candidate_numbers = [pr.number for pr in candidates] - assert 188 in candidate_numbers - assert 189 in candidate_numbers - assert 195 not in candidate_numbers + candidate_ids = [pr.qualified_id for pr in candidates] + assert "datacommons#188" in candidate_ids + assert "datacommons#189" in candidate_ids + assert "datacommons#195" not in candidate_ids @patch("google.genai.Client") def test_synthesize_features_with_pro(self, mock_client_cls): @@ -100,7 +100,7 @@ def test_synthesize_features_with_pro(self, mock_client_cls): "description": "Configured Dataflow worker auto-scaling via Terraform variables and resolved premature success status marking.", "category": "Ingestion & Safety", "target_components": ["dcp", "dataflow_worker"], - "included_prs": [188, 189], + "included_prs": ["datacommons#188", "datacommons#189"], "is_dcp_relevant": True, "breaking_changes": None, } @@ -148,7 +148,7 @@ def test_synthesize_features_with_pro(self, mock_client_cls): assert isinstance(feature, FeatureUpdate) assert feature.id == "ingestion_dataflow_scaling" assert feature.category == "Ingestion & Safety" - assert feature.included_prs == [188, 189] + assert feature.included_prs == ["datacommons#188", "datacommons#189"] assert feature.is_dcp_relevant is True From b5be976dc8c537da60e18c15bcdfa7435dfc2f2d Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:04:32 -0700 Subject: [PATCH 07/70] feat(deploy): add per-PR contribution summaries (pr_contributions) to FeatureUpdate model and Stage 2 Gemini prompt --- deploy/generate_release_notes/feature_extractor.py | 11 ++++++++++- deploy/generate_release_notes/models.py | 1 + .../tests/test_feature_extractor.py | 6 ++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index 1de238b3..66520b25 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -198,9 +198,13 @@ def synthesize_features_with_pro( - Use the `merged_at` timestamps to understand commit order. - If a PR was superseded or modified by a later PR in this release, describe only the FINAL state at {manifest.new_version}. -4. **Technical Writing**: +4. **Technical Writing & Comprehensive Context**: - Write clear, concise, engineer-style titles and descriptions. Avoid marketing fluff or non-technical summaries. - Set `is_dcp_relevant: true` for all platform-relevant features, or `false` for base-only features. + - If a feature contains only ONE PR, ensure the `description` is rich and comprehensive enough for release notes generation to understand all capabilities implemented. + +5. **Per-PR Contribution Summaries**: + - For EVERY PR listed in `included_prs`, provide a specific 1-2 sentence contribution summary under `pr_contributions` mapping the qualified PR ID to its specific capability contribution (e.g., `{{"datacommons#188": "Removed premature success status set at end of dataflow stage", "datacommons#189": "Added max_workers Terraform variable for Dataflow auto-scaling"}}`). {instructions_context} @@ -216,6 +220,10 @@ def synthesize_features_with_pro( "category": "Spanner Graph & APIs | Ingestion & Safety | Search & Website | Infra & Tooling", "target_components": ["dcp", "services", "preprocessing", "dataflow_worker", "ingestion_helper", "postprocessing"], "included_prs": ["datacommons#188", "datacommons#189"], + "pr_contributions": {{ + "datacommons#188": "Removed premature success status set at end of dataflow stage", + "datacommons#189": "Added max_workers Terraform variable for Dataflow auto-scaling" + }}, "is_dcp_relevant": true, "breaking_changes": "Optional string describing breaking change if any, else null" }} @@ -254,6 +262,7 @@ def synthesize_features_with_pro( category=cat, target_components=item.get("target_components", []), included_prs=item.get("included_prs", []), + pr_contributions=item.get("pr_contributions", {}), is_dcp_relevant=item.get("is_dcp_relevant", True), breaking_changes=item.get("breaking_changes"), ) diff --git a/deploy/generate_release_notes/models.py b/deploy/generate_release_notes/models.py index 6d77e6f2..774584a4 100644 --- a/deploy/generate_release_notes/models.py +++ b/deploy/generate_release_notes/models.py @@ -77,6 +77,7 @@ class FeatureUpdate: category: str # Must match one of SOPCategory values target_components: List[str] = field(default_factory=list) included_prs: List[str] = field(default_factory=list) # Qualified PR IDs e.g. ["datacommons#188"] + pr_contributions: Dict[str, str] = field(default_factory=dict) # Maps PR ID -> Specific contribution summary is_dcp_relevant: bool = True breaking_changes: Optional[str] = None diff --git a/deploy/generate_release_notes/tests/test_feature_extractor.py b/deploy/generate_release_notes/tests/test_feature_extractor.py index 905dca9b..77c01e45 100644 --- a/deploy/generate_release_notes/tests/test_feature_extractor.py +++ b/deploy/generate_release_notes/tests/test_feature_extractor.py @@ -101,6 +101,10 @@ def test_synthesize_features_with_pro(self, mock_client_cls): "category": "Ingestion & Safety", "target_components": ["dcp", "dataflow_worker"], "included_prs": ["datacommons#188", "datacommons#189"], + "pr_contributions": { + "datacommons#188": "Removed premature success status set at end of dataflow stage", + "datacommons#189": "Added max_workers Terraform variable for Dataflow auto-scaling", + }, "is_dcp_relevant": True, "breaking_changes": None, } @@ -149,6 +153,8 @@ def test_synthesize_features_with_pro(self, mock_client_cls): assert feature.id == "ingestion_dataflow_scaling" assert feature.category == "Ingestion & Safety" assert feature.included_prs == ["datacommons#188", "datacommons#189"] + assert "datacommons#188" in feature.pr_contributions + assert "datacommons#189" in feature.pr_contributions assert feature.is_dcp_relevant is True From 5ac65ee19d7edec882ce7365df61aa1c185662ad Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:14:53 -0700 Subject: [PATCH 08/70] feat(deploy): implement Step 3 Streamlined Agentic Release Notes Writer using Gemini Pro - Add ReleaseNotesWriter in release_notes_writer.py with agentic prompt following streamlined DCP template - Add unit and integration test suite in test_release_notes_writer.py --- .../release_notes_writer.py | 268 ++++++++++++++++++ .../tests/test_release_notes_writer.py | 240 ++++++++++++++++ 2 files changed, 508 insertions(+) create mode 100644 deploy/generate_release_notes/release_notes_writer.py create mode 100644 deploy/generate_release_notes/tests/test_release_notes_writer.py diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py new file mode 100644 index 00000000..e0a02a24 --- /dev/null +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -0,0 +1,268 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Step 3: Release Notes Writer for Data Commons Platform (DCP) release notes generation. + +Uses an Agentic Writer (Gemini Pro) to generate clean, partner-facing, non-technical +release notes following the streamlined Data Commons Platform format. +""" + +import json +import logging +import os +from typing import Any, Dict, List, Optional + +from google import genai +from google.genai import types + +from deploy.generate_release_notes.models import ( + FeatureUpdate, + PullRequest, + ReleaseInfoManifest, +) + +logger = logging.getLogger(__name__) + +DEFAULT_WRITER_MODEL = "gemini-2.5-pro" + + +class ReleaseNotesWriter: + """Agentic Release Notes Writer powered by Gemini Pro.""" + + def __init__( + self, + api_key: Optional[str] = None, + model_name: str = DEFAULT_WRITER_MODEL, + include_audit_log: bool = False, + ): + key = api_key or os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY") + if not key: + logger.warning( + "Neither GEMINI_API_KEY nor GOOGLE_API_KEY set in environment. Gemini API calls will fail if not authenticated via GCP default credentials." + ) + self.client = genai.Client() + else: + self.client = genai.Client(api_key=key) + + self.model_name = model_name + self.include_audit_log = include_audit_log + + def _extract_bug_fixes( + self, manifest: ReleaseInfoManifest, features: List[FeatureUpdate] + ) -> List[Dict[str, Any]]: + """Extracts PRs that are bug fixes or one-off improvements not covered in major features.""" + included_pr_ids = set() + for feat in features: + included_pr_ids.update(feat.included_prs) + + bug_fix_prs = [] + for pr in manifest.all_pull_requests: + if pr.qualified_id in included_pr_ids: + continue + + title_lower = pr.title.lower() + # Identify bug fixes or minor partner-relevant PRs + is_fix = ( + "fix" in title_lower + or "bug" in title_lower + or "resolve" in title_lower + or "correct" in title_lower + or "patch" in title_lower + ) + is_bot = ( + "bump version" in title_lower + or pr.author == "datacommons-robot-author" + or "dependabot" in pr.author.lower() + or "renovate" in pr.author.lower() + ) + + if is_fix and not is_bot: + bug_fix_prs.append( + { + "id": pr.qualified_id, + "title": pr.title, + "author": pr.author, + "repo": pr.repo_name, + "url": pr.url, + "body_snippet": pr.body[:300] if pr.body else "", + } + ) + + return bug_fix_prs + + def build_audit_table_markdown( + self, manifest: ReleaseInfoManifest, features: List[FeatureUpdate] + ) -> str: + """Generates an optional Markdown audit table listing all raw PRs and their status.""" + feature_pr_map: Dict[str, FeatureUpdate] = {} + for feat in features: + for pr_id in feat.included_prs: + feature_pr_map[pr_id] = feat + + rows = [] + for pr in manifest.all_pull_requests: + repo_short = pr.repo_name.split("/")[-1] + title_sanitized = pr.title.replace("|", "\\|").replace("<", "<").replace(">", ">") + + if pr.qualified_id in feature_pr_map: + feat = feature_pr_map[pr.qualified_id] + status = "Substantive Feature" if feat.is_dcp_relevant else "Base DC Only" + category = feat.category + elif "bump version" in pr.title.lower() or "bot" in pr.author.lower(): + status = "Bot Version Bump" + category = "Infra & Tooling" + else: + status = "Refactor / Minor" + category = "Infra & Tooling" + + components_str = ", ".join(pr.target_components) if pr.target_components else "infra" + rows.append( + f"| `{repo_short}` | [{pr.number}]({pr.url}) | {title_sanitized} | @{pr.author} | {components_str} | {category} | {status} |" + ) + + header = ( + "\n---\n\n## Complete Release Audit Log\n\n" + "| Repo | PR # | Title | Author | Components | Category / Type | Status |\n" + "| :--- | :--- | :--- | :--- | :--- | :--- | :--- |\n" + ) + return header + "\n".join(rows) + "\n" + + def render( + self, + manifest: ReleaseInfoManifest, + features: List[FeatureUpdate], + additional_instructions: Optional[str] = None, + release_date: str = "2026-07-29", + ) -> str: + """Calls Gemini Pro to generate publication-ready Markdown release notes according to the streamlined template.""" + logger.info( + f"Step 3: Rendering release notes for {manifest.new_version} using {self.model_name}..." + ) + + # Build payloads for prompt + features_payload = [] + for feat in features: + features_payload.append( + { + "id": feat.id, + "title": feat.title, + "description": feat.description, + "category": feat.category, + "target_components": feat.target_components, + "included_prs": feat.included_prs, + "pr_contributions": feat.pr_contributions, + "is_dcp_relevant": feat.is_dcp_relevant, + "breaking_changes": feat.breaking_changes, + } + ) + + bug_fixes_payload = self._extract_bug_fixes(manifest, features) + + instructions_context = "" + if additional_instructions or manifest.additional_instructions: + instructions_text = additional_instructions or manifest.additional_instructions + instructions_context = ( + f"\n#### Additional User Instructions & High-Priority Highlights:\n" + f"{instructions_text}\n" + ) + + prompt = f"""You are an expert Technical Release Manager and Product Documentation Specialist for the Data Commons Platform (DCP). +Your task is to write publication-ready, partner-facing release notes for Data Commons Platform release {manifest.new_version} ({release_date}). + +### Core Writing Guidelines & Tone: +1. **Tone & Style**: Write in a clear, positive, partner-facing tone using plain language. Use second person ("You can now...") and active voice for new features and improvements. Use past tense for bug fixes ("Fixed...", "Resolved..."). +2. **Target Audience**: Technical and non-technical partners, platform operators, and stakeholders who need to understand what changed, why it matters, and how to use it. +3. **Level of Detail**: + - For major features: Provide an engaging "What's New", "Why it Matters" (business/technical benefit), and a bulleted list of "Capabilities & Changes". + - For single-PR features: Provide rich, self-contained descriptions so partners do not need to look up code diffs. + - For bug fixes: Focus on what was broken, how it was resolved, and how the system behaves now. +4. **Link Formatting**: Every PR reference MUST be formatted as a clickable Markdown link using the format `[#]()` (e.g. `[datacommons#188](https://github.com/datacommonsorg/datacommons/pull/188)`). +5. **Strict Constraints**: + - DO NOT use any emojis anywhere in the document. + - DO NOT include a component version table or git commit/SHA table. + - DO NOT include release range commit text (e.g., "Release range: v1.1.0 to v1.1.1"). + - Place the 1-2 sentence Executive Summary directly beneath the main title (`# Data Commons Platform Release {manifest.new_version} ({release_date})`). + - Output ONLY clean, valid GitHub Flavored Markdown (GFM). + +--- + +### Input Release Data: + +#### Synthesized Features & Updates: +{json.dumps(features_payload, indent=2)} + +#### Bug Fixes & Refactors: +{json.dumps(bug_fixes_payload, indent=2)} + +{instructions_context} + +--- + +### Required Output Markdown Structure: + +# Data Commons Platform Release {manifest.new_version} ({release_date}) + +*(1-2 sentences highlighting the most important capabilities, improvements, and fixes in this release for partners and platform operators.)* + +--- + +## Key Feature Updates + +*(Group major feature updates here. For each feature, use the following structure:)* + +### [Feature Title] +**What's New**: [Clear description of what partners or operators can now do] +**Why it Matters**: [Business benefit and technical impact] +**Capabilities & Changes**: +- [Capability 1] ([`repo#PR`](URL)) +- [Capability 2] ([`repo#PR`](URL)) + +--- + +## Improvements & Configuration Updates + +*(List enhancements, performance updates, or required Terraform/Admin Panel configuration changes as concise bullet points:)* + +- **[Improvement Title]**: [Summary of update, configuration instructions if required, and benefit] ([`repo#PR`](URL)) + +--- + +## Bug Fixes + +*(List bug fixes as concise bullet points in past tense describing what was resolved and why it helps partners and operators:)* + +- **[Component / Scope]**: [Description of what was fixed and how the system behaves now] ([`repo#PR`](URL)) +""" + + try: + config = types.GenerateContentConfig( + temperature=0.2, + ) + res = self.client.models.generate_content( + model=self.model_name, + contents=prompt, + config=config, + ) + markdown_content = res.text.strip() + + # Append optional audit table if requested + if self.include_audit_log: + audit_table_md = self.build_audit_table_markdown(manifest, features) + markdown_content += "\n" + audit_table_md + + logger.info("Step 3 Complete: Release notes successfully generated.") + return markdown_content + except Exception as e: + logger.error(f"Step 3 Release Notes Writer failed with Gemini Pro: {e}") + raise RuntimeError(f"Failed to generate release notes with Gemini Pro: {e}") diff --git a/deploy/generate_release_notes/tests/test_release_notes_writer.py b/deploy/generate_release_notes/tests/test_release_notes_writer.py new file mode 100644 index 00000000..8035e000 --- /dev/null +++ b/deploy/generate_release_notes/tests/test_release_notes_writer.py @@ -0,0 +1,240 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit and Integration tests for Release Notes Writer (deploy/generate_release_notes/release_notes_writer.py).""" + +import json +import os +from unittest.mock import MagicMock, patch +import pytest + +from deploy.generate_release_notes.models import ( + FeatureUpdate, + PullRequest, + ReleaseInfoManifest, +) +from deploy.generate_release_notes.release_notes_writer import ReleaseNotesWriter + + +class TestReleaseNotesWriterUnit: + """Unit tests for ReleaseNotesWriter with mocked Gemini API calls.""" + + @patch("google.genai.Client") + def test_render_with_mock_gemini(self, mock_client_cls): + """Test ReleaseNotesWriter rendering with mocked Gemini Pro response.""" + mock_client = MagicMock() + mock_client_cls.return_value = mock_client + + mock_markdown_output = """# Data Commons Platform Release v1.1.1 (2026-07-29) + +This release introduces Dataflow worker auto-scaling and Google Maps JavaScript API enablement for interactive map rendering. + +--- + +## Key Feature Updates + +### Dataflow Worker Auto-Scaling & Pipeline Safety +**What's New**: Platform operators can now configure Dataflow worker auto-scaling via Terraform variables. +**Why it Matters**: Improves ingestion throughput while preventing status race conditions. +**Capabilities & Changes**: +- Added `max_workers` Terraform variable ([`datacommons#189`](https://github.com/datacommonsorg/datacommons/pull/189)) +- Removed premature success status setting at end of dataflow stage ([`datacommons#188`](https://github.com/datacommonsorg/datacommons/pull/188)) + +--- + +## Improvements & Configuration Updates + +- **Google Maps API Enablement**: Enabled Google Maps JavaScript and Places APIs by default in Terraform ([`datacommons#198`](https://github.com/datacommonsorg/datacommons/pull/198)) + +--- + +## Bug Fixes + +- **[Services]**: Resolved null pointer exception when querying empty StatVar observations ([`website#145`](https://github.com/datacommonsorg/website/pull/145)) +""" + mock_res = MagicMock() + mock_res.text = mock_markdown_output + mock_client.models.generate_content.return_value = mock_res + + pr188 = PullRequest( + number=188, + title="[DCP Ingestion] Remove status set to Success at end of dataflow stage", + body="Fixes status race condition", + author="gmechali", + url="https://github.com/datacommonsorg/datacommons/pull/188", + merged_at="2026-07-24T18:05:13Z", + repo_name="datacommonsorg/datacommons", + ) + pr189 = PullRequest( + number=189, + title="[DCP Ingestion] Allow dataflow to scale workers based on Terraform Variables", + body="Adds max_workers variable", + author="gmechali", + url="https://github.com/datacommonsorg/datacommons/pull/189", + merged_at="2026-07-24T19:13:17Z", + repo_name="datacommonsorg/datacommons", + ) + + manifest = ReleaseInfoManifest( + previous_version="v1.1.0", + new_version="v1.1.1", + all_pull_requests=[pr188, pr189], + ) + + feature = FeatureUpdate( + id="ingestion_dataflow_scaling", + title="Dataflow Worker Auto-Scaling & Pipeline Safety", + description="Configured Dataflow worker auto-scaling via Terraform variables.", + category="Ingestion & Safety", + target_components=["dcp", "dataflow_worker"], + included_prs=["datacommons#188", "datacommons#189"], + pr_contributions={ + "datacommons#188": "Removed premature success status set at end of dataflow stage", + "datacommons#189": "Added max_workers Terraform variable for Dataflow auto-scaling", + }, + is_dcp_relevant=True, + ) + + writer = ReleaseNotesWriter(api_key="mock_key") + output_md = writer.render(manifest=manifest, features=[feature], release_date="2026-07-29") + + assert "# Data Commons Platform Release v1.1.1 (2026-07-29)" in output_md + assert "## Key Feature Updates" in output_md + assert "## Improvements & Configuration Updates" in output_md + assert "## Bug Fixes" in output_md + assert "Dataflow Worker Auto-Scaling & Pipeline Safety" in output_md + assert "[`datacommons#189`](https://github.com/datacommonsorg/datacommons/pull/189)" in output_md + + def test_build_audit_table_markdown(self): + """Test generating optional Markdown audit table.""" + pr188 = PullRequest( + number=188, + title="Remove status set to Success | Dataflow Fix", + body="", + author="gmechali", + url="https://github.com/datacommonsorg/datacommons/pull/188", + merged_at="2026-07-24T18:05:13Z", + repo_name="datacommonsorg/datacommons", + target_components=["dcp"], + ) + pr195 = PullRequest( + number=195, + title="chore: bump version to 1.1.1", + body="", + author="datacommons-robot-author", + url="https://github.com/datacommonsorg/datacommons/pull/195", + merged_at="2026-07-29T00:37:18Z", + repo_name="datacommonsorg/datacommons", + target_components=["dcp"], + ) + + manifest = ReleaseInfoManifest( + previous_version="v1.1.0", + new_version="v1.1.1", + all_pull_requests=[pr188, pr195], + ) + + feature = FeatureUpdate( + id="ingestion_fix", + title="Dataflow Fix", + description="Fixed dataflow status race condition.", + category="Ingestion & Safety", + included_prs=["datacommons#188"], + is_dcp_relevant=True, + ) + + writer = ReleaseNotesWriter(api_key="mock_key", include_audit_log=True) + table_md = writer.build_audit_table_markdown(manifest=manifest, features=[feature]) + + assert "## Complete Release Audit Log" in table_md + assert "| `datacommons` | [188](https://github.com/datacommonsorg/datacommons/pull/188) | Remove status set to Success \\| Dataflow Fix | @gmechali | dcp | Ingestion & Safety | Substantive Feature |" in table_md + assert "Bot Version Bump" in table_md + + +class TestReleaseNotesWriterIntegration: + """Integration test running ReleaseNotesWriter against real manifest and features from Steps 1 & 2.""" + + @pytest.mark.integration + def test_real_release_notes_generation_v1_1_0_to_v1_1_1(self): + """Runs Gemini Pro Release Notes Writer against real /tmp/test_manifest_v1.1.1.json and /tmp/test_features_v1.1.1.json.""" + if not os.getenv("GEMINI_API_KEY") and not os.getenv("GOOGLE_API_KEY"): + pytest.skip("GEMINI_API_KEY or GOOGLE_API_KEY not set in environment. Skipping real Gemini API integration test.") + + manifest_file = "/tmp/test_manifest_v1.1.1.json" + features_file = "/tmp/test_features_v1.1.1.json" + + if not os.path.exists(manifest_file) or not os.path.exists(features_file): + pytest.skip("Manifest or Features JSON file missing. Run Steps 1 and 2 tests first.") + + with open(manifest_file, "r") as f: + manifest_dict = json.load(f) + + with open(features_file, "r") as f: + features_dict = json.load(f) + + prs = [ + PullRequest( + number=item["number"], + title=item["title"], + body="", + author=item.get("author", "unknown"), + url=f"https://github.com/{item['repo']}/pull/{item['number']}", + merged_at="2026-07-25T00:00:00Z", + repo_name=item["repo"], + target_components=item.get("target_components", []), + ) + for item in manifest_dict.get("sample_prs", []) + ] + + manifest = ReleaseInfoManifest( + previous_version=manifest_dict["previous_version"], + new_version=manifest_dict["new_version"], + all_pull_requests=prs, + ) + + features = [ + FeatureUpdate( + id=item["id"], + title=item["title"], + description=item["description"], + category=item["category"], + target_components=item.get("target_components", []), + included_prs=item.get("included_prs", []), + pr_contributions=item.get("pr_contributions", {}), + is_dcp_relevant=item.get("is_dcp_relevant", True), + ) + for item in features_dict + ] + + writer = ReleaseNotesWriter() + markdown_output = writer.render( + manifest=manifest, + features=features, + additional_instructions="Highlight Google Maps API enablement and Dataflow worker scaling.", + release_date="2026-07-29", + ) + + assert len(markdown_output) > 100 + print(f"\nSuccessfully generated release notes ({len(markdown_output)} chars):") + print("=" * 60) + print(markdown_output[:500] + "\n...\n") + print("=" * 60) + + # Save to /tmp/RELEASE_NOTES_v1.1.1.md + output_file = "/tmp/RELEASE_NOTES_v1.1.1.md" + with open(output_file, "w") as f: + f.write(markdown_output) + + print(f"Saved complete release notes to {output_file}") + assert os.path.exists(output_file) From f7888589d2c36f55d037643351386448af71d0c4 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:15:42 -0700 Subject: [PATCH 09/70] feat(deploy): implement CLI entry point main.py for release notes generation tool - Connect Step 1 (PRExtractor), Step 2 (FeatureExtractor), and Step 3 (ReleaseNotesWriter) into click CLI - Add test_main.py unit test suite --- deploy/generate_release_notes/main.py | 256 ++++++++++++++++++ .../generate_release_notes/tests/test_main.py | 93 +++++++ 2 files changed, 349 insertions(+) create mode 100644 deploy/generate_release_notes/main.py create mode 100644 deploy/generate_release_notes/tests/test_main.py diff --git a/deploy/generate_release_notes/main.py b/deploy/generate_release_notes/main.py new file mode 100644 index 00000000..e4d0b3e0 --- /dev/null +++ b/deploy/generate_release_notes/main.py @@ -0,0 +1,256 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CLI Entry Point for Data Commons Platform (DCP) Release Notes Generator. + +Orchestrates Step 1 (PR Extractor), Step 2 (Feature Extractor), and Step 3 (Release Notes Writer). + +Usage: + uv run --group generate-release-notes python -m deploy.generate_release_notes \\ + --prev v1.1.0 --new v1.1.1 \\ + [--out ./RELEASE_NOTES_v1.1.1.md] \\ + [--additional-instructions ./context.md] \\ + [--allow-missing-images] \\ + [--include-audit-log] +""" + +import json +import logging +import os +import sys +from typing import Optional + +import click + +from deploy.generate_release_notes.feature_extractor import ( + DEFAULT_FILTER_MODEL, + DEFAULT_SYNTHESIS_MODEL, + FeatureExtractor, +) +from deploy.generate_release_notes.pr_extractor import PRExtractor +from deploy.generate_release_notes.release_notes_writer import ( + DEFAULT_WRITER_MODEL, + ReleaseNotesWriter, +) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", + datefmt="%H:%M:%S", +) +logger = logging.getLogger("generate_release_notes") + + +@click.command( + help="Generate publication-ready Data Commons Platform (DCP) release notes between two release versions." +) +@click.option( + "--prev", + "prev_version", + required=True, + help="Previous release tag (e.g. v1.1.0 or 1.1.0).", +) +@click.option( + "--new", + "new_version", + required=True, + help="New release tag (e.g. v1.1.1 or 1.1.1).", +) +@click.option( + "--out", + "output_path", + default=None, + help="Output file path for generated Markdown release notes (default: ./RELEASE_NOTES_.md).", +) +@click.option( + "--additional-instructions", + "additional_instructions", + default=None, + help="Path to markdown file or raw text containing additional context, highlights, or custom notes.", +) +@click.option( + "--allow-missing-images", + is_flag=True, + default=False, + help="Bypass missing container image tag errors during staging/testing.", +) +@click.option( + "--include-audit-log", + is_flag=True, + default=False, + help="Append complete raw PR audit log table at the bottom of the release notes.", +) +@click.option( + "--filter-model", + default=DEFAULT_FILTER_MODEL, + help=f"Gemini model for Stage 1 noise filtering (default: {DEFAULT_FILTER_MODEL}).", +) +@click.option( + "--synthesis-model", + default=DEFAULT_SYNTHESIS_MODEL, + help=f"Gemini model for Stage 2 feature synthesis (default: {DEFAULT_SYNTHESIS_MODEL}).", +) +@click.option( + "--writer-model", + default=DEFAULT_WRITER_MODEL, + help=f"Gemini model for Step 3 release notes writing (default: {DEFAULT_WRITER_MODEL}).", +) +@click.option( + "--manifest-out", + default=None, + help="Optional path to save raw Step 1 ReleaseInfoManifest JSON.", +) +@click.option( + "--features-out", + default=None, + help="Optional path to save synthesized Step 2 FeatureUpdate JSON.", +) +def main( + prev_version: str, + new_version: str, + output_path: Optional[str], + additional_instructions: Optional[str], + allow_missing_images: bool, + include_audit_log: bool, + filter_model: str, + synthesis_model: str, + writer_model: str, + manifest_out: Optional[str], + features_out: Optional[str], +): + """Executes the 3-step DCP Release Notes Generation Pipeline.""" + # Normalize versions + if not prev_version.startswith("v"): + prev_version = f"v{prev_version}" + if not new_version.startswith("v"): + new_version = f"v{new_version}" + + if not output_path: + output_path = f"./RELEASE_NOTES_{new_version}.md" + + # Read additional instructions file if path provided + instructions_text = None + if additional_instructions: + if os.path.exists(additional_instructions): + with open(additional_instructions, "r") as f: + instructions_text = f.read().strip() + logger.info(f"Loaded additional instructions from {additional_instructions}") + else: + instructions_text = additional_instructions + logger.info("Using inline additional instructions.") + + logger.info("=" * 60) + logger.info(f" Starting DCP Release Notes Generation: {prev_version} -> {new_version}") + logger.info("=" * 60) + + # ---------------------------------------------------- + # STEP 1: Sourcing PRs & Component Version Info + # ---------------------------------------------------- + logger.info("\n--- STEP 1: Sourcing PRs & Resolving Image Tags ---") + pr_extractor = PRExtractor(skip_missing_images=allow_missing_images) + try: + manifest = pr_extractor.extract( + prev_version=prev_version, + new_version=new_version, + additional_instructions=instructions_text, + ) + except Exception as e: + logger.error(f"Step 1 Sourcing Failed: {e}") + sys.exit(1) + + if manifest_out: + manifest_dict = { + "previous_version": manifest.previous_version, + "new_version": manifest.new_version, + "total_prs": len(manifest.all_pull_requests), + "components": { + k: { + "name": v.component_name, + "repo": v.repo_name, + "prev_sha": v.previous_sha, + "new_sha": v.new_sha, + "image_uri": v.image_uri, + } + for k, v in manifest.components.items() + }, + } + with open(manifest_out, "w") as f: + json.dump(manifest_dict, f, indent=2) + logger.info(f"Saved manifest summary to {manifest_out}") + + # ---------------------------------------------------- + # STEP 2: Feature Extraction & SOP Classification + # ---------------------------------------------------- + logger.info("\n--- STEP 2: Feature Extraction & SOP Classification ---") + feature_extractor = FeatureExtractor( + filter_model=filter_model, + synthesis_model=synthesis_model, + ) + try: + features = feature_extractor.extract_features( + manifest=manifest, + additional_instructions=instructions_text, + ) + except Exception as e: + logger.error(f"Step 2 Feature Extraction Failed: {e}") + sys.exit(1) + + if features_out: + features_dict = [ + { + "id": f.id, + "title": f.title, + "description": f.description, + "category": f.category, + "included_prs": f.included_prs, + "pr_contributions": f.pr_contributions, + "is_dcp_relevant": f.is_dcp_relevant, + } + for f in features + ] + with open(features_out, "w") as f: + json.dump(features_dict, f, indent=2) + logger.info(f"Saved synthesized features to {features_out}") + + # ---------------------------------------------------- + # STEP 3: Release Notes Writing + # ---------------------------------------------------- + logger.info("\n--- STEP 3: Release Notes Writing ---") + writer = ReleaseNotesWriter( + model_name=writer_model, + include_audit_log=include_audit_log, + ) + try: + markdown_notes = writer.render( + manifest=manifest, + features=features, + additional_instructions=instructions_text, + ) + except Exception as e: + logger.error(f"Step 3 Release Notes Writing Failed: {e}") + sys.exit(1) + + # Save output to disk + with open(output_path, "w") as f: + f.write(markdown_notes) + + logger.info("=" * 60) + logger.info(f"🎉 Success! Publication-ready release notes written to: {output_path}") + logger.info(f" - Total PRs Processed: {len(manifest.all_pull_requests)}") + logger.info(f" - Synthesized Feature Updates: {len(features)}") + logger.info("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/deploy/generate_release_notes/tests/test_main.py b/deploy/generate_release_notes/tests/test_main.py new file mode 100644 index 00000000..133a9757 --- /dev/null +++ b/deploy/generate_release_notes/tests/test_main.py @@ -0,0 +1,93 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for CLI Entry Point (deploy/generate_release_notes/main.py).""" + +from unittest.mock import MagicMock, patch +from click.testing import CliRunner + +from deploy.generate_release_notes.main import main +from deploy.generate_release_notes.models import ( + FeatureUpdate, + PullRequest, + ReleaseInfoManifest, +) + + +class TestMainCLI: + """Tests for main CLI entry point.""" + + @patch("deploy.generate_release_notes.main.PRExtractor") + @patch("deploy.generate_release_notes.main.FeatureExtractor") + @patch("deploy.generate_release_notes.main.ReleaseNotesWriter") + def test_main_cli_success( + self, mock_writer_cls, mock_feature_cls, mock_pr_cls, tmp_path + ): + """Test end-to-end CLI execution with mocked Step 1, Step 2, and Step 3.""" + # 1. Mock PRExtractor + mock_pr_instance = MagicMock() + mock_pr_cls.return_value = mock_pr_instance + manifest = ReleaseInfoManifest( + previous_version="v1.1.0", + new_version="v1.1.1", + all_pull_requests=[ + PullRequest( + number=188, + title="Remove status set to Success", + body="", + author="gmechali", + url="https://github.com/datacommonsorg/datacommons/pull/188", + merged_at="2026-07-24T18:05:13Z", + repo_name="datacommonsorg/datacommons", + ) + ], + ) + mock_pr_instance.extract.return_value = manifest + + # 2. Mock FeatureExtractor + mock_feature_instance = MagicMock() + mock_feature_cls.return_value = mock_feature_instance + feature = FeatureUpdate( + id="dataflow_fix", + title="Dataflow Fix", + description="Fixed dataflow status set.", + category="Ingestion & Safety", + included_prs=["datacommons#188"], + is_dcp_relevant=True, + ) + mock_feature_instance.extract_features.return_value = [feature] + + # 3. Mock ReleaseNotesWriter + mock_writer_instance = MagicMock() + mock_writer_cls.return_value = mock_writer_instance + mock_writer_instance.render.return_value = "# Data Commons Platform Release v1.1.1 (2026-07-29)\n\nSample release notes." + + out_file = tmp_path / "RELEASE_NOTES_v1.1.1.md" + runner = CliRunner() + result = runner.invoke( + main, + [ + "--prev", + "1.1.0", + "--new", + "1.1.1", + "--out", + str(out_file), + "--allow-missing-images", + ], + ) + + assert result.exit_code == 0 + assert out_file.exists() + assert out_file.read_text() == "# Data Commons Platform Release v1.1.1 (2026-07-29)\n\nSample release notes." From 4dfd05a467e12929d89707baf39da276495a374a Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:18:18 -0700 Subject: [PATCH 10/70] fix(deploy): add __main__.py to enable python -m deploy.generate_release_notes execution --- deploy/generate_release_notes/__main__.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 deploy/generate_release_notes/__main__.py diff --git a/deploy/generate_release_notes/__main__.py b/deploy/generate_release_notes/__main__.py new file mode 100644 index 00000000..926afb1e --- /dev/null +++ b/deploy/generate_release_notes/__main__.py @@ -0,0 +1,20 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Entry point when executing package as module: python -m deploy.generate_release_notes.""" + +from deploy.generate_release_notes.main import main + +if __name__ == "__main__": + main() From f451ae3c44bf766f933c6826d70c956f5392a479 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:19:42 -0700 Subject: [PATCH 11/70] fix(deploy): update default Gemini models to gemini-3.5-flash and gemini-3.5-pro --- deploy/generate_release_notes/feature_extractor.py | 4 ++-- deploy/generate_release_notes/release_notes_writer.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index 66520b25..29612dbb 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -35,8 +35,8 @@ logger = logging.getLogger(__name__) -DEFAULT_FILTER_MODEL = "gemini-2.5-flash" -DEFAULT_SYNTHESIS_MODEL = "gemini-2.5-pro" +DEFAULT_FILTER_MODEL = "gemini-3.5-flash" +DEFAULT_SYNTHESIS_MODEL = "gemini-3.5-pro" VALID_SOP_CATEGORIES = {cat.value for cat in SOPCategory} diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index e0a02a24..38ca5f5b 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -34,7 +34,7 @@ logger = logging.getLogger(__name__) -DEFAULT_WRITER_MODEL = "gemini-2.5-pro" +DEFAULT_WRITER_MODEL = "gemini-3.5-pro" class ReleaseNotesWriter: From 40230e402d2aa59e62647f2b2ed1aa21de0b0887 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:20:35 -0700 Subject: [PATCH 12/70] fix(deploy): update default Gemini models to gemini-3-flash and gemini-3-pro --- deploy/generate_release_notes/feature_extractor.py | 4 ++-- deploy/generate_release_notes/release_notes_writer.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index 29612dbb..43fcbdcd 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -35,8 +35,8 @@ logger = logging.getLogger(__name__) -DEFAULT_FILTER_MODEL = "gemini-3.5-flash" -DEFAULT_SYNTHESIS_MODEL = "gemini-3.5-pro" +DEFAULT_FILTER_MODEL = "gemini-3-flash" +DEFAULT_SYNTHESIS_MODEL = "gemini-3-pro" VALID_SOP_CATEGORIES = {cat.value for cat in SOPCategory} diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index 38ca5f5b..2a1d6c75 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -34,7 +34,7 @@ logger = logging.getLogger(__name__) -DEFAULT_WRITER_MODEL = "gemini-3.5-pro" +DEFAULT_WRITER_MODEL = "gemini-3-pro" class ReleaseNotesWriter: From 7d862691a7675c2289f97c17e8cbd71b8e19b4cf Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:22:15 -0700 Subject: [PATCH 13/70] fix(deploy): set default Gemini models to gemini-2.5-flash and gemini-2.5-pro for API compatibility --- deploy/generate_release_notes/feature_extractor.py | 4 ++-- deploy/generate_release_notes/release_notes_writer.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index 43fcbdcd..66520b25 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -35,8 +35,8 @@ logger = logging.getLogger(__name__) -DEFAULT_FILTER_MODEL = "gemini-3-flash" -DEFAULT_SYNTHESIS_MODEL = "gemini-3-pro" +DEFAULT_FILTER_MODEL = "gemini-2.5-flash" +DEFAULT_SYNTHESIS_MODEL = "gemini-2.5-pro" VALID_SOP_CATEGORIES = {cat.value for cat in SOPCategory} diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index 2a1d6c75..e0a02a24 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -34,7 +34,7 @@ logger = logging.getLogger(__name__) -DEFAULT_WRITER_MODEL = "gemini-3-pro" +DEFAULT_WRITER_MODEL = "gemini-2.5-pro" class ReleaseNotesWriter: From 7a5385c5f12b681e7a5e169b64549a6369ee64d4 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:22:40 -0700 Subject: [PATCH 14/70] fix(deploy): update default filter model to stable gemini-3.5-flash and synthesis model to gemini-2.5-pro --- deploy/generate_release_notes/feature_extractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index 66520b25..b448f455 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -35,7 +35,7 @@ logger = logging.getLogger(__name__) -DEFAULT_FILTER_MODEL = "gemini-2.5-flash" +DEFAULT_FILTER_MODEL = "gemini-3.5-flash" DEFAULT_SYNTHESIS_MODEL = "gemini-2.5-pro" VALID_SOP_CATEGORIES = {cat.value for cat in SOPCategory} From fbf33c6e6529960b7ee179e48a1f99a09aa22cf6 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:23:50 -0700 Subject: [PATCH 15/70] fix(deploy): set default Gemini models to gemini-3.5-flash for Stage 2 synthesis and Step 3 writing --- deploy/generate_release_notes/feature_extractor.py | 2 +- deploy/generate_release_notes/release_notes_writer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index b448f455..fa824d89 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -36,7 +36,7 @@ logger = logging.getLogger(__name__) DEFAULT_FILTER_MODEL = "gemini-3.5-flash" -DEFAULT_SYNTHESIS_MODEL = "gemini-2.5-pro" +DEFAULT_SYNTHESIS_MODEL = "gemini-3.5-flash" VALID_SOP_CATEGORIES = {cat.value for cat in SOPCategory} diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index e0a02a24..26ecab3a 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -34,7 +34,7 @@ logger = logging.getLogger(__name__) -DEFAULT_WRITER_MODEL = "gemini-2.5-pro" +DEFAULT_WRITER_MODEL = "gemini-3.5-flash" class ReleaseNotesWriter: From e0c5c77f15a7c9bc128b76694aeaa421232d3884 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:24:37 -0700 Subject: [PATCH 16/70] fix(deploy): set default Gemini models to gemini-3.6-flash across all steps --- deploy/generate_release_notes/feature_extractor.py | 4 ++-- deploy/generate_release_notes/release_notes_writer.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index fa824d89..1f26e537 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -35,8 +35,8 @@ logger = logging.getLogger(__name__) -DEFAULT_FILTER_MODEL = "gemini-3.5-flash" -DEFAULT_SYNTHESIS_MODEL = "gemini-3.5-flash" +DEFAULT_FILTER_MODEL = "gemini-3.6-flash" +DEFAULT_SYNTHESIS_MODEL = "gemini-3.6-flash" VALID_SOP_CATEGORIES = {cat.value for cat in SOPCategory} diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index 26ecab3a..6ae95a7b 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -34,7 +34,7 @@ logger = logging.getLogger(__name__) -DEFAULT_WRITER_MODEL = "gemini-3.5-flash" +DEFAULT_WRITER_MODEL = "gemini-3.6-flash" class ReleaseNotesWriter: From 70dae84b890fb6a0e80445f2a57a497cd09d72f1 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:26:25 -0700 Subject: [PATCH 17/70] feat(deploy): inject rich DCP domain knowledge, repository maps, and file path rules into Stage 2 prompt --- .../feature_extractor.py | 52 ++++++++++++++----- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index 1f26e537..58001c41 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -181,29 +181,55 @@ def synthesize_features_with_pro( f"{instructions_text}\n" ) - prompt = f"""You are an expert Technical Release Manager drafting official release notes for Data Commons Platform (DCP) release {manifest.new_version} (previous version: {manifest.previous_version}). + prompt = f"""You are an expert Technical Release Manager for Data Commons Platform (DCP) drafting official release notes for release {manifest.new_version} (previous version: {manifest.previous_version}). + +### Context & Domain Knowledge — What is Data Commons Platform (DCP)? +Data Commons Platform (DCP) is a self-hosted, Cloud Spanner-backed deployment of Data Commons. It replaces legacy Bigtable with Cloud Spanner graph tables and vector embeddings, featuring custom data ingestion pipelines, specialized serving APIs, and deployment automation across 6 key repositories: + +1. **`datacommonsorg/datacommons` (Monorepo)**: + - Contains DCP Terraform modules (`infra/dcp/`, `infra/modules/`), CLI tools (`packages/datacommons-cli/`), and Admin Portal (`packages/datacommons-admin/`). +2. **`datacommonsorg/website`**: + - Web application serving UI and APIs (`server/`, `static/`, `build/cdc_services/`, `build/cdc_data/`). +3. **`datacommonsorg/mixer`**: + - Core Spanner gRPC graph and StatVar serving engine (`internal/server/`, `proto/`). +4. **`datacommonsorg/import`**: + - Data processing pipelines: Simple importer (`simple/`), Dataflow Java worker (`pipeline/ingestion/`), Ingestion Helper (`pipeline/workflow/ingestion-helper/`), Aggregation Helper (`pipeline/workflow/aggregation-helper/`). +5. **`datacommonsorg/agent-toolkit`**: + - Datacommons Model Context Protocol (MCP) server and tools (`src/datacommons_mcp/`). +6. **`datacommonsorg/datacommons-data`**: + - Data preprocessor container image built from `import/simple/` and `website/build/cdc_data/`. + +--- + +### Classification Rules & SOP Categories: +Categorize EVERY feature into EXACTLY ONE of these 4 standard SOP categories based on its files and description: + +1. **"Spanner Graph & APIs"**: + - Features touching SDMX 3.0 REST endpoints, `/v2/observation` StatVar data retrieval, Spanner gRPC graph serving, `proto/` definitions, or `agent-toolkit` MCP server/tools. +2. **"Ingestion & Safety"**: + - Features touching Dataflow Java worker (`pipeline/ingestion/`), `ingestion-helper`, `aggregation-helper`, Spanner table loading, timestamp bounds, data validation, or health probes. +3. **"Search & Website"**: + - Features touching Spanner vector embeddings (`NodeEmbedding`), private instance `detect-and-fulfill`, Nginx/Envoy, Website UI, or Admin Portal UI. +4. **"Infra & Tooling"**: + - Features touching `infra/dcp/` (Terraform), `datacommons-cli`/`admin` PyPI packages, monorepo root configs, or Cloud Build release pipelines. + +--- ### Task Instructions: -1. **Semantic Classification**: Categorize each feature into EXACTLY ONE of the 4 standard DCP SOP categories (use exact category names): - - "Spanner Graph & APIs" (SDMX 3.0 REST API, `/v2/observation` StatVars, MCP server/tools, Spanner gRPC serving/protos) - - "Ingestion & Safety" (Ingestion Helper, Aggregation Helper, Dataflow Java worker, timestamp bounds, safety checks, Spanner loading) - - "Search & Website" (Spanner vector embeddings / `NodeEmbedding`, private instance `detect-and-fulfill`, Website UI, Nginx/Envoy) - - "Infra & Tooling" (DCP Terraform modules, `datacommons admin`/`cli` PyPI packages, monorepo packages, Cloud Build release pipelines) - -2. **Feature Grouping & Deduplication**: +1. **Feature Grouping & Deduplication**: - Combine related PRs (e.g., an initial feature PR + follow-up bug fixes + test PRs) into a SINGLE cohesive `FeatureUpdate`. - List all included PR qualified IDs in `included_prs` (e.g. `["datacommons#188", "datacommons#189"]`). -3. **Supersede Resolution & Chronology**: - - Use the `merged_at` timestamps to understand commit order. +2. **Supersede Resolution & Chronology**: + - Use `merged_at` timestamps to understand commit order. - If a PR was superseded or modified by a later PR in this release, describe only the FINAL state at {manifest.new_version}. -4. **Technical Writing & Comprehensive Context**: +3. **Technical Writing & Comprehensive Context**: - Write clear, concise, engineer-style titles and descriptions. Avoid marketing fluff or non-technical summaries. - Set `is_dcp_relevant: true` for all platform-relevant features, or `false` for base-only features. - - If a feature contains only ONE PR, ensure the `description` is rich and comprehensive enough for release notes generation to understand all capabilities implemented. + - If a feature contains only ONE PR, ensure `description` is rich and comprehensive enough for release notes generation to understand all capabilities implemented. -5. **Per-PR Contribution Summaries**: +4. **Per-PR Contribution Summaries**: - For EVERY PR listed in `included_prs`, provide a specific 1-2 sentence contribution summary under `pr_contributions` mapping the qualified PR ID to its specific capability contribution (e.g., `{{"datacommons#188": "Removed premature success status set at end of dataflow stage", "datacommons#189": "Added max_workers Terraform variable for Dataflow auto-scaling"}}`). {instructions_context} From 7dc1f13cbb5456a58e2353d7572ce3e56b64a41e Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:28:49 -0700 Subject: [PATCH 18/70] fix(deploy): enforce User-First focus over developer implementation details across Feature Extractor and Release Notes Writer --- deploy/generate_release_notes/feature_extractor.py | 4 ++-- deploy/generate_release_notes/release_notes_writer.py | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index 58001c41..f15a3566 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -224,8 +224,8 @@ def synthesize_features_with_pro( - Use `merged_at` timestamps to understand commit order. - If a PR was superseded or modified by a later PR in this release, describe only the FINAL state at {manifest.new_version}. -3. **Technical Writing & Comprehensive Context**: - - Write clear, concise, engineer-style titles and descriptions. Avoid marketing fluff or non-technical summaries. +3. **User-First Technical Writing**: + - Focus feature titles and descriptions on **User Capabilities, Platform Benefits, and Operator Configurations**, NOT internal developer implementation details (e.g. avoid 'Refactored helper function X' or 'Updated internal class Y'). - Set `is_dcp_relevant: true` for all platform-relevant features, or `false` for base-only features. - If a feature contains only ONE PR, ensure `description` is rich and comprehensive enough for release notes generation to understand all capabilities implemented. diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index 6ae95a7b..a0473be6 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -183,12 +183,16 @@ def render( ### Core Writing Guidelines & Tone: 1. **Tone & Style**: Write in a clear, positive, partner-facing tone using plain language. Use second person ("You can now...") and active voice for new features and improvements. Use past tense for bug fixes ("Fixed...", "Resolved..."). 2. **Target Audience**: Technical and non-technical partners, platform operators, and stakeholders who need to understand what changed, why it matters, and how to use it. -3. **Level of Detail**: +3. **User-First Focus over Implementation Details**: + - Focus strictly on user-visible capabilities, platform operator configuration changes, and business/technical benefits. + - DO NOT mention internal code refactoring details, internal class names, or developer-only function changes. + - Translate developer PR titles (e.g. 'Refactor ObservationMap helper') into partner/user outcomes (e.g. 'Improved StatVar query performance and concurrency under high traffic'). +4. **Level of Detail**: - For major features: Provide an engaging "What's New", "Why it Matters" (business/technical benefit), and a bulleted list of "Capabilities & Changes". - For single-PR features: Provide rich, self-contained descriptions so partners do not need to look up code diffs. - For bug fixes: Focus on what was broken, how it was resolved, and how the system behaves now. -4. **Link Formatting**: Every PR reference MUST be formatted as a clickable Markdown link using the format `[#]()` (e.g. `[datacommons#188](https://github.com/datacommonsorg/datacommons/pull/188)`). -5. **Strict Constraints**: +5. **Link Formatting**: Every PR reference MUST be formatted as a clickable Markdown link using the format `[#]()` (e.g. `[datacommons#188](https://github.com/datacommonsorg/datacommons/pull/188)`). +6. **Strict Constraints**: - DO NOT use any emojis anywhere in the document. - DO NOT include a component version table or git commit/SHA table. - DO NOT include release range commit text (e.g., "Release range: v1.1.0 to v1.1.1"). From 54ce1e51b4d471d6b927c750eeffbb84749f480e Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:31:22 -0700 Subject: [PATCH 19/70] refactor(deploy): simplify Feature Extractor to single-stage Gemini pipeline with direct bot/noise filtering instructions --- .../feature_extractor.py | 148 ++++-------------- deploy/generate_release_notes/main.py | 18 +-- .../tests/test_feature_extractor.py | 76 ++------- 3 files changed, 49 insertions(+), 193 deletions(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index f15a3566..73fb5f42 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -14,8 +14,8 @@ """Step 2: Feature Extractor for Data Commons Platform (DCP) release notes generation. -Executes a Two-Stage Gemini LLM Pipeline (Flash + Pro) for noise filtering, -SOP classification, feature grouping, and technical release notes synthesis. +Synthesizes raw PullRequests into structured FeatureUpdate objects using Gemini LLM, +classifying features into standard SOP categories and handling feature grouping. """ import json @@ -35,20 +35,21 @@ logger = logging.getLogger(__name__) -DEFAULT_FILTER_MODEL = "gemini-3.6-flash" -DEFAULT_SYNTHESIS_MODEL = "gemini-3.6-flash" +DEFAULT_MODEL = "gemini-3.6-flash" VALID_SOP_CATEGORIES = {cat.value for cat in SOPCategory} class FeatureExtractor: - """Two-Stage Gemini LLM Pipeline for filtering, classifying, and synthesizing DCP release features.""" + """Single-stage Gemini LLM Pipeline for filtering, classifying, and synthesizing DCP release features.""" def __init__( self, api_key: Optional[str] = None, - filter_model: str = DEFAULT_FILTER_MODEL, - synthesis_model: str = DEFAULT_SYNTHESIS_MODEL, + model_name: str = DEFAULT_MODEL, + # Backward compatibility aliases + filter_model: Optional[str] = None, + synthesis_model: Optional[str] = None, ): key = api_key or os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY") if not key: @@ -59,98 +60,25 @@ def __init__( else: self.client = genai.Client(api_key=key) - self.filter_model = filter_model - self.synthesis_model = synthesis_model + self.model_name = synthesis_model or model_name or DEFAULT_MODEL - def filter_prs_with_flash(self, manifest: ReleaseInfoManifest) -> List[PullRequest]: - """Stage 1 (Flash Model): Rapidly triages all raw PRs to weed out bot bumps, typo fixes, and non-informative noise.""" - if not manifest.all_pull_requests: - logger.warning("No PRs provided in manifest for Stage 1 filtering.") - return [] - - logger.info( - f"Stage 1: Filtering {len(manifest.all_pull_requests)} raw PRs using {self.filter_model}..." - ) - - # Build compact representation for Flash model using qualified PR IDs (e.g. 'datacommons#188') - pr_summaries = [] - for pr in manifest.all_pull_requests: - pr_summaries.append( - { - "id": pr.qualified_id, - "title": pr.title, - "repo": pr.repo_name, - "author": pr.author, - "files_changed_count": len(pr.files_changed), - "sample_files": pr.files_changed[:10], - } - ) - - prompt = f"""You are a Senior Technical Release Engineer for Data Commons Platform (DCP). -Analyze the following list of merged Pull Requests for release range {manifest.previous_version} -> {manifest.new_version}. - -Goal: Identify all SUBSTANTIVE, meaningful Pull Requests that represent feature additions, bug fixes, infrastructure changes, configuration updates, or Data Commons Platform/Base capabilities. - -Filter OUT ONLY non-informative noise: -- Automated bot version bumps (e.g., 'chore: bump version to 1.1.1', dependabot, renovate). -- Trivial formatting, linting, or typo fixes in documentation/README (e.g., 'fix typo in README'). -- Trivial internal test-only refactors with zero functional impact. - -DO NOT filter out base Data Commons features or infrastructure PRs — keep all substantive PRs! - -Here is the list of PRs: -{json.dumps(pr_summaries, indent=2)} - -Respond ONLY with a JSON object containing a single key "relevant_pr_ids" with an array of qualified PR ID strings (e.g., ["datacommons#188", "import#42"]). -Example: {{"relevant_pr_ids": ["datacommons#188", "import#42"]}} -""" - - try: - config = types.GenerateContentConfig( - response_mime_type="application/json", - temperature=0.1, - ) - res = self.client.models.generate_content( - model=self.filter_model, - contents=prompt, - config=config, - ) - data = json.loads(res.text) - relevant_ids = set(data.get("relevant_pr_ids", [])) - - candidate_prs = [ - pr for pr in manifest.all_pull_requests if pr.qualified_id in relevant_ids - ] - logger.info( - f"Stage 1 Complete: Retained {len(candidate_prs)} / {len(manifest.all_pull_requests)} substantive PRs." - ) - return candidate_prs - except Exception as e: - logger.error(f"Stage 1 Flash filtering failed: {e}. Falling back to all non-bot PRs.") - # Basic fallback for error resilience - return [ - pr for pr in manifest.all_pull_requests - if "bump version" not in pr.title.lower() and pr.author != "datacommons-robot-author" - ] - - def synthesize_features_with_pro( + def extract_features( self, manifest: ReleaseInfoManifest, - candidate_prs: List[PullRequest], additional_instructions: Optional[str] = None, ) -> List[FeatureUpdate]: - """Stage 2 (Pro Model): Performs deep semantic classification, feature grouping, override resolution, and SOP drafting.""" - if not candidate_prs: - logger.warning("No candidate PRs provided for Stage 2 synthesis.") + """Classifies, groups, and synthesizes raw PRs into FeatureUpdate objects in 1 Gemini call.""" + if not manifest.all_pull_requests: + logger.warning("No PRs provided in manifest for feature extraction.") return [] logger.info( - f"Stage 2: Synthesizing features from {len(candidate_prs)} candidate PRs using {self.synthesis_model}..." + f"Step 2: Extracting features from {len(manifest.all_pull_requests)} PRs using {self.model_name}..." ) - # Build detailed PR context for Pro model with merged_at timestamps and qualified IDs + # Build detailed PR context for model with merged_at timestamps and qualified IDs detailed_prs = [] - for pr in candidate_prs: + for pr in manifest.all_pull_requests: # Preserve "BREAKING CHANGE:" sections if present in body body_text = pr.body or "" body_snippet = body_text[:500] @@ -216,25 +144,29 @@ def synthesize_features_with_pro( --- ### Task Instructions: -1. **Feature Grouping & Deduplication**: +1. **Filter Out & Ignore Irrelevant PRs**: + - Completely IGNORE automated bot PRs (e.g. dependabot, renovate, 'chore: bump version to 1.1.1'). + - Completely IGNORE trivial formatting, typo fixes, or non-informative refactors with zero user impact. + +2. **Feature Grouping & Deduplication**: - Combine related PRs (e.g., an initial feature PR + follow-up bug fixes + test PRs) into a SINGLE cohesive `FeatureUpdate`. - List all included PR qualified IDs in `included_prs` (e.g. `["datacommons#188", "datacommons#189"]`). -2. **Supersede Resolution & Chronology**: +3. **Supersede Resolution & Chronology**: - Use `merged_at` timestamps to understand commit order. - If a PR was superseded or modified by a later PR in this release, describe only the FINAL state at {manifest.new_version}. -3. **User-First Technical Writing**: +4. **User-First Technical Writing**: - Focus feature titles and descriptions on **User Capabilities, Platform Benefits, and Operator Configurations**, NOT internal developer implementation details (e.g. avoid 'Refactored helper function X' or 'Updated internal class Y'). - Set `is_dcp_relevant: true` for all platform-relevant features, or `false` for base-only features. - If a feature contains only ONE PR, ensure `description` is rich and comprehensive enough for release notes generation to understand all capabilities implemented. -4. **Per-PR Contribution Summaries**: +5. **Per-PR Contribution Summaries**: - For EVERY PR listed in `included_prs`, provide a specific 1-2 sentence contribution summary under `pr_contributions` mapping the qualified PR ID to its specific capability contribution (e.g., `{{"datacommons#188": "Removed premature success status set at end of dataflow stage", "datacommons#189": "Added max_workers Terraform variable for Dataflow auto-scaling"}}`). {instructions_context} -### Substantive Candidate PRs: +### Raw Merged PRs: {json.dumps(detailed_prs, indent=2)} Respond ONLY with a JSON array of FeatureUpdate objects with the following schema: @@ -262,7 +194,7 @@ def synthesize_features_with_pro( temperature=0.2, ) res = self.client.models.generate_content( - model=self.synthesis_model, + model=self.model_name, contents=prompt, config=config, ) @@ -295,31 +227,9 @@ def synthesize_features_with_pro( features.append(feature) logger.info( - f"Stage 2 Complete: Synthesized {len(features)} structured FeatureUpdate objects across categories." + f"Step 2 Complete: Synthesized {len(features)} structured FeatureUpdate objects across categories." ) return features except Exception as e: - logger.error(f"Stage 2 Pro synthesis failed: {e}") - raise RuntimeError(f"Failed to synthesize release features with Gemini Pro: {e}") - - def extract_features( - self, - manifest: ReleaseInfoManifest, - additional_instructions: Optional[str] = None, - ) -> List[FeatureUpdate]: - """Main entry point: orchestrates Stage 1 Flash filtering -> Stage 2 Pro synthesis -> List[FeatureUpdate].""" - logger.info( - f"Starting Step 2 Feature Extraction for release {manifest.previous_version} -> {manifest.new_version}..." - ) - - # Stage 1: Fast Flash Noise Filter - candidate_prs = self.filter_prs_with_flash(manifest) - - # Stage 2: Deep Pro Synthesis & SOP Classification - features = self.synthesize_features_with_pro( - manifest=manifest, - candidate_prs=candidate_prs, - additional_instructions=additional_instructions, - ) - - return features + logger.error(f"Step 2 Feature extraction failed: {e}") + raise RuntimeError(f"Failed to extract release features with Gemini: {e}") diff --git a/deploy/generate_release_notes/main.py b/deploy/generate_release_notes/main.py index e4d0b3e0..338bf523 100644 --- a/deploy/generate_release_notes/main.py +++ b/deploy/generate_release_notes/main.py @@ -34,8 +34,7 @@ import click from deploy.generate_release_notes.feature_extractor import ( - DEFAULT_FILTER_MODEL, - DEFAULT_SYNTHESIS_MODEL, + DEFAULT_MODEL as DEFAULT_FEATURE_MODEL, FeatureExtractor, ) from deploy.generate_release_notes.pr_extractor import PRExtractor @@ -91,15 +90,12 @@ default=False, help="Append complete raw PR audit log table at the bottom of the release notes.", ) -@click.option( - "--filter-model", - default=DEFAULT_FILTER_MODEL, - help=f"Gemini model for Stage 1 noise filtering (default: {DEFAULT_FILTER_MODEL}).", -) @click.option( "--synthesis-model", - default=DEFAULT_SYNTHESIS_MODEL, - help=f"Gemini model for Stage 2 feature synthesis (default: {DEFAULT_SYNTHESIS_MODEL}).", + "--filter-model", + "synthesis_model", + default=DEFAULT_FEATURE_MODEL, + help=f"Gemini model for Step 2 feature synthesis (default: {DEFAULT_FEATURE_MODEL}).", ) @click.option( "--writer-model", @@ -123,7 +119,6 @@ def main( additional_instructions: Optional[str], allow_missing_images: bool, include_audit_log: bool, - filter_model: str, synthesis_model: str, writer_model: str, manifest_out: Optional[str], @@ -194,8 +189,7 @@ def main( # ---------------------------------------------------- logger.info("\n--- STEP 2: Feature Extraction & SOP Classification ---") feature_extractor = FeatureExtractor( - filter_model=filter_model, - synthesis_model=synthesis_model, + model_name=synthesis_model, ) try: features = feature_extractor.extract_features( diff --git a/deploy/generate_release_notes/tests/test_feature_extractor.py b/deploy/generate_release_notes/tests/test_feature_extractor.py index 77c01e45..53c0671d 100644 --- a/deploy/generate_release_notes/tests/test_feature_extractor.py +++ b/deploy/generate_release_notes/tests/test_feature_extractor.py @@ -31,65 +31,8 @@ class TestFeatureExtractorUnit: """Unit tests for FeatureExtractor with mocked Gemini API calls.""" @patch("google.genai.Client") - def test_filter_prs_with_flash(self, mock_client_cls): - """Test Stage 1 Flash model filtering out bot version bumps and noise.""" - mock_client = MagicMock() - mock_client_cls.return_value = mock_client - - # Mock Flash model response returning PRs datacommons#188 and datacommons#189 - mock_res = MagicMock() - mock_res.text = json.dumps({"relevant_pr_ids": ["datacommons#188", "datacommons#189"]}) - mock_client.models.generate_content.return_value = mock_res - - pr188 = PullRequest( - number=188, - title="[DCP Ingestion] Remove status set to Success at end of dataflow stage", - body="Fixes status race condition", - author="gmechali", - url="https://github.com/datacommonsorg/datacommons/pull/188", - merged_at="2026-07-24T18:05:13Z", - repo_name="datacommonsorg/datacommons", - files_changed=["infra/dcp/dataflow_job.tf"], - ) - pr189 = PullRequest( - number=189, - title="[DCP Ingestion] Allow dataflow to scale workers based on Terraform Variables", - body="Adds max_workers variable", - author="gmechali", - url="https://github.com/datacommonsorg/datacommons/pull/189", - merged_at="2026-07-24T19:13:17Z", - repo_name="datacommonsorg/datacommons", - files_changed=["infra/dcp/variables.tf"], - ) - pr195_bot = PullRequest( - number=195, - title="chore: bump version to 1.1.1", - body="Automated release bump", - author="datacommons-robot-author", - url="https://github.com/datacommonsorg/datacommons/pull/195", - merged_at="2026-07-29T00:37:18Z", - repo_name="datacommonsorg/datacommons", - files_changed=["VERSION"], - ) - - manifest = ReleaseInfoManifest( - previous_version="v1.1.0", - new_version="v1.1.1", - all_pull_requests=[pr188, pr189, pr195_bot], - ) - - extractor = FeatureExtractor(api_key="mock_key") - candidates = extractor.filter_prs_with_flash(manifest) - - assert len(candidates) == 2 - candidate_ids = [pr.qualified_id for pr in candidates] - assert "datacommons#188" in candidate_ids - assert "datacommons#189" in candidate_ids - assert "datacommons#195" not in candidate_ids - - @patch("google.genai.Client") - def test_synthesize_features_with_pro(self, mock_client_cls): - """Test Stage 2 Pro model feature grouping and SOP classification.""" + def test_extract_features_single_stage(self, mock_client_cls): + """Test single-stage feature extraction and SOP classification while filtering bot PRs.""" mock_client = MagicMock() mock_client_cls.return_value = mock_client @@ -133,17 +76,26 @@ def test_synthesize_features_with_pro(self, mock_client_cls): repo_name="datacommonsorg/datacommons", files_changed=["infra/dcp/variables.tf"], ) + pr195_bot = PullRequest( + number=195, + title="chore: bump version to 1.1.1", + body="Automated release bump", + author="datacommons-robot-author", + url="https://github.com/datacommonsorg/datacommons/pull/195", + merged_at="2026-07-29T00:37:18Z", + repo_name="datacommonsorg/datacommons", + files_changed=["VERSION"], + ) manifest = ReleaseInfoManifest( previous_version="v1.1.0", new_version="v1.1.1", - all_pull_requests=[pr188, pr189], + all_pull_requests=[pr188, pr189, pr195_bot], ) extractor = FeatureExtractor(api_key="mock_key") - features = extractor.synthesize_features_with_pro( + features = extractor.extract_features( manifest=manifest, - candidate_prs=[pr188, pr189], additional_instructions="Focus on Dataflow scaling", ) From 9593d43d434665853356d1c0219efd30cfed1840 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:36:15 -0700 Subject: [PATCH 20/70] fix(deploy): update prompts to write for users building ON TOP OF platform (focus on Ingestion Inputs & APIs, de-emphasize Spanner DB layer, drop internal testing) --- .../feature_extractor.py | 17 ++++++++------- deploy/generate_release_notes/pr_extractor.py | 8 +++++-- .../release_notes_writer.py | 21 +++++++++++-------- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index 73fb5f42..69bcfccf 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -144,23 +144,24 @@ def extract_features( --- ### Task Instructions: -1. **Filter Out & Ignore Irrelevant PRs**: +1. **Filter Out & Ignore Internal Dev & Testing PRs**: - Completely IGNORE automated bot PRs (e.g. dependabot, renovate, 'chore: bump version to 1.1.1'). + - Completely IGNORE internal integration test setups, Spanner Omni test suites, CI sandbox workflows, local test harnesses, and test-only sample data updates. - Completely IGNORE trivial formatting, typo fixes, or non-informative refactors with zero user impact. -2. **Feature Grouping & Deduplication**: +2. **User Persona Focus (Building ON TOP OF Platform)**: + - Write for people **building ON TOP OF the platform** (data engineers, API consumers, instance operators). + - Focus on **Ingestion Inputs & Pipelines** (CSV/SDMX inputs, import workflows, validation rules) and **APIs & Tooling** (REST APIs, SDMX 3.0 endpoints, `/v2/observation`, MCP tools, Web UI, Admin CLI). + - **De-emphasize Database Layer Details**: Minimize mentions of Spanner database internals (e.g. Spanner graph schema, KeyValueStore cutover). Focus instead on the user-facing API or Ingestion behavior change. + +3. **Feature Grouping & Deduplication**: - Combine related PRs (e.g., an initial feature PR + follow-up bug fixes + test PRs) into a SINGLE cohesive `FeatureUpdate`. - List all included PR qualified IDs in `included_prs` (e.g. `["datacommons#188", "datacommons#189"]`). -3. **Supersede Resolution & Chronology**: +4. **Supersede Resolution & Chronology**: - Use `merged_at` timestamps to understand commit order. - If a PR was superseded or modified by a later PR in this release, describe only the FINAL state at {manifest.new_version}. -4. **User-First Technical Writing**: - - Focus feature titles and descriptions on **User Capabilities, Platform Benefits, and Operator Configurations**, NOT internal developer implementation details (e.g. avoid 'Refactored helper function X' or 'Updated internal class Y'). - - Set `is_dcp_relevant: true` for all platform-relevant features, or `false` for base-only features. - - If a feature contains only ONE PR, ensure `description` is rich and comprehensive enough for release notes generation to understand all capabilities implemented. - 5. **Per-PR Contribution Summaries**: - For EVERY PR listed in `included_prs`, provide a specific 1-2 sentence contribution summary under `pr_contributions` mapping the qualified PR ID to its specific capability contribution (e.g., `{{"datacommons#188": "Removed premature success status set at end of dataflow stage", "datacommons#189": "Added max_workers Terraform variable for Dataflow auto-scaling"}}`). diff --git a/deploy/generate_release_notes/pr_extractor.py b/deploy/generate_release_notes/pr_extractor.py index 3281a458..fc8adf7b 100644 --- a/deploy/generate_release_notes/pr_extractor.py +++ b/deploy/generate_release_notes/pr_extractor.py @@ -18,7 +18,7 @@ gcloud container image tags and GitHub CLI (gh pr list). """ -from datetime import datetime +from datetime import datetime, timezone import json import logging import subprocess @@ -335,7 +335,11 @@ def extract( if not ts_list: # Default to fallback timestamp if image tags missing t_min = "2026-01-01T00:00:00Z" - t_max = datetime.utcnow().isoformat() + "Z" + t_max = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + elif len(ts_list) == 1: + # If only prev_timestamp exists (e.g. new_version tag missing during staging/test), search from prev to NOW + t_min = ts_list[0] + t_max = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") else: sorted_ts = sorted(ts_list) t_min = sorted_ts[0] diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index a0473be6..04a2a197 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -182,17 +182,20 @@ def render( ### Core Writing Guidelines & Tone: 1. **Tone & Style**: Write in a clear, positive, partner-facing tone using plain language. Use second person ("You can now...") and active voice for new features and improvements. Use past tense for bug fixes ("Fixed...", "Resolved..."). -2. **Target Audience**: Technical and non-technical partners, platform operators, and stakeholders who need to understand what changed, why it matters, and how to use it. -3. **User-First Focus over Implementation Details**: - - Focus strictly on user-visible capabilities, platform operator configuration changes, and business/technical benefits. - - DO NOT mention internal code refactoring details, internal class names, or developer-only function changes. - - Translate developer PR titles (e.g. 'Refactor ObservationMap helper') into partner/user outcomes (e.g. 'Improved StatVar query performance and concurrency under high traffic'). -4. **Level of Detail**: +2. **Target Audience (Building ON TOP OF Platform)**: + - Write for external developers, data engineers, and instance operators building ON TOP OF Data Commons Platform (NOT internal platform maintainers). + - Primary Focus: **Ingestion Inputs & Pipelines** (CSV/SDMX inputs, data loading, workflow parameters) and **APIs & Tooling** (REST APIs, SDMX 3.0 endpoints, `/v2/observation`, MCP tools, Web UI, Admin CLI). +3. **De-emphasize Database Layer**: + - DO NOT focus on Spanner database layer mechanics (e.g. Spanner graph schema, KeyValueStore cutovers, Spanner table internals). Frame changes around how they affect API response speed, data availability, or ingestion inputs! +4. **Strict Exclusions**: + - DO NOT include internal integration test suites, Spanner Omni test setups, CI sandbox workflows, or developer-only test sample data updates. + - DO NOT include internal code refactors or unused example file cleanups. +5. **Level of Detail**: - For major features: Provide an engaging "What's New", "Why it Matters" (business/technical benefit), and a bulleted list of "Capabilities & Changes". - For single-PR features: Provide rich, self-contained descriptions so partners do not need to look up code diffs. - For bug fixes: Focus on what was broken, how it was resolved, and how the system behaves now. -5. **Link Formatting**: Every PR reference MUST be formatted as a clickable Markdown link using the format `[#]()` (e.g. `[datacommons#188](https://github.com/datacommonsorg/datacommons/pull/188)`). -6. **Strict Constraints**: +6. **Link Formatting**: Every PR reference MUST be formatted as a clickable Markdown link using the format `[#]()` (e.g. `[datacommons#188](https://github.com/datacommonsorg/datacommons/pull/188)`). +7. **Strict Constraints**: - DO NOT use any emojis anywhere in the document. - DO NOT include a component version table or git commit/SHA table. - DO NOT include release range commit text (e.g., "Release range: v1.1.0 to v1.1.1"). @@ -244,7 +247,7 @@ def render( ## Bug Fixes -*(List bug fixes as concise bullet points in past tense describing what was resolved and why it helps partners and operators:)* +*(List ONLY substantive bug fixes that resolve user-facing errors, data issues, or platform operator failures in past tense. DO NOT include internal dev cleanups, test refactors, or unused example file removals:)* - **[Component / Scope]**: [Description of what was fixed and how the system behaves now] ([`repo#PR`](URL)) """ From d033d8f24c47b66d28974c59e9cd1e57d4e34e42 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:36:48 -0700 Subject: [PATCH 21/70] fix(deploy): fix date range calculation when new_version tag is missing so t_max extends to current time for all repos --- deploy/generate_release_notes/feature_extractor.py | 4 ++-- deploy/generate_release_notes/pr_extractor.py | 11 ++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index 69bcfccf..c5231699 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -144,9 +144,9 @@ def extract_features( --- ### Task Instructions: -1. **Filter Out & Ignore Internal Dev & Testing PRs**: +1. **Filter Out & Ignore Internal Dev & Testing PRs (STRICT)**: - Completely IGNORE automated bot PRs (e.g. dependabot, renovate, 'chore: bump version to 1.1.1'). - - Completely IGNORE internal integration test setups, Spanner Omni test suites, CI sandbox workflows, local test harnesses, and test-only sample data updates. + - Completely IGNORE all test-only PRs: integration test setups, Spanner Omni test conversions, CI sandbox workflows, local test harnesses, hermetic test refactors, and test-only sample data updates (e.g. OECD wage sample data). DO NOT output any FeatureUpdate for test-only PRs! - Completely IGNORE trivial formatting, typo fixes, or non-informative refactors with zero user impact. 2. **User Persona Focus (Building ON TOP OF Platform)**: diff --git a/deploy/generate_release_notes/pr_extractor.py b/deploy/generate_release_notes/pr_extractor.py index fc8adf7b..2667ea11 100644 --- a/deploy/generate_release_notes/pr_extractor.py +++ b/deploy/generate_release_notes/pr_extractor.py @@ -312,6 +312,7 @@ def extract( # 1. Resolve version info and timestamps across all components repo_timestamps: Dict[str, List[str]] = {} + repo_has_missing_new: Dict[str, bool] = {} for comp_id, comp in COMPONENTS.items(): comp_info = self.resolve_component_version( comp, prev_version, new_version @@ -322,10 +323,13 @@ def extract( for rule in comp.sources: if rule.repo not in repo_timestamps: repo_timestamps[rule.repo] = [] + repo_has_missing_new[rule.repo] = False if comp_info.prev_timestamp: repo_timestamps[rule.repo].append(comp_info.prev_timestamp) if comp_info.new_timestamp: repo_timestamps[rule.repo].append(comp_info.new_timestamp) + else: + repo_has_missing_new[rule.repo] = True # 2. For each repository, compute min & max timestamps and fetch PRs in 1 single call raw_prs_by_repo: Dict[str, List[PullRequest]] = {} @@ -336,9 +340,10 @@ def extract( # Default to fallback timestamp if image tags missing t_min = "2026-01-01T00:00:00Z" t_max = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - elif len(ts_list) == 1: - # If only prev_timestamp exists (e.g. new_version tag missing during staging/test), search from prev to NOW - t_min = ts_list[0] + elif repo_has_missing_new.get(repo, False): + # If new_version tag is missing for this repo, extend t_max to current time NOW + sorted_ts = sorted(ts_list) + t_min = sorted_ts[0] t_max = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") else: sorted_ts = sorted(ts_list) From 3ff5e32ca1e31dbcff68dbac17834fe67a54b4d0 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:37:19 -0700 Subject: [PATCH 22/70] fix(deploy): filter out non-DCP-relevant features and test-only PRs, de-emphasize Spanner DB layer in Release Notes Writer --- deploy/generate_release_notes/release_notes_writer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index 04a2a197..d7a249ba 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -150,9 +150,12 @@ def render( f"Step 3: Rendering release notes for {manifest.new_version} using {self.model_name}..." ) - # Build payloads for prompt + # Build payloads for prompt — include ONLY DCP-relevant user/partner features features_payload = [] for feat in features: + if not feat.is_dcp_relevant: + logger.info(f"Omitting non-DCP-relevant feature from release notes: {feat.title}") + continue features_payload.append( { "id": feat.id, From e4d0331e4ca2480f2fda095c49aafef2e07bbd3c Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:40:31 -0700 Subject: [PATCH 23/70] fix(deploy): fix link formatting template in ReleaseNotesWriter to remove backticks around PR links --- .../generate_release_notes/release_notes_writer.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index d7a249ba..8c6ef7f8 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -197,7 +197,9 @@ def render( - For major features: Provide an engaging "What's New", "Why it Matters" (business/technical benefit), and a bulleted list of "Capabilities & Changes". - For single-PR features: Provide rich, self-contained descriptions so partners do not need to look up code diffs. - For bug fixes: Focus on what was broken, how it was resolved, and how the system behaves now. -6. **Link Formatting**: Every PR reference MUST be formatted as a clickable Markdown link using the format `[#]()` (e.g. `[datacommons#188](https://github.com/datacommonsorg/datacommons/pull/188)`). +6. **Link Formatting**: + - Every PR reference MUST be formatted as a clickable Markdown link: `[#]()` (e.g. `[datacommons#188](https://github.com/datacommonsorg/datacommons/pull/188)`). + - DO NOT put backticks around or inside the link text (e.g. write `[datacommons#188](URL)`, NEVER `[`datacommons#188`](URL)` or `` `[datacommons#188](URL)` ``). 7. **Strict Constraints**: - DO NOT use any emojis anywhere in the document. - DO NOT include a component version table or git commit/SHA table. @@ -235,8 +237,8 @@ def render( **What's New**: [Clear description of what partners or operators can now do] **Why it Matters**: [Business benefit and technical impact] **Capabilities & Changes**: -- [Capability 1] ([`repo#PR`](URL)) -- [Capability 2] ([`repo#PR`](URL)) +- [Capability 1] ([repo#PR](URL)) +- [Capability 2] ([repo#PR](URL)) --- @@ -244,7 +246,7 @@ def render( *(List enhancements, performance updates, or required Terraform/Admin Panel configuration changes as concise bullet points:)* -- **[Improvement Title]**: [Summary of update, configuration instructions if required, and benefit] ([`repo#PR`](URL)) +- **[Improvement Title]**: [Summary of update, configuration instructions if required, and benefit] ([repo#PR](URL)) --- @@ -252,7 +254,7 @@ def render( *(List ONLY substantive bug fixes that resolve user-facing errors, data issues, or platform operator failures in past tense. DO NOT include internal dev cleanups, test refactors, or unused example file removals:)* -- **[Component / Scope]**: [Description of what was fixed and how the system behaves now] ([`repo#PR`](URL)) +- **[Component / Scope]**: [Description of what was fixed and how the system behaves now] ([repo#PR](URL)) """ try: From 46fa01be6a7159e1b26587aa770457d7bc537621 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:41:36 -0700 Subject: [PATCH 24/70] fix(deploy): update Capabilities section to Capabilities & Use Cases Enabled, focusing on explicit user actions and input types --- deploy/generate_release_notes/feature_extractor.py | 4 ++-- deploy/generate_release_notes/release_notes_writer.py | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index c5231699..b14048bc 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -162,8 +162,8 @@ def extract_features( - Use `merged_at` timestamps to understand commit order. - If a PR was superseded or modified by a later PR in this release, describe only the FINAL state at {manifest.new_version}. -5. **Per-PR Contribution Summaries**: - - For EVERY PR listed in `included_prs`, provide a specific 1-2 sentence contribution summary under `pr_contributions` mapping the qualified PR ID to its specific capability contribution (e.g., `{{"datacommons#188": "Removed premature success status set at end of dataflow stage", "datacommons#189": "Added max_workers Terraform variable for Dataflow auto-scaling"}}`). +5. **Actionable Per-PR Capability & Use Case Summaries**: + - For EVERY PR listed in `included_prs`, provide a specific 1-2 sentence summary under `pr_contributions` describing an **explicit thing the user can DO or input format supported** because of this PR (e.g., `{{"agent-toolkit#211": "Query bilateral trade and migration relationships between multiple entities", "datacommons#189": "Configure max_workers in Terraform to scale Dataflow workers automatically for large imports"}}`). DO NOT list internal code refactors! {instructions_context} diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index 8c6ef7f8..8630d356 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -193,8 +193,9 @@ def render( 4. **Strict Exclusions**: - DO NOT include internal integration test suites, Spanner Omni test setups, CI sandbox workflows, or developer-only test sample data updates. - DO NOT include internal code refactors or unused example file cleanups. -5. **Level of Detail**: - - For major features: Provide an engaging "What's New", "Why it Matters" (business/technical benefit), and a bulleted list of "Capabilities & Changes". +5. **Level of Detail & Use Case Focus**: + - For major features: Provide an engaging "What's New", "Why it Matters" (business/technical benefit), and a bulleted list under "**Capabilities & Use Cases Enabled**". + - **DO NOT write a laundry list of code changes or PR descriptions!** Every bullet under Capabilities MUST describe an explicit thing the user can DO (e.g., 'Query bilateral trade flows between two countries', 'Import SDMX 3.0 CSV files directly', 'Run targeted single-entity vs child-place research playbooks'). Focus on supported input types, query capabilities, and real use cases! - For single-PR features: Provide rich, self-contained descriptions so partners do not need to look up code diffs. - For bug fixes: Focus on what was broken, how it was resolved, and how the system behaves now. 6. **Link Formatting**: @@ -236,9 +237,9 @@ def render( ### [Feature Title] **What's New**: [Clear description of what partners or operators can now do] **Why it Matters**: [Business benefit and technical impact] -**Capabilities & Changes**: -- [Capability 1] ([repo#PR](URL)) -- [Capability 2] ([repo#PR](URL)) +**Capabilities & Use Cases Enabled**: +- [Actionable Use Case / Input Capability 1] ([repo#PR](URL)) +- [Actionable Use Case / Input Capability 2] ([repo#PR](URL)) --- From cf1cf25c4ac88181fb079351054a91256e2f5a59 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:42:01 -0700 Subject: [PATCH 25/70] fix(deploy): update pr_contributions example in FeatureExtractor JSON schema to use actionable use cases --- deploy/generate_release_notes/feature_extractor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index b14048bc..e02b0fa2 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -180,8 +180,8 @@ def extract_features( "target_components": ["dcp", "services", "preprocessing", "dataflow_worker", "ingestion_helper", "postprocessing"], "included_prs": ["datacommons#188", "datacommons#189"], "pr_contributions": {{ - "datacommons#188": "Removed premature success status set at end of dataflow stage", - "datacommons#189": "Added max_workers Terraform variable for Dataflow auto-scaling" + "agent-toolkit#211": "Query bilateral trade and migration relationships between multiple entities", + "datacommons#189": "Configure max_workers in Terraform to scale Dataflow workers automatically for large imports" }}, "is_dcp_relevant": true, "breaking_changes": "Optional string describing breaking change if any, else null" From 6f6484a674c5482a18bdd857a30788a6213f2f22 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:44:26 -0700 Subject: [PATCH 26/70] feat(deploy): apply user's clean, structured FeatureExtractor prompt to feature_extractor.py --- .../feature_extractor.py | 94 +++++++++---------- 1 file changed, 43 insertions(+), 51 deletions(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index e02b0fa2..4e6b39c3 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -109,82 +109,74 @@ def extract_features( f"{instructions_text}\n" ) - prompt = f"""You are an expert Technical Release Manager for Data Commons Platform (DCP) drafting official release notes for release {manifest.new_version} (previous version: {manifest.previous_version}). - -### Context & Domain Knowledge — What is Data Commons Platform (DCP)? -Data Commons Platform (DCP) is a self-hosted, Cloud Spanner-backed deployment of Data Commons. It replaces legacy Bigtable with Cloud Spanner graph tables and vector embeddings, featuring custom data ingestion pipelines, specialized serving APIs, and deployment automation across 6 key repositories: - -1. **`datacommonsorg/datacommons` (Monorepo)**: - - Contains DCP Terraform modules (`infra/dcp/`, `infra/modules/`), CLI tools (`packages/datacommons-cli/`), and Admin Portal (`packages/datacommons-admin/`). -2. **`datacommonsorg/website`**: - - Web application serving UI and APIs (`server/`, `static/`, `build/cdc_services/`, `build/cdc_data/`). -3. **`datacommonsorg/mixer`**: - - Core Spanner gRPC graph and StatVar serving engine (`internal/server/`, `proto/`). -4. **`datacommonsorg/import`**: - - Data processing pipelines: Simple importer (`simple/`), Dataflow Java worker (`pipeline/ingestion/`), Ingestion Helper (`pipeline/workflow/ingestion-helper/`), Aggregation Helper (`pipeline/workflow/aggregation-helper/`). -5. **`datacommonsorg/agent-toolkit`**: - - Datacommons Model Context Protocol (MCP) server and tools (`src/datacommons_mcp/`). -6. **`datacommonsorg/datacommons-data`**: - - Data preprocessor container image built from `import/simple/` and `website/build/cdc_data/`. + prompt = f"""You are an expert Technical Release Manager for the Data Commons Platform (DCP). Your task is to analyze raw PR metadata and draft structured, user-centric release notes for version `{manifest.new_version}` (previous version: `{manifest.previous_version}`). --- -### Classification Rules & SOP Categories: -Categorize EVERY feature into EXACTLY ONE of these 4 standard SOP categories based on its files and description: - -1. **"Spanner Graph & APIs"**: - - Features touching SDMX 3.0 REST endpoints, `/v2/observation` StatVar data retrieval, Spanner gRPC graph serving, `proto/` definitions, or `agent-toolkit` MCP server/tools. -2. **"Ingestion & Safety"**: - - Features touching Dataflow Java worker (`pipeline/ingestion/`), `ingestion-helper`, `aggregation-helper`, Spanner table loading, timestamp bounds, data validation, or health probes. -3. **"Search & Website"**: - - Features touching Spanner vector embeddings (`NodeEmbedding`), private instance `detect-and-fulfill`, Nginx/Envoy, Website UI, or Admin Portal UI. -4. **"Infra & Tooling"**: - - Features touching `infra/dcp/` (Terraform), `datacommons-cli`/`admin` PyPI packages, monorepo root configs, or Cloud Build release pipelines. +### 1. CONTEXT & DOMAIN KNOWLEDGE +Data Commons Platform (DCP) is a self-hosted, Cloud Spanner-backed deployment of Data Commons. It replaces legacy Bigtable with Cloud Spanner graph tables and vector embeddings. It features custom data ingestion pipelines, specialized serving APIs, and deployment automation across 6 repositories: +1. `datacommonsorg/datacommons` (Monorepo): Terraform modules (`infra/dcp/`, `infra/modules/`), CLI tools (`packages/datacommons-cli/`), and Admin Portal (`packages/datacommons-admin/`). +2. `datacommonsorg/website`: Web application serving UI and APIs (`server/`, `static/`, `build/cdc_services/`, `build/cdc_data/`). +3. `datacommonsorg/mixer`: Core Spanner gRPC graph and StatVar serving engine (`internal/server/`, `proto/`). +4. `datacommonsorg/import`: Data processing pipelines (`simple/`, `pipeline/ingestion/` Dataflow Java worker, `pipeline/workflow/ingestion-helper/`, `pipeline/workflow/aggregation-helper/`). +5. `datacommonsorg/agent-toolkit`: Datacommons Model Context Protocol (MCP) server and tools (`src/datacommons_mcp/`). +6. `datacommonsorg/datacommons-data`: Data preprocessor container image built from `import/simple/` and `website/build/cdc_data/`. --- -### Task Instructions: -1. **Filter Out & Ignore Internal Dev & Testing PRs (STRICT)**: - - Completely IGNORE automated bot PRs (e.g. dependabot, renovate, 'chore: bump version to 1.1.1'). - - Completely IGNORE all test-only PRs: integration test setups, Spanner Omni test conversions, CI sandbox workflows, local test harnesses, hermetic test refactors, and test-only sample data updates (e.g. OECD wage sample data). DO NOT output any FeatureUpdate for test-only PRs! - - Completely IGNORE trivial formatting, typo fixes, or non-informative refactors with zero user impact. +### 2. CLASSIFICATION RULES (SOP CATEGORIES) +Categorize EVERY valid feature into EXACTLY ONE of these 4 categories based on its files and description: +* **"Spanner Graph & APIs"**: Features touching SDMX 3.0 REST endpoints, `/v2/observation` StatVar data retrieval, Spanner gRPC graph serving, `proto/` definitions, or `agent-toolkit` MCP server/tools. +* **"Ingestion & Safety"**: Features touching Dataflow Java worker (`pipeline/ingestion/`), `ingestion-helper`, `aggregation-helper`, Spanner table loading, timestamp bounds, data validation, or health probes. +* **"Search & Website"**: Features touching Spanner vector embeddings (`NodeEmbedding`), private instance `detect-and-fulfill`, Nginx/Envoy, Website UI, or Admin Portal UI. +* **"Infra & Tooling"**: Features touching `infra/dcp/` (Terraform), `datacommons-cli`/`admin` PyPI packages, monorepo root configs, or Cloud Build release pipelines. -2. **User Persona Focus (Building ON TOP OF Platform)**: - - Write for people **building ON TOP OF the platform** (data engineers, API consumers, instance operators). - - Focus on **Ingestion Inputs & Pipelines** (CSV/SDMX inputs, import workflows, validation rules) and **APIs & Tooling** (REST APIs, SDMX 3.0 endpoints, `/v2/observation`, MCP tools, Web UI, Admin CLI). - - **De-emphasize Database Layer Details**: Minimize mentions of Spanner database internals (e.g. Spanner graph schema, KeyValueStore cutover). Focus instead on the user-facing API or Ingestion behavior change. +--- -3. **Feature Grouping & Deduplication**: - - Combine related PRs (e.g., an initial feature PR + follow-up bug fixes + test PRs) into a SINGLE cohesive `FeatureUpdate`. - - List all included PR qualified IDs in `included_prs` (e.g. `["datacommons#188", "datacommons#189"]`). +### 3. STRICT FILTERING & DEDUPLICATION RULES +* **EXCLUDE Internal Dev & Testing PRs**: + * Ignore automated bot PRs (e.g., Dependabot, Renovate, "chore: bump version"). + * Ignore all test-only PRs (e.g., integration test setups, Spanner Omni test conversions, CI sandbox workflows, local test harnesses, hermetic test refactors, and test-only sample data updates like OECD wage sample data). + * Ignore formatting, typos, or non-informative refactors with zero user impact. +* **Deduplicate & Group Related PRs**: + * Combine related PRs (e.g., an initial feature PR + follow-up bug fixes + post-feature adjustments) into a SINGLE cohesive `FeatureUpdate`. + * If PRs conflict or supersede each other, describe ONLY the final chronological state at `{manifest.new_version}`. -4. **Supersede Resolution & Chronology**: - - Use `merged_at` timestamps to understand commit order. - - If a PR was superseded or modified by a later PR in this release, describe only the FINAL state at {manifest.new_version}. +--- -5. **Actionable Per-PR Capability & Use Case Summaries**: - - For EVERY PR listed in `included_prs`, provide a specific 1-2 sentence summary under `pr_contributions` describing an **explicit thing the user can DO or input format supported** because of this PR (e.g., `{{"agent-toolkit#211": "Query bilateral trade and migration relationships between multiple entities", "datacommons#189": "Configure max_workers in Terraform to scale Dataflow workers automatically for large imports"}}`). DO NOT list internal code refactors! +### 4. WRITING STYLE & PERSONA FOCUS +* **Target Audience**: Write for platform users, data engineers, API consumers, and instance operators building ON TOP of DCP. +* **User Capability Focus**: Frame the "description" and "pr_contributions" around what the user can *actually do* or what *input formats* are now supported (e.g., "Configure max_workers in Terraform to scale Dataflow workers automatically" instead of "Added Terraform max_workers variable"). +* **De-emphasize DB Internals**: Minimize mentions of Spanner database internals (e.g., Spanner graph schema, KeyValueStore cutover). Focus instead on the user-facing API or Ingestion behavior change. -{instructions_context} +--- -### Raw Merged PRs: +### 5. INPUT DATA +* **Instructions Context**: +{instructions_context} +* **Raw Merged PRs**: {json.dumps(detailed_prs, indent=2)} -Respond ONLY with a JSON array of FeatureUpdate objects with the following schema: +--- + +### 6. OUTPUT FORMAT +Respond ONLY with a valid JSON array of `FeatureUpdate` objects conforming strictly to the schema below. +Do not include markdown code block formatting (such as ```json) or any conversational text before or after the JSON. Output raw JSON only. + [ {{ "id": "short_unique_snake_case_id", "title": "Clear Technical Feature Title", - "description": "2-3 sentence technical description of the feature, changes, and impact.", + "description": "2-3 sentence technical description of the feature, explaining the change and its user-facing impact.", "category": "Spanner Graph & APIs | Ingestion & Safety | Search & Website | Infra & Tooling", "target_components": ["dcp", "services", "preprocessing", "dataflow_worker", "ingestion_helper", "postprocessing"], - "included_prs": ["datacommons#188", "datacommons#189"], + "included_prs": ["agent-toolkit#211", "datacommons#189"], "pr_contributions": {{ "agent-toolkit#211": "Query bilateral trade and migration relationships between multiple entities", "datacommons#189": "Configure max_workers in Terraform to scale Dataflow workers automatically for large imports" }}, "is_dcp_relevant": true, - "breaking_changes": "Optional string describing breaking change if any, else null" + "breaking_changes": "Detailed description of the breaking change if any, otherwise null" }} ] """ From 3408128911dff28e5d1229fba6b0f839dc477935 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:46:15 -0700 Subject: [PATCH 27/70] feat(deploy): apply user's master Release Notes Writer prompt with DO vs DON'T table to release_notes_writer.py --- .../release_notes_writer.py | 98 +++++++++++-------- 1 file changed, 56 insertions(+), 42 deletions(-) diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index 8630d356..88abccea 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -180,39 +180,55 @@ def render( f"{instructions_text}\n" ) - prompt = f"""You are an expert Technical Release Manager and Product Documentation Specialist for the Data Commons Platform (DCP). -Your task is to write publication-ready, partner-facing release notes for Data Commons Platform release {manifest.new_version} ({release_date}). - -### Core Writing Guidelines & Tone: -1. **Tone & Style**: Write in a clear, positive, partner-facing tone using plain language. Use second person ("You can now...") and active voice for new features and improvements. Use past tense for bug fixes ("Fixed...", "Resolved..."). -2. **Target Audience (Building ON TOP OF Platform)**: - - Write for external developers, data engineers, and instance operators building ON TOP OF Data Commons Platform (NOT internal platform maintainers). - - Primary Focus: **Ingestion Inputs & Pipelines** (CSV/SDMX inputs, data loading, workflow parameters) and **APIs & Tooling** (REST APIs, SDMX 3.0 endpoints, `/v2/observation`, MCP tools, Web UI, Admin CLI). -3. **De-emphasize Database Layer**: - - DO NOT focus on Spanner database layer mechanics (e.g. Spanner graph schema, KeyValueStore cutovers, Spanner table internals). Frame changes around how they affect API response speed, data availability, or ingestion inputs! -4. **Strict Exclusions**: - - DO NOT include internal integration test suites, Spanner Omni test setups, CI sandbox workflows, or developer-only test sample data updates. - - DO NOT include internal code refactors or unused example file cleanups. -5. **Level of Detail & Use Case Focus**: - - For major features: Provide an engaging "What's New", "Why it Matters" (business/technical benefit), and a bulleted list under "**Capabilities & Use Cases Enabled**". - - **DO NOT write a laundry list of code changes or PR descriptions!** Every bullet under Capabilities MUST describe an explicit thing the user can DO (e.g., 'Query bilateral trade flows between two countries', 'Import SDMX 3.0 CSV files directly', 'Run targeted single-entity vs child-place research playbooks'). Focus on supported input types, query capabilities, and real use cases! - - For single-PR features: Provide rich, self-contained descriptions so partners do not need to look up code diffs. - - For bug fixes: Focus on what was broken, how it was resolved, and how the system behaves now. -6. **Link Formatting**: - - Every PR reference MUST be formatted as a clickable Markdown link: `[#]()` (e.g. `[datacommons#188](https://github.com/datacommonsorg/datacommons/pull/188)`). - - DO NOT put backticks around or inside the link text (e.g. write `[datacommons#188](URL)`, NEVER `[`datacommons#188`](URL)` or `` `[datacommons#188](URL)` ``). -7. **Strict Constraints**: - - DO NOT use any emojis anywhere in the document. - - DO NOT include a component version table or git commit/SHA table. - - DO NOT include release range commit text (e.g., "Release range: v1.1.0 to v1.1.1"). - - Place the 1-2 sentence Executive Summary directly beneath the main title (`# Data Commons Platform Release {manifest.new_version} ({release_date})`). - - Output ONLY clean, valid GitHub Flavored Markdown (GFM). + prompt = f"""You are an expert Technical Release Manager and Product Documentation Specialist for the Data Commons Platform (DCP). + +Your objective is to generate publication-ready, partner-facing release notes for DCP version `{manifest.new_version}` ({release_date}) using GFM (GitHub Flavored Markdown). + +--- + +### 1. CORE WRITING STYLE & TONE +* **Perspective & Tone**: Use a clear, authoritative, yet welcoming and positive tone. Write in the active voice and use the second person ("You can now...") for new features or configuration updates. Use the past tense ("Fixed...", "Resolved...") for bug fixes. +* **Audience Focus**: Write specifically for external developers, data engineers, and instance operators building ON TOP OF the platform. +* **The "So What?" Rule**: Do not just list code changes. Frame every update around user capability (e.g., *what* can the developer do now, *which* inputs are accepted, or *how* does this affect query performance/scalability?). +* **De-emphasize DB Internals**: Do not write about Spanner database mechanics (e.g., "Spanner graph schema modifications," "KeyValueStore cutovers," or Spanner internal table indexing). Instead, frame these improvements around API response speed, easier configuration, or expanded data ingestion inputs. + +--- + +### 2. STRICT CONSTRAINTS (ZERO-TOLERANCE RULES) +* **NO Emojis**: Do not use emojis anywhere in the document. +* **NO Version/Commit Tables**: Do not include a component version table, git commit hashes, or SHA tables. +* **NO Commit Range Text**: Do not include text like "Release range: v1.1.0 to v1.1.1". +* **NO Code-Fenced Links**: Every PR reference MUST be a clean, clickable GFM link. Do not wrap backticks around or inside link text. + * **CORRECT**: `[website#123](https://github.com/...)` + * **INCORRECT**: `[`website#123`](https://github.com/...)` or `[`website#123` (https://github.com/...)]` + +--- + +### 3. INPUT MAPPING RULES +You will process two payload inputs. Map them to the final release note sections as follows: + +1. **`features_payload`**: + * Major, highly impactful items must be grouped as detailed features under **Key Feature Updates**. + * Minor enhancements, optimizations, or configuration instructions must be formatted as concise bullet points under **Improvements & Configuration Updates**. +2. **`bug_fixes_payload`**: + * Substantive bug fixes that address user-facing errors, data inaccuracies, or platform operator crashes must be mapped to **Bug Fixes**. + * *STRICT EXCLUSION*: Completely ignore internal development chores, test refactors, CI sandbox workflows, local test setups, or unused sample data removals. --- -### Input Release Data: +### 4. DO vs. DON'T CONTENT SAMPLES -#### Synthesized Features & Updates: +| Section | ❌ DO NOT Write (Internal Developer Focus) | DO Write (Partner & Operator Focus) | +| :--- | :--- | :--- | +| **Key Features** | "Merged PR to implement SDMX 3.0 CSV parser in import repo." | **Import SDMX 3.0 CSV files directly** to ingest standard-compliant macroeconomic datasets into your private instance with zero manual preprocessing. | +| **Improvements** | "Added Terraform variable max_workers." | **Scalable Dataflow Import Pipelines**: Configure `max_workers` in your Terraform configurations to scale compute resources automatically during large-scale imports. | +| **Bug Fixes** | "Fixed NullPointerException in observation API when entity is empty." | **Observation Serving**: Resolved a crash in the `/v2/observation` endpoint when querying empty entities; the API now gracefully returns an empty payload with a 200 OK. | + +--- + +### 5. INPUT DATA + +#### Major Features & Updates: {json.dumps(features_payload, indent=2)} #### Bug Fixes & Refactors: @@ -222,40 +238,38 @@ def render( --- -### Required Output Markdown Structure: +### 6. REQUIRED OUTPUT STRUCTURE +Generate GFM matching the exact structure below. Do not add any greeting, intro, or concluding conversational text outside this structure. # Data Commons Platform Release {manifest.new_version} ({release_date}) -*(1-2 sentences highlighting the most important capabilities, improvements, and fixes in this release for partners and platform operators.)* +[Provide a high-impact, 1-2 sentence Executive Summary highlighting the most important capabilities, performance boosts, and critical fixes introduced in this release for partners and platform operators.] --- ## Key Feature Updates -*(Group major feature updates here. For each feature, use the following structure:)* - ### [Feature Title] + **What's New**: [Clear description of what partners or operators can now do] -**Why it Matters**: [Business benefit and technical impact] + +**Why it Matters**: [Business benefit, technical impact, or performance advantage] + **Capabilities & Use Cases Enabled**: -- [Actionable Use Case / Input Capability 1] ([repo#PR](URL)) -- [Actionable Use Case / Input Capability 2] ([repo#PR](URL)) +- [Actionable Use Case / Input Capability 1] ([repo_short#PR](URL)) +- [Actionable Use Case / Input Capability 2] ([repo_short#PR](URL)) --- ## Improvements & Configuration Updates -*(List enhancements, performance updates, or required Terraform/Admin Panel configuration changes as concise bullet points:)* - -- **[Improvement Title]**: [Summary of update, configuration instructions if required, and benefit] ([repo#PR](URL)) +- **[Improvement Title]**: [Summary of update, step-by-step configuration instructions if required, and direct benefit] ([repo_short#PR](URL)) --- ## Bug Fixes -*(List ONLY substantive bug fixes that resolve user-facing errors, data issues, or platform operator failures in past tense. DO NOT include internal dev cleanups, test refactors, or unused example file removals:)* - -- **[Component / Scope]**: [Description of what was fixed and how the system behaves now] ([repo#PR](URL)) +- **[Component / Scope]**: [Description of what was broken, how it was resolved, and how the system behaves now] ([repo_short#PR](URL)) """ try: From 2a5258d9e555cf4c2b5985084d251d08e1aa5187 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:47:21 -0700 Subject: [PATCH 28/70] feat(deploy): add Anti-AI-Fluff word allowlist and strict word count budgets to release notes prompts --- deploy/generate_release_notes/feature_extractor.py | 4 +++- deploy/generate_release_notes/release_notes_writer.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index 4e6b39c3..594a355a 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -144,8 +144,10 @@ def extract_features( --- -### 4. WRITING STYLE & PERSONA FOCUS +### 4. WRITING STYLE & PERSONA FOCUS (CONCISE & ANTI-FLUFF) * **Target Audience**: Write for platform users, data engineers, API consumers, and instance operators building ON TOP of DCP. +* **BANNED AI FLUFF WORDS (STRICT)**: DO NOT use AI cliché words: `seamlessly`, `empower`, `leveraging`, `robust`, `overhaul`, `delivers a major`, `comprehensive`, `fosters`, `game-changing`, `cutting-edge`, `paradigm`. Write simple, direct sentences instead! +* **Concise Sentence Budget**: Keep `description` to 1-2 punchy sentences (max 25 words). Keep `pr_contributions` summaries to 1 short sentence (12-15 words max per PR). * **User Capability Focus**: Frame the "description" and "pr_contributions" around what the user can *actually do* or what *input formats* are now supported (e.g., "Configure max_workers in Terraform to scale Dataflow workers automatically" instead of "Added Terraform max_workers variable"). * **De-emphasize DB Internals**: Minimize mentions of Spanner database internals (e.g., Spanner graph schema, KeyValueStore cutover). Focus instead on the user-facing API or Ingestion behavior change. diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index 88abccea..240e8843 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -186,9 +186,15 @@ def render( --- -### 1. CORE WRITING STYLE & TONE -* **Perspective & Tone**: Use a clear, authoritative, yet welcoming and positive tone. Write in the active voice and use the second person ("You can now...") for new features or configuration updates. Use the past tense ("Fixed...", "Resolved...") for bug fixes. +### 1. CORE WRITING STYLE & TONE (CONCISE & ANTI-FLUFF) +* **Perspective & Tone**: Write like a senior Google engineer writing a concise technical changelog — direct, factual, punchy, and zero fluff. Use active voice ("You can now...") for features, past tense ("Fixed...") for bugs. * **Audience Focus**: Write specifically for external developers, data engineers, and instance operators building ON TOP OF the platform. +* **BANNED AI FLUFF WORDS (STRICT)**: DO NOT use AI cliché words: `seamlessly`, `empower`, `leveraging`, `robust`, `overhaul`, `delivers a major`, `comprehensive`, `fosters`, `game-changing`, `cutting-edge`, `paradigm`. Write simple, direct sentences instead! +* **STRICT WORD COUNT BUDGETS**: + * **Executive Summary**: Maximum 25 words (1 single, punchy sentence). + * **What's New**: 15-20 words max (1 direct sentence). + * **Why it Matters**: 15-20 words max (1 direct sentence). + * **Capabilities & Changes Bullets**: 12-15 words max per bullet. * **The "So What?" Rule**: Do not just list code changes. Frame every update around user capability (e.g., *what* can the developer do now, *which* inputs are accepted, or *how* does this affect query performance/scalability?). * **De-emphasize DB Internals**: Do not write about Spanner database mechanics (e.g., "Spanner graph schema modifications," "KeyValueStore cutovers," or Spanner internal table indexing). Instead, frame these improvements around API response speed, easier configuration, or expanded data ingestion inputs. From 223f9d2a42e787db91bad06f10cb74a040045305 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 19:51:46 -0700 Subject: [PATCH 29/70] fix(deploy): add internal iteration bug filtering rule and expand SDMX ESPv2 mixer context --- deploy/generate_release_notes/feature_extractor.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index 594a355a..46f3ed67 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -117,7 +117,7 @@ def extract_features( Data Commons Platform (DCP) is a self-hosted, Cloud Spanner-backed deployment of Data Commons. It replaces legacy Bigtable with Cloud Spanner graph tables and vector embeddings. It features custom data ingestion pipelines, specialized serving APIs, and deployment automation across 6 repositories: 1. `datacommonsorg/datacommons` (Monorepo): Terraform modules (`infra/dcp/`, `infra/modules/`), CLI tools (`packages/datacommons-cli/`), and Admin Portal (`packages/datacommons-admin/`). 2. `datacommonsorg/website`: Web application serving UI and APIs (`server/`, `static/`, `build/cdc_services/`, `build/cdc_data/`). -3. `datacommonsorg/mixer`: Core Spanner gRPC graph and StatVar serving engine (`internal/server/`, `proto/`). +3. `datacommonsorg/mixer`: Core Spanner gRPC graph, StatVar serving engine, and ESPv2 gateway (`internal/server/`, `proto/`, `deploy/helm_charts/`). 4. `datacommonsorg/import`: Data processing pipelines (`simple/`, `pipeline/ingestion/` Dataflow Java worker, `pipeline/workflow/ingestion-helper/`, `pipeline/workflow/aggregation-helper/`). 5. `datacommonsorg/agent-toolkit`: Datacommons Model Context Protocol (MCP) server and tools (`src/datacommons_mcp/`). 6. `datacommonsorg/datacommons-data`: Data preprocessor container image built from `import/simple/` and `website/build/cdc_data/`. @@ -126,7 +126,7 @@ def extract_features( ### 2. CLASSIFICATION RULES (SOP CATEGORIES) Categorize EVERY valid feature into EXACTLY ONE of these 4 categories based on its files and description: -* **"Spanner Graph & APIs"**: Features touching SDMX 3.0 REST endpoints, `/v2/observation` StatVar data retrieval, Spanner gRPC graph serving, `proto/` definitions, or `agent-toolkit` MCP server/tools. +* **"Spanner Graph & APIs"**: Features touching SDMX 3.0 REST endpoints, ESPv2 query parameter handling, `/v2/observation` StatVar data retrieval, Spanner gRPC graph serving, `proto/` definitions, or `agent-toolkit` MCP server/tools. * **"Ingestion & Safety"**: Features touching Dataflow Java worker (`pipeline/ingestion/`), `ingestion-helper`, `aggregation-helper`, Spanner table loading, timestamp bounds, data validation, or health probes. * **"Search & Website"**: Features touching Spanner vector embeddings (`NodeEmbedding`), private instance `detect-and-fulfill`, Nginx/Envoy, Website UI, or Admin Portal UI. * **"Infra & Tooling"**: Features touching `infra/dcp/` (Terraform), `datacommons-cli`/`admin` PyPI packages, monorepo root configs, or Cloud Build release pipelines. @@ -138,6 +138,10 @@ def extract_features( * Ignore automated bot PRs (e.g., Dependabot, Renovate, "chore: bump version"). * Ignore all test-only PRs (e.g., integration test setups, Spanner Omni test conversions, CI sandbox workflows, local test harnesses, hermetic test refactors, and test-only sample data updates like OECD wage sample data). * Ignore formatting, typos, or non-informative refactors with zero user impact. +* **EXCLUDE Internal Iteration Bug Fixes (Release Window Regressions)**: + * If a bug fix PR addresses a bug or regression introduced *within this same release window* (i.e. introduced after `{manifest.previous_version}` and fixed before `{manifest.new_version}`), DO NOT list it as a standalone Bug Fix! + * Fold it into the parent `FeatureUpdate` as part of that feature's development, or drop it if it was just an internal dev fix. + * ONLY list bugs under 'Bug Fixes' if the bug was present in `{manifest.previous_version}` or an earlier published release! * **Deduplicate & Group Related PRs**: * Combine related PRs (e.g., an initial feature PR + follow-up bug fixes + post-feature adjustments) into a SINGLE cohesive `FeatureUpdate`. * If PRs conflict or supersede each other, describe ONLY the final chronological state at `{manifest.new_version}`. From 27cb6f2b94f1214b234df3d1232d4311982fba04 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 20:01:24 -0700 Subject: [PATCH 30/70] feat(deploy): implement deterministic Python-level detect_internal_regressions helper in FeatureExtractor --- .../feature_extractor.py | 72 +++++++++++++++---- 1 file changed, 59 insertions(+), 13 deletions(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index 46f3ed67..9fca6c2c 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -40,6 +40,48 @@ VALID_SOP_CATEGORIES = {cat.value for cat in SOPCategory} +def detect_internal_regressions(prs: List[PullRequest]) -> Dict[str, str]: + """Deterministically identifies PRs that fix regressions introduced by earlier PRs in the same release window. + + Returns a dict mapping pr.qualified_id -> parent_feature_pr.qualified_id. + """ + sorted_prs = sorted(prs, key=lambda p: p.merged_at or "") + file_to_prs: Dict[str, List[PullRequest]] = {} + regression_map: Dict[str, str] = {} + + for pr in sorted_prs: + title_lower = pr.title.lower() + is_fix = any( + w in title_lower + for w in ["fix", "bug", "resolve", "patch", "repair", "correct"] + ) + + if is_fix and pr.files_changed: + matching_earlier_prs = [] + for f in pr.files_changed: + if f in file_to_prs: + for prev_pr in file_to_prs[f]: + if ( + prev_pr.number != pr.number + and prev_pr.repo_name == pr.repo_name + ): + matching_earlier_prs.append(prev_pr) + + if matching_earlier_prs: + parent_pr = matching_earlier_prs[-1] + regression_map[pr.qualified_id] = parent_pr.qualified_id + logger.info( + f"Deterministically detected internal regression: {pr.qualified_id} fixes intermediate PR {parent_pr.qualified_id}" + ) + + for f in pr.files_changed: + if f not in file_to_prs: + file_to_prs[f] = [] + file_to_prs[f].append(pr) + + return regression_map + + class FeatureExtractor: """Single-stage Gemini LLM Pipeline for filtering, classifying, and synthesizing DCP release features.""" @@ -76,6 +118,9 @@ def extract_features( f"Step 2: Extracting features from {len(manifest.all_pull_requests)} PRs using {self.model_name}..." ) + # Pre-process PRs to deterministically detect internal regressions within this release window + regression_map = detect_internal_regressions(manifest.all_pull_requests) + # Build detailed PR context for model with merged_at timestamps and qualified IDs detailed_prs = [] for pr in manifest.all_pull_requests: @@ -86,19 +131,20 @@ def extract_features( bc_start = body_text.find("BREAKING CHANGE") body_snippet += "\n...\n" + body_text[bc_start : bc_start + 300] - detailed_prs.append( - { - "id": pr.qualified_id, - "title": pr.title, - "author": pr.author, - "repo": pr.repo_name, - "url": pr.url, - "merged_at": pr.merged_at, - "target_components": pr.target_components, - "files_changed": pr.files_changed[:10], - "body_summary": body_snippet, - } - ) + pr_dict = { + "id": pr.qualified_id, + "title": pr.title, + "author": pr.author, + "repo": pr.repo_name, + "url": pr.url, + "merged_at": pr.merged_at, + "target_components": pr.target_components, + "files_changed": pr.files_changed[:10], + "body_summary": body_snippet, + "is_internal_regression": pr.qualified_id in regression_map, + "fixes_intermediate_pr": regression_map.get(pr.qualified_id), + } + detailed_prs.append(pr_dict) instructions_context = "" if additional_instructions or manifest.additional_instructions: From 683c5dbab77c8cdc85e384266ba20103886cfcc2 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 20:05:14 -0700 Subject: [PATCH 31/70] fix(deploy): remove invalid base:main search qualifier in PRExtractor allowing all 36 Mixer PRs including 13 SDMX PRs to be extracted --- deploy/generate_release_notes/pr_extractor.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/deploy/generate_release_notes/pr_extractor.py b/deploy/generate_release_notes/pr_extractor.py index 2667ea11..e6030d88 100644 --- a/deploy/generate_release_notes/pr_extractor.py +++ b/deploy/generate_release_notes/pr_extractor.py @@ -202,15 +202,20 @@ def fetch_prs_for_date_range( t_prev = ( prev_timestamp.split(".")[0].replace(" ", "T") if prev_timestamp - else "" + else "2026-01-01T00:00:00Z" ) t_new = ( new_timestamp.split(".")[0].replace(" ", "T") if new_timestamp - else "" + else datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") ) - search_query = f"merged:{t_prev}..{t_new} base:main" + if not t_prev.endswith("Z") and "+" not in t_prev and "-" not in t_prev[10:]: + t_prev += "Z" + if not t_new.endswith("Z") and "+" not in t_new and "-" not in t_new[10:]: + t_new += "Z" + + search_query = f"merged:{t_prev}..{t_new}" cmd = [ "gh", "pr", From ca81ccef68dda3a0efd0f336f5cdbdd37cbd7ed3 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 20:06:34 -0700 Subject: [PATCH 32/70] fix(deploy): enforce rendering ALL features from features_payload without dropping SDMX in ReleaseNotesWriter --- .../release_notes_writer.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index 240e8843..e29fdbff 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -210,15 +210,16 @@ def render( --- -### 3. INPUT MAPPING RULES +### 3. INPUT MAPPING RULES (RENDER ALL FEATURES) You will process two payload inputs. Map them to the final release note sections as follows: -1. **`features_payload`**: - * Major, highly impactful items must be grouped as detailed features under **Key Feature Updates**. - * Minor enhancements, optimizations, or configuration instructions must be formatted as concise bullet points under **Improvements & Configuration Updates**. -2. **`bug_fixes_payload`**: - * Substantive bug fixes that address user-facing errors, data inaccuracies, or platform operator crashes must be mapped to **Bug Fixes**. - * *STRICT EXCLUSION*: Completely ignore internal development chores, test refactors, CI sandbox workflows, local test setups, or unused sample data removals. +1. **`features_payload`**: + * **STRICT REQUIREMENT**: You MUST render EVERY item provided in `features_payload`. DO NOT drop or omit any feature! + * Major, highly impactful items (e.g. SDMX 3.0 REST Endpoints, Agent MCP Toolkit Overhaul, Modular Ingestion Workflows) MUST be rendered as detailed feature sections under **Key Feature Updates**. + * Minor enhancements, optimizations, or configuration instructions must be formatted as concise bullet points under **Improvements & Configuration Updates**. +2. **`bug_fixes_payload`**: + * Substantive bug fixes that address user-facing errors, data inaccuracies, or platform operator crashes must be mapped to **Bug Fixes**. + * *STRICT EXCLUSION*: Completely ignore internal development chores, test refactors, CI sandbox workflows, local test setups, or unused sample data removals. --- From 6b8d3e8e36d3d8d5e28701e651d13dac6b1608df Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 20:11:44 -0700 Subject: [PATCH 33/70] fix(deploy): add custom_only variable filtering example to DO vs DON'T table in ReleaseNotesWriter --- deploy/generate_release_notes/release_notes_writer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index e29fdbff..c5615d6b 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -229,6 +229,7 @@ def render( | :--- | :--- | :--- | | **Key Features** | "Merged PR to implement SDMX 3.0 CSV parser in import repo." | **Import SDMX 3.0 CSV files directly** to ingest standard-compliant macroeconomic datasets into your private instance with zero manual preprocessing. | | **Improvements** | "Added Terraform variable max_workers." | **Scalable Dataflow Import Pipelines**: Configure `max_workers` in your Terraform configurations to scale compute resources automatically during large-scale imports. | +| **Improvements** | "Propagated V2_RESOLVE_INDICATORS_TARGET to website." | **Filter Website Explore to Custom Variables**: Set `datacommons_services_website_search_scope` to `custom_only` in Terraform to restrict website search and explore results strictly to your instance's custom variables. | | **Bug Fixes** | "Fixed NullPointerException in observation API when entity is empty." | **Observation Serving**: Resolved a crash in the `/v2/observation` endpoint when querying empty entities; the API now gracefully returns an empty payload with a 200 OK. | --- From bc8bb23e8008e401f70a9124b6479a4e11335b94 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 20:13:05 -0700 Subject: [PATCH 34/70] feat(deploy): add generalized Enum & Configuration Value extraction rule to FeatureExtractor and ReleaseNotesWriter --- deploy/generate_release_notes/feature_extractor.py | 1 + deploy/generate_release_notes/release_notes_writer.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index 9fca6c2c..8a3afe99 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -199,6 +199,7 @@ def extract_features( * **BANNED AI FLUFF WORDS (STRICT)**: DO NOT use AI cliché words: `seamlessly`, `empower`, `leveraging`, `robust`, `overhaul`, `delivers a major`, `comprehensive`, `fosters`, `game-changing`, `cutting-edge`, `paradigm`. Write simple, direct sentences instead! * **Concise Sentence Budget**: Keep `description` to 1-2 punchy sentences (max 25 words). Keep `pr_contributions` summaries to 1 short sentence (12-15 words max per PR). * **User Capability Focus**: Frame the "description" and "pr_contributions" around what the user can *actually do* or what *input formats* are now supported (e.g., "Configure max_workers in Terraform to scale Dataflow workers automatically" instead of "Added Terraform max_workers variable"). +* **Extract Concrete Enums & Configuration Values (STRICT)**: Whenever an update introduces or modifies a configuration variable, CLI flag, environment variable, or Terraform setting, DO NOT summarize it generically. ALWAYS extract and list the specific valid values or enums (e.g., `custom_only`, `base_only`, `base_and_custom`, `--instance_name`, processing unit bounds) and explain the exact behavior or filtering capability each option enables for operators! * **De-emphasize DB Internals**: Minimize mentions of Spanner database internals (e.g., Spanner graph schema, KeyValueStore cutover). Focus instead on the user-facing API or Ingestion behavior change. --- diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index c5615d6b..07f74782 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -196,6 +196,7 @@ def render( * **Why it Matters**: 15-20 words max (1 direct sentence). * **Capabilities & Changes Bullets**: 12-15 words max per bullet. * **The "So What?" Rule**: Do not just list code changes. Frame every update around user capability (e.g., *what* can the developer do now, *which* inputs are accepted, or *how* does this affect query performance/scalability?). +* **Extract Concrete Enums & Configuration Values (STRICT)**: Whenever an update introduces or modifies a configuration variable, CLI flag, environment variable, or Terraform setting, DO NOT summarize it generically (e.g., "Configured search scope"). ALWAYS extract and list the specific valid values or enums (e.g., `custom_only`, `base_only`, `base_and_custom`, `--instance_name`, processing unit bounds) and explain the exact behavior or filtering capability each option enables for operators! * **De-emphasize DB Internals**: Do not write about Spanner database mechanics (e.g., "Spanner graph schema modifications," "KeyValueStore cutovers," or Spanner internal table indexing). Instead, frame these improvements around API response speed, easier configuration, or expanded data ingestion inputs. --- @@ -229,7 +230,6 @@ def render( | :--- | :--- | :--- | | **Key Features** | "Merged PR to implement SDMX 3.0 CSV parser in import repo." | **Import SDMX 3.0 CSV files directly** to ingest standard-compliant macroeconomic datasets into your private instance with zero manual preprocessing. | | **Improvements** | "Added Terraform variable max_workers." | **Scalable Dataflow Import Pipelines**: Configure `max_workers` in your Terraform configurations to scale compute resources automatically during large-scale imports. | -| **Improvements** | "Propagated V2_RESOLVE_INDICATORS_TARGET to website." | **Filter Website Explore to Custom Variables**: Set `datacommons_services_website_search_scope` to `custom_only` in Terraform to restrict website search and explore results strictly to your instance's custom variables. | | **Bug Fixes** | "Fixed NullPointerException in observation API when entity is empty." | **Observation Serving**: Resolved a crash in the `/v2/observation` endpoint when querying empty entities; the API now gracefully returns an empty payload with a 200 OK. | --- From da8f81a56a00dd7429e31e089aa7e8a6496079dd Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 20:15:30 -0700 Subject: [PATCH 35/70] docs(deploy): add comprehensive README.md for generate_release_notes tool --- deploy/generate_release_notes/README.md | 141 ++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 deploy/generate_release_notes/README.md diff --git a/deploy/generate_release_notes/README.md b/deploy/generate_release_notes/README.md new file mode 100644 index 00000000..085f4a33 --- /dev/null +++ b/deploy/generate_release_notes/README.md @@ -0,0 +1,141 @@ +# Data Commons Platform (DCP) Release Notes Generator + +An agentic, multi-repository tool for generating publication-ready, partner-facing release notes for the Data Commons Platform (DCP). + +The tool automatically extracts merged Pull Requests across all 6 core Data Commons repositories, classifies them according to standard SOP categories using Gemini 3.6 Flash, filters out internal test noise and regressions, and formats concise release notes tailored for developers and platform operators building on top of DCP. + +--- + +## Architecture Overview + +The tool operates as a structured 3-step pipeline: + +``` +┌─────────────────────────┐ ┌──────────────────────────────┐ ┌────────────────────────────┐ +│ Step 1: PR Extractor │ ───►│ Step 2: Feature Extractor │ ───►│ Step 3: Release Notes Writer│ +│ (gh CLI + gcloud tags) │ │ (Gemini 3.6 Flash LLM) │ │ (Gemini 3.6 Flash GFM) │ +└─────────────────────────┘ └──────────────────────────────┘ └────────────────────────────┘ +``` + +1. **Step 1: PR Extractor (`pr_extractor.py`)**: Resolves container image tags across Artifact Registry (`gcr.io/datcom-ci/datacommons-services`, `datacommons-data`, etc.) and Git tags. Queries GitHub CLI (`gh pr list`) in a single date-range search per repository to fetch all merged PRs between versions. +2. **Step 2: Feature Extractor (`feature_extractor.py`)**: Uses Gemini 3.6 Flash to filter out Dependabot PRs, test-only refactors, and intermediate release-window regressions (via deterministic file diff footprint matching). Synthesizes remaining PRs into structured `FeatureUpdate` objects categorized under: + - **Spanner Graph & APIs** (SDMX 3.0 REST endpoints, `/v2/observation`, Mixer gRPC, MCP tools) + - **Ingestion & Safety** (Dataflow workers, workflow orchestration, preprocessor, health probes) + - **Search & Website** (Vector embeddings, search scope targeting, Website UI) + - **Infra & Tooling** (Terraform modules, Admin CLI, monorepo versioning) +3. **Step 3: Release Notes Writer (`release_notes_writer.py`)**: Renders publication-ready GitHub Flavored Markdown (GFM) using the **"So What?" Rule**, strict **Anti-AI-Fluff Word Budgets**, side-by-side **DO vs. DON'T guidelines**, and clickable `[repo#PR](URL)` links. + +--- + +## Prerequisites + +Before running the tool, ensure you have the following installed and authenticated: + +1. **Python 3.11+** and [`uv`](https://github.com/astral-sh/uv) (or `pip`). +2. **GitHub CLI (`gh`)**: Must be installed and authenticated to read pull requests: + ```bash + gh auth status + # If not authenticated: + gh auth login + ``` +3. **Google Cloud SDK (`gcloud`)**: Must be authenticated to query Artifact Registry image tags: + ```bash + gcloud auth list + # If not authenticated: + gcloud auth login + gcloud auth application-default login + ``` +4. **Gemini API Key or GCP Credentials**: + ```bash + export GEMINI_API_KEY="your_gemini_api_key_here" + ``` + +--- + +## Typical Usage Commands + +### 1. Basic Production Release Notes Generation +Generate release notes between two published release tags (e.g. `v1.1.0` and `v1.1.1`): +```bash +uv run --group generate-release-notes python -m deploy.generate_release_notes \ + --prev v1.1.0 \ + --new v1.1.1 \ + --out ./RELEASE_NOTES_v1.1.1.md +``` + +### 2. Pre-Release / Staging Generation (Allow Missing Images) +Generate release notes for a staging release before container images have been tagged in Artifact Registry (extends search window to current time `NOW()`): +```bash +uv run --group generate-release-notes python -m deploy.generate_release_notes \ + --prev v1.1.0 \ + --new v1.1.1 \ + --allow-missing-images \ + --out ./RELEASE_NOTES_v1.1.1.md +``` + +### 3. Including High-Priority Release Highlights / Additional Instructions +Provide custom context or release highlights via a markdown file: +```bash +uv run --group generate-release-notes python -m deploy.generate_release_notes \ + --prev v1.1.0 \ + --new v1.1.1 \ + --additional-instructions ./release_highlights.md \ + --out ./RELEASE_NOTES_v1.1.1.md +``` + +### 4. Generating Release Notes with Full Audit Log Table +Include an append-only PR audit table at the bottom cross-referencing all 50+ processed PRs: +```bash +uv run --group generate-release-notes python -m deploy.generate_release_notes \ + --prev v1.1.0 \ + --new v1.1.1 \ + --include-audit-log \ + --out ./RELEASE_NOTES_v1.1.1.md +``` + +--- + +## Command-Line Options & Flags + +| Flag / Option | Type | Required | Description | +| :--- | :--- | :---: | :--- | +| `--prev` | `STRING` | **Yes** | Previous release version tag (e.g. `v1.1.0`). | +| `--new` | `STRING` | **Yes** | Target release version tag (e.g. `v1.1.1`). | +| `--out`, `-o` | `PATH` | No | Path to write output markdown file (default: `./RELEASE_NOTES_.md`). | +| `--allow-missing-images` | `BOOLEAN` | No | Bypass errors if container image tags are missing in Artifact Registry and extend date range to current time `NOW()`. | +| `--synthesis-model` | `STRING` | No | Gemini model to use for feature extraction & writing (default: `gemini-3.6-flash`). | +| `--additional-instructions` | `STRING/PATH` | No | Path to a markdown file or raw text containing high-priority user instructions or release highlights. | +| `--include-audit-log` | `BOOLEAN` | No | Append an audit log table mapping all raw PRs to their release classification status. | +| `--use-cache / --no-cache` | `BOOLEAN` | No | Enable or disable local disk caching for GitHub PR queries (default: `True`). | +| `--help` | `FLAG` | No | Display CLI help and exit. | + +--- + +## Repository Coverage + +The tool automatically tracks and correlates PRs across all 6 core Data Commons repositories: + +| Repository | Scope / Path Filter | Target Component | +| :--- | :--- | :--- | +| `datacommonsorg/datacommons` | All PRs (`infra/dcp/`, `packages/`) | DCP Monorepo & Infra (`dcp`) | +| `datacommonsorg/website` | All PRs (excluding `cdc_data/`) | Core Services (`services`) | +| `datacommonsorg/mixer` | All PRs (`internal/server/`, `proto/`, `deploy/`) | Core Services (`services`) | +| `datacommonsorg/agent-toolkit` | All PRs (`src/datacommons_mcp/`) | Core Services (`services`) | +| `datacommonsorg/import` | `simple/` | Data Preprocessor (`preprocessing`) | +| `datacommonsorg/import` | `pipeline/ingestion/` | Dataflow Worker (`dataflow_worker`) | +| `datacommonsorg/import` | `pipeline/workflow/ingestion-helper/` | Ingestion Helper (`ingestion_helper`) | +| `datacommonsorg/import` | `pipeline/workflow/aggregation-helper/` | Postprocessing Helper (`postprocessing`) | + +--- + +## Development & Testing + +Run the unit test suite (excludes live network/GitHub integration tests): +```bash +uv run pytest deploy/generate_release_notes/tests/ -m "not integration" -v +``` + +Run the full test suite (including live GitHub integration tests): +```bash +uv run pytest deploy/generate_release_notes/tests/ -v +``` From d735e20beccc0daa00cb368ca98227ab70c8bad2 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 20:16:24 -0700 Subject: [PATCH 36/70] docs(deploy): add Configuration & Component Registry (config.py) section to README.md --- deploy/generate_release_notes/README.md | 81 +++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/deploy/generate_release_notes/README.md b/deploy/generate_release_notes/README.md index 085f4a33..9dd99d92 100644 --- a/deploy/generate_release_notes/README.md +++ b/deploy/generate_release_notes/README.md @@ -111,6 +111,87 @@ uv run --group generate-release-notes python -m deploy.generate_release_notes \ --- +## Configuration & Component Registry (`config.py`) + +The tool's multi-repository mappings, image URIs, and source rules are centrally configured in [`config.py`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/config.py). + +### Core Data Structures in `config.py`: + +1. **`SourceRule`**: Defines a GitHub repository and an optional sub-directory `path_filter` for mapping monorepo or multi-repo PRs: + ```python + SourceRule(repo="datacommonsorg/import", path_filter="simple/") + ``` +2. **`ComponentConfig`**: Configures a tracked release component: + - `id`: Internal component key (e.g. `services`, `preprocessing`). + - `name`: Human-readable display name. + - `artifact_type`: Artifact category (`dcp_platform`, `docker_services`, `docker_data`, `dataflow_template`, `docker_ingestion_helper`, `docker_postprocessing`). + - `image_uri`: Primary container image URI in Google Artifact Registry or GCR (e.g. `gcr.io/datcom-ci/datacommons-services`). + - `default_tag_prefix`: Version tag prefix (e.g. `"v"` for `v1.1.1`). + - `sources`: List of contributing `SourceRule` objects. + +### Master `COMPONENTS` Registry: + +```python +COMPONENTS: Dict[str, ComponentConfig] = { + "dcp": ComponentConfig( + id="dcp", + name="DCP Monorepo & Infra (CLI, Admin, DB, Terraform)", + artifact_type="dcp_platform", + image_uri=None, + default_tag_prefix="v", + sources=[SourceRule(repo="datacommonsorg/datacommons")], + ), + "services": ComponentConfig( + id="services", + name="Core Services (Website, Mixer, MCP)", + artifact_type="docker_services", + image_uri="gcr.io/datcom-ci/datacommons-services", + default_tag_prefix="v", + sources=[ + SourceRule(repo="datacommonsorg/website"), + SourceRule(repo="datacommonsorg/mixer"), + SourceRule(repo="datacommonsorg/agent-toolkit"), + ], + ), + "preprocessing": ComponentConfig( + id="preprocessing", + name="Data Preprocessor (datacommons-data)", + artifact_type="docker_data", + image_uri="gcr.io/datcom-ci/datacommons-data", + default_tag_prefix="v", + sources=[SourceRule(repo="datacommonsorg/import", path_filter="simple/")], + ), + "dataflow_worker": ComponentConfig( + id="dataflow_worker", + name="Dataflow Ingestion Worker", + artifact_type="dataflow_template", + image_uri="us-docker.pkg.dev/datcom-ci/gcr.io/dataflow-templates/ingestion", + default_tag_prefix="v", + sources=[SourceRule(repo="datacommonsorg/import", path_filter="pipeline/ingestion/")], + ), + "ingestion_helper": ComponentConfig( + id="ingestion_helper", + name="Ingestion Helper Service", + artifact_type="docker_ingestion_helper", + image_uri="gcr.io/datcom-ci/datacommons-ingestion-helper", + default_tag_prefix="v", + sources=[SourceRule(repo="datacommonsorg/import", path_filter="pipeline/workflow/ingestion-helper/")], + ), + "postprocessing": ComponentConfig( + id="postprocessing", + name="Postprocessing Aggregation Helper Service", + artifact_type="docker_postprocessing", + image_uri="gcr.io/datcom-ci/datacommons-aggregation-helper", + default_tag_prefix="v", + sources=[SourceRule(repo="datacommonsorg/import", path_filter="pipeline/workflow/aggregation-helper/")], + ), +} +``` + +To add a new component or track an additional repository, simply define a new `ComponentConfig` in [`config.py`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/config.py). + +--- + ## Repository Coverage The tool automatically tracks and correlates PRs across all 6 core Data Commons repositories: From 85ec7e7e70b93eeaf6cb306587b79863feb95b38 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 20:18:59 -0700 Subject: [PATCH 37/70] fix(deploy): enforce strict DB Internal Name Ban and Grouped Bug Fixes (max 4-5 bullets) in FeatureExtractor & ReleaseNotesWriter --- deploy/generate_release_notes/feature_extractor.py | 2 +- deploy/generate_release_notes/release_notes_writer.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index 8a3afe99..32a0501a 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -200,7 +200,7 @@ def extract_features( * **Concise Sentence Budget**: Keep `description` to 1-2 punchy sentences (max 25 words). Keep `pr_contributions` summaries to 1 short sentence (12-15 words max per PR). * **User Capability Focus**: Frame the "description" and "pr_contributions" around what the user can *actually do* or what *input formats* are now supported (e.g., "Configure max_workers in Terraform to scale Dataflow workers automatically" instead of "Added Terraform max_workers variable"). * **Extract Concrete Enums & Configuration Values (STRICT)**: Whenever an update introduces or modifies a configuration variable, CLI flag, environment variable, or Terraform setting, DO NOT summarize it generically. ALWAYS extract and list the specific valid values or enums (e.g., `custom_only`, `base_only`, `base_and_custom`, `--instance_name`, processing unit bounds) and explain the exact behavior or filtering capability each option enables for operators! -* **De-emphasize DB Internals**: Minimize mentions of Spanner database internals (e.g., Spanner graph schema, KeyValueStore cutover). Focus instead on the user-facing API or Ingestion behavior change. +* **STRICT BAN ON DATABASE INTERNAL NAMES (ZERO TOLERANCE)**: NEVER output feature titles or section names containing internal database terms (e.g., "KeyValueStore", "Spanner Graph DDL", "Bigtable Migration", "Spanner Key Value Store", "Database Schema Modification"). Internal database storage details MUST NOT be exposed to platform users or partners! If a database change improves performance or latency, title it around user impact (e.g., "API Serving Latency & Latency Optimization") and describe the speedup without naming internal database tables or storage layers! --- diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index 07f74782..e5cdeb99 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -197,7 +197,7 @@ def render( * **Capabilities & Changes Bullets**: 12-15 words max per bullet. * **The "So What?" Rule**: Do not just list code changes. Frame every update around user capability (e.g., *what* can the developer do now, *which* inputs are accepted, or *how* does this affect query performance/scalability?). * **Extract Concrete Enums & Configuration Values (STRICT)**: Whenever an update introduces or modifies a configuration variable, CLI flag, environment variable, or Terraform setting, DO NOT summarize it generically (e.g., "Configured search scope"). ALWAYS extract and list the specific valid values or enums (e.g., `custom_only`, `base_only`, `base_and_custom`, `--instance_name`, processing unit bounds) and explain the exact behavior or filtering capability each option enables for operators! -* **De-emphasize DB Internals**: Do not write about Spanner database mechanics (e.g., "Spanner graph schema modifications," "KeyValueStore cutovers," or Spanner internal table indexing). Instead, frame these improvements around API response speed, easier configuration, or expanded data ingestion inputs. +* **STRICT BAN ON DATABASE INTERNAL NAMES (ZERO TOLERANCE)**: NEVER output feature titles or section names containing internal database terms (e.g., "KeyValueStore", "Spanner Graph DDL", "Bigtable Migration", "Spanner Key Value Store", "Database Schema Modification"). Internal database storage details MUST NOT be exposed to platform users or partners! If a database change improves performance or latency, title it around user impact (e.g., "API Serving Latency & Latency Optimization") and describe the speedup without naming internal database tables or storage layers! --- @@ -219,7 +219,9 @@ def render( * Major, highly impactful items (e.g. SDMX 3.0 REST Endpoints, Agent MCP Toolkit Overhaul, Modular Ingestion Workflows) MUST be rendered as detailed feature sections under **Key Feature Updates**. * Minor enhancements, optimizations, or configuration instructions must be formatted as concise bullet points under **Improvements & Configuration Updates**. 2. **`bug_fixes_payload`**: - * Substantive bug fixes that address user-facing errors, data inaccuracies, or platform operator crashes must be mapped to **Bug Fixes**. + * **HIGH-LEVEL GROUPING (MAX 4-5 BULLETS TOTAL - NO LAUNDRY LIST)**: DO NOT output a laundry list of dozens of individual PRs! + * Aggregate and group all raw bug fixes into 3 to 5 high-impact, functional bullet points (e.g., **Deployment & Infrastructure**, **Ingestion Pipeline Reliability**, **Serving API & Query Robustness**, **Web UI & Visualization**). + * Each grouped bullet must synthesize the common issue and fix in 1-2 concise sentences for the user, linking all relevant PRs together (e.g., `([datacommons#163](https://github.com/...), [datacommons#178](https://github.com/...))`). * *STRICT EXCLUSION*: Completely ignore internal development chores, test refactors, CI sandbox workflows, local test setups, or unused sample data removals. --- @@ -230,7 +232,7 @@ def render( | :--- | :--- | :--- | | **Key Features** | "Merged PR to implement SDMX 3.0 CSV parser in import repo." | **Import SDMX 3.0 CSV files directly** to ingest standard-compliant macroeconomic datasets into your private instance with zero manual preprocessing. | | **Improvements** | "Added Terraform variable max_workers." | **Scalable Dataflow Import Pipelines**: Configure `max_workers` in your Terraform configurations to scale compute resources automatically during large-scale imports. | -| **Bug Fixes** | "Fixed NullPointerException in observation API when entity is empty." | **Observation Serving**: Resolved a crash in the `/v2/observation` endpoint when querying empty entities; the API now gracefully returns an empty payload with a 200 OK. | +| **Bug Fixes** | "- Fixed NullPointerException in observation API when entity is empty. [PR 1]
- Fixed timeout in preprocessor polling. [PR 2]
- Fixed IAM count error. [PR 3]" | **Deployment & Infrastructure Reliability**: Resolved Terraform IAM race conditions during fresh deployments and fixed preprocessor polling timeouts during long-running ingestion workflows ([datacommons#163](https://github.com/...), [datacommons#176](https://github.com/...)). | --- From 69d3ca7aee40e6a8a8437b2cb721835c26e17c18 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 20:22:48 -0700 Subject: [PATCH 38/70] refactor(deploy): replace specific term list with generalized External Contracts & Capabilities rule in FeatureExtractor & ReleaseNotesWriter --- deploy/generate_release_notes/feature_extractor.py | 5 ++++- deploy/generate_release_notes/release_notes_writer.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index 32a0501a..f3db419a 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -200,7 +200,10 @@ def extract_features( * **Concise Sentence Budget**: Keep `description` to 1-2 punchy sentences (max 25 words). Keep `pr_contributions` summaries to 1 short sentence (12-15 words max per PR). * **User Capability Focus**: Frame the "description" and "pr_contributions" around what the user can *actually do* or what *input formats* are now supported (e.g., "Configure max_workers in Terraform to scale Dataflow workers automatically" instead of "Added Terraform max_workers variable"). * **Extract Concrete Enums & Configuration Values (STRICT)**: Whenever an update introduces or modifies a configuration variable, CLI flag, environment variable, or Terraform setting, DO NOT summarize it generically. ALWAYS extract and list the specific valid values or enums (e.g., `custom_only`, `base_only`, `base_and_custom`, `--instance_name`, processing unit bounds) and explain the exact behavior or filtering capability each option enables for operators! -* **STRICT BAN ON DATABASE INTERNAL NAMES (ZERO TOLERANCE)**: NEVER output feature titles or section names containing internal database terms (e.g., "KeyValueStore", "Spanner Graph DDL", "Bigtable Migration", "Spanner Key Value Store", "Database Schema Modification"). Internal database storage details MUST NOT be exposed to platform users or partners! If a database change improves performance or latency, title it around user impact (e.g., "API Serving Latency & Latency Optimization") and describe the speedup without naming internal database tables or storage layers! +* **FOCUS ON EXTERNAL CONTRACTS & CAPABILITIES (ZERO INTERNAL IMPLEMENTATION MECHANICS)**: + - NEVER output feature titles or section names named after internal storage implementations, database table names, schema migrations, or low-level data structures. + - External partners and operators interact with HTTP/gRPC APIs, Terraform modules, and CLI tools — they do not care about internal database tables, cache formats, or storage engine cutovers. + - Frame all storage or performance improvements strictly around user-facing impact (e.g., "API Serving Latency & Query Throughput", "Data Ingestion Speed", "Cache Freshness"). --- diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index e5cdeb99..78657eac 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -197,7 +197,10 @@ def render( * **Capabilities & Changes Bullets**: 12-15 words max per bullet. * **The "So What?" Rule**: Do not just list code changes. Frame every update around user capability (e.g., *what* can the developer do now, *which* inputs are accepted, or *how* does this affect query performance/scalability?). * **Extract Concrete Enums & Configuration Values (STRICT)**: Whenever an update introduces or modifies a configuration variable, CLI flag, environment variable, or Terraform setting, DO NOT summarize it generically (e.g., "Configured search scope"). ALWAYS extract and list the specific valid values or enums (e.g., `custom_only`, `base_only`, `base_and_custom`, `--instance_name`, processing unit bounds) and explain the exact behavior or filtering capability each option enables for operators! -* **STRICT BAN ON DATABASE INTERNAL NAMES (ZERO TOLERANCE)**: NEVER output feature titles or section names containing internal database terms (e.g., "KeyValueStore", "Spanner Graph DDL", "Bigtable Migration", "Spanner Key Value Store", "Database Schema Modification"). Internal database storage details MUST NOT be exposed to platform users or partners! If a database change improves performance or latency, title it around user impact (e.g., "API Serving Latency & Latency Optimization") and describe the speedup without naming internal database tables or storage layers! +* **FOCUS ON EXTERNAL CONTRACTS & CAPABILITIES (ZERO INTERNAL IMPLEMENTATION MECHANICS)**: + - NEVER output feature titles or section names named after internal storage implementations, database table names, schema migrations, or low-level data structures. + - External partners and operators interact with HTTP/gRPC APIs, Terraform modules, and CLI tools — they do not care about internal database tables, cache formats, or storage engine cutovers. + - Frame all storage or performance improvements strictly around user-facing impact (e.g., "API Serving Latency & Query Throughput", "Data Ingestion Speed", "Cache Freshness"). --- From 67a861757eb51487165d2499d991ea7c3ebcec64 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 20:25:38 -0700 Subject: [PATCH 39/70] refactor(deploy): apply subagent audit recommendations to generalize sample data rules and strengthen external contract focus in prompts --- deploy/generate_release_notes/feature_extractor.py | 2 +- deploy/generate_release_notes/release_notes_writer.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index f3db419a..b66c6499 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -182,7 +182,7 @@ def extract_features( ### 3. STRICT FILTERING & DEDUPLICATION RULES * **EXCLUDE Internal Dev & Testing PRs**: * Ignore automated bot PRs (e.g., Dependabot, Renovate, "chore: bump version"). - * Ignore all test-only PRs (e.g., integration test setups, Spanner Omni test conversions, CI sandbox workflows, local test harnesses, hermetic test refactors, and test-only sample data updates like OECD wage sample data). + * Ignore all test-only PRs (e.g., integration test setups, Spanner Omni test conversions, CI sandbox workflows, local test harnesses, hermetic test refactors, and test-only sample data or benchmark fixtures). * Ignore formatting, typos, or non-informative refactors with zero user impact. * **EXCLUDE Internal Iteration Bug Fixes (Release Window Regressions)**: * If a bug fix PR addresses a bug or regression introduced *within this same release window* (i.e. introduced after `{manifest.previous_version}` and fixed before `{manifest.new_version}`), DO NOT list it as a standalone Bug Fix! diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index 78657eac..f6a77bd0 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -201,6 +201,7 @@ def render( - NEVER output feature titles or section names named after internal storage implementations, database table names, schema migrations, or low-level data structures. - External partners and operators interact with HTTP/gRPC APIs, Terraform modules, and CLI tools — they do not care about internal database tables, cache formats, or storage engine cutovers. - Frame all storage or performance improvements strictly around user-facing impact (e.g., "API Serving Latency & Query Throughput", "Data Ingestion Speed", "Cache Freshness"). + - If a PR modifies an internal storage or cache layer to achieve faster serving, describe the benefit as "Faster API Query Execution & Higher Serving Throughput" without naming internal storage tables or schemas. --- From 9871d3e3938f247bf3405f48a02d36a90488f6fd Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 20:26:09 -0700 Subject: [PATCH 40/70] fix(deploy): pass pr_urls payload, inject 6-repo domain context into ReleaseNotesWriter, and align sentence budget constraints --- .../feature_extractor.py | 2 +- .../release_notes_writer.py | 30 +++++++++++++------ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py index b66c6499..f6ed816a 100644 --- a/deploy/generate_release_notes/feature_extractor.py +++ b/deploy/generate_release_notes/feature_extractor.py @@ -223,7 +223,7 @@ def extract_features( {{ "id": "short_unique_snake_case_id", "title": "Clear Technical Feature Title", - "description": "2-3 sentence technical description of the feature, explaining the change and its user-facing impact.", + "description": "1-2 sentence technical description of the feature (max 25 words), explaining the change and its user-facing impact.", "category": "Spanner Graph & APIs | Ingestion & Safety | Search & Website | Infra & Tooling", "target_components": ["dcp", "services", "preprocessing", "dataflow_worker", "ingestion_helper", "postprocessing"], "included_prs": ["agent-toolkit#211", "datacommons#189"], diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index f6a77bd0..5f529135 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -151,6 +151,7 @@ def render( ) # Build payloads for prompt — include ONLY DCP-relevant user/partner features + pr_url_map = {pr.qualified_id: pr.url for pr in manifest.all_pull_requests} features_payload = [] for feat in features: if not feat.is_dcp_relevant: @@ -164,6 +165,7 @@ def render( "category": feat.category, "target_components": feat.target_components, "included_prs": feat.included_prs, + "pr_urls": {pr_id: pr_url_map.get(pr_id, "") for pr_id in feat.included_prs}, "pr_contributions": feat.pr_contributions, "is_dcp_relevant": feat.is_dcp_relevant, "breaking_changes": feat.breaking_changes, @@ -186,7 +188,17 @@ def render( --- -### 1. CORE WRITING STYLE & TONE (CONCISE & ANTI-FLUFF) +### 1. CONTEXT & DOMAIN KNOWLEDGE +Data Commons Platform (DCP) is a self-hosted, Cloud Spanner-backed deployment of Data Commons. It replaces legacy Bigtable with Cloud Spanner graph tables and vector embeddings. It features custom data ingestion pipelines, specialized serving APIs, and deployment automation across 6 repositories: +1. `datacommonsorg/datacommons` (Monorepo): Terraform modules (`infra/dcp/`, `infra/modules/`), CLI tools (`packages/datacommons-cli/`), and Admin Portal (`packages/datacommons-admin/`). +2. `datacommonsorg/website`: Web application serving UI and APIs (`server/`, `static/`, `build/cdc_services/`, `build/cdc_data/`). +3. `datacommonsorg/mixer`: Core Spanner gRPC graph and StatVar serving engine (`internal/server/`, `proto/`, `deploy/helm_charts/`, ESPv2 gateway). +4. `datacommonsorg/agent-toolkit`: Model Context Protocol (MCP) server & FastMCP tools for AI agent integrations (`src/datacommons_mcp/`). +5. `datacommonsorg/import`: Data ingestion preprocessor (`simple/`), Dataflow ingestion worker (`pipeline/ingestion/`), and Cloud Workflow helpers (`pipeline/workflow/ingestion-helper/`, `pipeline/workflow/aggregation-helper/`). + +--- + +### 2. CORE WRITING STYLE & TONE (CONCISE & ANTI-FLUFF) * **Perspective & Tone**: Write like a senior Google engineer writing a concise technical changelog — direct, factual, punchy, and zero fluff. Use active voice ("You can now...") for features, past tense ("Fixed...") for bugs. * **Audience Focus**: Write specifically for external developers, data engineers, and instance operators building ON TOP OF the platform. * **BANNED AI FLUFF WORDS (STRICT)**: DO NOT use AI cliché words: `seamlessly`, `empower`, `leveraging`, `robust`, `overhaul`, `delivers a major`, `comprehensive`, `fosters`, `game-changing`, `cutting-edge`, `paradigm`. Write simple, direct sentences instead! @@ -205,22 +217,22 @@ def render( --- -### 2. STRICT CONSTRAINTS (ZERO-TOLERANCE RULES) +### 3. STRICT CONSTRAINTS (ZERO-TOLERANCE RULES) * **NO Emojis**: Do not use emojis anywhere in the document. * **NO Version/Commit Tables**: Do not include a component version table, git commit hashes, or SHA tables. * **NO Commit Range Text**: Do not include text like "Release range: v1.1.0 to v1.1.1". -* **NO Code-Fenced Links**: Every PR reference MUST be a clean, clickable GFM link. Do not wrap backticks around or inside link text. +* **NO Code-Fenced Links**: Every PR reference MUST be a clean, clickable GFM link. Do not wrap backticks around or inside link text. Use the provided full URL for each PR in `pr_urls` or `url` fields! * **CORRECT**: `[website#123](https://github.com/...)` * **INCORRECT**: `[`website#123`](https://github.com/...)` or `[`website#123` (https://github.com/...)]` --- -### 3. INPUT MAPPING RULES (RENDER ALL FEATURES) +### 4. INPUT MAPPING RULES (RENDER ALL FEATURES) You will process two payload inputs. Map them to the final release note sections as follows: 1. **`features_payload`**: * **STRICT REQUIREMENT**: You MUST render EVERY item provided in `features_payload`. DO NOT drop or omit any feature! - * Major, highly impactful items (e.g. SDMX 3.0 REST Endpoints, Agent MCP Toolkit Overhaul, Modular Ingestion Workflows) MUST be rendered as detailed feature sections under **Key Feature Updates**. + * Major, highly impactful items (e.g. new external API protocols, major component architecture overhauls, core pipeline workflow changes) MUST be rendered as detailed feature sections under **Key Feature Updates**. * Minor enhancements, optimizations, or configuration instructions must be formatted as concise bullet points under **Improvements & Configuration Updates**. 2. **`bug_fixes_payload`**: * **HIGH-LEVEL GROUPING (MAX 4-5 BULLETS TOTAL - NO LAUNDRY LIST)**: DO NOT output a laundry list of dozens of individual PRs! @@ -230,7 +242,7 @@ def render( --- -### 4. DO vs. DON'T CONTENT SAMPLES +### 5. DO vs. DON'T CONTENT SAMPLES | Section | ❌ DO NOT Write (Internal Developer Focus) | DO Write (Partner & Operator Focus) | | :--- | :--- | :--- | @@ -240,7 +252,7 @@ def render( --- -### 5. INPUT DATA +### 6. INPUT DATA #### Major Features & Updates: {json.dumps(features_payload, indent=2)} @@ -252,12 +264,12 @@ def render( --- -### 6. REQUIRED OUTPUT STRUCTURE +### 7. REQUIRED OUTPUT STRUCTURE Generate GFM matching the exact structure below. Do not add any greeting, intro, or concluding conversational text outside this structure. # Data Commons Platform Release {manifest.new_version} ({release_date}) -[Provide a high-impact, 1-2 sentence Executive Summary highlighting the most important capabilities, performance boosts, and critical fixes introduced in this release for partners and platform operators.] +[Provide a high-impact, 1-sentence Executive Summary (max 25 words) highlighting the most important capabilities, performance boosts, and critical fixes introduced in this release for partners and platform operators.] --- From 6bfd99b5d5fa99cea31658c86f9039357f4ccb8a Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 20:28:08 -0700 Subject: [PATCH 41/70] refactor(deploy): streamline Feature Updates format into What's New and Specific Capabilities in ReleaseNotesWriter --- deploy/generate_release_notes/release_notes_writer.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index 5f529135..7953f410 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -204,9 +204,8 @@ def render( * **BANNED AI FLUFF WORDS (STRICT)**: DO NOT use AI cliché words: `seamlessly`, `empower`, `leveraging`, `robust`, `overhaul`, `delivers a major`, `comprehensive`, `fosters`, `game-changing`, `cutting-edge`, `paradigm`. Write simple, direct sentences instead! * **STRICT WORD COUNT BUDGETS**: * **Executive Summary**: Maximum 25 words (1 single, punchy sentence). - * **What's New**: 15-20 words max (1 direct sentence). - * **Why it Matters**: 15-20 words max (1 direct sentence). - * **Capabilities & Changes Bullets**: 12-15 words max per bullet. + * **What's New**: Combine description and user benefit into 1 concise paragraph (25-35 words max). + * **Specific Capabilities Bullets**: 12-15 words max per bullet. * **The "So What?" Rule**: Do not just list code changes. Frame every update around user capability (e.g., *what* can the developer do now, *which* inputs are accepted, or *how* does this affect query performance/scalability?). * **Extract Concrete Enums & Configuration Values (STRICT)**: Whenever an update introduces or modifies a configuration variable, CLI flag, environment variable, or Terraform setting, DO NOT summarize it generically (e.g., "Configured search scope"). ALWAYS extract and list the specific valid values or enums (e.g., `custom_only`, `base_only`, `base_and_custom`, `--instance_name`, processing unit bounds) and explain the exact behavior or filtering capability each option enables for operators! * **FOCUS ON EXTERNAL CONTRACTS & CAPABILITIES (ZERO INTERNAL IMPLEMENTATION MECHANICS)**: @@ -277,11 +276,9 @@ def render( ### [Feature Title] -**What's New**: [Clear description of what partners or operators can now do] +**What's New**: [Clear 1-2 sentence description combining what changed and why it is important / user capability enabled.] -**Why it Matters**: [Business benefit, technical impact, or performance advantage] - -**Capabilities & Use Cases Enabled**: +**Specific Capabilities**: - [Actionable Use Case / Input Capability 1] ([repo_short#PR](URL)) - [Actionable Use Case / Input Capability 2] ([repo_short#PR](URL)) From 152e770cc83571357692f6d0adec00009f691fd2 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 20:28:25 -0700 Subject: [PATCH 42/70] fix(deploy): add NO Horizontal Dividers Between Features constraint to ReleaseNotesWriter --- deploy/generate_release_notes/release_notes_writer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index 7953f410..0f5b6130 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -220,6 +220,7 @@ def render( * **NO Emojis**: Do not use emojis anywhere in the document. * **NO Version/Commit Tables**: Do not include a component version table, git commit hashes, or SHA tables. * **NO Commit Range Text**: Do not include text like "Release range: v1.1.0 to v1.1.1". +* **NO Horizontal Dividers Between Features**: Do not place horizontal rule lines (`---`) between individual feature sections under Key Feature Updates. Use standard Markdown headers (`### Feature Title`) with single blank lines only! * **NO Code-Fenced Links**: Every PR reference MUST be a clean, clickable GFM link. Do not wrap backticks around or inside link text. Use the provided full URL for each PR in `pr_urls` or `url` fields! * **CORRECT**: `[website#123](https://github.com/...)` * **INCORRECT**: `[`website#123`](https://github.com/...)` or `[`website#123` (https://github.com/...)]` From 4a8c5569c85ece1611b2cce6e6f3d5f432b93bba Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Wed, 29 Jul 2026 20:34:53 -0700 Subject: [PATCH 43/70] fix(deploy): deterministically filter out internal regression fixes from Bug Fixes payload in ReleaseNotesWriter --- deploy/generate_release_notes/release_notes_writer.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py index 0f5b6130..754752d8 100644 --- a/deploy/generate_release_notes/release_notes_writer.py +++ b/deploy/generate_release_notes/release_notes_writer.py @@ -62,15 +62,26 @@ def _extract_bug_fixes( self, manifest: ReleaseInfoManifest, features: List[FeatureUpdate] ) -> List[Dict[str, Any]]: """Extracts PRs that are bug fixes or one-off improvements not covered in major features.""" + from deploy.generate_release_notes.feature_extractor import detect_internal_regressions + included_pr_ids = set() for feat in features: included_pr_ids.update(feat.included_prs) + # Detect internal regressions to exclude intermediate fixes from Bug Fixes + regression_map = detect_internal_regressions(manifest.all_pull_requests) + bug_fix_prs = [] for pr in manifest.all_pull_requests: if pr.qualified_id in included_pr_ids: continue + if pr.qualified_id in regression_map: + logger.info( + f"Excluding intermediate regression fix {pr.qualified_id} (fixes {regression_map[pr.qualified_id]}) from public Bug Fixes." + ) + continue + title_lower = pr.title.lower() # Identify bug fixes or minor partner-relevant PRs is_fix = ( From 3a9ee686b8e2069b3c2dc37442bef45bccceeed0 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 10:55:26 -0700 Subject: [PATCH 44/70] feat(deploy): include full list of pull_requests per component/image in --manifest-out JSON --- deploy/generate_release_notes/main.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/deploy/generate_release_notes/main.py b/deploy/generate_release_notes/main.py index 338bf523..9462e352 100644 --- a/deploy/generate_release_notes/main.py +++ b/deploy/generate_release_notes/main.py @@ -176,13 +176,26 @@ def main( "prev_sha": v.previous_sha, "new_sha": v.new_sha, "image_uri": v.image_uri, + "pull_requests_count": len(manifest.pull_requests_by_component.get(k, [])), + "pull_requests": [ + { + "id": pr.qualified_id, + "title": pr.title, + "author": pr.author, + "repo": pr.repo_name, + "url": pr.url, + "merged_at": pr.merged_at, + "files_changed": pr.files_changed, + } + for pr in manifest.pull_requests_by_component.get(k, []) + ], } for k, v in manifest.components.items() }, } with open(manifest_out, "w") as f: json.dump(manifest_dict, f, indent=2) - logger.info(f"Saved manifest summary to {manifest_out}") + logger.info(f"Saved manifest with PRs per image to {manifest_out}") # ---------------------------------------------------- # STEP 2: Feature Extraction & SOP Classification From 7148b4293933cbe3086508e6c6b7d400ef23e4db Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 11:08:19 -0700 Subject: [PATCH 45/70] feat(deploy): transform DCP Release Notes Generator into a modular Jetski Agentic Skill Suite --- deploy/generate_release_notes/README.md | 50 ++++++++++--- deploy/generate_release_notes/SKILL.md | 50 +++++++++++++ .../skills/dcp-context/SKILL.md | 61 +++++++++++++++ .../skills/pr-extraction/SKILL.md | 69 +++++++++++++++++ .../skills/release-writer/SKILL.md | 74 +++++++++++++++++++ 5 files changed, 293 insertions(+), 11 deletions(-) create mode 100644 deploy/generate_release_notes/SKILL.md create mode 100644 deploy/generate_release_notes/skills/dcp-context/SKILL.md create mode 100644 deploy/generate_release_notes/skills/pr-extraction/SKILL.md create mode 100644 deploy/generate_release_notes/skills/release-writer/SKILL.md diff --git a/deploy/generate_release_notes/README.md b/deploy/generate_release_notes/README.md index 9dd99d92..2cb1bea5 100644 --- a/deploy/generate_release_notes/README.md +++ b/deploy/generate_release_notes/README.md @@ -1,14 +1,46 @@ # Data Commons Platform (DCP) Release Notes Generator -An agentic, multi-repository tool for generating publication-ready, partner-facing release notes for the Data Commons Platform (DCP). +An agentic, skill-driven tool suite for generating publication-ready, partner-facing release notes for the Data Commons Platform (DCP). -The tool automatically extracts merged Pull Requests across all 6 core Data Commons repositories, classifies them according to standard SOP categories using Gemini 3.6 Flash, filters out internal test noise and regressions, and formats concise release notes tailored for developers and platform operators building on top of DCP. +The tool automatically extracts merged Pull Requests across all 6 core Data Commons repositories, classifies them according to standard SOP categories, filters out internal test noise and regressions, writes human-verifiable PR lists per container image (`output/prs_.txt`), and formats concise release notes tailored for developers and platform operators building on top of DCP. --- -## Architecture Overview +## Agentic Skill Suite Architecture -The tool operates as a structured 3-step pipeline: +The tool can be executed natively by Jetski using 4 specialized `SKILL.md` instruction sets: + +``` +deploy/generate_release_notes/ +├── SKILL.md <-- 1. Orchestrator Skill (Master Entrypoint) +├── skills/ +│ ├── pr-extraction/ +│ │ └── SKILL.md <-- 2. PR Extraction & Image Tag Resolution Skill +│ ├── dcp-context/ +│ │ └── SKILL.md <-- 3. DCP Domain Context & Architectural Map +│ └── release-writer/ +│ └── SKILL.md <-- 4. Partner-Facing Release Notes Writer +└── output/ <-- Verification & Output Directory + ├── prs_services.txt <-- Verified PRs for Core Services (Website, Mixer, MCP) + ├── prs_preprocessing.txt <-- Verified PRs for Data Preprocessor (datacommons-data) + ├── prs_dataflow_worker.txt <-- Verified PRs for Dataflow Worker + ├── prs_ingestion_helper.txt <-- Verified PRs for Ingestion Helper + ├── prs_postprocessing.txt <-- Verified PRs for Postprocessing Helper + ├── prs_dcp_monorepo.txt <-- Verified PRs for DCP Monorepo & Infra + └── RELEASE_NOTES_v1.1.1.md <-- Final Publication-Ready Release Notes +``` + +### How to Run via Jetski: +Simply ask Jetski: +> *"Jetski, generate release notes for v1.1.0 to v1.1.1 using the dcp-release-notes skill."* + +Jetski will orchestrate specialized subagents, resolve container image tags via `gcloud`, extract PRs via `gh pr list`, write human-verifiable `output/prs_.txt` files per image, apply domain context, and author publication-ready GFM release notes! + +--- + +## CLI Execution (Python Pipeline) + +Alternatively, you can run the standalone Python CLI tool: ``` ┌─────────────────────────┐ ┌──────────────────────────────┐ ┌────────────────────────────┐ @@ -17,13 +49,9 @@ The tool operates as a structured 3-step pipeline: └─────────────────────────┘ └──────────────────────────────┘ └────────────────────────────┘ ``` -1. **Step 1: PR Extractor (`pr_extractor.py`)**: Resolves container image tags across Artifact Registry (`gcr.io/datcom-ci/datacommons-services`, `datacommons-data`, etc.) and Git tags. Queries GitHub CLI (`gh pr list`) in a single date-range search per repository to fetch all merged PRs between versions. -2. **Step 2: Feature Extractor (`feature_extractor.py`)**: Uses Gemini 3.6 Flash to filter out Dependabot PRs, test-only refactors, and intermediate release-window regressions (via deterministic file diff footprint matching). Synthesizes remaining PRs into structured `FeatureUpdate` objects categorized under: - - **Spanner Graph & APIs** (SDMX 3.0 REST endpoints, `/v2/observation`, Mixer gRPC, MCP tools) - - **Ingestion & Safety** (Dataflow workers, workflow orchestration, preprocessor, health probes) - - **Search & Website** (Vector embeddings, search scope targeting, Website UI) - - **Infra & Tooling** (Terraform modules, Admin CLI, monorepo versioning) -3. **Step 3: Release Notes Writer (`release_notes_writer.py`)**: Renders publication-ready GitHub Flavored Markdown (GFM) using the **"So What?" Rule**, strict **Anti-AI-Fluff Word Budgets**, side-by-side **DO vs. DON'T guidelines**, and clickable `[repo#PR](URL)` links. +1. **Step 1: PR Extractor (`pr_extractor.py`)**: Resolves container image tags across Artifact Registry and Git tags. Queries GitHub CLI (`gh pr list`) in a single date-range search per repository to fetch all merged PRs between versions. +2. **Step 2: Feature Extractor (`feature_extractor.py`)**: Uses Gemini 3.6 Flash to filter out Dependabot PRs, test-only refactors, and intermediate release-window regressions (via deterministic file diff footprint matching). +3. **Step 3: Release Notes Writer (`release_notes_writer.py`)**: Renders publication-ready GitHub Flavored Markdown (GFM) using the **"So What?" Rule**, strict **Anti-AI-Fluff Word Budgets**, and clickable `[repo#PR](URL)` links. --- diff --git a/deploy/generate_release_notes/SKILL.md b/deploy/generate_release_notes/SKILL.md new file mode 100644 index 00000000..7639c5cc --- /dev/null +++ b/deploy/generate_release_notes/SKILL.md @@ -0,0 +1,50 @@ +--- +name: dcp-release-notes +description: Master orchestrator skill for generating publication-ready, partner-facing Data Commons Platform (DCP) release notes across all 6 core repositories using agentic subagents. +--- + +# DCP Release Notes Generator (Orchestrator Skill) + +This skill orchestrates the end-to-end generation of publication-ready, partner-facing release notes for the Data Commons Platform (DCP). It Coordinates specialized subagents to extract PRs per container image, verify PR lists with the developer in human-readable `.txt` files, apply domain context, and author concise release notes. + +--- + +## Workflow Instructions + +When the user asks to generate release notes (e.g., *"Generate release notes for v1.1.0 to v1.1.1"*): + +### Step 1: Target Version Resolution +1. Identify the previous release tag (``, e.g., `v1.1.0`) and target release tag (``, e.g., `v1.1.1`). +2. Create the output directory `deploy/generate_release_notes/output/` if it does not exist. + +### Step 2: PR Extraction & Image Verification (Subagent Delegation) +1. Read the **PR Extraction Skill**: [SKILL.md](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/pr-extraction/SKILL.md). +2. Spawn subagent(s) using `invoke_subagent` equipped with `gcloud` and `gh` CLI commands to: + - Resolve Artifact Registry image tags (`gcr.io/datcom-ci/datacommons-services`, `datacommons-data`, etc.). + - Execute date-range PR queries (`gh pr list --search "merged:.."`) across all 6 repositories. + - Filter out Dependabot, automated version bumps, and test-only fixtures. + - Deterministically detect intermediate release-window regressions. + - Write verified PR lists to component text files in `deploy/generate_release_notes/output/`: + * `prs_services.txt` (Core Services: Website, Mixer, MCP Agent) + * `prs_preprocessing.txt` (Data Preprocessor: datacommons-data) + * `prs_dataflow_worker.txt` (Dataflow Ingestion Worker) + * `prs_ingestion_helper.txt` (Ingestion Helper Service) + * `prs_postprocessing.txt` (Postprocessing Aggregation Helper) + * `prs_dcp_monorepo.txt` (DCP Monorepo & Terraform Infra) + +3. Inform the developer that PR verification files are written in `deploy/generate_release_notes/output/` for review. + +### Step 3: Load Domain Context & Architectural Principles +1. Read the **DCP Domain Context Skill**: [SKILL.md](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/dcp-context/SKILL.md). +2. Ensure strict adherence to: + - **External Contracts & Operator Capabilities**: Focus strictly on APIs, Terraform variables, CLI commands, and operator capabilities. + - **Zero Internal Implementation Mechanics**: Never output internal database table names (e.g., KeyValueStore, DDLs, schema cutovers). + +### Step 4: Author Publication-Ready Release Notes +1. Read the **Release Writer Skill**: [SKILL.md](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/release-writer/SKILL.md). +2. Synthesize the verified PR lists from `deploy/generate_release_notes/output/prs_*.txt` into the final release notes document: `deploy/generate_release_notes/output/RELEASE_NOTES_.md`. +3. Format according to the streamlined template: + - **Executive Summary**: 1 single sentence (max 25 words). + - **Key Feature Updates**: Non-verbose format (**What's New**: 1 paragraph combining description + benefit, followed by **Specific Capabilities**: bullets with `[repo#PR](URL)` links, NO horizontal rule dividers between features). + - **Improvements & Configuration Updates**: Bullet points extracting concrete enums (`custom_only`, `base_only`) and scaling limits (`max_workers`). + - **Bug Fixes**: 3–5 high-level functional categories (Deployment, Ingestion, Serving APIs, UI). diff --git a/deploy/generate_release_notes/skills/dcp-context/SKILL.md b/deploy/generate_release_notes/skills/dcp-context/SKILL.md new file mode 100644 index 00000000..c4b0d63e --- /dev/null +++ b/deploy/generate_release_notes/skills/dcp-context/SKILL.md @@ -0,0 +1,61 @@ +--- +name: dcp-context +description: Architectural reference and domain context for Data Commons Platform (DCP) release notes generation. +--- + +# Data Commons Platform (DCP) Domain Context & Architectural Map + +This skill provides the domain context, repository mapping, and architectural principles for writing publication-ready, partner-facing DCP release notes. + +--- + +## 1. Core Architectural Overview + +Data Commons Platform (DCP) is a self-hosted, Cloud Spanner-backed deployment of Data Commons. It replaces legacy Bigtable with Cloud Spanner graph tables and vector embeddings. It features custom data ingestion pipelines, specialized serving APIs, and deployment automation across 6 core repositories: + +1. `datacommonsorg/datacommons` (Monorepo & Infra): + - **Terraform Modules** (`infra/dcp/`, `infra/modules/`): Infrastructure provisioning for Spanner, Cloud Run, BigQuery, and Dataflow. + - **CLI Tools** (`packages/datacommons-cli/`): `datacommons admin init`, `datacommons admin deploy`. + - **Admin Portal** (`packages/datacommons-admin/`): Web management interface. + +2. `datacommonsorg/website` (Web Application & Frontend): + - Serves UI pages (Explore, Visualization Tools, Place Browser) and REST API routing (`server/`, `static/`, `build/cdc_services/`). + +3. `datacommonsorg/mixer` (Core Serving Engine): + - High-performance gRPC graph and StatVar serving engine (`internal/server/`, `proto/`, `deploy/helm_charts/`, ESPv2 gateway). + - Serves SDMX 3.0 REST Data & Availability endpoints, `/v2/observation`, and vector search embeddings. + +4. `datacommonsorg/agent-toolkit` (Model Context Protocol / MCP): + - Model Context Protocol (MCP) server & FastMCP tools for AI agent integrations (`src/datacommons_mcp/`). + - Enables agentic research playbooks, multi-entity observation retrieval, and indicator search across custom or base instances. + +5. `datacommonsorg/import` (Ingestion Stack & Cloud Workflows): + - **Data Preprocessor** (`simple/`): CSV/MCF validation and streaming JSON-LD batching (built into `datacommons-data` container image). + - **Dataflow Ingestion Worker** (`pipeline/ingestion/`): Parallelized BigQuery/Spanner graph loading. + - **Cloud Workflow Helpers** (`pipeline/workflow/ingestion-helper/`, `pipeline/workflow/aggregation-helper/`): Ingestion status tracking and postprocessing aggregations (StatVar, Place, Entity rollups). + +--- + +## 2. Architectural Boundary & Persona Principles + +### Focus on External Contracts & Operator Capabilities +- **Partner & Operator Focus**: Write specifically for external developers, data engineers, and instance operators building ON TOP OF DCP. +- **User Capabilities**: Frame every feature and improvement around *what the user can now do*, *which input formats are supported*, or *how compute resources scale*. +- **Extract Concrete Enums & Configuration Values**: Always extract valid enums (`custom_only`, `base_only`, `base_and_custom`), CLI flags (`--instance_name`), and scaling bounds (`max_workers`, BigQuery slots). + +### Zero Internal Implementation Mechanics (STRICT) +- **NO Internal Database Terms**: NEVER output feature titles or section names containing internal database table names, schema DDLs, or storage migration mechanics (e.g. no "KeyValueStore", "Spanner Graph DDL", "Bigtable Cutover", "Database Schema Modification"). +- **Frame Performance Speedups Around User Impact**: If an internal storage or cache layer change improves serving speed, title it around user impact: **"API Serving Latency & Query Throughput"** or **"Faster API Response Speed"** without naming internal database tables. + +--- + +## 3. SOP Categories for Feature Classification + +1. **Spanner Graph & APIs**: + - SDMX 3.0 REST endpoints, `/v2/observation`, Mixer gRPC graph serving, FastMCP tools, MCP agent research skills. +2. **Ingestion & Safety**: + - Preprocessing, Dataflow workers, Cloud Workflows orchestration, postprocessing aggregations, health probes. +3. **Search & Website**: + - Vector embeddings, semantic search, search target scope (`V2_RESOLVE_INDICATORS_TARGET`), Explore UI, Download Tool. +4. **Infra & Tooling**: + - Terraform modules, Admin CLI (`datacommons admin`), monorepo versioning, IAM role provisioning. diff --git a/deploy/generate_release_notes/skills/pr-extraction/SKILL.md b/deploy/generate_release_notes/skills/pr-extraction/SKILL.md new file mode 100644 index 00000000..ab9909be --- /dev/null +++ b/deploy/generate_release_notes/skills/pr-extraction/SKILL.md @@ -0,0 +1,69 @@ +--- +name: dcp-pr-extraction +description: Instructions for extracting, filtering, and verifying merged Pull Requests per container image across Data Commons repositories for release notes generation. +--- + +# DCP PR Extraction & Image Verification Skill + +This skill provides step-by-step instructions for extracting merged Pull Requests across all 6 Data Commons repositories and mapping them to their corresponding container images and components. + +--- + +## Component & Image Source Rules + +| Component Key | Component Name | Container Image URI / Artifact | Source Repos & Path Filters | +| :--- | :--- | :--- | :--- | +| `services` | Core Services (Website, Mixer, MCP Agent) | `gcr.io/datcom-ci/datacommons-services` | `datacommonsorg/website`
`datacommonsorg/mixer`
`datacommonsorg/agent-toolkit` | +| `preprocessing` | Data Preprocessor | `gcr.io/datcom-ci/datacommons-data` | `datacommonsorg/import` (filter: `simple/`) | +| `dataflow_worker` | Dataflow Ingestion Worker | `us-docker.pkg.dev/datcom-ci/gcr.io/dataflow-templates/ingestion` | `datacommonsorg/import` (filter: `pipeline/ingestion/`) | +| `ingestion_helper` | Ingestion Helper Service | `gcr.io/datcom-ci/datacommons-ingestion-helper` | `datacommonsorg/import` (filter: `pipeline/workflow/ingestion-helper/`) | +| `postprocessing` | Postprocessing Helper Service | `gcr.io/datcom-ci/datacommons-aggregation-helper` | `datacommonsorg/import` (filter: `pipeline/workflow/aggregation-helper/`) | +| `dcp` | DCP Monorepo & Terraform Infra | DCP Monorepo | `datacommonsorg/datacommons` | + +--- + +## Extraction Steps + +### 1. Image Tag & Timestamp Resolution +1. For each container image URI above, resolve the creation timestamp of `` and ``: + ```bash + gcloud container images list-tags --filter="tags:" --format="value(timestamp.datetime)" + ``` +2. If an image tag is missing (e.g. during staging before image tagging), set `t_new` to the current time `NOW()`. + +### 2. Single Date-Range PR Search per Repository +For each repository, run a single `gh pr list` query spanning `[t_prev .. t_new]`: +```bash +gh pr list --repo --state merged --search "merged:.." --json number,title,body,author,url,labels,files,mergedAt --limit 200 +``` +*(IMPORTANT: Do NOT pass `base:main` inside `--search`; use `--search "merged:.."` directly to prevent GitHub Search API parse errors!)* + +### 3. Intermediate Regression & Noise Filtering +1. **Filter Out Bot & Non-Production PRs**: + - Exclude Dependabot, Renovate, and automated version bumps (`"bump version"`, `datacommons-robot-author`). + - Exclude test-only PRs (unit/integration test harnesses, hermetic test refactors, test-only sample data, or benchmark fixtures). +2. **Filter Out Intermediate Release-Window Regressions**: + - If a bug fix PR addresses a feature or code modified *within the same release window* (`[t_prev..t_new]`), mark it as an internal regression and DO NOT include it in public Bug Fixes! + +### 4. Write Verification Files (`prs_.txt`) +Write clean, human-readable verification text files to `deploy/generate_release_notes/output/`: + +Format for each file: +``` +================================================================================ +Component: Core Services (Website, Mixer, MCP Agent) +Image URI: gcr.io/datcom-ci/datacommons-services +Release Range: v1.1.0 (2026-06-22) -> v1.1.1 (2026-07-28) +Total PRs: 86 +================================================================================ + +[mixer#2027] Support containedInPlace+ expansion in SDMX availability queries +Author: calinc | Merged: 2026-07-23T10:11:31Z +URL: https://github.com/datacommonsorg/mixer/pull/2027 +Files Changed: internal/server/sdmx/availability.go + +[agent-toolkit#211] Query bilateral entity observations through get_multi_entity_observations tool +Author: calinc | Merged: 2026-07-21T18:00:00Z +URL: https://github.com/datacommonsorg/agent-toolkit/pull/211 +Files Changed: src/datacommons_mcp/tools.py +``` diff --git a/deploy/generate_release_notes/skills/release-writer/SKILL.md b/deploy/generate_release_notes/skills/release-writer/SKILL.md new file mode 100644 index 00000000..ab197f8c --- /dev/null +++ b/deploy/generate_release_notes/skills/release-writer/SKILL.md @@ -0,0 +1,74 @@ +--- +name: dcp-release-writer +description: Instructions for authoring publication-ready, partner-facing Data Commons Platform (DCP) release notes using GFM markdown. +--- + +# DCP Release Notes Writer Skill + +This skill provides step-by-step instructions for authoring non-verbose, publication-ready, partner-facing release notes in GitHub Flavored Markdown (GFM). + +--- + +## 1. Writing Style & Tone Constraints + +- **Tone**: Direct, factual, punchy, senior-engineer technical changelog. Active voice for features ("You can now..."), past tense for bugs ("Resolved..."). +- **BANNED AI FLUFF WORDS (STRICT)**: DO NOT use AI cliché words: `seamlessly`, `empower`, `leveraging`, `robust`, `overhaul`, `delivers a major`, `comprehensive`, `fosters`, `game-changing`, `cutting-edge`, `paradigm`. +- **STRICT WORD COUNT BUDGETS**: + - **Executive Summary**: Maximum 25 words (1 single, punchy sentence). + - **What's New**: Combine description and user benefit into 1 concise paragraph (25-35 words max). + - **Specific Capabilities Bullets**: 12-15 words max per bullet. +- **GFM Link Rules**: Every PR reference MUST be a clean, clickable link: `[repo_short#PR](URL)`. NEVER wrap backticks around or inside link text (`[`repo#123`](URL)` is forbidden!). +- **NO Horizontal Dividers Between Features**: Do NOT place horizontal rule lines (`---`) between individual feature sections under Key Feature Updates. Use standard Markdown headers (`### Feature Title`) with single blank lines only! + +--- + +## 2. Document Template & Section Structure + +```markdown +# Data Commons Platform Release {new_version} ({release_date}) + +[Provide a high-impact, 1-sentence Executive Summary (max 25 words) highlighting the most important capabilities, performance boosts, and critical fixes introduced in this release for partners and platform operators.] + +--- + +## Key Feature Updates + +### [Feature Title] + +**What's New**: [Clear 1-2 sentence description combining what changed and why it is important / user capability enabled.] + +**Specific Capabilities**: +- [Actionable Use Case / Input Capability 1] ([repo_short#PR](URL)) +- [Actionable Use Case / Input Capability 2] ([repo_short#PR](URL)) + +### [Next Feature Title] + +**What's New**: [Clear 1-2 sentence description...] + +**Specific Capabilities**: +- [Actionable Use Case / Input Capability 1] ([repo_short#PR](URL)) + +--- + +## Improvements & Configuration Updates + +- **[Improvement Title]**: [Summary of update, step-by-step configuration instructions if required, and direct benefit] ([repo_short#PR](URL)) +- **[Terraform & Scaling]**: [Concrete scaling parameters, enums like custom_only/base_only, max_workers, processing units] ([repo_short#PR](URL)) + +--- + +## Bug Fixes + +- **[Deployment & Infrastructure]**: [Synthesized 1-2 sentence summary of deployment/IAM fixes] ([datacommons#163](URL), [datacommons#178](URL)) +- **[Ingestion Pipeline Reliability]**: [Synthesized 1-2 sentence summary of workflow/preprocessor fixes] ([import#636](URL), [import#637](URL)) +- **[Serving API & Query Robustness]**: [Synthesized 1-2 sentence summary of API/serving fixes] ([mixer#1995](URL), [mixer#2007](URL)) +- **[Web UI & Visualization]**: [Synthesized 1-2 sentence summary of UI/Explore fixes] ([website#6411](URL), [website#6474](URL)) +``` + +--- + +## 3. High-Level Bug Fix Grouping Rule + +- **MAX 3–5 BULLETS TOTAL**: DO NOT output a laundry list of dozens of individual PRs! +- Group all raw bug fixes into 3 to 5 functional categories (*Deployment & Infrastructure*, *Ingestion Pipeline Reliability*, *Serving API & Query Robustness*, *Web UI & Visualization*). +- Combine all related PR links into the single grouped bullet point. From 63baaaa32c8db8f8b4ebbe39e704dda80a2ef888 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 11:10:07 -0700 Subject: [PATCH 46/70] docs(deploy): update README with LLM agent skill instructions and zero Jetski mentions --- deploy/generate_release_notes/README.md | 256 ++++++------------------ 1 file changed, 63 insertions(+), 193 deletions(-) diff --git a/deploy/generate_release_notes/README.md b/deploy/generate_release_notes/README.md index 2cb1bea5..70a993f8 100644 --- a/deploy/generate_release_notes/README.md +++ b/deploy/generate_release_notes/README.md @@ -8,11 +8,11 @@ The tool automatically extracts merged Pull Requests across all 6 core Data Comm ## Agentic Skill Suite Architecture -The tool can be executed natively by Jetski using 4 specialized `SKILL.md` instruction sets: +The release notes generation pipeline is structured into 4 modular `SKILL.md` instruction sets. Point your LLM agent at these skills to execute the generation process: ``` deploy/generate_release_notes/ -├── SKILL.md <-- 1. Orchestrator Skill (Master Entrypoint) +├── SKILL.md <-- 1. Master Orchestrator Skill (Entrypoint) ├── skills/ │ ├── pr-extraction/ │ │ └── SKILL.md <-- 2. PR Extraction & Image Tag Resolution Skill @@ -30,221 +30,91 @@ deploy/generate_release_notes/ └── RELEASE_NOTES_v1.1.1.md <-- Final Publication-Ready Release Notes ``` -### How to Run via Jetski: -Simply ask Jetski: -> *"Jetski, generate release notes for v1.1.0 to v1.1.1 using the dcp-release-notes skill."* - -Jetski will orchestrate specialized subagents, resolve container image tags via `gcloud`, extract PRs via `gh pr list`, write human-verifiable `output/prs_.txt` files per image, apply domain context, and author publication-ready GFM release notes! - --- -## CLI Execution (Python Pipeline) +## Developer Usage Instructions (Prompting Your LLM Agent) -Alternatively, you can run the standalone Python CLI tool: +### Step 1: Point Your LLM Agent at the Orchestrator Skill +To generate release notes for a release range, point your LLM agent at [`deploy/generate_release_notes/SKILL.md`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/SKILL.md): -``` -┌─────────────────────────┐ ┌──────────────────────────────┐ ┌────────────────────────────┐ -│ Step 1: PR Extractor │ ───►│ Step 2: Feature Extractor │ ───►│ Step 3: Release Notes Writer│ -│ (gh CLI + gcloud tags) │ │ (Gemini 3.6 Flash LLM) │ │ (Gemini 3.6 Flash GFM) │ -└─────────────────────────┘ └──────────────────────────────┘ └────────────────────────────┘ -``` +> **Prompt Example**: +> *"Please read `deploy/generate_release_notes/SKILL.md` and generate release notes for version v1.1.0 to v1.1.1."* -1. **Step 1: PR Extractor (`pr_extractor.py`)**: Resolves container image tags across Artifact Registry and Git tags. Queries GitHub CLI (`gh pr list`) in a single date-range search per repository to fetch all merged PRs between versions. -2. **Step 2: Feature Extractor (`feature_extractor.py`)**: Uses Gemini 3.6 Flash to filter out Dependabot PRs, test-only refactors, and intermediate release-window regressions (via deterministic file diff footprint matching). -3. **Step 3: Release Notes Writer (`release_notes_writer.py`)**: Renders publication-ready GitHub Flavored Markdown (GFM) using the **"So What?" Rule**, strict **Anti-AI-Fluff Word Budgets**, and clickable `[repo#PR](URL)` links. +### Step 2: PR Extraction & Intermediate Verification Files +Your LLM agent will follow the PR extraction skill (`skills/pr-extraction/SKILL.md`) to: +1. Resolve container image tags across Artifact Registry via `gcloud`. +2. Query merged Pull Requests across all 6 Data Commons repositories via `gh pr list`. +3. Filter out non-production test fixtures and intermediate release-window regressions. +4. Output human-verifiable text files per container image into `deploy/generate_release_notes/output/`: + - `prs_services.txt` (Core Services: Website, Mixer, MCP Agent) + - `prs_preprocessing.txt` (Data Preprocessor: `datacommons-data`) + - `prs_dataflow_worker.txt` (Dataflow Ingestion Worker) + - `prs_ingestion_helper.txt` (Ingestion Helper Service) + - `prs_postprocessing.txt` (Postprocessing Aggregation Helper) + - `prs_dcp_monorepo.txt` (DCP Monorepo & Terraform Infra) ---- +Developers can open and inspect these `.txt` files to verify that all relevant PRs for each image are correctly captured before the final release notes are written. -## Prerequisites - -Before running the tool, ensure you have the following installed and authenticated: - -1. **Python 3.11+** and [`uv`](https://github.com/astral-sh/uv) (or `pip`). -2. **GitHub CLI (`gh`)**: Must be installed and authenticated to read pull requests: - ```bash - gh auth status - # If not authenticated: - gh auth login - ``` -3. **Google Cloud SDK (`gcloud`)**: Must be authenticated to query Artifact Registry image tags: - ```bash - gcloud auth list - # If not authenticated: - gcloud auth login - gcloud auth application-default login - ``` -4. **Gemini API Key or GCP Credentials**: - ```bash - export GEMINI_API_KEY="your_gemini_api_key_here" - ``` +### Step 3: Domain Context & Final Writing +Your LLM agent will load the domain context (`skills/dcp-context/SKILL.md`) and author the final release notes according to `skills/release-writer/SKILL.md`, outputting: +`deploy/generate_release_notes/output/RELEASE_NOTES_.md` --- -## Typical Usage Commands +## Modular Skill Breakdown -### 1. Basic Production Release Notes Generation -Generate release notes between two published release tags (e.g. `v1.1.0` and `v1.1.1`): -```bash -uv run --group generate-release-notes python -m deploy.generate_release_notes \ - --prev v1.1.0 \ - --new v1.1.1 \ - --out ./RELEASE_NOTES_v1.1.1.md -``` +### 1. Orchestrator Skill (`SKILL.md`) +The master entrypoint skill that coordinates subagent execution, manages the step-by-step pipeline, and ensures intermediate verification files are generated before writing the final release notes. -### 2. Pre-Release / Staging Generation (Allow Missing Images) -Generate release notes for a staging release before container images have been tagged in Artifact Registry (extends search window to current time `NOW()`): -```bash -uv run --group generate-release-notes python -m deploy.generate_release_notes \ - --prev v1.1.0 \ - --new v1.1.1 \ - --allow-missing-images \ - --out ./RELEASE_NOTES_v1.1.1.md -``` +### 2. PR Extraction Skill (`skills/pr-extraction/SKILL.md`) +Contains exact commands and rules for `gcloud container images list-tags` resolution, date-range `gh pr list` queries, non-production test filtering, and intermediate regression exclusion. -### 3. Including High-Priority Release Highlights / Additional Instructions -Provide custom context or release highlights via a markdown file: -```bash -uv run --group generate-release-notes python -m deploy.generate_release_notes \ - --prev v1.1.0 \ - --new v1.1.1 \ - --additional-instructions ./release_highlights.md \ - --out ./RELEASE_NOTES_v1.1.1.md -``` +### 3. DCP Domain Context Skill (`skills/dcp-context/SKILL.md`) +Defines the architectural map across all 6 core repositories (`datacommons`, `website`, `mixer`, `agent-toolkit`, `import`) and enforces strict **External Contracts & Operator Capabilities** vs. **Zero Internal Implementation Mechanics** (no internal database table names or DDLs). -### 4. Generating Release Notes with Full Audit Log Table -Include an append-only PR audit table at the bottom cross-referencing all 50+ processed PRs: -```bash -uv run --group generate-release-notes python -m deploy.generate_release_notes \ - --prev v1.1.0 \ - --new v1.1.1 \ - --include-audit-log \ - --out ./RELEASE_NOTES_v1.1.1.md -``` +### 4. Release Writer Skill (`skills/release-writer/SKILL.md`) +Defines the non-verbose, partner-facing GFM format: +- **Executive Summary**: 1 single sentence (max 25 words). +- **Key Feature Updates**: **What's New** (1 paragraph combining description + benefit) followed by **Specific Capabilities** (bullet points with `[repo#PR](URL)` links). +- **Improvements & Configuration Updates**: Bullet points extracting concrete enums (`custom_only`, `base_only`) and scaling limits (`max_workers`). +- **Bug Fixes**: 3–5 high-level functional categories (Deployment, Ingestion, Serving APIs, UI). --- -## Command-Line Options & Flags - -| Flag / Option | Type | Required | Description | -| :--- | :--- | :---: | :--- | -| `--prev` | `STRING` | **Yes** | Previous release version tag (e.g. `v1.1.0`). | -| `--new` | `STRING` | **Yes** | Target release version tag (e.g. `v1.1.1`). | -| `--out`, `-o` | `PATH` | No | Path to write output markdown file (default: `./RELEASE_NOTES_.md`). | -| `--allow-missing-images` | `BOOLEAN` | No | Bypass errors if container image tags are missing in Artifact Registry and extend date range to current time `NOW()`. | -| `--synthesis-model` | `STRING` | No | Gemini model to use for feature extraction & writing (default: `gemini-3.6-flash`). | -| `--additional-instructions` | `STRING/PATH` | No | Path to a markdown file or raw text containing high-priority user instructions or release highlights. | -| `--include-audit-log` | `BOOLEAN` | No | Append an audit log table mapping all raw PRs to their release classification status. | -| `--use-cache / --no-cache` | `BOOLEAN` | No | Enable or disable local disk caching for GitHub PR queries (default: `True`). | -| `--help` | `FLAG` | No | Display CLI help and exit. | +## Standalone CLI Execution (Python Pipeline) ---- +Alternatively, for non-LLM or CI/CD automated environments, run the standalone Python CLI tool: -## Configuration & Component Registry (`config.py`) - -The tool's multi-repository mappings, image URIs, and source rules are centrally configured in [`config.py`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/config.py). - -### Core Data Structures in `config.py`: - -1. **`SourceRule`**: Defines a GitHub repository and an optional sub-directory `path_filter` for mapping monorepo or multi-repo PRs: - ```python - SourceRule(repo="datacommonsorg/import", path_filter="simple/") - ``` -2. **`ComponentConfig`**: Configures a tracked release component: - - `id`: Internal component key (e.g. `services`, `preprocessing`). - - `name`: Human-readable display name. - - `artifact_type`: Artifact category (`dcp_platform`, `docker_services`, `docker_data`, `dataflow_template`, `docker_ingestion_helper`, `docker_postprocessing`). - - `image_uri`: Primary container image URI in Google Artifact Registry or GCR (e.g. `gcr.io/datcom-ci/datacommons-services`). - - `default_tag_prefix`: Version tag prefix (e.g. `"v"` for `v1.1.1`). - - `sources`: List of contributing `SourceRule` objects. - -### Master `COMPONENTS` Registry: - -```python -COMPONENTS: Dict[str, ComponentConfig] = { - "dcp": ComponentConfig( - id="dcp", - name="DCP Monorepo & Infra (CLI, Admin, DB, Terraform)", - artifact_type="dcp_platform", - image_uri=None, - default_tag_prefix="v", - sources=[SourceRule(repo="datacommonsorg/datacommons")], - ), - "services": ComponentConfig( - id="services", - name="Core Services (Website, Mixer, MCP)", - artifact_type="docker_services", - image_uri="gcr.io/datcom-ci/datacommons-services", - default_tag_prefix="v", - sources=[ - SourceRule(repo="datacommonsorg/website"), - SourceRule(repo="datacommonsorg/mixer"), - SourceRule(repo="datacommonsorg/agent-toolkit"), - ], - ), - "preprocessing": ComponentConfig( - id="preprocessing", - name="Data Preprocessor (datacommons-data)", - artifact_type="docker_data", - image_uri="gcr.io/datcom-ci/datacommons-data", - default_tag_prefix="v", - sources=[SourceRule(repo="datacommonsorg/import", path_filter="simple/")], - ), - "dataflow_worker": ComponentConfig( - id="dataflow_worker", - name="Dataflow Ingestion Worker", - artifact_type="dataflow_template", - image_uri="us-docker.pkg.dev/datcom-ci/gcr.io/dataflow-templates/ingestion", - default_tag_prefix="v", - sources=[SourceRule(repo="datacommonsorg/import", path_filter="pipeline/ingestion/")], - ), - "ingestion_helper": ComponentConfig( - id="ingestion_helper", - name="Ingestion Helper Service", - artifact_type="docker_ingestion_helper", - image_uri="gcr.io/datcom-ci/datacommons-ingestion-helper", - default_tag_prefix="v", - sources=[SourceRule(repo="datacommonsorg/import", path_filter="pipeline/workflow/ingestion-helper/")], - ), - "postprocessing": ComponentConfig( - id="postprocessing", - name="Postprocessing Aggregation Helper Service", - artifact_type="docker_postprocessing", - image_uri="gcr.io/datcom-ci/datacommons-aggregation-helper", - default_tag_prefix="v", - sources=[SourceRule(repo="datacommonsorg/import", path_filter="pipeline/workflow/aggregation-helper/")], - ), -} +```bash +uv run --group generate-release-notes python -m deploy.generate_release_notes \ + --prev v1.1.0 \ + --new v1.1.1 \ + --allow-missing-images \ + --manifest-out ./output/manifest_v1.1.1.json \ + --out ./output/RELEASE_NOTES_v1.1.1.md ``` -To add a new component or track an additional repository, simply define a new `ComponentConfig` in [`config.py`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/config.py). +### CLI Options Reference: +| Flag / Option | Type | Description | +| :--- | :--- | :--- | +| `--prev` | `STRING` | Previous release version tag (e.g. `v1.1.0`). | +| `--new` | `STRING` | Target release version tag (e.g. `v1.1.1`). | +| `--out`, `-o` | `PATH` | Output markdown file path (default: `./RELEASE_NOTES_.md`). | +| `--allow-missing-images` | `BOOLEAN` | Bypass missing container image tag errors during staging. | +| `--manifest-out` | `PATH` | Export JSON manifest mapping all PRs to their container image URIs. | +| `--include-audit-log` | `BOOLEAN` | Append raw PR audit log table at the bottom of the release notes. | --- -## Repository Coverage - -The tool automatically tracks and correlates PRs across all 6 core Data Commons repositories: +## Repository Mapping -| Repository | Scope / Path Filter | Target Component | +| Repository | Scope / Path Filter | Target Component & Image | | :--- | :--- | :--- | | `datacommonsorg/datacommons` | All PRs (`infra/dcp/`, `packages/`) | DCP Monorepo & Infra (`dcp`) | -| `datacommonsorg/website` | All PRs (excluding `cdc_data/`) | Core Services (`services`) | -| `datacommonsorg/mixer` | All PRs (`internal/server/`, `proto/`, `deploy/`) | Core Services (`services`) | -| `datacommonsorg/agent-toolkit` | All PRs (`src/datacommons_mcp/`) | Core Services (`services`) | -| `datacommonsorg/import` | `simple/` | Data Preprocessor (`preprocessing`) | -| `datacommonsorg/import` | `pipeline/ingestion/` | Dataflow Worker (`dataflow_worker`) | -| `datacommonsorg/import` | `pipeline/workflow/ingestion-helper/` | Ingestion Helper (`ingestion_helper`) | -| `datacommonsorg/import` | `pipeline/workflow/aggregation-helper/` | Postprocessing Helper (`postprocessing`) | - ---- - -## Development & Testing - -Run the unit test suite (excludes live network/GitHub integration tests): -```bash -uv run pytest deploy/generate_release_notes/tests/ -m "not integration" -v -``` - -Run the full test suite (including live GitHub integration tests): -```bash -uv run pytest deploy/generate_release_notes/tests/ -v -``` +| `datacommonsorg/website` | All PRs (excluding `cdc_data/`) | Core Services (`services`) $\rightarrow$ `gcr.io/datcom-ci/datacommons-services` | +| `datacommonsorg/mixer` | All PRs (`internal/server/`, `proto/`, `deploy/`) | Core Services (`services`) $\rightarrow$ `gcr.io/datcom-ci/datacommons-services` | +| `datacommonsorg/agent-toolkit` | All PRs (`src/datacommons_mcp/`) | Core Services (`services`) $\rightarrow$ `gcr.io/datcom-ci/datacommons-services` | +| `datacommonsorg/import` | `simple/` | Data Preprocessor (`preprocessing`) $\rightarrow$ `gcr.io/datcom-ci/datacommons-data` | +| `datacommonsorg/import` | `pipeline/ingestion/` | Dataflow Worker (`dataflow_worker`) $\rightarrow$ Dataflow Templates | +| `datacommonsorg/import` | `pipeline/workflow/ingestion-helper/` | Ingestion Helper (`ingestion_helper`) $\rightarrow$ `datacommons-ingestion-helper` | +| `datacommonsorg/import` | `pipeline/workflow/aggregation-helper/` | Postprocessing Helper (`postprocessing`) $\rightarrow$ `datacommons-aggregation-helper` | From 3c4c9fbce80bd9f54049429f7ecdbac9a5863898 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 11:10:45 -0700 Subject: [PATCH 47/70] feat(deploy): mandate concurrent subagent spawning via invoke_subagent for PR extraction skills --- deploy/generate_release_notes/SKILL.md | 25 +++++++++---------- .../skills/pr-extraction/SKILL.md | 9 +++++++ 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/deploy/generate_release_notes/SKILL.md b/deploy/generate_release_notes/SKILL.md index 7639c5cc..356480c3 100644 --- a/deploy/generate_release_notes/SKILL.md +++ b/deploy/generate_release_notes/SKILL.md @@ -17,22 +17,21 @@ When the user asks to generate release notes (e.g., *"Generate release notes for 1. Identify the previous release tag (``, e.g., `v1.1.0`) and target release tag (``, e.g., `v1.1.1`). 2. Create the output directory `deploy/generate_release_notes/output/` if it does not exist. -### Step 2: PR Extraction & Image Verification (Subagent Delegation) +### Step 2: PR Extraction & Image Verification (Concurrent Subagent Spawning) 1. Read the **PR Extraction Skill**: [SKILL.md](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/pr-extraction/SKILL.md). -2. Spawn subagent(s) using `invoke_subagent` equipped with `gcloud` and `gh` CLI commands to: +2. **Mandatory Subagent Spawning**: Call `invoke_subagent` to spawn 3 concurrent subagents in parallel: + - **Subagent 1 (`services-extractor`)**: Extract PRs for `website`, `mixer`, `agent-toolkit` $\rightarrow$ write `deploy/generate_release_notes/output/prs_services.txt`. + - **Subagent 2 (`import-extractor`)**: Extract PRs for `import` repo across `simple/`, `pipeline/ingestion/`, and `pipeline/workflow/` $\rightarrow$ write `prs_preprocessing.txt`, `prs_dataflow_worker.txt`, `prs_ingestion_helper.txt`, `prs_postprocessing.txt`. + - **Subagent 3 (`monorepo-extractor`)**: Extract PRs for `datacommons` monorepo & Terraform infra $\rightarrow$ write `prs_dcp_monorepo.txt`. + +3. Each subagent will use `gcloud` and `gh` CLI commands to: - Resolve Artifact Registry image tags (`gcr.io/datcom-ci/datacommons-services`, `datacommons-data`, etc.). - - Execute date-range PR queries (`gh pr list --search "merged:.."`) across all 6 repositories. - - Filter out Dependabot, automated version bumps, and test-only fixtures. + - Execute date-range PR queries (`gh pr list --search "merged:.."`) across their assigned repositories. + - Filter out Dependabot, automated version bumps, and non-production test fixtures. - Deterministically detect intermediate release-window regressions. - - Write verified PR lists to component text files in `deploy/generate_release_notes/output/`: - * `prs_services.txt` (Core Services: Website, Mixer, MCP Agent) - * `prs_preprocessing.txt` (Data Preprocessor: datacommons-data) - * `prs_dataflow_worker.txt` (Dataflow Ingestion Worker) - * `prs_ingestion_helper.txt` (Ingestion Helper Service) - * `prs_postprocessing.txt` (Postprocessing Aggregation Helper) - * `prs_dcp_monorepo.txt` (DCP Monorepo & Terraform Infra) - -3. Inform the developer that PR verification files are written in `deploy/generate_release_notes/output/` for review. + - Output human-verifiable text files per container image into `deploy/generate_release_notes/output/`. + +4. Inform the developer that PR verification files are written in `deploy/generate_release_notes/output/` for review. ### Step 3: Load Domain Context & Architectural Principles 1. Read the **DCP Domain Context Skill**: [SKILL.md](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/dcp-context/SKILL.md). diff --git a/deploy/generate_release_notes/skills/pr-extraction/SKILL.md b/deploy/generate_release_notes/skills/pr-extraction/SKILL.md index ab9909be..f5ad4779 100644 --- a/deploy/generate_release_notes/skills/pr-extraction/SKILL.md +++ b/deploy/generate_release_notes/skills/pr-extraction/SKILL.md @@ -7,6 +7,15 @@ description: Instructions for extracting, filtering, and verifying merged Pull R This skill provides step-by-step instructions for extracting merged Pull Requests across all 6 Data Commons repositories and mapping them to their corresponding container images and components. +## Concurrent Subagent Execution Mandate + +To extract PRs efficiently across all 6 repositories without blocking the main agent context, **you MUST spawn concurrent subagents using `invoke_subagent`**. + +Spawn subagents concurrently to handle component extraction in parallel: +- **Subagent 1 (`services-extractor`)**: Handles `datacommonsorg/website`, `mixer`, `agent-toolkit` $\rightarrow$ writes `prs_services.txt`. +- **Subagent 2 (`import-extractor`)**: Handles `datacommonsorg/import` path rules (`simple/`, `pipeline/ingestion/`, `pipeline/workflow/`) $\rightarrow$ writes `prs_preprocessing.txt`, `prs_dataflow_worker.txt`, `prs_ingestion_helper.txt`, `prs_postprocessing.txt`. +- **Subagent 3 (`monorepo-extractor`)**: Handles `datacommonsorg/datacommons` monorepo & Terraform infra $\rightarrow$ writes `prs_dcp_monorepo.txt`. + --- ## Component & Image Source Rules From c98542e797d5a95cc23aace0f02ced6534ac5afc Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 11:11:32 -0700 Subject: [PATCH 48/70] refactor(deploy): delete Python CLI tool and streamline release notes generator into pure Agentic Skill Suite --- deploy/generate_release_notes/README.md | 25 -- deploy/generate_release_notes/__init__.py | 3 - deploy/generate_release_notes/__main__.py | 20 - deploy/generate_release_notes/config.py | 124 ------ .../feature_extractor.py | 284 ------------- deploy/generate_release_notes/main.py | 263 ------------ deploy/generate_release_notes/models.py | 96 ----- deploy/generate_release_notes/pr_extractor.py | 384 ------------------ .../release_notes_writer.py | 330 --------------- .../generate_release_notes/tests/__init__.py | 1 - .../tests/test_feature_extractor.py | 179 -------- .../generate_release_notes/tests/test_main.py | 93 ----- .../tests/test_pr_extractor.py | 187 --------- .../tests/test_release_notes_writer.py | 240 ----------- pyproject.toml | 8 +- 15 files changed, 1 insertion(+), 2236 deletions(-) delete mode 100644 deploy/generate_release_notes/__init__.py delete mode 100644 deploy/generate_release_notes/__main__.py delete mode 100644 deploy/generate_release_notes/config.py delete mode 100644 deploy/generate_release_notes/feature_extractor.py delete mode 100644 deploy/generate_release_notes/main.py delete mode 100644 deploy/generate_release_notes/models.py delete mode 100644 deploy/generate_release_notes/pr_extractor.py delete mode 100644 deploy/generate_release_notes/release_notes_writer.py delete mode 100644 deploy/generate_release_notes/tests/__init__.py delete mode 100644 deploy/generate_release_notes/tests/test_feature_extractor.py delete mode 100644 deploy/generate_release_notes/tests/test_main.py delete mode 100644 deploy/generate_release_notes/tests/test_pr_extractor.py delete mode 100644 deploy/generate_release_notes/tests/test_release_notes_writer.py diff --git a/deploy/generate_release_notes/README.md b/deploy/generate_release_notes/README.md index 70a993f8..5960f1db 100644 --- a/deploy/generate_release_notes/README.md +++ b/deploy/generate_release_notes/README.md @@ -81,31 +81,6 @@ Defines the non-verbose, partner-facing GFM format: --- -## Standalone CLI Execution (Python Pipeline) - -Alternatively, for non-LLM or CI/CD automated environments, run the standalone Python CLI tool: - -```bash -uv run --group generate-release-notes python -m deploy.generate_release_notes \ - --prev v1.1.0 \ - --new v1.1.1 \ - --allow-missing-images \ - --manifest-out ./output/manifest_v1.1.1.json \ - --out ./output/RELEASE_NOTES_v1.1.1.md -``` - -### CLI Options Reference: -| Flag / Option | Type | Description | -| :--- | :--- | :--- | -| `--prev` | `STRING` | Previous release version tag (e.g. `v1.1.0`). | -| `--new` | `STRING` | Target release version tag (e.g. `v1.1.1`). | -| `--out`, `-o` | `PATH` | Output markdown file path (default: `./RELEASE_NOTES_.md`). | -| `--allow-missing-images` | `BOOLEAN` | Bypass missing container image tag errors during staging. | -| `--manifest-out` | `PATH` | Export JSON manifest mapping all PRs to their container image URIs. | -| `--include-audit-log` | `BOOLEAN` | Append raw PR audit log table at the bottom of the release notes. | - ---- - ## Repository Mapping | Repository | Scope / Path Filter | Target Component & Image | diff --git a/deploy/generate_release_notes/__init__.py b/deploy/generate_release_notes/__init__.py deleted file mode 100644 index af11b22a..00000000 --- a/deploy/generate_release_notes/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Data Commons Platform (DCP) Release Notes Generator Package.""" - -__version__ = "0.1.0" diff --git a/deploy/generate_release_notes/__main__.py b/deploy/generate_release_notes/__main__.py deleted file mode 100644 index 926afb1e..00000000 --- a/deploy/generate_release_notes/__main__.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Entry point when executing package as module: python -m deploy.generate_release_notes.""" - -from deploy.generate_release_notes.main import main - -if __name__ == "__main__": - main() diff --git a/deploy/generate_release_notes/config.py b/deploy/generate_release_notes/config.py deleted file mode 100644 index 6f7a6628..00000000 --- a/deploy/generate_release_notes/config.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Configuration and multi-repository component mappings for DCP release notes generation.""" - -from dataclasses import dataclass, field -from typing import Dict, List, Optional - -DEFAULT_GITHUB_ORG = "datacommonsorg" - - -@dataclass -class SourceRule: - """Source repository rule with optional file path filter.""" - - repo: str - path_filter: Optional[str] = ( - None # If set, only PRs modifying this path match this component - ) - - -@dataclass -class ComponentConfig: - """Configuration for a tracked repository/component in DCP releases.""" - - id: str # Unique key (e.g. 'dcp', 'preprocessing', 'services') - name: str # Human-readable name for release notes - artifact_type: ( - str # 'dcp_platform', 'docker_services', 'docker_data', etc. - ) - image_uri: Optional[str] = None # Primary container image URI - default_tag_prefix: str = "v" # Tag prefix (e.g. 'v' for 'v1.1.2') - sources: List[SourceRule] = field( - default_factory=list - ) # Multi-repo contributing sources - - -# Master registry of all tracked components across Data Commons repositories -COMPONENTS: Dict[str, ComponentConfig] = { - "dcp": ComponentConfig( - id="dcp", - name="DCP Monorepo & Infra (CLI, Admin, DB, Terraform)", - artifact_type="dcp_platform", - image_uri=None, - default_tag_prefix="v", - sources=[ - SourceRule(repo="datacommonsorg/datacommons"), - ], - ), - "services": ComponentConfig( - id="services", - name="Core Services (Website, Mixer, MCP)", - artifact_type="docker_services", - image_uri="gcr.io/datcom-ci/datacommons-services", - default_tag_prefix="v", - sources=[ - SourceRule(repo="datacommonsorg/website"), # All non-cdc_data website PRs - SourceRule(repo="datacommonsorg/mixer"), # All mixer PRs - SourceRule(repo="datacommonsorg/agent-toolkit"), # All MCP PRs - ], - ), - "preprocessing": ComponentConfig( - id="preprocessing", - name="Data Preprocessor (datacommons-data)", - artifact_type="docker_data", - image_uri="gcr.io/datcom-ci/datacommons-data", - default_tag_prefix="v", - sources=[ - SourceRule(repo="datacommonsorg/import", path_filter="simple/"), - SourceRule( - repo="datacommonsorg/website", path_filter="build/cdc_data/" - ), - ], - ), - "dataflow_worker": ComponentConfig( - id="dataflow_worker", - name="Dataflow Ingestion Worker", - artifact_type="dataflow_template", - image_uri="us-docker.pkg.dev/datcom-ci/gcr.io/dataflow-templates/ingestion", - default_tag_prefix="v", - sources=[ - SourceRule( - repo="datacommonsorg/import", path_filter="pipeline/ingestion/" - ), - ], - ), - "ingestion_helper": ComponentConfig( - id="ingestion_helper", - name="Ingestion Helper Service", - artifact_type="docker_helper", - image_uri="gcr.io/datcom-ci/datacommons-ingestion-helper", - default_tag_prefix="v", - sources=[ - SourceRule( - repo="datacommonsorg/import", - path_filter="pipeline/workflow/ingestion-helper/", - ), - ], - ), - "postprocessing": ComponentConfig( - id="postprocessing", - name="Postprocessing Aggregation Helper Service", - artifact_type="docker_helper", - image_uri="gcr.io/datcom-ci/datacommons-aggregation-helper", - default_tag_prefix="v", - sources=[ - SourceRule( - repo="datacommonsorg/import", - path_filter="pipeline/workflow/aggregation-helper/", - ), - ], - ), -} diff --git a/deploy/generate_release_notes/feature_extractor.py b/deploy/generate_release_notes/feature_extractor.py deleted file mode 100644 index f6ed816a..00000000 --- a/deploy/generate_release_notes/feature_extractor.py +++ /dev/null @@ -1,284 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Step 2: Feature Extractor for Data Commons Platform (DCP) release notes generation. - -Synthesizes raw PullRequests into structured FeatureUpdate objects using Gemini LLM, -classifying features into standard SOP categories and handling feature grouping. -""" - -import json -import logging -import os -from typing import Dict, List, Optional, Any - -from google import genai -from google.genai import types - -from deploy.generate_release_notes.models import ( - FeatureUpdate, - PullRequest, - ReleaseInfoManifest, - SOPCategory, -) - -logger = logging.getLogger(__name__) - -DEFAULT_MODEL = "gemini-3.6-flash" - -VALID_SOP_CATEGORIES = {cat.value for cat in SOPCategory} - - -def detect_internal_regressions(prs: List[PullRequest]) -> Dict[str, str]: - """Deterministically identifies PRs that fix regressions introduced by earlier PRs in the same release window. - - Returns a dict mapping pr.qualified_id -> parent_feature_pr.qualified_id. - """ - sorted_prs = sorted(prs, key=lambda p: p.merged_at or "") - file_to_prs: Dict[str, List[PullRequest]] = {} - regression_map: Dict[str, str] = {} - - for pr in sorted_prs: - title_lower = pr.title.lower() - is_fix = any( - w in title_lower - for w in ["fix", "bug", "resolve", "patch", "repair", "correct"] - ) - - if is_fix and pr.files_changed: - matching_earlier_prs = [] - for f in pr.files_changed: - if f in file_to_prs: - for prev_pr in file_to_prs[f]: - if ( - prev_pr.number != pr.number - and prev_pr.repo_name == pr.repo_name - ): - matching_earlier_prs.append(prev_pr) - - if matching_earlier_prs: - parent_pr = matching_earlier_prs[-1] - regression_map[pr.qualified_id] = parent_pr.qualified_id - logger.info( - f"Deterministically detected internal regression: {pr.qualified_id} fixes intermediate PR {parent_pr.qualified_id}" - ) - - for f in pr.files_changed: - if f not in file_to_prs: - file_to_prs[f] = [] - file_to_prs[f].append(pr) - - return regression_map - - -class FeatureExtractor: - """Single-stage Gemini LLM Pipeline for filtering, classifying, and synthesizing DCP release features.""" - - def __init__( - self, - api_key: Optional[str] = None, - model_name: str = DEFAULT_MODEL, - # Backward compatibility aliases - filter_model: Optional[str] = None, - synthesis_model: Optional[str] = None, - ): - key = api_key or os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY") - if not key: - logger.warning( - "Neither GEMINI_API_KEY nor GOOGLE_API_KEY set in environment. Gemini API calls will fail if not authenticated via GCP default credentials." - ) - self.client = genai.Client() - else: - self.client = genai.Client(api_key=key) - - self.model_name = synthesis_model or model_name or DEFAULT_MODEL - - def extract_features( - self, - manifest: ReleaseInfoManifest, - additional_instructions: Optional[str] = None, - ) -> List[FeatureUpdate]: - """Classifies, groups, and synthesizes raw PRs into FeatureUpdate objects in 1 Gemini call.""" - if not manifest.all_pull_requests: - logger.warning("No PRs provided in manifest for feature extraction.") - return [] - - logger.info( - f"Step 2: Extracting features from {len(manifest.all_pull_requests)} PRs using {self.model_name}..." - ) - - # Pre-process PRs to deterministically detect internal regressions within this release window - regression_map = detect_internal_regressions(manifest.all_pull_requests) - - # Build detailed PR context for model with merged_at timestamps and qualified IDs - detailed_prs = [] - for pr in manifest.all_pull_requests: - # Preserve "BREAKING CHANGE:" sections if present in body - body_text = pr.body or "" - body_snippet = body_text[:500] - if "BREAKING CHANGE" in body_text and "BREAKING CHANGE" not in body_snippet: - bc_start = body_text.find("BREAKING CHANGE") - body_snippet += "\n...\n" + body_text[bc_start : bc_start + 300] - - pr_dict = { - "id": pr.qualified_id, - "title": pr.title, - "author": pr.author, - "repo": pr.repo_name, - "url": pr.url, - "merged_at": pr.merged_at, - "target_components": pr.target_components, - "files_changed": pr.files_changed[:10], - "body_summary": body_snippet, - "is_internal_regression": pr.qualified_id in regression_map, - "fixes_intermediate_pr": regression_map.get(pr.qualified_id), - } - detailed_prs.append(pr_dict) - - instructions_context = "" - if additional_instructions or manifest.additional_instructions: - instructions_text = additional_instructions or manifest.additional_instructions - instructions_context = ( - f"\n### Additional User Context & High-Priority Highlights:\n" - f"Note: User instructions have top priority and override default classification or filtering where applicable:\n" - f"{instructions_text}\n" - ) - - prompt = f"""You are an expert Technical Release Manager for the Data Commons Platform (DCP). Your task is to analyze raw PR metadata and draft structured, user-centric release notes for version `{manifest.new_version}` (previous version: `{manifest.previous_version}`). - ---- - -### 1. CONTEXT & DOMAIN KNOWLEDGE -Data Commons Platform (DCP) is a self-hosted, Cloud Spanner-backed deployment of Data Commons. It replaces legacy Bigtable with Cloud Spanner graph tables and vector embeddings. It features custom data ingestion pipelines, specialized serving APIs, and deployment automation across 6 repositories: -1. `datacommonsorg/datacommons` (Monorepo): Terraform modules (`infra/dcp/`, `infra/modules/`), CLI tools (`packages/datacommons-cli/`), and Admin Portal (`packages/datacommons-admin/`). -2. `datacommonsorg/website`: Web application serving UI and APIs (`server/`, `static/`, `build/cdc_services/`, `build/cdc_data/`). -3. `datacommonsorg/mixer`: Core Spanner gRPC graph, StatVar serving engine, and ESPv2 gateway (`internal/server/`, `proto/`, `deploy/helm_charts/`). -4. `datacommonsorg/import`: Data processing pipelines (`simple/`, `pipeline/ingestion/` Dataflow Java worker, `pipeline/workflow/ingestion-helper/`, `pipeline/workflow/aggregation-helper/`). -5. `datacommonsorg/agent-toolkit`: Datacommons Model Context Protocol (MCP) server and tools (`src/datacommons_mcp/`). -6. `datacommonsorg/datacommons-data`: Data preprocessor container image built from `import/simple/` and `website/build/cdc_data/`. - ---- - -### 2. CLASSIFICATION RULES (SOP CATEGORIES) -Categorize EVERY valid feature into EXACTLY ONE of these 4 categories based on its files and description: -* **"Spanner Graph & APIs"**: Features touching SDMX 3.0 REST endpoints, ESPv2 query parameter handling, `/v2/observation` StatVar data retrieval, Spanner gRPC graph serving, `proto/` definitions, or `agent-toolkit` MCP server/tools. -* **"Ingestion & Safety"**: Features touching Dataflow Java worker (`pipeline/ingestion/`), `ingestion-helper`, `aggregation-helper`, Spanner table loading, timestamp bounds, data validation, or health probes. -* **"Search & Website"**: Features touching Spanner vector embeddings (`NodeEmbedding`), private instance `detect-and-fulfill`, Nginx/Envoy, Website UI, or Admin Portal UI. -* **"Infra & Tooling"**: Features touching `infra/dcp/` (Terraform), `datacommons-cli`/`admin` PyPI packages, monorepo root configs, or Cloud Build release pipelines. - ---- - -### 3. STRICT FILTERING & DEDUPLICATION RULES -* **EXCLUDE Internal Dev & Testing PRs**: - * Ignore automated bot PRs (e.g., Dependabot, Renovate, "chore: bump version"). - * Ignore all test-only PRs (e.g., integration test setups, Spanner Omni test conversions, CI sandbox workflows, local test harnesses, hermetic test refactors, and test-only sample data or benchmark fixtures). - * Ignore formatting, typos, or non-informative refactors with zero user impact. -* **EXCLUDE Internal Iteration Bug Fixes (Release Window Regressions)**: - * If a bug fix PR addresses a bug or regression introduced *within this same release window* (i.e. introduced after `{manifest.previous_version}` and fixed before `{manifest.new_version}`), DO NOT list it as a standalone Bug Fix! - * Fold it into the parent `FeatureUpdate` as part of that feature's development, or drop it if it was just an internal dev fix. - * ONLY list bugs under 'Bug Fixes' if the bug was present in `{manifest.previous_version}` or an earlier published release! -* **Deduplicate & Group Related PRs**: - * Combine related PRs (e.g., an initial feature PR + follow-up bug fixes + post-feature adjustments) into a SINGLE cohesive `FeatureUpdate`. - * If PRs conflict or supersede each other, describe ONLY the final chronological state at `{manifest.new_version}`. - ---- - -### 4. WRITING STYLE & PERSONA FOCUS (CONCISE & ANTI-FLUFF) -* **Target Audience**: Write for platform users, data engineers, API consumers, and instance operators building ON TOP of DCP. -* **BANNED AI FLUFF WORDS (STRICT)**: DO NOT use AI cliché words: `seamlessly`, `empower`, `leveraging`, `robust`, `overhaul`, `delivers a major`, `comprehensive`, `fosters`, `game-changing`, `cutting-edge`, `paradigm`. Write simple, direct sentences instead! -* **Concise Sentence Budget**: Keep `description` to 1-2 punchy sentences (max 25 words). Keep `pr_contributions` summaries to 1 short sentence (12-15 words max per PR). -* **User Capability Focus**: Frame the "description" and "pr_contributions" around what the user can *actually do* or what *input formats* are now supported (e.g., "Configure max_workers in Terraform to scale Dataflow workers automatically" instead of "Added Terraform max_workers variable"). -* **Extract Concrete Enums & Configuration Values (STRICT)**: Whenever an update introduces or modifies a configuration variable, CLI flag, environment variable, or Terraform setting, DO NOT summarize it generically. ALWAYS extract and list the specific valid values or enums (e.g., `custom_only`, `base_only`, `base_and_custom`, `--instance_name`, processing unit bounds) and explain the exact behavior or filtering capability each option enables for operators! -* **FOCUS ON EXTERNAL CONTRACTS & CAPABILITIES (ZERO INTERNAL IMPLEMENTATION MECHANICS)**: - - NEVER output feature titles or section names named after internal storage implementations, database table names, schema migrations, or low-level data structures. - - External partners and operators interact with HTTP/gRPC APIs, Terraform modules, and CLI tools — they do not care about internal database tables, cache formats, or storage engine cutovers. - - Frame all storage or performance improvements strictly around user-facing impact (e.g., "API Serving Latency & Query Throughput", "Data Ingestion Speed", "Cache Freshness"). - ---- - -### 5. INPUT DATA -* **Instructions Context**: -{instructions_context} -* **Raw Merged PRs**: -{json.dumps(detailed_prs, indent=2)} - ---- - -### 6. OUTPUT FORMAT -Respond ONLY with a valid JSON array of `FeatureUpdate` objects conforming strictly to the schema below. -Do not include markdown code block formatting (such as ```json) or any conversational text before or after the JSON. Output raw JSON only. - -[ - {{ - "id": "short_unique_snake_case_id", - "title": "Clear Technical Feature Title", - "description": "1-2 sentence technical description of the feature (max 25 words), explaining the change and its user-facing impact.", - "category": "Spanner Graph & APIs | Ingestion & Safety | Search & Website | Infra & Tooling", - "target_components": ["dcp", "services", "preprocessing", "dataflow_worker", "ingestion_helper", "postprocessing"], - "included_prs": ["agent-toolkit#211", "datacommons#189"], - "pr_contributions": {{ - "agent-toolkit#211": "Query bilateral trade and migration relationships between multiple entities", - "datacommons#189": "Configure max_workers in Terraform to scale Dataflow workers automatically for large imports" - }}, - "is_dcp_relevant": true, - "breaking_changes": "Detailed description of the breaking change if any, otherwise null" - }} -] -""" - - try: - config = types.GenerateContentConfig( - response_mime_type="application/json", - temperature=0.2, - ) - res = self.client.models.generate_content( - model=self.model_name, - contents=prompt, - config=config, - ) - raw_features = json.loads(res.text) - features: List[FeatureUpdate] = [] - for item in raw_features: - cat = item.get("category", "Infra & Tooling") - if cat not in VALID_SOP_CATEGORIES: - # Fuzzy match or fallback to Infra & Tooling - matched = False - for valid_cat in VALID_SOP_CATEGORIES: - if valid_cat.lower() in cat.lower() or cat.lower() in valid_cat.lower(): - cat = valid_cat - matched = True - break - if not matched: - cat = SOPCategory.INFRA_TOOLING.value - - feature = FeatureUpdate( - id=item.get("id", f"feature_{len(features)+1}"), - title=item.get("title", "Untitled Feature"), - description=item.get("description", ""), - category=cat, - target_components=item.get("target_components", []), - included_prs=item.get("included_prs", []), - pr_contributions=item.get("pr_contributions", {}), - is_dcp_relevant=item.get("is_dcp_relevant", True), - breaking_changes=item.get("breaking_changes"), - ) - features.append(feature) - - logger.info( - f"Step 2 Complete: Synthesized {len(features)} structured FeatureUpdate objects across categories." - ) - return features - except Exception as e: - logger.error(f"Step 2 Feature extraction failed: {e}") - raise RuntimeError(f"Failed to extract release features with Gemini: {e}") diff --git a/deploy/generate_release_notes/main.py b/deploy/generate_release_notes/main.py deleted file mode 100644 index 9462e352..00000000 --- a/deploy/generate_release_notes/main.py +++ /dev/null @@ -1,263 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""CLI Entry Point for Data Commons Platform (DCP) Release Notes Generator. - -Orchestrates Step 1 (PR Extractor), Step 2 (Feature Extractor), and Step 3 (Release Notes Writer). - -Usage: - uv run --group generate-release-notes python -m deploy.generate_release_notes \\ - --prev v1.1.0 --new v1.1.1 \\ - [--out ./RELEASE_NOTES_v1.1.1.md] \\ - [--additional-instructions ./context.md] \\ - [--allow-missing-images] \\ - [--include-audit-log] -""" - -import json -import logging -import os -import sys -from typing import Optional - -import click - -from deploy.generate_release_notes.feature_extractor import ( - DEFAULT_MODEL as DEFAULT_FEATURE_MODEL, - FeatureExtractor, -) -from deploy.generate_release_notes.pr_extractor import PRExtractor -from deploy.generate_release_notes.release_notes_writer import ( - DEFAULT_WRITER_MODEL, - ReleaseNotesWriter, -) - -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", - datefmt="%H:%M:%S", -) -logger = logging.getLogger("generate_release_notes") - - -@click.command( - help="Generate publication-ready Data Commons Platform (DCP) release notes between two release versions." -) -@click.option( - "--prev", - "prev_version", - required=True, - help="Previous release tag (e.g. v1.1.0 or 1.1.0).", -) -@click.option( - "--new", - "new_version", - required=True, - help="New release tag (e.g. v1.1.1 or 1.1.1).", -) -@click.option( - "--out", - "output_path", - default=None, - help="Output file path for generated Markdown release notes (default: ./RELEASE_NOTES_.md).", -) -@click.option( - "--additional-instructions", - "additional_instructions", - default=None, - help="Path to markdown file or raw text containing additional context, highlights, or custom notes.", -) -@click.option( - "--allow-missing-images", - is_flag=True, - default=False, - help="Bypass missing container image tag errors during staging/testing.", -) -@click.option( - "--include-audit-log", - is_flag=True, - default=False, - help="Append complete raw PR audit log table at the bottom of the release notes.", -) -@click.option( - "--synthesis-model", - "--filter-model", - "synthesis_model", - default=DEFAULT_FEATURE_MODEL, - help=f"Gemini model for Step 2 feature synthesis (default: {DEFAULT_FEATURE_MODEL}).", -) -@click.option( - "--writer-model", - default=DEFAULT_WRITER_MODEL, - help=f"Gemini model for Step 3 release notes writing (default: {DEFAULT_WRITER_MODEL}).", -) -@click.option( - "--manifest-out", - default=None, - help="Optional path to save raw Step 1 ReleaseInfoManifest JSON.", -) -@click.option( - "--features-out", - default=None, - help="Optional path to save synthesized Step 2 FeatureUpdate JSON.", -) -def main( - prev_version: str, - new_version: str, - output_path: Optional[str], - additional_instructions: Optional[str], - allow_missing_images: bool, - include_audit_log: bool, - synthesis_model: str, - writer_model: str, - manifest_out: Optional[str], - features_out: Optional[str], -): - """Executes the 3-step DCP Release Notes Generation Pipeline.""" - # Normalize versions - if not prev_version.startswith("v"): - prev_version = f"v{prev_version}" - if not new_version.startswith("v"): - new_version = f"v{new_version}" - - if not output_path: - output_path = f"./RELEASE_NOTES_{new_version}.md" - - # Read additional instructions file if path provided - instructions_text = None - if additional_instructions: - if os.path.exists(additional_instructions): - with open(additional_instructions, "r") as f: - instructions_text = f.read().strip() - logger.info(f"Loaded additional instructions from {additional_instructions}") - else: - instructions_text = additional_instructions - logger.info("Using inline additional instructions.") - - logger.info("=" * 60) - logger.info(f" Starting DCP Release Notes Generation: {prev_version} -> {new_version}") - logger.info("=" * 60) - - # ---------------------------------------------------- - # STEP 1: Sourcing PRs & Component Version Info - # ---------------------------------------------------- - logger.info("\n--- STEP 1: Sourcing PRs & Resolving Image Tags ---") - pr_extractor = PRExtractor(skip_missing_images=allow_missing_images) - try: - manifest = pr_extractor.extract( - prev_version=prev_version, - new_version=new_version, - additional_instructions=instructions_text, - ) - except Exception as e: - logger.error(f"Step 1 Sourcing Failed: {e}") - sys.exit(1) - - if manifest_out: - manifest_dict = { - "previous_version": manifest.previous_version, - "new_version": manifest.new_version, - "total_prs": len(manifest.all_pull_requests), - "components": { - k: { - "name": v.component_name, - "repo": v.repo_name, - "prev_sha": v.previous_sha, - "new_sha": v.new_sha, - "image_uri": v.image_uri, - "pull_requests_count": len(manifest.pull_requests_by_component.get(k, [])), - "pull_requests": [ - { - "id": pr.qualified_id, - "title": pr.title, - "author": pr.author, - "repo": pr.repo_name, - "url": pr.url, - "merged_at": pr.merged_at, - "files_changed": pr.files_changed, - } - for pr in manifest.pull_requests_by_component.get(k, []) - ], - } - for k, v in manifest.components.items() - }, - } - with open(manifest_out, "w") as f: - json.dump(manifest_dict, f, indent=2) - logger.info(f"Saved manifest with PRs per image to {manifest_out}") - - # ---------------------------------------------------- - # STEP 2: Feature Extraction & SOP Classification - # ---------------------------------------------------- - logger.info("\n--- STEP 2: Feature Extraction & SOP Classification ---") - feature_extractor = FeatureExtractor( - model_name=synthesis_model, - ) - try: - features = feature_extractor.extract_features( - manifest=manifest, - additional_instructions=instructions_text, - ) - except Exception as e: - logger.error(f"Step 2 Feature Extraction Failed: {e}") - sys.exit(1) - - if features_out: - features_dict = [ - { - "id": f.id, - "title": f.title, - "description": f.description, - "category": f.category, - "included_prs": f.included_prs, - "pr_contributions": f.pr_contributions, - "is_dcp_relevant": f.is_dcp_relevant, - } - for f in features - ] - with open(features_out, "w") as f: - json.dump(features_dict, f, indent=2) - logger.info(f"Saved synthesized features to {features_out}") - - # ---------------------------------------------------- - # STEP 3: Release Notes Writing - # ---------------------------------------------------- - logger.info("\n--- STEP 3: Release Notes Writing ---") - writer = ReleaseNotesWriter( - model_name=writer_model, - include_audit_log=include_audit_log, - ) - try: - markdown_notes = writer.render( - manifest=manifest, - features=features, - additional_instructions=instructions_text, - ) - except Exception as e: - logger.error(f"Step 3 Release Notes Writing Failed: {e}") - sys.exit(1) - - # Save output to disk - with open(output_path, "w") as f: - f.write(markdown_notes) - - logger.info("=" * 60) - logger.info(f"🎉 Success! Publication-ready release notes written to: {output_path}") - logger.info(f" - Total PRs Processed: {len(manifest.all_pull_requests)}") - logger.info(f" - Synthesized Feature Updates: {len(features)}") - logger.info("=" * 60) - - -if __name__ == "__main__": - main() diff --git a/deploy/generate_release_notes/models.py b/deploy/generate_release_notes/models.py deleted file mode 100644 index 774584a4..00000000 --- a/deploy/generate_release_notes/models.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Data models for Data Commons Platform (DCP) release notes generation.""" - -from dataclasses import dataclass, field -from enum import Enum -from typing import Dict, List, Optional - - -class SOPCategory(str, Enum): - """Standard SOP Categories for Data Commons Platform Release Notes.""" - - SPANNER_APIS = "Spanner Graph & APIs" - INGESTION_SAFETY = "Ingestion & Safety" - SEARCH_WEBSITE = "Search & Website" - INFRA_TOOLING = "Infra & Tooling" - - -@dataclass -class PullRequest: - """Represents a single merged GitHub Pull Request.""" - - number: int - title: str - body: str - author: str - url: str - merged_at: str - repo_name: str - labels: List[str] = field(default_factory=list) - files_changed: List[str] = field(default_factory=list) - commit_shas: List[str] = field(default_factory=list) - target_components: List[str] = field(default_factory=list) - - @property - def qualified_id(self) -> str: - """Returns a qualified repo#number identifier (e.g. 'datacommons#188' or 'import#42').""" - repo_short = self.repo_name.split("/")[-1] - return f"{repo_short}#{self.number}" - - -@dataclass -class ComponentVersionInfo: - """Version, SHA, and timestamp details for a single component/image.""" - - component_id: str - component_name: str - repo_name: str - previous_version: str - new_version: str - previous_sha: Optional[str] = None - new_sha: Optional[str] = None - prev_timestamp: Optional[str] = None - new_timestamp: Optional[str] = None - image_uri: Optional[str] = None - - -@dataclass -class FeatureUpdate: - """Represents a synthesized feature update combining one or more PRs.""" - - id: str - title: str - description: str - category: str # Must match one of SOPCategory values - target_components: List[str] = field(default_factory=list) - included_prs: List[str] = field(default_factory=list) # Qualified PR IDs e.g. ["datacommons#188"] - pr_contributions: Dict[str, str] = field(default_factory=dict) # Maps PR ID -> Specific contribution summary - is_dcp_relevant: bool = True - breaking_changes: Optional[str] = None - - -@dataclass -class ReleaseInfoManifest: - """Container for all raw sourced information and mapped PRs for a release.""" - - previous_version: str - new_version: str - components: Dict[str, ComponentVersionInfo] = field(default_factory=dict) - pull_requests_by_component: Dict[str, List[PullRequest]] = field( - default_factory=dict - ) - all_pull_requests: List[PullRequest] = field(default_factory=list) - additional_instructions: Optional[str] = None diff --git a/deploy/generate_release_notes/pr_extractor.py b/deploy/generate_release_notes/pr_extractor.py deleted file mode 100644 index e6030d88..00000000 --- a/deploy/generate_release_notes/pr_extractor.py +++ /dev/null @@ -1,384 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Step 1: PR Extractor for Data Commons Platform (DCP) release notes generation. - -Extracts Pull Requests and image mappings across Data Commons repositories using -gcloud container image tags and GitHub CLI (gh pr list). -""" - -from datetime import datetime, timezone -import json -import logging -import subprocess -from typing import Dict, List, Optional, Set, Tuple - -from deploy.generate_release_notes.config import ( - COMPONENTS, - ComponentConfig, - SourceRule, -) -from deploy.generate_release_notes.models import ( - ComponentVersionInfo, - PullRequest, - ReleaseInfoManifest, -) - -logger = logging.getLogger(__name__) - - -def normalize_version(version: str) -> str: - """Strip leading 'v' if present to normalize version string (e.g. 'v1.1.2' -> '1.1.2').""" - return version[1:] if version.startswith("v") else version - - -def format_version_tag(version: str) -> str: - """Ensure leading 'v' is present for Git tags (e.g. '1.1.2' -> 'v1.1.2').""" - return version if version.startswith("v") else f"v{version}" - - -class PRExtractor: - """Extracts all PRs and image mappings between two release versions using gcloud and gh pr list.""" - - def __init__(self, use_cache: bool = True, skip_missing_images: bool = False): - self.use_cache = use_cache - self.skip_missing_images = skip_missing_images - - def resolve_image_tag_info( - self, image_uri: str, version: str - ) -> Optional[Dict]: - """Resolves container image tag in Artifact Registry/GCR via gcloud container images list-tags. - - Returns dict with 'digest', 'tags', and 'timestamp' if found, else None. - """ - raw_version = normalize_version(version) - cmd = [ - "gcloud", - "container", - "images", - "list-tags", - image_uri, - f"--filter=tags={raw_version}", - "--format=json", - ] - try: - res = subprocess.run( - cmd, capture_output=True, text=True, check=True - ) - data = json.loads(res.stdout) - if data and isinstance(data, list) and len(data) > 0: - return data[0] - except Exception as e: - logger.warning( - f"Could not resolve tag '{raw_version}' for image '{image_uri}': {e}" - ) - return None - - def get_git_tag_timestamp( - self, repo: str, version: str - ) -> Optional[Tuple[str, str]]: - """Gets Git commit SHA and ISO timestamp for a git tag via gh api. - - Returns (commit_sha, iso_timestamp) or None. - """ - tag_name = format_version_tag(version) - cmd = [ - "gh", - "api", - f"repos/{repo}/git/matching-refs/tags/{tag_name}", - "--jq", - ".[0].object.sha", - ] - try: - res = subprocess.run( - cmd, capture_output=True, text=True, check=True - ) - sha = res.stdout.strip() - if not sha: - return None - - # Fetch commit details to get timestamp - commit_cmd = [ - "gh", - "api", - f"repos/{repo}/commits/{sha}", - "--jq", - ".commit.committer.date", - ] - commit_res = subprocess.run( - commit_cmd, capture_output=True, text=True, check=True - ) - timestamp = commit_res.stdout.strip() - return sha, timestamp - except Exception as e: - logger.warning( - f"Could not resolve git tag '{tag_name}' for repo '{repo}': {e}" - ) - return None - - def resolve_component_version( - self, comp: ComponentConfig, prev_version: str, new_version: str - ) -> ComponentVersionInfo: - """Resolves version info (SHAs, timestamps, image URI) for a component.""" - info = ComponentVersionInfo( - component_id=comp.id, - component_name=comp.name, - repo_name=comp.sources[0].repo if comp.sources else "", - previous_version=prev_version, - new_version=new_version, - image_uri=comp.image_uri, - ) - - # 1. Image-based component resolution (must find tag in Container Registry) - if comp.image_uri: - prev_data = self.resolve_image_tag_info( - comp.image_uri, prev_version - ) - new_data = self.resolve_image_tag_info(comp.image_uri, new_version) - - if not prev_data: - logger.warning( - f"Container image tag '{prev_version}' not found for {comp.name} at {comp.image_uri}" - ) - else: - info.prev_timestamp = prev_data.get("timestamp", {}).get("datetime") - info.previous_sha = prev_data.get("digest") - - if not new_data: - if self.skip_missing_images: - logger.warning( - f"Skipping missing image component '{comp.name}' ({comp.image_uri}:{new_version}) because skip_missing_images is set." - ) - else: - raise ValueError( - f"Required container image tag '{new_version}' not found for '{comp.name}' at {comp.image_uri}. " - f"Ensure the release container image has been built and tagged before generating release notes, " - f"or pass --skip-missing-images / skip_missing_images=True to bypass." - ) - else: - info.new_timestamp = new_data.get("timestamp", {}).get("datetime") - info.new_sha = new_data.get("digest") - - # 2. Non-image component resolution (e.g. monorepo packages, infra) via Git tags - else: - if comp.sources: - git_prev = self.get_git_tag_timestamp( - comp.sources[0].repo, prev_version - ) - if git_prev: - info.previous_sha, info.prev_timestamp = git_prev - - git_new = self.get_git_tag_timestamp( - comp.sources[0].repo, new_version - ) - if git_new: - info.new_sha, info.new_timestamp = git_new - else: - raise ValueError( - f"Required Git tag '{new_version}' not found for '{comp.name}' in repo '{comp.sources[0].repo}'." - ) - - return info - - def fetch_prs_for_date_range( - self, repo: str, prev_timestamp: str, new_timestamp: str - ) -> List[PullRequest]: - """Fetches all merged PRs for a repository between prev_timestamp and new_timestamp in 1 single call. - - Executes `gh pr list --search 'merged:T_prev..T_new base:main'`. - """ - # Format timestamps for GitHub search API (YYYY-MM-DDTHH:MM:SSZ) - t_prev = ( - prev_timestamp.split(".")[0].replace(" ", "T") - if prev_timestamp - else "2026-01-01T00:00:00Z" - ) - t_new = ( - new_timestamp.split(".")[0].replace(" ", "T") - if new_timestamp - else datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - ) - - if not t_prev.endswith("Z") and "+" not in t_prev and "-" not in t_prev[10:]: - t_prev += "Z" - if not t_new.endswith("Z") and "+" not in t_new and "-" not in t_new[10:]: - t_new += "Z" - - search_query = f"merged:{t_prev}..{t_new}" - cmd = [ - "gh", - "pr", - "list", - "--repo", - repo, - "--state", - "merged", - "--search", - search_query, - "--json", - "number,title,body,author,url,labels,files,mergedAt", - "--limit", - "200", - ] - - logger.info(f"Fetching PRs for {repo} with query '{search_query}'...") - try: - res = subprocess.run( - cmd, capture_output=True, text=True, check=True - ) - raw_prs = json.loads(res.stdout) - prs: List[PullRequest] = [] - for item in raw_prs: - merged_at = item.get("mergedAt", "") - # Strict timestamp check: omit PRs merged after new_timestamp if new_timestamp is set - if new_timestamp and merged_at: - if merged_at > t_new: - continue - if prev_timestamp and merged_at: - if merged_at < t_prev: - continue - - author_login = ( - item.get("author", {}).get("login", "unknown") - if isinstance(item.get("author"), dict) - else "unknown" - ) - labels = [ - l.get("name", "") - for l in item.get("labels", []) - if isinstance(l, dict) - ] - files = [ - f.get("path", "") - for f in item.get("files", []) - if isinstance(f, dict) - ] - - pr = PullRequest( - number=item["number"], - title=item.get("title", ""), - body=item.get("body", ""), - author=author_login, - url=item.get("url", ""), - merged_at=merged_at, - repo_name=repo, - labels=labels, - files_changed=files, - ) - prs.append(pr) - return prs - except Exception as e: - logger.error(f"Failed to fetch PRs for {repo}: {e}") - return [] - - def is_pr_matching_rule(self, pr: PullRequest, rule: SourceRule) -> bool: - """Checks if a PR matches a component's SourceRule (repo name and optional path filter).""" - if pr.repo_name != rule.repo: - return False - - # If no path filter, match all PRs in that repo - if not rule.path_filter: - return True - - # If path filter specified, check if any changed file matches - path = rule.path_filter.rstrip("/") - for f in pr.files_changed: - if f.startswith(path) or f.startswith(f"{path}/"): - return True - return False - - def extract( - self, - prev_version: str, - new_version: str, - additional_instructions: Optional[str] = None, - ) -> ReleaseInfoManifest: - """Main entry point: orchestrates tag resolution, date-range PR fetching, and manifest assembly.""" - logger.info( - f"Starting PR extraction for release range {prev_version} -> {new_version}..." - ) - - manifest = ReleaseInfoManifest( - previous_version=prev_version, - new_version=new_version, - additional_instructions=additional_instructions, - ) - - # 1. Resolve version info and timestamps across all components - repo_timestamps: Dict[str, List[str]] = {} - repo_has_missing_new: Dict[str, bool] = {} - for comp_id, comp in COMPONENTS.items(): - comp_info = self.resolve_component_version( - comp, prev_version, new_version - ) - manifest.components[comp_id] = comp_info - - # Track timestamps per repo to compute widest range - for rule in comp.sources: - if rule.repo not in repo_timestamps: - repo_timestamps[rule.repo] = [] - repo_has_missing_new[rule.repo] = False - if comp_info.prev_timestamp: - repo_timestamps[rule.repo].append(comp_info.prev_timestamp) - if comp_info.new_timestamp: - repo_timestamps[rule.repo].append(comp_info.new_timestamp) - else: - repo_has_missing_new[rule.repo] = True - - # 2. For each repository, compute min & max timestamps and fetch PRs in 1 single call - raw_prs_by_repo: Dict[str, List[PullRequest]] = {} - all_prs_set: Dict[Tuple[str, int], PullRequest] = {} - - for repo, ts_list in repo_timestamps.items(): - if not ts_list: - # Default to fallback timestamp if image tags missing - t_min = "2026-01-01T00:00:00Z" - t_max = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - elif repo_has_missing_new.get(repo, False): - # If new_version tag is missing for this repo, extend t_max to current time NOW - sorted_ts = sorted(ts_list) - t_min = sorted_ts[0] - t_max = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - else: - sorted_ts = sorted(ts_list) - t_min = sorted_ts[0] - t_max = sorted_ts[-1] - - prs = self.fetch_prs_for_date_range(repo, t_min, t_max) - raw_prs_by_repo[repo] = prs - for pr in prs: - all_prs_set[(pr.repo_name, pr.number)] = pr - - # 3. Map PRs to components based on SourceRules - for comp_id, comp in COMPONENTS.items(): - comp_prs: List[PullRequest] = [] - comp_info = manifest.components.get(comp_id) - - for rule in comp.sources: - prs_for_repo = raw_prs_by_repo.get(rule.repo, []) - for pr in prs_for_repo: - if self.is_pr_matching_rule(pr, rule): - # Add target component tag to PR - if comp_id not in pr.target_components: - pr.target_components.append(comp_id) - if pr not in comp_prs: - comp_prs.append(pr) - - manifest.pull_requests_by_component[comp_id] = comp_prs - - manifest.all_pull_requests = list(all_prs_set.values()) - logger.info( - f"Successfully extracted {len(manifest.all_pull_requests)} unique PRs across {len(manifest.components)} components." - ) - return manifest diff --git a/deploy/generate_release_notes/release_notes_writer.py b/deploy/generate_release_notes/release_notes_writer.py deleted file mode 100644 index 754752d8..00000000 --- a/deploy/generate_release_notes/release_notes_writer.py +++ /dev/null @@ -1,330 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Step 3: Release Notes Writer for Data Commons Platform (DCP) release notes generation. - -Uses an Agentic Writer (Gemini Pro) to generate clean, partner-facing, non-technical -release notes following the streamlined Data Commons Platform format. -""" - -import json -import logging -import os -from typing import Any, Dict, List, Optional - -from google import genai -from google.genai import types - -from deploy.generate_release_notes.models import ( - FeatureUpdate, - PullRequest, - ReleaseInfoManifest, -) - -logger = logging.getLogger(__name__) - -DEFAULT_WRITER_MODEL = "gemini-3.6-flash" - - -class ReleaseNotesWriter: - """Agentic Release Notes Writer powered by Gemini Pro.""" - - def __init__( - self, - api_key: Optional[str] = None, - model_name: str = DEFAULT_WRITER_MODEL, - include_audit_log: bool = False, - ): - key = api_key or os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY") - if not key: - logger.warning( - "Neither GEMINI_API_KEY nor GOOGLE_API_KEY set in environment. Gemini API calls will fail if not authenticated via GCP default credentials." - ) - self.client = genai.Client() - else: - self.client = genai.Client(api_key=key) - - self.model_name = model_name - self.include_audit_log = include_audit_log - - def _extract_bug_fixes( - self, manifest: ReleaseInfoManifest, features: List[FeatureUpdate] - ) -> List[Dict[str, Any]]: - """Extracts PRs that are bug fixes or one-off improvements not covered in major features.""" - from deploy.generate_release_notes.feature_extractor import detect_internal_regressions - - included_pr_ids = set() - for feat in features: - included_pr_ids.update(feat.included_prs) - - # Detect internal regressions to exclude intermediate fixes from Bug Fixes - regression_map = detect_internal_regressions(manifest.all_pull_requests) - - bug_fix_prs = [] - for pr in manifest.all_pull_requests: - if pr.qualified_id in included_pr_ids: - continue - - if pr.qualified_id in regression_map: - logger.info( - f"Excluding intermediate regression fix {pr.qualified_id} (fixes {regression_map[pr.qualified_id]}) from public Bug Fixes." - ) - continue - - title_lower = pr.title.lower() - # Identify bug fixes or minor partner-relevant PRs - is_fix = ( - "fix" in title_lower - or "bug" in title_lower - or "resolve" in title_lower - or "correct" in title_lower - or "patch" in title_lower - ) - is_bot = ( - "bump version" in title_lower - or pr.author == "datacommons-robot-author" - or "dependabot" in pr.author.lower() - or "renovate" in pr.author.lower() - ) - - if is_fix and not is_bot: - bug_fix_prs.append( - { - "id": pr.qualified_id, - "title": pr.title, - "author": pr.author, - "repo": pr.repo_name, - "url": pr.url, - "body_snippet": pr.body[:300] if pr.body else "", - } - ) - - return bug_fix_prs - - def build_audit_table_markdown( - self, manifest: ReleaseInfoManifest, features: List[FeatureUpdate] - ) -> str: - """Generates an optional Markdown audit table listing all raw PRs and their status.""" - feature_pr_map: Dict[str, FeatureUpdate] = {} - for feat in features: - for pr_id in feat.included_prs: - feature_pr_map[pr_id] = feat - - rows = [] - for pr in manifest.all_pull_requests: - repo_short = pr.repo_name.split("/")[-1] - title_sanitized = pr.title.replace("|", "\\|").replace("<", "<").replace(">", ">") - - if pr.qualified_id in feature_pr_map: - feat = feature_pr_map[pr.qualified_id] - status = "Substantive Feature" if feat.is_dcp_relevant else "Base DC Only" - category = feat.category - elif "bump version" in pr.title.lower() or "bot" in pr.author.lower(): - status = "Bot Version Bump" - category = "Infra & Tooling" - else: - status = "Refactor / Minor" - category = "Infra & Tooling" - - components_str = ", ".join(pr.target_components) if pr.target_components else "infra" - rows.append( - f"| `{repo_short}` | [{pr.number}]({pr.url}) | {title_sanitized} | @{pr.author} | {components_str} | {category} | {status} |" - ) - - header = ( - "\n---\n\n## Complete Release Audit Log\n\n" - "| Repo | PR # | Title | Author | Components | Category / Type | Status |\n" - "| :--- | :--- | :--- | :--- | :--- | :--- | :--- |\n" - ) - return header + "\n".join(rows) + "\n" - - def render( - self, - manifest: ReleaseInfoManifest, - features: List[FeatureUpdate], - additional_instructions: Optional[str] = None, - release_date: str = "2026-07-29", - ) -> str: - """Calls Gemini Pro to generate publication-ready Markdown release notes according to the streamlined template.""" - logger.info( - f"Step 3: Rendering release notes for {manifest.new_version} using {self.model_name}..." - ) - - # Build payloads for prompt — include ONLY DCP-relevant user/partner features - pr_url_map = {pr.qualified_id: pr.url for pr in manifest.all_pull_requests} - features_payload = [] - for feat in features: - if not feat.is_dcp_relevant: - logger.info(f"Omitting non-DCP-relevant feature from release notes: {feat.title}") - continue - features_payload.append( - { - "id": feat.id, - "title": feat.title, - "description": feat.description, - "category": feat.category, - "target_components": feat.target_components, - "included_prs": feat.included_prs, - "pr_urls": {pr_id: pr_url_map.get(pr_id, "") for pr_id in feat.included_prs}, - "pr_contributions": feat.pr_contributions, - "is_dcp_relevant": feat.is_dcp_relevant, - "breaking_changes": feat.breaking_changes, - } - ) - - bug_fixes_payload = self._extract_bug_fixes(manifest, features) - - instructions_context = "" - if additional_instructions or manifest.additional_instructions: - instructions_text = additional_instructions or manifest.additional_instructions - instructions_context = ( - f"\n#### Additional User Instructions & High-Priority Highlights:\n" - f"{instructions_text}\n" - ) - - prompt = f"""You are an expert Technical Release Manager and Product Documentation Specialist for the Data Commons Platform (DCP). - -Your objective is to generate publication-ready, partner-facing release notes for DCP version `{manifest.new_version}` ({release_date}) using GFM (GitHub Flavored Markdown). - ---- - -### 1. CONTEXT & DOMAIN KNOWLEDGE -Data Commons Platform (DCP) is a self-hosted, Cloud Spanner-backed deployment of Data Commons. It replaces legacy Bigtable with Cloud Spanner graph tables and vector embeddings. It features custom data ingestion pipelines, specialized serving APIs, and deployment automation across 6 repositories: -1. `datacommonsorg/datacommons` (Monorepo): Terraform modules (`infra/dcp/`, `infra/modules/`), CLI tools (`packages/datacommons-cli/`), and Admin Portal (`packages/datacommons-admin/`). -2. `datacommonsorg/website`: Web application serving UI and APIs (`server/`, `static/`, `build/cdc_services/`, `build/cdc_data/`). -3. `datacommonsorg/mixer`: Core Spanner gRPC graph and StatVar serving engine (`internal/server/`, `proto/`, `deploy/helm_charts/`, ESPv2 gateway). -4. `datacommonsorg/agent-toolkit`: Model Context Protocol (MCP) server & FastMCP tools for AI agent integrations (`src/datacommons_mcp/`). -5. `datacommonsorg/import`: Data ingestion preprocessor (`simple/`), Dataflow ingestion worker (`pipeline/ingestion/`), and Cloud Workflow helpers (`pipeline/workflow/ingestion-helper/`, `pipeline/workflow/aggregation-helper/`). - ---- - -### 2. CORE WRITING STYLE & TONE (CONCISE & ANTI-FLUFF) -* **Perspective & Tone**: Write like a senior Google engineer writing a concise technical changelog — direct, factual, punchy, and zero fluff. Use active voice ("You can now...") for features, past tense ("Fixed...") for bugs. -* **Audience Focus**: Write specifically for external developers, data engineers, and instance operators building ON TOP OF the platform. -* **BANNED AI FLUFF WORDS (STRICT)**: DO NOT use AI cliché words: `seamlessly`, `empower`, `leveraging`, `robust`, `overhaul`, `delivers a major`, `comprehensive`, `fosters`, `game-changing`, `cutting-edge`, `paradigm`. Write simple, direct sentences instead! -* **STRICT WORD COUNT BUDGETS**: - * **Executive Summary**: Maximum 25 words (1 single, punchy sentence). - * **What's New**: Combine description and user benefit into 1 concise paragraph (25-35 words max). - * **Specific Capabilities Bullets**: 12-15 words max per bullet. -* **The "So What?" Rule**: Do not just list code changes. Frame every update around user capability (e.g., *what* can the developer do now, *which* inputs are accepted, or *how* does this affect query performance/scalability?). -* **Extract Concrete Enums & Configuration Values (STRICT)**: Whenever an update introduces or modifies a configuration variable, CLI flag, environment variable, or Terraform setting, DO NOT summarize it generically (e.g., "Configured search scope"). ALWAYS extract and list the specific valid values or enums (e.g., `custom_only`, `base_only`, `base_and_custom`, `--instance_name`, processing unit bounds) and explain the exact behavior or filtering capability each option enables for operators! -* **FOCUS ON EXTERNAL CONTRACTS & CAPABILITIES (ZERO INTERNAL IMPLEMENTATION MECHANICS)**: - - NEVER output feature titles or section names named after internal storage implementations, database table names, schema migrations, or low-level data structures. - - External partners and operators interact with HTTP/gRPC APIs, Terraform modules, and CLI tools — they do not care about internal database tables, cache formats, or storage engine cutovers. - - Frame all storage or performance improvements strictly around user-facing impact (e.g., "API Serving Latency & Query Throughput", "Data Ingestion Speed", "Cache Freshness"). - - If a PR modifies an internal storage or cache layer to achieve faster serving, describe the benefit as "Faster API Query Execution & Higher Serving Throughput" without naming internal storage tables or schemas. - ---- - -### 3. STRICT CONSTRAINTS (ZERO-TOLERANCE RULES) -* **NO Emojis**: Do not use emojis anywhere in the document. -* **NO Version/Commit Tables**: Do not include a component version table, git commit hashes, or SHA tables. -* **NO Commit Range Text**: Do not include text like "Release range: v1.1.0 to v1.1.1". -* **NO Horizontal Dividers Between Features**: Do not place horizontal rule lines (`---`) between individual feature sections under Key Feature Updates. Use standard Markdown headers (`### Feature Title`) with single blank lines only! -* **NO Code-Fenced Links**: Every PR reference MUST be a clean, clickable GFM link. Do not wrap backticks around or inside link text. Use the provided full URL for each PR in `pr_urls` or `url` fields! - * **CORRECT**: `[website#123](https://github.com/...)` - * **INCORRECT**: `[`website#123`](https://github.com/...)` or `[`website#123` (https://github.com/...)]` - ---- - -### 4. INPUT MAPPING RULES (RENDER ALL FEATURES) -You will process two payload inputs. Map them to the final release note sections as follows: - -1. **`features_payload`**: - * **STRICT REQUIREMENT**: You MUST render EVERY item provided in `features_payload`. DO NOT drop or omit any feature! - * Major, highly impactful items (e.g. new external API protocols, major component architecture overhauls, core pipeline workflow changes) MUST be rendered as detailed feature sections under **Key Feature Updates**. - * Minor enhancements, optimizations, or configuration instructions must be formatted as concise bullet points under **Improvements & Configuration Updates**. -2. **`bug_fixes_payload`**: - * **HIGH-LEVEL GROUPING (MAX 4-5 BULLETS TOTAL - NO LAUNDRY LIST)**: DO NOT output a laundry list of dozens of individual PRs! - * Aggregate and group all raw bug fixes into 3 to 5 high-impact, functional bullet points (e.g., **Deployment & Infrastructure**, **Ingestion Pipeline Reliability**, **Serving API & Query Robustness**, **Web UI & Visualization**). - * Each grouped bullet must synthesize the common issue and fix in 1-2 concise sentences for the user, linking all relevant PRs together (e.g., `([datacommons#163](https://github.com/...), [datacommons#178](https://github.com/...))`). - * *STRICT EXCLUSION*: Completely ignore internal development chores, test refactors, CI sandbox workflows, local test setups, or unused sample data removals. - ---- - -### 5. DO vs. DON'T CONTENT SAMPLES - -| Section | ❌ DO NOT Write (Internal Developer Focus) | DO Write (Partner & Operator Focus) | -| :--- | :--- | :--- | -| **Key Features** | "Merged PR to implement SDMX 3.0 CSV parser in import repo." | **Import SDMX 3.0 CSV files directly** to ingest standard-compliant macroeconomic datasets into your private instance with zero manual preprocessing. | -| **Improvements** | "Added Terraform variable max_workers." | **Scalable Dataflow Import Pipelines**: Configure `max_workers` in your Terraform configurations to scale compute resources automatically during large-scale imports. | -| **Bug Fixes** | "- Fixed NullPointerException in observation API when entity is empty. [PR 1]
- Fixed timeout in preprocessor polling. [PR 2]
- Fixed IAM count error. [PR 3]" | **Deployment & Infrastructure Reliability**: Resolved Terraform IAM race conditions during fresh deployments and fixed preprocessor polling timeouts during long-running ingestion workflows ([datacommons#163](https://github.com/...), [datacommons#176](https://github.com/...)). | - ---- - -### 6. INPUT DATA - -#### Major Features & Updates: -{json.dumps(features_payload, indent=2)} - -#### Bug Fixes & Refactors: -{json.dumps(bug_fixes_payload, indent=2)} - -{instructions_context} - ---- - -### 7. REQUIRED OUTPUT STRUCTURE -Generate GFM matching the exact structure below. Do not add any greeting, intro, or concluding conversational text outside this structure. - -# Data Commons Platform Release {manifest.new_version} ({release_date}) - -[Provide a high-impact, 1-sentence Executive Summary (max 25 words) highlighting the most important capabilities, performance boosts, and critical fixes introduced in this release for partners and platform operators.] - ---- - -## Key Feature Updates - -### [Feature Title] - -**What's New**: [Clear 1-2 sentence description combining what changed and why it is important / user capability enabled.] - -**Specific Capabilities**: -- [Actionable Use Case / Input Capability 1] ([repo_short#PR](URL)) -- [Actionable Use Case / Input Capability 2] ([repo_short#PR](URL)) - ---- - -## Improvements & Configuration Updates - -- **[Improvement Title]**: [Summary of update, step-by-step configuration instructions if required, and direct benefit] ([repo_short#PR](URL)) - ---- - -## Bug Fixes - -- **[Component / Scope]**: [Description of what was broken, how it was resolved, and how the system behaves now] ([repo_short#PR](URL)) -""" - - try: - config = types.GenerateContentConfig( - temperature=0.2, - ) - res = self.client.models.generate_content( - model=self.model_name, - contents=prompt, - config=config, - ) - markdown_content = res.text.strip() - - # Append optional audit table if requested - if self.include_audit_log: - audit_table_md = self.build_audit_table_markdown(manifest, features) - markdown_content += "\n" + audit_table_md - - logger.info("Step 3 Complete: Release notes successfully generated.") - return markdown_content - except Exception as e: - logger.error(f"Step 3 Release Notes Writer failed with Gemini Pro: {e}") - raise RuntimeError(f"Failed to generate release notes with Gemini Pro: {e}") diff --git a/deploy/generate_release_notes/tests/__init__.py b/deploy/generate_release_notes/tests/__init__.py deleted file mode 100644 index d6f7fe06..00000000 --- a/deploy/generate_release_notes/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Tests package for deploy/generate_release_notes.""" diff --git a/deploy/generate_release_notes/tests/test_feature_extractor.py b/deploy/generate_release_notes/tests/test_feature_extractor.py deleted file mode 100644 index 53c0671d..00000000 --- a/deploy/generate_release_notes/tests/test_feature_extractor.py +++ /dev/null @@ -1,179 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Unit and Integration tests for Feature Extractor (deploy/generate_release_notes/feature_extractor.py).""" - -import json -import os -from unittest.mock import MagicMock, patch -import pytest - -from deploy.generate_release_notes.feature_extractor import FeatureExtractor -from deploy.generate_release_notes.models import ( - FeatureUpdate, - PullRequest, - ReleaseInfoManifest, -) - - -class TestFeatureExtractorUnit: - """Unit tests for FeatureExtractor with mocked Gemini API calls.""" - - @patch("google.genai.Client") - def test_extract_features_single_stage(self, mock_client_cls): - """Test single-stage feature extraction and SOP classification while filtering bot PRs.""" - mock_client = MagicMock() - mock_client_cls.return_value = mock_client - - mock_synthesis_output = [ - { - "id": "ingestion_dataflow_scaling", - "title": "Dataflow Worker Auto-Scaling & Pipeline Safety", - "description": "Configured Dataflow worker auto-scaling via Terraform variables and resolved premature success status marking.", - "category": "Ingestion & Safety", - "target_components": ["dcp", "dataflow_worker"], - "included_prs": ["datacommons#188", "datacommons#189"], - "pr_contributions": { - "datacommons#188": "Removed premature success status set at end of dataflow stage", - "datacommons#189": "Added max_workers Terraform variable for Dataflow auto-scaling", - }, - "is_dcp_relevant": True, - "breaking_changes": None, - } - ] - mock_res = MagicMock() - mock_res.text = json.dumps(mock_synthesis_output) - mock_client.models.generate_content.return_value = mock_res - - pr188 = PullRequest( - number=188, - title="[DCP Ingestion] Remove status set to Success at end of dataflow stage", - body="Fixes status race condition", - author="gmechali", - url="https://github.com/datacommonsorg/datacommons/pull/188", - merged_at="2026-07-24T18:05:13Z", - repo_name="datacommonsorg/datacommons", - files_changed=["infra/dcp/dataflow_job.tf"], - ) - pr189 = PullRequest( - number=189, - title="[DCP Ingestion] Allow dataflow to scale workers based on Terraform Variables", - body="Adds max_workers variable", - author="gmechali", - url="https://github.com/datacommonsorg/datacommons/pull/189", - merged_at="2026-07-24T19:13:17Z", - repo_name="datacommonsorg/datacommons", - files_changed=["infra/dcp/variables.tf"], - ) - pr195_bot = PullRequest( - number=195, - title="chore: bump version to 1.1.1", - body="Automated release bump", - author="datacommons-robot-author", - url="https://github.com/datacommonsorg/datacommons/pull/195", - merged_at="2026-07-29T00:37:18Z", - repo_name="datacommonsorg/datacommons", - files_changed=["VERSION"], - ) - - manifest = ReleaseInfoManifest( - previous_version="v1.1.0", - new_version="v1.1.1", - all_pull_requests=[pr188, pr189, pr195_bot], - ) - - extractor = FeatureExtractor(api_key="mock_key") - features = extractor.extract_features( - manifest=manifest, - additional_instructions="Focus on Dataflow scaling", - ) - - assert len(features) == 1 - feature = features[0] - assert isinstance(feature, FeatureUpdate) - assert feature.id == "ingestion_dataflow_scaling" - assert feature.category == "Ingestion & Safety" - assert feature.included_prs == ["datacommons#188", "datacommons#189"] - assert "datacommons#188" in feature.pr_contributions - assert "datacommons#189" in feature.pr_contributions - assert feature.is_dcp_relevant is True - - -class TestFeatureExtractorIntegration: - """Integration test running FeatureExtractor against real manifest from Step 1 with Gemini API.""" - - @pytest.mark.integration - def test_real_feature_extraction_v1_1_0_to_v1_1_1(self): - """Runs 2-stage Gemini pipeline against real /tmp/test_manifest_v1.1.1.json.""" - if not os.getenv("GEMINI_API_KEY") and not os.getenv("GOOGLE_API_KEY"): - pytest.skip("GEMINI_API_KEY or GOOGLE_API_KEY not set in environment. Skipping real Gemini API integration test.") - - manifest_file = "/tmp/test_manifest_v1.1.1.json" - if not os.path.exists(manifest_file): - pytest.skip(f"Manifest file {manifest_file} not found. Run Step 1 test first.") - - # Re-construct ReleaseInfoManifest from test manifest summary - with open(manifest_file, "r") as f: - manifest_dict = json.load(f) - - prs = [ - PullRequest( - number=item["number"], - title=item["title"], - body="", - author=item.get("author", "unknown"), - url=f"https://github.com/{item['repo']}/pull/{item['number']}", - merged_at="2026-07-25T00:00:00Z", - repo_name=item["repo"], - target_components=item.get("target_components", []), - ) - for item in manifest_dict.get("sample_prs", []) - ] - - manifest = ReleaseInfoManifest( - previous_version=manifest_dict["previous_version"], - new_version=manifest_dict["new_version"], - all_pull_requests=prs, - ) - - extractor = FeatureExtractor() - features = extractor.extract_features( - manifest=manifest, - additional_instructions="Integrate Maps API and Dataflow scaling highlights.", - ) - - assert len(features) > 0 - print(f"\nSuccessfully extracted {len(features)} feature updates using Gemini pipeline:") - for feat in features: - print(f" - [{feat.category}] {feat.title} (PRs: {feat.included_prs})") - - # Save features to /tmp for inspection - output_file = "/tmp/test_features_v1.1.1.json" - features_dict = [ - { - "id": f.id, - "title": f.title, - "description": f.description, - "category": f.category, - "target_components": f.target_components, - "included_prs": f.included_prs, - "is_dcp_relevant": f.is_dcp_relevant, - } - for f in features - ] - with open(output_file, "w") as f: - json.dump(features_dict, f, indent=2) - - print(f"Saved synthesized features summary to {output_file}") - assert os.path.exists(output_file) diff --git a/deploy/generate_release_notes/tests/test_main.py b/deploy/generate_release_notes/tests/test_main.py deleted file mode 100644 index 133a9757..00000000 --- a/deploy/generate_release_notes/tests/test_main.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Unit tests for CLI Entry Point (deploy/generate_release_notes/main.py).""" - -from unittest.mock import MagicMock, patch -from click.testing import CliRunner - -from deploy.generate_release_notes.main import main -from deploy.generate_release_notes.models import ( - FeatureUpdate, - PullRequest, - ReleaseInfoManifest, -) - - -class TestMainCLI: - """Tests for main CLI entry point.""" - - @patch("deploy.generate_release_notes.main.PRExtractor") - @patch("deploy.generate_release_notes.main.FeatureExtractor") - @patch("deploy.generate_release_notes.main.ReleaseNotesWriter") - def test_main_cli_success( - self, mock_writer_cls, mock_feature_cls, mock_pr_cls, tmp_path - ): - """Test end-to-end CLI execution with mocked Step 1, Step 2, and Step 3.""" - # 1. Mock PRExtractor - mock_pr_instance = MagicMock() - mock_pr_cls.return_value = mock_pr_instance - manifest = ReleaseInfoManifest( - previous_version="v1.1.0", - new_version="v1.1.1", - all_pull_requests=[ - PullRequest( - number=188, - title="Remove status set to Success", - body="", - author="gmechali", - url="https://github.com/datacommonsorg/datacommons/pull/188", - merged_at="2026-07-24T18:05:13Z", - repo_name="datacommonsorg/datacommons", - ) - ], - ) - mock_pr_instance.extract.return_value = manifest - - # 2. Mock FeatureExtractor - mock_feature_instance = MagicMock() - mock_feature_cls.return_value = mock_feature_instance - feature = FeatureUpdate( - id="dataflow_fix", - title="Dataflow Fix", - description="Fixed dataflow status set.", - category="Ingestion & Safety", - included_prs=["datacommons#188"], - is_dcp_relevant=True, - ) - mock_feature_instance.extract_features.return_value = [feature] - - # 3. Mock ReleaseNotesWriter - mock_writer_instance = MagicMock() - mock_writer_cls.return_value = mock_writer_instance - mock_writer_instance.render.return_value = "# Data Commons Platform Release v1.1.1 (2026-07-29)\n\nSample release notes." - - out_file = tmp_path / "RELEASE_NOTES_v1.1.1.md" - runner = CliRunner() - result = runner.invoke( - main, - [ - "--prev", - "1.1.0", - "--new", - "1.1.1", - "--out", - str(out_file), - "--allow-missing-images", - ], - ) - - assert result.exit_code == 0 - assert out_file.exists() - assert out_file.read_text() == "# Data Commons Platform Release v1.1.1 (2026-07-29)\n\nSample release notes." diff --git a/deploy/generate_release_notes/tests/test_pr_extractor.py b/deploy/generate_release_notes/tests/test_pr_extractor.py deleted file mode 100644 index 9b67ed17..00000000 --- a/deploy/generate_release_notes/tests/test_pr_extractor.py +++ /dev/null @@ -1,187 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Unit and Integration tests for PR Extractor (deploy/generate_release_notes/pr_extractor.py).""" - -import json -import os -from unittest.mock import MagicMock, patch -import pytest - -from deploy.generate_release_notes.config import COMPONENTS, SourceRule -from deploy.generate_release_notes.models import PullRequest, ReleaseInfoManifest -from deploy.generate_release_notes.pr_extractor import PRExtractor - - -class TestPRExtractorUnit: - """Unit tests for PRExtractor helper functions and SourceRule matching.""" - - def test_source_rule_path_filtering(self): - """Test multi-source path filtering rules across website and import repos.""" - extractor = PRExtractor() - - # 1. Website PR modifying build/cdc_data/ -> Preprocessor - pr_cdc_data = PullRequest( - number=101, - title="Update cdc_data Dockerfile", - body="", - author="testuser", - url="https://github.com/datacommonsorg/website/pull/101", - merged_at="2026-07-20T10:00:00Z", - repo_name="datacommonsorg/website", - files_changed=["build/cdc_data/Dockerfile", "build/cdc_data/run.sh"], - ) - - # 2. Website PR modifying server/ -> Services - pr_website_server = PullRequest( - number=102, - title="Update Flask routes", - body="", - author="testuser", - url="https://github.com/datacommonsorg/website/pull/102", - merged_at="2026-07-21T10:00:00Z", - repo_name="datacommonsorg/website", - files_changed=["server/routes.py", "static/js/app.js"], - ) - - # 3. Import PR modifying simple/ -> Preprocessor - pr_import_simple = PullRequest( - number=201, - title="Update CSV parser in simple importer", - body="", - author="testuser", - url="https://github.com/datacommonsorg/import/pull/201", - merged_at="2026-07-22T10:00:00Z", - repo_name="datacommonsorg/import", - files_changed=["simple/parser.py", "simple/main.go"], - ) - - # 4. Import PR modifying pipeline/workflow/ingestion-helper/ -> Ingestion Helper - pr_import_ingestion = PullRequest( - number=202, - title="Fix ingestion helper Spanner query", - body="", - author="testuser", - url="https://github.com/datacommonsorg/import/pull/202", - merged_at="2026-07-23T10:00:00Z", - repo_name="datacommonsorg/import", - files_changed=["pipeline/workflow/ingestion-helper/main.go"], - ) - - # Rules - rule_prep_website = SourceRule(repo="datacommonsorg/website", path_filter="build/cdc_data/") - rule_services_website = SourceRule(repo="datacommonsorg/website", path_filter=None) - rule_prep_import = SourceRule(repo="datacommonsorg/import", path_filter="simple/") - rule_ingestion_helper = SourceRule(repo="datacommonsorg/import", path_filter="pipeline/workflow/ingestion-helper/") - - # Assertions - assert extractor.is_pr_matching_rule(pr_cdc_data, rule_prep_website) is True - assert extractor.is_pr_matching_rule(pr_website_server, rule_prep_website) is False - assert extractor.is_pr_matching_rule(pr_website_server, rule_services_website) is True - - assert extractor.is_pr_matching_rule(pr_import_simple, rule_prep_import) is True - assert extractor.is_pr_matching_rule(pr_import_ingestion, rule_prep_import) is False - assert extractor.is_pr_matching_rule(pr_import_ingestion, rule_ingestion_helper) is True - - @patch("deploy.generate_release_notes.pr_extractor.subprocess.run") - def test_gcloud_image_tag_resolution(self, mock_run): - """Test resolving container image tags via gcloud list-tags mock.""" - mock_output = [ - { - "digest": "sha256:1234567890abcdef", - "tags": ["1.1.1", "latest"], - "timestamp": {"datetime": "2026-07-15 12:00:00-07:00"}, - } - ] - mock_res = MagicMock() - mock_res.stdout = json.dumps(mock_output) - mock_run.return_value = mock_res - - extractor = PRExtractor() - info = extractor.resolve_image_tag_info("gcr.io/datcom-ci/datacommons-services", "1.1.1") - - assert info is not None - assert info["digest"] == "sha256:1234567890abcdef" - assert info["timestamp"]["datetime"] == "2026-07-15 12:00:00-07:00" - - @patch("deploy.generate_release_notes.pr_extractor.PRExtractor.resolve_image_tag_info") - def test_missing_image_tag_handling(self, mock_resolve): - """Test error raising and bypass flag when container image tag is missing.""" - mock_resolve.return_value = None # Image tag not found - comp = COMPONENTS["services"] - - # 1. Default mode: should raise ValueError - extractor_strict = PRExtractor(skip_missing_images=False) - with pytest.raises(ValueError, match="Required container image tag '9.9.9' not found"): - extractor_strict.resolve_component_version(comp, "1.1.0", "9.9.9") - - # 2. Skip mode: should log warning and not raise error - extractor_permissive = PRExtractor(skip_missing_images=True) - info = extractor_permissive.resolve_component_version(comp, "1.1.0", "9.9.9") - assert info.new_timestamp is None - - -class TestPRExtractorIntegration: - """Integration test executing PRExtractor against real GitHub repositories.""" - - @pytest.mark.integration - def test_real_pr_extraction_v1_1_0_to_v1_1_1(self): - """Extracts PRs between v1.1.0 and v1.1.1 across public Data Commons repos.""" - extractor = PRExtractor() - manifest = extractor.extract( - prev_version="v1.1.0", - new_version="v1.1.1", - additional_instructions="Integration test run for v1.1.0 -> v1.1.1", - ) - - assert isinstance(manifest, ReleaseInfoManifest) - assert manifest.previous_version == "v1.1.0" - assert manifest.new_version == "v1.1.1" - assert len(manifest.components) > 0 - - # Dump manifest to /tmp for schema inspection - output_file = "/tmp/test_manifest_v1.1.1.json" - manifest_dict = { - "previous_version": manifest.previous_version, - "new_version": manifest.new_version, - "total_prs_extracted": len(manifest.all_pull_requests), - "components": { - k: { - "id": v.component_id, - "name": v.component_name, - "prev_timestamp": v.prev_timestamp, - "new_timestamp": v.new_timestamp, - } - for k, v in manifest.components.items() - }, - "pull_requests_count_by_component": { - k: len(v) for k, v in manifest.pull_requests_by_component.items() - }, - "sample_prs": [ - { - "number": pr.number, - "title": pr.title, - "author": pr.author, - "repo": pr.repo_name, - "target_components": pr.target_components, - } - for pr in manifest.all_pull_requests[:10] - ], - } - - with open(output_file, "w") as f: - json.dump(manifest_dict, f, indent=2) - - print(f"\nSaved integration test manifest summary to {output_file}") - assert os.path.exists(output_file) diff --git a/deploy/generate_release_notes/tests/test_release_notes_writer.py b/deploy/generate_release_notes/tests/test_release_notes_writer.py deleted file mode 100644 index 8035e000..00000000 --- a/deploy/generate_release_notes/tests/test_release_notes_writer.py +++ /dev/null @@ -1,240 +0,0 @@ -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Unit and Integration tests for Release Notes Writer (deploy/generate_release_notes/release_notes_writer.py).""" - -import json -import os -from unittest.mock import MagicMock, patch -import pytest - -from deploy.generate_release_notes.models import ( - FeatureUpdate, - PullRequest, - ReleaseInfoManifest, -) -from deploy.generate_release_notes.release_notes_writer import ReleaseNotesWriter - - -class TestReleaseNotesWriterUnit: - """Unit tests for ReleaseNotesWriter with mocked Gemini API calls.""" - - @patch("google.genai.Client") - def test_render_with_mock_gemini(self, mock_client_cls): - """Test ReleaseNotesWriter rendering with mocked Gemini Pro response.""" - mock_client = MagicMock() - mock_client_cls.return_value = mock_client - - mock_markdown_output = """# Data Commons Platform Release v1.1.1 (2026-07-29) - -This release introduces Dataflow worker auto-scaling and Google Maps JavaScript API enablement for interactive map rendering. - ---- - -## Key Feature Updates - -### Dataflow Worker Auto-Scaling & Pipeline Safety -**What's New**: Platform operators can now configure Dataflow worker auto-scaling via Terraform variables. -**Why it Matters**: Improves ingestion throughput while preventing status race conditions. -**Capabilities & Changes**: -- Added `max_workers` Terraform variable ([`datacommons#189`](https://github.com/datacommonsorg/datacommons/pull/189)) -- Removed premature success status setting at end of dataflow stage ([`datacommons#188`](https://github.com/datacommonsorg/datacommons/pull/188)) - ---- - -## Improvements & Configuration Updates - -- **Google Maps API Enablement**: Enabled Google Maps JavaScript and Places APIs by default in Terraform ([`datacommons#198`](https://github.com/datacommonsorg/datacommons/pull/198)) - ---- - -## Bug Fixes - -- **[Services]**: Resolved null pointer exception when querying empty StatVar observations ([`website#145`](https://github.com/datacommonsorg/website/pull/145)) -""" - mock_res = MagicMock() - mock_res.text = mock_markdown_output - mock_client.models.generate_content.return_value = mock_res - - pr188 = PullRequest( - number=188, - title="[DCP Ingestion] Remove status set to Success at end of dataflow stage", - body="Fixes status race condition", - author="gmechali", - url="https://github.com/datacommonsorg/datacommons/pull/188", - merged_at="2026-07-24T18:05:13Z", - repo_name="datacommonsorg/datacommons", - ) - pr189 = PullRequest( - number=189, - title="[DCP Ingestion] Allow dataflow to scale workers based on Terraform Variables", - body="Adds max_workers variable", - author="gmechali", - url="https://github.com/datacommonsorg/datacommons/pull/189", - merged_at="2026-07-24T19:13:17Z", - repo_name="datacommonsorg/datacommons", - ) - - manifest = ReleaseInfoManifest( - previous_version="v1.1.0", - new_version="v1.1.1", - all_pull_requests=[pr188, pr189], - ) - - feature = FeatureUpdate( - id="ingestion_dataflow_scaling", - title="Dataflow Worker Auto-Scaling & Pipeline Safety", - description="Configured Dataflow worker auto-scaling via Terraform variables.", - category="Ingestion & Safety", - target_components=["dcp", "dataflow_worker"], - included_prs=["datacommons#188", "datacommons#189"], - pr_contributions={ - "datacommons#188": "Removed premature success status set at end of dataflow stage", - "datacommons#189": "Added max_workers Terraform variable for Dataflow auto-scaling", - }, - is_dcp_relevant=True, - ) - - writer = ReleaseNotesWriter(api_key="mock_key") - output_md = writer.render(manifest=manifest, features=[feature], release_date="2026-07-29") - - assert "# Data Commons Platform Release v1.1.1 (2026-07-29)" in output_md - assert "## Key Feature Updates" in output_md - assert "## Improvements & Configuration Updates" in output_md - assert "## Bug Fixes" in output_md - assert "Dataflow Worker Auto-Scaling & Pipeline Safety" in output_md - assert "[`datacommons#189`](https://github.com/datacommonsorg/datacommons/pull/189)" in output_md - - def test_build_audit_table_markdown(self): - """Test generating optional Markdown audit table.""" - pr188 = PullRequest( - number=188, - title="Remove status set to Success | Dataflow Fix", - body="", - author="gmechali", - url="https://github.com/datacommonsorg/datacommons/pull/188", - merged_at="2026-07-24T18:05:13Z", - repo_name="datacommonsorg/datacommons", - target_components=["dcp"], - ) - pr195 = PullRequest( - number=195, - title="chore: bump version to 1.1.1", - body="", - author="datacommons-robot-author", - url="https://github.com/datacommonsorg/datacommons/pull/195", - merged_at="2026-07-29T00:37:18Z", - repo_name="datacommonsorg/datacommons", - target_components=["dcp"], - ) - - manifest = ReleaseInfoManifest( - previous_version="v1.1.0", - new_version="v1.1.1", - all_pull_requests=[pr188, pr195], - ) - - feature = FeatureUpdate( - id="ingestion_fix", - title="Dataflow Fix", - description="Fixed dataflow status race condition.", - category="Ingestion & Safety", - included_prs=["datacommons#188"], - is_dcp_relevant=True, - ) - - writer = ReleaseNotesWriter(api_key="mock_key", include_audit_log=True) - table_md = writer.build_audit_table_markdown(manifest=manifest, features=[feature]) - - assert "## Complete Release Audit Log" in table_md - assert "| `datacommons` | [188](https://github.com/datacommonsorg/datacommons/pull/188) | Remove status set to Success \\| Dataflow Fix | @gmechali | dcp | Ingestion & Safety | Substantive Feature |" in table_md - assert "Bot Version Bump" in table_md - - -class TestReleaseNotesWriterIntegration: - """Integration test running ReleaseNotesWriter against real manifest and features from Steps 1 & 2.""" - - @pytest.mark.integration - def test_real_release_notes_generation_v1_1_0_to_v1_1_1(self): - """Runs Gemini Pro Release Notes Writer against real /tmp/test_manifest_v1.1.1.json and /tmp/test_features_v1.1.1.json.""" - if not os.getenv("GEMINI_API_KEY") and not os.getenv("GOOGLE_API_KEY"): - pytest.skip("GEMINI_API_KEY or GOOGLE_API_KEY not set in environment. Skipping real Gemini API integration test.") - - manifest_file = "/tmp/test_manifest_v1.1.1.json" - features_file = "/tmp/test_features_v1.1.1.json" - - if not os.path.exists(manifest_file) or not os.path.exists(features_file): - pytest.skip("Manifest or Features JSON file missing. Run Steps 1 and 2 tests first.") - - with open(manifest_file, "r") as f: - manifest_dict = json.load(f) - - with open(features_file, "r") as f: - features_dict = json.load(f) - - prs = [ - PullRequest( - number=item["number"], - title=item["title"], - body="", - author=item.get("author", "unknown"), - url=f"https://github.com/{item['repo']}/pull/{item['number']}", - merged_at="2026-07-25T00:00:00Z", - repo_name=item["repo"], - target_components=item.get("target_components", []), - ) - for item in manifest_dict.get("sample_prs", []) - ] - - manifest = ReleaseInfoManifest( - previous_version=manifest_dict["previous_version"], - new_version=manifest_dict["new_version"], - all_pull_requests=prs, - ) - - features = [ - FeatureUpdate( - id=item["id"], - title=item["title"], - description=item["description"], - category=item["category"], - target_components=item.get("target_components", []), - included_prs=item.get("included_prs", []), - pr_contributions=item.get("pr_contributions", {}), - is_dcp_relevant=item.get("is_dcp_relevant", True), - ) - for item in features_dict - ] - - writer = ReleaseNotesWriter() - markdown_output = writer.render( - manifest=manifest, - features=features, - additional_instructions="Highlight Google Maps API enablement and Dataflow worker scaling.", - release_date="2026-07-29", - ) - - assert len(markdown_output) > 100 - print(f"\nSuccessfully generated release notes ({len(markdown_output)} chars):") - print("=" * 60) - print(markdown_output[:500] + "\n...\n") - print("=" * 60) - - # Save to /tmp/RELEASE_NOTES_v1.1.1.md - output_file = "/tmp/RELEASE_NOTES_v1.1.1.md" - with open(output_file, "w") as f: - f.write(markdown_output) - - print(f"Saved complete release notes to {output_file}") - assert os.path.exists(output_file) diff --git a/pyproject.toml b/pyproject.toml index 47da7b9f..7e7906e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,12 +23,6 @@ dependencies = [ dev = [ {include-group = "lint"}, {include-group = "test"}, - {include-group = "generate-release-notes"} -] -generate-release-notes = [ - "google-genai>=0.1.0", - "click>=8.1.7", - "jinja2>=3.1.0", ] lint = [ "pre-commit>=4.5.1", @@ -153,7 +147,7 @@ packages = [] [tool.pytest.ini_options] pythonpath = ["."] -testpaths = ["packages/*/tests", "deploy/generate_release_notes/tests"] +testpaths = ["packages/*/tests"] [tool.setuptools.dynamic] version = {file = "VERSION"} From 71f1e6e632ffda46cae2c9feae3e085695eefce0 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 11:15:49 -0700 Subject: [PATCH 49/70] refactor(deploy): clarify Orchestrator vs Subagent skill boundary for PR extraction --- deploy/generate_release_notes/SKILL.md | 72 ++++++++-------- .../skills/pr-extraction/SKILL.md | 82 +++++++++---------- 2 files changed, 76 insertions(+), 78 deletions(-) diff --git a/deploy/generate_release_notes/SKILL.md b/deploy/generate_release_notes/SKILL.md index 356480c3..05528bac 100644 --- a/deploy/generate_release_notes/SKILL.md +++ b/deploy/generate_release_notes/SKILL.md @@ -5,45 +5,47 @@ description: Master orchestrator skill for generating publication-ready, partner # DCP Release Notes Generator (Orchestrator Skill) -This skill orchestrates the end-to-end generation of publication-ready, partner-facing release notes for the Data Commons Platform (DCP). It Coordinates specialized subagents to extract PRs per container image, verify PR lists with the developer in human-readable `.txt` files, apply domain context, and author concise release notes. +This skill orchestrates the end-to-end generation of publication-ready, partner-facing release notes for the Data Commons Platform (DCP). It coordinates specialized subagents to extract PRs per container image, write human-readable verification `.txt` files for developer review, apply domain context, and author concise release notes. + +--- + +## Component & Container Image Registry + +| Component Key | Component Name | Container Image URI / Artifact | Source Repos & Path Filters | Output Verification File | +| :--- | :--- | :--- | :--- | :--- | +| `services` | Core Services (Website, Mixer, MCP Agent) | `gcr.io/datcom-ci/datacommons-services` | `datacommonsorg/website`
`datacommonsorg/mixer`
`datacommonsorg/agent-toolkit` | `output/prs_services.txt` | +| `preprocessing` | Data Preprocessor | `gcr.io/datcom-ci/datacommons-data` | `datacommonsorg/import` (filter: `simple/`) | `output/prs_preprocessing.txt` | +| `dataflow_worker` | Dataflow Ingestion Worker | `us-docker.pkg.dev/datcom-ci/gcr.io/dataflow-templates/ingestion` | `datacommonsorg/import` (filter: `pipeline/ingestion/`) | `output/prs_dataflow_worker.txt` | +| `ingestion_helper` | Ingestion Helper Service | `gcr.io/datcom-ci/datacommons-ingestion-helper` | `datacommonsorg/import` (filter: `pipeline/workflow/ingestion-helper/`) | `output/prs_ingestion_helper.txt` | +| `postprocessing` | Postprocessing Helper Service | `gcr.io/datcom-ci/datacommons-aggregation-helper` | `datacommonsorg/import` (filter: `pipeline/workflow/aggregation-helper/`) | `output/prs_postprocessing.txt` | +| `dcp_monorepo` | DCP Monorepo & Terraform Infra | DCP Monorepo | `datacommonsorg/datacommons` | `output/prs_dcp_monorepo.txt` | --- ## Workflow Instructions -When the user asks to generate release notes (e.g., *"Generate release notes for v1.1.0 to v1.1.1"*): +When requested to generate release notes (e.g., *"Generate release notes for v1.1.0 to v1.1.1"*): -### Step 1: Target Version Resolution +### Step 1: Version Resolution & Output Directory Setup 1. Identify the previous release tag (``, e.g., `v1.1.0`) and target release tag (``, e.g., `v1.1.1`). -2. Create the output directory `deploy/generate_release_notes/output/` if it does not exist. - -### Step 2: PR Extraction & Image Verification (Concurrent Subagent Spawning) -1. Read the **PR Extraction Skill**: [SKILL.md](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/pr-extraction/SKILL.md). -2. **Mandatory Subagent Spawning**: Call `invoke_subagent` to spawn 3 concurrent subagents in parallel: - - **Subagent 1 (`services-extractor`)**: Extract PRs for `website`, `mixer`, `agent-toolkit` $\rightarrow$ write `deploy/generate_release_notes/output/prs_services.txt`. - - **Subagent 2 (`import-extractor`)**: Extract PRs for `import` repo across `simple/`, `pipeline/ingestion/`, and `pipeline/workflow/` $\rightarrow$ write `prs_preprocessing.txt`, `prs_dataflow_worker.txt`, `prs_ingestion_helper.txt`, `prs_postprocessing.txt`. - - **Subagent 3 (`monorepo-extractor`)**: Extract PRs for `datacommons` monorepo & Terraform infra $\rightarrow$ write `prs_dcp_monorepo.txt`. - -3. Each subagent will use `gcloud` and `gh` CLI commands to: - - Resolve Artifact Registry image tags (`gcr.io/datcom-ci/datacommons-services`, `datacommons-data`, etc.). - - Execute date-range PR queries (`gh pr list --search "merged:.."`) across their assigned repositories. - - Filter out Dependabot, automated version bumps, and non-production test fixtures. - - Deterministically detect intermediate release-window regressions. - - Output human-verifiable text files per container image into `deploy/generate_release_notes/output/`. - -4. Inform the developer that PR verification files are written in `deploy/generate_release_notes/output/` for review. - -### Step 3: Load Domain Context & Architectural Principles -1. Read the **DCP Domain Context Skill**: [SKILL.md](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/dcp-context/SKILL.md). -2. Ensure strict adherence to: - - **External Contracts & Operator Capabilities**: Focus strictly on APIs, Terraform variables, CLI commands, and operator capabilities. - - **Zero Internal Implementation Mechanics**: Never output internal database table names (e.g., KeyValueStore, DDLs, schema cutovers). - -### Step 4: Author Publication-Ready Release Notes -1. Read the **Release Writer Skill**: [SKILL.md](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/release-writer/SKILL.md). -2. Synthesize the verified PR lists from `deploy/generate_release_notes/output/prs_*.txt` into the final release notes document: `deploy/generate_release_notes/output/RELEASE_NOTES_.md`. -3. Format according to the streamlined template: - - **Executive Summary**: 1 single sentence (max 25 words). - - **Key Feature Updates**: Non-verbose format (**What's New**: 1 paragraph combining description + benefit, followed by **Specific Capabilities**: bullets with `[repo#PR](URL)` links, NO horizontal rule dividers between features). - - **Improvements & Configuration Updates**: Bullet points extracting concrete enums (`custom_only`, `base_only`) and scaling limits (`max_workers`). - - **Bug Fixes**: 3–5 high-level functional categories (Deployment, Ingestion, Serving APIs, UI). +2. Ensure `deploy/generate_release_notes/output/` directory exists. + +### Step 2: Spawn PR Extraction Subagents +Call `invoke_subagent` to spawn subagents concurrently across the components above. + +Provide each subagent with: +1. The **PR Extraction Skill**: [`deploy/generate_release_notes/skills/pr-extraction/SKILL.md`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/pr-extraction/SKILL.md). +2. Its assigned **Component Name**, **Image URI**, **Source Repos & Path Filters**, ``, ``, and target **Output Verification File**. + +#### Subagent Tasks: +- **Subagent 1 (`services-extractor`)**: Extract PRs for `gcr.io/datcom-ci/datacommons-services` from `website`, `mixer`, `agent-toolkit` $\rightarrow$ write `output/prs_services.txt`. +- **Subagent 2 (`import-extractor`)**: Extract PRs for `import` repo across `simple/`, `pipeline/ingestion/`, `pipeline/workflow/ingestion-helper/`, `pipeline/workflow/aggregation-helper/` $\rightarrow$ write `output/prs_preprocessing.txt`, `output/prs_dataflow_worker.txt`, `output/prs_ingestion_helper.txt`, `output/prs_postprocessing.txt`. +- **Subagent 3 (`monorepo-extractor`)**: Extract PRs for `datacommonsorg/datacommons` monorepo & Terraform infra $\rightarrow$ write `output/prs_dcp_monorepo.txt`. + +### Step 3: Verification Checkpoint +Notify the developer that the PR verification files have been generated under `deploy/generate_release_notes/output/prs_*.txt` for review. + +### Step 4: Apply Domain Context & Author Release Notes +1. Read the **DCP Domain Context Skill**: [`deploy/generate_release_notes/skills/dcp-context/SKILL.md`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/dcp-context/SKILL.md). +2. Read the **Release Writer Skill**: [`deploy/generate_release_notes/skills/release-writer/SKILL.md`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/release-writer/SKILL.md). +3. Author the final release notes from the verified PR text files into: `deploy/generate_release_notes/output/RELEASE_NOTES_.md`. diff --git a/deploy/generate_release_notes/skills/pr-extraction/SKILL.md b/deploy/generate_release_notes/skills/pr-extraction/SKILL.md index f5ad4779..9e22123c 100644 --- a/deploy/generate_release_notes/skills/pr-extraction/SKILL.md +++ b/deploy/generate_release_notes/skills/pr-extraction/SKILL.md @@ -1,78 +1,74 @@ --- name: dcp-pr-extraction -description: Instructions for extracting, filtering, and verifying merged Pull Requests per container image across Data Commons repositories for release notes generation. +description: Subagent instruction skill for extracting, filtering, and verifying merged Pull Requests for assigned container images and repositories. --- -# DCP PR Extraction & Image Verification Skill +# DCP PR Extraction & Image Verification Skill (Subagent Skill) -This skill provides step-by-step instructions for extracting merged Pull Requests across all 6 Data Commons repositories and mapping them to their corresponding container images and components. - -## Concurrent Subagent Execution Mandate - -To extract PRs efficiently across all 6 repositories without blocking the main agent context, **you MUST spawn concurrent subagents using `invoke_subagent`**. - -Spawn subagents concurrently to handle component extraction in parallel: -- **Subagent 1 (`services-extractor`)**: Handles `datacommonsorg/website`, `mixer`, `agent-toolkit` $\rightarrow$ writes `prs_services.txt`. -- **Subagent 2 (`import-extractor`)**: Handles `datacommonsorg/import` path rules (`simple/`, `pipeline/ingestion/`, `pipeline/workflow/`) $\rightarrow$ writes `prs_preprocessing.txt`, `prs_dataflow_worker.txt`, `prs_ingestion_helper.txt`, `prs_postprocessing.txt`. -- **Subagent 3 (`monorepo-extractor`)**: Handles `datacommonsorg/datacommons` monorepo & Terraform infra $\rightarrow$ writes `prs_dcp_monorepo.txt`. +This skill provides step-by-step instructions for an individual subagent to extract merged Pull Requests for its assigned container image(s) and repository path(s), filter noise/regressions, and write a human-readable verification `.txt` file. --- -## Component & Image Source Rules - -| Component Key | Component Name | Container Image URI / Artifact | Source Repos & Path Filters | -| :--- | :--- | :--- | :--- | -| `services` | Core Services (Website, Mixer, MCP Agent) | `gcr.io/datcom-ci/datacommons-services` | `datacommonsorg/website`
`datacommonsorg/mixer`
`datacommonsorg/agent-toolkit` | -| `preprocessing` | Data Preprocessor | `gcr.io/datcom-ci/datacommons-data` | `datacommonsorg/import` (filter: `simple/`) | -| `dataflow_worker` | Dataflow Ingestion Worker | `us-docker.pkg.dev/datcom-ci/gcr.io/dataflow-templates/ingestion` | `datacommonsorg/import` (filter: `pipeline/ingestion/`) | -| `ingestion_helper` | Ingestion Helper Service | `gcr.io/datcom-ci/datacommons-ingestion-helper` | `datacommonsorg/import` (filter: `pipeline/workflow/ingestion-helper/`) | -| `postprocessing` | Postprocessing Helper Service | `gcr.io/datcom-ci/datacommons-aggregation-helper` | `datacommonsorg/import` (filter: `pipeline/workflow/aggregation-helper/`) | -| `dcp` | DCP Monorepo & Terraform Infra | DCP Monorepo | `datacommonsorg/datacommons` | +## Input Parameters Provided by Orchestrator +When invoked, you will receive the following parameters: +- **`component_name`**: Human-readable component name (e.g. `Core Services (Website, Mixer, MCP Agent)`). +- **`image_uri`**: Container Image URI in Artifact Registry (e.g. `gcr.io/datcom-ci/datacommons-services`). +- **`source_repos`**: List of source repositories and path filters to extract PRs from. +- **`prev_version`**: Previous release tag (e.g. `v1.1.0`). +- **`new_version`**: Target release tag (e.g. `v1.1.1`). +- **`output_file`**: Output file path (e.g. `deploy/generate_release_notes/output/prs_services.txt`). --- -## Extraction Steps +## Execution Steps -### 1. Image Tag & Timestamp Resolution -1. For each container image URI above, resolve the creation timestamp of `` and ``: +### 1. Container Image Tag & Timestamp Resolution +1. Resolve the creation timestamp for `` and `` for your assigned `image_uri`: ```bash gcloud container images list-tags --filter="tags:" --format="value(timestamp.datetime)" ``` 2. If an image tag is missing (e.g. during staging before image tagging), set `t_new` to the current time `NOW()`. ### 2. Single Date-Range PR Search per Repository -For each repository, run a single `gh pr list` query spanning `[t_prev .. t_new]`: +For each assigned source repository, execute a single `gh pr list` query spanning `[t_prev .. t_new]`: ```bash gh pr list --repo --state merged --search "merged:.." --json number,title,body,author,url,labels,files,mergedAt --limit 200 ``` *(IMPORTANT: Do NOT pass `base:main` inside `--search`; use `--search "merged:.."` directly to prevent GitHub Search API parse errors!)* -### 3. Intermediate Regression & Noise Filtering +### 3. Path & Directory Filtering +If your assigned component specifies a path filter (e.g. `simple/` for preprocessor, `pipeline/ingestion/` for Dataflow worker, `pipeline/workflow/ingestion-helper/` for ingestion helper): +- Inspect `files[].path` for each PR. +- Keep ONLY PRs that modify files within your assigned path filter! + +### 4. Noise & Intermediate Regression Filtering 1. **Filter Out Bot & Non-Production PRs**: - Exclude Dependabot, Renovate, and automated version bumps (`"bump version"`, `datacommons-robot-author`). - Exclude test-only PRs (unit/integration test harnesses, hermetic test refactors, test-only sample data, or benchmark fixtures). 2. **Filter Out Intermediate Release-Window Regressions**: - - If a bug fix PR addresses a feature or code modified *within the same release window* (`[t_prev..t_new]`), mark it as an internal regression and DO NOT include it in public Bug Fixes! + - Check if a bug fix PR addresses a feature or code modified *within the same release window* (`[t_prev..t_new]`). + - If it fixes an intermediate PR merged earlier in `[t_prev..t_new]`, tag it as an internal regression and DO NOT include it in public Bug Fixes! -### 4. Write Verification Files (`prs_.txt`) -Write clean, human-readable verification text files to `deploy/generate_release_notes/output/`: +### 5. Write Verification File (`prs_.txt`) +Format and write the extracted PRs into your assigned `output_file`: -Format for each file: ``` ================================================================================ -Component: Core Services (Website, Mixer, MCP Agent) -Image URI: gcr.io/datcom-ci/datacommons-services -Release Range: v1.1.0 (2026-06-22) -> v1.1.1 (2026-07-28) -Total PRs: 86 +Component: {component_name} +Image URI: {image_uri} +Release Range: {prev_version} ({t_prev}) -> {new_version} ({t_new}) +Total PRs Extracted: {count} ================================================================================ -[mixer#2027] Support containedInPlace+ expansion in SDMX availability queries -Author: calinc | Merged: 2026-07-23T10:11:31Z -URL: https://github.com/datacommonsorg/mixer/pull/2027 -Files Changed: internal/server/sdmx/availability.go +[{repo_short}#{number}] {title} +Author: {author} | Merged: {merged_at} +URL: {url} +Files Changed: {files_summary} -[agent-toolkit#211] Query bilateral entity observations through get_multi_entity_observations tool -Author: calinc | Merged: 2026-07-21T18:00:00Z -URL: https://github.com/datacommonsorg/agent-toolkit/pull/211 -Files Changed: src/datacommons_mcp/tools.py +[{repo_short}#{number}] {title} +Author: {author} | Merged: {merged_at} +URL: {url} +Files Changed: {files_summary} ``` + +Confirm when your assigned verification file has been written cleanly to `output_file`. From d13b35601047fd99d697a3645b18f0c103d7903c Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 11:16:59 -0700 Subject: [PATCH 50/70] feat(deploy): replace rigid path rules with PR content & semantic relevance analysis in extraction skills --- deploy/generate_release_notes/SKILL.md | 14 +++++++------- .../skills/pr-extraction/SKILL.md | 13 +++++++++---- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/deploy/generate_release_notes/SKILL.md b/deploy/generate_release_notes/SKILL.md index 05528bac..900fcff3 100644 --- a/deploy/generate_release_notes/SKILL.md +++ b/deploy/generate_release_notes/SKILL.md @@ -11,14 +11,14 @@ This skill orchestrates the end-to-end generation of publication-ready, partner- ## Component & Container Image Registry -| Component Key | Component Name | Container Image URI / Artifact | Source Repos & Path Filters | Output Verification File | +| Component Key | Component Name | Container Image URI / Artifact | Source Repos & Content Focus | Output Verification File | | :--- | :--- | :--- | :--- | :--- | -| `services` | Core Services (Website, Mixer, MCP Agent) | `gcr.io/datcom-ci/datacommons-services` | `datacommonsorg/website`
`datacommonsorg/mixer`
`datacommonsorg/agent-toolkit` | `output/prs_services.txt` | -| `preprocessing` | Data Preprocessor | `gcr.io/datcom-ci/datacommons-data` | `datacommonsorg/import` (filter: `simple/`) | `output/prs_preprocessing.txt` | -| `dataflow_worker` | Dataflow Ingestion Worker | `us-docker.pkg.dev/datcom-ci/gcr.io/dataflow-templates/ingestion` | `datacommonsorg/import` (filter: `pipeline/ingestion/`) | `output/prs_dataflow_worker.txt` | -| `ingestion_helper` | Ingestion Helper Service | `gcr.io/datcom-ci/datacommons-ingestion-helper` | `datacommonsorg/import` (filter: `pipeline/workflow/ingestion-helper/`) | `output/prs_ingestion_helper.txt` | -| `postprocessing` | Postprocessing Helper Service | `gcr.io/datcom-ci/datacommons-aggregation-helper` | `datacommonsorg/import` (filter: `pipeline/workflow/aggregation-helper/`) | `output/prs_postprocessing.txt` | -| `dcp_monorepo` | DCP Monorepo & Terraform Infra | DCP Monorepo | `datacommonsorg/datacommons` | `output/prs_dcp_monorepo.txt` | +| `services` | Core Services (Website, Mixer, MCP Agent) | `gcr.io/datcom-ci/datacommons-services` | `datacommonsorg/website`
`datacommonsorg/mixer`
`datacommonsorg/agent-toolkit`
*(Serving APIs, SDMX 3.0, FastMCP, UI)* | `output/prs_services.txt` | +| `preprocessing` | Data Preprocessor | `gcr.io/datcom-ci/datacommons-data` | `datacommonsorg/import`
*(CSV/MCF validation, JSON-LD streaming batching)* | `output/prs_preprocessing.txt` | +| `dataflow_worker` | Dataflow Ingestion Worker | `us-docker.pkg.dev/datcom-ci/gcr.io/dataflow-templates/ingestion` | `datacommonsorg/import`
*(Dataflow pipelines, TFRecord loading, Spanner graph transforms)* | `output/prs_dataflow_worker.txt` | +| `ingestion_helper` | Ingestion Helper Service | `gcr.io/datcom-ci/datacommons-ingestion-helper` | `datacommonsorg/import`
*(Cloud Workflows status tracking, run history tables)* | `output/prs_ingestion_helper.txt` | +| `postprocessing` | Postprocessing Helper Service | `gcr.io/datcom-ci/datacommons-aggregation-helper` | `datacommonsorg/import`
*(Graph postprocessing rollups, StatVar/Place aggregations, summary store)* | `output/prs_postprocessing.txt` | +| `dcp_monorepo` | DCP Monorepo & Terraform Infra | DCP Monorepo | `datacommonsorg/datacommons`
*(Terraform modules, Admin CLI, deployment infra)* | `output/prs_dcp_monorepo.txt` | --- diff --git a/deploy/generate_release_notes/skills/pr-extraction/SKILL.md b/deploy/generate_release_notes/skills/pr-extraction/SKILL.md index 9e22123c..fe6ed0b5 100644 --- a/deploy/generate_release_notes/skills/pr-extraction/SKILL.md +++ b/deploy/generate_release_notes/skills/pr-extraction/SKILL.md @@ -36,10 +36,15 @@ gh pr list --repo --state merged --search "merged:.." ``` *(IMPORTANT: Do NOT pass `base:main` inside `--search`; use `--search "merged:.."` directly to prevent GitHub Search API parse errors!)* -### 3. Path & Directory Filtering -If your assigned component specifies a path filter (e.g. `simple/` for preprocessor, `pipeline/ingestion/` for Dataflow worker, `pipeline/workflow/ingestion-helper/` for ingestion helper): -- Inspect `files[].path` for each PR. -- Keep ONLY PRs that modify files within your assigned path filter! +### 3. PR Content & Component Relevance Analysis +Do NOT rely on rigid directory path matching. Instead, **analyze the actual content of each PR** (title, description body, labels, and changed files) to determine its technical relevance to your assigned component: + +- **Data Preprocessor (`preprocessing` / `datacommons-data`)**: Include PRs whose content affects CSV/MCF parsing, streaming JSON-LD batching, schema validation, column mapping, or preprocessor container execution. +- **Dataflow Ingestion Worker (`dataflow_worker`)**: Include PRs whose content affects Dataflow pipelines, TFRecord loading, BigQuery/Spanner graph transformations, batch import scaling (`max_workers`), or ingestion transforms. +- **Ingestion Helper Service (`ingestion_helper`)**: Include PRs whose content affects Cloud Workflows orchestration, ingestion status tracking, execution IDs, status polling, or run history tables. +- **Postprocessing Helper Service (`postprocessing`)**: Include PRs whose content affects graph postprocessing rollups, StatVar/Place/Entity aggregations, Data-Point Vectors (DPVs), or pre-computed summary stores. +- **Core Services (`services` / `datacommons-services`)**: Include PRs whose content affects serving APIs (Mixer gRPC, SDMX 3.0 REST, MCP agent tools, `/v2/observation`), vector embeddings, or Website Explore UI tools. +- **DCP Monorepo & Infra (`dcp_monorepo`)**: Include PRs whose content affects Terraform modules, Admin CLI tools (`datacommons admin`), or deployment infrastructure. ### 4. Noise & Intermediate Regression Filtering 1. **Filter Out Bot & Non-Production PRs**: From 8dbc1b659d50610973a2b7b14bb99181f0c7e897 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 11:19:11 -0700 Subject: [PATCH 51/70] feat(deploy): provide DCP Context Skill to subagents and add Excluded PRs audit section to prs_*.txt --- deploy/generate_release_notes/SKILL.md | 5 +- .../skills/pr-extraction/SKILL.md | 56 ++++++++++++------- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/deploy/generate_release_notes/SKILL.md b/deploy/generate_release_notes/SKILL.md index 900fcff3..1f40b7f8 100644 --- a/deploy/generate_release_notes/SKILL.md +++ b/deploy/generate_release_notes/SKILL.md @@ -35,11 +35,12 @@ Call `invoke_subagent` to spawn subagents concurrently across the components abo Provide each subagent with: 1. The **PR Extraction Skill**: [`deploy/generate_release_notes/skills/pr-extraction/SKILL.md`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/pr-extraction/SKILL.md). -2. Its assigned **Component Name**, **Image URI**, **Source Repos & Path Filters**, ``, ``, and target **Output Verification File**. +2. The **DCP Context Skill**: [`deploy/generate_release_notes/skills/dcp-context/SKILL.md`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/dcp-context/SKILL.md). +3. Its assigned **Component Name**, **Image URI**, **Source Repos**, ``, ``, and target **Output Verification File**. #### Subagent Tasks: - **Subagent 1 (`services-extractor`)**: Extract PRs for `gcr.io/datcom-ci/datacommons-services` from `website`, `mixer`, `agent-toolkit` $\rightarrow$ write `output/prs_services.txt`. -- **Subagent 2 (`import-extractor`)**: Extract PRs for `import` repo across `simple/`, `pipeline/ingestion/`, `pipeline/workflow/ingestion-helper/`, `pipeline/workflow/aggregation-helper/` $\rightarrow$ write `output/prs_preprocessing.txt`, `output/prs_dataflow_worker.txt`, `output/prs_ingestion_helper.txt`, `output/prs_postprocessing.txt`. +- **Subagent 2 (`import-extractor`)**: Extract PRs for `import` repo for preprocessor, Dataflow worker, ingestion helper, and postprocessing helper $\rightarrow$ write `output/prs_preprocessing.txt`, `output/prs_dataflow_worker.txt`, `output/prs_ingestion_helper.txt`, `output/prs_postprocessing.txt`. - **Subagent 3 (`monorepo-extractor`)**: Extract PRs for `datacommonsorg/datacommons` monorepo & Terraform infra $\rightarrow$ write `output/prs_dcp_monorepo.txt`. ### Step 3: Verification Checkpoint diff --git a/deploy/generate_release_notes/skills/pr-extraction/SKILL.md b/deploy/generate_release_notes/skills/pr-extraction/SKILL.md index fe6ed0b5..bb039364 100644 --- a/deploy/generate_release_notes/skills/pr-extraction/SKILL.md +++ b/deploy/generate_release_notes/skills/pr-extraction/SKILL.md @@ -36,35 +36,37 @@ gh pr list --repo --state merged --search "merged:.." ``` *(IMPORTANT: Do NOT pass `base:main` inside `--search`; use `--search "merged:.."` directly to prevent GitHub Search API parse errors!)* -### 3. PR Content & Component Relevance Analysis -Do NOT rely on rigid directory path matching. Instead, **analyze the actual content of each PR** (title, description body, labels, and changed files) to determine its technical relevance to your assigned component: - -- **Data Preprocessor (`preprocessing` / `datacommons-data`)**: Include PRs whose content affects CSV/MCF parsing, streaming JSON-LD batching, schema validation, column mapping, or preprocessor container execution. -- **Dataflow Ingestion Worker (`dataflow_worker`)**: Include PRs whose content affects Dataflow pipelines, TFRecord loading, BigQuery/Spanner graph transformations, batch import scaling (`max_workers`), or ingestion transforms. -- **Ingestion Helper Service (`ingestion_helper`)**: Include PRs whose content affects Cloud Workflows orchestration, ingestion status tracking, execution IDs, status polling, or run history tables. -- **Postprocessing Helper Service (`postprocessing`)**: Include PRs whose content affects graph postprocessing rollups, StatVar/Place/Entity aggregations, Data-Point Vectors (DPVs), or pre-computed summary stores. -- **Core Services (`services` / `datacommons-services`)**: Include PRs whose content affects serving APIs (Mixer gRPC, SDMX 3.0 REST, MCP agent tools, `/v2/observation`), vector embeddings, or Website Explore UI tools. -- **DCP Monorepo & Infra (`dcp_monorepo`)**: Include PRs whose content affects Terraform modules, Admin CLI tools (`datacommons admin`), or deployment infrastructure. - -### 4. Noise & Intermediate Regression Filtering -1. **Filter Out Bot & Non-Production PRs**: - - Exclude Dependabot, Renovate, and automated version bumps (`"bump version"`, `datacommons-robot-author`). - - Exclude test-only PRs (unit/integration test harnesses, hermetic test refactors, test-only sample data, or benchmark fixtures). -2. **Filter Out Intermediate Release-Window Regressions**: - - Check if a bug fix PR addresses a feature or code modified *within the same release window* (`[t_prev..t_new]`). - - If it fixes an intermediate PR merged earlier in `[t_prev..t_new]`, tag it as an internal regression and DO NOT include it in public Bug Fixes! +### 3. DCP Context & Semantic Content Analysis +1. **Read DCP Context Skill**: Before analyzing PRs, you MUST read [`deploy/generate_release_notes/skills/dcp-context/SKILL.md`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/dcp-context/SKILL.md) to understand how your assigned component fits into the platform architecture. +2. **Analyze PR Content**: Analyze the actual content of each PR (title, description body, labels, and changed code context) against the DCP context to evaluate relevance: + - **Data Preprocessor (`preprocessing` / `datacommons-data`)**: Include PRs affecting CSV/MCF parsing, streaming JSON-LD batching, schema validation, column mapping, or preprocessor execution. + - **Dataflow Ingestion Worker (`dataflow_worker`)**: Include PRs affecting Dataflow pipelines, TFRecord loading, BigQuery/Spanner graph transformations, or batch import scaling (`max_workers`). + - **Ingestion Helper Service (`ingestion_helper`)**: Include PRs affecting Cloud Workflows orchestration, ingestion status tracking, execution IDs, status polling, or run history tables. + - **Postprocessing Helper Service (`postprocessing`)**: Include PRs affecting graph postprocessing rollups, StatVar/Place/Entity aggregations, Data-Point Vectors (DPVs), or pre-computed summary stores. + - **Core Services (`services` / `datacommons-services`)**: Include PRs affecting serving APIs (Mixer gRPC, SDMX 3.0 REST, MCP agent tools, `/v2/observation`), vector embeddings, or Website Explore UI tools. + - **DCP Monorepo & Infra (`dcp_monorepo`)**: Include PRs affecting Terraform modules, Admin CLI tools (`datacommons admin`), or deployment infrastructure. + +### 4. Noise, Base DC, and Regression Categorization +Categorize every PR into either **Relevant PRs** or **Excluded PRs**: +1. **Relevant PRs**: Direct partner/operator features, configuration capabilities, or true platform bug fixes. +2. **Excluded PRs**: + - **Base DC Only / Flag Flips**: PRs that only affect internal Google-hosted Base DC or internal flag flips without platform impact. + - **Intermediate Regressions**: Bug fix PRs that address features/code introduced within the same release window (`[t_prev..t_new]`). + - **Bot & Non-Production Chores**: Dependabot bumps, automated version bumps, unit/integration test harness refactors, or test sample data removals. ### 5. Write Verification File (`prs_.txt`) -Format and write the extracted PRs into your assigned `output_file`: +Format and write the extracted PRs into your assigned `output_file`, including an **Irrelevant / Excluded PRs** section at the bottom for developer audit: ``` ================================================================================ Component: {component_name} Image URI: {image_uri} Release Range: {prev_version} ({t_prev}) -> {new_version} ({t_new}) -Total PRs Extracted: {count} +Total Relevant PRs: {relevant_count} | Total Excluded PRs: {excluded_count} ================================================================================ +--- RELEVANT PRODUCTION PRS --- + [{repo_short}#{number}] {title} Author: {author} | Merged: {merged_at} URL: {url} @@ -74,6 +76,22 @@ Files Changed: {files_summary} Author: {author} | Merged: {merged_at} URL: {url} Files Changed: {files_summary} + +================================================================================ +--- IRRELEVANT / EXCLUDED PRS (AUDIT LOG) --- +================================================================================ + +[{repo_short}#{number}] {title} +Reason: Excluded - Base DC-only flag flip / internal feature toggle +URL: {url} + +[{repo_short}#{number}] {title} +Reason: Excluded - Intermediate regression fix for PR {parent_pr_id} merged in current release window +URL: {url} + +[{repo_short}#{number}] {title} +Reason: Excluded - Unit test harness refactor / test sample data update +URL: {url} ``` Confirm when your assigned verification file has been written cleanly to `output_file`. From 658613cbc0ca4f7c28528174887306ee3d2af09c Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 11:26:42 -0700 Subject: [PATCH 52/70] feat(deploy): add Release Delta Synthesis skill, clean prs_*.txt format (Change Summary + DCP Impact), and add IMAGE_DELTAS.txt pipeline stage --- deploy/generate_release_notes/README.md | 38 +++++---- deploy/generate_release_notes/SKILL.md | 18 +++-- .../skills/pr-extraction/SKILL.md | 23 +++--- .../skills/release-delta-synthesis/SKILL.md | 80 +++++++++++++++++++ 4 files changed, 125 insertions(+), 34 deletions(-) create mode 100644 deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md diff --git a/deploy/generate_release_notes/README.md b/deploy/generate_release_notes/README.md index 5960f1db..1df2494b 100644 --- a/deploy/generate_release_notes/README.md +++ b/deploy/generate_release_notes/README.md @@ -15,11 +15,13 @@ deploy/generate_release_notes/ ├── SKILL.md <-- 1. Master Orchestrator Skill (Entrypoint) ├── skills/ │ ├── pr-extraction/ -│ │ └── SKILL.md <-- 2. PR Extraction & Image Tag Resolution Skill +│ │ └── SKILL.md <-- 2. PR Extraction Skill (Subagent Extraction) +│ ├── release-delta-synthesis/ +│ │ └── SKILL.md <-- 3. Release Delta Synthesis Skill (Image Delta Analysis) │ ├── dcp-context/ -│ │ └── SKILL.md <-- 3. DCP Domain Context & Architectural Map +│ │ └── SKILL.md <-- 4. DCP Domain Context & Architectural Map │ └── release-writer/ -│ └── SKILL.md <-- 4. Partner-Facing Release Notes Writer +│ └── SKILL.md <-- 5. Partner-Facing Release Notes Writer └── output/ <-- Verification & Output Directory ├── prs_services.txt <-- Verified PRs for Core Services (Website, Mixer, MCP) ├── prs_preprocessing.txt <-- Verified PRs for Data Preprocessor (datacommons-data) @@ -27,6 +29,7 @@ deploy/generate_release_notes/ ├── prs_ingestion_helper.txt <-- Verified PRs for Ingestion Helper ├── prs_postprocessing.txt <-- Verified PRs for Postprocessing Helper ├── prs_dcp_monorepo.txt <-- Verified PRs for DCP Monorepo & Infra + ├── IMAGE_DELTAS_v1.1.1.txt <-- Intermediate Image Delta Summary (Delta vs. Previous Release) └── RELEASE_NOTES_v1.1.1.md <-- Final Publication-Ready Release Notes ``` @@ -41,22 +44,23 @@ To generate release notes for a release range, point your LLM agent at [`deploy/ > *"Please read `deploy/generate_release_notes/SKILL.md` and generate release notes for version v1.1.0 to v1.1.1."* ### Step 2: PR Extraction & Intermediate Verification Files -Your LLM agent will follow the PR extraction skill (`skills/pr-extraction/SKILL.md`) to: +Your LLM agent will spawn concurrent subagents using `skills/pr-extraction/SKILL.md` and `skills/dcp-context/SKILL.md` to: 1. Resolve container image tags across Artifact Registry via `gcloud`. 2. Query merged Pull Requests across all 6 Data Commons repositories via `gh pr list`. -3. Filter out non-production test fixtures and intermediate release-window regressions. -4. Output human-verifiable text files per container image into `deploy/generate_release_notes/output/`: - - `prs_services.txt` (Core Services: Website, Mixer, MCP Agent) - - `prs_preprocessing.txt` (Data Preprocessor: `datacommons-data`) - - `prs_dataflow_worker.txt` (Dataflow Ingestion Worker) - - `prs_ingestion_helper.txt` (Ingestion Helper Service) - - `prs_postprocessing.txt` (Postprocessing Aggregation Helper) - - `prs_dcp_monorepo.txt` (DCP Monorepo & Terraform Infra) - -Developers can open and inspect these `.txt` files to verify that all relevant PRs for each image are correctly captured before the final release notes are written. - -### Step 3: Domain Context & Final Writing -Your LLM agent will load the domain context (`skills/dcp-context/SKILL.md`) and author the final release notes according to `skills/release-writer/SKILL.md`, outputting: +3. Analyze PR content, change summary, and DCP impact. +4. Output human-verifiable text files per container image into `deploy/generate_release_notes/output/prs_*.txt`. + +Developers can open and inspect these `.txt` files to verify that all relevant PRs for each image are correctly captured, and inspect the **Irrelevant / Excluded PRs** audit log at the bottom. + +### Step 3: Release Delta Synthesis (Delta vs. Previously Published Image) +Your LLM agent will spawn a **Release Delta Synthesis Subagent** using `skills/release-delta-synthesis/SKILL.md` to: +1. Analyze all `output/prs_*.txt` files. +2. Distinguish true platform bug fixes present in `` vs. intermediate bug fixes introduced and resolved within `` (omitting intra-release fixes). +3. Summarize salient features and operator capabilities added to each container image relative to ``. +4. Output the intermediate delta summary file: `deploy/generate_release_notes/output/IMAGE_DELTAS_.txt`. + +### Step 4: Final Release Notes Authoring +Your LLM agent will load `skills/release-writer/SKILL.md` and author the publication-ready release notes: `deploy/generate_release_notes/output/RELEASE_NOTES_.md` --- diff --git a/deploy/generate_release_notes/SKILL.md b/deploy/generate_release_notes/SKILL.md index 1f40b7f8..bf435908 100644 --- a/deploy/generate_release_notes/SKILL.md +++ b/deploy/generate_release_notes/SKILL.md @@ -43,10 +43,18 @@ Provide each subagent with: - **Subagent 2 (`import-extractor`)**: Extract PRs for `import` repo for preprocessor, Dataflow worker, ingestion helper, and postprocessing helper $\rightarrow$ write `output/prs_preprocessing.txt`, `output/prs_dataflow_worker.txt`, `output/prs_ingestion_helper.txt`, `output/prs_postprocessing.txt`. - **Subagent 3 (`monorepo-extractor`)**: Extract PRs for `datacommonsorg/datacommons` monorepo & Terraform infra $\rightarrow$ write `output/prs_dcp_monorepo.txt`. -### Step 3: Verification Checkpoint -Notify the developer that the PR verification files have been generated under `deploy/generate_release_notes/output/prs_*.txt` for review. - -### Step 4: Apply Domain Context & Author Release Notes +### Step 3: Verification Checkpoint & Release Delta Synthesis Subagent +1. Notify the developer that raw PR verification files have been generated under `deploy/generate_release_notes/output/prs_*.txt` for review. +2. Call `invoke_subagent` to spawn a specialized **Release Delta Synthesis Subagent** (`delta-synthesizer`). +3. Provide the subagent with the **Release Delta Synthesis Skill**: [`deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md). +4. The subagent will: + - Read all `output/prs_*.txt` files. + - Investigate and distinguish true bug fixes present in `` vs. intermediate bug fixes introduced and fixed within `` (omitting intra-release fixes). + - Summarize salient features and configuration updates per container image relative to ``. + - Output the unified image delta summary to: `deploy/generate_release_notes/output/IMAGE_DELTAS_.txt`. + +### Step 4: Author Publication-Ready Release Notes 1. Read the **DCP Domain Context Skill**: [`deploy/generate_release_notes/skills/dcp-context/SKILL.md`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/dcp-context/SKILL.md). 2. Read the **Release Writer Skill**: [`deploy/generate_release_notes/skills/release-writer/SKILL.md`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/release-writer/SKILL.md). -3. Author the final release notes from the verified PR text files into: `deploy/generate_release_notes/output/RELEASE_NOTES_.md`. +3. Read `deploy/generate_release_notes/output/IMAGE_DELTAS_.txt`. +4. Author the final release notes from the verified image delta summary into: `deploy/generate_release_notes/output/RELEASE_NOTES_.md`. diff --git a/deploy/generate_release_notes/skills/pr-extraction/SKILL.md b/deploy/generate_release_notes/skills/pr-extraction/SKILL.md index bb039364..161686e6 100644 --- a/deploy/generate_release_notes/skills/pr-extraction/SKILL.md +++ b/deploy/generate_release_notes/skills/pr-extraction/SKILL.md @@ -55,7 +55,11 @@ Categorize every PR into either **Relevant PRs** or **Excluded PRs**: - **Bot & Non-Production Chores**: Dependabot bumps, automated version bumps, unit/integration test harness refactors, or test sample data removals. ### 5. Write Verification File (`prs_.txt`) -Format and write the extracted PRs into your assigned `output_file`, including an **Irrelevant / Excluded PRs** section at the bottom for developer audit: +Format and write the extracted PRs into your assigned `output_file`, including an **Irrelevant / Excluded PRs** section at the bottom for developer audit. + +For each relevant PR, provide: +1. **Change Summary**: Concise description of what changed in the code. +2. **DCP Impact**: Direct impact on platform operators, developers, or end-users. ``` ================================================================================ @@ -67,15 +71,13 @@ Total Relevant PRs: {relevant_count} | Total Excluded PRs: {excluded_count} --- RELEVANT PRODUCTION PRS --- -[{repo_short}#{number}] {title} -Author: {author} | Merged: {merged_at} -URL: {url} -Files Changed: {files_summary} +[{repo_short}#{number}] {title} (Author: {author} | Merged: {merged_at}) +- Change Summary: {1-2 sentence summary of what changed in this PR} +- DCP Impact: {1-2 sentence explanation of user capability, API contract, or operator benefit} -[{repo_short}#{number}] {title} -Author: {author} | Merged: {merged_at} -URL: {url} -Files Changed: {files_summary} +[{repo_short}#{number}] {title} (Author: {author} | Merged: {merged_at}) +- Change Summary: {1-2 sentence summary of what changed in this PR} +- DCP Impact: {1-2 sentence explanation of user capability, API contract, or operator benefit} ================================================================================ --- IRRELEVANT / EXCLUDED PRS (AUDIT LOG) --- @@ -83,15 +85,12 @@ Files Changed: {files_summary} [{repo_short}#{number}] {title} Reason: Excluded - Base DC-only flag flip / internal feature toggle -URL: {url} [{repo_short}#{number}] {title} Reason: Excluded - Intermediate regression fix for PR {parent_pr_id} merged in current release window -URL: {url} [{repo_short}#{number}] {title} Reason: Excluded - Unit test harness refactor / test sample data update -URL: {url} ``` Confirm when your assigned verification file has been written cleanly to `output_file`. diff --git a/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md b/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md new file mode 100644 index 00000000..7edf4a31 --- /dev/null +++ b/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md @@ -0,0 +1,80 @@ +--- +name: dcp-release-delta-synthesis +description: Subagent skill for analyzing raw PR verification files (prs_*.txt) and synthesizing image-level release deltas relative to the previously published release image. +--- + +# DCP Release Delta Synthesis Skill (Image Impact & Delta Analysis) + +This skill provides step-by-step instructions for a subagent to analyze all raw PR verification files (`deploy/generate_release_notes/output/prs_*.txt`), evaluate component-level changes relative to the previously published container image, filter out intra-release intermediate bug fixes, and synthesize a unified `IMAGE_DELTAS_.txt` document. + +--- + +## Input & Output Files + +- **Input Files**: `deploy/generate_release_notes/output/prs_*.txt` (`prs_services.txt`, `prs_preprocessing.txt`, `prs_dataflow_worker.txt`, `prs_ingestion_helper.txt`, `prs_postprocessing.txt`, `prs_dcp_monorepo.txt`). +- **Context Reference**: [`deploy/generate_release_notes/skills/dcp-context/SKILL.md`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/dcp-context/SKILL.md). +- **Target Output File**: `deploy/generate_release_notes/output/IMAGE_DELTAS_.txt`. + +--- + +## Execution Steps + +### 1. Read & Consolidate PR Verification Files +1. Read all `prs_*.txt` files from `deploy/generate_release_notes/output/`. +2. Group PRs by container image / component key (`services`, `preprocessing`, `dataflow_worker`, `ingestion_helper`, `postprocessing`, `dcp_monorepo`). + +### 2. Intra-Release vs. Prior-Release Bug Fix Investigation +Perform deep investigation into every bug fix PR: +- **Intra-Release Intermediate Fix (EXCLUDE)**: Was the bug introduced by a feature/PR *merged within this current release window* (`[v_prev..v_new]`)? If so, exclude it! Platform users running `v_prev` were never exposed to this bug, so listing it creates noise. +- **Prior-Release True Fix (INCLUDE)**: Did the bug exist in the previously published container image (`v_prev` or earlier)? If so, synthesize it under true platform bug fixes for that component! + +### 3. Synthesize Salient Image Deltas (Per Container Image) +For each container image / component, summarize the salient changes from the perspective of an operator upgrading from `` to ``: + +1. **Major Feature Capabilities Added**: + - What new capabilities exist in this image that were not present in ``? + - What new API endpoints, protocols (e.g. SDMX 3.0, FastMCP), or UI tools are now available? +2. **Configuration & Infra Updates**: + - What new Terraform variables, CLI parameters (`--instance_name`), or environment variables were added? + - What scaling bounds (`max_workers`, BigQuery slots) or memory optimizations were introduced? +3. **True Platform Bug Fixes**: + - What issues present in `` were resolved in this container image? + +--- + +## Output Document Structure (`IMAGE_DELTAS_.txt`) + +Write `deploy/generate_release_notes/output/IMAGE_DELTAS_.txt` using the exact structure below: + +``` +================================================================================ +DCP RELEASE DELTA SUMMARY: {prev_version} -> {new_version} +Generated Date: {date} +================================================================================ + +-------------------------------------------------------------------------------- +1. COMPONENT: Core Services (Website, Mixer, MCP Agent) + Container Image: gcr.io/datcom-ci/datacommons-services +-------------------------------------------------------------------------------- + +SALIENT FEATURES & CAPABILITIES (vs. {prev_version}): +- SDMX 3.0 REST Data & Availability Endpoints: Serves multi-entity observations in SDMX-CSV 2.0 format with facetId filtering and containedInPlace+ expansion ([mixer#1976], [mixer#1988], [mixer#2000]). +- FastMCP Agent Integration: Exposes get_multi_entity_observations tool and indicator search with custom_only/base_only target scopes ([agent-toolkit#211], [agent-toolkit#212]). + +CONFIGURATION & OPERATOR UPDATES: +- Vector Search Profiles: Support custom embedding profiles via --spanner_search_config_path ([mixer#2039]). + +TRUE PLATFORM BUG FIXES (Fixes issues present in {prev_version}): +- Serving SQL Optimization: Unroll SQL array parameters for size <= 10 to resolve latency spikes ([mixer#1993]). +- Place Browser Duplication: Fixed duplicate place rendering when multiple provenances exist ([website#6474]). + +[INTRA-RELEASE FIXES EXCLUDED FROM PUBLIC NOTES: mixer#2025, mixer#2070] + +-------------------------------------------------------------------------------- +2. COMPONENT: Data Preprocessor + Container Image: gcr.io/datcom-ci/datacommons-data +-------------------------------------------------------------------------------- +... +``` + +Confirm when `IMAGE_DELTAS_.txt` has been written cleanly. From 40024056e60a542ef3bb5054314110b0e2682573 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 11:28:41 -0700 Subject: [PATCH 53/70] feat(deploy): separate Website, Mixer, and MCP Agent Toolkit into distinct component sections in Release Delta Synthesis skill --- .../skills/release-delta-synthesis/SKILL.md | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md b/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md index 7edf4a31..7ebf3200 100644 --- a/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md +++ b/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md @@ -42,6 +42,21 @@ For each container image / component, summarize the salient changes from the per --- +## Component & Service Separation Rules + +Do NOT club Website, Mixer, and MCP Agent Toolkit together in the output synthesis. Separate them into distinct, dedicated component sections so developers and operators can clearly see changes per layer: + +1. **`Mixer Serving Engine & SDMX APIs`** (`datacommonsorg/mixer`): Core gRPC serving engine, SDMX 3.0 REST Data/Availability endpoints, `/v2/observation`, vector search indexing, SQL query planner optimizations. +2. **`MCP Agent Toolkit & FastMCP Tools`** (`datacommonsorg/agent-toolkit`): FastMCP tools, `get_multi_entity_observations`, indicator search tools, target scope resolution (`custom_only`, `base_only`). +3. **`Website UI & Exploration Tools`** (`datacommonsorg/website`): Explore UI, Download Tool, Place Browser, Croissant dataset metadata, web server routing and caching. +4. **`Data Preprocessor`** (`datacommonsorg/import` - `datacommons-data`): CSV/MCF validation, 10k-node streaming JSON-LD batching, namespace mapping. +5. **`Dataflow Ingestion Worker`** (`datacommonsorg/import` - Dataflow Templates): TFRecord loading, Spanner graph transformations, `max_workers` auto-scaling. +6. **`Ingestion Helper Service`** (`datacommonsorg/import` - `datacommons-ingestion-helper`): Cloud Workflows status tracking, execution IDs, history tables. +7. **`Postprocessing Aggregation Helper`** (`datacommonsorg/import` - `datacommons-aggregation-helper`): StatVar/Place/Entity rollups, summary store, DPV aggregations. +8. **`DCP Monorepo & Infrastructure`** (`datacommonsorg/datacommons`): Terraform modules, Admin CLI (`datacommons admin`), Cloud Run job orchestration. + +--- + ## Output Document Structure (`IMAGE_DELTAS_.txt`) Write `deploy/generate_release_notes/output/IMAGE_DELTAS_.txt` using the exact structure below: @@ -53,25 +68,46 @@ Generated Date: {date} ================================================================================ -------------------------------------------------------------------------------- -1. COMPONENT: Core Services (Website, Mixer, MCP Agent) - Container Image: gcr.io/datcom-ci/datacommons-services +1. COMPONENT: Mixer Serving Engine & SDMX APIs (datacommonsorg/mixer) + Container Image: gcr.io/datcom-ci/datacommons-services (Mixer binary) -------------------------------------------------------------------------------- SALIENT FEATURES & CAPABILITIES (vs. {prev_version}): - SDMX 3.0 REST Data & Availability Endpoints: Serves multi-entity observations in SDMX-CSV 2.0 format with facetId filtering and containedInPlace+ expansion ([mixer#1976], [mixer#1988], [mixer#2000]). -- FastMCP Agent Integration: Exposes get_multi_entity_observations tool and indicator search with custom_only/base_only target scopes ([agent-toolkit#211], [agent-toolkit#212]). CONFIGURATION & OPERATOR UPDATES: - Vector Search Profiles: Support custom embedding profiles via --spanner_search_config_path ([mixer#2039]). TRUE PLATFORM BUG FIXES (Fixes issues present in {prev_version}): - Serving SQL Optimization: Unroll SQL array parameters for size <= 10 to resolve latency spikes ([mixer#1993]). -- Place Browser Duplication: Fixed duplicate place rendering when multiple provenances exist ([website#6474]). [INTRA-RELEASE FIXES EXCLUDED FROM PUBLIC NOTES: mixer#2025, mixer#2070] -------------------------------------------------------------------------------- -2. COMPONENT: Data Preprocessor +2. COMPONENT: MCP Agent Toolkit & FastMCP Tools (datacommonsorg/agent-toolkit) + Container Image: gcr.io/datcom-ci/datacommons-services (MCP binary) +-------------------------------------------------------------------------------- + +SALIENT FEATURES & CAPABILITIES (vs. {prev_version}): +- FastMCP Agent Integration: Exposes get_multi_entity_observations tool and indicator search with custom_only/base_only target scopes ([agent-toolkit#211], [agent-toolkit#212]). + +TRUE PLATFORM BUG FIXES (Fixes issues present in {prev_version}): +- Agent API Protocol: Updated V2AgentGetObservations to HTTP POST for large payload handling ([agent-toolkit#213]). + +-------------------------------------------------------------------------------- +3. COMPONENT: Website UI & Exploration Tools (datacommonsorg/website) + Container Image: gcr.io/datcom-ci/datacommons-services (Website binary) +-------------------------------------------------------------------------------- + +SALIENT FEATURES & CAPABILITIES (vs. {prev_version}): +- Download Tool Redesign: Enhanced export interface for custom variable datasets ([website#6411]). +- Croissant Dataset Metadata: Inject Croissant JSON-LD for dataset indexing ([website#6443]). + +TRUE PLATFORM BUG FIXES (Fixes issues present in {prev_version}): +- Place Browser Duplication: Fixed duplicate place rendering when multiple provenances exist ([website#6474]). + +-------------------------------------------------------------------------------- +4. COMPONENT: Data Preprocessor Container Image: gcr.io/datcom-ci/datacommons-data -------------------------------------------------------------------------------- ... From 0b0ee69d7890e719ee36672f9245169d73fe80b7 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 13:21:41 -0700 Subject: [PATCH 54/70] fix(deploy): address all audit recommendations - URL preservation, tag resolution fallback, 6 dedicated subagents, and SOP mapping matrix --- deploy/generate_release_notes/SKILL.md | 9 ++++--- .../skills/dcp-context/SKILL.md | 20 +++++++------- .../skills/pr-extraction/SKILL.md | 26 +++++++++++++++---- .../skills/release-delta-synthesis/SKILL.md | 22 +++++++++++----- 4 files changed, 53 insertions(+), 24 deletions(-) diff --git a/deploy/generate_release_notes/SKILL.md b/deploy/generate_release_notes/SKILL.md index bf435908..22908aef 100644 --- a/deploy/generate_release_notes/SKILL.md +++ b/deploy/generate_release_notes/SKILL.md @@ -38,10 +38,13 @@ Provide each subagent with: 2. The **DCP Context Skill**: [`deploy/generate_release_notes/skills/dcp-context/SKILL.md`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/dcp-context/SKILL.md). 3. Its assigned **Component Name**, **Image URI**, **Source Repos**, ``, ``, and target **Output Verification File**. -#### Subagent Tasks: +#### Dedicated Subagent Tasks (1-to-1 with Component Output Files): - **Subagent 1 (`services-extractor`)**: Extract PRs for `gcr.io/datcom-ci/datacommons-services` from `website`, `mixer`, `agent-toolkit` $\rightarrow$ write `output/prs_services.txt`. -- **Subagent 2 (`import-extractor`)**: Extract PRs for `import` repo for preprocessor, Dataflow worker, ingestion helper, and postprocessing helper $\rightarrow$ write `output/prs_preprocessing.txt`, `output/prs_dataflow_worker.txt`, `output/prs_ingestion_helper.txt`, `output/prs_postprocessing.txt`. -- **Subagent 3 (`monorepo-extractor`)**: Extract PRs for `datacommonsorg/datacommons` monorepo & Terraform infra $\rightarrow$ write `output/prs_dcp_monorepo.txt`. +- **Subagent 2 (`preprocessing-extractor`)**: Extract PRs for `gcr.io/datcom-ci/datacommons-data` (preprocessor) from `import` repo $\rightarrow$ write `output/prs_preprocessing.txt`. +- **Subagent 3 (`dataflow-worker-extractor`)**: Extract PRs for Dataflow Ingestion Worker from `import` repo $\rightarrow$ write `output/prs_dataflow_worker.txt`. +- **Subagent 4 (`ingestion-helper-extractor`)**: Extract PRs for Ingestion Helper Service from `import` repo $\rightarrow$ write `output/prs_ingestion_helper.txt`. +- **Subagent 5 (`postprocessing-extractor`)**: Extract PRs for Postprocessing Helper Service from `import` repo $\rightarrow$ write `output/prs_postprocessing.txt`. +- **Subagent 6 (`monorepo-extractor`)**: Extract PRs for `datacommonsorg/datacommons` monorepo & Terraform infra $\rightarrow$ write `output/prs_dcp_monorepo.txt`. ### Step 3: Verification Checkpoint & Release Delta Synthesis Subagent 1. Notify the developer that raw PR verification files have been generated under `deploy/generate_release_notes/output/prs_*.txt` for review. diff --git a/deploy/generate_release_notes/skills/dcp-context/SKILL.md b/deploy/generate_release_notes/skills/dcp-context/SKILL.md index c4b0d63e..da5f6b99 100644 --- a/deploy/generate_release_notes/skills/dcp-context/SKILL.md +++ b/deploy/generate_release_notes/skills/dcp-context/SKILL.md @@ -49,13 +49,13 @@ Data Commons Platform (DCP) is a self-hosted, Cloud Spanner-backed deployment of --- -## 3. SOP Categories for Feature Classification - -1. **Spanner Graph & APIs**: - - SDMX 3.0 REST endpoints, `/v2/observation`, Mixer gRPC graph serving, FastMCP tools, MCP agent research skills. -2. **Ingestion & Safety**: - - Preprocessing, Dataflow workers, Cloud Workflows orchestration, postprocessing aggregations, health probes. -3. **Search & Website**: - - Vector embeddings, semantic search, search target scope (`V2_RESOLVE_INDICATORS_TARGET`), Explore UI, Download Tool. -4. **Infra & Tooling**: - - Terraform modules, Admin CLI (`datacommons admin`), monorepo versioning, IAM role provisioning. +## 3. SOP Categories & Release Notes Mapping Matrix + +The 4 SOP categories translate directly into the sections of the final release notes document: + +| SOP Category | Scope & Included Components | Release Notes Section Mapping | +| :--- | :--- | :--- | +| **Spanner Graph & APIs** | Mixer gRPC, SDMX 3.0 REST, `/v2/observation`, FastMCP tools | **Key Feature Updates** (Major API/MCP Features)
**Bug Fixes**: *Serving API & Query Robustness* | +| **Ingestion & Safety** | Preprocessor, Dataflow workers, Cloud Workflows, postprocessing rollups | **Key Feature Updates** (Major Pipeline Features)
**Improvements**: *Dataflow Transformations / Postprocessing*
**Bug Fixes**: *Ingestion Pipeline Reliability* | +| **Search & Website** | Vector embeddings, semantic search, Explore UI, Download Tool, Place Browser | **Key Feature Updates** (Search/Embeddings)
**Improvements**: *Website Exploration Tools*
**Bug Fixes**: *Web UI & Visualization* | +| **Infra & Tooling** | Terraform modules, Admin CLI (`datacommons admin`), Cloud Run, IAM roles | **Improvements**: *Terraform Infrastructure & Auto-Scaling*
**Bug Fixes**: *Deployment & Infrastructure Reliability* | diff --git a/deploy/generate_release_notes/skills/pr-extraction/SKILL.md b/deploy/generate_release_notes/skills/pr-extraction/SKILL.md index 161686e6..08b71385 100644 --- a/deploy/generate_release_notes/skills/pr-extraction/SKILL.md +++ b/deploy/generate_release_notes/skills/pr-extraction/SKILL.md @@ -23,11 +23,20 @@ When invoked, you will receive the following parameters: ## Execution Steps ### 1. Container Image Tag & Timestamp Resolution -1. Resolve the creation timestamp for `` and `` for your assigned `image_uri`: +1. **Primary Tag Resolution (Artifact Registry / GCR)**: + Attempt to resolve the creation timestamp for `` and `` for your assigned `image_uri`: ```bash gcloud container images list-tags --filter="tags:" --format="value(timestamp.datetime)" ``` -2. If an image tag is missing (e.g. during staging before image tagging), set `t_new` to the current time `NOW()`. +2. **Fallback Tag Resolution (Git Release Tags)**: + If `gcloud` returns no timestamp (e.g. tag naming difference like `v1.1.0` vs `1.1.0` or missing image), resolve the timestamp directly from git or GitHub release: + ```bash + gh release view --repo --json publishedAt --jq '.publishedAt' + # Or via git tag: + git log -1 --format=%cI + ``` +3. **Staging / Unreleased Target Tag**: + If `` is unreleased or image tag is missing during staging, set `t_new` to the current time `NOW()`. ### 2. Single Date-Range PR Search per Repository For each assigned source repository, execute a single `gh pr list` query spanning `[t_prev .. t_new]`: @@ -46,10 +55,11 @@ gh pr list --repo --state merged --search "merged:.." - **Core Services (`services` / `datacommons-services`)**: Include PRs affecting serving APIs (Mixer gRPC, SDMX 3.0 REST, MCP agent tools, `/v2/observation`), vector embeddings, or Website Explore UI tools. - **DCP Monorepo & Infra (`dcp_monorepo`)**: Include PRs affecting Terraform modules, Admin CLI tools (`datacommons admin`), or deployment infrastructure. -### 4. Noise, Base DC, and Regression Categorization +### 4. Noise, Revert PRs, and Regression Categorization Categorize every PR into either **Relevant PRs** or **Excluded PRs**: 1. **Relevant PRs**: Direct partner/operator features, configuration capabilities, or true platform bug fixes. 2. **Excluded PRs**: + - **Revert / Superseded PR Pairs**: If a PR reverts or supersedes another PR merged *within the same release window* (`[t_prev..t_new]`), exclude BOTH PRs. - **Base DC Only / Flag Flips**: PRs that only affect internal Google-hosted Base DC or internal flag flips without platform impact. - **Intermediate Regressions**: Bug fix PRs that address features/code introduced within the same release window (`[t_prev..t_new]`). - **Bot & Non-Production Chores**: Dependabot bumps, automated version bumps, unit/integration test harness refactors, or test sample data removals. @@ -58,8 +68,9 @@ Categorize every PR into either **Relevant PRs** or **Excluded PRs**: Format and write the extracted PRs into your assigned `output_file`, including an **Irrelevant / Excluded PRs** section at the bottom for developer audit. For each relevant PR, provide: -1. **Change Summary**: Concise description of what changed in the code. -2. **DCP Impact**: Direct impact on platform operators, developers, or end-users. +1. **URL**: Explicit GitHub PR URL for GFM link generation (`https://github.com/...`). +2. **Change Summary**: Concise description of what changed in the code. +3. **DCP Impact**: Direct impact on platform operators, developers, or end-users. ``` ================================================================================ @@ -72,10 +83,12 @@ Total Relevant PRs: {relevant_count} | Total Excluded PRs: {excluded_count} --- RELEVANT PRODUCTION PRS --- [{repo_short}#{number}] {title} (Author: {author} | Merged: {merged_at}) +URL: {url} - Change Summary: {1-2 sentence summary of what changed in this PR} - DCP Impact: {1-2 sentence explanation of user capability, API contract, or operator benefit} [{repo_short}#{number}] {title} (Author: {author} | Merged: {merged_at}) +URL: {url} - Change Summary: {1-2 sentence summary of what changed in this PR} - DCP Impact: {1-2 sentence explanation of user capability, API contract, or operator benefit} @@ -85,12 +98,15 @@ Total Relevant PRs: {relevant_count} | Total Excluded PRs: {excluded_count} [{repo_short}#{number}] {title} Reason: Excluded - Base DC-only flag flip / internal feature toggle +URL: {url} [{repo_short}#{number}] {title} Reason: Excluded - Intermediate regression fix for PR {parent_pr_id} merged in current release window +URL: {url} [{repo_short}#{number}] {title} Reason: Excluded - Unit test harness refactor / test sample data update +URL: {url} ``` Confirm when your assigned verification file has been written cleanly to `output_file`. diff --git a/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md b/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md index 7ebf3200..0a95a041 100644 --- a/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md +++ b/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md @@ -24,12 +24,22 @@ This skill provides step-by-step instructions for a subagent to analyze all raw 2. Group PRs by container image / component key (`services`, `preprocessing`, `dataflow_worker`, `ingestion_helper`, `postprocessing`, `dcp_monorepo`). ### 2. Intra-Release vs. Prior-Release Bug Fix Investigation -Perform deep investigation into every bug fix PR: -- **Intra-Release Intermediate Fix (EXCLUDE)**: Was the bug introduced by a feature/PR *merged within this current release window* (`[v_prev..v_new]`)? If so, exclude it! Platform users running `v_prev` were never exposed to this bug, so listing it creates noise. -- **Prior-Release True Fix (INCLUDE)**: Did the bug exist in the previously published container image (`v_prev` or earlier)? If so, synthesize it under true platform bug fixes for that component! +Perform step-by-step investigation into every bug fix PR using the following concrete verification procedure: -### 3. Synthesize Salient Image Deltas (Per Container Image) -For each container image / component, summarize the salient changes from the perspective of an operator upgrading from `` to ``: +1. **Check PR Title & Description Cross-References**: + - Inspect if the PR description references a PR merged in the current range (e.g., *"Fixes #2015"*, *"Follow up to #2000"*, *"Regression introduced by #1967"*). + - If it references an intermediate PR merged within `[t_prev..t_new]`, classify it as **Intra-Release Intermediate Fix (EXCLUDE)**. +2. **Footprint & Feature Matching**: + - Compare the PR's `Change Summary` and modified files against major features in `prs_*.txt` merged earlier in the window. + - If the bug is for a feature first introduced in this release window (e.g. SDMX 3.0 REST endpoints, FastMCP tools), classify it as **Intra-Release Intermediate Fix (EXCLUDE)** because users running `` were never exposed to this bug. +3. **Git History Verification (If Ambiguous)**: + - If a bug fix is ambiguous, run `git log -S "" ` or inspect `git diff ..` via shell to check if the code existed in ``. + - If the code existed in `` and was broken, classify it as **Prior-Release True Fix (INCLUDE)**. + +### 3. Synthesize Salient Image Deltas & Preserve URLs +For each container image / component, summarize the salient changes from the perspective of an operator upgrading from `` to ``. + +*CRITICAL MANDATE*: Preserve the full `URL` string for EVERY PR referenced in the delta summary so the final Release Writer can format clean GFM links `[repo#PR](URL)` without guessing! 1. **Major Feature Capabilities Added**: - What new capabilities exist in this image that were not present in ``? @@ -38,7 +48,7 @@ For each container image / component, summarize the salient changes from the per - What new Terraform variables, CLI parameters (`--instance_name`), or environment variables were added? - What scaling bounds (`max_workers`, BigQuery slots) or memory optimizations were introduced? 3. **True Platform Bug Fixes**: - - What issues present in `` were resolved in this container image? + - What issues present in `` were resolved in this container image? Include full PR URLs for each fix! --- From e5ad81a374e677c123f8a6fd5b168bc6cd6ff898 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 13:26:22 -0700 Subject: [PATCH 55/70] fix(deploy): enforce prompt user on missing image tags, mandate reasons for all ignored PRs, and simplify section mapping principles in dcp-context --- .../skills/dcp-context/SKILL.md | 21 ++++++++------ .../skills/pr-extraction/SKILL.md | 28 +++++++++---------- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/deploy/generate_release_notes/skills/dcp-context/SKILL.md b/deploy/generate_release_notes/skills/dcp-context/SKILL.md index da5f6b99..d62bd89a 100644 --- a/deploy/generate_release_notes/skills/dcp-context/SKILL.md +++ b/deploy/generate_release_notes/skills/dcp-context/SKILL.md @@ -49,13 +49,18 @@ Data Commons Platform (DCP) is a self-hosted, Cloud Spanner-backed deployment of --- -## 3. SOP Categories & Release Notes Mapping Matrix +## 3. Section Mapping & Release Content Principles -The 4 SOP categories translate directly into the sections of the final release notes document: +Rather than using arbitrary internal categories, map changes directly into the three standard release notes sections based on technical impact: -| SOP Category | Scope & Included Components | Release Notes Section Mapping | -| :--- | :--- | :--- | -| **Spanner Graph & APIs** | Mixer gRPC, SDMX 3.0 REST, `/v2/observation`, FastMCP tools | **Key Feature Updates** (Major API/MCP Features)
**Bug Fixes**: *Serving API & Query Robustness* | -| **Ingestion & Safety** | Preprocessor, Dataflow workers, Cloud Workflows, postprocessing rollups | **Key Feature Updates** (Major Pipeline Features)
**Improvements**: *Dataflow Transformations / Postprocessing*
**Bug Fixes**: *Ingestion Pipeline Reliability* | -| **Search & Website** | Vector embeddings, semantic search, Explore UI, Download Tool, Place Browser | **Key Feature Updates** (Search/Embeddings)
**Improvements**: *Website Exploration Tools*
**Bug Fixes**: *Web UI & Visualization* | -| **Infra & Tooling** | Terraform modules, Admin CLI (`datacommons admin`), Cloud Run, IAM roles | **Improvements**: *Terraform Infrastructure & Auto-Scaling*
**Bug Fixes**: *Deployment & Infrastructure Reliability* | +1. **Key Feature Updates**: + - Major, high-impact capabilities introduced in this release (e.g., SDMX 3.0 REST Data & Availability APIs, FastMCP AI agent tools, streaming JSON-LD preprocessors, vector search embeddings). + - Must follow the non-verbose format: **What's New** (1 paragraph combining description + benefit) followed by **Specific Capabilities** (bullet points with `[repo#PR](URL)` links). + +2. **Improvements & Configuration Updates**: + - Incremental enhancements, operator tools, Terraform variables (`max_workers`, BigQuery slots), CLI flags (`--instance_name`), and scaling optimizations. + - Must extract concrete enums (e.g. `custom_only`, `base_only`) and explicit configuration parameters. + +3. **Bug Fixes**: + - Synthesized into 3 to 5 functional categories (*Deployment & Infrastructure*, *Ingestion Pipeline Reliability*, *Serving API & Query Robustness*, *Web UI & Visualization*). + - Must ONLY include true platform bug fixes present in prior releases (excluding intra-release intermediate fixes). diff --git a/deploy/generate_release_notes/skills/pr-extraction/SKILL.md b/deploy/generate_release_notes/skills/pr-extraction/SKILL.md index 08b71385..b4e306fc 100644 --- a/deploy/generate_release_notes/skills/pr-extraction/SKILL.md +++ b/deploy/generate_release_notes/skills/pr-extraction/SKILL.md @@ -22,21 +22,15 @@ When invoked, you will receive the following parameters: ## Execution Steps -### 1. Container Image Tag & Timestamp Resolution -1. **Primary Tag Resolution (Artifact Registry / GCR)**: - Attempt to resolve the creation timestamp for `` and `` for your assigned `image_uri`: +### 1. Container Image Tag & Timestamp Resolution (NO AUTOMATIC FALLBACK) +1. **Artifact Registry Tag Resolution**: + Resolve the creation timestamp for `` and `` for your assigned `image_uri`: ```bash gcloud container images list-tags --filter="tags:" --format="value(timestamp.datetime)" ``` -2. **Fallback Tag Resolution (Git Release Tags)**: - If `gcloud` returns no timestamp (e.g. tag naming difference like `v1.1.0` vs `1.1.0` or missing image), resolve the timestamp directly from git or GitHub release: - ```bash - gh release view --repo --json publishedAt --jq '.publishedAt' - # Or via git tag: - git log -1 --format=%cI - ``` -3. **Staging / Unreleased Target Tag**: - If `` is unreleased or image tag is missing during staging, set `t_new` to the current time `NOW()`. +2. **STRICT MANDATE — ASK USER ON MISSING TAGS**: + If an image tag does NOT exist in Artifact Registry for `` or ``, **DO NOT automatically guess, synthesize, or fall back to git tags**. + Stop immediately and ask the user how to proceed (e.g., provide an alternative tag, specify custom date boundaries, or pass `--allow-missing-images` to use `NOW()`). ### 2. Single Date-Range PR Search per Repository For each assigned source repository, execute a single `gh pr list` query spanning `[t_prev .. t_new]`: @@ -65,7 +59,9 @@ Categorize every PR into either **Relevant PRs** or **Excluded PRs**: - **Bot & Non-Production Chores**: Dependabot bumps, automated version bumps, unit/integration test harness refactors, or test sample data removals. ### 5. Write Verification File (`prs_.txt`) -Format and write the extracted PRs into your assigned `output_file`, including an **Irrelevant / Excluded PRs** section at the bottom for developer audit. +Format and write the extracted PRs into your assigned `output_file`, including a complete **Irrelevant / Excluded PRs** section at the bottom for developer audit. + +*MANDATE*: Every single PR that is NOT included in Relevant Production PRs MUST be listed under Excluded PRs with an explicit, 1-sentence `Reason:` explaining why it was ignored (e.g., Base DC flag flip, revert pair, intermediate regression fix, bot bump, or test harness refactor). For each relevant PR, provide: 1. **URL**: Explicit GitHub PR URL for GFM link generation (`https://github.com/...`). @@ -97,7 +93,11 @@ URL: {url} ================================================================================ [{repo_short}#{number}] {title} -Reason: Excluded - Base DC-only flag flip / internal feature toggle +Reason: Excluded - Base DC-only flag flip / internal feature toggle without platform impact +URL: {url} + +[{repo_short}#{number}] {title} +Reason: Excluded - Revert PR pair (reverted by PR {revert_pr_id} merged in current release window) URL: {url} [{repo_short}#{number}] {title} From ad9f99c2714940b6c2bc3c76de97fdbc4e8789ae Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 16:17:55 -0700 Subject: [PATCH 56/70] docs(deploy): sanitize internal Spanner table names in generated RELEASE_NOTES_v1.1.1.md --- deploy/generate_release_notes/SKILL.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deploy/generate_release_notes/SKILL.md b/deploy/generate_release_notes/SKILL.md index 22908aef..3e09091b 100644 --- a/deploy/generate_release_notes/SKILL.md +++ b/deploy/generate_release_notes/SKILL.md @@ -30,6 +30,10 @@ When requested to generate release notes (e.g., *"Generate release notes for v1. 1. Identify the previous release tag (``, e.g., `v1.1.0`) and target release tag (``, e.g., `v1.1.1`). 2. Ensure `deploy/generate_release_notes/output/` directory exists. +> [!IMPORTANT] +> **DO NOT resolve image tags or run `gcloud` commands in the orchestrator.** +> The orchestrator MUST NOT query Artifact Registry or inspect image creation timestamps up front. Simply pass the raw version strings (`` and ``) to each subagent and let them resolve their assigned image tags concurrently. + ### Step 2: Spawn PR Extraction Subagents Call `invoke_subagent` to spawn subagents concurrently across the components above. From 46aa9c2806d6baa8c54f85657b1239ba121ff47b Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 16:19:34 -0700 Subject: [PATCH 57/70] fix(deploy): replace all local file URIs with relative links across release notes skills and README --- deploy/generate_release_notes/README.md | 2 +- deploy/generate_release_notes/SKILL.md | 10 +++++----- .../skills/pr-extraction/SKILL.md | 2 +- .../skills/release-delta-synthesis/SKILL.md | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/deploy/generate_release_notes/README.md b/deploy/generate_release_notes/README.md index 1df2494b..69d9c29a 100644 --- a/deploy/generate_release_notes/README.md +++ b/deploy/generate_release_notes/README.md @@ -38,7 +38,7 @@ deploy/generate_release_notes/ ## Developer Usage Instructions (Prompting Your LLM Agent) ### Step 1: Point Your LLM Agent at the Orchestrator Skill -To generate release notes for a release range, point your LLM agent at [`deploy/generate_release_notes/SKILL.md`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/SKILL.md): +To generate release notes for a release range, point your LLM agent at [`SKILL.md`](SKILL.md): > **Prompt Example**: > *"Please read `deploy/generate_release_notes/SKILL.md` and generate release notes for version v1.1.0 to v1.1.1."* diff --git a/deploy/generate_release_notes/SKILL.md b/deploy/generate_release_notes/SKILL.md index 3e09091b..3be98784 100644 --- a/deploy/generate_release_notes/SKILL.md +++ b/deploy/generate_release_notes/SKILL.md @@ -38,8 +38,8 @@ When requested to generate release notes (e.g., *"Generate release notes for v1. Call `invoke_subagent` to spawn subagents concurrently across the components above. Provide each subagent with: -1. The **PR Extraction Skill**: [`deploy/generate_release_notes/skills/pr-extraction/SKILL.md`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/pr-extraction/SKILL.md). -2. The **DCP Context Skill**: [`deploy/generate_release_notes/skills/dcp-context/SKILL.md`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/dcp-context/SKILL.md). +1. The **PR Extraction Skill**: [`skills/pr-extraction/SKILL.md`](skills/pr-extraction/SKILL.md). +2. The **DCP Context Skill**: [`skills/dcp-context/SKILL.md`](skills/dcp-context/SKILL.md). 3. Its assigned **Component Name**, **Image URI**, **Source Repos**, ``, ``, and target **Output Verification File**. #### Dedicated Subagent Tasks (1-to-1 with Component Output Files): @@ -53,7 +53,7 @@ Provide each subagent with: ### Step 3: Verification Checkpoint & Release Delta Synthesis Subagent 1. Notify the developer that raw PR verification files have been generated under `deploy/generate_release_notes/output/prs_*.txt` for review. 2. Call `invoke_subagent` to spawn a specialized **Release Delta Synthesis Subagent** (`delta-synthesizer`). -3. Provide the subagent with the **Release Delta Synthesis Skill**: [`deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md). +3. Provide the subagent with the **Release Delta Synthesis Skill**: [`skills/release-delta-synthesis/SKILL.md`](skills/release-delta-synthesis/SKILL.md). 4. The subagent will: - Read all `output/prs_*.txt` files. - Investigate and distinguish true bug fixes present in `` vs. intermediate bug fixes introduced and fixed within `` (omitting intra-release fixes). @@ -61,7 +61,7 @@ Provide each subagent with: - Output the unified image delta summary to: `deploy/generate_release_notes/output/IMAGE_DELTAS_.txt`. ### Step 4: Author Publication-Ready Release Notes -1. Read the **DCP Domain Context Skill**: [`deploy/generate_release_notes/skills/dcp-context/SKILL.md`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/dcp-context/SKILL.md). -2. Read the **Release Writer Skill**: [`deploy/generate_release_notes/skills/release-writer/SKILL.md`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/release-writer/SKILL.md). +1. Read the **DCP Domain Context Skill**: [`skills/dcp-context/SKILL.md`](skills/dcp-context/SKILL.md). +2. Read the **Release Writer Skill**: [`skills/release-writer/SKILL.md`](skills/release-writer/SKILL.md). 3. Read `deploy/generate_release_notes/output/IMAGE_DELTAS_.txt`. 4. Author the final release notes from the verified image delta summary into: `deploy/generate_release_notes/output/RELEASE_NOTES_.md`. diff --git a/deploy/generate_release_notes/skills/pr-extraction/SKILL.md b/deploy/generate_release_notes/skills/pr-extraction/SKILL.md index b4e306fc..10a7a163 100644 --- a/deploy/generate_release_notes/skills/pr-extraction/SKILL.md +++ b/deploy/generate_release_notes/skills/pr-extraction/SKILL.md @@ -40,7 +40,7 @@ gh pr list --repo --state merged --search "merged:.." *(IMPORTANT: Do NOT pass `base:main` inside `--search`; use `--search "merged:.."` directly to prevent GitHub Search API parse errors!)* ### 3. DCP Context & Semantic Content Analysis -1. **Read DCP Context Skill**: Before analyzing PRs, you MUST read [`deploy/generate_release_notes/skills/dcp-context/SKILL.md`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/dcp-context/SKILL.md) to understand how your assigned component fits into the platform architecture. +1. **Read DCP Context Skill**: Before analyzing PRs, you MUST read [`skills/dcp-context/SKILL.md`](../dcp-context/SKILL.md) to understand how your assigned component fits into the platform architecture. 2. **Analyze PR Content**: Analyze the actual content of each PR (title, description body, labels, and changed code context) against the DCP context to evaluate relevance: - **Data Preprocessor (`preprocessing` / `datacommons-data`)**: Include PRs affecting CSV/MCF parsing, streaming JSON-LD batching, schema validation, column mapping, or preprocessor execution. - **Dataflow Ingestion Worker (`dataflow_worker`)**: Include PRs affecting Dataflow pipelines, TFRecord loading, BigQuery/Spanner graph transformations, or batch import scaling (`max_workers`). diff --git a/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md b/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md index 0a95a041..fd0603ca 100644 --- a/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md +++ b/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md @@ -12,7 +12,7 @@ This skill provides step-by-step instructions for a subagent to analyze all raw ## Input & Output Files - **Input Files**: `deploy/generate_release_notes/output/prs_*.txt` (`prs_services.txt`, `prs_preprocessing.txt`, `prs_dataflow_worker.txt`, `prs_ingestion_helper.txt`, `prs_postprocessing.txt`, `prs_dcp_monorepo.txt`). -- **Context Reference**: [`deploy/generate_release_notes/skills/dcp-context/SKILL.md`](file:///Users/calinc/datcom-datacommons/deploy/generate_release_notes/skills/dcp-context/SKILL.md). +- **Context Reference**: [`skills/dcp-context/SKILL.md`](../dcp-context/SKILL.md). - **Target Output File**: `deploy/generate_release_notes/output/IMAGE_DELTAS_.txt`. --- From a01200ee49e5540e36bcd6febec72dc0112b3e0c Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 16:19:58 -0700 Subject: [PATCH 58/70] feat(deploy): update Release Writer skill to scale Executive Summary dynamically with release size --- .../skills/release-writer/SKILL.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/deploy/generate_release_notes/skills/release-writer/SKILL.md b/deploy/generate_release_notes/skills/release-writer/SKILL.md index ab197f8c..4678231b 100644 --- a/deploy/generate_release_notes/skills/release-writer/SKILL.md +++ b/deploy/generate_release_notes/skills/release-writer/SKILL.md @@ -13,10 +13,12 @@ This skill provides step-by-step instructions for authoring non-verbose, publica - **Tone**: Direct, factual, punchy, senior-engineer technical changelog. Active voice for features ("You can now..."), past tense for bugs ("Resolved..."). - **BANNED AI FLUFF WORDS (STRICT)**: DO NOT use AI cliché words: `seamlessly`, `empower`, `leveraging`, `robust`, `overhaul`, `delivers a major`, `comprehensive`, `fosters`, `game-changing`, `cutting-edge`, `paradigm`. -- **STRICT WORD COUNT BUDGETS**: - - **Executive Summary**: Maximum 25 words (1 single, punchy sentence). - - **What's New**: Combine description and user benefit into 1 concise paragraph (25-35 words max). - - **Specific Capabilities Bullets**: 12-15 words max per bullet. +- **DYNAMIC EXECUTIVE SUMMARY**: + - The summary length and detail level MUST scale dynamically with the scope of the release. + - **Large / Feature-Rich Releases**: Provide a comprehensive 2–3 sentence overview highlighting all major capabilities, API protocols, preprocessor boosts, and critical fixes without an artificial word count cap. + - **Small / Patch Releases**: Provide a short, single-sentence summary (15–25 words) without unnecessary verbosity or fluff. +- **What's New Paragraphs**: Combine technical change and user benefit into 1 concise, punchy paragraph (25–45 words). +- **Specific Capabilities Bullets**: 12–20 words max per bullet point. - **GFM Link Rules**: Every PR reference MUST be a clean, clickable link: `[repo_short#PR](URL)`. NEVER wrap backticks around or inside link text (`[`repo#123`](URL)` is forbidden!). - **NO Horizontal Dividers Between Features**: Do NOT place horizontal rule lines (`---`) between individual feature sections under Key Feature Updates. Use standard Markdown headers (`### Feature Title`) with single blank lines only! @@ -27,7 +29,7 @@ This skill provides step-by-step instructions for authoring non-verbose, publica ```markdown # Data Commons Platform Release {new_version} ({release_date}) -[Provide a high-impact, 1-sentence Executive Summary (max 25 words) highlighting the most important capabilities, performance boosts, and critical fixes introduced in this release for partners and platform operators.] +[Provide a high-impact Executive Summary highlighting the most important capabilities, performance boosts, and critical fixes introduced in this release. Adjust summary length dynamically based on release size: 2-3 sentences for major releases, 1 punchy sentence for patch releases.] --- From 1591261ed6d6773260f71d1ab0354459e40068e4 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 16:20:59 -0700 Subject: [PATCH 59/70] chore(deploy): add .gitignore and .gitkeep for generate_release_notes output directory --- deploy/generate_release_notes/output/.gitignore | 4 ++++ deploy/generate_release_notes/output/.gitkeep | 1 + 2 files changed, 5 insertions(+) create mode 100644 deploy/generate_release_notes/output/.gitignore create mode 100644 deploy/generate_release_notes/output/.gitkeep diff --git a/deploy/generate_release_notes/output/.gitignore b/deploy/generate_release_notes/output/.gitignore new file mode 100644 index 00000000..6e9fd814 --- /dev/null +++ b/deploy/generate_release_notes/output/.gitignore @@ -0,0 +1,4 @@ +# Ignore generated release notes outputs and verification logs +*.txt +*.md +!.gitkeep diff --git a/deploy/generate_release_notes/output/.gitkeep b/deploy/generate_release_notes/output/.gitkeep new file mode 100644 index 00000000..8ab527fd --- /dev/null +++ b/deploy/generate_release_notes/output/.gitkeep @@ -0,0 +1 @@ +# Preserve output directory structure for generated release notes artifacts From 75482ccf45ce41ccb1c3efd03291e422801886a1 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 16:23:31 -0700 Subject: [PATCH 60/70] revert(deploy): restore pyproject.toml and uv.lock to match upstream/main --- pyproject.toml | 9 ++-- uv.lock | 121 +++++++------------------------------------------ 2 files changed, 20 insertions(+), 110 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7e7906e4..309df685 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ [dependency-groups] dev = [ {include-group = "lint"}, - {include-group = "test"}, + {include-group = "test"} ] lint = [ "pre-commit>=4.5.1", @@ -145,9 +145,8 @@ build-backend = "setuptools.build_meta" # Tell setuptools this is a meta-package and to stop looking for code packages = [] -[tool.pytest.ini_options] -pythonpath = ["."] -testpaths = ["packages/*/tests"] - [tool.setuptools.dynamic] version = {file = "VERSION"} + +[tool.pytest.ini_options] +testpaths = ["packages"] diff --git a/uv.lock b/uv.lock index 62377e6c..fdb58b1b 100644 --- a/uv.lock +++ b/uv.lock @@ -425,22 +425,14 @@ dependencies = [ [package.dev-dependencies] dev = [ - { name = "click" }, { name = "datacommons-admin" }, { name = "datacommons-api" }, { name = "datacommons-db" }, { name = "datacommons-schema" }, - { name = "google-genai" }, - { name = "jinja2" }, { name = "pre-commit" }, { name = "pytest" }, { name = "ruff" }, ] -generate-release-notes = [ - { name = "click" }, - { name = "google-genai" }, - { name = "jinja2" }, -] lint = [ { name = "pre-commit" }, { name = "ruff" }, @@ -458,22 +450,14 @@ requires-dist = [{ name = "datacommons-cli", editable = "packages/datacommons-cl [package.metadata.requires-dev] dev = [ - { name = "click", specifier = ">=8.1.7" }, { name = "datacommons-admin", editable = "packages/datacommons-admin" }, { name = "datacommons-api", editable = "packages/datacommons-api" }, { name = "datacommons-db", editable = "packages/datacommons-db" }, { name = "datacommons-schema", editable = "packages/datacommons-schema" }, - { name = "google-genai", specifier = ">=0.1.0" }, - { name = "jinja2", specifier = ">=3.1.0" }, { name = "pre-commit", specifier = ">=4.5.1" }, { name = "pytest", specifier = ">=9.0.3" }, { name = "ruff", specifier = ">=0.15.0" }, ] -generate-release-notes = [ - { name = "click", specifier = ">=8.1.7" }, - { name = "google-genai", specifier = ">=0.1.0" }, - { name = "jinja2", specifier = ">=3.1.0" }, -] lint = [ { name = "pre-commit", specifier = ">=4.5.1" }, { name = "ruff", specifier = ">=0.15.0" }, @@ -521,15 +505,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, ] -[[package]] -name = "distro" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, -] - [[package]] name = "fastapi" version = "0.128.6" @@ -579,20 +554,16 @@ grpc = [ [[package]] name = "google-auth" -version = "2.56.2" +version = "2.48.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography" }, { name = "pyasn1-modules" }, + { name = "rsa" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c7/33/dbc946a407401b975f0719658f18e664ece2109f79ffd1ff3bf226c205f4/google_auth-2.56.2.tar.gz", hash = "sha256:e28f103ca8091fb7012b99c44243d7366c29863713b8e34a220c3322b7a07051", size = 365820, upload-time = "2026-07-21T21:53:28.188Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0c/41/242044323fbd746615884b1c16639749e73665b718209946ebad7ba8a813/google_auth-2.48.0.tar.gz", hash = "sha256:4f7e706b0cd3208a3d940a19a822c37a476ddba5450156c3e6624a71f7c841ce", size = 326522, upload-time = "2026-01-26T19:22:47.157Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/88/63/50636aae68c9bf17c891c7eb18b49baa9bd6b31d2a97b8de4813a9fc8d1c/google_auth-2.56.2-py3-none-any.whl", hash = "sha256:c8270ea95b2697b74e3d8438ae9c5b898e38b623b915c7b5c5635921e7de68a6", size = 258588, upload-time = "2026-07-21T21:53:26.399Z" }, -] - -[package.optional-dependencies] -requests = [ - { name = "requests" }, + { url = "https://files.pythonhosted.org/packages/83/1d/d6466de3a5249d35e832a52834115ca9d1d0de6abc22065f049707516d47/google_auth-2.48.0-py3-none-any.whl", hash = "sha256:2e2a537873d449434252a9632c28bfc268b0adb1e53f9fb62afc5333a975903f", size = 236499, upload-time = "2026-01-26T19:22:45.099Z" }, ] [[package]] @@ -697,27 +668,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9c/97/7d75fe37a7a6ed171a2cf17117177e7aab7e6e0d115858741b41e9dd4254/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c", size = 28800, upload-time = "2025-12-16T00:40:30.322Z" }, ] -[[package]] -name = "google-genai" -version = "2.15.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "google-auth", extra = ["requests"] }, - { name = "httpx" }, - { name = "pydantic" }, - { name = "requests" }, - { name = "sniffio" }, - { name = "tenacity" }, - { name = "typing-extensions" }, - { name = "websockets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/69/53/b2c9b0a74b817a393d388a2303ec4da8bda27ea744b23914480d1b024d84/google_genai-2.15.0.tar.gz", hash = "sha256:ef71bdb79ce9931bca1cf0a393c8cfb606e1075b6100fcdde02b7b467db8235d", size = 640674, upload-time = "2026-07-29T17:43:21.853Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/86/2ef6d992955307bf525f305b44cb3e2066eb01636549607082a0407f7047/google_genai-2.15.0-py3-none-any.whl", hash = "sha256:f322a94c3c1ddb1b1cc536086708f5f8e13101347d062f63dcd7947b598c1d16", size = 1030459, upload-time = "2026-07-29T17:43:20.286Z" }, -] - [[package]] name = "google-resumable-media" version = "2.9.0" @@ -756,7 +706,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/e8/2e1462c8fdbe0f210feb5ac7ad2d9029af8be3bf45bd9fa39765f821642f/greenlet-3.3.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:5fd23b9bc6d37b563211c6abbb1b3cab27db385a4449af5c32e932f93017080c", size = 274974, upload-time = "2026-01-23T15:31:02.891Z" }, { url = "https://files.pythonhosted.org/packages/7e/a8/530a401419a6b302af59f67aaf0b9ba1015855ea7e56c036b5928793c5bd/greenlet-3.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09f51496a0bfbaa9d74d36a52d2580d1ef5ed4fdfcff0a73730abfbbbe1403dd", size = 577175, upload-time = "2026-01-23T16:00:56.213Z" }, { url = "https://files.pythonhosted.org/packages/8e/89/7e812bb9c05e1aaef9b597ac1d0962b9021d2c6269354966451e885c4e6b/greenlet-3.3.1-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb0feb07fe6e6a74615ee62a880007d976cf739b6669cce95daa7373d4fc69c5", size = 590401, upload-time = "2026-01-23T16:05:26.365Z" }, - { url = "https://files.pythonhosted.org/packages/70/ae/e2d5f0e59b94a2269b68a629173263fa40b63da32f5c231307c349315871/greenlet-3.3.1-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:67ea3fc73c8cd92f42467a72b75e8f05ed51a0e9b1d15398c913416f2dafd49f", size = 601161, upload-time = "2026-01-23T16:15:53.456Z" }, { url = "https://files.pythonhosted.org/packages/5c/ae/8d472e1f5ac5efe55c563f3eabb38c98a44b832602e12910750a7c025802/greenlet-3.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:39eda9ba259cc9801da05351eaa8576e9aa83eb9411e8f0c299e05d712a210f2", size = 590272, upload-time = "2026-01-23T15:32:49.411Z" }, { url = "https://files.pythonhosted.org/packages/a8/51/0fde34bebfcadc833550717eade64e35ec8738e6b097d5d248274a01258b/greenlet-3.3.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e2e7e882f83149f0a71ac822ebf156d902e7a5d22c9045e3e0d1daf59cee2cc9", size = 1550729, upload-time = "2026-01-23T16:04:20.867Z" }, { url = "https://files.pythonhosted.org/packages/16/c9/2fb47bee83b25b119d5a35d580807bb8b92480a54b68fef009a02945629f/greenlet-3.3.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:80aa4d79eb5564f2e0a6144fcc744b5a37c56c4a92d60920720e99210d88db0f", size = 1615552, upload-time = "2026-01-23T15:33:45.743Z" }, @@ -765,7 +714,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/c8/9d76a66421d1ae24340dfae7e79c313957f6e3195c144d2c73333b5bfe34/greenlet-3.3.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:7e806ca53acf6d15a888405880766ec84721aa4181261cd11a457dfe9a7a4975", size = 276443, upload-time = "2026-01-23T15:30:10.066Z" }, { url = "https://files.pythonhosted.org/packages/81/99/401ff34bb3c032d1f10477d199724f5e5f6fbfb59816ad1455c79c1eb8e7/greenlet-3.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d842c94b9155f1c9b3058036c24ffb8ff78b428414a19792b2380be9cecf4f36", size = 597359, upload-time = "2026-01-23T16:00:57.394Z" }, { url = "https://files.pythonhosted.org/packages/2b/bc/4dcc0871ed557792d304f50be0f7487a14e017952ec689effe2180a6ff35/greenlet-3.3.1-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:20fedaadd422fa02695f82093f9a98bad3dab5fcda793c658b945fcde2ab27ba", size = 607805, upload-time = "2026-01-23T16:05:28.068Z" }, - { url = "https://files.pythonhosted.org/packages/3b/cd/7a7ca57588dac3389e97f7c9521cb6641fd8b6602faf1eaa4188384757df/greenlet-3.3.1-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c620051669fd04ac6b60ebc70478210119c56e2d5d5df848baec4312e260e4ca", size = 622363, upload-time = "2026-01-23T16:15:54.754Z" }, { url = "https://files.pythonhosted.org/packages/cf/05/821587cf19e2ce1f2b24945d890b164401e5085f9d09cbd969b0c193cd20/greenlet-3.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14194f5f4305800ff329cbf02c5fcc88f01886cadd29941b807668a45f0d2336", size = 609947, upload-time = "2026-01-23T15:32:51.004Z" }, { url = "https://files.pythonhosted.org/packages/a4/52/ee8c46ed9f8babaa93a19e577f26e3d28a519feac6350ed6f25f1afee7e9/greenlet-3.3.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7b2fe4150a0cf59f847a67db8c155ac36aed89080a6a639e9f16df5d6c6096f1", size = 1567487, upload-time = "2026-01-23T16:04:22.125Z" }, { url = "https://files.pythonhosted.org/packages/8f/7c/456a74f07029597626f3a6db71b273a3632aecb9afafeeca452cfa633197/greenlet-3.3.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:49f4ad195d45f4a66a0eb9c1ba4832bb380570d361912fa3554746830d332149", size = 1636087, upload-time = "2026-01-23T15:33:47.486Z" }, @@ -774,7 +722,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/ab/d26750f2b7242c2b90ea2ad71de70cfcd73a948a49513188a0fc0d6fc15a/greenlet-3.3.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:7ab327905cabb0622adca5971e488064e35115430cec2c35a50fd36e72a315b3", size = 275205, upload-time = "2026-01-23T15:30:24.556Z" }, { url = "https://files.pythonhosted.org/packages/10/d3/be7d19e8fad7c5a78eeefb2d896a08cd4643e1e90c605c4be3b46264998f/greenlet-3.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:65be2f026ca6a176f88fb935ee23c18333ccea97048076aef4db1ef5bc0713ac", size = 599284, upload-time = "2026-01-23T16:00:58.584Z" }, { url = "https://files.pythonhosted.org/packages/ae/21/fe703aaa056fdb0f17e5afd4b5c80195bbdab701208918938bd15b00d39b/greenlet-3.3.1-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7a3ae05b3d225b4155bda56b072ceb09d05e974bc74be6c3fc15463cf69f33fd", size = 610274, upload-time = "2026-01-23T16:05:29.312Z" }, - { url = "https://files.pythonhosted.org/packages/06/00/95df0b6a935103c0452dad2203f5be8377e551b8466a29650c4c5a5af6cc/greenlet-3.3.1-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:12184c61e5d64268a160226fb4818af4df02cfead8379d7f8b99a56c3a54ff3e", size = 624375, upload-time = "2026-01-23T16:15:55.915Z" }, { url = "https://files.pythonhosted.org/packages/cb/86/5c6ab23bb3c28c21ed6bebad006515cfe08b04613eb105ca0041fecca852/greenlet-3.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6423481193bbbe871313de5fd06a082f2649e7ce6e08015d2a76c1e9186ca5b3", size = 612904, upload-time = "2026-01-23T15:32:52.317Z" }, { url = "https://files.pythonhosted.org/packages/c2/f3/7949994264e22639e40718c2daf6f6df5169bf48fb038c008a489ec53a50/greenlet-3.3.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:33a956fe78bbbda82bfc95e128d61129b32d66bcf0a20a1f0c08aa4839ffa951", size = 1567316, upload-time = "2026-01-23T16:04:23.316Z" }, { url = "https://files.pythonhosted.org/packages/8d/6e/d73c94d13b6465e9f7cd6231c68abde838bb22408596c05d9059830b7872/greenlet-3.3.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4b065d3284be43728dd280f6f9a13990b56470b81be20375a207cdc814a983f2", size = 1636549, upload-time = "2026-01-23T15:33:48.643Z" }, @@ -783,7 +730,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/fb/011c7c717213182caf78084a9bea51c8590b0afda98001f69d9f853a495b/greenlet-3.3.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:bd59acd8529b372775cd0fcbc5f420ae20681c5b045ce25bd453ed8455ab99b5", size = 275737, upload-time = "2026-01-23T15:32:16.889Z" }, { url = "https://files.pythonhosted.org/packages/41/2e/a3a417d620363fdbb08a48b1dd582956a46a61bf8fd27ee8164f9dfe87c2/greenlet-3.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b31c05dd84ef6871dd47120386aed35323c944d86c3d91a17c4b8d23df62f15b", size = 646422, upload-time = "2026-01-23T16:01:00.354Z" }, { url = "https://files.pythonhosted.org/packages/b4/09/c6c4a0db47defafd2d6bab8ddfe47ad19963b4e30f5bed84d75328059f8c/greenlet-3.3.1-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:02925a0bfffc41e542c70aa14c7eda3593e4d7e274bfcccca1827e6c0875902e", size = 658219, upload-time = "2026-01-23T16:05:30.956Z" }, - { url = "https://files.pythonhosted.org/packages/e2/89/b95f2ddcc5f3c2bc09c8ee8d77be312df7f9e7175703ab780f2014a0e781/greenlet-3.3.1-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3e0f3878ca3a3ff63ab4ea478585942b53df66ddde327b59ecb191b19dbbd62d", size = 671455, upload-time = "2026-01-23T16:15:57.232Z" }, { url = "https://files.pythonhosted.org/packages/80/38/9d42d60dffb04b45f03dbab9430898352dba277758640751dc5cc316c521/greenlet-3.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34a729e2e4e4ffe9ae2408d5ecaf12f944853f40ad724929b7585bca808a9d6f", size = 660237, upload-time = "2026-01-23T15:32:53.967Z" }, { url = "https://files.pythonhosted.org/packages/96/61/373c30b7197f9e756e4c81ae90a8d55dc3598c17673f91f4d31c3c689c3f/greenlet-3.3.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:aec9ab04e82918e623415947921dea15851b152b822661cce3f8e4393c3df683", size = 1615261, upload-time = "2026-01-23T16:04:25.066Z" }, { url = "https://files.pythonhosted.org/packages/fd/d3/ca534310343f5945316f9451e953dcd89b36fe7a19de652a1dc5a0eeef3f/greenlet-3.3.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:71c767cf281a80d02b6c1bdc41c9468e1f5a494fb11bc8688c360524e273d7b1", size = 1683719, upload-time = "2026-01-23T15:33:50.61Z" }, @@ -792,7 +738,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/28/24/cbbec49bacdcc9ec652a81d3efef7b59f326697e7edf6ed775a5e08e54c2/greenlet-3.3.1-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:3e63252943c921b90abb035ebe9de832c436401d9c45f262d80e2d06cc659242", size = 282706, upload-time = "2026-01-23T15:33:05.525Z" }, { url = "https://files.pythonhosted.org/packages/86/2e/4f2b9323c144c4fe8842a4e0d92121465485c3c2c5b9e9b30a52e80f523f/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:76e39058e68eb125de10c92524573924e827927df5d3891fbc97bd55764a8774", size = 651209, upload-time = "2026-01-23T16:01:01.517Z" }, { url = "https://files.pythonhosted.org/packages/d9/87/50ca60e515f5bb55a2fbc5f0c9b5b156de7d2fc51a0a69abc9d23914a237/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9f9d5e7a9310b7a2f416dd13d2e3fd8b42d803968ea580b7c0f322ccb389b97", size = 654300, upload-time = "2026-01-23T16:05:32.199Z" }, - { url = "https://files.pythonhosted.org/packages/7c/25/c51a63f3f463171e09cb586eb64db0861eb06667ab01a7968371a24c4f3b/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b9721549a95db96689458a1e0ae32412ca18776ed004463df3a9299c1b257ab", size = 662574, upload-time = "2026-01-23T16:15:58.364Z" }, { url = "https://files.pythonhosted.org/packages/1d/94/74310866dfa2b73dd08659a3d18762f83985ad3281901ba0ee9a815194fb/greenlet-3.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92497c78adf3ac703b57f1e3813c2d874f27f71a178f9ea5887855da413cd6d2", size = 653842, upload-time = "2026-01-23T15:32:55.671Z" }, { url = "https://files.pythonhosted.org/packages/97/43/8bf0ffa3d498eeee4c58c212a3905dd6146c01c8dc0b0a046481ca29b18c/greenlet-3.3.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ed6b402bc74d6557a705e197d47f9063733091ed6357b3de33619d8a8d93ac53", size = 1614917, upload-time = "2026-01-23T16:04:26.276Z" }, { url = "https://files.pythonhosted.org/packages/89/90/a3be7a5f378fc6e84abe4dcfb2ba32b07786861172e502388b4c90000d1b/greenlet-3.3.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:59913f1e5ada20fde795ba906916aea25d442abcc0593fba7e26c92b7ad76249", size = 1676092, upload-time = "2026-01-23T15:33:52.176Z" }, @@ -908,19 +853,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7d/c9/f6e1e8567660bc5b0aba281f2b0017b2a7665fcad6bf3ed67286a0c72cd4/html5rdf-1.2.1-py2.py3-none-any.whl", hash = "sha256:1f519121bc366af3e485310dc8041d2e86e5173c1a320fac3dc9d2604069b83e", size = 109765, upload-time = "2024-10-30T05:06:52.507Z" }, ] -[[package]] -name = "httpcore" -version = "1.0.9" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "certifi" }, - { name = "h11" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, -] - [[package]] name = "httptools" version = "0.7.1" @@ -957,21 +889,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/53/cf/878f3b91e4e6e011eff6d1fa9ca39f7eb17d19c9d7971b04873734112f30/httptools-0.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:cfabda2a5bb85aa2a904ce06d974a3f30fb36cc63d7feaddec05d2050acede96", size = 88205, upload-time = "2025-10-10T03:55:00.389Z" }, ] -[[package]] -name = "httpx" -version = "0.28.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "certifi" }, - { name = "httpcore" }, - { name = "idna" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, -] - [[package]] name = "identify" version = "2.6.16" @@ -1664,6 +1581,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/56/5d/c814546c2333ceea4ba42262d8c4d55763003e767fa169adc693bd524478/requests-2.33.0-py3-none-any.whl", hash = "sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b", size = 65017, upload-time = "2026-03-25T15:10:40.382Z" }, ] +[[package]] +name = "rsa" +version = "4.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/8a/22b7beea3ee0d44b1916c0c1cb0ee3af23b700b6da9f04991899d0c555d4/rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75", size = 29034, upload-time = "2025-04-16T09:51:18.218Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", size = 34696, upload-time = "2025-04-16T09:51:17.142Z" }, +] + [[package]] name = "ruff" version = "0.15.0" @@ -1698,15 +1627,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/23/63/5517029d6696ddf2bd378d46f63f479be001c31b462303170a1da57650cb/setuptools-80.0.0-py3-none-any.whl", hash = "sha256:a38f898dcd6e5380f4da4381a87ec90bd0a7eec23d204a5552e80ee3cab6bd27", size = 1240907, upload-time = "2025-04-27T17:21:09.175Z" }, ] -[[package]] -name = "sniffio" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, -] - [[package]] name = "sqlalchemy" version = "2.0.46" @@ -1792,15 +1712,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/0d/13d1d239a25cbfb19e740db83143e95c772a1fe10202dda4b76792b114dd/starlette-0.52.1-py3-none-any.whl", hash = "sha256:0029d43eb3d273bc4f83a08720b4912ea4b071087a3b48db01b7c839f7954d74", size = 74272, upload-time = "2026-01-18T13:34:09.188Z" }, ] -[[package]] -name = "tenacity" -version = "9.1.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, -] - [[package]] name = "typing-extensions" version = "4.15.0" From 95a61b75e88fb7526a7bcce69e39a970092c5da6 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 16:24:47 -0700 Subject: [PATCH 61/70] docs(deploy): remove redundant Repository Mapping table from README and update skill descriptions --- deploy/generate_release_notes/README.md | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/deploy/generate_release_notes/README.md b/deploy/generate_release_notes/README.md index 69d9c29a..01deb31c 100644 --- a/deploy/generate_release_notes/README.md +++ b/deploy/generate_release_notes/README.md @@ -2,7 +2,7 @@ An agentic, skill-driven tool suite for generating publication-ready, partner-facing release notes for the Data Commons Platform (DCP). -The tool automatically extracts merged Pull Requests across all 6 core Data Commons repositories, classifies them according to standard SOP categories, filters out internal test noise and regressions, writes human-verifiable PR lists per container image (`output/prs_.txt`), and formats concise release notes tailored for developers and platform operators building on top of DCP. +The tool automatically extracts merged Pull Requests across all 6 core Data Commons repositories, classifies them according to platform layer and technical impact, filters out internal test noise and regressions, writes human-verifiable PR lists per container image (`output/prs_.txt`), and formats concise release notes tailored for developers and platform operators building on top of DCP. --- @@ -71,29 +71,14 @@ Your LLM agent will load `skills/release-writer/SKILL.md` and author the publica The master entrypoint skill that coordinates subagent execution, manages the step-by-step pipeline, and ensures intermediate verification files are generated before writing the final release notes. ### 2. PR Extraction Skill (`skills/pr-extraction/SKILL.md`) -Contains exact commands and rules for `gcloud container images list-tags` resolution, date-range `gh pr list` queries, non-production test filtering, and intermediate regression exclusion. +Contains exact commands and rules for `gcloud container images list-tags` resolution (with strict user prompt on missing tags), date-range `gh pr list` queries, non-production test filtering, and intermediate regression exclusion. ### 3. DCP Domain Context Skill (`skills/dcp-context/SKILL.md`) Defines the architectural map across all 6 core repositories (`datacommons`, `website`, `mixer`, `agent-toolkit`, `import`) and enforces strict **External Contracts & Operator Capabilities** vs. **Zero Internal Implementation Mechanics** (no internal database table names or DDLs). ### 4. Release Writer Skill (`skills/release-writer/SKILL.md`) Defines the non-verbose, partner-facing GFM format: -- **Executive Summary**: 1 single sentence (max 25 words). +- **Dynamic Executive Summary**: Scales length dynamically with release size (2–3 sentences for major releases, 1 punchy sentence for patch releases). - **Key Feature Updates**: **What's New** (1 paragraph combining description + benefit) followed by **Specific Capabilities** (bullet points with `[repo#PR](URL)` links). - **Improvements & Configuration Updates**: Bullet points extracting concrete enums (`custom_only`, `base_only`) and scaling limits (`max_workers`). - **Bug Fixes**: 3–5 high-level functional categories (Deployment, Ingestion, Serving APIs, UI). - ---- - -## Repository Mapping - -| Repository | Scope / Path Filter | Target Component & Image | -| :--- | :--- | :--- | -| `datacommonsorg/datacommons` | All PRs (`infra/dcp/`, `packages/`) | DCP Monorepo & Infra (`dcp`) | -| `datacommonsorg/website` | All PRs (excluding `cdc_data/`) | Core Services (`services`) $\rightarrow$ `gcr.io/datcom-ci/datacommons-services` | -| `datacommonsorg/mixer` | All PRs (`internal/server/`, `proto/`, `deploy/`) | Core Services (`services`) $\rightarrow$ `gcr.io/datcom-ci/datacommons-services` | -| `datacommonsorg/agent-toolkit` | All PRs (`src/datacommons_mcp/`) | Core Services (`services`) $\rightarrow$ `gcr.io/datcom-ci/datacommons-services` | -| `datacommonsorg/import` | `simple/` | Data Preprocessor (`preprocessing`) $\rightarrow$ `gcr.io/datcom-ci/datacommons-data` | -| `datacommonsorg/import` | `pipeline/ingestion/` | Dataflow Worker (`dataflow_worker`) $\rightarrow$ Dataflow Templates | -| `datacommonsorg/import` | `pipeline/workflow/ingestion-helper/` | Ingestion Helper (`ingestion_helper`) $\rightarrow$ `datacommons-ingestion-helper` | -| `datacommonsorg/import` | `pipeline/workflow/aggregation-helper/` | Postprocessing Helper (`postprocessing`) $\rightarrow$ `datacommons-aggregation-helper` | From f9e0b66686a13f5ff55daed6f3c25beedfd0d292 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 16:26:07 -0700 Subject: [PATCH 62/70] feat(deploy): establish dcp-context SKILL.md as Single Source of Truth for component and repository mapping --- deploy/generate_release_notes/SKILL.md | 24 ++++++++------ .../skills/dcp-context/SKILL.md | 33 +++++++------------ 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/deploy/generate_release_notes/SKILL.md b/deploy/generate_release_notes/SKILL.md index 3be98784..64415efc 100644 --- a/deploy/generate_release_notes/SKILL.md +++ b/deploy/generate_release_notes/SKILL.md @@ -9,16 +9,20 @@ This skill orchestrates the end-to-end generation of publication-ready, partner- --- -## Component & Container Image Registry - -| Component Key | Component Name | Container Image URI / Artifact | Source Repos & Content Focus | Output Verification File | -| :--- | :--- | :--- | :--- | :--- | -| `services` | Core Services (Website, Mixer, MCP Agent) | `gcr.io/datcom-ci/datacommons-services` | `datacommonsorg/website`
`datacommonsorg/mixer`
`datacommonsorg/agent-toolkit`
*(Serving APIs, SDMX 3.0, FastMCP, UI)* | `output/prs_services.txt` | -| `preprocessing` | Data Preprocessor | `gcr.io/datcom-ci/datacommons-data` | `datacommonsorg/import`
*(CSV/MCF validation, JSON-LD streaming batching)* | `output/prs_preprocessing.txt` | -| `dataflow_worker` | Dataflow Ingestion Worker | `us-docker.pkg.dev/datcom-ci/gcr.io/dataflow-templates/ingestion` | `datacommonsorg/import`
*(Dataflow pipelines, TFRecord loading, Spanner graph transforms)* | `output/prs_dataflow_worker.txt` | -| `ingestion_helper` | Ingestion Helper Service | `gcr.io/datcom-ci/datacommons-ingestion-helper` | `datacommonsorg/import`
*(Cloud Workflows status tracking, run history tables)* | `output/prs_ingestion_helper.txt` | -| `postprocessing` | Postprocessing Helper Service | `gcr.io/datcom-ci/datacommons-aggregation-helper` | `datacommonsorg/import`
*(Graph postprocessing rollups, StatVar/Place aggregations, summary store)* | `output/prs_postprocessing.txt` | -| `dcp_monorepo` | DCP Monorepo & Terraform Infra | DCP Monorepo | `datacommonsorg/datacommons`
*(Terraform modules, Admin CLI, deployment infra)* | `output/prs_dcp_monorepo.txt` | +## Component & Container Image Registry (Single Source of Truth) + +The authoritative mapping of component keys, container image URIs, source repositories, subdirectory path filters, and output verification files is defined in [`skills/dcp-context/SKILL.md`](skills/dcp-context/SKILL.md). + +Subagents must inspect `skills/dcp-context/SKILL.md` for full path filter and image URI details. Below is the active component key overview: + +| Component Key | Component Name | Container Image URI / Artifact | Target Verification File | +| :--- | :--- | :--- | :--- | +| `services` | Core Services (Website, Mixer, MCP Agent) | `gcr.io/datcom-ci/datacommons-services` | `output/prs_services.txt` | +| `preprocessing` | Data Preprocessor | `gcr.io/datcom-ci/datacommons-data` | `output/prs_preprocessing.txt` | +| `dataflow_worker` | Dataflow Ingestion Worker | `us-docker.pkg.dev/datcom-ci/gcr.io/dataflow-templates/ingestion` | `output/prs_dataflow_worker.txt` | +| `ingestion_helper` | Ingestion Helper Service | `gcr.io/datcom-ci/datacommons-ingestion-helper` | `output/prs_ingestion_helper.txt` | +| `postprocessing` | Postprocessing Helper Service | `gcr.io/datcom-ci/datacommons-aggregation-helper` | `output/prs_postprocessing.txt` | +| `dcp_monorepo` | DCP Monorepo & Terraform Infra | DCP Monorepo & Terraform Modules | `output/prs_dcp_monorepo.txt` | --- diff --git a/deploy/generate_release_notes/skills/dcp-context/SKILL.md b/deploy/generate_release_notes/skills/dcp-context/SKILL.md index d62bd89a..60de1434 100644 --- a/deploy/generate_release_notes/skills/dcp-context/SKILL.md +++ b/deploy/generate_release_notes/skills/dcp-context/SKILL.md @@ -9,30 +9,21 @@ This skill provides the domain context, repository mapping, and architectural pr --- -## 1. Core Architectural Overview +## 1. Core Architectural Overview & Single Source of Truth Mapping -Data Commons Platform (DCP) is a self-hosted, Cloud Spanner-backed deployment of Data Commons. It replaces legacy Bigtable with Cloud Spanner graph tables and vector embeddings. It features custom data ingestion pipelines, specialized serving APIs, and deployment automation across 6 core repositories: +Data Commons Platform (DCP) is a self-hosted, Cloud Spanner-backed deployment of Data Commons. It replaces legacy Bigtable with Cloud Spanner graph tables and vector embeddings. It features custom data ingestion pipelines, specialized serving APIs, and deployment automation across 6 core repositories. -1. `datacommonsorg/datacommons` (Monorepo & Infra): - - **Terraform Modules** (`infra/dcp/`, `infra/modules/`): Infrastructure provisioning for Spanner, Cloud Run, BigQuery, and Dataflow. - - **CLI Tools** (`packages/datacommons-cli/`): `datacommons admin init`, `datacommons admin deploy`. - - **Admin Portal** (`packages/datacommons-admin/`): Web management interface. +### 📌 Component & Repository Mapping Registry (SINGLE SOURCE OF TRUTH) +All skills and subagents MUST use this table as the single authoritative source of truth for component keys, source repositories, subdirectory path filters, target container images, and output verification files: -2. `datacommonsorg/website` (Web Application & Frontend): - - Serves UI pages (Explore, Visualization Tools, Place Browser) and REST API routing (`server/`, `static/`, `build/cdc_services/`). - -3. `datacommonsorg/mixer` (Core Serving Engine): - - High-performance gRPC graph and StatVar serving engine (`internal/server/`, `proto/`, `deploy/helm_charts/`, ESPv2 gateway). - - Serves SDMX 3.0 REST Data & Availability endpoints, `/v2/observation`, and vector search embeddings. - -4. `datacommonsorg/agent-toolkit` (Model Context Protocol / MCP): - - Model Context Protocol (MCP) server & FastMCP tools for AI agent integrations (`src/datacommons_mcp/`). - - Enables agentic research playbooks, multi-entity observation retrieval, and indicator search across custom or base instances. - -5. `datacommonsorg/import` (Ingestion Stack & Cloud Workflows): - - **Data Preprocessor** (`simple/`): CSV/MCF validation and streaming JSON-LD batching (built into `datacommons-data` container image). - - **Dataflow Ingestion Worker** (`pipeline/ingestion/`): Parallelized BigQuery/Spanner graph loading. - - **Cloud Workflow Helpers** (`pipeline/workflow/ingestion-helper/`, `pipeline/workflow/aggregation-helper/`): Ingestion status tracking and postprocessing aggregations (StatVar, Place, Entity rollups). +| Component Key | Component Name | Source Repositories & Subdirectory Filters | Container Image URI / Release Artifact | Output Verification File | +| :--- | :--- | :--- | :--- | :--- | +| `services` | Core Services (Website, Mixer, MCP Agent) | `datacommonsorg/website` (`server/`, `static/`, `build/cdc_services/`)
`datacommonsorg/mixer` (`internal/server/`, `proto/`, `deploy/`)
`datacommonsorg/agent-toolkit` (`src/datacommons_mcp/`) | `gcr.io/datcom-ci/datacommons-services` | `output/prs_services.txt` | +| `preprocessing` | Data Preprocessor | `datacommonsorg/import` (`simple/`) | `gcr.io/datcom-ci/datacommons-data` | `output/prs_preprocessing.txt` | +| `dataflow_worker` | Dataflow Ingestion Worker | `datacommonsorg/import` (`pipeline/ingestion/`) | `us-docker.pkg.dev/datcom-ci/gcr.io/dataflow-templates/ingestion` | `output/prs_dataflow_worker.txt` | +| `ingestion_helper` | Ingestion Helper Service | `datacommonsorg/import` (`pipeline/workflow/ingestion-helper/`) | `gcr.io/datcom-ci/datacommons-ingestion-helper` | `output/prs_ingestion_helper.txt` | +| `postprocessing` | Postprocessing Helper Service | `datacommonsorg/import` (`pipeline/workflow/aggregation-helper/`) | `gcr.io/datcom-ci/datacommons-aggregation-helper` | `output/prs_postprocessing.txt` | +| `dcp_monorepo` | DCP Monorepo & Terraform Infra | `datacommonsorg/datacommons` (`infra/dcp/`, `infra/modules/`, `packages/`) | DCP Monorepo & Terraform Modules | `output/prs_dcp_monorepo.txt` | --- From adcd05f0f4583d0a46467130451fa08069e0cc06 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 16:27:51 -0700 Subject: [PATCH 63/70] docs(deploy): rewrite README.md for developer readability with Quick Start at top and architecture at end --- deploy/generate_release_notes/README.md | 99 +++++++++---------------- 1 file changed, 37 insertions(+), 62 deletions(-) diff --git a/deploy/generate_release_notes/README.md b/deploy/generate_release_notes/README.md index 01deb31c..7e152b81 100644 --- a/deploy/generate_release_notes/README.md +++ b/deploy/generate_release_notes/README.md @@ -1,14 +1,40 @@ # Data Commons Platform (DCP) Release Notes Generator -An agentic, skill-driven tool suite for generating publication-ready, partner-facing release notes for the Data Commons Platform (DCP). +An agentic, skill-driven tool suite for generating publication-ready, partner-facing release notes for Data Commons Platform (DCP) releases. -The tool automatically extracts merged Pull Requests across all 6 core Data Commons repositories, classifies them according to platform layer and technical impact, filters out internal test noise and regressions, writes human-verifiable PR lists per container image (`output/prs_.txt`), and formats concise release notes tailored for developers and platform operators building on top of DCP. +--- + +## Quick Start (How to Use) + +Simply point your LLM coding assistant (e.g. Jetski / Gemini) at [`SKILL.md`](SKILL.md): + +> **Prompt Example**: +> *"Please read `deploy/generate_release_notes/SKILL.md` and generate release notes for version v1.1.0 to v1.1.1."* + +--- + +## How It Works (4 Automated Steps) + +1. **PR Extraction**: The agent spawns dedicated subagents to query merged Pull Requests across all Data Commons repositories for each component layer within the release window (`[prev_version .. new_version]`). +2. **Developer Verification Checkpoint**: The agent generates human-readable text files per component under `output/prs_*.txt`. You can open and inspect these files to verify extracted PRs, change summaries, DCP impact, and excluded noise. +3. **Release Delta Synthesis**: An agent analyzes the verified PR lists to distinguish true platform bug fixes present in prior releases from intermediate intra-release fixes, generating `output/IMAGE_DELTAS_.txt`. +4. **Final Release Notes Authoring**: The agent applies domain context and release-writing guidelines to generate the final publication-ready release notes: `output/RELEASE_NOTES_.md`. + +--- + +## Output Artifacts & Verification + +All output artifacts are generated into `deploy/generate_release_notes/output/`: + +- `output/prs_.txt`: Extracted PRs per component with **Change Summary**, **DCP Impact**, and an **Excluded PRs Audit Log** (with explicit reasons for every ignored PR). +- `output/IMAGE_DELTAS_.txt`: Intermediate summary of salient features, configuration updates, and true platform bug fixes per container image. +- `output/RELEASE_NOTES_.md`: Final release notes formatted for external developers and instance operators. --- -## Agentic Skill Suite Architecture +## Architecture & Skill Reference -The release notes generation pipeline is structured into 4 modular `SKILL.md` instruction sets. Point your LLM agent at these skills to execute the generation process: +The pipeline is organized into modular skill instruction sets under `deploy/generate_release_notes/`: ``` deploy/generate_release_notes/ @@ -19,66 +45,15 @@ deploy/generate_release_notes/ │ ├── release-delta-synthesis/ │ │ └── SKILL.md <-- 3. Release Delta Synthesis Skill (Image Delta Analysis) │ ├── dcp-context/ -│ │ └── SKILL.md <-- 4. DCP Domain Context & Architectural Map +│ │ └── SKILL.md <-- 4. DCP Domain Context & Architectural Map (Single Source of Truth) │ └── release-writer/ │ └── SKILL.md <-- 5. Partner-Facing Release Notes Writer └── output/ <-- Verification & Output Directory - ├── prs_services.txt <-- Verified PRs for Core Services (Website, Mixer, MCP) - ├── prs_preprocessing.txt <-- Verified PRs for Data Preprocessor (datacommons-data) - ├── prs_dataflow_worker.txt <-- Verified PRs for Dataflow Worker - ├── prs_ingestion_helper.txt <-- Verified PRs for Ingestion Helper - ├── prs_postprocessing.txt <-- Verified PRs for Postprocessing Helper - ├── prs_dcp_monorepo.txt <-- Verified PRs for DCP Monorepo & Infra - ├── IMAGE_DELTAS_v1.1.1.txt <-- Intermediate Image Delta Summary (Delta vs. Previous Release) - └── RELEASE_NOTES_v1.1.1.md <-- Final Publication-Ready Release Notes ``` ---- - -## Developer Usage Instructions (Prompting Your LLM Agent) - -### Step 1: Point Your LLM Agent at the Orchestrator Skill -To generate release notes for a release range, point your LLM agent at [`SKILL.md`](SKILL.md): - -> **Prompt Example**: -> *"Please read `deploy/generate_release_notes/SKILL.md` and generate release notes for version v1.1.0 to v1.1.1."* - -### Step 2: PR Extraction & Intermediate Verification Files -Your LLM agent will spawn concurrent subagents using `skills/pr-extraction/SKILL.md` and `skills/dcp-context/SKILL.md` to: -1. Resolve container image tags across Artifact Registry via `gcloud`. -2. Query merged Pull Requests across all 6 Data Commons repositories via `gh pr list`. -3. Analyze PR content, change summary, and DCP impact. -4. Output human-verifiable text files per container image into `deploy/generate_release_notes/output/prs_*.txt`. - -Developers can open and inspect these `.txt` files to verify that all relevant PRs for each image are correctly captured, and inspect the **Irrelevant / Excluded PRs** audit log at the bottom. - -### Step 3: Release Delta Synthesis (Delta vs. Previously Published Image) -Your LLM agent will spawn a **Release Delta Synthesis Subagent** using `skills/release-delta-synthesis/SKILL.md` to: -1. Analyze all `output/prs_*.txt` files. -2. Distinguish true platform bug fixes present in `` vs. intermediate bug fixes introduced and resolved within `` (omitting intra-release fixes). -3. Summarize salient features and operator capabilities added to each container image relative to ``. -4. Output the intermediate delta summary file: `deploy/generate_release_notes/output/IMAGE_DELTAS_.txt`. - -### Step 4: Final Release Notes Authoring -Your LLM agent will load `skills/release-writer/SKILL.md` and author the publication-ready release notes: -`deploy/generate_release_notes/output/RELEASE_NOTES_.md` - ---- - -## Modular Skill Breakdown - -### 1. Orchestrator Skill (`SKILL.md`) -The master entrypoint skill that coordinates subagent execution, manages the step-by-step pipeline, and ensures intermediate verification files are generated before writing the final release notes. - -### 2. PR Extraction Skill (`skills/pr-extraction/SKILL.md`) -Contains exact commands and rules for `gcloud container images list-tags` resolution (with strict user prompt on missing tags), date-range `gh pr list` queries, non-production test filtering, and intermediate regression exclusion. - -### 3. DCP Domain Context Skill (`skills/dcp-context/SKILL.md`) -Defines the architectural map across all 6 core repositories (`datacommons`, `website`, `mixer`, `agent-toolkit`, `import`) and enforces strict **External Contracts & Operator Capabilities** vs. **Zero Internal Implementation Mechanics** (no internal database table names or DDLs). - -### 4. Release Writer Skill (`skills/release-writer/SKILL.md`) -Defines the non-verbose, partner-facing GFM format: -- **Dynamic Executive Summary**: Scales length dynamically with release size (2–3 sentences for major releases, 1 punchy sentence for patch releases). -- **Key Feature Updates**: **What's New** (1 paragraph combining description + benefit) followed by **Specific Capabilities** (bullet points with `[repo#PR](URL)` links). -- **Improvements & Configuration Updates**: Bullet points extracting concrete enums (`custom_only`, `base_only`) and scaling limits (`max_workers`). -- **Bug Fixes**: 3–5 high-level functional categories (Deployment, Ingestion, Serving APIs, UI). +### Skill Breakdown: +- **Orchestrator (`SKILL.md`)**: Coordinates subagents across component layers and manages the step-by-step workflow. +- **PR Extraction (`skills/pr-extraction/SKILL.md`)**: Instructions for date-range `gh pr list` queries, Artifact Registry tag resolution (with prompt on missing tags), and noise filtering. +- **Release Delta Synthesis (`skills/release-delta-synthesis/SKILL.md`)**: Rules for image delta synthesis, separating Mixer, MCP Agent Toolkit, and Website UI into dedicated sections. +- **DCP Domain Context (`skills/dcp-context/SKILL.md`)**: **Single Source of Truth** for component keys, repository mappings, subdirectory path filters, container image URIs, and persona guidelines. +- **Release Writer (`skills/release-writer/SKILL.md`)**: Guidelines for authoring publication-ready release notes with dynamic Executive Summary scaling and two-tier feature formatting (**What's New** + **Specific Capabilities** with `[repo#PR](URL)` links). From 0027cac029f8ed564df617451813556e290a7c50 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 16:33:25 -0700 Subject: [PATCH 64/70] docs(deploy): replace Jetski with Antigravity in README.md --- deploy/generate_release_notes/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/generate_release_notes/README.md b/deploy/generate_release_notes/README.md index 7e152b81..941c1e20 100644 --- a/deploy/generate_release_notes/README.md +++ b/deploy/generate_release_notes/README.md @@ -6,7 +6,7 @@ An agentic, skill-driven tool suite for generating publication-ready, partner-fa ## Quick Start (How to Use) -Simply point your LLM coding assistant (e.g. Jetski / Gemini) at [`SKILL.md`](SKILL.md): +Simply point your LLM coding assistant (e.g. Antigravity / Gemini) at [`SKILL.md`](SKILL.md): > **Prompt Example**: > *"Please read `deploy/generate_release_notes/SKILL.md` and generate release notes for version v1.1.0 to v1.1.1."* From 48b438a8e1d6f8fb61fd82cd9d92dc4c39f6d36d Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 16:41:33 -0700 Subject: [PATCH 65/70] docs(deploy): remove hardcoded repository count from SKILL.md description --- deploy/generate_release_notes/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/generate_release_notes/SKILL.md b/deploy/generate_release_notes/SKILL.md index 64415efc..72c04b3c 100644 --- a/deploy/generate_release_notes/SKILL.md +++ b/deploy/generate_release_notes/SKILL.md @@ -1,6 +1,6 @@ --- name: dcp-release-notes -description: Master orchestrator skill for generating publication-ready, partner-facing Data Commons Platform (DCP) release notes across all 6 core repositories using agentic subagents. +description: Master orchestrator skill for generating publication-ready, partner-facing Data Commons Platform (DCP) release notes across core repositories and platform components using agentic subagents. --- # DCP Release Notes Generator (Orchestrator Skill) From fc11ca7db911867a6f69cdea2437f8d9c875549c Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 16:46:04 -0700 Subject: [PATCH 66/70] fix(deploy): correct infra path from infra/modules/ to infra/dcp/ in dcp-context SKILL.md --- deploy/generate_release_notes/skills/dcp-context/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/generate_release_notes/skills/dcp-context/SKILL.md b/deploy/generate_release_notes/skills/dcp-context/SKILL.md index 60de1434..ae0fe779 100644 --- a/deploy/generate_release_notes/skills/dcp-context/SKILL.md +++ b/deploy/generate_release_notes/skills/dcp-context/SKILL.md @@ -23,7 +23,7 @@ All skills and subagents MUST use this table as the single authoritative source | `dataflow_worker` | Dataflow Ingestion Worker | `datacommonsorg/import` (`pipeline/ingestion/`) | `us-docker.pkg.dev/datcom-ci/gcr.io/dataflow-templates/ingestion` | `output/prs_dataflow_worker.txt` | | `ingestion_helper` | Ingestion Helper Service | `datacommonsorg/import` (`pipeline/workflow/ingestion-helper/`) | `gcr.io/datcom-ci/datacommons-ingestion-helper` | `output/prs_ingestion_helper.txt` | | `postprocessing` | Postprocessing Helper Service | `datacommonsorg/import` (`pipeline/workflow/aggregation-helper/`) | `gcr.io/datcom-ci/datacommons-aggregation-helper` | `output/prs_postprocessing.txt` | -| `dcp_monorepo` | DCP Monorepo & Terraform Infra | `datacommonsorg/datacommons` (`infra/dcp/`, `infra/modules/`, `packages/`) | DCP Monorepo & Terraform Modules | `output/prs_dcp_monorepo.txt` | +| `dcp_monorepo` | DCP Monorepo & Terraform Infra | `datacommonsorg/datacommons` (`infra/dcp/`, `packages/`) | DCP Monorepo & Terraform Modules | `output/prs_dcp_monorepo.txt` | --- From 760bfc7b22d1fa02d161e355a8dca0e5e25a58ef Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 16:47:28 -0700 Subject: [PATCH 67/70] refactor(deploy): move persona and contract principles to release-writer SKILL.md and clean dcp-context SKILL.md --- .../skills/dcp-context/SKILL.md | 31 ------------------- .../skills/release-writer/SKILL.md | 4 ++- 2 files changed, 3 insertions(+), 32 deletions(-) diff --git a/deploy/generate_release_notes/skills/dcp-context/SKILL.md b/deploy/generate_release_notes/skills/dcp-context/SKILL.md index ae0fe779..e653de86 100644 --- a/deploy/generate_release_notes/skills/dcp-context/SKILL.md +++ b/deploy/generate_release_notes/skills/dcp-context/SKILL.md @@ -24,34 +24,3 @@ All skills and subagents MUST use this table as the single authoritative source | `ingestion_helper` | Ingestion Helper Service | `datacommonsorg/import` (`pipeline/workflow/ingestion-helper/`) | `gcr.io/datcom-ci/datacommons-ingestion-helper` | `output/prs_ingestion_helper.txt` | | `postprocessing` | Postprocessing Helper Service | `datacommonsorg/import` (`pipeline/workflow/aggregation-helper/`) | `gcr.io/datcom-ci/datacommons-aggregation-helper` | `output/prs_postprocessing.txt` | | `dcp_monorepo` | DCP Monorepo & Terraform Infra | `datacommonsorg/datacommons` (`infra/dcp/`, `packages/`) | DCP Monorepo & Terraform Modules | `output/prs_dcp_monorepo.txt` | - ---- - -## 2. Architectural Boundary & Persona Principles - -### Focus on External Contracts & Operator Capabilities -- **Partner & Operator Focus**: Write specifically for external developers, data engineers, and instance operators building ON TOP OF DCP. -- **User Capabilities**: Frame every feature and improvement around *what the user can now do*, *which input formats are supported*, or *how compute resources scale*. -- **Extract Concrete Enums & Configuration Values**: Always extract valid enums (`custom_only`, `base_only`, `base_and_custom`), CLI flags (`--instance_name`), and scaling bounds (`max_workers`, BigQuery slots). - -### Zero Internal Implementation Mechanics (STRICT) -- **NO Internal Database Terms**: NEVER output feature titles or section names containing internal database table names, schema DDLs, or storage migration mechanics (e.g. no "KeyValueStore", "Spanner Graph DDL", "Bigtable Cutover", "Database Schema Modification"). -- **Frame Performance Speedups Around User Impact**: If an internal storage or cache layer change improves serving speed, title it around user impact: **"API Serving Latency & Query Throughput"** or **"Faster API Response Speed"** without naming internal database tables. - ---- - -## 3. Section Mapping & Release Content Principles - -Rather than using arbitrary internal categories, map changes directly into the three standard release notes sections based on technical impact: - -1. **Key Feature Updates**: - - Major, high-impact capabilities introduced in this release (e.g., SDMX 3.0 REST Data & Availability APIs, FastMCP AI agent tools, streaming JSON-LD preprocessors, vector search embeddings). - - Must follow the non-verbose format: **What's New** (1 paragraph combining description + benefit) followed by **Specific Capabilities** (bullet points with `[repo#PR](URL)` links). - -2. **Improvements & Configuration Updates**: - - Incremental enhancements, operator tools, Terraform variables (`max_workers`, BigQuery slots), CLI flags (`--instance_name`), and scaling optimizations. - - Must extract concrete enums (e.g. `custom_only`, `base_only`) and explicit configuration parameters. - -3. **Bug Fixes**: - - Synthesized into 3 to 5 functional categories (*Deployment & Infrastructure*, *Ingestion Pipeline Reliability*, *Serving API & Query Robustness*, *Web UI & Visualization*). - - Must ONLY include true platform bug fixes present in prior releases (excluding intra-release intermediate fixes). diff --git a/deploy/generate_release_notes/skills/release-writer/SKILL.md b/deploy/generate_release_notes/skills/release-writer/SKILL.md index 4678231b..a7c7f0d6 100644 --- a/deploy/generate_release_notes/skills/release-writer/SKILL.md +++ b/deploy/generate_release_notes/skills/release-writer/SKILL.md @@ -9,8 +9,10 @@ This skill provides step-by-step instructions for authoring non-verbose, publica --- -## 1. Writing Style & Tone Constraints +## 1. Persona & Writing Style Constraints +- **Partner & Operator Persona**: Write specifically for external developers, data engineers, and instance operators building ON TOP OF DCP. Frame features around user capabilities (*"what the user can now do"*, *input formats supported*, *scaling controls*). +- **Zero Internal Database Terms (STRICT)**: NEVER output feature titles or section names containing internal database table names, schema DDLs, or storage migration mechanics (e.g. no "KeyValueStore", "Spanner Graph DDL", "Bigtable Cutover"). Frame latency improvements around user impact (e.g. *"API Serving Latency & Query Throughput"*). - **Tone**: Direct, factual, punchy, senior-engineer technical changelog. Active voice for features ("You can now..."), past tense for bugs ("Resolved..."). - **BANNED AI FLUFF WORDS (STRICT)**: DO NOT use AI cliché words: `seamlessly`, `empower`, `leveraging`, `robust`, `overhaul`, `delivers a major`, `comprehensive`, `fosters`, `game-changing`, `cutting-edge`, `paradigm`. - **DYNAMIC EXECUTIVE SUMMARY**: From 79a46790908df74b7054a18eb1cec3b954431065 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 16:50:04 -0700 Subject: [PATCH 68/70] docs(deploy): rewrite dcp-context SKILL.md with DCP definition, user touchpoints vs internal mechanics, and single source of truth mapping --- .../skills/dcp-context/SKILL.md | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/deploy/generate_release_notes/skills/dcp-context/SKILL.md b/deploy/generate_release_notes/skills/dcp-context/SKILL.md index e653de86..be60154e 100644 --- a/deploy/generate_release_notes/skills/dcp-context/SKILL.md +++ b/deploy/generate_release_notes/skills/dcp-context/SKILL.md @@ -1,19 +1,55 @@ --- name: dcp-context -description: Architectural reference and domain context for Data Commons Platform (DCP) release notes generation. +description: Architectural reference, domain context, and component map for Data Commons Platform (DCP) release notes generation. --- # Data Commons Platform (DCP) Domain Context & Architectural Map -This skill provides the domain context, repository mapping, and architectural principles for writing publication-ready, partner-facing DCP release notes. +This skill provides the domain context, platform architecture, user touchpoints, and single source of truth component mapping for evaluating Pull Request relevance and generating partner-facing release notes. --- -## 1. Core Architectural Overview & Single Source of Truth Mapping +## 1. What is Data Commons Platform (DCP)? -Data Commons Platform (DCP) is a self-hosted, Cloud Spanner-backed deployment of Data Commons. It replaces legacy Bigtable with Cloud Spanner graph tables and vector embeddings. It features custom data ingestion pipelines, specialized serving APIs, and deployment automation across 6 core repositories. +- **Data Commons**: An open-knowledge graph that unifies public datasets across demographics, economics, climate, health, and geography into a standardized, interconnected graph structure. +- **Data Commons Platform (DCP)**: The self-hosted, enterprise-grade deployment of Data Commons. It allows organizations and partners to deploy an isolated Data Commons instance backed by Cloud Spanner, load custom proprietary datasets alongside public Data Commons data, expose standardized SDMX 3.0 REST and FastMCP AI agent interfaces, and manage infrastructure via Terraform and CLI automation. + +--- + +## 2. User & Operator Touchpoints vs. Internal Implementation + +When analyzing Pull Requests and synthesizing release notes, agents MUST distinguish between **external user/operator touchpoints** (what partners interact with) and **internal implementation mechanics** (non-user facing code). + +### A. External User & Operator Touchpoints (PUBLIC RELEASE RELEVANT) +These represent the interfaces, contracts, and capabilities that partners, developers, data engineers, and instance operators directly interact with: + +1. **Serving APIs & Protocols**: + - SDMX 3.0 REST Data and Availability endpoints (`/sdmx/v3/rest/data/...`, `/sdmx/v3/rest/availability/...`). + - Observations V2 API (`/v2/observation`) and Mixer gRPC graph endpoints. + - Place containment expansion (`containedInPlace+`), time-series filtering (`TIME_PERIOD`). +2. **AI Agent Integration (MCP / Model Context Protocol)**: + - FastMCP tools for AI agent research playbooks (`get_multi_entity_observations`, `search_indicators`, `get_variable_metadata`). + - Indicator search target scopes (`custom_only`, `base_only`, `base_and_custom`). +3. **Web Applications & Exploration Tools**: + - Explore UI, Download Tool, Place Browser, Croissant JSON-LD dataset metadata. +4. **Infrastructure & Deployment Automation**: + - Terraform modules (`infra/dcp/`), variables (`ingestion_dataflow_max_workers`, `spanner_processing_units`), and IAM role configurations. + - Admin CLI (`datacommons admin init`, `datacommons admin deploy`) and Admin Portal web interface. + - Vector search profile configurations (`--spanner_search_config_path`). +5. **Data Ingestion Inputs**: + - Custom CSV/MCF dataset formats, column mapping definitions, and batch import job configurations. + +### B. Internal Implementation Mechanics (NON-USER FACING — DO NOT EXPOSE) +These are internal engine mechanics that partners do NOT interact with directly. They should be framed around high-level user impact (e.g. *"98% lower query latency"*) without exposing internal table names or DDLs: +- Internal Cloud Spanner DDL graph table schemas and KeyValueStore tables. +- Dataflow TFRecord chunking and intermediate GCS staging paths. +- Cloud Workflows internal execution IDs and status tracking tables (`IngestionHistory`). +- Internal SQL parameter unrolling and join ordering optimizations. + +--- + +## 3. Component & Repository Registry (SINGLE SOURCE OF TRUTH) -### 📌 Component & Repository Mapping Registry (SINGLE SOURCE OF TRUTH) All skills and subagents MUST use this table as the single authoritative source of truth for component keys, source repositories, subdirectory path filters, target container images, and output verification files: | Component Key | Component Name | Source Repositories & Subdirectory Filters | Container Image URI / Release Artifact | Output Verification File | From d327f4cb418820b51e55a04f7168bc910dbb45a2 Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 17:12:22 -0700 Subject: [PATCH 69/70] refactor(deploy): update release notes skill suite to 10/10 SOP standards with Prime Directives, CoT phases, and zero fluff guidelines --- deploy/generate_release_notes/SKILL.md | 83 ++++++++++--------- .../skills/dcp-context/SKILL.md | 72 ++++++++++------ .../skills/pr-extraction/SKILL.md | 60 ++++++++------ .../skills/release-delta-synthesis/SKILL.md | 63 +++++++------- .../skills/release-writer/SKILL.md | 52 +++++++++--- 5 files changed, 201 insertions(+), 129 deletions(-) diff --git a/deploy/generate_release_notes/SKILL.md b/deploy/generate_release_notes/SKILL.md index 72c04b3c..3f12c8ed 100644 --- a/deploy/generate_release_notes/SKILL.md +++ b/deploy/generate_release_notes/SKILL.md @@ -5,67 +5,70 @@ description: Master orchestrator skill for generating publication-ready, partner # DCP Release Notes Generator (Orchestrator Skill) -This skill orchestrates the end-to-end generation of publication-ready, partner-facing release notes for the Data Commons Platform (DCP). It coordinates specialized subagents to extract PRs per container image, write human-readable verification `.txt` files for developer review, apply domain context, and author concise release notes. +**PRIME DIRECTIVE**: You are an expert Data Commons Release Engineer. Your objective is to orchestrate the end-to-end generation of publication-ready, partner-facing Data Commons Platform (DCP) release notes by coordinating specialized subagents across core repositories and platform components. --- -## Component & Container Image Registry (Single Source of Truth) +## Input & Output Contracts -The authoritative mapping of component keys, container image URIs, source repositories, subdirectory path filters, and output verification files is defined in [`skills/dcp-context/SKILL.md`](skills/dcp-context/SKILL.md). +### Inputs +- **`prev_version`**: Previous release tag (e.g., `v1.1.0`). +- **`new_version`**: Target release tag (e.g., `v1.1.1`). -Subagents must inspect `skills/dcp-context/SKILL.md` for full path filter and image URI details. Below is the active component key overview: +### Target Output Artifacts +- **Raw PR Verification Files**: `deploy/generate_release_notes/output/prs_*.txt` +- **Unified Image Delta Summary**: `deploy/generate_release_notes/output/IMAGE_DELTAS_.txt` +- **Publication-Ready Release Notes**: `deploy/generate_release_notes/output/RELEASE_NOTES_.md` -| Component Key | Component Name | Container Image URI / Artifact | Target Verification File | -| :--- | :--- | :--- | :--- | -| `services` | Core Services (Website, Mixer, MCP Agent) | `gcr.io/datcom-ci/datacommons-services` | `output/prs_services.txt` | -| `preprocessing` | Data Preprocessor | `gcr.io/datcom-ci/datacommons-data` | `output/prs_preprocessing.txt` | -| `dataflow_worker` | Dataflow Ingestion Worker | `us-docker.pkg.dev/datcom-ci/gcr.io/dataflow-templates/ingestion` | `output/prs_dataflow_worker.txt` | -| `ingestion_helper` | Ingestion Helper Service | `gcr.io/datcom-ci/datacommons-ingestion-helper` | `output/prs_ingestion_helper.txt` | -| `postprocessing` | Postprocessing Helper Service | `gcr.io/datcom-ci/datacommons-aggregation-helper` | `output/prs_postprocessing.txt` | -| `dcp_monorepo` | DCP Monorepo & Terraform Infra | DCP Monorepo & Terraform Modules | `output/prs_dcp_monorepo.txt` | +--- + +## Component & Container Image Registry Reference + +The authoritative mapping of component keys, container image URIs, source repositories, subdirectory path filters, and output verification files is defined strictly in **[`skills/dcp-context/SKILL.md`](skills/dcp-context/SKILL.md)** (Single Source of Truth). + +The orchestrator MUST read `skills/dcp-context/SKILL.md` dynamically to inspect the active component registry without hardcoding component lists in this file. --- -## Workflow Instructions +## Workflow Execution SOP -When requested to generate release notes (e.g., *"Generate release notes for v1.1.0 to v1.1.1"*): +### Step 0: Mandated Orchestrator Thinking Phase +Before executing steps, open a `` block to record: +1. Format validation for `` and `` (verify both follow semver `vX.Y.Z` format). +2. Verification of `deploy/generate_release_notes/output/` directory creation. +3. Verification of `skills/dcp-context/SKILL.md` accessibility. +4. Orchestration plan to spawn extraction subagents concurrently across all registry rows. ### Step 1: Version Resolution & Output Directory Setup -1. Identify the previous release tag (``, e.g., `v1.1.0`) and target release tag (``, e.g., `v1.1.1`). -2. Ensure `deploy/generate_release_notes/output/` directory exists. +1. Validate the previous release tag (``, e.g., `v1.1.0`) and target release tag (``, e.g., `v1.1.1`). +2. Create the `deploy/generate_release_notes/output/` directory if it does not already exist. > [!IMPORTANT] > **DO NOT resolve image tags or run `gcloud` commands in the orchestrator.** > The orchestrator MUST NOT query Artifact Registry or inspect image creation timestamps up front. Simply pass the raw version strings (`` and ``) to each subagent and let them resolve their assigned image tags concurrently. -### Step 2: Spawn PR Extraction Subagents -Call `invoke_subagent` to spawn subagents concurrently across the components above. +### Step 2: Dynamically Spawn PR Extraction Subagents +1. Read the **Component & Repository Registry** table in [`skills/dcp-context/SKILL.md`](skills/dcp-context/SKILL.md). +2. For **each row** in the Component Registry table, call `invoke_subagent` to spawn a dedicated extraction subagent concurrently. Provide each subagent with: -1. The **PR Extraction Skill**: [`skills/pr-extraction/SKILL.md`](skills/pr-extraction/SKILL.md). -2. The **DCP Context Skill**: [`skills/dcp-context/SKILL.md`](skills/dcp-context/SKILL.md). -3. Its assigned **Component Name**, **Image URI**, **Source Repos**, ``, ``, and target **Output Verification File**. - -#### Dedicated Subagent Tasks (1-to-1 with Component Output Files): -- **Subagent 1 (`services-extractor`)**: Extract PRs for `gcr.io/datcom-ci/datacommons-services` from `website`, `mixer`, `agent-toolkit` $\rightarrow$ write `output/prs_services.txt`. -- **Subagent 2 (`preprocessing-extractor`)**: Extract PRs for `gcr.io/datcom-ci/datacommons-data` (preprocessor) from `import` repo $\rightarrow$ write `output/prs_preprocessing.txt`. -- **Subagent 3 (`dataflow-worker-extractor`)**: Extract PRs for Dataflow Ingestion Worker from `import` repo $\rightarrow$ write `output/prs_dataflow_worker.txt`. -- **Subagent 4 (`ingestion-helper-extractor`)**: Extract PRs for Ingestion Helper Service from `import` repo $\rightarrow$ write `output/prs_ingestion_helper.txt`. -- **Subagent 5 (`postprocessing-extractor`)**: Extract PRs for Postprocessing Helper Service from `import` repo $\rightarrow$ write `output/prs_postprocessing.txt`. -- **Subagent 6 (`monorepo-extractor`)**: Extract PRs for `datacommonsorg/datacommons` monorepo & Terraform infra $\rightarrow$ write `output/prs_dcp_monorepo.txt`. - -### Step 3: Verification Checkpoint & Release Delta Synthesis Subagent -1. Notify the developer that raw PR verification files have been generated under `deploy/generate_release_notes/output/prs_*.txt` for review. -2. Call `invoke_subagent` to spawn a specialized **Release Delta Synthesis Subagent** (`delta-synthesizer`). -3. Provide the subagent with the **Release Delta Synthesis Skill**: [`skills/release-delta-synthesis/SKILL.md`](skills/release-delta-synthesis/SKILL.md). -4. The subagent will: - - Read all `output/prs_*.txt` files. - - Investigate and distinguish true bug fixes present in `` vs. intermediate bug fixes introduced and fixed within `` (omitting intra-release fixes). - - Summarize salient features and configuration updates per container image relative to ``. - - Output the unified image delta summary to: `deploy/generate_release_notes/output/IMAGE_DELTAS_.txt`. +- The **PR Extraction Skill**: [`skills/pr-extraction/SKILL.md`](skills/pr-extraction/SKILL.md). +- The **DCP Context Skill**: [`skills/dcp-context/SKILL.md`](skills/dcp-context/SKILL.md). +- Assigned `component_key`, `component_name`, `image_uri`, `source_repos`, ``, ``, and `output_file` from that row. + +### Step 3: Verification Checkpoint & Release Delta Synthesis +1. Verify that all expected `deploy/generate_release_notes/output/prs_*.txt` files have been written successfully by the subagents. +2. Notify the developer that raw PR verification files under `deploy/generate_release_notes/output/prs_*.txt` are ready for review. +3. Call `invoke_subagent` to spawn a specialized **Release Delta Synthesis Subagent** (`delta-synthesizer`). +4. Provide the subagent with the **Release Delta Synthesis Skill**: [`skills/release-delta-synthesis/SKILL.md`](skills/release-delta-synthesis/SKILL.md). +5. Verify that the subagent outputs the unified image delta summary to `deploy/generate_release_notes/output/IMAGE_DELTAS_.txt`. + +> [!WARNING] +> If any extraction subagent fails or fails to write its output verification file, DO NOT proceed to Step 4 silently. Log an explicit warning to the developer detailing which component failed and ask how to proceed. ### Step 4: Author Publication-Ready Release Notes 1. Read the **DCP Domain Context Skill**: [`skills/dcp-context/SKILL.md`](skills/dcp-context/SKILL.md). 2. Read the **Release Writer Skill**: [`skills/release-writer/SKILL.md`](skills/release-writer/SKILL.md). 3. Read `deploy/generate_release_notes/output/IMAGE_DELTAS_.txt`. -4. Author the final release notes from the verified image delta summary into: `deploy/generate_release_notes/output/RELEASE_NOTES_.md`. +4. Author the final release notes from the verified image delta summary into `deploy/generate_release_notes/output/RELEASE_NOTES_.md`. +5. Display a summary of generated artifacts to the developer. diff --git a/deploy/generate_release_notes/skills/dcp-context/SKILL.md b/deploy/generate_release_notes/skills/dcp-context/SKILL.md index be60154e..51f2225a 100644 --- a/deploy/generate_release_notes/skills/dcp-context/SKILL.md +++ b/deploy/generate_release_notes/skills/dcp-context/SKILL.md @@ -5,7 +5,17 @@ description: Architectural reference, domain context, and component map for Data # Data Commons Platform (DCP) Domain Context & Architectural Map -This skill provides the domain context, platform architecture, user touchpoints, and single source of truth component mapping for evaluating Pull Request relevance and generating partner-facing release notes. +**PRIME DIRECTIVE**: You are an expert Data Commons Architectural & Domain Analyst. Your objective is to provide the authoritative architectural context, user touchpoint principles, and component registry for evaluating PR relevance and framing partner-facing release notes across all Data Commons Platform components. + +--- + +## Input & Output Contracts + +### Inputs +- **PR Metadata & Code Footprints**: PR title, body, changed files, diffs, and labels extracted from source repositories. + +### Target Output Context +- **Relevance Classification**: `RELEVANT_USER_CAPABILITY`, `RELEVANT_OPERATOR_TOOL`, `RELEVANT_BUG_FIX`, or `EXCLUDED_INTERNAL_MECHANIC`. --- @@ -16,36 +26,46 @@ This skill provides the domain context, platform architecture, user touchpoints, --- -## 2. User & Operator Touchpoints vs. Internal Implementation - -When analyzing Pull Requests and synthesizing release notes, agents MUST distinguish between **external user/operator touchpoints** (what partners interact with) and **internal implementation mechanics** (non-user facing code). - -### A. External User & Operator Touchpoints (PUBLIC RELEASE RELEVANT) -These represent the interfaces, contracts, and capabilities that partners, developers, data engineers, and instance operators directly interact with: - -1. **Serving APIs & Protocols**: - - SDMX 3.0 REST Data and Availability endpoints (`/sdmx/v3/rest/data/...`, `/sdmx/v3/rest/availability/...`). - - Observations V2 API (`/v2/observation`) and Mixer gRPC graph endpoints. - - Place containment expansion (`containedInPlace+`), time-series filtering (`TIME_PERIOD`). -2. **AI Agent Integration (MCP / Model Context Protocol)**: - - FastMCP tools for AI agent research playbooks (`get_multi_entity_observations`, `search_indicators`, `get_variable_metadata`). - - Indicator search target scopes (`custom_only`, `base_only`, `base_and_custom`). -3. **Web Applications & Exploration Tools**: - - Explore UI, Download Tool, Place Browser, Croissant JSON-LD dataset metadata. -4. **Infrastructure & Deployment Automation**: - - Terraform modules (`infra/dcp/`), variables (`ingestion_dataflow_max_workers`, `spanner_processing_units`), and IAM role configurations. - - Admin CLI (`datacommons admin init`, `datacommons admin deploy`) and Admin Portal web interface. - - Vector search profile configurations (`--spanner_search_config_path`). -5. **Data Ingestion Inputs**: - - Custom CSV/MCF dataset formats, column mapping definitions, and batch import job configurations. - -### B. Internal Implementation Mechanics (NON-USER FACING — DO NOT EXPOSE) -These are internal engine mechanics that partners do NOT interact with directly. They should be framed around high-level user impact (e.g. *"98% lower query latency"*) without exposing internal table names or DDLs: +## 2. User & Operator Touchpoint Principles for PR Relevance + +When analyzing Pull Requests and synthesizing release notes, agents MUST categorize changes based on **where and how the user or operator interacts with the platform**: + +### A. Data Input & Ingestion Pipeline (What Data Engineers & Operators Care About) +- **Data Input Configurations & Schemas**: Anything that changes **what types of input are accepted** by the preprocessor (custom CSV/MCF formats, column mapping definitions, schema validation rules, subject node integrity). +- **CLI & Operational Control**: `datacommons admin` CLI parameters, flags (`--instance_name`), and deployment automation. +- **Ingestion Speed, Performance, & Accuracy**: While import is running, operators care deeply about **throughput, execution speed, multi-threaded parsing, streaming JSON-LD batching, failure resilience, and data accuracy**. + +### B. Serving & Data Access (How Users & AI Interact With Their Data) +- **Mixer Serving APIs (Primary Data Touchpoint)**: Users care deeply about the **shape and speed** of Mixer APIs (SDMX 3.0 REST Data & Availability endpoints, all `/v2/` Mixer REST & gRPC endpoints, place containment expansion `containedInPlace+`, query latency). This is their primary avenue for interacting with their data! +- **MCP Agent Tools & Capabilities (AI Touchpoint)**: FastMCP tools (`get_multi_entity_observations`, `search_indicators`, `get_variable_metadata`) and target scopes (`custom_only`, `base_only`) — because this is a primary avenue for how AI agents and researchers query and analyze their data! +- **Web Applications & Exploration UI**: Explore UI, Download Tool, Place Browser, Croissant JSON-LD dataset metadata — how end-users visualize, query, and export datasets. +- **Infrastructure & Scaling Controls**: Terraform modules (`infra/dcp/`), variables (`ingestion_dataflow_max_workers`, `spanner_processing_units`), and vector search profile configurations (`--spanner_search_config_path`). + +### C. Internal Implementation Mechanics (Non-User Facing Noise — DO NOT EXPOSE) +These are internal engine mechanics that partners do NOT interact with directly. They should be framed around **high-level user impact** (e.g., *"98% lower query latency"*) without exposing internal table names or DDLs: - Internal Cloud Spanner DDL graph table schemas and KeyValueStore tables. - Dataflow TFRecord chunking and intermediate GCS staging paths. - Cloud Workflows internal execution IDs and status tracking tables (`IngestionHistory`). - Internal SQL parameter unrolling and join ordering optimizations. +### D. Sequential Decision SOP for Evaluating PR Relevance + +When evaluating any PR against domain context, follow this exact step-by-step sequence: + +1. **Step 1: Identify Target Component Layer**: Match modified paths against the Component Registry table (Section 3). +2. **Step 2: Evaluate Touchpoint Category**: + - Check if the PR alters Data Input / Ingestion (Section 2.A) $\rightarrow$ Classify as **`RELEVANT_DATA_INPUT_OR_INGESTION`**. + - Check if the PR alters Serving APIs, MCP tools, or UI (Section 2.B) $\rightarrow$ Classify as **`RELEVANT_SERVING_OR_UI`**. + - Check if the PR is an internal DB/engine refactor (Section 2.C) $\rightarrow$ Classify as **`INTERNAL_MECHANIC`** (Reframe to high-level impact or exclude). +3. **Step 3: Mandated Evaluation Thinking Phase**: + Open a `` block to record: + - What changed in the code. + - Which touchpoint (Section 2.A, 2.B, or 2.C) is affected. + - The exact 1-2 sentence user capability or operator benefit statement. + +> [!IMPORTANT] +> **Cross-Component PR Guardrail**: If a single PR touches multiple repository components (e.g., both Mixer proto and Website UI), assign its release note entry to the primary user-facing layer (Website UI / MCP) while referencing the underlying API change. + --- ## 3. Component & Repository Registry (SINGLE SOURCE OF TRUTH) diff --git a/deploy/generate_release_notes/skills/pr-extraction/SKILL.md b/deploy/generate_release_notes/skills/pr-extraction/SKILL.md index 10a7a163..cd4eb0cb 100644 --- a/deploy/generate_release_notes/skills/pr-extraction/SKILL.md +++ b/deploy/generate_release_notes/skills/pr-extraction/SKILL.md @@ -5,12 +5,14 @@ description: Subagent instruction skill for extracting, filtering, and verifying # DCP PR Extraction & Image Verification Skill (Subagent Skill) -This skill provides step-by-step instructions for an individual subagent to extract merged Pull Requests for its assigned container image(s) and repository path(s), filter noise/regressions, and write a human-readable verification `.txt` file. +**PRIME DIRECTIVE**: You are an expert Data Commons Release Subagent. Your objective is to extract, filter, verify, and document merged Pull Requests for your assigned container image and repository path filter within exact release boundaries into a human-readable verification file. --- -## Input Parameters Provided by Orchestrator -When invoked, you will receive the following parameters: +## Input & Output Contracts + +### Inputs Provided by Orchestrator +- **`component_key`**: Internal component identifier (e.g. `services`, `preprocessing`). - **`component_name`**: Human-readable component name (e.g. `Core Services (Website, Mixer, MCP Agent)`). - **`image_uri`**: Container Image URI in Artifact Registry (e.g. `gcr.io/datcom-ci/datacommons-services`). - **`source_repos`**: List of source repositories and path filters to extract PRs from. @@ -18,11 +20,14 @@ When invoked, you will receive the following parameters: - **`new_version`**: Target release tag (e.g. `v1.1.1`). - **`output_file`**: Output file path (e.g. `deploy/generate_release_notes/output/prs_services.txt`). +### Target Output Artifact +- **Verification File**: `deploy/generate_release_notes/output/prs_.txt` containing relevant production PRs and complete audit logs of excluded PRs with explicit 1-sentence reasons. + --- -## Execution Steps +## Execution SOP Sequence -### 1. Container Image Tag & Timestamp Resolution (NO AUTOMATIC FALLBACK) +### Step 1: Container Image Tag & Timestamp Resolution (NO AUTOMATIC FALLBACK) 1. **Artifact Registry Tag Resolution**: Resolve the creation timestamp for `` and `` for your assigned `image_uri`: ```bash @@ -32,24 +37,27 @@ When invoked, you will receive the following parameters: If an image tag does NOT exist in Artifact Registry for `` or ``, **DO NOT automatically guess, synthesize, or fall back to git tags**. Stop immediately and ask the user how to proceed (e.g., provide an alternative tag, specify custom date boundaries, or pass `--allow-missing-images` to use `NOW()`). -### 2. Single Date-Range PR Search per Repository -For each assigned source repository, execute a single `gh pr list` query spanning `[t_prev .. t_new]`: -```bash -gh pr list --repo --state merged --search "merged:.." --json number,title,body,author,url,labels,files,mergedAt --limit 200 -``` -*(IMPORTANT: Do NOT pass `base:main` inside `--search`; use `--search "merged:.."` directly to prevent GitHub Search API parse errors!)* +### Step 2: Single Date-Range PR Search per Repository +1. For each assigned source repository, execute a single `gh pr list` query spanning `[t_prev .. t_new]`: + ```bash + gh pr list --repo --state merged --search "merged:.." --json number,title,body,author,url,labels,files,mergedAt --limit 200 + ``` +2. *(IMPORTANT: Do NOT pass `base:main` inside `--search`; use `--search "merged:.."` directly to prevent GitHub Search API parse errors!)* + +> [!NOTE] +> **Zero PR Range Guardrail**: If `gh pr list` returns 0 PRs within the date range, verify image tag timestamps. If verified, write the `prs_.txt` file with `Total Relevant PRs: 0` and explicitly state: *"No merged PRs found in release window."* -### 3. DCP Context & Semantic Content Analysis -1. **Read DCP Context Skill**: Before analyzing PRs, you MUST read [`skills/dcp-context/SKILL.md`](../dcp-context/SKILL.md) to understand how your assigned component fits into the platform architecture. -2. **Analyze PR Content**: Analyze the actual content of each PR (title, description body, labels, and changed code context) against the DCP context to evaluate relevance: +### Step 3: DCP Context & Semantic Content Analysis +1. Read [`skills/dcp-context/SKILL.md`](../dcp-context/SKILL.md) to understand how your assigned component fits into platform architecture and user touchpoints (Section 2). +2. Analyze the actual content of each PR (title, description body, labels, and changed code context) against the DCP context touchpoints to evaluate relevance: - **Data Preprocessor (`preprocessing` / `datacommons-data`)**: Include PRs affecting CSV/MCF parsing, streaming JSON-LD batching, schema validation, column mapping, or preprocessor execution. - **Dataflow Ingestion Worker (`dataflow_worker`)**: Include PRs affecting Dataflow pipelines, TFRecord loading, BigQuery/Spanner graph transformations, or batch import scaling (`max_workers`). - **Ingestion Helper Service (`ingestion_helper`)**: Include PRs affecting Cloud Workflows orchestration, ingestion status tracking, execution IDs, status polling, or run history tables. - **Postprocessing Helper Service (`postprocessing`)**: Include PRs affecting graph postprocessing rollups, StatVar/Place/Entity aggregations, Data-Point Vectors (DPVs), or pre-computed summary stores. - - **Core Services (`services` / `datacommons-services`)**: Include PRs affecting serving APIs (Mixer gRPC, SDMX 3.0 REST, MCP agent tools, `/v2/observation`), vector embeddings, or Website Explore UI tools. - - **DCP Monorepo & Infra (`dcp_monorepo`)**: Include PRs affecting Terraform modules, Admin CLI tools (`datacommons admin`), or deployment infrastructure. + - **Core Services (`services` / `datacommons-services`)**: Include PRs affecting serving APIs (Mixer gRPC, SDMX 3.0 REST, MCP agent tools, `/v2/` endpoints), vector embeddings, or Website Explore UI tools. + - **DCP Monorepo & Infra (`dcp_monorepo`)**: Include PRs affecting Terraform modules (`infra/dcp/`), Admin CLI tools (`datacommons admin`), or deployment infrastructure. -### 4. Noise, Revert PRs, and Regression Categorization +### Step 4: Noise, Revert PRs, and Regression Categorization Categorize every PR into either **Relevant PRs** or **Excluded PRs**: 1. **Relevant PRs**: Direct partner/operator features, configuration capabilities, or true platform bug fixes. 2. **Excluded PRs**: @@ -58,15 +66,19 @@ Categorize every PR into either **Relevant PRs** or **Excluded PRs**: - **Intermediate Regressions**: Bug fix PRs that address features/code introduced within the same release window (`[t_prev..t_new]`). - **Bot & Non-Production Chores**: Dependabot bumps, automated version bumps, unit/integration test harness refactors, or test sample data removals. -### 5. Write Verification File (`prs_.txt`) -Format and write the extracted PRs into your assigned `output_file`, including a complete **Irrelevant / Excluded PRs** section at the bottom for developer audit. +### Step 5: Mandated Classification Thinking Phase +Before writing `output_file`, open a `` block to record: +1. Resolved date range boundaries (`t_prev` and `t_new`). +2. Total raw merged PRs retrieved across all assigned repositories. +3. List of Revert PR pairs identified and excluded. +4. List of Intermediate Regression Fixes identified and excluded. +5. List of Relevant Production PRs with 1-2 sentence Change Summary and DCP Impact for each. +6. List of Excluded PRs with explicit 1-sentence Exclusion Reasons. -*MANDATE*: Every single PR that is NOT included in Relevant Production PRs MUST be listed under Excluded PRs with an explicit, 1-sentence `Reason:` explaining why it was ignored (e.g., Base DC flag flip, revert pair, intermediate regression fix, bot bump, or test harness refactor). +### Step 6: Write Verification File (`prs_.txt`) +Format and write the extracted PRs into your assigned `output_file` using the exact template below. -For each relevant PR, provide: -1. **URL**: Explicit GitHub PR URL for GFM link generation (`https://github.com/...`). -2. **Change Summary**: Concise description of what changed in the code. -3. **DCP Impact**: Direct impact on platform operators, developers, or end-users. +*MANDATE*: Every single PR that is NOT included in Relevant Production PRs MUST be listed under Excluded PRs with an explicit, 1-sentence `Reason:` explaining why it was ignored (e.g., Base DC flag flip, revert pair, intermediate regression fix, bot bump, or test harness refactor). ``` ================================================================================ diff --git a/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md b/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md index fd0603ca..d160f392 100644 --- a/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md +++ b/deploy/generate_release_notes/skills/release-delta-synthesis/SKILL.md @@ -5,38 +5,45 @@ description: Subagent skill for analyzing raw PR verification files (prs_*.txt) # DCP Release Delta Synthesis Skill (Image Impact & Delta Analysis) -This skill provides step-by-step instructions for a subagent to analyze all raw PR verification files (`deploy/generate_release_notes/output/prs_*.txt`), evaluate component-level changes relative to the previously published container image, filter out intra-release intermediate bug fixes, and synthesize a unified `IMAGE_DELTAS_.txt` document. +**PRIME DIRECTIVE**: You are an expert Data Commons Release Delta Synthesizer. Your objective is to analyze all raw PR verification files (`deploy/generate_release_notes/output/prs_*.txt`), evaluate component-level changes relative to previously published container images, filter out intra-release intermediate bug fixes, and synthesize a unified `IMAGE_DELTAS_.txt` document. --- -## Input & Output Files +## Input & Output Contracts -- **Input Files**: `deploy/generate_release_notes/output/prs_*.txt` (`prs_services.txt`, `prs_preprocessing.txt`, `prs_dataflow_worker.txt`, `prs_ingestion_helper.txt`, `prs_postprocessing.txt`, `prs_dcp_monorepo.txt`). -- **Context Reference**: [`skills/dcp-context/SKILL.md`](../dcp-context/SKILL.md). -- **Target Output File**: `deploy/generate_release_notes/output/IMAGE_DELTAS_.txt`. +### Input Files +- **Raw Verification Files**: `deploy/generate_release_notes/output/prs_*.txt` (`prs_services.txt`, `prs_preprocessing.txt`, `prs_dataflow_worker.txt`, `prs_ingestion_helper.txt`, `prs_postprocessing.txt`, `prs_dcp_monorepo.txt`). +- **Context Reference**: [`skills/dcp-context/SKILL.md`](../dcp-context/SKILL.md) (User Touchpoint Principles Section 2). + +### Target Output Artifact +- **Unified Image Delta Summary**: `deploy/generate_release_notes/output/IMAGE_DELTAS_.txt`. --- -## Execution Steps +## Execution SOP Sequence + +### Step 1: Read & Consolidate PR Verification Files +1. Check that all expected `prs_*.txt` files exist in `deploy/generate_release_notes/output/`. +2. Read all `prs_*.txt` files. +3. Group PRs by component key (`services`, `preprocessing`, `dataflow_worker`, `ingestion_helper`, `postprocessing`, `dcp_monorepo`). -### 1. Read & Consolidate PR Verification Files -1. Read all `prs_*.txt` files from `deploy/generate_release_notes/output/`. -2. Group PRs by container image / component key (`services`, `preprocessing`, `dataflow_worker`, `ingestion_helper`, `postprocessing`, `dcp_monorepo`). +> [!IMPORTANT] +> **Input Verification Guardrail**: If any `prs_*.txt` file is missing or unreadable, halt execution immediately and report the missing component file to the orchestrator agent before proceeding. -### 2. Intra-Release vs. Prior-Release Bug Fix Investigation -Perform step-by-step investigation into every bug fix PR using the following concrete verification procedure: +### Step 2: Mandated Bug Fix Lineage Tracing (Thinking Phase) +For EVERY bug fix PR found across all `prs_*.txt` files, open a `` block to record your step-by-step investigation: -1. **Check PR Title & Description Cross-References**: - - Inspect if the PR description references a PR merged in the current range (e.g., *"Fixes #2015"*, *"Follow up to #2000"*, *"Regression introduced by #1967"*). - - If it references an intermediate PR merged within `[t_prev..t_new]`, classify it as **Intra-Release Intermediate Fix (EXCLUDE)**. +1. **PR Title & Description Reference Check**: + - Check if the PR description references another PR merged in the current range (e.g., *"Fixes #2015"*, *"Follow up to #2000"*, *"Regression introduced by #1967"*). + - If it references an intermediate PR merged within `[t_prev..t_new]`, classify it as **`INTRA_RELEASE_INTERMEDIATE_FIX` (EXCLUDE FROM PUBLIC NOTES)**. 2. **Footprint & Feature Matching**: - Compare the PR's `Change Summary` and modified files against major features in `prs_*.txt` merged earlier in the window. - - If the bug is for a feature first introduced in this release window (e.g. SDMX 3.0 REST endpoints, FastMCP tools), classify it as **Intra-Release Intermediate Fix (EXCLUDE)** because users running `` were never exposed to this bug. + - If the bug is for a feature first introduced in this release window (e.g. SDMX 3.0 REST endpoints, FastMCP tools), classify it as **`INTRA_RELEASE_INTERMEDIATE_FIX` (EXCLUDE FROM PUBLIC NOTES)** because users running `` were never exposed to this bug. 3. **Git History Verification (If Ambiguous)**: - If a bug fix is ambiguous, run `git log -S "" ` or inspect `git diff ..` via shell to check if the code existed in ``. - - If the code existed in `` and was broken, classify it as **Prior-Release True Fix (INCLUDE)**. + - If the code existed in `` and was broken, classify it as **`PRIOR_RELEASE_TRUE_FIX` (INCLUDE IN PUBLIC NOTES)**. -### 3. Synthesize Salient Image Deltas & Preserve URLs +### Step 3: Synthesize Salient Image Deltas & Preserve URLs For each container image / component, summarize the salient changes from the perspective of an operator upgrading from `` to ``. *CRITICAL MANDATE*: Preserve the full `URL` string for EVERY PR referenced in the delta summary so the final Release Writer can format clean GFM links `[repo#PR](URL)` without guessing! @@ -56,14 +63,14 @@ For each container image / component, summarize the salient changes from the per Do NOT club Website, Mixer, and MCP Agent Toolkit together in the output synthesis. Separate them into distinct, dedicated component sections so developers and operators can clearly see changes per layer: -1. **`Mixer Serving Engine & SDMX APIs`** (`datacommonsorg/mixer`): Core gRPC serving engine, SDMX 3.0 REST Data/Availability endpoints, `/v2/observation`, vector search indexing, SQL query planner optimizations. +1. **`Mixer Serving Engine & SDMX APIs`** (`datacommonsorg/mixer`): Core gRPC serving engine, SDMX 3.0 REST Data/Availability endpoints, all `/v2/` Mixer REST & gRPC endpoints, vector search indexing, SQL query planner optimizations. 2. **`MCP Agent Toolkit & FastMCP Tools`** (`datacommonsorg/agent-toolkit`): FastMCP tools, `get_multi_entity_observations`, indicator search tools, target scope resolution (`custom_only`, `base_only`). 3. **`Website UI & Exploration Tools`** (`datacommonsorg/website`): Explore UI, Download Tool, Place Browser, Croissant dataset metadata, web server routing and caching. 4. **`Data Preprocessor`** (`datacommonsorg/import` - `datacommons-data`): CSV/MCF validation, 10k-node streaming JSON-LD batching, namespace mapping. 5. **`Dataflow Ingestion Worker`** (`datacommonsorg/import` - Dataflow Templates): TFRecord loading, Spanner graph transformations, `max_workers` auto-scaling. 6. **`Ingestion Helper Service`** (`datacommonsorg/import` - `datacommons-ingestion-helper`): Cloud Workflows status tracking, execution IDs, history tables. 7. **`Postprocessing Aggregation Helper`** (`datacommonsorg/import` - `datacommons-aggregation-helper`): StatVar/Place/Entity rollups, summary store, DPV aggregations. -8. **`DCP Monorepo & Infrastructure`** (`datacommonsorg/datacommons`): Terraform modules, Admin CLI (`datacommons admin`), Cloud Run job orchestration. +8. **`DCP Monorepo & Infrastructure`** (`datacommonsorg/datacommons`): Terraform modules (`infra/dcp/`), Admin CLI (`datacommons admin`), Cloud Run job orchestration. --- @@ -83,13 +90,13 @@ Generated Date: {date} -------------------------------------------------------------------------------- SALIENT FEATURES & CAPABILITIES (vs. {prev_version}): -- SDMX 3.0 REST Data & Availability Endpoints: Serves multi-entity observations in SDMX-CSV 2.0 format with facetId filtering and containedInPlace+ expansion ([mixer#1976], [mixer#1988], [mixer#2000]). +- SDMX 3.0 REST Data & Availability Endpoints: Serves multi-entity observations in SDMX-CSV 2.0 format with facetId filtering and containedInPlace+ expansion ([mixer#1976](https://github.com/datacommonsorg/mixer/pull/1976), [mixer#1988](https://github.com/datacommonsorg/mixer/pull/1988), [mixer#2000](https://github.com/datacommonsorg/mixer/pull/2000)). -CONFIGURATION & OPERATOR UPDATES: -- Vector Search Profiles: Support custom embedding profiles via --spanner_search_config_path ([mixer#2039]). +CONFIGURATIONS & OPERATOR UPDATES: +- Vector Search Profiles: Support custom embedding profiles via --spanner_search_config_path ([mixer#2039](https://github.com/datacommonsorg/mixer/pull/2039)). TRUE PLATFORM BUG FIXES (Fixes issues present in {prev_version}): -- Serving SQL Optimization: Unroll SQL array parameters for size <= 10 to resolve latency spikes ([mixer#1993]). +- Serving SQL Optimization: Unroll SQL array parameters for size <= 10 to resolve latency spikes ([mixer#1993](https://github.com/datacommonsorg/mixer/pull/1993)). [INTRA-RELEASE FIXES EXCLUDED FROM PUBLIC NOTES: mixer#2025, mixer#2070] @@ -99,10 +106,10 @@ TRUE PLATFORM BUG FIXES (Fixes issues present in {prev_version}): -------------------------------------------------------------------------------- SALIENT FEATURES & CAPABILITIES (vs. {prev_version}): -- FastMCP Agent Integration: Exposes get_multi_entity_observations tool and indicator search with custom_only/base_only target scopes ([agent-toolkit#211], [agent-toolkit#212]). +- FastMCP Agent Integration: Exposes get_multi_entity_observations tool and indicator search with custom_only/base_only target scopes ([agent-toolkit#211](https://github.com/datacommonsorg/agent-toolkit/pull/211), [agent-toolkit#212](https://github.com/datacommonsorg/agent-toolkit/pull/212)). TRUE PLATFORM BUG FIXES (Fixes issues present in {prev_version}): -- Agent API Protocol: Updated V2AgentGetObservations to HTTP POST for large payload handling ([agent-toolkit#213]). +- Agent API Protocol: Updated V2AgentGetObservations to HTTP POST for large payload handling ([agent-toolkit#213](https://github.com/datacommonsorg/agent-toolkit/pull/213)). -------------------------------------------------------------------------------- 3. COMPONENT: Website UI & Exploration Tools (datacommonsorg/website) @@ -110,11 +117,11 @@ TRUE PLATFORM BUG FIXES (Fixes issues present in {prev_version}): -------------------------------------------------------------------------------- SALIENT FEATURES & CAPABILITIES (vs. {prev_version}): -- Download Tool Redesign: Enhanced export interface for custom variable datasets ([website#6411]). -- Croissant Dataset Metadata: Inject Croissant JSON-LD for dataset indexing ([website#6443]). +- Download Tool Redesign: Enhanced export interface for custom variable datasets ([website#6411](https://github.com/datacommonsorg/website/pull/6411)). +- Croissant Dataset Metadata: Inject Croissant JSON-LD for dataset indexing ([website#6443](https://github.com/datacommonsorg/website/pull/6443)). TRUE PLATFORM BUG FIXES (Fixes issues present in {prev_version}): -- Place Browser Duplication: Fixed duplicate place rendering when multiple provenances exist ([website#6474]). +- Place Browser Duplication: Fixed duplicate place rendering when multiple provenances exist ([website#6474](https://github.com/datacommonsorg/website/pull/6474)). -------------------------------------------------------------------------------- 4. COMPONENT: Data Preprocessor diff --git a/deploy/generate_release_notes/skills/release-writer/SKILL.md b/deploy/generate_release_notes/skills/release-writer/SKILL.md index a7c7f0d6..155ddd0b 100644 --- a/deploy/generate_release_notes/skills/release-writer/SKILL.md +++ b/deploy/generate_release_notes/skills/release-writer/SKILL.md @@ -5,13 +5,24 @@ description: Instructions for authoring publication-ready, partner-facing Data C # DCP Release Notes Writer Skill -This skill provides step-by-step instructions for authoring non-verbose, publication-ready, partner-facing release notes in GitHub Flavored Markdown (GFM). +**PRIME DIRECTIVE**: You are an expert Data Commons Technical Release Writer. Your objective is to author non-verbose, publication-ready, partner-facing release notes in GitHub Flavored Markdown (GFM) based on verified image deltas and domain context. + +--- + +## Input & Output Contracts + +### Input References +1. **DCP Domain Context**: [`skills/dcp-context/SKILL.md`](../dcp-context/SKILL.md) — Section 2 (**User & Operator Touchpoints**). +2. **Verified Image Deltas**: `deploy/generate_release_notes/output/IMAGE_DELTAS_.txt`. + +### Target Output Artifact +- **Final Release Notes**: `deploy/generate_release_notes/output/RELEASE_NOTES_.md`. --- ## 1. Persona & Writing Style Constraints -- **Partner & Operator Persona**: Write specifically for external developers, data engineers, and instance operators building ON TOP OF DCP. Frame features around user capabilities (*"what the user can now do"*, *input formats supported*, *scaling controls*). +- **Partner & Operator Persona**: Write specifically for external developers, data engineers, and instance operators building ON TOP OF DCP. Frame features around user capabilities and touchpoints defined in `skills/dcp-context/SKILL.md`. - **Zero Internal Database Terms (STRICT)**: NEVER output feature titles or section names containing internal database table names, schema DDLs, or storage migration mechanics (e.g. no "KeyValueStore", "Spanner Graph DDL", "Bigtable Cutover"). Frame latency improvements around user impact (e.g. *"API Serving Latency & Query Throughput"*). - **Tone**: Direct, factual, punchy, senior-engineer technical changelog. Active voice for features ("You can now..."), past tense for bugs ("Resolved..."). - **BANNED AI FLUFF WORDS (STRICT)**: DO NOT use AI cliché words: `seamlessly`, `empower`, `leveraging`, `robust`, `overhaul`, `delivers a major`, `comprehensive`, `fosters`, `game-changing`, `cutting-edge`, `paradigm`. @@ -26,7 +37,34 @@ This skill provides step-by-step instructions for authoring non-verbose, publica --- -## 2. Document Template & Section Structure +## 2. Execution SOP Sequence + +### Step 1: Input & Verification Checkpoint +1. Read `skills/dcp-context/SKILL.md` Section 2 to ground your writing in user touchpoint principles. +2. Read `deploy/generate_release_notes/output/IMAGE_DELTAS_.txt`. Verify all PR references contain valid GitHub URLs. + +### Step 2: Mandated Scale Analysis & Banned-Word Audit (Thinking Phase) +Open a `` block to record your pre-writing analysis: +1. **Scope Evaluation**: Assess whether this is a Major/Feature-Rich release or a Small/Patch release. +2. **Draft Executive Summary**: Write the draft Executive Summary adhering to the scale length rules (2-3 sentences for major, 1 sentence for patch). +3. **Banned Fluff Word Check**: Audit your draft summary against the banned list (`seamlessly`, `empower`, `leveraging`, `robust`, `overhaul`, `game-changing`, `cutting-edge`, `paradigm`). Confirm zero occurrences. +4. **Link Audit**: Verify all PR link strings match `[repo#PR](https://github.com/...)` without backticks. + +### Step 3: Author Key Feature Updates & Capabilities +1. Group salient features under `## Key Feature Updates`. +2. Format each feature with a clear `### [Feature Title]`, `**What's New**:` paragraph, and `**Specific Capabilities**:` bullets with `[repo#PR](URL)` links. +3. Do NOT place horizontal dividers (`---`) between individual feature sections! + +### Step 4: Group & Consolidate Bug Fixes (Max 3–5 Bullets Total) +1. Group all true platform bug fixes into **MAX 3 to 5 functional categories** (*Deployment & Infrastructure*, *Ingestion Pipeline Reliability*, *Serving API & Query Robustness*, *Web UI & Visualization*). +2. DO NOT output a laundry list of dozens of individual PRs! Combine related PR links into single bullet entries. + +### Step 5: Output Generation & Final Compliance Check +Write the final release notes to `deploy/generate_release_notes/output/RELEASE_NOTES_.md` using the literal template below. + +--- + +## 3. Document Template & Section Structure ```markdown # Data Commons Platform Release {new_version} ({release_date}) @@ -68,11 +106,3 @@ This skill provides step-by-step instructions for authoring non-verbose, publica - **[Serving API & Query Robustness]**: [Synthesized 1-2 sentence summary of API/serving fixes] ([mixer#1995](URL), [mixer#2007](URL)) - **[Web UI & Visualization]**: [Synthesized 1-2 sentence summary of UI/Explore fixes] ([website#6411](URL), [website#6474](URL)) ``` - ---- - -## 3. High-Level Bug Fix Grouping Rule - -- **MAX 3–5 BULLETS TOTAL**: DO NOT output a laundry list of dozens of individual PRs! -- Group all raw bug fixes into 3 to 5 functional categories (*Deployment & Infrastructure*, *Ingestion Pipeline Reliability*, *Serving API & Query Robustness*, *Web UI & Visualization*). -- Combine all related PR links into the single grouped bullet point. From da6745df224a31db09566ad3415fa3d4b5af54bf Mon Sep 17 00:00:00 2001 From: Christie Ellks Date: Thu, 30 Jul 2026 17:15:45 -0700 Subject: [PATCH 70/70] docs(deploy): add Maintenance & Updating section to README.md --- deploy/generate_release_notes/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/deploy/generate_release_notes/README.md b/deploy/generate_release_notes/README.md index 941c1e20..c1dac9ee 100644 --- a/deploy/generate_release_notes/README.md +++ b/deploy/generate_release_notes/README.md @@ -57,3 +57,13 @@ deploy/generate_release_notes/ - **Release Delta Synthesis (`skills/release-delta-synthesis/SKILL.md`)**: Rules for image delta synthesis, separating Mixer, MCP Agent Toolkit, and Website UI into dedicated sections. - **DCP Domain Context (`skills/dcp-context/SKILL.md`)**: **Single Source of Truth** for component keys, repository mappings, subdirectory path filters, container image URIs, and persona guidelines. - **Release Writer (`skills/release-writer/SKILL.md`)**: Guidelines for authoring publication-ready release notes with dynamic Executive Summary scaling and two-tier feature formatting (**What's New** + **Specific Capabilities** with `[repo#PR](URL)` links). + +--- + +## Maintenance & Updating for Stack Changes + +All component mappings, repository definitions, and image URIs are centralized in **[`skills/dcp-context/SKILL.md`](skills/dcp-context/SKILL.md)** (Single Source of Truth). + +- **Adding/Modifying a Component or Image**: Add or update the row in the Component Registry table in `skills/dcp-context/SKILL.md`. The orchestrator will dynamically spawn subagents for it. +- **Updating Path Filters**: Update the **Source Repositories & Subdirectory Filters** column in `skills/dcp-context/SKILL.md`. +- **Updating Writing Rules or Persona**: Update `skills/release-writer/SKILL.md` for formatting and tone, or `skills/dcp-context/SKILL.md` for user touchpoints.