diff --git a/.github/scripts/audit-ci-settings.mjs b/.github/scripts/audit-ci-settings.mjs new file mode 100644 index 0000000..40ade01 --- /dev/null +++ b/.github/scripts/audit-ci-settings.mjs @@ -0,0 +1,72 @@ +import { execFileSync } from "node:child_process"; +import { pathToFileURL } from "node:url"; + +export function auditSettings(repository, { branchRules, rulesets, environment, actions }) { + const issues = []; + // GitHub resolves include/exclude patterns, default-branch aliases and inherited + // rules. Never infer effective protection from a ruleset's declarations. + const rules = Array.isArray(branchRules) ? branchRules : []; + if (!Array.isArray(branchRules)) issues.push("Effective main rules could not be verified."); + if (!rules.length) issues.push("No active ruleset protects main."); + const ids = [...new Set(rules.map((rule) => rule.ruleset_id))]; + const active = ids.map((id) => { + const set = rulesets.find((entry) => entry.id === id && entry.enforcement === "active"); + if (!set || !Array.isArray(set.bypass_actors)) issues.push("Applicable ruleset details could not be verified: " + id); + return set; + }).filter(Boolean); + for (const type of ["deletion", "non_fast_forward", "required_linear_history"]) { + if (!rules.some((rule) => rule.type === type)) issues.push("Missing protection: " + type); + } + if (rules.some((rule) => rule.type === "merge_queue")) issues.push("Merge queue is enabled for main."); + const pulls = rules.filter((rule) => rule.type === "pull_request").map((rule) => rule.parameters); + if (!pulls.some((pull) => pull?.require_code_owner_review)) issues.push("Independent CODEOWNERS review is not required."); + if (!pulls.some((pull) => pull?.dismiss_stale_reviews_on_push)) issues.push("Stale approvals are not dismissed."); + if (!pulls.some((pull) => pull?.required_review_thread_resolution)) issues.push("Review thread resolution is not required."); + const checks = rules.filter((rule) => rule.type === "required_status_checks").map((rule) => rule.parameters); + const context = repository === ".github" ? "Organization controls verification" : "OpenBoa Coffee trusted required / OpenBoa Coffee trusted required"; + if (!checks.some((check) => check?.strict_required_status_checks_policy && check.required_status_checks?.length)) issues.push("Required checks do not require an up-to-date branch."); + if (!checks.some((set) => set?.required_status_checks?.some((check) => check.context === context && check.integration_id === 15368))) issues.push("Required check/source is missing: " + context); + if (active.some((rule) => rule.bypass_actors?.length)) issues.push("Ruleset has bypass actors; explicit exception review is needed."); + if (repository !== ".github") { + const protectionRules = Array.isArray(environment?.protection_rules) ? environment.protection_rules : []; + const required = protectionRules.filter((rule) => rule?.type === "required_reviewers"); + const reviewers = required.length === 1 && Array.isArray(required[0].reviewers) ? required[0].reviewers : []; + if (!reviewers.length) issues.push("coffee-security has no required reviewers."); + const login = reviewers[0]?.reviewer?.login; + // GitHub accepts approval by any listed principal, so an additional user or + // team would make the documented owner's confirmation optional. + if (required.length !== 1 || reviewers.length !== 1 || reviewers[0]?.type !== "User" || typeof login !== "string" || login.toLowerCase() !== "sonsangjoon") { + issues.push("coffee-security must require only User SonSangjoon."); + } + if (environment?.can_admins_bypass === true) issues.push("coffee-security allows administrator bypass."); + else if (environment?.can_admins_bypass !== false) issues.push("coffee-security administrator bypass setting could not be verified."); + } + if (actions.default_workflow_permissions !== "read") issues.push("Default workflow token is not read-only."); + if (actions.can_approve_pull_request_reviews !== false) issues.push("Actions are allowed to approve pull requests."); + return { repository, issues, preservedAdditionalRules: rules.filter((rule) => /coverage|code_quality/u.test(rule.type)).map((rule) => rule.type) }; +} + +async function main() { + const repositories = process.argv.slice(2); + if (!repositories.length) repositories.push(".github", "coffee-chat", "coffee-chat-roastery", "coffee-chat-eval", "coffee-chat-bench"); + const api = (path, paginated = false) => { + const data = JSON.parse(execFileSync("gh", ["api", ...(paginated ? ["--paginate", "--slurp"] : []), path], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] })); + return paginated ? data.flat() : data; + }; + const reports = []; + for (const repository of repositories) { + if (![".github", "coffee-chat", "coffee-chat-roastery", "coffee-chat-eval", "coffee-chat-bench"].includes(repository)) throw Error("Unsupported repository"); + const prefix = "repos/openboa-ai/" + repository; + try { + const branchRules = api(prefix + "/rules/branches/main?per_page=100", true); + const ids = [...new Set(branchRules.map((rule) => rule.ruleset_id))]; + const rulesets = ids.map((id) => api(prefix + "/rulesets/" + id)); + const environment = repository === ".github" ? null : api(prefix + "/environments/coffee-security"); + const actions = api(prefix + "/actions/permissions/workflow"); + reports.push(auditSettings(repository, { branchRules, rulesets, environment, actions })); + } catch { reports.push({ repository, issues: ["Live settings could not be fully read; not verified."] }); } + } + console.log(JSON.stringify(reports, null, 2)); + if (reports.some((report) => report.issues.length)) process.exitCode = 1; +} +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main().catch((error) => { console.error(error.message); process.exitCode = 1; }); diff --git a/.github/scripts/check-candidate-policy.mjs b/.github/scripts/check-candidate-policy.mjs new file mode 100644 index 0000000..bc6657a --- /dev/null +++ b/.github/scripts/check-candidate-policy.mjs @@ -0,0 +1,147 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { existsSync, lstatSync, readFileSync, readdirSync } from "node:fs"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { validateCandidateWorkflowDelegation } from "./classify-candidate.mjs"; + +export function readJson(root, path) { + const file = resolve(root, path); + assert.ok(lstatSync(file).isFile(), `${path}: regular file required`); + const bytes = readFileSync(file); + assert.ok(bytes.length <= 8 * 1024 * 1024, `${path}: oversized control data`); + return JSON.parse(bytes.toString("utf8")); +} + +export function validatePackage(root) { + const pkg = readJson(root, "package.json"); + const lock = readJson(root, "package-lock.json"); + assert.equal(pkg.private, true, "verification packages must remain private"); + assert.equal(typeof pkg.scripts?.verify, "string", "npm run verify is required"); + assert.ok(pkg.scripts.verify.trim(), "verify must not be empty"); + for (const script of ["preinstall", "install", "postinstall", "prepare", "preverify", "postverify"]) { + assert.equal(pkg.scripts[script], undefined, `${script}: implicit execution is not allowed`); + } + assert.equal(pkg.workspaces, undefined, "workspace installation requires a reviewed execution contract"); + assert.equal(lock.lockfileVersion, 3, "lockfile v3 is required"); + assert.equal(lock.name, pkg.name, "lockfile name mismatch"); + assert.equal(lock.version, pkg.version, "lockfile version mismatch"); + assert.ok(lock.packages?.[""], "lockfile root is required"); + for (const field of ["dependencies", "devDependencies", "optionalDependencies"]) { + assert.deepEqual(lock.packages[""][field] ?? {}, pkg[field] ?? {}, `${field}: package/lock mismatch`); + for (const version of Object.values(pkg[field] ?? {})) { + assert.match(version, /^\d+\.\d+\.\d+$/, "direct dependencies must use exact registry versions"); + } + } + for (const [path, dependency] of Object.entries(lock.packages)) { + if (path === "") continue; + assert.ok(path.startsWith("node_modules/") && !path.split("/").includes(".."), "invalid dependency path"); + assert.equal(dependency.link, undefined, "linked dependencies are not allowed"); + assert.equal(dependency.hasInstallScript, undefined, "dependency installation scripts are not allowed"); + const url = new URL(dependency.resolved); + assert.equal(url.origin, "https://registry.npmjs.org", "dependencies must come from the npm registry"); + assert.equal(url.username + url.password + url.search + url.hash, "", "dependency URL authority override"); + assert.match(dependency.integrity, /^sha512-[A-Za-z0-9+/]+={0,2}$/, "dependency integrity is required"); + } + return { pkg, lock }; +} + +export function validateMergePolicy(policy) { + assert.equal(policy.merge_method, "squash"); + assert.equal(policy.merge_queue, false); + assert.deepEqual(policy.eligible_author_associations, ["OWNER", "MEMBER"]); + assert.deepEqual(policy.eligible_bot_logins, ["dependabot[bot]"]); + assert.ok(policy.required_checks?.some((check) => + check.context === "OpenBoa Coffee trusted required / OpenBoa Coffee trusted required" && + check.integration_id === 15368), "trusted required check must remain enforced"); + assert.equal(policy.sensitive_review?.enforcement, "github_environment"); + assert.equal(policy.sensitive_review.environment, "coffee-security"); + assert.equal(policy.sensitive_review.required_approvals, 1); + assert.ok(Array.isArray(policy.protected_paths) && policy.protected_paths.length > 0); + for (const path of policy.protected_paths) { + assert.equal(typeof path, "string"); + assert.ok(path.length > 0 && !path.startsWith("!") && !/[\r\n\0]/u.test(path), "invalid protected path"); + } + const normalized = policy.protected_paths.map((path) => path.replace(/^\//u, "")); + for (const path of [".github/**", "AGENTS.md", "SECURITY.md", "CODEOWNERS", "package.json", "package-lock.json"]) { + assert.ok(normalized.includes(path), `protected control missing: ${path}`); + } +} + +function readOwnership(root) { + const locations = [".github/CODEOWNERS", "CODEOWNERS", "docs/CODEOWNERS"].filter((path) => existsSync(resolve(root, path))); + assert.equal(locations.length, 1, "exactly one CODEOWNERS location is required; competing authority is forbidden"); + const path = locations[0]; + assert.ok(lstatSync(resolve(root, path)).isFile(), "CODEOWNERS must be a regular file"); + const bytes = readFileSync(resolve(root, path)); + assert.ok(bytes.length < 3_000_000, "CODEOWNERS must remain below GitHub's file-size limit"); + const text = bytes.toString("utf8"); + assert.ok(!/[\0-\x08\x0b\x0c\x0e-\x1f\x7f]/u.test(text), "invalid CODEOWNERS control character"); + const routes = text.split(/\r?\n/u).map((line) => line.split("#", 1)[0].trim()).filter(Boolean).map((line) => line.split(/[ \t]+/u)); + assert.ok(routes.length, "CODEOWNERS must retain review routes"); + for (const [, ...owners] of routes) for (const owner of owners) { + // These repositories use GitHub handles, not email aliases. Invalid added + // syntax can make GitHub discard the entire line, including retained owners. + assert.match(owner, /^@[A-Za-z0-9](?:[A-Za-z0-9]|-(?=[A-Za-z0-9])){0,38}(?:\/[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)?$/u, "CODEOWNERS owners must use valid user/team handles"); + } + return { path, routes }; +} + +export function validateCandidatePolicy(baseRoot, candidateRoot) { + // No repository code, YAML constructors, package imports or base bootstrap is executed here. + const records = execFileSync("git", ["-C", candidateRoot, "ls-files", "--stage", "-z"], { encoding: "utf8" }).split("\0").filter(Boolean); + for (const record of records) { + const [metadata, path] = record.split("\t"); + assert.match(metadata, /^100(?:644|755) [0-9a-f]{40} 0$/, `${path}: only regular tracked files are allowed`); + assert.ok(lstatSync(resolve(candidateRoot, path)).isFile(), `${path}: checkout must be regular`); + assert.ok(!/(^|\/)(?:\.npmrc|npm-shrinkwrap\.json|\.gitleaks\.toml)$/u.test(path), `${path}: alternate authority is forbidden`); + } + const base = readJson(baseRoot, ".github/merge-policy.json"); + const candidate = readJson(candidateRoot, ".github/merge-policy.json"); + validateMergePolicy(base); + validateMergePolicy(candidate); + const normalized = new Set(candidate.protected_paths.map((path) => path.replace(/^\//u, ""))); + for (const path of base.protected_paths) { + assert.ok(normalized.has(path.replace(/^\//u, "")), `protected path removal requires a scoped exception: ${path}`); + } + for (const check of base.required_checks) { + assert.ok(candidate.required_checks.some((entry) => entry.context === check.context && entry.integration_id === check.integration_id), "required check removal is forbidden"); + } + assert.ok((candidate.required_approvals ?? 0) >= (base.required_approvals ?? 0), "review requirement cannot decrease"); + assert.ok((candidate.required_code_owner_reviews ?? 0) >= (base.required_code_owner_reviews ?? 0), "code-owner review cannot decrease"); + if (base.sensitive_review.prevent_self_review === true) assert.equal(candidate.sensitive_review.prevent_self_review, true); + assert.deepEqual(readdirSync(resolve(candidateRoot, ".github/workflows")).sort(), ["trusted.yml"], "only the inert wrapper is allowed"); + validateCandidateWorkflowDelegation(readFileSync(resolve(candidateRoot, ".github/workflows/trusted.yml"), "utf8")); + // Preserve installed local safeguards while policy implementation moves to the center. + for (const path of [".githooks/pre-commit", ".github/dependabot.yml"]) { + assert.equal(readFileSync(resolve(candidateRoot, path), "utf8"), readFileSync(resolve(baseRoot, path), "utf8"), `${path}: control evolution requires a separately reviewed central contract`); + } + assert.ok(lstatSync(resolve(candidateRoot, ".githooks/pre-commit")).mode & 0o111, "secret hook must remain executable"); + const ignore = (root) => readFileSync(resolve(root, ".gitignore"), "utf8").split("\n").filter((line) => line && !line.startsWith("#")); + const baseIgnore = ignore(baseRoot); + const candidateIgnore = ignore(candidateRoot); + for (const pattern of baseIgnore) assert.ok(candidateIgnore.includes(pattern), `.gitignore removed protection: ${pattern}`); + for (const pattern of candidateIgnore.filter((line) => line.startsWith("!"))) assert.ok(baseIgnore.includes(pattern), ".gitignore cannot add exclusion overrides"); + const baseOwners = readOwnership(baseRoot), candidateOwners = readOwnership(candidateRoot); + assert.equal(candidateOwners.path, baseOwners.path, "CODEOWNERS authority cannot move"); + // GitHub uses the last matching route. New routes may precede the existing + // ordered suffix, so they cannot shadow ownership of current or future paths. + // No approximation of GitHub's glob grammar is needed. + const retainedRoutes = candidateOwners.routes.slice(-baseOwners.routes.length); + assert.deepEqual(retainedRoutes.map(([pattern]) => pattern), baseOwners.routes.map(([pattern]) => pattern), "existing ownership routes must remain the ordered suffix; add new routes before them"); + for (const [index, [pattern, ...reviewers]] of baseOwners.routes.entries()) { + for (const reviewer of reviewers) assert.ok(retainedRoutes[index].slice(1).includes(reviewer), `owner removed for ${pattern}`); + } + for (const path of ["AGENTS.md", "SECURITY.md"]) assert.ok(readFileSync(resolve(candidateRoot, path), "utf8").trim(), `${path} must remain nonempty`); + validatePackage(candidateRoot); + return { status: "policy-passed" }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + console.log(JSON.stringify(validateCandidatePolicy(process.argv[2], process.argv[3]))); + } catch (error) { + console.error(`Policy violation or invalid control data: ${error.message}`); + process.exitCode = 1; + } +} diff --git a/.github/scripts/check-required-results.mjs b/.github/scripts/check-required-results.mjs new file mode 100644 index 0000000..09e33b3 --- /dev/null +++ b/.github/scripts/check-required-results.mjs @@ -0,0 +1,18 @@ +import assert from "node:assert/strict"; +import { pathToFileURL } from "node:url"; + +export function checkRequiredResults(needs) { + for (const job of ["authorize", "dependency-review", "codeql", "quality"]) { + assert.equal(needs[job]?.result, "success", `${job}: required success is missing`); + } + const sensitive = needs.authorize.outputs?.sensitive; + assert.ok(sensitive === "true" || sensitive === "false", "classification must be explicit"); + assert.equal(needs["sensitive-review"]?.result, sensitive === "true" ? "success" : "skipped", + "sensitive approval must match the exact run classification"); + return { status: "required-checks-passed" }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { console.log(JSON.stringify(checkRequiredResults(JSON.parse(process.env.NEEDS_JSON)))); } + catch (error) { console.error(error.message); process.exitCode = 1; } +} diff --git a/.github/scripts/classify-candidate.mjs b/.github/scripts/classify-candidate.mjs index 12dfad6..eebaa9b 100644 --- a/.github/scripts/classify-candidate.mjs +++ b/.github/scripts/classify-candidate.mjs @@ -47,7 +47,7 @@ function changedPaths(candidateRoot, baseSha, headSha) { return paths; } -function trustedWrapper(controlSha) { +export function trustedWrapper(controlSha) { return `name: OpenBoa Coffee trusted gate on: @@ -97,12 +97,31 @@ function requireTrustedCandidateWorkflow(candidateRoot) { ); } +function safePackageUpdate(trustedRoot, candidateRoot) { + const read = (root) => JSON.parse(readFileSync(resolve(root, "package.json"), "utf8")); + const base = read(trustedRoot); + const candidate = read(candidateRoot); + for (const field of ["dependencies", "devDependencies", "optionalDependencies"]) { + const before = base[field] ?? {}; + const after = candidate[field] ?? {}; + if (JSON.stringify(Object.keys(before).sort()) !== JSON.stringify(Object.keys(after).sort())) return false; + for (const name of Object.keys(before)) { + if (!/^\d+\.\d+\.\d+$/u.test(before[name]) || !/^\d+\.\d+\.\d+$/u.test(after[name])) return false; + const a = before[name].split(".").map(Number); + const b = after[name].split(".").map(Number); + if (a[0] !== b[0] || b[1] < a[1] || (a[1] === b[1] && b[2] < a[2])) return false; + } + delete base[field]; + delete candidate[field]; + } + return JSON.stringify(base) === JSON.stringify(candidate); +} + export function classifyCandidate({ actor, baseRepository, baseSha, candidateRoot, - exactPolicyOutcome, headRepository, headSha, prAuthor, @@ -123,22 +142,19 @@ export function classifyCandidate({ }); const paths = changedPaths(candidateRoot, baseSha, headSha); const protectedChanges = paths.filter((path) => - matchers.some((matcher) => matcher.test(path)), + matchers.some((matcher) => matcher.test(path)) || [".gitignore", ".gitattributes"].includes(path), ); const dependabotPackageOnly = - exactPolicyOutcome === "success" && actor === "dependabot[bot]" && prAuthor === "dependabot[bot]" && typeof baseRepository === "string" && baseRepository.length > 0 && headRepository === baseRepository && protectedChanges.length > 0 && - protectedChanges.every( + safePackageUpdate(trustedRoot, candidateRoot) && protectedChanges.every( (path) => path === "package.json" || path === "package-lock.json", ); - const sensitive = - exactPolicyOutcome !== "success" || - (protectedChanges.length > 0 && !dependabotPackageOnly); + const sensitive = protectedChanges.length > 0 && !dependabotPackageOnly; return Object.freeze({ sensitive, protectedChanges: Object.freeze(protectedChanges), @@ -152,7 +168,6 @@ function main() { baseRepository: process.env.BASE_REPOSITORY, baseSha: process.env.BASE_SHA, candidateRoot: process.env.CANDIDATE_ROOT, - exactPolicyOutcome: process.env.EXACT_POLICY_OUTCOME, headRepository: process.env.HEAD_REPOSITORY, headSha: process.env.HEAD_SHA, prAuthor: process.env.PR_AUTHOR, diff --git a/.github/scripts/install-actionlint.sh b/.github/scripts/install-actionlint.sh new file mode 100644 index 0000000..c6bcdc8 --- /dev/null +++ b/.github/scripts/install-actionlint.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail +version=1.7.12 +case "$(uname -s)-$(uname -m)" in + Linux-x86_64) platform=linux_amd64; digest=8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 ;; + Darwin-arm64) platform=darwin_arm64; digest=aba9ced2dee8d27fecca3dc7feb1a7f9a52caefa1eb46f3271ea66b6e0e6953f ;; + *) echo 'Unsupported actionlint platform' >&2; exit 1 ;; +esac +install_dir="${RUNNER_TEMP:?}/actionlint-$version" +mkdir -p "$install_dir" +archive="$install_dir/archive.tar.gz" +curl --fail --silent --show-error --location --proto '=https' --tlsv1.2 \ + "https://github.com/rhysd/actionlint/releases/download/v$version/actionlint_${version}_${platform}.tar.gz" --output "$archive" +printf '%s %s\n' "$digest" "$archive" | shasum -a 256 --check +tar -xzf "$archive" -C "$install_dir" actionlint +if test -n "${GITHUB_PATH:-}"; then printf '%s\n' "$install_dir" >> "$GITHUB_PATH"; fi +printf '%s\n' "$install_dir/actionlint" diff --git a/.github/scripts/reject-candidate-authorities.sh b/.github/scripts/reject-candidate-authorities.sh index e4493f2..a321327 100755 --- a/.github/scripts/reject-candidate-authorities.sh +++ b/.github/scripts/reject-candidate-authorities.sh @@ -10,6 +10,10 @@ if awk '$1 == "120000" { found = 1 } END { exit !found }' "$index_entries"; then echo 'candidate symlinks are not allowed across the trusted data boundary' >&2 exit 1 fi +if awk '$1 == "160000" { found = 1 } END { exit !found }' "$index_entries"; then + echo 'candidate gitlinks are not allowed across the trusted data boundary' >&2 + exit 1 +fi test ! -e "$candidate_root/.npmrc" test ! -e "$candidate_root/.github/policy-parser/.npmrc" diff --git a/.github/scripts/run-candidate-quality.sh b/.github/scripts/run-candidate-quality.sh deleted file mode 100755 index 100f609..0000000 --- a/.github/scripts/run-candidate-quality.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -repository="${1:?target repository is required}" -candidate_root="${2:?candidate repository root is required}" -candidate_home="${HOME:?}" -candidate_tmp="${RUNNER_TEMP:?}" -npm_globalconfig="$candidate_tmp/npm-globalconfig" -npm_userconfig="$candidate_tmp/npm-userconfig" - -: > "$npm_globalconfig" -: > "$npm_userconfig" - -candidate_environment=( - "CI=true" - "HOME=$candidate_home" - "LANG=${LANG:-C.UTF-8}" - "NPM_CONFIG_CACHE=$candidate_tmp/npm-cache" - "NPM_CONFIG_GLOBALCONFIG=$npm_globalconfig" - "NPM_CONFIG_REGISTRY=https://registry.npmjs.org" - "NPM_CONFIG_REPLACE_REGISTRY_HOST=never" - "NPM_CONFIG_USERCONFIG=$npm_userconfig" - "PATH=$PATH" - "PWD=$candidate_root" - "RUNNER_TEMP=$candidate_tmp" - "TMPDIR=${TMPDIR:-$candidate_tmp}" -) -if test -n "${DOCKER_HOST:-}"; then - candidate_environment+=("DOCKER_HOST=$DOCKER_HOST") -fi - -run_clean() { - env -i "${candidate_environment[@]}" "$@" -} - -zero_base_layout() { - case "$repository" in - openboa-ai/coffee-chat) - test -f plugin.json - test -f skills/roast/SKILL.md - test -f skills/brew/SKILL.md - ;; - openboa-ai/coffee-chat-roastery) - test -f origins/.gitkeep - test -f beans/.gitkeep - ;; - openboa-ai/coffee-chat-eval) - test -f iterations/README.md - ;; - openboa-ai/coffee-chat-bench) - test -f evals/README.md - test -f graders/README.md - test -f research/README.md - test -f evals/output-quality/perspective-capture/.gitkeep - test -f evals/output-quality/perspective-application/human-understanding/.gitkeep - test -f evals/output-quality/perspective-application/agent-judgment-action/.gitkeep - test -f evals/triggering/perspective-capture/.gitkeep - test -f evals/triggering/perspective-application/.gitkeep - ;; - *) - return 1 - ;; - esac -} - -cd "$candidate_root" -run_clean npm ci --ignore-scripts --no-bin-links -run_clean npm audit --audit-level=moderate - -if test ! -f .github/policy-parser/package.json; then - if ! zero_base_layout; then - echo "candidate is neither a legacy Coffee repository nor a recognized zero-base layout" >&2 - exit 1 - fi - run_clean node .github/ci-policy.mjs - exit 0 -fi - -run_clean npm ci --ignore-scripts --no-bin-links --prefix .github/policy-parser -run_clean npm audit --audit-level=moderate --prefix .github/policy-parser - -case "$repository" in - openboa-ai/coffee-chat) - run_clean node node_modules/prettier/bin/prettier.cjs --check . - run_clean node node_modules/typescript/bin/tsc --noEmit - run_clean node --test tests/*.test.mjs - run_clean node scripts/verify-readme-assets.mjs - run_clean node scripts/build-package.mjs - run_clean node scripts/package-smoke.mjs - ;; - openboa-ai/coffee-chat-roastery) - run_clean node node_modules/prettier/bin/prettier.cjs --check . - run_clean node node_modules/typescript/bin/tsc --noEmit - run_clean node scripts/build.mjs - run_clean git diff --exit-code -- dist - run_clean node scripts/check-repository-state.mjs --root . - run_clean node --test tests/*.test.mjs - run_clean node scripts/check-package.mjs - ;; - openboa-ai/coffee-chat-eval) - run_clean node node_modules/prettier/bin/prettier.cjs --check . - run_clean node node_modules/typescript/bin/tsc --noEmit - run_clean npm test - run_clean npm run dry-run - run_clean npm run smoke - run_clean npm run ci:policy - ;; - openboa-ai/coffee-chat-bench) - run_clean node node_modules/prettier/bin/prettier.cjs --check AGENTS.md README.md DATA-CARD.md PREREGISTRATION.md OVERLAP-REPORT.json package.json package-lock.json tsconfig.json prettier.config.mjs docs/*.md docs/validity/*.md harbor/*.md qualification/*.md qualification/*.json "bank/**/*.json" harbor/*.ts schemas/*.json scripts/*.mjs src/*.ts tests/*.mjs tests/*.ts - run_clean node scripts/check-inactive-boundary.mjs --root . - run_clean node node_modules/typescript/bin/tsc --noEmit - run_clean node --experimental-strip-types --test tests/*.test.mjs tests/*.test.ts - ;; - *) - echo "unsupported Coffee repository: $repository" >&2 - exit 1 - ;; -esac diff --git a/.github/scripts/run-eval-harbor.sh b/.github/scripts/run-eval-harbor.sh deleted file mode 100755 index d8ee78f..0000000 --- a/.github/scripts/run-eval-harbor.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -candidate_root="${1:?candidate repository root is required}" -candidate_tmp="${RUNNER_TEMP:?}" -uv_root="$candidate_tmp/uv-venv" -harbor_root="$candidate_tmp/harbor-venv" -bench_root="$candidate_tmp/bench-source" -projection_root="$candidate_tmp/bench-projection" -eval_root="$candidate_tmp/eval-oracle" -jobs_root="$eval_root/jobs" -bench_repository="https://github.com/openboa-ai/coffee-chat-bench.git" -bench_commit="1bc71605964770bbd1bd96e049b8412b6ee068fc" -npm_globalconfig="$candidate_tmp/bench-npm-globalconfig" -npm_userconfig="$candidate_tmp/bench-npm-userconfig" - -clean_environment=( - "CI=true" - "GIT_CONFIG_GLOBAL=/dev/null" - "GIT_CONFIG_NOSYSTEM=1" - "GIT_TERMINAL_PROMPT=0" - "HOME=${HOME:?}" - "LANG=${LANG:-C.UTF-8}" - "NPM_CONFIG_CACHE=$candidate_tmp/bench-npm-cache" - "NPM_CONFIG_GLOBALCONFIG=$npm_globalconfig" - "NPM_CONFIG_REGISTRY=https://registry.npmjs.org" - "NPM_CONFIG_REPLACE_REGISTRY_HOST=never" - "NPM_CONFIG_USERCONFIG=$npm_userconfig" - "PATH=$PATH" - "PIP_CONFIG_FILE=/dev/null" - "PIP_DISABLE_PIP_VERSION_CHECK=1" - "PIP_INDEX_URL=https://pypi.org/simple" - "RUNNER_TEMP=$candidate_tmp" - "TMPDIR=${TMPDIR:-$candidate_tmp}" - "UV_INDEX_URL=https://pypi.org/simple" - "UV_NO_CONFIG=1" -) -if test -n "${DOCKER_HOST:-}"; then - clean_environment+=("DOCKER_HOST=$DOCKER_HOST") -fi - -: > "$npm_globalconfig" -: > "$npm_userconfig" - -run_clean() { - env -i "${clean_environment[@]}" "$@" -} - -run_git() { - run_clean git -c protocol.file.allow=never -c credential.helper= "$@" -} - -if test ! -f "$candidate_root/.github/uv-requirements.txt"; then - test ! -e "$candidate_root/.github/harbor-requirements.txt" - test -f "$candidate_root/iterations/README.md" - printf '%s\n' 'Eval Harbor calibration not applicable to the zero-base evaluator layout.' - exit 0 -fi - -test -f "$candidate_root/.github/uv-requirements.txt" -test -f "$candidate_root/.github/harbor-requirements.txt" -test -f "$candidate_root/src/bench.ts" -test -f "$candidate_root/src/cli.ts" -test -f "$candidate_root/src/harbor.ts" -test -f "$candidate_root/src/runner.ts" -test ! -e "$bench_root" -test ! -e "$projection_root" -test ! -e "$eval_root" - -mkdir "$bench_root" -run_git init --quiet "$bench_root" -run_git -C "$bench_root" remote add origin "$bench_repository" -run_git -C "$bench_root" fetch --quiet --no-tags --depth=1 origin "$bench_commit" -run_git -C "$bench_root" checkout --quiet --detach FETCH_HEAD -test "$(run_git -C "$bench_root" rev-parse HEAD)" = "$bench_commit" -test -f "$bench_root/package-lock.json" -test -f "$bench_root/harbor/project.ts" -test -d "$bench_root/bank" - -run_clean python3 -m venv "$uv_root" -run_clean "$uv_root/bin/python" -m pip install \ - --disable-pip-version-check --require-hashes --no-deps \ - -r "$candidate_root/.github/uv-requirements.txt" -run_clean "$uv_root/bin/uv" venv --python python3 --no-python-downloads \ - "$harbor_root" -run_clean "$uv_root/bin/uv" pip install --require-hashes --no-deps \ - --only-binary :all: --python "$harbor_root/bin/python" \ - -r "$candidate_root/.github/harbor-requirements.txt" - -cd "$bench_root" -run_clean npm ci --ignore-scripts --no-audit --no-fund --no-bin-links -run_clean node --experimental-strip-types harbor/project.ts bank "$projection_root" -test -f "$projection_root/projection-manifest.json" - -mkdir "$eval_root" -cd "$candidate_root" -run_clean node --experimental-strip-types src/cli.ts oracle-control \ - --projection-root "$projection_root" \ - --case-id "ccbench-ra-s1-dialogue-01" \ - --diagnostic-target a \ - --bench-commit "$bench_commit" \ - --harbor-command "$harbor_root/bin/harbor" \ - --jobs-root "$jobs_root" -test -s "$jobs_root/receipts.json" diff --git a/.github/scripts/run-repository-verify.mjs b/.github/scripts/run-repository-verify.mjs new file mode 100644 index 0000000..8316b87 --- /dev/null +++ b/.github/scripts/run-repository-verify.mjs @@ -0,0 +1,80 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { validatePackage } from "./check-candidate-policy.mjs"; + +// Official Node image, resolved from node:24-bookworm. Update through control review. +export const IMAGE = "node@sha256:be23f54a88d34e8824c741b19b91064094f92c1c97b194144bfc8b50d67258e2"; + +export function containerArgs(volume, network, command, { user = "1000:1000", stdin = false } = {}) { + return ["run", "--rm", ...(stdin ? ["--interactive"] : []), + "--name", `${volume}-job`, "--network", network, "--user", user, + "--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges:true", + "--pids-limit=256", "--memory=2g", "--cpus=2", + "--tmpfs", "/tmp:rw,nosuid,nodev,size=512m,mode=1777", + "--mount", `type=volume,source=${volume},target=/work`, + "--workdir", "/work", "--env", "CI=true", "--env", "HOME=/tmp", + "--env", "NPM_CONFIG_USERCONFIG=/tmp/npm-userconfig", "--env", "NPM_CONFIG_GLOBALCONFIG=/tmp/npm-globalconfig", + "--env", "NPM_CONFIG_CACHE=/tmp/npm-cache", "--env", "NPM_CONFIG_REGISTRY=https://registry.npmjs.org", + "--env", "NPM_CONFIG_UPDATE_NOTIFIER=false", "--entrypoint", "/bin/sh", + IMAGE, "-ec", command]; +} + +export function repositorySnapshot(root) { + validatePackage(root); + const gitEnv = { ...process.env, GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1" }; + const git = (args, options = {}) => execFileSync("git", ["-C", root, ...args], { env: gitEnv, ...options }); + git(["diff", "--exit-code"]); // Include only the staged, reviewed tree, never local scratch files. + const tree = git(["write-tree"], { encoding: "utf8" }).trim(); + // -z disables Git's C quoting. Latin-1 preserves every pathname byte, including + // non-UTF8 names; only ASCII metadata, separators and basenames are interpreted. + const records = git(["ls-tree", "-r", "-z", tree]).toString("latin1").split("\0").filter(Boolean); + for (const record of records) { + const tab = record.indexOf("\t"); + const metadata = record.slice(0, tab).match(/^100(?:644|755) blob ([0-9a-f]{40})$/u); + if (!metadata) throw Error("snapshot contains a non-regular entry"); + const path = record.slice(tab + 1); + const basename = path.slice(path.lastIndexOf("/") + 1); + if ([".npmrc", "npm-shrinkwrap.json"].includes(basename)) throw Error("alternate installation authority"); + if (basename === ".gitattributes" && /export-ignore|export-subst/u.test(git(["cat-file", "blob", metadata[1]], { encoding: "utf8", maxBuffer: 128 * 1024 * 1024 }))) { + throw Error("archive transformations are not allowed"); + } + } + const archive = git(["archive", "--format=tar", tree], { maxBuffer: 128 * 1024 * 1024 }); + return { tree, archive }; +} + +export function runRepositoryVerify(root) { + const { tree, archive } = repositorySnapshot(root); + const volume = `coffee-verify-${randomUUID()}`; + const resume = randomUUID(); + const docker = (args, input) => { + const result = spawnSync("docker", args, { + stdio: input ? ["pipe", "inherit", "inherit"] : "inherit", input, + timeout: 20 * 60 * 1000, + }); + if (result.error || result.status !== 0) throw Error(`isolated verification failed (${result.status ?? result.error?.code})`); + }; + try { + docker(["volume", "create", volume]); + docker(containerArgs(volume, "none", "chmod 0777 /work; touch /work/.initialized", { user: "0:0" })); + docker(containerArgs(volume, "none", "mkdir repo; tar -xf - -C repo; cd repo; git -c core.hooksPath=/dev/null init -q; git -c core.hooksPath=/dev/null add -f --all", { stdin: true }), archive); + // The installer sees public package data, never credentials. No package lifecycle runs. + // Logs cannot issue workflow commands; the resume nonce never enters the container. + if (process.env.GITHUB_ACTIONS === "true") console.log(`::stop-commands::${resume}`); + docker(containerArgs(volume, "bridge", "cd repo; npm ci --ignore-scripts --no-bin-links --no-fund; npm audit --audit-level=moderate")); + // Candidate programs run only here: no network, host mount, socket, token or runner command file. + docker(containerArgs(volume, "none", "cd repo; npm --ignore-scripts run verify")); + return { tree, status: "verified", image: IMAGE }; + } finally { + spawnSync("docker", ["rm", "--force", `${volume}-job`], { stdio: "ignore" }); + spawnSync("docker", ["volume", "rm", volume], { stdio: "ignore" }); + if (process.env.GITHUB_ACTIONS === "true") console.log(`::${resume}::`); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { console.log(JSON.stringify(runRepositoryVerify(resolve(process.argv[2] ?? ".")))); } + catch (error) { console.error(error.message); process.exitCode = 1; } +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4185552 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,71 @@ +name: Organization controls CI + +on: + pull_request_target: + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: [main] + +permissions: {} + +jobs: + verify: + name: Organization controls verification + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + steps: + - name: Admit same-repository control changes + if: ${{ github.event_name == 'pull_request_target' }} + env: + HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }} + REPOSITORY: ${{ github.repository }} + ASSOCIATION: ${{ github.event.pull_request.author_association }} + PR_AUTHOR: ${{ github.event.pull_request.user.login }} + ACTOR: ${{ github.actor }} + run: | + test "$HEAD_REPOSITORY" = "$REPOSITORY" + case "$ASSOCIATION" in + OWNER|MEMBER) ;; + *) test "$PR_AUTHOR" = 'dependabot[bot]'; test "$ACTOR" = 'dependabot[bot]' ;; + esac + - name: Check out approved controls + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ github.event.pull_request.base.sha || github.sha }} + persist-credentials: false + path: control + - name: Check out candidate data + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + fetch-depth: 0 + path: candidate + - name: Set up Node.js for approved controls + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - name: Reject alternate candidate authority + run: bash control/.github/scripts/reject-candidate-authorities.sh "$GITHUB_WORKSPACE/candidate" + - name: Install pinned workflow linter + run: bash control/.github/scripts/install-actionlint.sh + - name: Validate candidate workflows as data + shell: bash + run: | + shopt -s nullglob dotglob + workflows=(candidate/.github/workflows/*.yml candidate/.github/workflows/*.yaml) + test "${#workflows[@]}" -gt 0 + actionlint -shellcheck= -pyflakes= "${workflows[@]}" + - name: Install trusted secret scanner + run: bash control/.github/scripts/install-gitleaks.sh + - name: Scan central history and candidate without repository configuration + run: | + gitleaks git --config "$GITLEAKS_TRUSTED_CONFIG" --gitleaks-ignore-path /dev/null --ignore-gitleaks-allow --redact --no-banner candidate + gitleaks dir --config "$GITLEAKS_TRUSTED_CONFIG" --gitleaks-ignore-path /dev/null --ignore-gitleaks-allow --redact --no-banner candidate + - name: Execute candidate regressions only in isolation + env: + NODE_OPTIONS: "" + NODE_PATH: "" + run: node control/.github/scripts/run-repository-verify.mjs "$GITHUB_WORKSPACE/candidate" diff --git a/.github/workflows/coffee-trusted-gate.yml b/.github/workflows/coffee-trusted-gate.yml index 7e06559..53f3b02 100644 --- a/.github/workflows/coffee-trusted-gate.yml +++ b/.github/workflows/coffee-trusted-gate.yml @@ -22,20 +22,27 @@ jobs: permissions: contents: read outputs: - sensitive: ${{ steps.classify.outputs.sensitive }} + sensitive: ${{ github.event_name == 'pull_request_target' && steps.classify.outputs.sensitive || 'false' }} steps: - name: Admit only the solo maintainer or in-repository Dependabot env: + EVENT_NAME: ${{ github.event_name }} + REF_NAME: ${{ github.ref }} ACTOR: ${{ github.actor }} AUTHOR_ASSOCIATION: ${{ github.event.pull_request.author_association }} BASE_REPOSITORY: ${{ github.repository }} - HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }} + HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name || github.repository }} PR_AUTHOR: ${{ github.event.pull_request.user.login }} run: | case "$BASE_REPOSITORY" in openboa-ai/coffee-chat|openboa-ai/coffee-chat-roastery|openboa-ai/coffee-chat-eval|openboa-ai/coffee-chat-bench) ;; *) exit 1 ;; esac + if test "$EVENT_NAME" = push; then + test "$REF_NAME" = refs/heads/main + exit 0 + fi + test "$EVENT_NAME" = pull_request_target case "$AUTHOR_ASSOCIATION" in OWNER|MEMBER) test "$ACTOR" = "$PR_AUTHOR" @@ -59,15 +66,15 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: repository: ${{ github.repository }} - ref: ${{ github.event.pull_request.base.sha }} + ref: ${{ github.event.pull_request.base.sha || github.event.before }} fetch-depth: 1 persist-credentials: false path: trusted-target - name: Check out the exact candidate as inert data uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} fetch-depth: 0 persist-credentials: false path: candidate @@ -79,9 +86,9 @@ jobs: run: bash control/.github/scripts/install-gitleaks.sh - name: Scan candidate history, worktree, and raw blobs without execution env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} BASE_REPOSITORY: ${{ github.repository }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} run: | set -o pipefail test ! -e candidate/.gitleaks.toml @@ -118,49 +125,26 @@ jobs: gitleaks dir --config "$GITLEAKS_TRUSTED_CONFIG" \ --gitleaks-ignore-path "$ignore_path" --ignore-gitleaks-allow \ --redact --no-banner "$blob_dir" - - name: Set up Node.js for trusted policy only + - name: Set up Node.js for trusted controls uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 with: node-version: 24 - - name: Authenticate and install the base policy parser - env: - NODE_OPTIONS: "" - NODE_PATH: "" - NPM_CONFIG_GLOBALCONFIG: ${{ runner.temp }}/trusted-npm-globalconfig - NPM_CONFIG_REGISTRY: https://registry.npmjs.org - NPM_CONFIG_REPLACE_REGISTRY_HOST: never - NPM_CONFIG_USERCONFIG: ${{ runner.temp }}/trusted-npm-userconfig - run: | - : > "$NPM_CONFIG_GLOBALCONFIG" - : > "$NPM_CONFIG_USERCONFIG" - test -f "$GITHUB_WORKSPACE/trusted-target/.github/policy-bootstrap.mjs" - test -f "$GITHUB_WORKSPACE/trusted-target/.github/ci-policy.mjs" - test ! -L "$GITHUB_WORKSPACE/trusted-target/.github/policy-bootstrap.mjs" - test ! -L "$GITHUB_WORKSPACE/trusted-target/.github/ci-policy.mjs" - node "$GITHUB_WORKSPACE/trusted-target/.github/policy-bootstrap.mjs" - npm ci --ignore-scripts --no-bin-links --prefix "$GITHUB_WORKSPACE/trusted-target/.github/policy-parser" - npm audit --audit-level=moderate --prefix "$GITHUB_WORKSPACE/trusted-target/.github/policy-parser" - - name: Evaluate exact trusted base policy against candidate data - id: exact-policy - continue-on-error: true + - name: Check central security policy against candidate data env: - BENCH_CI_POLICY_ROOT: ${{ github.workspace }}/candidate - CI_POLICY_ROOT: ${{ github.workspace }}/candidate - EVAL_CI_POLICY_ROOT: ${{ github.workspace }}/candidate NODE_OPTIONS: "" NODE_PATH: "" - ROASTERY_CI_POLICY_ROOT: ${{ github.workspace }}/candidate - run: node "$GITHUB_WORKSPACE/trusted-target/.github/ci-policy.mjs" + run: >- + node control/.github/scripts/check-candidate-policy.mjs + "$GITHUB_WORKSPACE/trusted-target" "$GITHUB_WORKSPACE/candidate" - name: Classify policy evolution and protected path changes id: classify env: ACTOR: ${{ github.actor }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} BASE_REPOSITORY: ${{ github.repository }} CANDIDATE_ROOT: ${{ github.workspace }}/candidate - EXACT_POLICY_OUTCOME: ${{ steps.exact-policy.outcome }} - HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} + HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} PR_AUTHOR: ${{ github.event.pull_request.user.login }} TRUSTED_ROOT: ${{ github.workspace }}/trusted-target run: node control/.github/scripts/classify-candidate.mjs @@ -190,12 +174,16 @@ jobs: contents: read steps: - name: Reject vulnerable dependency changes + if: ${{ github.event_name == 'pull_request_target' }} uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 with: comment-summary-in-pr: never fail-on-severity: moderate fail-on-scopes: runtime,development,unknown show-patched-versions: true + - name: Record post-merge dependency coverage + if: ${{ github.event_name == 'push' }} + run: echo 'The isolated quality lane audits the complete locked dependency graph on main.' codeql: name: Trusted CodeQL JavaScript-TypeScript @@ -222,8 +210,8 @@ jobs: - name: Check out candidate source without persisted credentials uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} persist-credentials: false path: candidate - name: Initialize CodeQL without building candidate code @@ -244,7 +232,7 @@ jobs: '${{ steps.codeql-analyze.outputs.sarif-output }}' quality: - name: Trusted deterministic quality + name: Isolated repository verification needs: - authorize - dependency-review @@ -270,8 +258,8 @@ jobs: - name: Check out the exact authorized candidate uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} fetch-depth: 0 persist-credentials: false path: candidate @@ -279,52 +267,11 @@ jobs: uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 with: node-version: 24 - - name: Install immutable Gitleaks for repository security tests - run: bash control/.github/scripts/install-gitleaks.sh - - name: Run repository-specific checks with a minimal environment - run: >- - control/.github/scripts/run-candidate-quality.sh - '${{ github.repository }}' "$GITHUB_WORKSPACE/candidate" - - eval-harbor: - name: Trusted Eval Harbor calibration - needs: - - authorize - - sensitive-review - if: >- - ${{ always() && github.repository == 'openboa-ai/coffee-chat-eval' && - needs.authorize.result == 'success' && - (needs.authorize.outputs.sensitive != 'true' || - needs.sensitive-review.result == 'success') }} - runs-on: ubuntu-24.04 - timeout-minutes: 30 - permissions: - contents: read - steps: - - name: Check out immutable organization controls - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - repository: openboa-ai/.github - ref: ${{ inputs.control_sha }} - fetch-depth: 1 - persist-credentials: false - path: control - - name: Check out the exact authorized Eval candidate - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 1 - persist-credentials: false - path: candidate - - name: Set up Node.js without candidate dependencies - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 - with: - node-version: 24 - - name: Install the hash-locked Harbor graph and calibrate - run: >- - control/.github/scripts/run-eval-harbor.sh - "$GITHUB_WORKSPACE/candidate" + - name: Run repository-owned verification in isolation + env: + NODE_OPTIONS: "" + NODE_PATH: "" + run: node control/.github/scripts/run-repository-verify.mjs "$GITHUB_WORKSPACE/candidate" required: name: OpenBoa Coffee trusted required @@ -337,34 +284,23 @@ jobs: - sensitive-review - dependency-review - codeql - - eval-harbor - quality runs-on: ubuntu-24.04 timeout-minutes: 5 - permissions: {} + permissions: + contents: read steps: + - name: Check out immutable organization controls + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + repository: openboa-ai/.github + ref: ${{ inputs.control_sha }} + fetch-depth: 1 + persist-credentials: false + path: control - name: Require every trusted lane env: - AUTHORIZE_RESULT: ${{ needs.authorize.result }} - CODEQL_RESULT: ${{ needs.codeql.result }} - DEPENDENCY_REVIEW_RESULT: ${{ needs.dependency-review.result }} - EVAL_HARBOR_RESULT: ${{ needs.eval-harbor.result }} - QUALITY_RESULT: ${{ needs.quality.result }} - REPOSITORY: ${{ github.repository }} - SENSITIVE: ${{ needs.authorize.outputs.sensitive }} - SENSITIVE_REVIEW_RESULT: ${{ needs.sensitive-review.result }} - run: | - test "$AUTHORIZE_RESULT" = success - test "$DEPENDENCY_REVIEW_RESULT" = success - test "$CODEQL_RESULT" = success - test "$QUALITY_RESULT" = success - if test "$REPOSITORY" = openboa-ai/coffee-chat-eval; then - test "$EVAL_HARBOR_RESULT" = success - else - test "$EVAL_HARBOR_RESULT" = skipped - fi - if test "$SENSITIVE" = true; then - test "$SENSITIVE_REVIEW_RESULT" = success - else - test "$SENSITIVE_REVIEW_RESULT" = skipped - fi + NEEDS_JSON: ${{ toJSON(needs) }} + NODE_OPTIONS: "" + NODE_PATH: "" + run: node control/.github/scripts/check-required-results.mjs diff --git a/AGENTS.md b/AGENTS.md index 19fd842..dd4740f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,19 +6,29 @@ definitions. Treat every executable or policy file as a sensitive control. - Target repository changes use pull requests. Routine changes remain eligible for native auto-merge; protected paths and policy evolution pause at the `coffee-security` GitHub Environment for the solo maintainer's confirmation. -- Required workflows execute pull-request content only as inert data. Controls - and parsers must come from the required workflow SHA or the target base SHA. +- Authorization and security jobs treat pull-request content only as inert data. + Controls and parsers come from the pinned workflow SHA or the target base SHA. + Repository-owned verification runs only through the trusted launcher in a + separate non-root container without a network, host mounts, credentials, + runner command files, shared privileged caches, or Docker socket. - Target repositories define one exact `pull_request_target` wrapper containing no executable steps. It calls this organization-owned reusable workflow by a - full commit SHA; authorization, secret scanning, dependency review, CodeQL, - and deterministic quality all remain here. + full commit SHA. Central code owns authorization, secret scanning, dependency + review, CodeQL and execution isolation; repositories own npm run verify. + Keep the required aggregate name and all protected paths. Policy failures + fail the run; approval never converts them to success. - Do not add secrets, OIDC, package publishing, deployment, or write-token permissions. The only write permission is `security-events: write` in the trusted CodeQL job. - Do not enable merge queue. Routine auto-merge applies in the target Coffee repositories only after this trusted workflow and their normal CI pass. -- Eval Harbor calibration runs on a fresh runner before any candidate program, - from its complete hash-locked dependency graph. -- Run `npm test`, `actionlint .github/workflows/*.yml`, `sh -n +- Product/evaluator-specific tests and calibration belong to their repository. + Do not restore historical product dispatchers or report structural checks as + a calibration, benchmark result, or Product lift. +- Central PR changes also run through base-owned isolation. The initial rollout + needs owner-reviewed local evidence because the old base has no self-CI. + Never bypass old required checks to bootstrap a new pin without a scoped, + expiring exception identifying the exact base, head and compensating checks. +- Run `npm test`, `actionlint` (all workflow extensions), `sh -n .github/scripts/*.sh`, `node --check .github/scripts/*.mjs`, and `git diff --check` before merging. diff --git a/CODEOWNERS b/CODEOWNERS index 0cf20ce..45df435 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,7 +1,9 @@ -/.github/** @openboa-ai/security-maintainers -/.npmrc @openboa-ai/security-maintainers -/AGENTS.md @openboa-ai/security-maintainers -/CODEOWNERS @openboa-ai/security-maintainers -/SECURITY.md @openboa-ai/security-maintainers -/package.json @openboa-ai/security-maintainers -/scripts/** @openboa-ai/security-maintainers +# Keep existing team ownership; add both maintainer accounts for independent review. +* @SonSangjoon @openboa +/.github/** @openboa-ai/security-maintainers @SonSangjoon @openboa +/.npmrc @openboa-ai/security-maintainers @SonSangjoon @openboa +/AGENTS.md @openboa-ai/security-maintainers @SonSangjoon @openboa +/CODEOWNERS @openboa-ai/security-maintainers @SonSangjoon @openboa +/SECURITY.md @openboa-ai/security-maintainers @SonSangjoon @openboa +/package.json @openboa-ai/security-maintainers @SonSangjoon @openboa +/scripts/** @openboa-ai/security-maintainers @SonSangjoon @openboa diff --git a/README.md b/README.md index 535fc8e..40b38b8 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,75 @@ -# .github -OpenBoa organization profile and shared GitHub community configuration. +# OpenBoa GitHub controls + +Central controls answer whether a change meets security and approval policy. +Each Coffee Chat repository answers whether its own product or data is correct. +Neither requires the center to know Product Skills, benchmark folders, or an +evaluator implementation. + +## Ownership and execution + +1. An exact, inert wrapper selects this reusable workflow by immutable SHA. +2. Approved central code checks candidate data, existing protected paths, secrets, + dependency changes and CodeQL findings. Invalid policy fails immediately. +3. Sensitive changes wait for the existing coffee-security approval. +4. The repository's npm run verify runs in an isolated container. Installation + uses its integrity-locked registry dependencies without lifecycle scripts; + candidate programs have no network, token, host mount or runner command files. +5. The unchanged required aggregate accepts only explicit successful results. + GitHub applies required checks and independent review. Target wrappers retain + only pull_request_target; they cannot add push or other execution triggers. + Post-merge observation belongs to organization-owned orchestration; the + target wrapper itself does not claim an automatic main-push check. + +Targets are coffee-chat (Product), coffee-chat-roastery (public data seed), +coffee-chat-bench (evaluation definitions), and coffee-chat-eval (execution/evidence). +The project owner is SonSangjoon. Existing team ownership remains in CODEOWNERS; +the PR author cannot supply their own independent review. + +## Verify the controls + +Run npm run verify, actionlint (all workflow extensions), shell/Node syntax checks +and git diff --check. scripts/test-isolation.mjs additionally exercises the real +Docker boundary; it is explicit because unit-test containers cannot access the +host Docker socket. The pinned installer for actionlint verifies its SHA256. +The test suite covers malformed control data, protected-path removal, workflow +spoofing, installation authority, approval and cancelled/missing job results. + +Trusted CI for this repository uses base-owned controls to execute candidate +regressions in isolation. Do not add a candidate-owned pull_request workflow as +a bootstrap: its author can change the host-runner steps before any guard or +container runs. The workflow inventory test detects drift during verification; +it is not a GitHub pre-execution enforcement barrier for new workflow files. + +The first introduction cannot retroactively create base-owned CI on the old +base. Before requesting owner approval, complete latest-head Codex review, +resolve code findings, and retain exact-tree local regression, lint, secret-scan +and real-isolation evidence. Report missing GitHub CI as unverified, not passing; +do not create a substitute success check or bypass existing required checks. +The owner must review the initial landing, including this evidence gap. After +landing, observe trusted main CI before upgrading callers. Subsequent central +PRs require the base-owned CI as well as Codex review. Source commits in support +are full SHAs, not branch names. An old caller still uses its old pin on a rerun. + +## Migration and evidence + +Do not weaken a required check to change its implementation. Keep the canonical +PR-only wrapper shape unchanged, validate each target's actual base/head, then +update only its immutable control SHA. Existing protected paths, secret hooks, update policy and +required-check identity are retained. Changes to those safeguards require an +explicitly reviewed successor contract, not approval of an unrelated failure. + +Use .github/scripts/audit-ci-settings.mjs to inspect live settings. It is read-only +and uses GitHub's effective main rules, including exclusions and inherited rules, +instead of treating declarations as enforcement. Missing ruleset/bypass details +remain unverified. CODEOWNERS keeps one location and its existing ordered routes +as a suffix; new routes go before them so they cannot override existing owners. +Owner additions use GitHub user/team handles; malformed tokens fail because they +can invalidate an entire route. Comments preserve those routes. Any initial +blocked-pin exception must be scoped to +an exact PR/base/head/control SHA, expire, include compensating verification and +leave all protections restored. Existing code-quality/coverage rules are not +deleted by this rollout; their producers must be verified before claiming success. + +Local tests, live GitHub checks, owner approval, merged main and post-merge runs +are separate evidence states. No passing CI check is a Ground Truth approval, +Judge qualification, evaluator calibration or Product-performance result. diff --git a/SECURITY.md b/SECURITY.md index 600b11d..5863501 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -10,7 +10,22 @@ to it. Routine changes may auto-merge only after every trusted lane succeeds; protected paths and policy evolution additionally require the solo maintainer's `coffee-security` GitHub Environment confirmation. -Candidate repositories may supply only the exact inert trusted-workflow wrapper, -never candidate steps, alternate npm authority, or symlinked control data. Eval -Harbor calibration uses a fresh runner and an authenticated dependency graph so -earlier candidate tests cannot mutate it. +Candidate repositories supply only the exact inert trusted-workflow wrapper, +never candidate steps, alternate npm authority, or symlinked control data. +The target wrapper permits only `pull_request_target` and uses an immutable +control SHA. Organization-owned `.github/workflows/ci.yml` observes this central +repository's `main` branch through `push` after owner-reviewed initial landing. +It does not add automatic main-push verification to target repositories. + +Authorization reads policy as data; it never imports repository validators. +Repository-owned npm run verify executes in a separate non-root, network-disabled +container. The trusted installer uses the locked public npm graph with lifecycle +scripts disabled. Neither stage receives host mounts, runner command files, +credentials, a Docker socket, or a privileged cache. Docker isolation does not +claim protection against a kernel or container-runtime vulnerability; use fresh +GitHub-hosted runners and keep the pinned runtime reviewed. + +CODEOWNERS protects merge-time review, not pre-review workflow execution. +The execution boundary is enforced by the approved launcher, not by candidate +YAML permission declarations. Same-repository administrators remain trusted +GitHub control-plane principals; a check name alone is not a workflow identity. diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..ed73ae4 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,12 @@ +{ + "name": "@openboa-ai/organization-controls", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@openboa-ai/organization-controls", + "version": "1.0.0" + } + } +} diff --git a/package.json b/package.json index 04503c8..0506589 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "version": "1.0.0", "type": "module", "scripts": { - "test": "node --test scripts/test-coffee-required-workflow.mjs" + "test": "node --test scripts/test-ci-boundary.mjs", + "verify": "npm test" } } diff --git a/scripts/test-ci-boundary.mjs b/scripts/test-ci-boundary.mjs new file mode 100644 index 0000000..2e44c87 --- /dev/null +++ b/scripts/test-ci-boundary.mjs @@ -0,0 +1,448 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { cpSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import test from "node:test"; +import { classifyCandidate, trustedWrapper, validateCandidateWorkflowDelegation } from "../.github/scripts/classify-candidate.mjs"; +import { validateCandidatePolicy, validatePackage } from "../.github/scripts/check-candidate-policy.mjs"; +import { checkCodeqlSarif } from "../.github/scripts/check-codeql-sarif.mjs"; +import { checkRequiredResults } from "../.github/scripts/check-required-results.mjs"; +import { containerArgs, IMAGE, repositorySnapshot } from "../.github/scripts/run-repository-verify.mjs"; +import { auditSettings } from "../.github/scripts/audit-ci-settings.mjs"; + +const source = resolve(import.meta.dirname, ".."); +const workflow = readFileSync(join(source, ".github/workflows/coffee-trusted-gate.yml"), "utf8"); +const write = (root, path, text) => { mkdirSync(dirname(join(root, path)), { recursive: true }); writeFileSync(join(root, path), text); }; +const json = (root, path, data) => write(root, path, JSON.stringify(data, null, 2) + "\n"); +const git = (root, ...args) => execFileSync("git", ["-C", root, "-c", "core.hooksPath=/dev/null", "-c", "commit.gpgsign=false", ...args], { encoding: "utf8" }); +function commit(root) { + git(root, "add", "-f", "--all"); + git(root, "-c", "user.name=CI test", "-c", "user.email=ci@example.invalid", "commit", "-qm", "fixture"); + return git(root, "rev-parse", "HEAD").trim(); +} +function fixture(run) { + const root = mkdtempSync(join(tmpdir(), "coffee-ci-policy-")); + const base = join(root, "base"), candidate = join(root, "candidate"); + try { + const policy = { + merge_method: "squash", merge_queue: false, required_approvals: 0, + eligible_author_associations: ["OWNER", "MEMBER"], eligible_bot_logins: ["dependabot[bot]"], + required_checks: [{ context: "OpenBoa Coffee trusted required / OpenBoa Coffee trusted required", integration_id: 15368 }], + protected_paths: ["/.github/**", "/AGENTS.md", "/SECURITY.md", "/CODEOWNERS", "/package.json", "/package-lock.json", "/data/**"], + sensitive_review: { enforcement: "github_environment", environment: "coffee-security", required_approvals: 1, prevent_self_review: false }, + }; + json(base, ".github/merge-policy.json", policy); + json(base, "package.json", { name: "fixture", version: "0.0.0", private: true, scripts: { verify: "node verify.mjs" } }); + json(base, "package-lock.json", { name: "fixture", version: "0.0.0", lockfileVersion: 3, packages: { "": { name: "fixture", version: "0.0.0" } } }); + write(base, ".github/workflows/trusted.yml", trustedWrapper("a".repeat(40))); + write(base, ".github/dependabot.yml", "version: 2\nupdates: []\n"); + write(base, ".githooks/pre-commit", "#!/bin/sh\nexit 0\n"); + execFileSync("chmod", ["755", join(base, ".githooks/pre-commit")]); + write(base, "CODEOWNERS", "/.github/** @owner\n"); + write(base, "AGENTS.md", "Keep the trust boundary.\n"); + write(base, "SECURITY.md", "Private vulnerability reporting.\n"); + write(base, ".gitignore", ".env\nnode_modules/\n"); + write(base, "README.md", "fixture\n"); + write(base, "verify.mjs", "throw Error('candidate code must never run in policy checks');\n"); + cpSync(base, candidate, { recursive: true }); + git(candidate, "init", "-q"); + return run({ root, base, candidate, baseSha: commit(candidate), policy }); + } finally { rmSync(root, { recursive: true, force: true }); } +} + +test("policy reads data without executing candidate or requiring a base parser", () => fixture(({ base, candidate }) => { + assert.deepEqual(validateCandidatePolicy(base, candidate), { status: "policy-passed" }); +})); +test("security policy protection is mandatory in both trusted and candidate policies", () => { + for (const scope of ["both", "base", "candidate"]) for (const replacement of [undefined, "/security.md", "/docs/SECURITY.md", "./SECURITY.md", "//SECURITY.md"]) fixture(({ base, candidate, policy }) => { + policy.protected_paths = policy.protected_paths.filter((path) => path !== "/SECURITY.md"); + if (replacement !== undefined) policy.protected_paths.push(replacement); + for (const root of scope === "both" ? [base, candidate] : [scope === "base" ? base : candidate]) json(root, ".github/merge-policy.json", policy); + git(candidate, "add", "-f", "--all"); + assert.throws(() => validateCandidatePolicy(base, candidate), /protected control missing: SECURITY\.md/u); + }); +}); +test("root security policy edits remain sensitive with either supported path spelling", () => { + for (const path of ["SECURITY.md", "/SECURITY.md"]) fixture(({ base, candidate, baseSha: initialBaseSha, policy }) => { + policy.protected_paths = policy.protected_paths.map((entry) => entry === "/SECURITY.md" ? path : entry); + for (const root of [base, candidate]) json(root, ".github/merge-policy.json", policy); + const baseSha = path === "/SECURITY.md" ? initialBaseSha : commit(candidate); + write(candidate, "SECURITY.md", "Changed security guidance.\n"); + const headSha = commit(candidate); + assert.equal(validateCandidatePolicy(base, candidate).status, "policy-passed"); + for (const actor of ["owner", "dependabot[bot]"]) { + const result = classifyCandidate({ baseRepository: "openboa-ai/coffee-chat", headRepository: "openboa-ai/coffee-chat", actor, prAuthor: actor, trustedRoot: base, candidateRoot: candidate, baseSha, headSha }); + assert.equal(result.sensitive, true); + assert.deepEqual(result.protectedChanges, ["SECURITY.md"]); + } + }); +}); +test("the PR-only wrapper is structurally exact and SHA-updatable", () => { + for (const sha of ["a".repeat(40), "b".repeat(40)]) assert.equal(validateCandidateWorkflowDelegation(trustedWrapper(sha)), sha); + for (const mutate of [ + (s) => s.replace("contents: read", "contents: write"), + (s) => s.replace("control_sha: " + "a".repeat(40), "control_sha: " + "b".repeat(40)), + (s) => s + " steps:\n - run: echo spoof\n", + (s) => s.replace("pull_request_target:", "pull_request:"), + (s) => s.replace("\n\npermissions: {}", "\n push:\n branches: [main]\n\npermissions: {}"), + ]) assert.throws(() => validateCandidateWorkflowDelegation(mutate(trustedWrapper("a".repeat(40))))); +}); +test("target trigger expansion fails policy and classification before approval", () => { + for (const trigger of [" push:\n branches: [main]\n", " workflow_dispatch: {}\n", " schedule:\n - cron: '0 0 * * *'\n"]) fixture(({ base, candidate, baseSha }) => { + const expanded = trustedWrapper("a".repeat(40)).replace("\n\npermissions: {}", "\n" + trigger + "\npermissions: {}"); + write(candidate, ".github/workflows/trusted.yml", expanded); + const headSha = commit(candidate); + assert.throws(() => validateCandidatePolicy(base, candidate), /exact trusted wrapper/u); + assert.throws(() => classifyCandidate({ baseRepository: "openboa-ai/coffee-chat", headRepository: "openboa-ai/coffee-chat", actor: "owner", prAuthor: "owner", trustedRoot: base, candidateRoot: candidate, baseSha, headSha }), /exact trusted wrapper/u); + }); +}); +test("invalid policy and removed safeguards fail rather than request approval", () => { + for (const mutate of [ + ({ candidate, policy }) => { policy.protected_paths.pop(); json(candidate, ".github/merge-policy.json", policy); }, + ({ candidate, policy }) => { policy.required_checks = []; json(candidate, ".github/merge-policy.json", policy); }, + ({ candidate, policy }) => { policy.sensitive_review.required_approvals = 0; json(candidate, ".github/merge-policy.json", policy); }, + ({ candidate }) => write(candidate, ".github/workflows/spoof.yml", "on: pull_request\n"), + ({ candidate }) => write(candidate, ".githooks/pre-commit", "disabled\n"), + ({ candidate }) => write(candidate, ".gitignore", "node_modules/\n"), + ({ candidate }) => write(candidate, "CODEOWNERS", "/.github/** @other\n"), + ({ candidate }) => write(candidate, ".github/merge-policy.json", "{malformed"), + ]) fixture((f) => { mutate(f); git(f.candidate, "add", "-f", "--all"); assert.throws(() => validateCandidatePolicy(f.base, f.candidate)); }); +}); +test("alternate authority, symlinks and gitlinks cannot cross the boundary", () => { + for (const path of [".npmrc", "npm-shrinkwrap.json", "nested/.npmrc", ".gitleaks.toml"]) fixture(({ base, candidate }) => { + write(candidate, path, "untrusted\n"); git(candidate, "add", "-f", "--all"); assert.throws(() => validateCandidatePolicy(base, candidate)); + }); + fixture(({ base, candidate }) => { + symlinkSync("package.json", join(candidate, "escape")); git(candidate, "add", "--all"); assert.throws(() => validateCandidatePolicy(base, candidate)); + }); + fixture(({ base, candidate, baseSha }) => { + git(candidate, "update-index", "--add", "--cacheinfo", "160000," + baseSha + ",external"); assert.throws(() => validateCandidatePolicy(base, candidate)); + }); +}); +test("ownership precedence, competing locations and unloaded files cannot remove review routes", () => { + for (const mutate of [ + ({ candidate }) => write(candidate, "CODEOWNERS", "/.github/** @owner\n* @other\n"), + ({ candidate }) => write(candidate, "CODEOWNERS", "/.github/** @owner\n/.github/workflows/** @other\n"), + ({ candidate }) => write(candidate, "CODEOWNERS", "/.github/** @owner\n/.github/workflows/**\n"), + ({ candidate }) => write(candidate, ".github/CODEOWNERS", "* @other\n"), + ({ candidate }) => write(candidate, "docs/CODEOWNERS", "* @other\n"), + ({ base, candidate }) => { + write(base, "CODEOWNERS", "* @global\n/.github/** @owner\n"); + write(candidate, "CODEOWNERS", "/.github/** @owner\n* @global\n"); + }, + ({ candidate }) => write(candidate, "CODEOWNERS", "/.github/** @owner\n#" + "x".repeat(3_000_000)), + ({ candidate }) => write(candidate, "CODEOWNERS", "/.github/** @other # @owner\n"), + ({ candidate }) => write(candidate, "CODEOWNERS", "/.github/** @owner docs@\n"), + ({ candidate }) => write(candidate, "CODEOWNERS", "/.github/** @owner @invalid--login\n"), + ]) fixture((f) => { + mutate(f); git(f.candidate, "add", "-f", "--all"); + assert.throws(() => validateCandidatePolicy(f.base, f.candidate)); + }); +}); +test("ownership preserves ordered routes and supports maintainer additions and GitHub locations", () => { + for (const path of ["CODEOWNERS", ".github/CODEOWNERS", "docs/CODEOWNERS"]) fixture(({ base, candidate }) => { + if (path !== "CODEOWNERS") for (const root of [base, candidate]) rmSync(join(root, "CODEOWNERS")); + write(base, path, "* @global\n/.github/** @owner\n"); + write(candidate, path, " # Review routes\r\n*\t@global @maintainer\r\n/.github/** @owner @maintainer # owners retained\r\n"); + git(candidate, "add", "-f", "--all"); + assert.equal(validateCandidatePolicy(base, candidate).status, "policy-passed"); + }); + fixture(({ base, candidate }) => { + write(candidate, "CODEOWNERS", "/new-file.md @maintainer\n/.github/** @owner\n"); + git(candidate, "add", "-f", "--all"); + assert.equal(validateCandidatePolicy(base, candidate).status, "policy-passed"); + }); +}); +test("lock rejects local dependencies, missing integrity, implicit scripts and drift", () => { + for (const mutate of [ + (pkg) => { pkg.private = false; }, + (pkg) => { pkg.scripts.preverify = "node malicious.mjs"; }, + (pkg, lock) => { lock.name = "other"; }, + (pkg, lock) => { lock.packages["node_modules/dep"] = { resolved: "file:../private", integrity: "sha512-YWJj" }; }, + (pkg, lock) => { lock.packages["node_modules/dep"] = { resolved: "https://registry.npmjs.org/dep/-/dep-1.0.0.tgz" }; }, + ]) fixture(({ candidate }) => { + const pkg = JSON.parse(readFileSync(join(candidate, "package.json"))); + const lock = JSON.parse(readFileSync(join(candidate, "package-lock.json"))); + mutate(pkg, lock); json(candidate, "package.json", pkg); json(candidate, "package-lock.json", lock); assert.throws(() => validatePackage(candidate)); + }); +}); +test("classification uses protected paths, not product formats or parser errors", () => { + for (const [path, sensitive] of [["README.md", false], ["data/new-format.json", true], [".gitignore", true]]) fixture(({ base, candidate, baseSha }) => { + write(candidate, path, "new data\n"); const headSha = commit(candidate); + assert.equal(classifyCandidate({ baseRepository: "openboa-ai/coffee-chat-bench", headRepository: "openboa-ai/coffee-chat-bench", actor: "owner", prAuthor: "owner", trustedRoot: base, candidateRoot: candidate, baseSha, headSha }).sensitive, sensitive); + }); +}); +test("Dependabot cannot change verify while posing as a package update", () => fixture(({ base, candidate, baseSha }) => { + const pkg = JSON.parse(readFileSync(join(candidate, "package.json"))); + pkg.scripts.verify = "echo bypass"; json(candidate, "package.json", pkg); + const headSha = commit(candidate); + assert.equal(classifyCandidate({ baseRepository: "openboa-ai/coffee-chat", headRepository: "openboa-ai/coffee-chat", actor: "dependabot[bot]", prAuthor: "dependabot[bot]", trustedRoot: base, candidateRoot: candidate, baseSha, headSha }).sensitive, true); +})); +test("Dependabot exact patch/minor bumps stay routine while major/range/downgrade changes stay sensitive", () => { + for (const field of ["dependencies", "devDependencies", "optionalDependencies"]) { + for (const [version, sensitive] of [["1.2.4", false], ["1.3.0", false], ["2.0.0", true], ["1.2.2", true], ["1.1.9", true], ["^1.2.4", true]]) fixture(({ base, candidate }) => { + const pkg = JSON.parse(readFileSync(join(base, "package.json"))); + pkg[field] = { dependency: "1.2.3" }; + json(base, "package.json", pkg); json(candidate, "package.json", pkg); + const baseSha = commit(candidate); + pkg[field].dependency = version; + json(candidate, "package.json", pkg); + const headSha = commit(candidate); + const options = { baseRepository: "openboa-ai/coffee-chat", headRepository: "openboa-ai/coffee-chat", actor: "dependabot[bot]", prAuthor: "dependabot[bot]", trustedRoot: base, candidateRoot: candidate, baseSha, headSha }; + assert.equal(classifyCandidate(options).sensitive, sensitive, `${field}: ${version}`); + if (!sensitive) { + assert.equal(classifyCandidate({ ...options, actor: "owner" }).sensitive, true); + assert.equal(classifyCandidate({ ...options, prAuthor: "owner" }).sensitive, true); + assert.equal(classifyCandidate({ ...options, headRepository: "outsider/coffee-chat" }).sensitive, true); + } + }); + } +}); +test("aggregate rejects missing, cancelled, failed and unapproved runs", () => { + const success = () => ({ authorize: { result: "success", outputs: { sensitive: "false" } }, "dependency-review": { result: "success" }, codeql: { result: "success" }, quality: { result: "success" }, "sensitive-review": { result: "skipped" } }); + assert.equal(checkRequiredResults(success()).status, "required-checks-passed"); + for (const job of ["authorize", "dependency-review", "codeql", "quality"]) for (const state of ["failure", "cancelled", "skipped", undefined]) { + const result = success(); result[job].result = state; assert.throws(() => checkRequiredResults(result)); + } + for (const state of ["failure", "cancelled", "skipped", undefined]) { + const result = success(); result.authorize.outputs.sensitive = "true"; result["sensitive-review"].result = state; assert.throws(() => checkRequiredResults(result)); + } + const result = success(); result.authorize.outputs.sensitive = "true"; result["sensitive-review"].result = "success"; assert.equal(checkRequiredResults(result).status, "required-checks-passed"); + delete result.authorize.outputs.sensitive; assert.throws(() => checkRequiredResults(result)); +}); +test("sandbox has no verification network, host bind, socket or inherited credentials", () => { + const args = containerArgs("coffee-test", "none", "cd repo; npm --ignore-scripts run verify"); + for (const flag of ["--read-only", "--cap-drop=ALL", "--security-opt=no-new-privileges:true", "--pids-limit=256"]) assert.ok(args.includes(flag)); + assert.equal(args[args.indexOf("--network") + 1], "none"); + assert.equal(args[args.indexOf("--user") + 1], "1000:1000"); + assert.equal(args[args.indexOf("--mount") + 1], "type=volume,source=coffee-test,target=/work"); + assert.equal(args.filter((arg) => arg === "--mount").length, 1); + assert.match(IMAGE, /^node@sha256:[0-9a-f]{64}$/u); + assert.doesNotMatch(args.join(" "), /docker\.sock|type=bind|GITHUB_TOKEN|GITHUB_OUTPUT|--privileged|--env-file/u); +}); +test("snapshot cannot hide archive attributes behind Git-quoted names", () => { + for (const directory of ["tést", "한글", "tab\tpath", "line\npath", 'quote"path', "back\\slash"]) { + for (const attribute of ["export-ignore", "export-subst"]) fixture(({ candidate }) => { + const path = `${directory}/reviewed.txt`; + write(candidate, path, "$Format:%H$\n"); + write(candidate, `${directory}/.gitattributes`, `reviewed.txt ${attribute}\n`); + commit(candidate); + if (directory === "tést" && attribute === "export-ignore") { + const archive = execFileSync("git", ["-C", candidate, "archive", "--format=tar", git(candidate, "write-tree").trim()]); + assert.ok(git(candidate, "ls-tree", "-r", "-z", "--name-only", "HEAD").split("\0").includes(path)); + assert.notEqual(spawnSync("tar", ["-xOf", "-", path], { input: archive }).status, 0, "unguarded archive omits the reviewed file"); + } + assert.throws(() => repositorySnapshot(candidate), /archive transformations are not allowed/u, JSON.stringify(directory)); + }); + for (const authority of [".npmrc", "npm-shrinkwrap.json"]) fixture(({ candidate }) => { + write(candidate, `${directory}/${authority}`, "untrusted\n"); + commit(candidate); + assert.throws(() => repositorySnapshot(candidate), /alternate installation authority/u, JSON.stringify(directory)); + }); + } +}); +test("snapshot preserves ordinary Unicode and control-character filenames", () => fixture(({ candidate }) => { + for (const path of ["한글/문서.txt", "tést/note.txt", "tab\tpath/note.txt", "line\npath/note.txt"]) write(candidate, path, "reviewed\n"); + const comments = "# harmless comment\n".repeat(60_000); + write(candidate, "tést/.gitattributes", comments + "note.txt -text\n"); + commit(candidate); + const snapshot = repositorySnapshot(candidate); + assert.equal(snapshot.tree, git(candidate, "write-tree").trim()); + for (const path of ["한글/문서.txt", "tést/note.txt", "tab\tpath/note.txt", "line\npath/note.txt"]) { + assert.equal(execFileSync("tar", ["-xOf", "-", path], { input: snapshot.archive, encoding: "utf8" }), "reviewed\n"); + } + write(candidate, "tést/.gitattributes", comments + "note.txt export-ignore\n"); + commit(candidate); + assert.throws(() => repositorySnapshot(candidate), /archive transformations are not allowed/u); +})); +test("snapshot checks raw non-UTF8 path bytes without lossy decoding", { skip: process.platform !== "linux" }, () => { + for (const basename of [".gitattributes", ".npmrc", "npm-shrinkwrap.json"]) fixture(({ candidate }) => { + const directory = Buffer.concat([Buffer.from(candidate + "/raw-"), Buffer.from([0xff])]); + mkdirSync(directory); + writeFileSync(Buffer.concat([directory, Buffer.from("/" + basename)]), "* export-ignore\n"); + commit(candidate); + assert.throws(() => repositorySnapshot(candidate), basename === ".gitattributes" ? /archive transformations/u : /alternate installation authority/u); + }); +}); +test("workflow preserves trusted scans and approval ordering without product coupling", () => { + for (const pattern of [/gitleaks git/u, /git -C candidate cat-file blob/u, /dependency-review-action@[0-9a-f]{40}/u, /build-mode: none/u, /check-codeql-sarif.mjs/u, /environment: coffee-security/u, /needs\.sensitive-review\.result == 'success'/u, /run-repository-verify.mjs/u]) assert.match(workflow, pattern); + assert.doesNotMatch(workflow, /continue-on-error|exact-policy|policy-bootstrap|ci-policy.mjs|run-candidate-quality|harbor/u); + for (const match of workflow.matchAll(/uses: ([^\s]+)/gu)) assert.match(match[1], /@[0-9a-f]{40}$/u); + assert.doesNotMatch(readFileSync(join(source, ".github/scripts/check-candidate-policy.mjs"), "utf8"), /skills\/|evals\/|iterations\/|perspective-capture|development\/|policy-parser/u); +}); +test("central PR orchestration stays base-owned without a candidate bootstrap", () => { + // Regression coverage, not a GitHub pre-execution policy barrier. + assert.deepEqual(readdirSync(join(source, ".github/workflows")).filter((file) => /\.ya?ml$/u.test(file)).sort(), ["ci.yml", "coffee-trusted-gate.yml"]); + const trusted = readFileSync(join(source, ".github/workflows/ci.yml"), "utf8"); + assert.equal(trusted.match(/^on:\n([\s\S]*?)\npermissions:/mu)[1], " pull_request_target:\n types: [opened, synchronize, reopened, ready_for_review]\n push:\n branches: [main]\n"); + assert.match(trusted, /name: Organization controls verification/u); + assert.match(trusted, /ref: \$\{\{ github\.event\.pull_request\.base\.sha \|\| github\.sha \}\}[\s\S]*?path: control/u); + assert.match(trusted, /ref: \$\{\{ github\.event\.pull_request\.head\.sha \|\| github\.sha \}\}[\s\S]*?path: candidate/u); + assert.match(trusted, /node control\/\.github\/scripts\/run-repository-verify\.mjs/u); + assert.doesNotMatch(trusted, / pull_request:|node candidate\/|bash candidate\/|working-directory: candidate|continue-on-error/u); + for (const match of trusted.matchAll(/uses: ([^\s]+)/gu)) assert.match(match[1], /@[0-9a-f]{40}$/u); + assert.match(workflow, /^on:\n workflow_call:\n/mu); +}); +test("actual lint step includes both workflow extensions and propagates failures", () => { + const trusted = readFileSync(join(source, ".github/workflows/ci.yml"), "utf8"); + const block = trusted.slice(trusted.indexOf(" - name: Validate candidate workflows as data"), trusted.indexOf(" - name: Install trusted secret scanner")); + const run = block.match(/ run: ([\s\S]*)/u)[1].trimEnd(); + const script = run.startsWith("|\n") ? run.slice(2).split("\n").map((line) => line.slice(10)).join("\n") : run; + const capture = "actionlint() { printf '%s\\0' \"$@\"; return \"$LINT_STATUS\"; }\n"; + for (const files of [["existing.yml", "new workflow.yaml", ".hidden.yaml"], ["only.yml"], ["only.yaml"], []]) { + const root = mkdtempSync(join(tmpdir(), "coffee-workflow-lint-")); + try { + const paths = files.map((file) => "candidate/.github/workflows/" + file); + for (const path of paths) write(root, path, "on: push\n"); + write(root, "candidate/.github/workflows/README.md", "Not a workflow.\n"); + for (const status of [0, 23]) { + const result = spawnSync("bash", ["-e", "-o", "pipefail", "-c", capture + script], { cwd: root, env: { PATH: process.env.PATH, LINT_STATUS: String(status) }, encoding: "utf8" }); + if (files.length === 0) { + assert.notEqual(result.status, 0); + assert.equal(result.stdout, ""); + } else { + assert.equal(result.status, status, result.stderr); + assert.deepEqual(result.stdout.split("\0").slice(0, -1).sort(), ["-shellcheck=", "-pyflakes=", ...paths].sort()); + } + } + } finally { rmSync(root, { recursive: true, force: true }); } + } +}); +test("CodeQL findings and missing, malformed or symlinked SARIF fail closed", () => { + const root = mkdtempSync(join(tmpdir(), "coffee-sarif-")); + try { + assert.throws(() => checkCodeqlSarif(root)); + const clean = { version: "2.1.0", runs: [{ tool: { driver: { name: "CodeQL" } }, results: [] }] }; + json(root, "javascript.sarif", clean); assert.deepEqual(checkCodeqlSarif(root), { files: 1, results: 0 }); + clean.runs[0].results = [{ message: { text: "finding" } }]; json(root, "javascript.sarif", clean); assert.throws(() => checkCodeqlSarif(root)); + write(root, "javascript.sarif", "{}"); assert.throws(() => checkCodeqlSarif(root)); + rmSync(join(root, "javascript.sarif")); symlinkSync("missing", join(root, "javascript.sarif")); assert.throws(() => checkCodeqlSarif(root)); + } finally { rmSync(root, { recursive: true, force: true }); } +}); +test("JavaScript and shell controls pass syntax checks", () => { + for (const file of readdirSync(join(source, ".github/scripts"))) { + const path = join(source, ".github/scripts", file); + if (file.endsWith(".mjs")) assert.equal(spawnSync(process.execPath, ["--check", path]).status, 0, file); + if (file.endsWith(".sh")) assert.equal(spawnSync("bash", ["-n", path]).status, 0, file); + } +}); + +function auditSnapshot(repository = "coffee-chat-bench") { + const snapshot = { + rulesets: [{ enforcement: "active", conditions: { ref_name: { include: ["refs/heads/main"] } }, bypass_actors: [], rules: [ + ...["deletion", "non_fast_forward", "required_linear_history"].map((type) => ({ type })), + { type: "pull_request", parameters: { require_code_owner_review: true, dismiss_stale_reviews_on_push: true, required_review_thread_resolution: true } }, + { type: "required_status_checks", parameters: { strict_required_status_checks_policy: true, required_status_checks: [{ context: repository === ".github" ? "Organization controls verification" : "OpenBoa Coffee trusted required / OpenBoa Coffee trusted required", integration_id: 15368 }] } }, + ] }], + environment: repository === ".github" ? null : { can_admins_bypass: false, protection_rules: [{ type: "required_reviewers", reviewers: [{ type: "User", reviewer: { login: "SonSangjoon" } }] }] }, + actions: { default_workflow_permissions: "read", can_approve_pull_request_reviews: false }, + }; + snapshot.rulesets[0].id = 42; + snapshot.branchRules = snapshot.rulesets[0].rules.map((rule) => ({ ...rule, ruleset_id: 42 })); + return snapshot; +} +test("live settings audit distinguishes declarations from enforced reviews and checks", () => { + const snapshot = auditSnapshot(); + assert.deepEqual(auditSettings("coffee-chat-bench", snapshot).issues, []); + for (const exclude of [["refs/heads/main"], ["~DEFAULT_BRANCH"], ["refs/heads/m*"], ["~ALL"]]) { + const excluded = structuredClone(snapshot); + excluded.rulesets[0].conditions.ref_name.exclude = exclude; + // GitHub's effective-branch endpoint returns no rules for these declarations. + excluded.branchRules = []; + assert.ok(auditSettings("coffee-chat-bench", excluded).issues.includes("No active ruleset protects main.")); + } + const missing = structuredClone(snapshot); + delete missing.branchRules; + assert.ok(auditSettings("coffee-chat-bench", missing).issues.includes("Effective main rules could not be verified.")); + const incomplete = structuredClone(snapshot); + incomplete.rulesets = []; + assert.ok(auditSettings("coffee-chat-bench", incomplete).issues.includes("Applicable ruleset details could not be verified: 42")); + const hiddenBypass = structuredClone(snapshot); + delete hiddenBypass.rulesets[0].bypass_actors; + assert.ok(auditSettings("coffee-chat-bench", hiddenBypass).issues.includes("Applicable ruleset details could not be verified: 42")); + const layered = structuredClone(snapshot); + layered.branchRules.unshift({ type: "pull_request", ruleset_id: 43, parameters: {} }, { type: "required_status_checks", ruleset_id: 43, parameters: {} }); + layered.rulesets.push({ id: 43, enforcement: "active", bypass_actors: [] }); + assert.deepEqual(auditSettings("coffee-chat-bench", layered).issues, []); + const emptyStrict = structuredClone(layered); + for (const rule of emptyStrict.branchRules.filter((rule) => rule.type === "required_status_checks")) { + rule.parameters.strict_required_status_checks_policy = rule.ruleset_id === 43; + if (rule.ruleset_id === 43) rule.parameters.required_status_checks = []; + } + assert.ok(auditSettings("coffee-chat-bench", emptyStrict).issues.includes("Required checks do not require an up-to-date branch.")); + snapshot.environment.protection_rules = []; + assert.ok(auditSettings("coffee-chat-bench", snapshot).issues.includes("coffee-security has no required reviewers.")); + snapshot.rulesets = []; + snapshot.branchRules = []; + assert.ok(auditSettings("coffee-chat-bench", snapshot).issues.includes("No active ruleset protects main.")); +}); + +test("environment audit requires the owner exclusively and verifies administrator bypass", () => { + const owner = { type: "User", reviewer: { login: "SonSangjoon" } }; + const other = { type: "User", reviewer: { login: "openboa" } }; + const team = { type: "Team", reviewer: { login: "SonSangjoon", slug: "owners" } }; + for (const repository of ["coffee-chat", "coffee-chat-roastery", "coffee-chat-eval", "coffee-chat-bench"]) { + assert.deepEqual(auditSettings(repository, auditSnapshot(repository)).issues, []); + for (const reviewers of [[other], [team], [owner, other], [owner, team], [other, owner], [], [{}], [{ type: "User", reviewer: {} }], "not-an-array", null]) { + const snapshot = auditSnapshot(repository); + snapshot.environment.protection_rules[0].reviewers = reviewers; + assert.ok(auditSettings(repository, snapshot).issues.includes("coffee-security must require only User SonSangjoon.")); + } + for (const bypass of [true, undefined, null, "false"]) { + const snapshot = auditSnapshot(repository); + snapshot.environment.can_admins_bypass = bypass; + assert.ok(auditSettings(repository, snapshot).issues.some((issue) => /administrator bypass/u.test(issue))); + } + const duplicate = auditSnapshot(repository); + duplicate.environment.protection_rules.push({ type: "required_reviewers", reviewers: [other] }); + assert.ok(auditSettings(repository, duplicate).issues.includes("coffee-security must require only User SonSangjoon.")); + const normal = auditSnapshot(repository); + normal.environment.protection_rules[0].reviewers[0].reviewer.login = "sonsangjoon"; + normal.environment.protection_rules[0].prevent_self_review = true; + normal.environment.protection_rules.push({ type: "branch_policy" }); + assert.deepEqual(auditSettings(repository, normal).issues, []); + } + assert.deepEqual(auditSettings(".github", auditSnapshot(".github")).issues, []); +}); + +test("settings audit rejects effective merge queues but ignores excluded declarations", () => { + for (const repository of [".github", "coffee-chat", "coffee-chat-roastery", "coffee-chat-eval", "coffee-chat-bench"]) { + for (const id of [42, 43]) { + const snapshot = auditSnapshot(repository); + if (id === 43) snapshot.rulesets.push({ id, enforcement: "active", bypass_actors: [] }); + snapshot.branchRules.push({ type: "merge_queue", ruleset_id: id }); + assert.ok(auditSettings(repository, snapshot).issues.includes("Merge queue is enabled for main.")); + } + const excluded = auditSnapshot(repository); + excluded.rulesets.push({ id: 43, enforcement: "active", bypass_actors: [], conditions: { ref_name: { include: ["~ALL"], exclude: ["refs/heads/main"] } }, rules: [{ type: "merge_queue" }] }); + assert.deepEqual(auditSettings(repository, excluded).issues, []); + } +}); + +test("early symlink in a large index still fails the executable authority guard", () => { + const root = mkdtempSync(join(tmpdir(), "coffee-index-")); + try { + git(root, "init", "-q"); + write(root, "package.json", "{}\n"); + symlinkSync("package.json", join(root, "000-escape")); + for (let i = 0; i < 5000; i++) write(root, "bulk/" + i, "x"); + git(root, "add", "--all"); + assert.notEqual(spawnSync("bash", [join(source, ".github/scripts/reject-candidate-authorities.sh"), root]).status, 0); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test("actual admission step rejects forks and unsupported events before checkout", () => { + const block = workflow.slice(workflow.indexOf(" - name: Admit only"), workflow.indexOf(" - name: Check out immutable")); + const script = block.slice(block.indexOf(" run: |\n") + " run: |\n".length).split("\n").map((line) => line.slice(10)).join("\n"); + const env = { PATH: process.env.PATH, BASE_REPOSITORY: "openboa-ai/coffee-chat", HEAD_REPOSITORY: "openboa-ai/coffee-chat", EVENT_NAME: "pull_request_target", AUTHOR_ASSOCIATION: "OWNER", ACTOR: "owner", PR_AUTHOR: "owner", REF_NAME: "refs/heads/main" }; + for (const [change, succeeds] of [ + [{}, true], + [{ HEAD_REPOSITORY: "outsider/coffee-chat" }, false], + [{ AUTHOR_ASSOCIATION: "CONTRIBUTOR" }, false], + [{ AUTHOR_ASSOCIATION: "CONTRIBUTOR", ACTOR: "dependabot[bot]", PR_AUTHOR: "dependabot[bot]" }, true], + [{ EVENT_NAME: "push" }, true], + [{ EVENT_NAME: "push", REF_NAME: "refs/heads/feature" }, false], + [{ EVENT_NAME: "workflow_dispatch" }, false], + ]) assert.equal(spawnSync("bash", ["-e", "-o", "pipefail", "-c", script], { env: { ...env, ...change } }).status === 0, succeeds); +}); diff --git a/scripts/test-coffee-required-workflow.mjs b/scripts/test-coffee-required-workflow.mjs deleted file mode 100644 index 2fb6ca1..0000000 --- a/scripts/test-coffee-required-workflow.mjs +++ /dev/null @@ -1,691 +0,0 @@ -import assert from "node:assert/strict"; -import { execFileSync, spawnSync } from "node:child_process"; -import { - mkdtempSync, - mkdirSync, - renameSync, - readFileSync, - rmSync, - symlinkSync, - writeFileSync, -} from "node:fs"; -import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; -import test from "node:test"; - -import { - classifyCandidate, - validateCandidateWorkflowDelegation, -} from "../.github/scripts/classify-candidate.mjs"; -import { checkCodeqlSarif } from "../.github/scripts/check-codeql-sarif.mjs"; - -const root = resolve(import.meta.dirname, ".."); -const workflowPath = resolve(root, ".github/workflows/coffee-trusted-gate.yml"); -const workflow = readFileSync(workflowPath, "utf8"); -const authorityCheck = resolve( - root, - ".github/scripts/reject-candidate-authorities.sh", -); -const qualityRunnerPath = resolve( - root, - ".github/scripts/run-candidate-quality.sh", -); -const qualityRunner = readFileSync(qualityRunnerPath, "utf8"); -const evalHarborRunner = readFileSync( - resolve(root, ".github/scripts/run-eval-harbor.sh"), - "utf8", -); - -function commit(repository, message) { - execFileSync("git", ["-C", repository, "add", "."]); - execFileSync("git", ["-C", repository, "commit", "-qm", message]); - return execFileSync("git", ["-C", repository, "rev-parse", "HEAD"], { - encoding: "utf8", - }).trim(); -} - -function trustedWrapper(controlSha) { - return `name: OpenBoa Coffee trusted gate - -on: - pull_request_target: - types: [opened, synchronize, reopened, ready_for_review] - -permissions: {} - -jobs: - trusted: - name: OpenBoa Coffee trusted required - permissions: - actions: read - contents: read - security-events: write - uses: openboa-ai/.github/.github/workflows/coffee-trusted-gate.yml@${controlSha} - with: - control_sha: ${controlSha} -`; -} - -function classificationFixture(mutate) { - const fixture = mkdtempSync(join(tmpdir(), "coffee-classifier-")); - const candidate = join(fixture, "candidate"); - const trusted = join(fixture, "trusted"); - mkdirSync(candidate); - mkdirSync(join(trusted, ".github"), { recursive: true }); - execFileSync("git", ["init", "-q", candidate]); - execFileSync("git", ["-C", candidate, "config", "user.email", "test@example.invalid"]); - execFileSync("git", ["-C", candidate, "config", "user.name", "Policy test"]); - mkdirSync(join(candidate, "src")); - mkdirSync(join(candidate, ".github/workflows"), { recursive: true }); - writeFileSync(join(candidate, "README.md"), "base\n"); - writeFileSync( - join(candidate, "package.json"), - `${JSON.stringify({ devDependencies: { prettier: "3.9.6" } })}\n`, - ); - writeFileSync( - join(candidate, "package-lock.json"), - `${JSON.stringify({ lockfileVersion: 3 })}\n`, - ); - writeFileSync(join(candidate, "src/control.ts"), "export const control = true;\n"); - writeFileSync( - join(candidate, ".github/workflows/trusted.yml"), - trustedWrapper("a".repeat(40)), - ); - const baseSha = commit(candidate, "base"); - writeFileSync( - join(trusted, ".github/merge-policy.json"), - `${JSON.stringify({ - protected_paths: [ - "/src/**", - "/.github/**", - "/package.json", - "/package-lock.json", - ], - })}\n`, - ); - mutate(candidate); - const headSha = commit(candidate, "candidate"); - return { baseSha, candidate, fixture, headSha, trusted }; -} - -function classifyFixture( - mutate, - exactPolicyOutcome = "success", - identity = { - actor: "solo-maintainer", - baseRepository: "openboa-ai/coffee-chat", - headRepository: "openboa-ai/coffee-chat", - prAuthor: "solo-maintainer", - }, -) { - const fixture = classificationFixture(mutate); - try { - return classifyCandidate({ - ...identity, - baseSha: fixture.baseSha, - candidateRoot: fixture.candidate, - exactPolicyOutcome, - headSha: fixture.headSha, - trustedRoot: fixture.trusted, - }); - } finally { - rmSync(fixture.fixture, { force: true, recursive: true }); - } -} - -test("trusted gate is callable only through a trusted target wrapper", () => { - assert.match(workflow, /^on:\n workflow_call:\n inputs:\n control_sha:/mu); - assert.match(workflow, /required: true/u); - assert.match(workflow, /type: string/u); - assert.doesNotMatch(workflow, /^ pull_request(?:_target)?:/mu); - assert.doesNotMatch(workflow, /^concurrency:/mu); - assert.match(workflow, /^permissions: \{\}$/mu); - assert.match(workflow, /openboa-ai\/coffee-chat-bench/gu); - assert.doesNotMatch(workflow, /github\.repository != ''/u); -}); - -test("candidate is data and all controls come from immutable trusted commits", () => { - const sourceCheckout = workflow.indexOf("ref: ${{ inputs.control_sha }}"); - const baseCheckout = workflow.indexOf( - "ref: ${{ github.event.pull_request.base.sha }}", - ); - const candidateCheckout = workflow.indexOf( - "ref: ${{ github.event.pull_request.head.sha }}", - ); - const policy = workflow.indexOf( - 'node "$GITHUB_WORKSPACE/trusted-target/.github/ci-policy.mjs"', - ); - assert.ok(sourceCheckout > 0); - assert.ok(baseCheckout > sourceCheckout); - assert.ok(candidateCheckout > baseCheckout); - assert.ok(policy > candidateCheckout); - assert.match(workflow, /repository: openboa-ai\/\.github/u); - assert.match(workflow, /ref: \$\{\{ inputs\.control_sha \}\}/gu); - assert.match(workflow, /path: control/u); - assert.match(workflow, /path: trusted-target/u); - assert.match(workflow, /path: candidate/u); - assert.match(workflow, /persist-credentials: false/gu); -}); - -test("trusted gate rejects alternate npm authority before any candidate install", () => { - assert.match( - workflow, - /control\/\.github\/scripts\/reject-candidate-authorities\.sh/u, - ); - assert.doesNotMatch(workflow, /npm ci --[^\n]*prefix candidate/u); - assert.doesNotMatch(workflow, /working-directory:\s*candidate/u); - assert.doesNotMatch(workflow, /npm run/u); -}); - -test("trusted gate rejects candidate symlink escapes before reading data", () => { - const rejection = workflow.indexOf("reject-candidate-authorities.sh"); - const secretScan = workflow.indexOf( - "Scan candidate history, worktree, and raw blobs", - ); - const policy = workflow.indexOf( - "Evaluate exact trusted base policy against candidate data", - ); - assert.ok(rejection > 0); - assert.ok(rejection < secretScan); - assert.ok(rejection < policy); -}); - -test("authority check rejects every alternate npm authority", () => { - for (const authority of [ - ".npmrc", - ".github/policy-parser/.npmrc", - "npm-shrinkwrap.json", - ]) { - const fixture = mkdtempSync(join(tmpdir(), "coffee-npm-authority-")); - try { - execFileSync("git", ["init", "-q", fixture]); - mkdirSync(resolve(fixture, authority, ".."), { recursive: true }); - writeFileSync(resolve(fixture, authority), "registry=https://attacker.invalid\n"); - execFileSync("git", ["-C", fixture, "add", "."]); - const result = spawnSync(authorityCheck, [fixture], { encoding: "utf8" }); - assert.equal(result.status, 1, `${authority} was accepted`); - } finally { - rmSync(fixture, { force: true, recursive: true }); - } - } -}); - -test("classifier keeps routine changes automatic", () => { - const result = classifyFixture((candidate) => { - writeFileSync(join(candidate, "README.md"), "routine\n"); - }); - assert.equal(result.sensitive, false); - assert.deepEqual(result.protectedChanges, []); -}); - -test("classifier routes protected edits and policy evolution to the Environment", () => { - const protectedEdit = classifyFixture((candidate) => { - writeFileSync(join(candidate, "src/control.ts"), "export const control = false;\n"); - }); - assert.equal(protectedEdit.sensitive, true); - assert.deepEqual(protectedEdit.protectedChanges, ["src/control.ts"]); - - const policyEvolution = classifyFixture((candidate) => { - writeFileSync(join(candidate, "README.md"), "new policy shape\n"); - }, "failure"); - assert.equal(policyEvolution.sensitive, true); -}); - -test("classifier exempts only exact in-repository Dependabot package changes", () => { - const packageChanges = (candidate) => { - writeFileSync( - join(candidate, "package.json"), - `${JSON.stringify({ devDependencies: { prettier: "3.9.7" } })}\n`, - ); - writeFileSync( - join(candidate, "package-lock.json"), - `${JSON.stringify({ lockfileVersion: 3, updated: true })}\n`, - ); - }; - const dependabotIdentity = { - actor: "dependabot[bot]", - baseRepository: "openboa-ai/coffee-chat", - headRepository: "openboa-ai/coffee-chat", - prAuthor: "dependabot[bot]", - }; - - const routine = classifyFixture( - packageChanges, - "success", - dependabotIdentity, - ); - assert.equal(routine.sensitive, false); - assert.deepEqual(routine.protectedChanges, [ - "package-lock.json", - "package.json", - ]); - - for (const [ - label, - exactPolicyOutcome, - identity, - mutate, - expectedProtectedChanges, - ] of [ - [ - "owner package update", - "success", - { - actor: "solo-maintainer", - baseRepository: "openboa-ai/coffee-chat", - headRepository: "openboa-ai/coffee-chat", - prAuthor: "solo-maintainer", - }, - packageChanges, - ["package-lock.json", "package.json"], - ], - [ - "Dependabot actor mismatch", - "success", - { ...dependabotIdentity, actor: "solo-maintainer" }, - packageChanges, - ["package-lock.json", "package.json"], - ], - [ - "Dependabot author mismatch", - "success", - { ...dependabotIdentity, prAuthor: "solo-maintainer" }, - packageChanges, - ["package-lock.json", "package.json"], - ], - [ - "Dependabot fork", - "success", - { ...dependabotIdentity, headRepository: "attacker/coffee-chat" }, - packageChanges, - ["package-lock.json", "package.json"], - ], - [ - "policy failure", - "failure", - dependabotIdentity, - packageChanges, - ["package-lock.json", "package.json"], - ], - [ - "additional protected path", - "success", - dependabotIdentity, - (candidate) => { - packageChanges(candidate); - writeFileSync( - join(candidate, "src/control.ts"), - "export const control = false;\n", - ); - }, - ["package-lock.json", "package.json", "src/control.ts"], - ], - ]) { - const result = classifyFixture(mutate, exactPolicyOutcome, identity); - assert.equal(result.sensitive, true, label); - assert.deepEqual( - result.protectedChanges, - expectedProtectedChanges, - label, - ); - } -}); - -test("classifier checks both sides of protected path renames", () => { - const result = classifyFixture((candidate) => { - mkdirSync(join(candidate, "lib")); - renameSync(join(candidate, "src/control.ts"), join(candidate, "lib/control.ts")); - }); - assert.equal(result.sensitive, true); - assert.ok(result.protectedChanges.includes("src/control.ts")); -}); - -test("classifier rejects any workflow except the exact inert trusted wrapper", () => { - const fixture = classificationFixture((candidate) => { - writeFileSync(join(candidate, ".github/workflows/spoof.yml"), "on: pull_request\n"); - }); - try { - assert.throws( - () => - classifyCandidate({ - baseSha: fixture.baseSha, - candidateRoot: fixture.candidate, - exactPolicyOutcome: "success", - headSha: fixture.headSha, - trustedRoot: fixture.trusted, - }), - /exact trusted wrapper/u, - ); - } finally { - rmSync(fixture.fixture, { force: true, recursive: true }); - } -}); - -test("trusted wrapper validation is version-updatable but structurally exact", () => { - const firstSha = "a".repeat(40); - const secondSha = "b".repeat(40); - assert.equal(validateCandidateWorkflowDelegation(trustedWrapper(firstSha)), firstSha); - assert.equal(validateCandidateWorkflowDelegation(trustedWrapper(secondSha)), secondSha); - assert.throws( - () => - validateCandidateWorkflowDelegation( - trustedWrapper(firstSha).replace( - `control_sha: ${firstSha}`, - `control_sha: ${secondSha}`, - ), - ), - /exact trusted wrapper/u, - ); - assert.throws( - () => - validateCandidateWorkflowDelegation( - `${trustedWrapper(firstSha)} - run: candidate-script\n`, - ), - /exact trusted wrapper/u, - ); -}); - -test("authority check rejects an early symlink in a large Git index", () => { - const fixture = mkdtempSync(join(tmpdir(), "coffee-authority-check-")); - try { - execFileSync("git", ["init", "-q", fixture]); - writeFileSync(join(fixture, "package.json"), "{}\n"); - writeFileSync(join(fixture, "package-lock.json"), "{}\n"); - symlinkSync("package.json", join(fixture, "000-escape")); - const bulk = join(fixture, "bulk"); - mkdirSync(bulk); - for (let index = 0; index < 5000; index += 1) { - writeFileSync(join(bulk, `${String(index).padStart(5, "0")}.txt`), "x\n"); - } - execFileSync("git", ["-C", fixture, "add", "."]); - const result = spawnSync(authorityCheck, [fixture], { encoding: "utf8" }); - assert.equal(result.status, 1, result.stderr); - assert.match(result.stderr, /candidate symlinks are not allowed/u); - } finally { - rmSync(fixture, { force: true, recursive: true }); - } -}); - -test("trusted gate clears Node injection paths and resolves parser from base", () => { - assert.match(workflow, /NODE_OPTIONS: ""/u); - assert.match(workflow, /NODE_PATH: ""/u); - assert.match( - workflow, - /NPM_CONFIG_REGISTRY: https:\/\/registry\.npmjs\.org/u, - ); - assert.match(workflow, /NPM_CONFIG_REPLACE_REGISTRY_HOST: never/u); - assert.match(workflow, /trusted-npm-globalconfig/u); - assert.match(workflow, /trusted-npm-userconfig/u); - assert.doesNotMatch( - workflow, - /NPM_CONFIG_(?:GLOBAL|USER)CONFIG: \/dev\/null/u, - ); - assert.match( - workflow, - /npm ci --ignore-scripts --no-bin-links --prefix "\$GITHUB_WORKSPACE\/trusted-target\/\.github\/policy-parser"/u, - ); - for (const rootVariable of [ - "CI_POLICY_ROOT", - "ROASTERY_CI_POLICY_ROOT", - "EVAL_CI_POLICY_ROOT", - "BENCH_CI_POLICY_ROOT", - ]) { - assert.match( - workflow, - new RegExp( - `${rootVariable}: \\$\\{\\{ github\\.workspace \\}\\}/candidate`, - "u", - ), - ); - } -}); - -test("trusted gate scans secrets, dependencies, and CodeQL without candidate code", () => { - assert.match( - workflow, - /run: bash control\/\.github\/scripts\/install-gitleaks\.sh/u, - ); - assert.match(workflow, /gitleaks git/u); - assert.match(workflow, /git -C candidate cat-file blob/u); - assert.match( - workflow, - /actions\/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294/u, - ); - assert.match(workflow, /build-mode: none/u); - assert.match(workflow, /security-events: write/u); - assert.match(workflow, /needs: authorize/gu); - assert.doesNotMatch(workflow, /^ pull_request_target:/mu); -}); - -test("trusted gate uses a strict scan when the candidate removes the base ignore file", () => { - assert.match( - workflow, - /if test -e trusted-target\/\.gitleaksignore; then\n\s+if test -e candidate\/\.gitleaksignore; then\n\s+cmp trusted-target\/\.gitleaksignore candidate\/\.gitleaksignore\n\s+ignore_path="\$GITHUB_WORKSPACE\/trusted-target\/\.gitleaksignore"\n\s+fi\n\s+history_ignore_path="\$GITHUB_WORKSPACE\/trusted-target\/\.gitleaksignore"\n\s+else\n\s+test ! -e candidate\/\.gitleaksignore/u, - ); - assert.match(workflow, /ignore_path=\/dev\/null/u); - assert.match(workflow, /history_ignore_path="\$GITHUB_WORKSPACE\/trusted-target\/\.gitleaksignore"/u); - assert.match( - workflow, - /gitleaks git --config "\$GITLEAKS_TRUSTED_CONFIG" \\\n\s+--gitleaks-ignore-path "\$history_ignore_path"/u, - ); -}); - -test("trusted CodeQL fails closed on local SARIF findings", () => { - const fixture = mkdtempSync(join(tmpdir(), "coffee-codeql-sarif-")); - try { - const sarifPath = join(fixture, "javascript.sarif"); - writeFileSync( - sarifPath, - `${JSON.stringify({ - version: "2.1.0", - runs: [{ tool: { driver: { name: "CodeQL" } }, results: [] }], - })}\n`, - ); - assert.deepEqual(checkCodeqlSarif(fixture), { files: 1, results: 0 }); - - writeFileSync( - sarifPath, - `${JSON.stringify({ - version: "2.1.0", - runs: [ - { - tool: { driver: { name: "CodeQL" } }, - results: [{ ruleId: "js/example" }], - }, - ], - })}\n`, - ); - assert.throws(() => checkCodeqlSarif(fixture), /CodeQL findings/u); - } finally { - rmSync(fixture, { force: true, recursive: true }); - } -}); - -test("trusted CodeQL requires regular bounded SARIF output", () => { - const fixture = mkdtempSync(join(tmpdir(), "coffee-codeql-bounds-")); - try { - assert.throws(() => checkCodeqlSarif(fixture), /SARIF file/u); - const outside = join(fixture, "outside.json"); - writeFileSync(outside, '{"version":"2.1.0","runs":[]}\n'); - symlinkSync(outside, join(fixture, "javascript.sarif")); - assert.throws(() => checkCodeqlSarif(fixture), /regular file/u); - } finally { - rmSync(fixture, { force: true, recursive: true }); - } -}); - -test("trusted CodeQL checks generated SARIF before aggregate success", () => { - const codeqlJob = workflow.slice( - workflow.indexOf(" codeql:"), - workflow.indexOf(" quality:"), - ); - const controlCheckout = codeqlJob.indexOf( - "repository: openboa-ai/.github", - ); - const candidateCheckout = codeqlJob.indexOf( - "repository: ${{ github.event.pull_request.head.repo.full_name }}", - ); - const analyze = codeqlJob.indexOf("id: codeql-analyze"); - const sarifCheck = codeqlJob.indexOf( - "node control/.github/scripts/check-codeql-sarif.mjs", - ); - assert.ok(controlCheckout > 0); - assert.ok(candidateCheckout > controlCheckout); - assert.ok(analyze > 0); - assert.ok(sarifCheck > analyze); - assert.match(codeqlJob, /ref: \$\{\{ inputs\.control_sha \}\}/u); - assert.match(codeqlJob, /path: control/u); - assert.match(codeqlJob, /path: candidate/u); - assert.match(codeqlJob, /source-root: candidate/u); - assert.match( - codeqlJob, - /checkout_path: \$\{\{ github\.workspace \}\}\/candidate/u, - ); - assert.match(codeqlJob, /output: \$\{\{ runner\.temp \}\}\/codeql-sarif/u); - assert.match(codeqlJob, /steps\.codeql-analyze\.outputs\.sarif-output/u); -}); - -test("exact policy failures and protected changes cannot bypass sensitive review", () => { - assert.match(workflow, /id: exact-policy\n\s+continue-on-error: true/u); - assert.match(workflow, /id: classify/u); - assert.match(workflow, /EXACT_POLICY_OUTCOME: \$\{\{ steps\.exact-policy\.outcome \}\}/u); - assert.match(workflow, /environment: coffee-security/u); - assert.match(workflow, /needs\.authorize\.outputs\.sensitive == 'true'/u); - assert.match(workflow, /SENSITIVE_REVIEW_RESULT/u); - assert.match(workflow, /test "\$SENSITIVE_REVIEW_RESULT" = success/u); - assert.match(workflow, /test "\$SENSITIVE_REVIEW_RESULT" = skipped/u); - const classifyStep = workflow.slice( - workflow.indexOf(" - name: Classify policy evolution"), - workflow.indexOf("\n\n sensitive-review:"), - ); - assert.match(classifyStep, /ACTOR: \$\{\{ github\.actor \}\}/u); - assert.match(classifyStep, /BASE_REPOSITORY: \$\{\{ github\.repository \}\}/u); - assert.match( - classifyStep, - /HEAD_REPOSITORY: \$\{\{ github\.event\.pull_request\.head\.repo\.full_name \}\}/u, - ); - assert.match( - classifyStep, - /PR_AUTHOR: \$\{\{ github\.event\.pull_request\.user\.login \}\}/u, - ); -}); - -test("trusted quality runs after authorization with fixed npm authority", () => { - const classifier = workflow.indexOf("Classify policy evolution and protected path changes"); - const quality = workflow.indexOf("Trusted deterministic quality"); - const qualityJob = workflow.slice( - workflow.indexOf(" quality:"), - workflow.indexOf(" eval-harbor:"), - ); - const gitleaksInstall = qualityJob.indexOf( - "bash control/.github/scripts/install-gitleaks.sh", - ); - const candidateQuality = qualityJob.indexOf( - "control/.github/scripts/run-candidate-quality.sh", - ); - assert.ok(classifier > 0); - assert.ok(quality > classifier); - assert.ok(gitleaksInstall > 0); - assert.ok(candidateQuality > gitleaksInstall); - assert.match(workflow, /needs\.sensitive-review\.result == 'success'/u); - assert.match(qualityRunner, /^set -euo pipefail$/mu); - assert.match(qualityRunner, /env -i/u); - assert.match(qualityRunner, /npm_userconfig="\$candidate_tmp\/npm-userconfig"/u); - assert.match(qualityRunner, /npm_globalconfig="\$candidate_tmp\/npm-globalconfig"/u); - assert.match(qualityRunner, /NPM_CONFIG_USERCONFIG=\$npm_userconfig/u); - assert.match(qualityRunner, /NPM_CONFIG_GLOBALCONFIG=\$npm_globalconfig/u); - assert.match(qualityRunner, /: > "\$npm_userconfig"/u); - assert.match(qualityRunner, /: > "\$npm_globalconfig"/u); - assert.match(qualityRunner, /NPM_CONFIG_REPLACE_REGISTRY_HOST=never/u); - assert.doesNotMatch(qualityRunner, /GITHUB_TOKEN|GH_TOKEN|ACTIONS_ID_TOKEN/u); - assert.match(qualityRunner, /run_clean npm ci --ignore-scripts --no-bin-links\n/u); - assert.match( - qualityRunner, - /run_clean npm ci --ignore-scripts --no-bin-links --prefix \.github\/policy-parser/u, - ); - assert.match(qualityRunner, /zero_base_layout\(\)/u); - assert.match(qualityRunner, /run_clean node \.github\/ci-policy\.mjs/u); - assert.match( - qualityRunner, - /candidate is neither a legacy Coffee repository nor a recognized zero-base layout/u, - ); - - const commandsByRepository = new Map([ - [ - "openboa-ai/coffee-chat", - [ - "run_clean node node_modules/prettier/bin/prettier.cjs --check .", - "run_clean node node_modules/typescript/bin/tsc --noEmit", - "run_clean node --test tests/*.test.mjs", - "run_clean node scripts/verify-readme-assets.mjs", - "run_clean node scripts/build-package.mjs", - "run_clean node scripts/package-smoke.mjs", - ], - ], - [ - "openboa-ai/coffee-chat-roastery", - [ - "run_clean node node_modules/prettier/bin/prettier.cjs --check .", - "run_clean node node_modules/typescript/bin/tsc --noEmit", - "run_clean node scripts/build.mjs", - "run_clean git diff --exit-code -- dist", - "run_clean node scripts/check-repository-state.mjs --root .", - "run_clean node --test tests/*.test.mjs", - "run_clean node scripts/check-package.mjs", - ], - ], - [ - "openboa-ai/coffee-chat-eval", - [ - "run_clean node node_modules/prettier/bin/prettier.cjs --check .", - "run_clean node node_modules/typescript/bin/tsc --noEmit", - "run_clean npm test", - "run_clean npm run dry-run", - "run_clean npm run smoke", - "run_clean npm run ci:policy", - ], - ], - [ - "openboa-ai/coffee-chat-bench", - [ - 'run_clean node node_modules/prettier/bin/prettier.cjs --check AGENTS.md README.md DATA-CARD.md PREREGISTRATION.md OVERLAP-REPORT.json package.json package-lock.json tsconfig.json prettier.config.mjs docs/*.md docs/validity/*.md harbor/*.md qualification/*.md qualification/*.json "bank/**/*.json" harbor/*.ts schemas/*.json scripts/*.mjs src/*.ts tests/*.mjs tests/*.ts', - "run_clean node scripts/check-inactive-boundary.mjs --root .", - "run_clean node node_modules/typescript/bin/tsc --noEmit", - "run_clean node --experimental-strip-types --test tests/*.test.mjs tests/*.test.ts", - ], - ], - ]); - - for (const [repository, commands] of commandsByRepository) { - const repositoryCase = qualityRunner.lastIndexOf('case "$repository" in'); - const start = qualityRunner.indexOf(` ${repository})`, repositoryCase); - const end = qualityRunner.indexOf(" ;;", start); - assert.ok(start > 0, `${repository}: command branch`); - assert.ok(end > start, `${repository}: command branch end`); - const branch = qualityRunner.slice(start, end); - const commandLines = branch - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.startsWith("run_clean ")); - assert.deepEqual(commandLines, commands, repository); - } -}); - -test("Eval Harbor calibration uses a fresh runner before any candidate program", () => { - assert.doesNotMatch(qualityRunner, /harbor-requirements|canary:calibrate|benchmark:calibrate/u); - assert.match(workflow, /eval-harbor:/u); - assert.match(workflow, /github\.repository == 'openboa-ai\/coffee-chat-eval'/u); - assert.match(workflow, /Trusted Eval Harbor calibration/u); - const harborJob = workflow.slice( - workflow.indexOf(" eval-harbor:"), - workflow.lastIndexOf(" required:"), - ); - assert.match(harborJob, /control\/\.github\/scripts\/run-eval-harbor\.sh/u); - assert.doesNotMatch(harborJob, /npm run|npm test|src\/cli\.ts|src\/pcda-cli\.ts/u); - assert.match(evalHarborRunner, /--require-hashes/u); - assert.match(evalHarborRunner, /--harbor-command/u); - assert.match(evalHarborRunner, /node --experimental-strip-types src\/cli\.ts oracle-control/u); - assert.match(evalHarborRunner, /candidate_root.*iterations\/README\.md/u); - assert.match(evalHarborRunner, /Eval Harbor calibration not applicable/u); - assert.match(workflow, /EVAL_HARBOR_RESULT/u); -}); diff --git a/scripts/test-isolation.mjs b/scripts/test-isolation.mjs new file mode 100644 index 0000000..3ee7ac8 --- /dev/null +++ b/scripts/test-isolation.mjs @@ -0,0 +1,50 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +const launcher = resolve(import.meta.dirname, "../.github/scripts/run-repository-verify.mjs"); +for (const shouldPass of [true, false]) { + const root = mkdtempSync(join(tmpdir(), "coffee-ci-isolation-")); + try { + writeFileSync(join(root, "package.json"), JSON.stringify({ name: "isolation-probe", version: "0.0.0", private: true, scripts: { verify: "node verify.mjs" } })); + writeFileSync(join(root, "package-lock.json"), JSON.stringify({ name: "isolation-probe", version: "0.0.0", lockfileVersion: 3, requires: true, packages: { "": { name: "isolation-probe", version: "0.0.0" } } })); + const hostOnly = join(root, "host-only-canary"); + const probe = [ + 'import assert from "node:assert/strict";', + 'import { existsSync, readFileSync, writeFileSync } from "node:fs";', + 'import { networkInterfaces } from "node:os";', + 'assert.notEqual(process.getuid(), 0);', + 'for (const key of ["COFFEE_CI_CANARY", "GITHUB_TOKEN", "GITHUB_OUTPUT", "GITHUB_ENV", "ACTIONS_RUNTIME_TOKEN", "DOCKER_HOST"]) assert.equal(process.env[key], undefined, key);', + 'assert.equal(existsSync("/var/run/docker.sock"), false);', + 'assert.equal(existsSync(' + JSON.stringify(hostOnly) + '), false);', + 'assert.deepEqual(Object.keys(networkInterfaces()), ["lo"]);', + 'const status = readFileSync("/proc/self/status", "utf8");', + 'assert.match(status, /CapEff:\\s+0+\\n/);', + 'assert.match(status, /NoNewPrivs:\\s+1\\n/);', + 'assert.throws(() => writeFileSync("/root-write-probe", "denied"));', + 'await assert.rejects(fetch("http://192.0.2.1", { signal: AbortSignal.timeout(1000) }));', + 'console.log("isolation assertions passed");', + 'console.log("::notice::candidate log must not become a workflow command");', + shouldPass ? 'process.exit(0);' : 'process.exit(23);', + ].join("\n"); + writeFileSync(join(root, "verify.mjs"), probe); + execFileSync("git", ["init", "-q", root]); + execFileSync("git", ["-C", root, "add", "--all"]); + writeFileSync(hostOnly, "non-secret local canary"); + const result = spawnSync(process.execPath, [launcher, root], { + env: { ...process.env, COFFEE_CI_CANARY: "non-secret-inheritance-probe", GITHUB_ACTIONS: "true" }, + encoding: "utf8", timeout: 120000, + }); + const output = (result.stdout ?? "") + (result.stderr ?? ""); + assert.match(output, /isolation assertions passed/, output); + assert.equal(result.status === 0, shouldPass, output); + const volume = output.match(/^coffee-verify-[0-9a-f-]+$/m)?.[0]; + assert.ok(volume, output); + assert.notEqual(spawnSync("docker", ["volume", "inspect", volume]).status, 0, "temporary volume leaked"); + assert.notEqual(spawnSync("docker", ["inspect", volume + "-job"]).status, 0, "temporary container leaked"); + assert.match(output, /::stop-commands::[0-9a-f-]+/); + console.log(shouldPass ? "PASS: real non-root, network, credential, host and log isolation" : "PASS: failed verification propagates and isolated state is removed"); + } finally { rmSync(root, { recursive: true, force: true }); } +}