From fbc21cc3b57adc773a9d2d896afe3657774f8f6f Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:17:26 +1200 Subject: [PATCH 01/47] Trigger isolated V111 fast screen --- competitions/trace_the_ace/V111_RUN_NOW.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 competitions/trace_the_ace/V111_RUN_NOW.md diff --git a/competitions/trace_the_ace/V111_RUN_NOW.md b/competitions/trace_the_ace/V111_RUN_NOW.md new file mode 100644 index 00000000..dd713525 --- /dev/null +++ b/competitions/trace_the_ace/V111_RUN_NOW.md @@ -0,0 +1,3 @@ +# V111 isolated runner trigger + +Run the frozen minutes-scale residual screen independently. From 717721d381c28d29174d2cde1518c951bf5c89af Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:17:44 +1200 Subject: [PATCH 02/47] Synchronize isolated V111 runner --- competitions/trace_the_ace/V111_RUN_NOW.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/competitions/trace_the_ace/V111_RUN_NOW.md b/competitions/trace_the_ace/V111_RUN_NOW.md index dd713525..e101740a 100644 --- a/competitions/trace_the_ace/V111_RUN_NOW.md +++ b/competitions/trace_the_ace/V111_RUN_NOW.md @@ -1,3 +1,5 @@ # V111 isolated runner trigger Run the frozen minutes-scale residual screen independently. + +Trigger: 2 From 438d2511502f25ce0dc559187516cc09aff01f47 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:51:57 +1200 Subject: [PATCH 03/47] Add V112 fast raw-observable collision screen --- .../v112_fast_raw_observable_screen.py | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 competitions/trace_the_ace/v112_fast_raw_observable_screen.py diff --git a/competitions/trace_the_ace/v112_fast_raw_observable_screen.py b/competitions/trace_the_ace/v112_fast_raw_observable_screen.py new file mode 100644 index 00000000..54a3a9c4 --- /dev/null +++ b/competitions/trace_the_ace/v112_fast_raw_observable_screen.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""V112 FAST RAW-OBSERVABLE SCREEN. +Frozen screen only: one shared transcript/V97 pass, deterministic 2500-row discovery sample, +objective-grouped OOF. Tests whether information discarded by V71 exists in raw transcripts. +Escalate only >= .003 V97 gain; >= .010 is phase-change candidate. +""" +from __future__ import annotations +import argparse, json, re +from pathlib import Path +import numpy as np +from scipy.sparse import hstack, csr_matrix +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GroupKFold +from sklearn.preprocessing import StandardScaler +from v111_fast_residual_screen import h +from v110_residual_collider_state_discovery import hb, ll, logit, p97_predict +from v71_mastery_events import load_transcript, normalize_roles +from v75_canonical_trajectory import load_training, SEED +from v81_target_segment_phase import choose_target_segment +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control, build_control +EPS=1e-5 +MATH=re.compile(r"\d|[+\-*/=×÷<>%]|\b(?:half|quarter|third|decimal|fraction|percent|times|divide|multiply)\b",re.I) + +def texts(df,obj): + d=normalize_roles(df).reset_index(drop=True); role=d.role_repaired.astype(str).tolist(); c=d.content.fillna('').astype(str).tolist() + student=' '.join(x for r,x in zip(role,c) if r=='student'); tutor=' '.join(x for r,x in zip(role,c) if r=='tutor'); full=' '.join(f'[{r}] {x}' for r,x in zip(role,c)) + seg,_=choose_target_segment(df,obj); sr=normalize_roles(seg).reset_index(drop=True); local=' '.join(f'[{r}] {x}' for r,x in zip(sr.role_repaired.astype(str),sr.content.fillna('').astype(str))) + last=' '.join(f'[{r}] {x}' for r,x in list(zip(role,c))[-8:]) + return student,tutor,full,local,last + +def numvec(df): + d=normalize_roles(df).reset_index(drop=True); role=d.role_repaired.astype(str).to_numpy(); c=d.content.fillna('').astype(str).tolist(); n=max(1,len(c)); stu=[x for r,x in zip(role,c) if r=='student']; tut=[x for r,x in zip(role,c) if r=='tutor'] + lens=np.array([len(x) for x in stu],float) if stu else np.zeros(1); words=np.array([len(x.split()) for x in stu],float) if stu else np.zeros(1) + return np.array([len(c),len(stu),len(tut),lens.mean(),lens.max(),words.mean(),np.mean([bool(MATH.search(x)) for x in stu]) if stu else 0,np.mean(d.role_changed),len(stu)/n,len(tut)/n],float) + +def sparse_oof(P,X,y,g): + q=np.zeros(len(y)); folds=GroupKFold(min(4,len(np.unique(g)))) + for tr,va in folds.split(np.zeros(len(y)),y,g): + m=LogisticRegression(C=.08,max_iter=180,solver='liblinear',random_state=SEED).fit(hstack([csr_matrix(logit(P[tr])[:,None]),X[tr]],format='csr'),y[tr]); q[va]=m.predict_proba(hstack([csr_matrix(logit(P[va])[:,None]),X[va]],format='csr'))[:,1] + return np.clip(q,EPS,1-EPS) +def dense_oof(P,X,y,g): + q=np.zeros(len(y)); folds=GroupKFold(min(4,len(np.unique(g)))) + for tr,va in folds.split(X,y,g): + sc=StandardScaler().fit(X[tr]); m=LogisticRegression(C=.15,max_iter=180,solver='liblinear',random_state=SEED).fit(np.c_[logit(P[tr]),sc.transform(X[tr])],y[tr]); q[va]=m.predict_proba(np.c_[logit(P[va]),sc.transform(X[va])])[:,1] + return np.clip(q,EPS,1-EPS) +def main(a): + f=load_training(a.features,a.labels).reset_index(drop=True); print('features columns',list(f.columns),flush=True) + obj0=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy(); cand=np.where(np.array([hb(x,5)!=0 for x in obj0]))[0]; ix=np.array(sorted(cand,key=lambda i:h(f.response_id.iloc[i]))[:a.rows]); f=f.iloc[ix].reset_index(drop=True) + y=f.target.to_numpy(int); obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy(); support=f.learning_objective.astype(str).to_numpy(); sess=f.session_id.astype(str).to_numpy(); cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)} + rt=[];rz=[]; T={k:[] for k in ['STUDENT','TUTOR','FULL','LOCAL','LAST8']}; N=[] + for _,r in f.iterrows(): + d=cache[str(r.session_id)]; t,z=segmented_control(d,str(r.learning_objective),'related');rt.append(t);rz.append(z); vals=texts(d,str(r.learning_objective)); + for k,v in zip(T,vals): T[k].append(v) + N.append(numvec(d)) + X75=build_v75(f,cache);Xr=build_control(rt,rz);P=np.zeros(len(f));splits=list(GroupKFold(min(4,len(np.unique(obj)))).split(np.zeros(len(f)),y,obj)) + for tr,va in splits:P[va],_=p97_predict(X75,Xr,y,tr,va,support) + base=ll(y,P);out={'rows':len(f),'objectives':len(np.unique(obj)),'v97':base,'tests':{}} + hv=HashingVectorizer(n_features=2**16,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + for k,txt in T.items(): + X=hv.transform(txt);q=sparse_oof(P,X,y,obj);out['tests'][k]={'ll':ll(y,q),'gain':base-ll(y,q)} + Xn=np.vstack(N);q=dense_oof(P,Xn,y,obj);out['tests']['STRUCTURE']={'ll':ll(y,q),'gain':base-ll(y,q)} + # Combined raw views: cheap union, tests whether complementary raw observables jointly matter. + X=hstack([hv.transform(T['STUDENT']),hv.transform(T['TUTOR']),hv.transform(T['LOCAL']),csr_matrix(StandardScaler().fit_transform(Xn))],format='csr');q=sparse_oof(P,X,y,obj);out['tests']['COMBINED']={'ll':ll(y,q),'gain':base-ll(y,q)} + # Tight collision geometry from the same frozen predictions. + ds=[] + for o in np.unique(obj): + z=np.where(obj==o)[0];a0=z[y[z]==0];a1=z[y[z]==1] + if len(a0) and len(a1): + p1=np.sort(P[a1]);ds.extend(float(np.min(np.abs(p1-P[i]))) for i in a0) + out['tight_collisions']={'basis':len(ds),'median_dp':float(np.median(ds)) if ds else None,'p10_dp':float(np.quantile(ds,.1)) if ds else None} + gains={k:v['gain'] for k,v in out['tests'].items()};win=max(gains,key=gains.get);g=gains[win];out['decision']={'winner':win,'winner_gain':g,'verdict':'PHASE_CHANGE_CANDIDATE' if g>=.01 else 'ESCALATE_RAW_OBSERVABLE' if g>=.003 else 'RAW_TRANSCRIPT_NOT_SEPARATING','rule':'Escalate >=.003; phase-change candidate >=.010; otherwise audit non-text metadata/test-regime/applicability.'} + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v112_fast_raw_observable_screen.json');main(p.parse_args()) From 467f675d2e425d54977ba29765f140bfe06bac02 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:52:07 +1200 Subject: [PATCH 04/47] Run V112 fast raw-observable screen --- .../trace-ace-v112-fast-raw-observable.yml | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/trace-ace-v112-fast-raw-observable.yml diff --git a/.github/workflows/trace-ace-v112-fast-raw-observable.yml b/.github/workflows/trace-ace-v112-fast-raw-observable.yml new file mode 100644 index 00000000..f68bef01 --- /dev/null +++ b/.github/workflows/trace-ace-v112-fast-raw-observable.yml @@ -0,0 +1,46 @@ +name: Trace Ace V112 Fast Raw Observable Screen +on: + pull_request: + branches: [main] + paths: + - 'competitions/trace_the_ace/v112_fast_raw_observable_screen.py' + - '.github/workflows/trace-ace-v112-fast-raw-observable.yml' + workflow_dispatch: +jobs: + screen: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Run V112 shared fast pass + run: | + cd competitions/trace_the_ace + python v112_fast_raw_observable_screen.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v112_fast_raw_observable_screen.json + - name: Show decision + run: cat v112_fast_raw_observable_screen.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v112-fast-raw-observable-screen + path: v112_fast_raw_observable_screen.json + retention-days: 14 From 2d1125f2cf807de3e57042869d6ae1e7cbb829ec Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:52:57 +1200 Subject: [PATCH 05/47] Repurpose fast runner for V112 raw-observable screen --- .../trace-ace-v111-fast-residual-screen.yml | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/workflows/trace-ace-v111-fast-residual-screen.yml b/.github/workflows/trace-ace-v111-fast-residual-screen.yml index a953c67b..e08e5426 100644 --- a/.github/workflows/trace-ace-v111-fast-residual-screen.yml +++ b/.github/workflows/trace-ace-v111-fast-residual-screen.yml @@ -1,15 +1,9 @@ -name: Trace Ace V111 Fast Residual Screen +name: Trace Ace V112 Fast Raw Observable Screen on: - push: - branches: [agent/trace-ace-mastery-events] - paths: - - 'competitions/trace_the_ace/v111_fast_residual_screen.py' - - '.github/workflows/trace-ace-v111-fast-residual-screen.yml' pull_request: branches: [agent/trace-ace-mastery-events] paths: - - 'competitions/trace_the_ace/V111_RUN_NOW.md' - - 'competitions/trace_the_ace/v111_fast_residual_screen.py' + - 'competitions/trace_the_ace/v112_fast_raw_observable_screen.py' - '.github/workflows/trace-ace-v111-fast-residual-screen.yml' workflow_dispatch: jobs: @@ -37,13 +31,13 @@ jobs: echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" echo "LABELS=$LABELS" >> "$GITHUB_ENV" echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" - - name: Run all fast screens + - name: Run V112 shared fast pass run: | cd competitions/trace_the_ace - python v111_fast_residual_screen.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v111_fast_residual_screen.json - - run: cat v111_fast_residual_screen.json + python v112_fast_raw_observable_screen.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v112_fast_raw_observable_screen.json + - run: cat v112_fast_raw_observable_screen.json - uses: actions/upload-artifact@v4 with: - name: trace-ace-v111-fast-residual-screen - path: v111_fast_residual_screen.json + name: trace-ace-v112-fast-raw-observable-screen + path: v112_fast_raw_observable_screen.json retention-days: 14 From 1e057ab78cf64566e55cb91604a5bc5240729a96 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:53:10 +1200 Subject: [PATCH 06/47] Route isolated fast runner to V112 screen --- .../v111_fast_residual_screen.py | 83 +------------------ 1 file changed, 4 insertions(+), 79 deletions(-) diff --git a/competitions/trace_the_ace/v111_fast_residual_screen.py b/competitions/trace_the_ace/v111_fast_residual_screen.py index b8a1b56e..50420a32 100644 --- a/competitions/trace_the_ace/v111_fast_residual_screen.py +++ b/competitions/trace_the_ace/v111_fast_residual_screen.py @@ -1,82 +1,7 @@ #!/usr/bin/env python3 -"""V111 FAST SCREEN — decision-changing residual tests in one shared pass. -Screen only; winners require untouched full verification before promotion. -Frozen before results: deterministic hash sample, V97 baseline, grouped OOF, no cross-test aggregates. -Primary question: which missing representation family can separate V97 residual collisions? -""" -from __future__ import annotations -import argparse, hashlib, json +"""Temporary isolated-runner shim: execute frozen V112 fast raw-observable screen.""" +from v112_fast_raw_observable_screen import main +import argparse from pathlib import Path -import numpy as np -from sklearn.linear_model import LogisticRegression -from sklearn.metrics import log_loss -from sklearn.model_selection import GroupKFold -from sklearn.preprocessing import StandardScaler -from v110_residual_collider_state_discovery import hb, ll, logit, p97_predict, state_vec -from v71_mastery_events import load_transcript -from v75_canonical_trajectory import load_training, SEED -from v81_target_segment_phase import choose_target_segment -from v85_evidence_state import build_v75, evidence_events -from v94_related_control import segmented_control, build_control - -EPS=1e-5 - -def h(x): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16) -def fit_oof(P,S,y,groups): - q=np.zeros(len(y)); k=min(4,len(np.unique(groups))) - for tr,va in GroupKFold(k).split(S,y,groups): - sc=StandardScaler().fit(S[tr]); A=sc.transform(S[tr]); B=sc.transform(S[va]) - m=LogisticRegression(C=.15,max_iter=250,solver='liblinear',random_state=SEED).fit(np.c_[logit(P[tr]),A],y[tr]) - q[va]=m.predict_proba(np.c_[logit(P[va]),B])[:,1] - return np.clip(q,EPS,1-EPS) -def evvec(E, mode): - if not E: return np.zeros(12) - pos=np.array([float(e['pos']) for e in E]); neg=np.array([float(e['neg']) for e in E]); ind=np.array([float(e['independent']) for e in E]); ass=np.array([float(e['assistance']) for e in E]); rel=np.array([float(e['rel']) for e in E]); n=len(E) - if mode=='CONTENT': - return np.array([n,pos.mean(),neg.mean(),ind.mean(),ass.mean(),rel.mean(),(pos*ind).mean(),(neg*rel).mean(),rel.max(),rel[-1],pos.sum()/max(1,n),neg.sum()/max(1,n)]) - if mode=='TERMINAL': - z=np.arange(n); lastp=np.max(np.where(pos>0,z,-1)); lastn=np.max(np.where(neg>0,z,-1)); lasti=np.max(np.where((pos*ind)>0,z,-1)); - return np.array([n,pos[-1],neg[-1],ind[-1],ass[-1],rel[-1],lastp/n,lastn/n,lasti/n,(lasti-lastn)/n,(lastp-lastn)/n,(pos[-min(3,n):]*ind[-min(3,n):]).mean()]) - if mode=='ORDER': - return state_vec(E,(.7,.5,.7,.75),False)[:12] - if mode=='ORDER_ABLATE': - return state_vec(E,(.7,.5,.7,.75),True)[:12] - raise ValueError(mode) - -def main(a): - f=load_training(a.features,a.labels).reset_index(drop=True) - print('features columns',list(f.columns),flush=True) - # deterministic balanced screen: discovery objectives only, <= N rows, preserve many objectives - obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() - cand=np.where(np.array([hb(x,5)!=0 for x in obj]))[0] - ix=np.array(sorted(cand,key=lambda i:h(f.response_id.iloc[i]))[:a.rows]); f=f.iloc[ix].reset_index(drop=True) - y=f.target.to_numpy(int); obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy(); support=f.learning_objective.astype(str).to_numpy(); sess=f.session_id.astype(str).to_numpy() - cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in np.unique(sess)} - rt=[]; rz=[]; EV=[] - for i,r in f.iterrows(): - t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related'); rt.append(t); rz.append(z) - seg,_=choose_target_segment(cache[str(r.session_id)],str(r.learning_objective)); EV.append(evidence_events(seg,str(r.learning_objective))) - X75=build_v75(f,cache); Xr=build_control(rt,rz) - P=np.zeros(len(f)); splits=list(GroupKFold(min(4,len(np.unique(obj)))).split(np.zeros(len(f)),y,obj)) - for tr,va in splits: P[va],_=p97_predict(X75,Xr,y,tr,va,support) - base=ll(y,P); out={'rows':len(f),'objectives':len(np.unique(obj)),'v97':base,'tests':{}} - mats={m:np.vstack([evvec(E,m) for E in EV]) for m in ['CONTENT','TERMINAL','ORDER','ORDER_ABLATE']} - # Highest-impact screen A: does event content carry missing information at all? - for m,S in mats.items(): - q=fit_oof(P,S,y,obj); out['tests'][m]={'ll':ll(y,q),'gain':base-ll(y,q)} - out['tests']['CHRONOLOGY_CAUSAL']={'gain':out['tests']['ORDER']['gain']-out['tests']['ORDER_ABLATE']['gain']} - # Screen B: objective relevance is the suspected bottleneck. Remove rel dimensions and compare. - S=mats['CONTENT'].copy(); Sr=S.copy(); Sr[:,5]=0; Sr[:,7]=0; Sr[:,8]=0; Sr[:,9]=0 - qr=fit_oof(P,Sr,y,obj); out['tests']['RELEVANCE_ABLATION']={'ll':ll(y,qr),'gain':base-ll(y,qr),'relevance_value':out['tests']['CONTENT']['gain']-(base-ll(y,qr))} - # Screen C: tight residual geometry. Opposite-label nearest neighbor distance in P within objective. - ds=[] - for o in np.unique(obj): - z=np.where(obj==o)[0]; a0=z[y[z]==0]; a1=z[y[z]==1] - if len(a0) and len(a1): - p1=np.sort(P[a1]); ds.extend([float(np.min(np.abs(p1-P[i]))) for i in a0]) - out['residual_geometry']={'opposite_nn_p_median':float(np.median(ds)) if ds else None,'opposite_nn_p_p10':float(np.quantile(ds,.1)) if ds else None,'pairs_basis':len(ds)} - gains={k:v.get('gain',-9) for k,v in out['tests'].items() if isinstance(v,dict)}; winner=max(gains,key=gains.get) - out['decision']={'winner':winner,'winner_gain':gains[winner],'rule':'Escalate only a family with >=.003 grouped-OOF screen gain; chronology requires >=.001 ORDER-over-ABLATE. Otherwise change observable/extraction, not grammar.'} - Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2),flush=True) if __name__=='__main__': - p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--rows',type=int,default=2500); p.add_argument('--out',default='v111_fast_residual_screen.json'); main(p.parse_args()) + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v111_fast_residual_screen.json');main(p.parse_args()) From dd24094da2d81fbc6384731505a11d5fd7e93bdc Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:59:44 +1200 Subject: [PATCH 07/47] Fix V112 circular import with local deterministic hash --- .../trace_the_ace/v112_fast_raw_observable_screen.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/competitions/trace_the_ace/v112_fast_raw_observable_screen.py b/competitions/trace_the_ace/v112_fast_raw_observable_screen.py index 54a3a9c4..e3494bb6 100644 --- a/competitions/trace_the_ace/v112_fast_raw_observable_screen.py +++ b/competitions/trace_the_ace/v112_fast_raw_observable_screen.py @@ -5,7 +5,7 @@ Escalate only >= .003 V97 gain; >= .010 is phase-change candidate. """ from __future__ import annotations -import argparse, json, re +import argparse, json, re, hashlib from pathlib import Path import numpy as np from scipy.sparse import hstack, csr_matrix @@ -13,7 +13,6 @@ from sklearn.linear_model import LogisticRegression from sklearn.model_selection import GroupKFold from sklearn.preprocessing import StandardScaler -from v111_fast_residual_screen import h from v110_residual_collider_state_discovery import hb, ll, logit, p97_predict from v71_mastery_events import load_transcript, normalize_roles from v75_canonical_trajectory import load_training, SEED @@ -23,6 +22,9 @@ EPS=1e-5 MATH=re.compile(r"\d|[+\-*/=×÷<>%]|\b(?:half|quarter|third|decimal|fraction|percent|times|divide|multiply)\b",re.I) +def h(x): + return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16) + def texts(df,obj): d=normalize_roles(df).reset_index(drop=True); role=d.role_repaired.astype(str).tolist(); c=d.content.fillna('').astype(str).tolist() student=' '.join(x for r,x in zip(role,c) if r=='student'); tutor=' '.join(x for r,x in zip(role,c) if r=='tutor'); full=' '.join(f'[{r}] {x}' for r,x in zip(role,c)) @@ -61,9 +63,7 @@ def main(a): for k,txt in T.items(): X=hv.transform(txt);q=sparse_oof(P,X,y,obj);out['tests'][k]={'ll':ll(y,q),'gain':base-ll(y,q)} Xn=np.vstack(N);q=dense_oof(P,Xn,y,obj);out['tests']['STRUCTURE']={'ll':ll(y,q),'gain':base-ll(y,q)} - # Combined raw views: cheap union, tests whether complementary raw observables jointly matter. X=hstack([hv.transform(T['STUDENT']),hv.transform(T['TUTOR']),hv.transform(T['LOCAL']),csr_matrix(StandardScaler().fit_transform(Xn))],format='csr');q=sparse_oof(P,X,y,obj);out['tests']['COMBINED']={'ll':ll(y,q),'gain':base-ll(y,q)} - # Tight collision geometry from the same frozen predictions. ds=[] for o in np.unique(obj): z=np.where(obj==o)[0];a0=z[y[z]==0];a1=z[y[z]==1] From 32fa84005da0800a2bad18c028063c81fd0a83db Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 18:22:21 +1200 Subject: [PATCH 08/47] Add frozen V113 applicability and regime fast pass --- .../v113_applicability_regime_fast.py | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 competitions/trace_the_ace/v113_applicability_regime_fast.py diff --git a/competitions/trace_the_ace/v113_applicability_regime_fast.py b/competitions/trace_the_ace/v113_applicability_regime_fast.py new file mode 100644 index 00000000..e4fc1762 --- /dev/null +++ b/competitions/trace_the_ace/v113_applicability_regime_fast.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""V113 frozen fast pass: applicability + regime fingerprint. +Primary: can sample-local non-text/support topology explain when RELATED beats V75? +Frozen: deterministic 2500 rows, objective-grouped 4-fold OOF, fixed HistGB gate. +Thresholds: phase >=.010 (and all folds nonnegative) or >=.008 + >=15% oracle recovery; +escalate >=.003 with >=3/4 positive folds and placebo <25% real gain; structured .001-.003 only +if a family ablation removes >=.001; otherwise suppress metadata router. +No cross-test aggregates enter prediction. +""" +from __future__ import annotations +import argparse,json,hashlib +from pathlib import Path +import numpy as np +from sklearn.ensemble import HistGradientBoostingClassifier +from sklearn.model_selection import GroupKFold +from sklearn.metrics import log_loss,roc_auc_score +from v71_mastery_events import load_transcript,normalize_roles +from v75_canonical_trajectory import load_training,SEED +from v81_target_segment_phase import choose_target_segment +from v85_evidence_state import build_v75 +from v93_shift_robust_validation import obj_family +from v94_related_control import segmented_control,build_control +from v110_residual_collider_state_discovery import hb,p97_predict,ll +EPS=1e-5 + +def H(x): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16) +def lossrow(y,p): return -(y*np.log(np.clip(p,EPS,1))+(1-y)*np.log(np.clip(1-p,EPS,1))) +def counts(train_vals, eval_vals): + u,n=np.unique(train_vals,return_counts=True);d=dict(zip(u,n));return np.array([d.get(x,0) for x in eval_vals],float) +def transcript_meta(d,obj): + z=normalize_roles(d).reset_index(drop=True); roles=z.role_repaired.astype(str).to_numpy(); n=max(1,len(z)); + seg,_=choose_target_segment(d,obj); ns=len(seg) + stu=float(np.sum(roles=='student')); tut=float(np.sum(roles=='tutor')) + # timestamp duration only when parseable; otherwise 0. + dur=0. + for c in ['timestamp','time','created_at']: + if c in z.columns: + try: + t=np.array(np.asarray(__import__('pandas').to_datetime(z[c],errors='coerce').astype('int64')),float); good=t>0 + if good.sum()>1: dur=float((t[good].max()-t[good].min())/1e9) + except Exception: pass + break + start=0. + if ns and len(seg): + try: start=float(seg.index.min())/n + except Exception: start=0. + return np.array([len(z),stu,tut,stu/n,tut/n,ns,ns/n,start,np.log1p(max(dur,0.))],float) +def geometry(p0,pr): + d=pr-p0 + return np.c_[p0,pr,d,np.abs(d),np.abs(p0-.5),np.abs(pr-.5),np.minimum(p0,pr),np.maximum(p0,pr)] +def fit_gate(X,ywin,sw,tr,va): + m=HistGradientBoostingClassifier(max_depth=2,max_iter=70,learning_rate=.05,min_samples_leaf=80,l2_regularization=2.,random_state=SEED) + m.fit(X[tr],ywin[tr],sample_weight=sw[tr]); return m.predict_proba(X[va])[:,1] +def route(p0,pr,g): + # fixed conservative interpolation; no sweep + w=np.clip(.65*g,0,.65); return np.clip((1-w)*p0+w*pr,EPS,1-EPS) +def run(a): + f0=load_training(a.features,a.labels).reset_index(drop=True); print('features columns',list(f0.columns),flush=True) + objall=(f0.learning_objective_id if 'learning_objective_id' in f0 else f0.learning_objective).astype(str).to_numpy(); cand=np.where(np.array([hb(x,5)!=0 for x in objall]))[0] + ix=np.array(sorted(cand,key=lambda i:H(f0.response_id.iloc[i]))[:a.rows]); f=f0.iloc[ix].reset_index(drop=True) + y=f.target.to_numpy(int); obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy(); key=f.learning_objective.astype(str).to_numpy(); fam=np.array([obj_family(x) for x in key]); sess=f.session_id.astype(str).to_numpy() + cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)} + rt=[];rz=[];meta=[] + for _,r in f.iterrows(): + d=cache[str(r.session_id)];t,z=segmented_control(d,str(r.learning_objective),'related');rt.append(t);rz.append(z);meta.append(transcript_meta(d,str(r.learning_objective))) + X75=build_v75(f,cache);Xr=build_control(rt,rz); P0=np.zeros(len(f));PR=np.zeros(len(f)); fold=np.full(len(f),-1,int) + splits=list(GroupKFold(min(4,len(np.unique(obj)))).split(np.zeros(len(y)),y,obj)) + # experts once, true outer OOF + for k,(tr,va) in enumerate(splits): + P0[va],_=p97_predict(X75,Xr,y,tr,va,key); # returns V97, so fit experts explicitly below is unavailable + # recover endpoints with same base learner via helper's ingredients: fixed V97 endpoint reconstruction impossible from p97 alone. + # use imported logistic expert fitter locally. + from sklearn.linear_model import LogisticRegression + m0=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr]); mr=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr]) + P0[va]=np.clip(m0.predict_proba(X75[va])[:,1],EPS,1-EPS);PR[va]=np.clip(mr.predict_proba(Xr[va])[:,1],EPS,1-EPS);fold[va]=k + base=np.where(np.array([np.sum(key[np.setdiff1d(np.arange(len(y)),np.where(fold==fold[i])[0])]==key[i]) for i in range(len(y))])==0,.65*P0+.35*PR,P0) + base_ll=ll(y,base); oracle=np.where(lossrow(y,PR)0,fc>0,np.divide(ec,fc+1.)] + G=geometry(P0,PR); I=np.c_[np.log1p(np.array([H(x)%997 for x in f.response_id.astype(str)])),np.array([H(x)%31 for x in sess])] + D=np.abs(PR-P0)[:,None]; SX=np.c_[D*support[:,:3],(PR-P0)[:,None]*support[:,:3],D*(support[:,3:5])] + allx=np.c_[G,support,M,SX] + mats={'GEOMETRY':G,'SUPPORT':support,'SESSION':M,'SUPPORT_X_DISAGREEMENT':SX,'ALL_APPLICABILITY':allx,'ID_PLACEBO':I} + win=(lossrow(y,PR)0))} + real=[x for x in families if x!='ID_PLACEBO']; winner=max(real,key=lambda n:tests[n]['gain']); gain=tests[winner]['gain']; rec=(gain/gap if gap>0 else 0.); placebo=tests['ID_PLACEBO']['gain'] + # family ablation criterion: ALL minus best non-all as observable diagnostic + ablation=max([tests[x]['gain'] for x in ['GEOMETRY','SUPPORT','SESSION','SUPPORT_X_DISAGREEMENT']]) + all_gain=tests['ALL_APPLICABILITY']['gain']; removal=all_gain-ablation + phase=(gain>=.010 and min(tests[winner]['fold_gains'])>=0) or (gain>=.008 and rec>=.15) + escalate=(gain>=.003 and tests[winner]['positive_folds']>=3 and placebo < .25*gain) + structured=(.001<=gain<.003 and abs(removal)>=.001) + verdict='PHASE_CHANGE_APPLICABILITY' if phase else 'ESCALATE_APPLICABILITY' if escalate else 'STRUCTURED_HINT' if structured else 'SUPPRESS_METADATA_ROUTER' + out={'rows':len(y),'objectives':len(np.unique(obj)),'v97':base_ll,'row_endpoint_oracle':oracle_ll,'oracle_gap':gap,'tests':tests,'winner':winner,'winner_gain':gain,'oracle_gap_recovered_fraction':rec,'all_vs_best_family_delta':removal,'decision':verdict,'precommit':{'phase':'>=.010 and all folds nonnegative OR >=.008 and >=15% oracle recovery','escalate':'>=.003, >=3/4 folds positive, placebo <25% real gain','structured':'.001-.003 only if family ablation >=.001','otherwise':'suppress metadata router'}} + # Optional label-free real-test fingerprint when test_features exists. + out['regime_fingerprint']={'status':'UNAVAILABLE_NO_TEST_FEATURES'} + if a.test_features and a.test_features.exists(): + te=__import__('pandas').read_csv(a.test_features); trkey=f0.learning_objective.astype(str).to_numpy(); trfam=np.array([obj_family(x) for x in trkey]); tek=te.learning_objective.astype(str).to_numpy(); tef=np.array([obj_family(x) for x in tek]) + ec=counts(trkey,tek);fc=counts(trfam,tef);out['regime_fingerprint']={'status':'AVAILABLE','rows':len(te),'exact_seen_rate':float(np.mean(ec>0)),'family_seen_rate':float(np.mean(fc>0)),'median_exact_support':float(np.median(ec)),'median_family_support':float(np.median(fc))} + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--test-features',type=Path,default=None);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v113_applicability_regime_fast.json');run(p.parse_args()) From 5f5a629d26f21f50d6068b33354191992db0da9c Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 18:22:34 +1200 Subject: [PATCH 09/47] Add V113 fast applicability workflow --- ...ace-ace-v113-applicability-regime-fast.yml | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/trace-ace-v113-applicability-regime-fast.yml diff --git a/.github/workflows/trace-ace-v113-applicability-regime-fast.yml b/.github/workflows/trace-ace-v113-applicability-regime-fast.yml new file mode 100644 index 00000000..271cd53a --- /dev/null +++ b/.github/workflows/trace-ace-v113-applicability-regime-fast.yml @@ -0,0 +1,53 @@ +name: Trace Ace V113 Applicability Regime Fast +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v113_applicability_regime_fast.py' + - '.github/workflows/trace-ace-v113-applicability-regime-fast.yml' + workflow_dispatch: +jobs: + screen: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + TEST=$(find data/meta -type f \( -name 'test_features*.csv' -o -name 'test*.csv' \) -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TEST=$TEST" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Preflight import + run: | + cd competitions/trace_the_ace + python -m py_compile v113_applicability_regime_fast.py + - name: Run V113 shared fast pass + run: | + cd competitions/trace_the_ace + if [ -n "$TEST" ]; then EXTRA="--test-features ../../$TEST"; else EXTRA=""; fi + python v113_applicability_regime_fast.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 $EXTRA --out ../../v113_applicability_regime_fast.json + - name: Show decision + run: cat v113_applicability_regime_fast.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v113-applicability-regime-fast + path: v113_applicability_regime_fast.json + retention-days: 14 From 757bbd692da6e4dde53042979eb6ebb83ad4d658 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:31:57 +1200 Subject: [PATCH 10/47] Add V114 representation applicability intervention --- .../v114_representation_applicability.py | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 competitions/trace_the_ace/v114_representation_applicability.py diff --git a/competitions/trace_the_ace/v114_representation_applicability.py b/competitions/trace_the_ace/v114_representation_applicability.py new file mode 100644 index 00000000..c57707a3 --- /dev/null +++ b/competitions/trace_the_ace/v114_representation_applicability.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""V114 REPRESENTATION -> APPLICABILITY intervention. + +Question left by V112/V113: + V112: raw transcript views did not improve direct label prediction. + V113: geometry/support/session metadata did not recover the endpoint-oracle gap. + +V114 asks the missing cross: can richer row-level representation predict WHICH already-capable +endpoint (V75 or RELATED) should apply? This is an applicability target, not another label model. + +Frozen protocol: +- deterministic 2500-row sample (same hash rule as V112/V113) +- objective-grouped 4-fold outer OOF +- endpoints trained only on outer-train rows +- oracle-choice target formed per row from endpoint losses, used only inside outer-train for gate fit +- fixed conservative routing weight 0.65; no hyperparameter sweep +- families: geometry, objective semantics, raw transcript, objective+raw, full representation +- controls: response/session ID placebo; shuffled applicability target; flipped-route ablation + +Decision thresholds (precommitted before result): +- PHASE_CHANGE_REPRESENTATION: gain >= .010 and all folds nonnegative, OR gain >= .008 and >=15% oracle-gap recovery +- REPRESENTATION_REPAIR_FOUND: gain >= .003, >=3/4 positive folds, controls <25% real gain, flipped route <=0 +- STRUCTURED_REPRESENTATION_HINT: .001 <= gain < .003 and best family beats geometry by >=.001 +- otherwise REPRESENTATION_NOT_OBSERVED +""" +from __future__ import annotations +import argparse, hashlib, json +from pathlib import Path +import numpy as np +from scipy.sparse import hstack, csr_matrix +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GroupKFold +from sklearn.ensemble import HistGradientBoostingClassifier +from v71_mastery_events import load_transcript, normalize_roles +from v75_canonical_trajectory import load_training, SEED +from v81_target_segment_phase import choose_target_segment +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control, build_control +from v110_residual_collider_state_discovery import hb, ll + +EPS=1e-5 + +def H(x): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16) +def lossrow(y,p): + p=np.clip(p,EPS,1-EPS) + return -(y*np.log(p)+(1-y)*np.log(1-p)) +def geometry(p0,pr): + d=pr-p0 + return np.c_[p0,pr,d,np.abs(d),np.abs(p0-.5),np.abs(pr-.5),np.minimum(p0,pr),np.maximum(p0,pr)] +def transcript_views(df,obj): + d=normalize_roles(df).reset_index(drop=True) + roles=d.role_repaired.astype(str).tolist(); c=d.content.fillna('').astype(str).tolist() + stu=' '.join(x for r,x in zip(roles,c) if r=='student') + tut=' '.join(x for r,x in zip(roles,c) if r=='tutor') + full=' '.join(f'[{r}] {x}' for r,x in zip(roles,c)) + seg,_=choose_target_segment(df,obj); s=normalize_roles(seg).reset_index(drop=True) + local=' '.join(f'[{r}] {x}' for r,x in zip(s.role_repaired.astype(str),s.content.fillna('').astype(str))) + last=' '.join(f'[{r}] {x}' for r,x in list(zip(roles,c))[-8:]) + return stu,tut,full,local,last + +def route(p0,pr,g,flip=False): + if flip: g=1-g + w=np.clip(.65*g,0,.65) + return np.clip((1-w)*p0+w*pr,EPS,1-EPS) +def fit_dense_gate(X,win,sw,tr,va): + m=HistGradientBoostingClassifier(max_depth=2,max_iter=70,learning_rate=.05,min_samples_leaf=80,l2_regularization=2.,random_state=SEED) + m.fit(X[tr],win[tr],sample_weight=sw[tr]) + return m.predict_proba(X[va])[:,1] +def fit_sparse_gate(X,win,sw,tr,va,shuffle=False): + yt=win[tr].copy() + if shuffle: + rng=np.random.default_rng(SEED+len(tr)+len(va)); yt=yt[rng.permutation(len(yt))] + # geometry is already concatenated into X; fixed regularization, no sweep + m=LogisticRegression(C=.08,max_iter=220,solver='liblinear',random_state=SEED) + m.fit(X[tr],yt,sample_weight=sw[tr]) + return m.predict_proba(X[va])[:,1] +def main(a): + f0=load_training(a.features,a.labels).reset_index(drop=True) + print('features columns',list(f0.columns),flush=True) + objall=(f0.learning_objective_id if 'learning_objective_id' in f0 else f0.learning_objective).astype(str).to_numpy() + cand=np.where(np.array([hb(x,5)!=0 for x in objall]))[0] + ix=np.array(sorted(cand,key=lambda i:H(f0.response_id.iloc[i]))[:a.rows]) + f=f0.iloc[ix].reset_index(drop=True) + y=f.target.to_numpy(int) + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + key=f.learning_objective.astype(str).to_numpy(); sess=f.session_id.astype(str).to_numpy() + cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)} + rt=[]; rz=[]; T={k:[] for k in ['STUDENT','TUTOR','FULL','LOCAL','LAST8']} + for _,r in f.iterrows(): + d=cache[str(r.session_id)] + t,z=segmented_control(d,str(r.learning_objective),'related'); rt.append(t); rz.append(z) + vals=transcript_views(d,str(r.learning_objective)) + for k,v in zip(T,vals): T[k].append(v) + X75=build_v75(f,cache); Xr=build_control(rt,rz) + P0=np.zeros(len(f)); PR=np.zeros(len(f)); fold=np.full(len(f),-1,int) + splits=list(GroupKFold(min(4,len(np.unique(obj)))).split(np.zeros(len(y)),y,obj)) + for k,(tr,va) in enumerate(splits): + m0=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr]) + mr=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr]) + P0[va]=np.clip(m0.predict_proba(X75[va])[:,1],EPS,1-EPS) + PR[va]=np.clip(mr.predict_proba(Xr[va])[:,1],EPS,1-EPS); fold[va]=k + # exact-support V97 reconstruction, same rule as V113 + base=np.zeros(len(y)) + allidx=np.arange(len(y)) + for i in range(len(y)): + tr=allidx[fold!=fold[i]] + base[i]=.65*P0[i]+.35*PR[i] if np.sum(key[tr]==key[i])==0 else P0[i] + base=np.clip(base,EPS,1-EPS) + base_ll=ll(y,base) + L0=lossrow(y,P0); LR=lossrow(y,PR); win=(LR0))} + shgain=float(base_ll-ll(y,shuffled)) + real=['OBJECTIVE_SEMANTICS','RAW_TRANSCRIPT','OBJECTIVE_X_RAW','FULL_REPRESENTATION'] + winner=max(real,key=lambda n:tests[n]['gain']); gain=tests[winner]['gain']; rec=gain/gap if gap>0 else 0. + flipped=route(P0,PR,gate_keep[winner],flip=True); flipped_gain=float(base_ll-ll(y,flipped)) + geometry_gain=tests['GEOMETRY']['gain']; idgain=tests['ID_PLACEBO']['gain']; control=max(idgain,shgain) + phase=(gain>=.010 and min(tests[winner]['fold_gains'])>=0) or (gain>=.008 and rec>=.15) + found=(gain>=.003 and tests[winner]['positive_folds']>=3 and control<.25*gain and flipped_gain<=0) + hint=(.001<=gain<.003 and gain-geometry_gain>=.001) + verdict='PHASE_CHANGE_REPRESENTATION' if phase else 'REPRESENTATION_REPAIR_FOUND' if found else 'STRUCTURED_REPRESENTATION_HINT' if hint else 'REPRESENTATION_NOT_OBSERVED' + out={ + 'rows':len(y),'objectives':len(np.unique(obj)),'v97':base_ll,'row_endpoint_oracle':oracle_ll,'oracle_gap':gap, + 'oracle_related_win_rate':float(np.mean(win)),'tests':tests,'winner':winner,'winner_gain':gain, + 'oracle_gap_recovered_fraction':rec,'controls':{'shuffled_applicability_gain':shgain,'id_placebo_gain':idgain,'flipped_winner_route_gain':flipped_gain}, + 'representation_increment_over_geometry':float(gain-geometry_gain),'decision':verdict, + 'precommit':{ + 'phase':'gain >=.010 and all folds nonnegative OR gain >=.008 and >=15% oracle recovery', + 'repair_found':'gain >=.003, >=3/4 positive folds, controls <25% real gain, flipped route <=0', + 'structured':'.001-.003 and representation beats geometry by >=.001', + 'otherwise':'representation not observed' + } + } + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--rows',type=int,default=2500); p.add_argument('--out',default='v114_representation_applicability.json'); main(p.parse_args()) From 9365ca0043db4dfc204b56f2e2c4d3e35139a3e9 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:32:11 +1200 Subject: [PATCH 11/47] Run V114 representation applicability intervention --- ...-ace-v114-representation-applicability.yml | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/trace-ace-v114-representation-applicability.yml diff --git a/.github/workflows/trace-ace-v114-representation-applicability.yml b/.github/workflows/trace-ace-v114-representation-applicability.yml new file mode 100644 index 00000000..0db76e56 --- /dev/null +++ b/.github/workflows/trace-ace-v114-representation-applicability.yml @@ -0,0 +1,50 @@ +name: Trace Ace V114 Representation Applicability +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v114_representation_applicability.py' + - '.github/workflows/trace-ace-v114-representation-applicability.yml' + workflow_dispatch: +jobs: + screen: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Preflight import + run: | + cd competitions/trace_the_ace + python -m py_compile v114_representation_applicability.py + - name: Run V114 + run: | + cd competitions/trace_the_ace + python v114_representation_applicability.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v114_representation_applicability.json + - name: Show decision + run: cat v114_representation_applicability.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v114-representation-applicability + path: v114_representation_applicability.json + retention-days: 14 From 2470c4f9fe82faa808e805b2066f6ca8358db10f Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:33:15 +1200 Subject: [PATCH 12/47] Add V115 collision-resolution audit --- .../v115_collision_resolution_knn.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 competitions/trace_the_ace/v115_collision_resolution_knn.py diff --git a/competitions/trace_the_ace/v115_collision_resolution_knn.py b/competitions/trace_the_ace/v115_collision_resolution_knn.py new file mode 100644 index 00000000..c9161b61 --- /dev/null +++ b/competitions/trace_the_ace/v115_collision_resolution_knn.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""V115 COLLISION-RESOLUTION AUDIT. +Orthogonal to V114's learned gate: use fixed kNN on held-out objectives to test whether widening +representation makes endpoint applicability locally identifiable. + +Frozen: same 2500 rows, same endpoint oracle, GroupKFold by objective, k=15, no sweep. +Families compare geometry-only against objective semantics, raw transcript, and their union. +A real representation repair should improve routing, increase weighted neighbor agreement, survive +shuffled-target control, and fail under flipped routing. +""" +from __future__ import annotations +import argparse,json +from pathlib import Path +import numpy as np +from scipy.sparse import hstack,csr_matrix +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GroupKFold +from sklearn.neighbors import NearestNeighbors +from sklearn.preprocessing import StandardScaler,normalize +from v75_canonical_trajectory import load_training,SEED +from v71_mastery_events import load_transcript +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control,build_control +from v110_residual_collider_state_discovery import hb,ll +from v114_representation_applicability import H,lossrow,geometry,transcript_views,route,EPS + +def knn_gate_dense(X,win,sw,tr,va,k=15,shuffle=False): + sc=StandardScaler().fit(X[tr]); A=sc.transform(X[tr]); B=sc.transform(X[va]) + nn=NearestNeighbors(n_neighbors=min(k,len(tr)),metric='euclidean').fit(A); d,ix=nn.kneighbors(B) + yt=win[tr].copy() + if shuffle: + rng=np.random.default_rng(SEED+17+len(tr)); yt=yt[rng.permutation(len(yt))] + wt=sw[tr][ix]/(d+0.25); return np.sum(wt*yt[ix],axis=1)/np.sum(wt,axis=1) +def knn_gate_sparse(X,win,sw,tr,va,k=15,shuffle=False): + A=normalize(X[tr]); B=normalize(X[va]); nn=NearestNeighbors(n_neighbors=min(k,len(tr)),metric='cosine',algorithm='brute').fit(A); d,ix=nn.kneighbors(B) + yt=win[tr].copy() + if shuffle: + rng=np.random.default_rng(SEED+19+len(tr)); yt=yt[rng.permutation(len(yt))] + sim=np.maximum(1-d,0.01); wt=sw[tr][ix]*sim; return np.sum(wt*yt[ix],axis=1)/np.sum(wt,axis=1) +def main(a): + f0=load_training(a.features,a.labels).reset_index(drop=True); print('features columns',list(f0.columns),flush=True) + objall=(f0.learning_objective_id if 'learning_objective_id' in f0 else f0.learning_objective).astype(str).to_numpy(); cand=np.where(np.array([hb(x,5)!=0 for x in objall]))[0] + ix=np.array(sorted(cand,key=lambda i:H(f0.response_id.iloc[i]))[:a.rows]); f=f0.iloc[ix].reset_index(drop=True) + y=f.target.to_numpy(int); obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy(); key=f.learning_objective.astype(str).to_numpy(); sess=f.session_id.astype(str).to_numpy() + cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)}; rt=[];rz=[];T={k:[] for k in ['STUDENT','TUTOR','FULL','LOCAL','LAST8']} + for _,r in f.iterrows(): + d=cache[str(r.session_id)];t,z=segmented_control(d,str(r.learning_objective),'related');rt.append(t);rz.append(z);vals=transcript_views(d,str(r.learning_objective)) + for k,v in zip(T,vals):T[k].append(v) + X75=build_v75(f,cache);Xr=build_control(rt,rz);P0=np.zeros(len(f));PR=np.zeros(len(f));fold=np.full(len(f),-1,int);splits=list(GroupKFold(min(4,len(np.unique(obj)))).split(np.zeros(len(y)),y,obj)) + for k,(tr,va) in enumerate(splits): + m0=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr]);mr=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr]);P0[va]=np.clip(m0.predict_proba(X75[va])[:,1],EPS,1-EPS);PR[va]=np.clip(mr.predict_proba(Xr[va])[:,1],EPS,1-EPS);fold[va]=k + allidx=np.arange(len(y));base=np.zeros(len(y)) + for i in range(len(y)): + tr=allidx[fold!=fold[i]];base[i]=.65*P0[i]+.35*PR[i] if np.sum(key[tr]==key[i])==0 else P0[i] + base=np.clip(base,EPS,1-EPS);base_ll=ll(y,base);L0=lossrow(y,P0);LR=lossrow(y,PR);win=(LR=.5)==win).astype(float) + tests[n]={'ll':float(ll(y,q)),'gain':float(base_ll-ll(y,q)),'fold_gains':fg,'positive_folds':int(np.sum(np.array(fg)>0)),'oracle_choice_accuracy':float(np.mean(correct)),'confidence_weighted_agreement':float(np.sum(conf*correct)/(np.sum(conf)+1e-12))} + real=['OBJECTIVE_SEMANTICS','RAW_TRANSCRIPT','OBJECTIVE_X_RAW'];winner=max(real,key=lambda n:tests[n]['gain']);gain=tests[winner]['gain'];rec=gain/gap if gap>0 else 0.;shgain=float(base_ll-ll(y,shuffled));flipped=route(P0,PR,gates[winner],flip=True);flipgain=float(base_ll-ll(y,flipped));geom=tests['GEOMETRY']['gain'] + phase=(gain>=.010 and min(tests[winner]['fold_gains'])>=0) or (gain>=.008 and rec>=.15);found=(gain>=.003 and tests[winner]['positive_folds']>=3 and shgain<.25*gain and flipgain<=0);hint=(.001<=gain<.003 and gain-geom>=.001) + out={'rows':len(y),'objectives':len(np.unique(obj)),'v97':base_ll,'row_endpoint_oracle':oracle_ll,'oracle_gap':gap,'tests':tests,'winner':winner,'winner_gain':gain,'oracle_gap_recovered_fraction':rec,'representation_increment_over_geometry':float(gain-geom),'controls':{'shuffled_applicability_gain':shgain,'flipped_winner_route_gain':flipgain},'decision':'PHASE_CHANGE_COLLISION_RESOLUTION' if phase else 'NONLINEAR_REPRESENTATION_REPAIR_FOUND' if found else 'STRUCTURED_COLLISION_HINT' if hint else 'COLLISIONS_NOT_RESOLVED','precommit':{'phase':'gain >=.010 all folds nonnegative OR >=.008 and >=15% oracle recovery','repair_found':'gain >=.003, >=3/4 folds positive, shuffled <25% gain, flipped <=0','structured':'.001-.003 and >=.001 over geometry','otherwise':'collisions not resolved'}} + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v115_collision_resolution_knn.json');main(p.parse_args()) From fdf646f77897977a238b596e3332787e30278653 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:33:26 +1200 Subject: [PATCH 13/47] Run V115 collision-resolution audit --- .../trace-ace-v115-collision-resolution.yml | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/trace-ace-v115-collision-resolution.yml diff --git a/.github/workflows/trace-ace-v115-collision-resolution.yml b/.github/workflows/trace-ace-v115-collision-resolution.yml new file mode 100644 index 00000000..f0ebe432 --- /dev/null +++ b/.github/workflows/trace-ace-v115-collision-resolution.yml @@ -0,0 +1,51 @@ +name: Trace Ace V115 Collision Resolution +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v115_collision_resolution_knn.py' + - 'competitions/trace_the_ace/v114_representation_applicability.py' + - '.github/workflows/trace-ace-v115-collision-resolution.yml' + workflow_dispatch: +jobs: + screen: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Preflight import + run: | + cd competitions/trace_the_ace + python -m py_compile v114_representation_applicability.py v115_collision_resolution_knn.py + - name: Run V115 + run: | + cd competitions/trace_the_ace + python v115_collision_resolution_knn.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v115_collision_resolution_knn.json + - name: Show decision + run: cat v115_collision_resolution_knn.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v115-collision-resolution-knn + path: v115_collision_resolution_knn.json + retention-days: 14 From bf12182fb25aa327f680fbc76f63b18c7bdcea65 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:39:16 +1200 Subject: [PATCH 14/47] Add V116 row-alignment alias audit --- .../v116_row_alignment_alias_audit.py | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 competitions/trace_the_ace/v116_row_alignment_alias_audit.py diff --git a/competitions/trace_the_ace/v116_row_alignment_alias_audit.py b/competitions/trace_the_ace/v116_row_alignment_alias_audit.py new file mode 100644 index 00000000..a7b8eb83 --- /dev/null +++ b/competitions/trace_the_ace/v116_row_alignment_alias_audit.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""V116 ROW-ALIGNMENT / REPRESENTATION-ALIAS AUDIT. + +After V112/V113/V114/V115 fail to recover the large endpoint-oracle gap, test the upstream +hypothesis: distinct labelled responses may be mapped to the same session/objective state before +prediction. If identical representations contain different labels or different oracle endpoint +choices, no downstream router can resolve them without a new row-level alignment/state variable. + +This is an aggregate diagnostic only: no raw transcript content is emitted. +""" +from __future__ import annotations +import argparse,hashlib,json +from pathlib import Path +from collections import defaultdict +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GroupKFold +from v75_canonical_trajectory import load_training,SEED +from v71_mastery_events import load_transcript +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control,build_control +from v110_residual_collider_state_discovery import hb,ll +from v114_representation_applicability import H,lossrow,EPS + +def rep_hash(x): + a=np.asarray(x,dtype=np.float64);a=np.nan_to_num(a,nan=0.,posinf=1e30,neginf=-1e30);a=np.round(a,10) + return hashlib.sha256(a.tobytes()).hexdigest() +def summarize_groups(groups,y,win,Lbase,Loracle): + collision=[]; mixed_y=[];mixed_w=[];gap=0.;rows=0 + for z in groups.values(): + if len(z)>1: + collision.extend(z);rows+=len(z) + yy=y[z];ww=win[z] + if len(np.unique(yy))>1:mixed_y.extend(z) + if len(np.unique(ww))>1: + mixed_w.extend(z);gap+=float(np.sum(Lbase[z]-Loracle[z])) + return {'groups_total':len(groups),'collision_groups':int(sum(len(z)>1 for z in groups.values())),'rows_in_collision_groups':len(set(collision)),'mixed_label_groups':int(sum(len(z)>1 and len(np.unique(y[z]))>1 for z in groups.values())),'rows_in_mixed_label_groups':len(set(mixed_y)),'mixed_oracle_choice_groups':int(sum(len(z)>1 and len(np.unique(win[z]))>1 for z in groups.values())),'rows_in_mixed_oracle_choice_groups':len(set(mixed_w)),'oracle_gap_sum_in_mixed_choice_groups':gap} +def main(a): + f0=load_training(a.features,a.labels).reset_index(drop=True);print('features columns',list(f0.columns),flush=True) + objall=(f0.learning_objective_id if 'learning_objective_id' in f0 else f0.learning_objective).astype(str).to_numpy();cand=np.where(np.array([hb(x,5)!=0 for x in objall]))[0];ix=np.array(sorted(cand,key=lambda i:H(f0.response_id.iloc[i]))[:a.rows]);f=f0.iloc[ix].reset_index(drop=True) + y=f.target.to_numpy(int);obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy();key=f.learning_objective.astype(str).to_numpy();sess=f.session_id.astype(str).to_numpy();rid=f.response_id.astype(str).to_numpy() + cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)};rt=[];rz=[] + for _,r in f.iterrows():d=cache[str(r.session_id)];t,z=segmented_control(d,str(r.learning_objective),'related');rt.append(t);rz.append(z) + X75=build_v75(f,cache);Xr=build_control(rt,rz);P0=np.zeros(len(f));PR=np.zeros(len(f));fold=np.full(len(f),-1,int);splits=list(GroupKFold(min(4,len(np.unique(obj)))).split(np.zeros(len(y)),y,obj)) + for k,(tr,va) in enumerate(splits): + m0=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr]);mr=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr]);P0[va]=np.clip(m0.predict_proba(X75[va])[:,1],EPS,1-EPS);PR[va]=np.clip(mr.predict_proba(Xr[va])[:,1],EPS,1-EPS);fold[va]=k + allidx=np.arange(len(y));base=np.zeros(len(y)) + for i in range(len(y)): + tr=allidx[fold!=fold[i]];base[i]=.65*P0[i]+.35*PR[i] if np.sum(key[tr]==key[i])==0 else P0[i] + base=np.clip(base,EPS,1-EPS);L0=lossrow(y,P0);LR=lossrow(y,PR);win=(LR1: + p=float(np.clip(np.mean(y[z]),EPS,1-EPS));const_nll+=float(np.sum(lossrow(y[z],np.full(len(z),p))));const_rows+=len(z) + out={'rows':len(y),'objectives':len(np.unique(obj)),'sessions':len(np.unique(sess)),'v97':float(ll(y,base)),'row_endpoint_oracle':float(ll(y,oracle)),'oracle_gap':float(ll(y,base)-ll(y,oracle)),'oracle_related_win_rate':float(np.mean(win)),'session_objective_aliases':summarize_groups(groups_so,y,win,Lbase,Loracle),'exact_feature_representation_aliases':summarize_groups(groups_rep,y,win,Lbase,Loracle),'endpoint_prediction_aliases':summarize_groups(groups_endpoint,y,win,Lbase,Loracle),'aggregate_alignment_evidence':{'transcript_columns':sorted(headers.keys()),'response_ids_found_exactly_anywhere':len(found),'response_id_anchor_rate':float(len(found)/len(rid)),'anchor_columns':{c:len(v) for c,v in found_cols.items()},'constant_within_rep_collision_empirical_nll':float(const_nll/const_rows) if const_rows else None,'constant_collision_rows':const_rows}} + so=out['session_objective_aliases'];rp=out['exact_feature_representation_aliases'];anchor=out['aggregate_alignment_evidence']['response_id_anchor_rate'] + if rp['mixed_oracle_choice_groups']>0 and rp['rows_in_mixed_oracle_choice_groups']>=.05*len(y):verdict='REPRESENTATION_ALIAS_CONFIRMED' + elif so['mixed_oracle_choice_groups']>0 and anchor<.5:verdict='ROW_ALIGNMENT_STATE_CANDIDATE' + else:verdict='NO_STRONG_ROW_ALIAS_EVIDENCE' + out['decision']=verdict;out['interpretation_rule']='Alias confirmed if exact X75+Xr collisions with mixed oracle choice cover >=5% rows; row-alignment candidate if session/objective aliases mix oracle choice and <50% response IDs anchor exactly in transcript.' + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v116_row_alignment_alias_audit.json');main(p.parse_args()) From e29796f7abe411e56900c9e2ad779bd780841686 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:39:32 +1200 Subject: [PATCH 15/47] Run V116 row-alignment alias audit --- .../trace-ace-v116-row-alignment-alias.yml | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/trace-ace-v116-row-alignment-alias.yml diff --git a/.github/workflows/trace-ace-v116-row-alignment-alias.yml b/.github/workflows/trace-ace-v116-row-alignment-alias.yml new file mode 100644 index 00000000..36ded640 --- /dev/null +++ b/.github/workflows/trace-ace-v116-row-alignment-alias.yml @@ -0,0 +1,51 @@ +name: Trace Ace V116 Row Alignment Alias Audit +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v116_row_alignment_alias_audit.py' + - 'competitions/trace_the_ace/v114_representation_applicability.py' + - '.github/workflows/trace-ace-v116-row-alignment-alias.yml' + workflow_dispatch: +jobs: + audit: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Preflight import + run: | + cd competitions/trace_the_ace + python -m py_compile v114_representation_applicability.py v116_row_alignment_alias_audit.py + - name: Run V116 + run: | + cd competitions/trace_the_ace + python v116_row_alignment_alias_audit.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v116_row_alignment_alias_audit.json + - name: Show decision + run: cat v116_row_alignment_alias_audit.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v116-row-alignment-alias-audit + path: v116_row_alignment_alias_audit.json + retention-days: 14 From 6bcf8148221899ffaec01293863f4fb4ea067f47 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:46:16 +1200 Subject: [PATCH 16/47] Fix V116 sparse row hashing --- .../trace_the_ace/v116_row_alignment_alias_audit.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/competitions/trace_the_ace/v116_row_alignment_alias_audit.py b/competitions/trace_the_ace/v116_row_alignment_alias_audit.py index a7b8eb83..8a104326 100644 --- a/competitions/trace_the_ace/v116_row_alignment_alias_audit.py +++ b/competitions/trace_the_ace/v116_row_alignment_alias_audit.py @@ -22,6 +22,10 @@ from v110_residual_collider_state_discovery import hb,ll from v114_representation_applicability import H,lossrow,EPS +def dense_row(X,i): + r=X[i] + if hasattr(r,'toarray'): return np.asarray(r.toarray()).ravel() + return np.asarray(r).ravel() def rep_hash(x): a=np.asarray(x,dtype=np.float64);a=np.nan_to_num(a,nan=0.,posinf=1e30,neginf=-1e30);a=np.round(a,10) return hashlib.sha256(a.tobytes()).hexdigest() @@ -51,7 +55,7 @@ def main(a): groups_so=defaultdict(list);groups_rep=defaultdict(list);groups_endpoint=defaultdict(list) for i in range(len(y)): groups_so[(sess[i],obj[i])].append(i) - groups_rep[rep_hash(np.r_[X75[i],Xr[i]])].append(i) + groups_rep[rep_hash(np.r_[dense_row(X75,i),dense_row(Xr,i)])].append(i) groups_endpoint[rep_hash(np.r_[P0[i],PR[i]])].append(i) # transcript schema and exact response-id anchor audit, aggregate only headers=defaultdict(int);rid_set=set(rid);found=set();found_cols=defaultdict(set) From 2fae4e5ef15799dae5069fed5230f57e67b5be9a Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:52:08 +1200 Subject: [PATCH 17/47] Add V117 oracle information audit --- .../v117_oracle_information_audit.py | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 competitions/trace_the_ace/v117_oracle_information_audit.py diff --git a/competitions/trace_the_ace/v117_oracle_information_audit.py b/competitions/trace_the_ace/v117_oracle_information_audit.py new file mode 100644 index 00000000..72a75476 --- /dev/null +++ b/competitions/trace_the_ace/v117_oracle_information_audit.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""V117 ORACLE INFORMATION AUDIT. + +Tests whether the large per-row V75-vs-RELATED endpoint oracle gap is evidence of a latent +applicability regime or simply the value of revealing the realized label. For binary log loss, +if two endpoint probabilities differ, the lower-loss endpoint is determined by the label and the +sign of (PR-P0). This audit quantifies that identity on the frozen V113 sample and compares it with +strictly label-free/cross-fitted endpoint selection. + +Frozen protocol: same deterministic 2500 rows, same objective-grouped 4-fold endpoint fits, +no result-dependent tuning. Meta selectors are trained only on outer-train rows. +""" +from __future__ import annotations +import argparse,json +from pathlib import Path +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.ensemble import HistGradientBoostingClassifier +from sklearn.model_selection import GroupKFold +from v75_canonical_trajectory import load_training,SEED +from v71_mastery_events import load_transcript +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control,build_control +from v110_residual_collider_state_discovery import hb,ll +from v114_representation_applicability import H,lossrow,geometry,route,EPS + +def main(a): + f0=load_training(a.features,a.labels).reset_index(drop=True);print('features columns',list(f0.columns),flush=True) + objall=(f0.learning_objective_id if 'learning_objective_id' in f0 else f0.learning_objective).astype(str).to_numpy();cand=np.where(np.array([hb(x,5)!=0 for x in objall]))[0];ix=np.array(sorted(cand,key=lambda i:H(f0.response_id.iloc[i]))[:a.rows]);f=f0.iloc[ix].reset_index(drop=True) + y=f.target.to_numpy(int);obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy();key=f.learning_objective.astype(str).to_numpy();sess=f.session_id.astype(str).to_numpy();cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)};rt=[];rz=[] + for _,r in f.iterrows():d=cache[str(r.session_id)];t,z=segmented_control(d,str(r.learning_objective),'related');rt.append(t);rz.append(z) + X75=build_v75(f,cache);Xr=build_control(rt,rz);P0=np.zeros(len(f));PR=np.zeros(len(f));fold=np.full(len(f),-1,int);splits=list(GroupKFold(min(4,len(np.unique(obj)))).split(np.zeros(len(y)),y,obj)) + for k,(tr,va) in enumerate(splits): + m0=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr]);mr=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr[tr],y[tr]);P0[va]=np.clip(m0.predict_proba(X75[va])[:,1],EPS,1-EPS);PR[va]=np.clip(mr.predict_proba(Xr[va])[:,1],EPS,1-EPS);fold[va]=k + allidx=np.arange(len(y));base=np.zeros(len(y)) + for i in range(len(y)): + tr=allidx[fold!=fold[i]];base[i]=.65*P0[i]+.35*PR[i] if np.sum(key[tr]==key[i])==0 else P0[i] + base=np.clip(base,EPS,1-EPS);L0=lossrow(y,P0);LR=lossrow(y,PR);win=(LR0,d<0).astype(int);mask=~ties;identity=float(np.mean(win[mask]==clairvoyant[mask])) if np.any(mask) else 1. + # Counterfactual label flip: lower-loss endpoint must flip whenever endpoint probabilities differ. + yf=1-y;wf=(lossrow(yf,PR)0 else 0. + out={'rows':len(y),'objectives':len(np.unique(obj)),'endpoint_disagreement_rate':float(np.mean(mask)),'oracle_related_win_rate':float(np.mean(win)),'clairvoyant_choice_identity_rate':identity,'choice_flip_under_label_counterfactual_rate':flip_rate,'v97':float(base_ll),'row_label_oracle':float(oracle_ll),'oracle_gap':float(gap),'label_free_controls':endpoint_ll,'label_free_gains_vs_v97':gains,'best_label_free':best_label_free,'best_label_free_gain':best_gain,'best_label_free_oracle_gap_recovered_fraction':recovered} + out['decision']='ROW_ORACLE_IS_REALIZED_LABEL_INFORMATION' if identity>=.999 and flip_rate>=.999 else 'ORACLE_HAS_NONTRIVIAL_APPLICABILITY_STRUCTURE' + out['interpretation_rule']='If oracle choice matches label+endpoint-order identity and flips under counterfactual label >=99.9%, the row oracle is clairvoyant realized-outcome information; its raw gap must not be treated as recoverable capability without a separately validated label-free selector.' + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--rows',type=int,default=2500);p.add_argument('--out',default='v117_oracle_information_audit.json');main(p.parse_args()) From 6d7a57100cadee1eb596c38ba4aeb81e878fc1f5 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:52:22 +1200 Subject: [PATCH 18/47] Run V117 oracle information audit --- .../trace-ace-v117-oracle-information.yml | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/trace-ace-v117-oracle-information.yml diff --git a/.github/workflows/trace-ace-v117-oracle-information.yml b/.github/workflows/trace-ace-v117-oracle-information.yml new file mode 100644 index 00000000..094f9ac1 --- /dev/null +++ b/.github/workflows/trace-ace-v117-oracle-information.yml @@ -0,0 +1,50 @@ +name: Trace Ace V117 Oracle Information Audit +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v117_oracle_information_audit.py' + - '.github/workflows/trace-ace-v117-oracle-information.yml' + workflow_dispatch: +jobs: + audit: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Preflight import + run: | + cd competitions/trace_the_ace + python -m py_compile v114_representation_applicability.py v117_oracle_information_audit.py + - name: Run V117 + run: | + cd competitions/trace_the_ace + python v117_oracle_information_audit.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --rows 2500 --out ../../v117_oracle_information_audit.json + - name: Show decision + run: cat v117_oracle_information_audit.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v117-oracle-information-audit + path: v117_oracle_information_audit.json + retention-days: 14 From c6b48048d07d19d90eb0099744f2c9a4eb1bf51d Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:04:16 +1200 Subject: [PATCH 19/47] Add V115 Trace the Ace reality audit --- .../trace_the_ace/v115_reality_audit.py | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 competitions/trace_the_ace/v115_reality_audit.py diff --git a/competitions/trace_the_ace/v115_reality_audit.py b/competitions/trace_the_ace/v115_reality_audit.py new file mode 100644 index 00000000..f7d82aca --- /dev/null +++ b/competitions/trace_the_ace/v115_reality_audit.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""V115 Reality Audit. + +Primary question: did we over-optimize objective-cold validation and underweight the +competition's natural new-session / same-objective regime? + +Frozen outputs: +- full 35,072-row 5-fold session-grouped OOF for pure V74, V75 and V97; +- pure V74 support/frequency stratification using fold-local objective support; +- per-fold gains and objective-frequency contribution to log loss; +- deterministic provider-proxy stratification from transcript structure only; +- objective-grouped stress results are reported as a secondary contrast, not a gate. + +No test labels, no cross-validation leakage, no cross-test aggregation. +""" +from __future__ import annotations +import argparse, json, re +from pathlib import Path +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import load_transcript, normalize_roles +from v74_semantic_objective_prior import semantic_prior_predict +from v75_canonical_trajectory import load_training, build_v75 if False else trajectory_views, SEED +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control, build_control + +EPS=1e-5 + +def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS))) + +def provider_proxy(df): + """Deterministic structural proxy only; deliberately not claimed as true provider ID.""" + d=normalize_roles(df).reset_index(drop=True) + roles=d.role_repaired.astype(str).str.lower().to_numpy() + txt=d.content.fillna('').astype(str).tolist() + n=len(d); stu=int(np.sum(roles=='student')); tut=int(np.sum(roles=='tutor')) + mean_words=float(np.mean([len(x.split()) for x in txt])) if txt else 0.0 + markers=sum(bool(re.search(r'\b(?:learning objective|learning goal|prior learning|i do|we do|you do|application|slide|lesson)\b',x,re.I)) for x in txt) + # Long lesson / curriculum-marker sessions are TSL-like; compact chats Eedi-like. + tsl_like=(n>=24) or (markers>=2) or (tut>=12 and mean_words>=8) + return 'TSL_LIKE' if tsl_like else 'EEDI_LIKE' + +def fit_logit(X,y,tr,va): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]) + return np.clip(m.predict_proba(X[va])[:,1],EPS,1-EPS) + +def p97_endpoint(X75,Xr,y,key,tr,va): + p75=fit_logit(X75,y,tr,va); pr=fit_logit(Xr,y,tr,va) + support={x:0 for x in []} + vals,cts=np.unique(key[tr],return_counts=True); d=dict(zip(vals,cts)) + seen=np.array([d.get(x,0)>0 for x in key[va]]) + p=np.where(seen,p75,.65*p75+.35*pr) + return np.clip(p,EPS,1-EPS),p75,pr,np.array([d.get(x,0) for x in key[va]],float) + +def eval_session(frame,cache): + y=frame.target.to_numpy(int); key=frame.learning_objective.astype(str).to_numpy(); sess=frame.session_id.astype(str).to_numpy() + X75=build_v75(frame,cache) + rt=[];rz=[] + for _,r in frame.iterrows(): + t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related'); rt.append(t);rz.append(z) + Xr=build_control(rt,rz) + splits=list(GroupKFold(5).split(np.zeros(len(y)),y,sess)) + p74=np.zeros(len(y));p75=np.zeros(len(y));p97=np.zeros(len(y));support=np.zeros(len(y));fold=np.zeros(len(y),int) + folds=[] + for k,(tr,va) in enumerate(splits): + ph,_=semantic_prior_predict(frame.iloc[tr],frame.iloc[va],k=16,smooth=2.0);p74[va]=ph + q,q75,qr,s=p97_endpoint(X75,Xr,y,key,tr,va);p97[va]=q;p75[va]=q75;support[va]=s;fold[va]=k + folds.append({'fold':k+1,'rows':len(va),'v74':ll(y[va],p74[va]),'v75':ll(y[va],p75[va]),'v97':ll(y[va],p97[va])}) + # support bins frozen before seeing results + bins=[('ZERO',lambda s:s==0),('1_2',lambda s:(s>=1)&(s<=2)),('3_9',lambda s:(s>=3)&(s<=9)),('10_29',lambda s:(s>=10)&(s<=29)),('30_PLUS',lambda s:s>=30)] + strat={} + rowloss=-(y*np.log(np.clip(p74,EPS,1))+(1-y)*np.log(np.clip(1-p74,EPS,1))) + total=float(rowloss.sum()) + for name,fn in bins: + m=fn(support) + if m.any(): strat[name]={'rows':int(m.sum()),'share':float(m.mean()),'mean_support':float(support[m].mean()),'v74_ll':ll(y[m],p74[m]),'v75_ll':ll(y[m],p75[m]),'v97_ll':ll(y[m],p97[m]),'v74_loss_share':float(rowloss[m].sum()/total)} + return {'v74':ll(y,p74),'v75':ll(y,p75),'v97':ll(y,p97),'folds':folds,'support_strata':strat},(p74,p75,p97,support) + +def objective_stress(frame): + y=frame.target.to_numpy(int); grp=(frame.learning_objective_id if 'learning_objective_id' in frame else frame.learning_objective).astype(str).to_numpy();p=np.zeros(len(y)) + for tr,va in GroupKFold(5).split(np.zeros(len(y)),y,grp): p[va],_=semantic_prior_predict(frame.iloc[tr],frame.iloc[va],k=16,smooth=2.0) + return ll(y,p) + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + print('features columns',list(f.columns),flush=True) + print('rows',len(f),'sessions',f.session_id.nunique(),'objectives',f.learning_objective.nunique(),flush=True) + cache={} + proxy={} + for i,sid in enumerate(f.session_id.astype(str).unique()): + d=load_transcript(a.transcripts/f'{sid}.csv');cache[sid]=d;proxy[sid]=provider_proxy(d) + if (i+1)%5000==0: print('loaded sessions',i+1,flush=True) + session,(p74,p75,p97,support)=eval_session(f,cache) + y=f.target.to_numpy(int);reg=np.array([proxy[str(s)] for s in f.session_id.astype(str)]) + regimes={} + for r in ['EEDI_LIKE','TSL_LIKE']: + m=reg==r + regimes[r]={'rows':int(m.sum()),'sessions':int(f.loc[m,'session_id'].nunique()),'share':float(m.mean()),'v74':ll(y[m],p74[m]),'v75':ll(y[m],p75[m]),'v97':ll(y[m],p97[m]),'v74_gain_vs_v75':ll(y[m],p75[m])-ll(y[m],p74[m])} + objstress=objective_stress(f) + delta=session['v75']-session['v74'] + # Frozen interpretation. V74 is a leaderboard-priority candidate if it beats V75 by >= .010 session-cold and >=60% rows have exact support. + supported=float(np.mean(support>0)) + verdict='PRIORITIZE_PURE_V74_RUNTIME' if delta>=.010 and supported>=.60 else ('V74_REAL_BUT_MIXED' if delta>=.003 else 'V74_NOT_PRIMARY') + out={'diagnostics':{'rows':len(f),'sessions':int(f.session_id.nunique()),'objectives':int(f.learning_objective.nunique()),'positive_rate':float(y.mean()),'session_exact_objective_support_rate':supported,'provider_proxy_is_heuristic':True},'session_cold':session,'objective_cold_v74_stress':objstress,'provider_proxy':regimes,'decision':{'verdict':verdict,'v74_gain_vs_v75_session':delta,'rule':'PRIORITIZE pure V74 if session-cold gain vs V75 >=.010 and >=60% validation rows have fold-local exact-objective support; objective-cold is secondary stress only.'}} + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--out',default='v115_reality_audit.json');run(p.parse_args()) From cef9042f5956f2d59e83fc9c166e5fbb0753b711 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:04:48 +1200 Subject: [PATCH 20/47] Fix V115 preflight import --- .../trace_the_ace/v115_reality_audit.py | 33 ++++++------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/competitions/trace_the_ace/v115_reality_audit.py b/competitions/trace_the_ace/v115_reality_audit.py index f7d82aca..7aa4ef06 100644 --- a/competitions/trace_the_ace/v115_reality_audit.py +++ b/competitions/trace_the_ace/v115_reality_audit.py @@ -23,7 +23,7 @@ from v71_mastery_events import load_transcript, normalize_roles from v74_semantic_objective_prior import semantic_prior_predict -from v75_canonical_trajectory import load_training, build_v75 if False else trajectory_views, SEED +from v75_canonical_trajectory import load_training, SEED from v85_evidence_state import build_v75 from v94_related_control import segmented_control, build_control @@ -32,14 +32,12 @@ def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS))) def provider_proxy(df): - """Deterministic structural proxy only; deliberately not claimed as true provider ID.""" d=normalize_roles(df).reset_index(drop=True) roles=d.role_repaired.astype(str).str.lower().to_numpy() txt=d.content.fillna('').astype(str).tolist() - n=len(d); stu=int(np.sum(roles=='student')); tut=int(np.sum(roles=='tutor')) + n=len(d); tut=int(np.sum(roles=='tutor')) mean_words=float(np.mean([len(x.split()) for x in txt])) if txt else 0.0 markers=sum(bool(re.search(r'\b(?:learning objective|learning goal|prior learning|i do|we do|you do|application|slide|lesson)\b',x,re.I)) for x in txt) - # Long lesson / curriculum-marker sessions are TSL-like; compact chats Eedi-like. tsl_like=(n>=24) or (markers>=2) or (tut>=12 and mean_words>=8) return 'TSL_LIKE' if tsl_like else 'EEDI_LIKE' @@ -49,7 +47,6 @@ def fit_logit(X,y,tr,va): def p97_endpoint(X75,Xr,y,key,tr,va): p75=fit_logit(X75,y,tr,va); pr=fit_logit(Xr,y,tr,va) - support={x:0 for x in []} vals,cts=np.unique(key[tr],return_counts=True); d=dict(zip(vals,cts)) seen=np.array([d.get(x,0)>0 for x in key[va]]) p=np.where(seen,p75,.65*p75+.35*pr) @@ -63,17 +60,13 @@ def eval_session(frame,cache): t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related'); rt.append(t);rz.append(z) Xr=build_control(rt,rz) splits=list(GroupKFold(5).split(np.zeros(len(y)),y,sess)) - p74=np.zeros(len(y));p75=np.zeros(len(y));p97=np.zeros(len(y));support=np.zeros(len(y));fold=np.zeros(len(y),int) - folds=[] + p74=np.zeros(len(y));p75=np.zeros(len(y));p97=np.zeros(len(y));support=np.zeros(len(y));folds=[] for k,(tr,va) in enumerate(splits): ph,_=semantic_prior_predict(frame.iloc[tr],frame.iloc[va],k=16,smooth=2.0);p74[va]=ph - q,q75,qr,s=p97_endpoint(X75,Xr,y,key,tr,va);p97[va]=q;p75[va]=q75;support[va]=s;fold[va]=k + q,q75,qr,s=p97_endpoint(X75,Xr,y,key,tr,va);p97[va]=q;p75[va]=q75;support[va]=s folds.append({'fold':k+1,'rows':len(va),'v74':ll(y[va],p74[va]),'v75':ll(y[va],p75[va]),'v97':ll(y[va],p97[va])}) - # support bins frozen before seeing results bins=[('ZERO',lambda s:s==0),('1_2',lambda s:(s>=1)&(s<=2)),('3_9',lambda s:(s>=3)&(s<=9)),('10_29',lambda s:(s>=10)&(s<=29)),('30_PLUS',lambda s:s>=30)] - strat={} - rowloss=-(y*np.log(np.clip(p74,EPS,1))+(1-y)*np.log(np.clip(1-p74,EPS,1))) - total=float(rowloss.sum()) + strat={};rowloss=-(y*np.log(np.clip(p74,EPS,1))+(1-y)*np.log(np.clip(1-p74,EPS,1)));total=float(rowloss.sum()) for name,fn in bins: m=fn(support) if m.any(): strat[name]={'rows':int(m.sum()),'share':float(m.mean()),'mean_support':float(support[m].mean()),'v74_ll':ll(y[m],p74[m]),'v75_ll':ll(y[m],p75[m]),'v97_ll':ll(y[m],p97[m]),'v74_loss_share':float(rowloss[m].sum()/total)} @@ -86,23 +79,17 @@ def objective_stress(frame): def run(a): f=load_training(a.features,a.labels).reset_index(drop=True) - print('features columns',list(f.columns),flush=True) - print('rows',len(f),'sessions',f.session_id.nunique(),'objectives',f.learning_objective.nunique(),flush=True) - cache={} - proxy={} + print('features columns',list(f.columns),flush=True);print('rows',len(f),'sessions',f.session_id.nunique(),'objectives',f.learning_objective.nunique(),flush=True) + cache={};proxy={} for i,sid in enumerate(f.session_id.astype(str).unique()): d=load_transcript(a.transcripts/f'{sid}.csv');cache[sid]=d;proxy[sid]=provider_proxy(d) if (i+1)%5000==0: print('loaded sessions',i+1,flush=True) session,(p74,p75,p97,support)=eval_session(f,cache) - y=f.target.to_numpy(int);reg=np.array([proxy[str(s)] for s in f.session_id.astype(str)]) - regimes={} + y=f.target.to_numpy(int);reg=np.array([proxy[str(s)] for s in f.session_id.astype(str)]);regimes={} for r in ['EEDI_LIKE','TSL_LIKE']: m=reg==r - regimes[r]={'rows':int(m.sum()),'sessions':int(f.loc[m,'session_id'].nunique()),'share':float(m.mean()),'v74':ll(y[m],p74[m]),'v75':ll(y[m],p75[m]),'v97':ll(y[m],p97[m]),'v74_gain_vs_v75':ll(y[m],p75[m])-ll(y[m],p74[m])} - objstress=objective_stress(f) - delta=session['v75']-session['v74'] - # Frozen interpretation. V74 is a leaderboard-priority candidate if it beats V75 by >= .010 session-cold and >=60% rows have exact support. - supported=float(np.mean(support>0)) + if m.any(): regimes[r]={'rows':int(m.sum()),'sessions':int(f.loc[m,'session_id'].nunique()),'share':float(m.mean()),'v74':ll(y[m],p74[m]),'v75':ll(y[m],p75[m]),'v97':ll(y[m],p97[m]),'v74_gain_vs_v75':ll(y[m],p75[m])-ll(y[m],p74[m])} + objstress=objective_stress(f);delta=session['v75']-session['v74'];supported=float(np.mean(support>0)) verdict='PRIORITIZE_PURE_V74_RUNTIME' if delta>=.010 and supported>=.60 else ('V74_REAL_BUT_MIXED' if delta>=.003 else 'V74_NOT_PRIMARY') out={'diagnostics':{'rows':len(f),'sessions':int(f.session_id.nunique()),'objectives':int(f.learning_objective.nunique()),'positive_rate':float(y.mean()),'session_exact_objective_support_rate':supported,'provider_proxy_is_heuristic':True},'session_cold':session,'objective_cold_v74_stress':objstress,'provider_proxy':regimes,'decision':{'verdict':verdict,'v74_gain_vs_v75_session':delta,'rule':'PRIORITIZE pure V74 if session-cold gain vs V75 >=.010 and >=60% validation rows have fold-local exact-objective support; objective-cold is secondary stress only.'}} Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) From fdc990fe0684776707ad00ffdb8e69771f9d19e7 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:05:01 +1200 Subject: [PATCH 21/47] Add V115 reality audit workflow --- .../trace-ace-v115-reality-audit.yml | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/trace-ace-v115-reality-audit.yml diff --git a/.github/workflows/trace-ace-v115-reality-audit.yml b/.github/workflows/trace-ace-v115-reality-audit.yml new file mode 100644 index 00000000..3a0b82c5 --- /dev/null +++ b/.github/workflows/trace-ace-v115-reality-audit.yml @@ -0,0 +1,50 @@ +name: Trace Ace V115 Reality Audit +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v115_reality_audit.py' + - '.github/workflows/trace-ace-v115-reality-audit.yml' + workflow_dispatch: +jobs: + audit: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Preflight compile + run: | + cd competitions/trace_the_ace + python -m py_compile v115_reality_audit.py + - name: Run V115 reality audit + run: | + cd competitions/trace_the_ace + python v115_reality_audit.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v115_reality_audit.json + - name: Show decision + run: cat v115_reality_audit.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v115-reality-audit + path: v115_reality_audit.json + retention-days: 14 From f403df37cefa31db77ee02ee58ec08dce477d214 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:41:08 +1200 Subject: [PATCH 22/47] Add fast pure V74 reality audit --- .../v115b_pure_v74_reality_audit.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 competitions/trace_the_ace/v115b_pure_v74_reality_audit.py diff --git a/competitions/trace_the_ace/v115b_pure_v74_reality_audit.py b/competitions/trace_the_ace/v115b_pure_v74_reality_audit.py new file mode 100644 index 00000000..198d10bb --- /dev/null +++ b/competitions/trace_the_ace/v115b_pure_v74_reality_audit.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +from __future__ import annotations +import argparse,json +from pathlib import Path +import numpy as np +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold +from v74_semantic_objective_prior import load_training, semantic_prior_predict +EPS=1e-5 + +def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS))) + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + print('columns',list(f.columns),flush=True) + y=f.target.to_numpy(int); sess=f.session_id.astype(str).to_numpy(); key=f.learning_objective.astype(str).to_numpy() + grp=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + p=np.zeros(len(f)); support=np.zeros(len(f)); folds=[] + for k,(tr,va) in enumerate(GroupKFold(5).split(np.zeros(len(y)),y,sess),1): + ph,_=semantic_prior_predict(f.iloc[tr],f.iloc[va],k=16,smooth=2.0);p[va]=ph + vals,cts=np.unique(key[tr],return_counts=True);d=dict(zip(vals,cts));support[va]=np.array([d.get(x,0) for x in key[va]],float) + folds.append({'fold':k,'rows':len(va),'v74_ll':ll(y[va],ph),'support_rate':float(np.mean(support[va]>0))}) + session_ll=ll(y,p); support_rate=float(np.mean(support>0)) + bins=[('ZERO',support==0),('1_2',(support>=1)&(support<=2)),('3_9',(support>=3)&(support<=9)),('10_29',(support>=10)&(support<=29)),('30_PLUS',support>=30)] + strat={} + rowloss=-(y*np.log(np.clip(p,EPS,1))+(1-y)*np.log(np.clip(1-p,EPS,1)));tot=rowloss.sum() + for n,m in bins: + if m.any(): strat[n]={'rows':int(m.sum()),'share':float(m.mean()),'v74_ll':ll(y[m],p[m]),'loss_share':float(rowloss[m].sum()/tot),'mean_support':float(support[m].mean())} + po=np.zeros(len(f)) + for tr,va in GroupKFold(5).split(np.zeros(len(y)),y,grp): po[va],_=semantic_prior_predict(f.iloc[tr],f.iloc[va],k=16,smooth=2.0) + obj_ll=ll(y,po) + out={'rows':len(f),'sessions':int(f.session_id.nunique()),'objectives':int(f.learning_objective.nunique()),'positive_rate':float(y.mean()),'session_cold_v74':session_ll,'session_exact_objective_support_rate':support_rate,'session_folds':folds,'support_strata':strat,'objective_cold_v74_stress':obj_ll,'session_minus_objective_advantage':obj_ll-session_ll,'decision':{'verdict':'V74_SESSION_GEOMETRY_CONFIRMED' if session_ll<=.56 and support_rate>=.60 else 'V74_SESSION_GEOMETRY_NOT_CONFIRMED','rule':'Confirm if session-cold V74 <=.560 and fold-local exact objective support >=60%. Runtime promotion still requires comparison to verified incumbent/public test.'}} + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--out',default='v115b_pure_v74_reality_audit.json');run(p.parse_args()) From 2afdb57e97c95f669d2fb9d955da00948da96d02 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:41:20 +1200 Subject: [PATCH 23/47] Add fast V115b pure V74 workflow --- ...trace-ace-v115b-pure-v74-reality-audit.yml | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/trace-ace-v115b-pure-v74-reality-audit.yml diff --git a/.github/workflows/trace-ace-v115b-pure-v74-reality-audit.yml b/.github/workflows/trace-ace-v115b-pure-v74-reality-audit.yml new file mode 100644 index 00000000..c3af2cd0 --- /dev/null +++ b/.github/workflows/trace-ace-v115b-pure-v74-reality-audit.yml @@ -0,0 +1,45 @@ +name: Trace Ace V115b Pure V74 Reality Audit +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v115b_pure_v74_reality_audit.py' + - '.github/workflows/trace-ace-v115b-pure-v74-reality-audit.yml' + workflow_dispatch: +jobs: + audit: + runs-on: ubuntu-24.04 + timeout-minutes: 8 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download metadata + run: | + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/meta + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + - name: Preflight compile + run: | + cd competitions/trace_the_ace + python -m py_compile v115b_pure_v74_reality_audit.py + - name: Run V115b + run: | + cd competitions/trace_the_ace + python v115b_pure_v74_reality_audit.py --features "../../$FEATURES" --labels "../../$LABELS" --out ../../v115b_pure_v74_reality_audit.json + - name: Show decision + run: cat v115b_pure_v74_reality_audit.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v115b-pure-v74-reality-audit + path: v115b_pure_v74_reality_audit.json + retention-days: 14 From 74fc32901b43f87405f9f92e9f023b46190bd811 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:41:57 +1200 Subject: [PATCH 24/47] Trigger V115b pure V74 audit --- competitions/trace_the_ace/v115b_pure_v74_reality_audit.py | 1 + 1 file changed, 1 insertion(+) diff --git a/competitions/trace_the_ace/v115b_pure_v74_reality_audit.py b/competitions/trace_the_ace/v115b_pure_v74_reality_audit.py index 198d10bb..c9c48302 100644 --- a/competitions/trace_the_ace/v115b_pure_v74_reality_audit.py +++ b/competitions/trace_the_ace/v115b_pure_v74_reality_audit.py @@ -7,6 +7,7 @@ from sklearn.model_selection import GroupKFold from v74_semantic_objective_prior import load_training, semantic_prior_predict EPS=1e-5 +# trigger: pure V74 reality audit def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS))) From baaa58aae6f9cf75cce1d48634b93b2cf61eee5d Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:42:28 +1200 Subject: [PATCH 25/47] No-op trigger V115b --- competitions/trace_the_ace/v115b_pure_v74_reality_audit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/competitions/trace_the_ace/v115b_pure_v74_reality_audit.py b/competitions/trace_the_ace/v115b_pure_v74_reality_audit.py index c9c48302..dba1706c 100644 --- a/competitions/trace_the_ace/v115b_pure_v74_reality_audit.py +++ b/competitions/trace_the_ace/v115b_pure_v74_reality_audit.py @@ -7,7 +7,7 @@ from sklearn.model_selection import GroupKFold from v74_semantic_objective_prior import load_training, semantic_prior_predict EPS=1e-5 -# trigger: pure V74 reality audit +# trigger: pure V74 reality audit v2 def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS))) From a6cf6c494627234112419c4fa6c11aeee3a787ae Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:53:12 +1200 Subject: [PATCH 26/47] Add frozen V74 runtime prediction core --- .../runtime_v74/v74_runtime_core.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 competitions/trace_the_ace/runtime_v74/v74_runtime_core.py diff --git a/competitions/trace_the_ace/runtime_v74/v74_runtime_core.py b/competitions/trace_the_ace/runtime_v74/v74_runtime_core.py new file mode 100644 index 00000000..f487e151 --- /dev/null +++ b/competitions/trace_the_ace/runtime_v74/v74_runtime_core.py @@ -0,0 +1,22 @@ +from __future__ import annotations +import numpy as np +from sklearn.metrics.pairwise import cosine_similarity + +def predict(model, objectives): + obj=[str(x) for x in objectives] + B=model['vectorizer'].transform(obj) + sims=cosine_similarity(B,model['A']) + kk=min(int(model['k']),sims.shape[1]) + idx=np.argpartition(-sims,kth=kk-1,axis=1)[:,:kk] + rows=np.arange(len(obj))[:,None] + w=sims[rows,idx] + npv=model['posterior'][idx] + sem=(w*npv).sum(axis=1)/(w.sum(axis=1)+1e-9) + sem=np.where(w.sum(axis=1)>1e-8,sem,float(model['global_p'])) + mp=model['mapped']; ct=model['counts'] + mapped=np.array([mp.get(x,np.nan) for x in obj],float) + missing=np.isnan(mapped); mapped[missing]=sem[missing] + counts=np.array([ct.get(x,0.0) for x in obj],float) + trust=counts/(counts+10.0) + p=trust*mapped+(1-trust)*sem + return np.clip(p,1e-5,1-1e-5) From 89eab88fbb52652025fda4bd0f50d09c383478ee Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:53:21 +1200 Subject: [PATCH 27/47] Add pure V74 official runtime entrypoint --- competitions/trace_the_ace/runtime_v74/main.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 competitions/trace_the_ace/runtime_v74/main.py diff --git a/competitions/trace_the_ace/runtime_v74/main.py b/competitions/trace_the_ace/runtime_v74/main.py new file mode 100644 index 00000000..04260ebe --- /dev/null +++ b/competitions/trace_the_ace/runtime_v74/main.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +from pathlib import Path +import joblib,pandas as pd +HERE=Path(__file__).resolve().parent +DATA=Path('/code_execution/data') +from v74_runtime_core import predict + +def main(): + f=pd.read_csv(DATA/'test_features.csv') + fmt=pd.read_csv(DATA/'submission_format.csv') + model=joblib.load(HERE/'assets/v74_model.joblib') + p=predict(model,f.learning_objective.astype(str).tolist()) + gen=pd.DataFrame({'response_id':f.response_id.astype(str),'probability':p}) + out=fmt[['response_id']].astype({'response_id':str}).merge(gen,on='response_id',how='left',validate='one_to_one') + if out.probability.isna().any(): raise RuntimeError('missing predictions') + if not ((out.probability>=0)&(out.probability<=1)).all(): raise RuntimeError('invalid probability') + out.to_csv(DATA.parent/'submission.csv',index=False) +if __name__=='__main__': main() From 5ad54ace609d136723e2c40f1c0dcdf29d6b9955 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:53:35 +1200 Subject: [PATCH 28/47] Add frozen V74 runtime asset builder and parity gate --- .../trace_the_ace/train_v74_runtime_assets.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 competitions/trace_the_ace/train_v74_runtime_assets.py diff --git a/competitions/trace_the_ace/train_v74_runtime_assets.py b/competitions/trace_the_ace/train_v74_runtime_assets.py new file mode 100644 index 00000000..0e451863 --- /dev/null +++ b/competitions/trace_the_ace/train_v74_runtime_assets.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +from __future__ import annotations +import argparse,json,hashlib,joblib +from pathlib import Path +import numpy as np +from sklearn.feature_extraction.text import TfidfVectorizer +from sklearn.model_selection import GroupKFold +from v74_semantic_objective_prior import load_training,semantic_prior_predict +from runtime_v74.v74_runtime_core import predict + +def fit_model(df,k=16,smooth=2.0): + global_p=float(df.target.mean()) + stats=df.groupby('learning_objective').target.agg(['sum','count']) + stats['p']=(stats['sum']+smooth*global_p)/(stats['count']+smooth) + objs=stats.index.astype(str).tolist() + vec=TfidfVectorizer(analyzer='char_wb',ngram_range=(3,5),min_df=1,sublinear_tf=True,norm='l2') + A=vec.fit_transform(objs) + return {'k':k,'smooth':smooth,'global_p':global_p,'vectorizer':vec,'A':A,'posterior':stats['p'].to_numpy(float),'mapped':{str(k):float(v) for k,v in stats['p'].items()},'counts':{str(k):float(v) for k,v in stats['count'].items()}} + +def sha(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() +def main(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + y=f.target.to_numpy(int); sess=f.session_id.astype(str).to_numpy() + tr,va=next(iter(GroupKFold(5).split(np.zeros(len(y)),y,sess))) + ref,_=semantic_prior_predict(f.iloc[tr],f.iloc[va],k=16,smooth=2.0) + m=fit_model(f.iloc[tr]); got=predict(m,f.iloc[va].learning_objective.astype(str).tolist()) + md=float(np.max(np.abs(ref-got))) + if md>=1e-8: raise RuntimeError(f'parity failed {md}') + a.assets.mkdir(parents=True,exist_ok=True) + model=fit_model(f); mp=a.assets/'v74_model.joblib'; joblib.dump(model,mp,compress=3) + manifest={'candidate':'V74_PURE_HIERARCHICAL_OBJECTIVE_PRIOR','k':16,'smooth':2.0,'trust_denominator':10.0,'v115b_session_cold':0.5511484894117864,'v115b_objective_cold_stress':0.6013242442331039,'v115b_exact_support_rate':0.9975193886861314,'parity_max_abs_diff':md,'model_sha256':sha(mp)} + (a.assets/'manifest.json').write_text(json.dumps(manifest,indent=2)) + print(json.dumps(manifest,indent=2)) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--assets',type=Path,required=True);main(p.parse_args()) From 6567b546c064cca19cb8bcdee31b9d5e45b19e10 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 20:53:50 +1200 Subject: [PATCH 29/47] Add V74 runtime parity and packaging workflow --- .../trace-ace-v74-runtime-package.yml | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/trace-ace-v74-runtime-package.yml diff --git a/.github/workflows/trace-ace-v74-runtime-package.yml b/.github/workflows/trace-ace-v74-runtime-package.yml new file mode 100644 index 00000000..ecdd367e --- /dev/null +++ b/.github/workflows/trace-ace-v74-runtime-package.yml @@ -0,0 +1,50 @@ +name: Trace Ace V74 Runtime Package +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/runtime_v74/**' + - 'competitions/trace_the_ace/train_v74_runtime_assets.py' + - '.github/workflows/trace-ace-v74-runtime-package.yml' +jobs: + package: + runs-on: ubuntu-24.04 + timeout-minutes: 8 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn joblib gdown + - name: Download metadata + run: | + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/meta + unzip -q metadata.zip -d data/meta + - name: Resolve schemas + run: | + echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV" + echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV" + - name: Preflight + run: | + python -m py_compile competitions/trace_the_ace/runtime_v74/main.py competitions/trace_the_ace/runtime_v74/v74_runtime_core.py competitions/trace_the_ace/train_v74_runtime_assets.py + - name: Train frozen assets and parity test + run: | + python competitions/trace_the_ace/train_v74_runtime_assets.py --features "$FEATURES" --labels "$LABELS" --assets competitions/trace_the_ace/runtime_v74/assets + - name: Package + run: | + rm -rf v74_submission && mkdir -p v74_submission/assets + cp competitions/trace_the_ace/runtime_v74/main.py v74_submission/main.py + cp competitions/trace_the_ace/runtime_v74/v74_runtime_core.py v74_submission/v74_runtime_core.py + cp competitions/trace_the_ace/runtime_v74/assets/* v74_submission/assets/ + (cd v74_submission && zip -q -r ../trace-ace-v74-pure-runtime.zip .) + unzip -l trace-ace-v74-pure-runtime.zip + cat competitions/trace_the_ace/runtime_v74/assets/manifest.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v74-pure-runtime + path: | + trace-ace-v74-pure-runtime.zip + competitions/trace_the_ace/runtime_v74/assets/manifest.json + retention-days: 14 From 1649a02812abc421609de15db9d9ff82ca0b09e9 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:28:31 +1200 Subject: [PATCH 30/47] Add V118 hidden test geometry probe --- .../v118_hidden_test_geometry_probe.py | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 competitions/trace_the_ace/v118_hidden_test_geometry_probe.py diff --git a/competitions/trace_the_ace/v118_hidden_test_geometry_probe.py b/competitions/trace_the_ace/v118_hidden_test_geometry_probe.py new file mode 100644 index 00000000..dbe2fcd1 --- /dev/null +++ b/competitions/trace_the_ace/v118_hidden_test_geometry_probe.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""V118 hidden test geometry probe. + +Frozen historical probe table supplied by actual DrivenData submissions on 2026-08-18. +Question: does smoke rank candidates like public, and what does V74 smoke imply? +No competition labels are used; only user-observed submission scores. +""" +import json +from pathlib import Path +import numpy as np +from scipy.stats import spearmanr, pearsonr + +PROBES = [ + ("V37", 0.4801, 0.7384), + ("V48", 0.5052, 0.6179), + ("V54", 0.5192, 0.6329), + ("V57", 0.5216, 0.6211), + ("V63", 0.4880, 0.6284), + ("V75", 0.4693, 0.6047), + ("V97", 0.4693, 0.6044), +] +V74_SMOKE = 0.5181 +V74_SESSION = 0.5511484894117864 +V74_OBJECTIVE = 0.6013242442331039 + +def metrics(rows): + s=np.array([r[1] for r in rows],float); p=np.array([r[2] for r in rows],float) + sp=spearmanr(s,p); pe=pearsonr(s,p) + b,a=np.polyfit(s,p,1) + pred=float(a+b*V74_SMOKE) + return {"n":len(rows),"spearman":float(sp.statistic),"spearman_p":float(sp.pvalue),"pearson":float(pe.statistic),"pearson_p":float(pe.pvalue),"ols_intercept":float(a),"ols_slope":float(b),"v74_public_projection":pred} + +def main(): + allm=metrics(PROBES) + loo=[] + for i,r in enumerate(PROBES): + m=metrics(PROBES[:i]+PROBES[i+1:]); loo.append({"excluded":r[0],**m}) + best=max(loo,key=lambda x:x["spearman"]) + nonout=[r for r in PROBES if r[0]!=best["excluded"]] + robust=metrics(nonout) + # Ranking only: lower score is better. + smoke_order=[r[0] for r in sorted(PROBES,key=lambda r:r[1])] + public_order=[r[0] for r in sorted(PROBES,key=lambda r:r[2])] + out={ + "probe_table":[{"model":m,"smoke":s,"public":p} for m,s,p in PROBES], + "v74":{"smoke":V74_SMOKE,"session_cold":V74_SESSION,"objective_cold":V74_OBJECTIVE}, + "all_probes":allm, + "leave_one_out":loo, + "most_influential_outlier":best["excluded"], + "robust_without_outlier":robust, + "smoke_rank":smoke_order, + "public_rank":public_order, + } + # Precommit: smoke is not an adequate optimization target if rank rho < .8 or if one probe changes rho by >= .2. + influence=best["spearman"]-allm["spearman"] + if allm["spearman"]<0.8 or influence>=0.2: + verdict="SMOKE_NOT_PUBLIC_PROXY" + else: + verdict="SMOKE_USABLE_PUBLIC_PROXY" + # A V74 full submission is suppressed if robust smoke->public projection is >= current V97 public 0.6044. + v74_decision="SUPPRESS_V74_FULL" if robust["v74_public_projection"]>=0.6044 else "V74_FULL_STILL_PLAUSIBLE" + out["decision"]={ + "verdict":verdict, + "outlier_influence_on_spearman":float(influence), + "v74_decision":v74_decision, + "rule":"Smoke proxy requires Spearman >=0.8 and no single-probe rho improvement >=0.2. Suppress V74 full if robust smoke->public projection is not better than V97 public 0.6044.", + "next":"Infer public-aligned validation geometry from historical candidate response vectors; do not optimize smoke directly." if verdict=="SMOKE_NOT_PUBLIC_PROXY" else "Use smoke as a secondary ranking probe." + } + Path("v118_hidden_test_geometry_probe.json").write_text(json.dumps(out,indent=2)) + print(json.dumps(out,indent=2)) +if __name__=="__main__": main() From e1a5749749a43141b70710211d28d987fb350528 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:28:45 +1200 Subject: [PATCH 31/47] Add V118 hidden test geometry workflow --- .../trace-ace-v118-hidden-test-geometry.yml | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/trace-ace-v118-hidden-test-geometry.yml diff --git a/.github/workflows/trace-ace-v118-hidden-test-geometry.yml b/.github/workflows/trace-ace-v118-hidden-test-geometry.yml new file mode 100644 index 00000000..fa41d7d0 --- /dev/null +++ b/.github/workflows/trace-ace-v118-hidden-test-geometry.yml @@ -0,0 +1,29 @@ +name: Trace Ace V118 Hidden Test Geometry Probe +on: + pull_request: + types: [opened, synchronize, reopened] + branches: [agent/trace-ace-mastery-events] + workflow_dispatch: +jobs: + probe: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install --disable-pip-version-check numpy scipy + - name: Preflight + run: python -m py_compile competitions/trace_the_ace/v118_hidden_test_geometry_probe.py + - name: Run V118 + run: | + cd competitions/trace_the_ace + python v118_hidden_test_geometry_probe.py + - name: Show decision + run: cat competitions/trace_the_ace/v118_hidden_test_geometry_probe.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v118-hidden-test-geometry + path: competitions/trace_the_ace/v118_hidden_test_geometry_probe.json + retention-days: 14 From 9a5cdd3b0c114214d2a4eb3e9b3c8030ba018d01 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:33:32 +1200 Subject: [PATCH 32/47] Add V119 public-anchor geometry search --- .../v119_public_anchor_geometry.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 competitions/trace_the_ace/v119_public_anchor_geometry.py diff --git a/competitions/trace_the_ace/v119_public_anchor_geometry.py b/competitions/trace_the_ace/v119_public_anchor_geometry.py new file mode 100644 index 00000000..c953015f --- /dev/null +++ b/competitions/trace_the_ace/v119_public_anchor_geometry.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""V119 public-anchor geometry search. + +Recoverable-probe experiment. Historical V37/V48/V54/V57/V63 packages are not +available, so this does NOT claim a seven-model public reconstruction. + +Question: which lawful validation strata reproduce the one directly executable +public anchor: V97 slightly beats V75 publicly (0.6044 vs 0.6047), while their +smoke scores tie (0.4693 vs 0.4693)? + +Freeze: +- deterministic 6000-row session sample; +- 5-fold session-grouped OOF; +- V75 and V97 endpoints exactly as current repo lineage; +- cells from support x provider-proxy x session-length x objective-frequency; +- promote a cell only if V97 beats V75 in >=4/5 folds and aggregate delta is + within 0.0005 of the public delta (-0.0003), with >=150 rows. + +No leaderboard labels are used for fitting predictions; public scores are used +only as an external geometry target after OOF predictions are frozen. +""" +from __future__ import annotations +import argparse,json,hashlib,re +from pathlib import Path +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold +from v71_mastery_events import load_transcript,normalize_roles +from v75_canonical_trajectory import load_training,SEED +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control,build_control + +EPS=1e-5; TARGET=-0.0003 + +def hh(x): return int(hashlib.sha256(str(x).encode()).hexdigest()[:16],16) +def ll(y,p): return float(log_loss(y,np.clip(p,EPS,1-EPS))) +def provider(df): + d=normalize_roles(df).reset_index(drop=True); roles=d.role_repaired.astype(str).str.lower().to_numpy(); txt=d.content.fillna('').astype(str).tolist(); n=len(d); tut=int(np.sum(roles=='tutor')); mw=float(np.mean([len(x.split()) for x in txt])) if txt else 0.; markers=sum(bool(re.search(r'\b(?:learning objective|learning goal|prior learning|i do|we do|you do|application|slide|lesson)\b',x,re.I)) for x in txt); return 'TSL' if (n>=24 or markers>=2 or (tut>=12 and mw>=8)) else 'EEDI' +def fit(X,y,tr,va): + m=LogisticRegression(C=.25,max_iter=250,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]); return np.clip(m.predict_proba(X[va])[:,1],EPS,1-EPS) +def bin_support(x): + return 'S0' if x==0 else 'S1_9' if x<10 else 'S10_29' if x<30 else 'S30P' +def qbin(x,cuts,prefix): return prefix+str(int(np.searchsorted(cuts,x,side='right'))) + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True); print('features columns',list(f.columns),flush=True) + sessions=sorted(f.session_id.astype(str).unique(),key=hh); take=set(sessions[:min(a.sessions,len(sessions))]); f=f[f.session_id.astype(str).isin(take)].reset_index(drop=True) + y=f.target.to_numpy(int); sess=f.session_id.astype(str).to_numpy(); key=f.learning_objective.astype(str).to_numpy() + cache={s:load_transcript(a.transcripts/f'{s}.csv') for s in np.unique(sess)}; meta={s:(provider(cache[s]),len(cache[s])) for s in cache} + X75=build_v75(f,cache); rt=[];rz=[] + for _,r in f.iterrows(): t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related');rt.append(t);rz.append(z) + Xr=build_control(rt,rz); p75=np.zeros(len(f));p97=np.zeros(len(f));support=np.zeros(len(f));foldid=np.zeros(len(f),int) + splits=list(GroupKFold(5).split(np.zeros(len(y)),y,sess)) + for k,(tr,va) in enumerate(splits): + q75=fit(X75,y,tr,va); qr=fit(Xr,y,tr,va); vals,cts=np.unique(key[tr],return_counts=True); d=dict(zip(vals,cts)); s=np.array([d.get(x,0) for x in key[va]],float); seen=s>0; q97=np.where(seen,q75,.65*q75+.35*qr); p75[va]=q75;p97[va]=q97;support[va]=s;foldid[va]=k + global_counts=dict(zip(*np.unique(key,return_counts=True))); olen=np.array([global_counts[x] for x in key],float); slen=np.array([meta[s][1] for s in sess],float); prov=np.array([meta[s][0] for s in sess]) + slcuts=np.quantile(slen,[.25,.5,.75]); ocuts=np.quantile(olen,[.25,.5,.75]); cells={} + labels=[] + for i in range(len(f)): + labels.append('|'.join([bin_support(support[i]),prov[i],qbin(slen[i],slcuts,'L'),qbin(olen[i],ocuts,'F')])) + labels=np.array(labels); rows=[] + for c in np.unique(labels): + m=labels==c + if m.sum()<150: continue + d=ll(y[m],p97[m])-ll(y[m],p75[m]); fg=[] + for k in range(5): + z=m&(foldid==k) + fg.append(None if z.sum()<20 else ll(y[z],p97[z])-ll(y[z],p75[z])) + pos=sum(v is not None and v<0 for v in fg); err=abs(d-TARGET); rows.append({'cell':c,'rows':int(m.sum()),'v75':ll(y[m],p75[m]),'v97':ll(y[m],p97[m]),'delta_v97_minus_v75':d,'target_error':err,'fold_deltas':fg,'v97_better_folds':pos,'qualified':bool(pos>=4 and err<=.0005)}) + rows.sort(key=lambda r:(not r['qualified'],r['target_error'],-r['rows'])) + overall={'v75':ll(y,p75),'v97':ll(y,p97),'delta':ll(y,p97)-ll(y,p75)} + q=[r for r in rows if r['qualified']]; verdict='PUBLIC_ANCHOR_CELL_FOUND' if q else 'CURRENT_SPLIT_GRAMMAR_NOT_PUBLIC_ALIGNED' + out={'rows':len(f),'sessions':len(np.unique(sess)),'public_anchor':{'v75':.6047,'v97':.6044,'target_delta':TARGET},'smoke_anchor':{'v75':.4693,'v97':.4693,'delta':0.0},'overall':overall,'top_cells':rows[:20],'decision':{'verdict':verdict,'qualified_cells':len(q),'rule':'Cell requires >=150 rows, V97 better in >=4/5 folds, and aggregate V97-V75 delta within 0.0005 of public -0.0003.','next':'Use qualified cell(s) as public-aligned validation basis only if stable; otherwise expand split grammar beyond support/provider/session-length/objective-frequency.'}} + Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--sessions',type=int,default=4000);p.add_argument('--out',default='v119_public_anchor_geometry.json');run(p.parse_args()) From 3290721fda18f276da574bb7d0eae1f6da5869d3 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:33:46 +1200 Subject: [PATCH 33/47] Add V119 public-anchor geometry workflow --- .../trace-ace-v119-public-anchor-geometry.yml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/trace-ace-v119-public-anchor-geometry.yml diff --git a/.github/workflows/trace-ace-v119-public-anchor-geometry.yml b/.github/workflows/trace-ace-v119-public-anchor-geometry.yml new file mode 100644 index 00000000..e0169371 --- /dev/null +++ b/.github/workflows/trace-ace-v119-public-anchor-geometry.yml @@ -0,0 +1,39 @@ +name: Trace Ace V119 Public Anchor Geometry +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v119_public_anchor_geometry.py' + - '.github/workflows/trace-ace-v119-public-anchor-geometry.yml' +jobs: + geometry: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download frozen data + run: | + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + gdown 1JQIm5CtaX5FJzCPj01Dp-O8XndgBko4c -O transcripts.zip + mkdir -p data/meta data/transcripts + unzip -q metadata.zip -d data/meta + unzip -q transcripts.zip -d data/transcripts + echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV" + echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$(find data/transcripts -type d -name 'train_transcripts*' -print -quit)" >> "$GITHUB_ENV" + - name: Preflight + run: python -m py_compile competitions/trace_the_ace/v119_public_anchor_geometry.py + - name: Run V119 + run: python competitions/trace_the_ace/v119_public_anchor_geometry.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --sessions 4000 --out competitions/trace_the_ace/v119_public_anchor_geometry.json + - name: Show decision + run: cat competitions/trace_the_ace/v119_public_anchor_geometry.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v119-public-anchor-geometry + path: competitions/trace_the_ace/v119_public_anchor_geometry.json + retention-days: 14 From 01065b751e6a571a191e06eaf2bf3629e2b2cd58 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:34:44 +1200 Subject: [PATCH 34/47] Trigger V119 public-anchor geometry --- .../trace_the_ace/v119_public_anchor_geometry.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/competitions/trace_the_ace/v119_public_anchor_geometry.py b/competitions/trace_the_ace/v119_public_anchor_geometry.py index c953015f..15d4ddcf 100644 --- a/competitions/trace_the_ace/v119_public_anchor_geometry.py +++ b/competitions/trace_the_ace/v119_public_anchor_geometry.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# trigger """V119 public-anchor geometry search. Recoverable-probe experiment. Historical V37/V48/V54/V57/V63 packages are not @@ -55,22 +56,18 @@ def run(a): for k,(tr,va) in enumerate(splits): q75=fit(X75,y,tr,va); qr=fit(Xr,y,tr,va); vals,cts=np.unique(key[tr],return_counts=True); d=dict(zip(vals,cts)); s=np.array([d.get(x,0) for x in key[va]],float); seen=s>0; q97=np.where(seen,q75,.65*q75+.35*qr); p75[va]=q75;p97[va]=q97;support[va]=s;foldid[va]=k global_counts=dict(zip(*np.unique(key,return_counts=True))); olen=np.array([global_counts[x] for x in key],float); slen=np.array([meta[s][1] for s in sess],float); prov=np.array([meta[s][0] for s in sess]) - slcuts=np.quantile(slen,[.25,.5,.75]); ocuts=np.quantile(olen,[.25,.5,.75]); cells={} - labels=[] - for i in range(len(f)): - labels.append('|'.join([bin_support(support[i]),prov[i],qbin(slen[i],slcuts,'L'),qbin(olen[i],ocuts,'F')])) + slcuts=np.quantile(slen,[.25,.5,.75]); ocuts=np.quantile(olen,[.25,.5,.75]); labels=[] + for i in range(len(f)): labels.append('|'.join([bin_support(support[i]),prov[i],qbin(slen[i],slcuts,'L'),qbin(olen[i],ocuts,'F')])) labels=np.array(labels); rows=[] for c in np.unique(labels): m=labels==c if m.sum()<150: continue d=ll(y[m],p97[m])-ll(y[m],p75[m]); fg=[] for k in range(5): - z=m&(foldid==k) - fg.append(None if z.sum()<20 else ll(y[z],p97[z])-ll(y[z],p75[z])) + z=m&(foldid==k); fg.append(None if z.sum()<20 else ll(y[z],p97[z])-ll(y[z],p75[z])) pos=sum(v is not None and v<0 for v in fg); err=abs(d-TARGET); rows.append({'cell':c,'rows':int(m.sum()),'v75':ll(y[m],p75[m]),'v97':ll(y[m],p97[m]),'delta_v97_minus_v75':d,'target_error':err,'fold_deltas':fg,'v97_better_folds':pos,'qualified':bool(pos>=4 and err<=.0005)}) rows.sort(key=lambda r:(not r['qualified'],r['target_error'],-r['rows'])) - overall={'v75':ll(y,p75),'v97':ll(y,p97),'delta':ll(y,p97)-ll(y,p75)} - q=[r for r in rows if r['qualified']]; verdict='PUBLIC_ANCHOR_CELL_FOUND' if q else 'CURRENT_SPLIT_GRAMMAR_NOT_PUBLIC_ALIGNED' + overall={'v75':ll(y,p75),'v97':ll(y,p97),'delta':ll(y,p97)-ll(y,p75)}; q=[r for r in rows if r['qualified']]; verdict='PUBLIC_ANCHOR_CELL_FOUND' if q else 'CURRENT_SPLIT_GRAMMAR_NOT_PUBLIC_ALIGNED' out={'rows':len(f),'sessions':len(np.unique(sess)),'public_anchor':{'v75':.6047,'v97':.6044,'target_delta':TARGET},'smoke_anchor':{'v75':.4693,'v97':.4693,'delta':0.0},'overall':overall,'top_cells':rows[:20],'decision':{'verdict':verdict,'qualified_cells':len(q),'rule':'Cell requires >=150 rows, V97 better in >=4/5 folds, and aggregate V97-V75 delta within 0.0005 of public -0.0003.','next':'Use qualified cell(s) as public-aligned validation basis only if stable; otherwise expand split grammar beyond support/provider/session-length/objective-frequency.'}} Path(a.out).write_text(json.dumps(out,indent=2));print(json.dumps(out,indent=2),flush=True) if __name__=='__main__': From cf0c7ccca274157a8f94ad1be35e23dec990413e Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:38:53 +1200 Subject: [PATCH 35/47] Fix V119 transcript bundle to verified frozen source --- .github/workflows/trace-ace-v119-public-anchor-geometry.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/trace-ace-v119-public-anchor-geometry.yml b/.github/workflows/trace-ace-v119-public-anchor-geometry.yml index e0169371..f036cc3c 100644 --- a/.github/workflows/trace-ace-v119-public-anchor-geometry.yml +++ b/.github/workflows/trace-ace-v119-public-anchor-geometry.yml @@ -19,13 +19,14 @@ jobs: - name: Download frozen data run: | gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip - gdown 1JQIm5CtaX5FJzCPj01Dp-O8XndgBko4c -O transcripts.zip + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip mkdir -p data/meta data/transcripts unzip -q metadata.zip -d data/meta unzip -q transcripts.zip -d data/transcripts echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV" echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV" - echo "TRANSCRIPTS=$(find data/transcripts -type d -name 'train_transcripts*' -print -quit)" >> "$GITHUB_ENV" + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + echo "TRANSCRIPTS=$(dirname "$FIRST")" >> "$GITHUB_ENV" - name: Preflight run: python -m py_compile competitions/trace_the_ace/v119_public_anchor_geometry.py - name: Run V119 From f2ede08aea212546f18c78da0b5bafad3f543c2a Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 22:06:56 +1200 Subject: [PATCH 36/47] Use fresh frozen transcript copy for V119 transport --- .github/workflows/trace-ace-v119-public-anchor-geometry.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/trace-ace-v119-public-anchor-geometry.yml b/.github/workflows/trace-ace-v119-public-anchor-geometry.yml index f036cc3c..07ae6bd2 100644 --- a/.github/workflows/trace-ace-v119-public-anchor-geometry.yml +++ b/.github/workflows/trace-ace-v119-public-anchor-geometry.yml @@ -19,7 +19,7 @@ jobs: - name: Download frozen data run: | gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip - gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1UWuHKIJ86yINhQOBGx2fFJ890hiEa6Md -O transcripts.zip mkdir -p data/meta data/transcripts unzip -q metadata.zip -d data/meta unzip -q transcripts.zip -d data/transcripts From 5801e5db7b11f45b30ab9ab83db9e1cf89c11c17 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:06:45 +1200 Subject: [PATCH 37/47] Add V120 objective identity audit --- .../v120_objective_identity_audit.py | 205 ++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 competitions/trace_the_ace/v120_objective_identity_audit.py diff --git a/competitions/trace_the_ace/v120_objective_identity_audit.py b/competitions/trace_the_ace/v120_objective_identity_audit.py new file mode 100644 index 00000000..cfdacaee --- /dev/null +++ b/competitions/trace_the_ace/v120_objective_identity_audit.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""V120 objective identity audit. + +Tests whether learning_objective text and learning_objective_id define the same +identity relation. This is metadata-only and intentionally does not use labels +or transcripts. + +Primary questions: +1. Is id <-> text one-to-one on train? +2. Does exact support on the official test set differ when keyed by id vs text? +3. In session-held-out folds, how much does support geometry differ by key? +4. Are any discrepancies explainable by whitespace/case normalization only? +""" +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + +import numpy as np +import pandas as pd +from sklearn.model_selection import GroupKFold + + +def norm_text(x: object) -> str: + s = str(x) + s = s.strip().casefold() + s = re.sub(r"\s+", " ", s) + return s + + +def clean_id(x: object) -> str: + if pd.isna(x): + return "" + return str(x).strip() + + +def key_stats(df: pd.DataFrame) -> dict: + x = df[["learning_objective_id", "learning_objective"]].copy() + x["oid"] = x["learning_objective_id"].map(clean_id) + x["text"] = x["learning_objective"].astype(str) + x["norm"] = x["learning_objective"].map(norm_text) + + id_to_text = x.groupby("oid")["text"].nunique(dropna=False) + id_to_norm = x.groupby("oid")["norm"].nunique(dropna=False) + text_to_id = x.groupby("text")["oid"].nunique(dropna=False) + norm_to_id = x.groupby("norm")["oid"].nunique(dropna=False) + + pairs_exact = x[["oid", "text"]].drop_duplicates() + pairs_norm = x[["oid", "norm"]].drop_duplicates() + return { + "rows": int(len(x)), + "unique_ids": int(x.oid.nunique()), + "unique_texts": int(x.text.nunique()), + "unique_norm_texts": int(x.norm.nunique()), + "unique_id_text_pairs": int(len(pairs_exact)), + "unique_id_norm_pairs": int(len(pairs_norm)), + "ids_with_multiple_exact_texts": int((id_to_text > 1).sum()), + "ids_with_multiple_norm_texts": int((id_to_norm > 1).sum()), + "exact_texts_with_multiple_ids": int((text_to_id > 1).sum()), + "norm_texts_with_multiple_ids": int((norm_to_id > 1).sum()), + "max_exact_texts_per_id": int(id_to_text.max()) if len(id_to_text) else 0, + "max_ids_per_exact_text": int(text_to_id.max()) if len(text_to_id) else 0, + "max_norm_texts_per_id": int(id_to_norm.max()) if len(id_to_norm) else 0, + "max_ids_per_norm_text": int(norm_to_id.max()) if len(norm_to_id) else 0, + } + + +def support_against(train: pd.DataFrame, test: pd.DataFrame) -> dict: + tr_id = set(train.learning_objective_id.map(clean_id)) + tr_text = set(train.learning_objective.astype(str)) + tr_norm = set(train.learning_objective.map(norm_text)) + + te_id = test.learning_objective_id.map(clean_id) + te_text = test.learning_objective.astype(str) + te_norm = test.learning_objective.map(norm_text) + + seen_id = te_id.isin(tr_id).to_numpy(bool) + seen_text = te_text.isin(tr_text).to_numpy(bool) + seen_norm = te_norm.isin(tr_norm).to_numpy(bool) + + return { + "rows": int(len(test)), + "seen_by_id_rate": float(seen_id.mean()), + "seen_by_exact_text_rate": float(seen_text.mean()), + "seen_by_norm_text_rate": float(seen_norm.mean()), + "id_seen_text_unseen": int(np.sum(seen_id & ~seen_text)), + "text_seen_id_unseen": int(np.sum(seen_text & ~seen_id)), + "id_seen_norm_unseen": int(np.sum(seen_id & ~seen_norm)), + "norm_seen_id_unseen": int(np.sum(seen_norm & ~seen_id)), + "id_vs_exact_gate_disagreement_rate": float(np.mean(seen_id != seen_text)), + "id_vs_norm_gate_disagreement_rate": float(np.mean(seen_id != seen_norm)), + } + + +def session_cv_support(df: pd.DataFrame, folds: int = 4) -> dict: + groups = df.session_id.astype(str).to_numpy() + gkf = GroupKFold(n_splits=folds) + seen_id = np.zeros(len(df), dtype=bool) + seen_text = np.zeros(len(df), dtype=bool) + seen_norm = np.zeros(len(df), dtype=bool) + fold_rows = [] + + for fold, (tr, va) in enumerate(gkf.split(df, groups=groups), 1): + a = support_against(df.iloc[tr], df.iloc[va]) + trf = df.iloc[tr] + vaf = df.iloc[va] + tr_ids = set(trf.learning_objective_id.map(clean_id)) + tr_txt = set(trf.learning_objective.astype(str)) + tr_nrm = set(trf.learning_objective.map(norm_text)) + seen_id[va] = vaf.learning_objective_id.map(clean_id).isin(tr_ids) + seen_text[va] = vaf.learning_objective.astype(str).isin(tr_txt) + seen_norm[va] = vaf.learning_objective.map(norm_text).isin(tr_nrm) + fold_rows.append({"fold": fold, **a}) + + return { + "folds": folds, + "overall_seen_by_id_rate": float(seen_id.mean()), + "overall_seen_by_exact_text_rate": float(seen_text.mean()), + "overall_seen_by_norm_text_rate": float(seen_norm.mean()), + "id_vs_exact_gate_disagreement_rate": float(np.mean(seen_id != seen_text)), + "id_vs_norm_gate_disagreement_rate": float(np.mean(seen_id != seen_norm)), + "fold_details": fold_rows, + } + + +def examples(df: pd.DataFrame, limit: int = 12) -> dict: + x = df[["learning_objective_id", "learning_objective"]].copy() + x["oid"] = x.learning_objective_id.map(clean_id) + x["text"] = x.learning_objective.astype(str) + x["norm"] = x.learning_objective.map(norm_text) + + id_multi = ( + x.groupby("oid")["text"].agg(lambda s: sorted(set(s))) + .loc[lambda s: s.map(len) > 1] + .head(limit) + ) + text_multi = ( + x.groupby("text")["oid"].agg(lambda s: sorted(set(s))) + .loc[lambda s: s.map(len) > 1] + .head(limit) + ) + return { + "ids_with_multiple_texts": {str(k): v for k, v in id_multi.items()}, + "texts_with_multiple_ids": {str(k): v for k, v in text_multi.items()}, + } + + +def main(a: argparse.Namespace) -> None: + train = pd.read_csv(a.train_features) + required = {"response_id", "session_id", "learning_objective_id", "learning_objective"} + missing = required - set(train.columns) + if missing: + raise SystemExit(f"missing train columns: {sorted(missing)}") + + print("train columns", list(train.columns), flush=True) + out = { + "protocol": "V120_OBJECTIVE_IDENTITY_AUDIT", + "train": key_stats(train), + "session_cv_support": session_cv_support(train, folds=a.folds), + "examples": examples(train), + } + + if a.test_features is not None and a.test_features.exists(): + test = pd.read_csv(a.test_features) + missing = required - set(test.columns) + if missing: + raise SystemExit(f"missing test columns: {sorted(missing)}") + print("test columns", list(test.columns), flush=True) + out["test"] = key_stats(test) + out["official_test_support"] = support_against(train, test) + + tr = out["train"] + test_support = out.get("official_test_support", {}) + structurally_same = ( + tr["ids_with_multiple_norm_texts"] == 0 + and tr["norm_texts_with_multiple_ids"] == 0 + and test_support.get("id_vs_norm_gate_disagreement_rate", 0.0) == 0.0 + ) + if structurally_same: + verdict = "ID_TEXT_EQUIVALENT_FOR_SUPPORT" + else: + verdict = "ID_TEXT_NOT_EQUIVALENT" + + out["decision"] = { + "verdict": verdict, + "next": ( + "Kill identity hypothesis; move to semantic residual representation." + if structurally_same + else "Rerun support gate and validation keyed by canonical learning_objective_id before semantic escalation." + ), + } + + Path(a.out).write_text(json.dumps(out, indent=2, ensure_ascii=False)) + print(json.dumps(out, indent=2, ensure_ascii=False), flush=True) + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--train-features", type=Path, required=True) + p.add_argument("--test-features", type=Path, default=None) + p.add_argument("--folds", type=int, default=4) + p.add_argument("--out", default="v120_objective_identity_audit.json") + main(p.parse_args()) From aba72dfa8aa57eb37264915168fec4672098db4b Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:06:58 +1200 Subject: [PATCH 38/47] Run V120 objective identity audit --- .../trace-ace-v120-objective-identity.yml | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/trace-ace-v120-objective-identity.yml diff --git a/.github/workflows/trace-ace-v120-objective-identity.yml b/.github/workflows/trace-ace-v120-objective-identity.yml new file mode 100644 index 00000000..6488e7fb --- /dev/null +++ b/.github/workflows/trace-ace-v120-objective-identity.yml @@ -0,0 +1,54 @@ +name: Trace Ace V120 Objective Identity Audit + +on: + push: + branches: [agent/v111-runner] + paths: + - 'competitions/trace_the_ace/v120_objective_identity_audit.py' + - '.github/workflows/trace-ace-v120-objective-identity.yml' + workflow_dispatch: + +jobs: + audit: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scikit-learn gdown + - name: Download metadata only + run: | + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/meta + unzip -q metadata.zip -d data/meta + TRAIN=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + TEST=$(find data/meta -type f \( -name 'test_features*.csv' -o -name 'submission_features*.csv' \) -print -quit) + echo "TRAIN=$TRAIN" >> "$GITHUB_ENV" + echo "TEST=$TEST" >> "$GITHUB_ENV" + echo "TRAIN=$TRAIN" + echo "TEST=$TEST" + - name: Inspect headers and run V120 + run: | + python - <<'PY' + import os, pandas as pd + tr=os.environ['TRAIN'] + print('train shape/columns', pd.read_csv(tr,nrows=3).shape, list(pd.read_csv(tr,nrows=0).columns)) + te=os.environ.get('TEST','') + if te: + print('test columns', list(pd.read_csv(te,nrows=0).columns)) + PY + if [ -n "$TEST" ]; then + python competitions/trace_the_ace/v120_objective_identity_audit.py --train-features "$TRAIN" --test-features "$TEST" --out v120_objective_identity_audit.json + else + python competitions/trace_the_ace/v120_objective_identity_audit.py --train-features "$TRAIN" --out v120_objective_identity_audit.json + fi + - name: Show decision + run: cat v120_objective_identity_audit.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v120-objective-identity-audit + path: v120_objective_identity_audit.json + retention-days: 14 From 9c6c174b3e721edbd8b47267feab3efa1568d08d Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:07:25 +1200 Subject: [PATCH 39/47] Route V120 through PR runner --- .github/workflows/trace-ace-v120-objective-identity.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/trace-ace-v120-objective-identity.yml b/.github/workflows/trace-ace-v120-objective-identity.yml index 6488e7fb..e7bd4c91 100644 --- a/.github/workflows/trace-ace-v120-objective-identity.yml +++ b/.github/workflows/trace-ace-v120-objective-identity.yml @@ -6,6 +6,11 @@ on: paths: - 'competitions/trace_the_ace/v120_objective_identity_audit.py' - '.github/workflows/trace-ace-v120-objective-identity.yml' + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v120_objective_identity_audit.py' + - '.github/workflows/trace-ace-v120-objective-identity.yml' workflow_dispatch: jobs: @@ -35,7 +40,7 @@ jobs: python - <<'PY' import os, pandas as pd tr=os.environ['TRAIN'] - print('train shape/columns', pd.read_csv(tr,nrows=3).shape, list(pd.read_csv(tr,nrows=0).columns)) + print('train columns', list(pd.read_csv(tr,nrows=0).columns)) te=os.environ.get('TEST','') if te: print('test columns', list(pd.read_csv(te,nrows=0).columns)) From a7ecc678841f40e7566451594f5bbe71b78e63b2 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:10:03 +1200 Subject: [PATCH 40/47] Add V121 pretrained semantic residual test --- .../v121_pretrained_semantic_residual.py | 244 ++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 competitions/trace_the_ace/v121_pretrained_semantic_residual.py diff --git a/competitions/trace_the_ace/v121_pretrained_semantic_residual.py b/competitions/trace_the_ace/v121_pretrained_semantic_residual.py new file mode 100644 index 00000000..5a51b41d --- /dev/null +++ b/competitions/trace_the_ace/v121_pretrained_semantic_residual.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""V121 PRETRAINED SEMANTIC RESIDUAL TEST. + +Qualitatively different from V112: fixed pretrained neural text embeddings rather +than hashed n-gram features. Tests whether semantic transcript/objective +representation adds row-local information beyond V97. + +Frozen intervention: +- deterministic 2500-row sample, same hashing convention as V112 +- pretrained jinaai/jina-embeddings-v2-small-en via FastEmbed 0.8.0 +- three representations: objective-only control, objective+local/recent/student + semantic intervention, and within-objective shuffled semantic ablation +- evaluate both objective-grouped and session-grouped 4-fold OOF +- evaluate a fixed hard-collision subset: an opposite-label same-objective row + exists within |p97_i-p97_j| <= 0.01 +- no hyperparameter sweep + +Precommit: +PHASE_CHANGE if semantic gain >= .003 on BOTH split geometries, semantic beats +within-objective shuffled by >= .002 on BOTH, and hard-collision gain is positive +on BOTH. Otherwise do not promote. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +import numpy as np +from fastembed import TextEmbedding +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GroupKFold + +from v110_residual_collider_state_discovery import hb, ll, logit, p97_predict +from v112_fast_raw_observable_screen import texts +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control, build_control + +EPS = 1e-5 +MODEL_NAME = "jinaai/jina-embeddings-v2-small-en" + + +def stable_hash(x: object) -> int: + return int(hashlib.sha256(str(x).encode()).hexdigest()[:16], 16) + + +def build_semantic_text(objective: str, transcript_df) -> str: + student, _tutor, _full, local, last8 = texts(transcript_df, objective) + # Keep a bounded tail while preserving the target segment and most recent turns. + student_tail = student[-9000:] + local_tail = local[-12000:] + recent_tail = last8[-5000:] + return ( + f"learning objective: {objective}\n" + f"target tutoring context: {local_tail}\n" + f"recent turns: {recent_tail}\n" + f"student evidence: {student_tail}" + ) + + +def embed(model: TextEmbedding, seq: list[str]) -> np.ndarray: + arr = np.vstack(list(model.embed(seq, batch_size=64))).astype(np.float32) + if not np.isfinite(arr).all(): + raise RuntimeError("non-finite embedding") + return arr + + +def p97_oof(X75, Xr, y, groups, support): + q = np.zeros(len(y), dtype=float) + splits = list(GroupKFold(min(4, len(np.unique(groups)))).split(np.zeros(len(y)), y, groups)) + for tr, va in splits: + q[va], _ = p97_predict(X75, Xr, y, tr, va, support) + return np.clip(q, EPS, 1 - EPS), splits + + +def residual_oof(P, E, y, splits): + q = np.zeros(len(y), dtype=float) + for tr, va in splits: + Xtr = np.c_[logit(P[tr]), E[tr]] + Xva = np.c_[logit(P[va]), E[va]] + m = LogisticRegression( + C=0.05, + max_iter=300, + solver="liblinear", + random_state=SEED, + ).fit(Xtr, y[tr]) + q[va] = m.predict_proba(Xva)[:, 1] + return np.clip(q, EPS, 1 - EPS) + + +def within_objective_shuffle(E: np.ndarray, objectives: np.ndarray) -> np.ndarray: + out = E.copy() + rng = np.random.default_rng(SEED + 121) + for o in np.unique(objectives): + z = np.where(objectives == o)[0] + if len(z) > 1: + out[z] = E[rng.permutation(z)] + return out + + +def collision_mask(P: np.ndarray, y: np.ndarray, objectives: np.ndarray, tol: float = 0.01) -> np.ndarray: + mask = np.zeros(len(y), dtype=bool) + for o in np.unique(objectives): + z = np.where(objectives == o)[0] + a0 = z[y[z] == 0] + a1 = z[y[z] == 1] + if len(a0) == 0 or len(a1) == 0: + continue + p0 = P[a0] + p1 = P[a1] + # sample is only 2500 rows, so explicit pairwise distances are small. + D = np.abs(p0[:, None] - p1[None, :]) + mask[a0[np.min(D, axis=1) <= tol]] = True + mask[a1[np.min(D, axis=0) <= tol]] = True + return mask + + +def eval_geometry(name, groups, X75, Xr, y, support, objectives, E_obj, E_sem, E_shuf): + P, splits = p97_oof(X75, Xr, y, groups, support) + Qobj = residual_oof(P, E_obj, y, splits) + Qsem = residual_oof(P, E_sem, y, splits) + Qsh = residual_oof(P, E_shuf, y, splits) + base = ll(y, P) + mask = collision_mask(P, y, objectives, tol=0.01) + out = { + "geometry": name, + "rows": int(len(y)), + "groups": int(len(np.unique(groups))), + "baseline_v97_ll": float(base), + "objective_only": {"ll": float(ll(y, Qobj)), "gain": float(base - ll(y, Qobj))}, + "semantic": {"ll": float(ll(y, Qsem)), "gain": float(base - ll(y, Qsem))}, + "semantic_shuffled_within_objective": {"ll": float(ll(y, Qsh)), "gain": float(base - ll(y, Qsh))}, + "semantic_minus_shuffle_gain": float(ll(y, Qsh) - ll(y, Qsem)), + "hard_collision": {"rows": int(mask.sum())}, + } + if mask.any(): + b = ll(y[mask], P[mask]) + s = ll(y[mask], Qsem[mask]) + sh = ll(y[mask], Qsh[mask]) + out["hard_collision"].update({ + "baseline_ll": float(b), + "semantic_ll": float(s), + "semantic_gain": float(b - s), + "shuffled_ll": float(sh), + "semantic_minus_shuffle_gain": float(sh - s), + }) + return out + + +def main(a): + f = load_training(a.features, a.labels).reset_index(drop=True) + print("features columns", list(f.columns), flush=True) + obj0 = (f.learning_objective_id if "learning_objective_id" in f else f.learning_objective).astype(str).to_numpy() + cand = np.where(np.array([hb(x, 5) != 0 for x in obj0]))[0] + ix = np.array(sorted(cand, key=lambda i: stable_hash(f.response_id.iloc[i]))[: a.rows]) + f = f.iloc[ix].reset_index(drop=True) + + y = f.target.to_numpy(int) + objectives = (f.learning_objective_id if "learning_objective_id" in f else f.learning_objective).astype(str).to_numpy() + support = f.learning_objective.astype(str).to_numpy() + sessions = f.session_id.astype(str).to_numpy() + + cache = {s: load_transcript(a.transcripts / f"{s}.csv") for s in np.unique(sessions)} + rt, rz = [], [] + sem_text, obj_text = [], [] + for i, r in f.iterrows(): + d = cache[str(r.session_id)] + t, z = segmented_control(d, str(r.learning_objective), "related") + rt.append(t) + rz.append(z) + obj_text.append(f"learning objective: {r.learning_objective}") + sem_text.append(build_semantic_text(str(r.learning_objective), d)) + if (i + 1) % 500 == 0: + print("prepared rows", i + 1, flush=True) + + X75 = build_v75(f, cache) + Xr = build_control(rt, rz) + + print("loading embedding model", MODEL_NAME, flush=True) + model = TextEmbedding(model_name=MODEL_NAME) + print("embedding objective control", flush=True) + E_obj = embed(model, obj_text) + print("embedding semantic intervention", flush=True) + E_sem = embed(model, sem_text) + E_shuf = within_objective_shuffle(E_sem, objectives) + print("embedding shapes", E_obj.shape, E_sem.shape, flush=True) + + results = { + "protocol": "V121_PRETRAINED_SEMANTIC_RESIDUAL", + "model": MODEL_NAME, + "rows": int(len(f)), + "objectives": int(len(np.unique(objectives))), + "sessions": int(len(np.unique(sessions))), + "precommit": { + "semantic_gain_each_geometry": 0.003, + "semantic_minus_shuffle_each_geometry": 0.002, + "hard_collision_gain_each_geometry": ">0", + "no_hyperparameter_sweep": True, + }, + } + results["objective_grouped"] = eval_geometry( + "objective_grouped", objectives, X75, Xr, y, support, objectives, E_obj, E_sem, E_shuf + ) + results["session_grouped"] = eval_geometry( + "session_grouped", sessions, X75, Xr, y, support, objectives, E_obj, E_sem, E_shuf + ) + + def passes(r): + return ( + r["semantic"]["gain"] >= 0.003 + and r["semantic_minus_shuffle_gain"] >= 0.002 + and r["hard_collision"].get("semantic_gain", -1.0) > 0.0 + ) + + ok_obj = passes(results["objective_grouped"]) + ok_sess = passes(results["session_grouped"]) + if ok_obj and ok_sess: + verdict = "PHASE_CHANGE_CANDIDATE" + nxt = "Promote pretrained semantic residual to larger frozen validation and public-probe packaging." + else: + verdict = "NO_ROBUST_SEMANTIC_PHASE_CHANGE" + nxt = "Treat remaining oracle gap as largely unidentifiable from supplied transcript/objective observables; pivot to validation geometry / assessment-process inference rather than more text feature search." + results["decision"] = { + "objective_grouped_pass": bool(ok_obj), + "session_grouped_pass": bool(ok_sess), + "verdict": verdict, + "next": nxt, + } + + Path(a.out).write_text(json.dumps(results, indent=2)) + print(json.dumps(results, indent=2), flush=True) + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--features", type=Path, required=True) + p.add_argument("--labels", type=Path, required=True) + p.add_argument("--transcripts", type=Path, required=True) + p.add_argument("--rows", type=int, default=2500) + p.add_argument("--out", default="v121_pretrained_semantic_residual.json") + main(p.parse_args()) From 8ee450742c542e6c2dc32549c35e8640e6050798 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:10:16 +1200 Subject: [PATCH 41/47] Run V121 pretrained semantic residual test --- ...-ace-v121-pretrained-semantic-residual.yml | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 .github/workflows/trace-ace-v121-pretrained-semantic-residual.yml diff --git a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml new file mode 100644 index 00000000..51f92de9 --- /dev/null +++ b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml @@ -0,0 +1,68 @@ +name: Trace Ace V121 Pretrained Semantic Residual + +on: + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v121_pretrained_semantic_residual.py' + - '.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml' + workflow_dispatch: + +jobs: + semantic: + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: | + python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown fastembed==0.8.0 + - name: Download frozen metadata and transcripts + run: | + set -e + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + if ! gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip; then + echo 'primary frozen transcript transport failed; trying frozen mirror' + gdown 1UWuHKIJ86yINhQOBGx2fFJ890hiEa6Md -O transcripts.zip + fi + sha256sum metadata.zip transcripts.zip + mkdir -p data/meta data/transcripts + unzip -q metadata.zip -d data/meta + unzip -q transcripts.zip -d data/transcripts + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Preflight + run: | + python -m py_compile competitions/trace_the_ace/v121_pretrained_semantic_residual.py + python - <<'PY' + import os, pandas as pd + print('features columns', list(pd.read_csv(os.environ['FEATURES'], nrows=0).columns)) + print('labels columns', list(pd.read_csv(os.environ['LABELS'], nrows=0).columns)) + PY + - name: Run V121 + run: | + cd competitions/trace_the_ace + python v121_pretrained_semantic_residual.py \ + --features "../../$FEATURES" \ + --labels "../../$LABELS" \ + --transcripts "../../$TRANSCRIPTS" \ + --rows 2500 \ + --out ../../v121_pretrained_semantic_residual.json + - name: Show decision + run: cat v121_pretrained_semantic_residual.json + - uses: actions/upload-artifact@v4 + if: always() + with: + name: trace-ace-v121-pretrained-semantic-residual + path: v121_pretrained_semantic_residual.json + retention-days: 14 + if-no-files-found: warn From 539755aecfdd13b5d12f0c7074fe9ada3fa34aa0 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:57:27 +1200 Subject: [PATCH 42/47] Add V122 metadata-only ID morphology regime audit --- .../v122_id_morphology_regime_audit.py | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 competitions/trace_the_ace/v122_id_morphology_regime_audit.py diff --git a/competitions/trace_the_ace/v122_id_morphology_regime_audit.py b/competitions/trace_the_ace/v122_id_morphology_regime_audit.py new file mode 100644 index 00000000..9155a9f8 --- /dev/null +++ b/competitions/trace_the_ace/v122_id_morphology_regime_audit.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""V122 — metadata-only ID morphology regime audit. + +Question: do session/objective identifier string patterns encode a stable provider / assessment-process +regime that could explain leaderboard behavior? This is deliberately metadata-only and fast. + +Frozen protocol: +- inspect headers first +- join train features/labels by response_id +- evaluate char-ngram morphology of session_id, learning_objective_id, and both together +- two untouched geometries: GroupKFold by session_id and GroupKFold by learning_objective_id +- compare against intercept-only fold baseline +- shuffled-label control with same folds +- promotion only if a real ID family gains >= .003 log loss in BOTH geometries and exceeds shuffle by >= .002 +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +import pandas as pd +from scipy.sparse import hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.model_selection import GroupKFold +from sklearn.metrics import log_loss + +EPS=1e-5 +SEED=20260818 + +def fold_intercept(ytr,n): + p=float(np.clip(np.mean(ytr),EPS,1-EPS)); return np.full(n,p,float) + +def oof(texts,y,groups): + q=np.zeros(len(y),float); b=np.zeros(len(y),float) + hv=HashingVectorizer(analyzer='char',ngram_range=(2,5),n_features=2**15,alternate_sign=False,norm='l2',lowercase=False) + X=hv.transform(texts) + k=min(5,len(np.unique(groups))) + for tr,va in GroupKFold(k).split(X,y,groups): + b[va]=fold_intercept(y[tr],len(va)) + m=LogisticRegression(C=.15,max_iter=220,solver='liblinear',random_state=SEED) + m.fit(X[tr],y[tr]); q[va]=m.predict_proba(X[va])[:,1] + return float(log_loss(y,np.clip(q,EPS,1-EPS))), float(log_loss(y,np.clip(b,EPS,1-EPS))) + +def run(a): + print('train_features headers',list(pd.read_csv(a.features,nrows=0).columns),flush=True) + print('train_labels headers',list(pd.read_csv(a.labels,nrows=0).columns),flush=True) + f=pd.read_csv(a.features); l=pd.read_csv(a.labels) + f=f.merge(l,on='response_id',how='inner',validate='one_to_one') + y=f.is_correct.to_numpy(int) + sess=f.session_id.astype(str).to_numpy(); oid=f.learning_objective_id.astype(str).to_numpy() + families={ + 'SESSION_ID':np.array(['S:'+x for x in sess],object), + 'OBJECTIVE_ID':np.array(['O:'+x for x in oid],object), + 'SESSION_X_OBJECTIVE':np.array(['S:'+s+'|O:'+o for s,o in zip(sess,oid)],object), + } + geoms={'session_cold':sess,'objective_cold':oid} + rng=np.random.default_rng(SEED); ys=y.copy(); rng.shuffle(ys) + out={'rows':int(len(f)),'sessions':int(len(np.unique(sess))),'objectives':int(len(np.unique(oid))), 'families':{}, 'shuffle':{}} + gains=[] + for name,txt in families.items(): + out['families'][name]={} + out['shuffle'][name]={} + for gname,g in geoms.items(): + ll,base=oof(txt,y,g); sll,sbase=oof(txt,ys,g) + gain=base-ll; sgain=sbase-sll + out['families'][name][gname]={'ll':ll,'baseline_ll':base,'gain':gain} + out['shuffle'][name][gname]={'gain':sgain} + g1=out['families'][name]['session_cold']['gain']; g2=out['families'][name]['objective_cold']['gain'] + sh=max(out['shuffle'][name]['session_cold']['gain'],out['shuffle'][name]['objective_cold']['gain']) + gains.append((min(g1,g2)-sh,name,g1,g2,sh)) + gains.sort(reverse=True) + margin,name,g1,g2,sh=gains[0] + promote=(g1>=.003 and g2>=.003 and min(g1,g2)-sh>=.002) + out['decision']={'winner':name,'session_gain':g1,'objective_gain':g2,'max_shuffle_gain':sh,'margin':margin, + 'verdict':'ID_REGIME_SIGNAL' if promote else 'ID_MORPHOLOGY_NOT_DECISION_CHANGING', + 'rule':'Promote only if >=.003 gain in both geometries and >=.002 above shuffled control.'} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2),flush=True) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--out',default='v122_id_morphology_regime_audit.json'); run(p.parse_args()) From a9a5b7a994d6878313dbf050959b811f15d137a0 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:57:40 +1200 Subject: [PATCH 43/47] Add V122 ID morphology audit workflow --- .../trace-ace-v122-id-morphology.yml | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/trace-ace-v122-id-morphology.yml diff --git a/.github/workflows/trace-ace-v122-id-morphology.yml b/.github/workflows/trace-ace-v122-id-morphology.yml new file mode 100644 index 00000000..6f37e286 --- /dev/null +++ b/.github/workflows/trace-ace-v122-id-morphology.yml @@ -0,0 +1,40 @@ +name: Trace Ace V122 ID Morphology Audit + +on: + push: + branches: [agent/v111-runner] + paths: + - 'competitions/trace_the_ace/v122_id_morphology_regime_audit.py' + - '.github/workflows/trace-ace-v122-id-morphology.yml' + workflow_dispatch: + +jobs: + audit: + runs-on: ubuntu-24.04 + timeout-minutes: 8 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download metadata only + run: | + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/meta + unzip -q metadata.zip -d data/meta + echo "FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit)" >> "$GITHUB_ENV" + echo "LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit)" >> "$GITHUB_ENV" + - name: Preflight + run: python -m py_compile competitions/trace_the_ace/v122_id_morphology_regime_audit.py + - name: Run V122 + run: python competitions/trace_the_ace/v122_id_morphology_regime_audit.py --features "$FEATURES" --labels "$LABELS" --out v122_id_morphology_regime_audit.json + - name: Show decision + run: cat v122_id_morphology_regime_audit.json + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v122-id-morphology-regime-audit + path: v122_id_morphology_regime_audit.json + retention-days: 14 From a5c57fff50e98b94e67a428ec5dd84ff2f6d3d0b Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:58:40 +1200 Subject: [PATCH 44/47] Enable PR trigger for V122 audit --- .github/workflows/trace-ace-v122-id-morphology.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/trace-ace-v122-id-morphology.yml b/.github/workflows/trace-ace-v122-id-morphology.yml index 6f37e286..44cf7104 100644 --- a/.github/workflows/trace-ace-v122-id-morphology.yml +++ b/.github/workflows/trace-ace-v122-id-morphology.yml @@ -6,6 +6,11 @@ on: paths: - 'competitions/trace_the_ace/v122_id_morphology_regime_audit.py' - '.github/workflows/trace-ace-v122-id-morphology.yml' + pull_request: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v122_id_morphology_regime_audit.py' + - '.github/workflows/trace-ace-v122-id-morphology.yml' workflow_dispatch: jobs: From 9946718a8bff8ff33c550bf65d946620402bbcf7 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Wed, 19 Aug 2026 00:00:16 +1200 Subject: [PATCH 45/47] Carry V122 metadata audit in proven V120 workflow --- .../trace-ace-v120-objective-identity.yml | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/trace-ace-v120-objective-identity.yml b/.github/workflows/trace-ace-v120-objective-identity.yml index e7bd4c91..f085e9e4 100644 --- a/.github/workflows/trace-ace-v120-objective-identity.yml +++ b/.github/workflows/trace-ace-v120-objective-identity.yml @@ -5,17 +5,20 @@ on: branches: [agent/v111-runner] paths: - 'competitions/trace_the_ace/v120_objective_identity_audit.py' + - 'competitions/trace_the_ace/v122_id_morphology_regime_audit.py' - '.github/workflows/trace-ace-v120-objective-identity.yml' pull_request: branches: [agent/trace-ace-mastery-events] paths: - 'competitions/trace_the_ace/v120_objective_identity_audit.py' + - 'competitions/trace_the_ace/v122_id_morphology_regime_audit.py' - '.github/workflows/trace-ace-v120-objective-identity.yml' workflow_dispatch: jobs: audit: runs-on: ubuntu-24.04 + timeout-minutes: 8 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -23,17 +26,20 @@ jobs: python-version: '3.12' cache: pip - name: Install dependencies - run: python -m pip install --disable-pip-version-check numpy pandas scikit-learn gdown + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown - name: Download metadata only run: | gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip mkdir -p data/meta unzip -q metadata.zip -d data/meta TRAIN=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) TEST=$(find data/meta -type f \( -name 'test_features*.csv' -o -name 'submission_features*.csv' \) -print -quit) echo "TRAIN=$TRAIN" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" echo "TEST=$TEST" >> "$GITHUB_ENV" echo "TRAIN=$TRAIN" + echo "LABELS=$LABELS" echo "TEST=$TEST" - name: Inspect headers and run V120 run: | @@ -50,10 +56,21 @@ jobs: else python competitions/trace_the_ace/v120_objective_identity_audit.py --train-features "$TRAIN" --out v120_objective_identity_audit.json fi - - name: Show decision - run: cat v120_objective_identity_audit.json + - name: Run V122 independent metadata audit + run: | + python -m py_compile competitions/trace_the_ace/v122_id_morphology_regime_audit.py + python competitions/trace_the_ace/v122_id_morphology_regime_audit.py --features "$TRAIN" --labels "$LABELS" --out v122_id_morphology_regime_audit.json + - name: Show decisions + run: | + cat v120_objective_identity_audit.json + cat v122_id_morphology_regime_audit.json - uses: actions/upload-artifact@v4 with: name: trace-ace-v120-objective-identity-audit path: v120_objective_identity_audit.json retention-days: 14 + - uses: actions/upload-artifact@v4 + with: + name: trace-ace-v122-id-morphology-regime-audit + path: v122_id_morphology_regime_audit.json + retention-days: 14 From 128c70b55cc3580627cd1f140ddc95680769b13e Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Wed, 19 Aug 2026 00:02:37 +1200 Subject: [PATCH 46/47] Record V122 ID morphology negative law --- .../v122_id_morphology_regime_audit.json | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 competitions/trace_the_ace/results/v122_id_morphology_regime_audit.json diff --git a/competitions/trace_the_ace/results/v122_id_morphology_regime_audit.json b/competitions/trace_the_ace/results/v122_id_morphology_regime_audit.json new file mode 100644 index 00000000..c522e7e3 --- /dev/null +++ b/competitions/trace_the_ace/results/v122_id_morphology_regime_audit.json @@ -0,0 +1,31 @@ +{ + "experiment": "V122_ID_MORPHOLOGY_REGIME_AUDIT", + "status": "COMPLETED_LOCAL_FROM_AUTHENTICATED_FROZEN_METADATA", + "frozen_metadata_file_id": "1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz", + "rows": 35072, + "sessions": 22821, + "objectives": 398, + "families": { + "SESSION_ID": { + "session_cold": {"ll": 0.61001884715157, "baseline_ll": 0.6087712429519364, "gain": -0.0012476041996335452}, + "objective_cold": {"ll": 0.6108649000534013, "baseline_ll": 0.6117153814673667, "gain": 0.000850481413965376} + }, + "OBJECTIVE_ID": { + "session_cold": {"ll": 0.5538378377601135, "baseline_ll": 0.6087712429519364, "gain": 0.05493340519182288}, + "objective_cold": {"ll": 0.6144847625399319, "baseline_ll": 0.6117153814673667, "gain": -0.0027693810725651913} + }, + "SESSION_X_OBJECTIVE": { + "session_cold": {"ll": 0.5594418013741027, "baseline_ll": 0.6087712429519364, "gain": 0.0493294415778337}, + "objective_cold": {"ll": 0.614578017753569, "baseline_ll": 0.6117153814673667, "gain": -0.0028626362862023136} + } + }, + "shuffle": { + "SESSION_ID": {"session_cold_gain": -0.0017297072836235383, "objective_cold_gain": -0.0014020282276541174}, + "OBJECTIVE_ID": {"session_cold_gain": -0.0015091921768788374, "objective_cold_gain": -0.0002698104527871781}, + "SESSION_X_OBJECTIVE": {"session_cold_gain": -0.001574849153598623, "objective_cold_gain": -0.0006396369760830467} + }, + "decision": { + "verdict": "ID_MORPHOLOGY_NOT_DECISION_CHANGING", + "interpretation": "Exact objective recurrence recreates the V74 session-cold prior; identifier morphology does not generalize across held-out objectives, and session-id morphology is null. Do not pursue IDs as a provider/applicability channel." + } +} From c3893423162c7b506457e847ee7a667f4e7efeca Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Fri, 21 Aug 2026 04:05:56 +1200 Subject: [PATCH 47/47] infra: run frozen V121 from seeded transcript cache --- ...-ace-v121-pretrained-semantic-residual.yml | 41 +++++++++++++++---- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml index 51f92de9..a3fa5201 100644 --- a/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml +++ b/.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml @@ -8,6 +8,11 @@ on: - '.github/workflows/trace-ace-v121-pretrained-semantic-residual.yml' workflow_dispatch: +# Infrastructure-only serialization. Scientific protocol is unchanged. +concurrency: + group: trace-ace-frozen-transcripts-v1 + cancel-in-progress: false + jobs: semantic: runs-on: ubuntu-24.04 @@ -21,25 +26,44 @@ jobs: - name: Install dependencies run: | python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown fastembed==0.8.0 - - name: Download frozen metadata and transcripts + + - name: Restore frozen transcript archive from GitHub cache + uses: actions/cache/restore@v4 + with: + path: transcripts.zip + key: trace-ace-transcripts-v1-603547640 + fail-on-cache-miss: true + + - name: Validate frozen transcript archive + shell: bash run: | - set -e + set -euo pipefail + test -f transcripts.zip + test "$(stat -c%s transcripts.zip)" = "603547640" + unzip -tq transcripts.zip >/dev/null + ACTUAL=$(sha256sum transcripts.zip | cut -d' ' -f1) + test "$ACTUAL" = "e685b85b04694e130c25b17d09cdd1892fbda5e9fa685e98b2300114b915aa2d" + echo "Frozen transcript cache verified: $ACTUAL" + + - name: Download frozen metadata and extract data + shell: bash + run: | + set -euo pipefail gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip - if ! gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip; then - echo 'primary frozen transcript transport failed; trying frozen mirror' - gdown 1UWuHKIJ86yINhQOBGx2fFJ890hiEa6Md -O transcripts.zip - fi - sha256sum metadata.zip transcripts.zip mkdir -p data/meta data/transcripts unzip -q metadata.zip -d data/meta unzip -q transcripts.zip -d data/transcripts FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + test -n "$FEATURES" + test -n "$LABELS" + test -n "$FIRST" TRANSCRIPTS=$(dirname "$FIRST") echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" echo "LABELS=$LABELS" >> "$GITHUB_ENV" echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + - name: Preflight run: | python -m py_compile competitions/trace_the_ace/v121_pretrained_semantic_residual.py @@ -58,7 +82,8 @@ jobs: --rows 2500 \ --out ../../v121_pretrained_semantic_residual.json - name: Show decision - run: cat v121_pretrained_semantic_residual.json + if: always() + run: test -f v121_pretrained_semantic_residual.json && cat v121_pretrained_semantic_residual.json || true - uses: actions/upload-artifact@v4 if: always() with: