Skip to content

Commit a674408

Browse files
committed
ci: coordinate interoperability startup
1 parent 62fc1ee commit a674408

8 files changed

Lines changed: 242 additions & 13 deletions

File tree

‎.github/INTEROPERABILITY.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,20 @@ We bind every record to the parent CI run, attempt and tested revision. Endpoint
1010

1111
Artefact names end in `attempt-N`. Context and lease artefacts use the controller attempt; other interoperability artefacts use the parent CI attempt. The suffix prevents a rerun from consuming records from an earlier attempt.
1212

13+
Recipient and enrolment waits allow 30 minutes without a newly observed endpoint job start. Each endpoint can renew that startup window once, using GitHub job identity and start metadata from the same CI attempt. Polling, artefact changes and transient request failures do not renew it. Transfer barriers retain their fixed deadlines. The controller and endpoint jobs retain their 75-minute limits.
14+
1315
We publish each phase result before waiting at the barrier, which keeps a failed endpoint available until its peers finish the current phase. We block later phases but still run local cleanup. A missing, failed or incomplete endpoint result fails the final check.
1416

17+
Blocked endpoint jobs fail after uploading their blocked record and attempting local cleanup. Qualification after a shared startup failure requires a full CI attempt. A failed-job rerun can leave successful endpoints from an earlier attempt, whose records remain ineligible.
18+
1519
## Authentication
1620

1721
The Network controller runs trusted default-branch code through a `workflow_run` listener and ignores CI runs that did not select the endpoint workflow. We configure the dedicated test tailnet outside this repository. The controller identity needs write access to `auth_keys`, `policy_file`, `devices:core` and `devices:posture_attributes`. We restrict enrolment to `tag:quarantine`.
1822

1923
The controller checks CI and endpoint liveness before obtaining authority, requesting keys and publishing enrolment.
2024

25+
The broker waits for every enrolment recipient before obtaining Tailscale authority. A startup timeout before that point does not create tailnet resources.
26+
2127
We give endpoint jobs read-only GitHub permissions and no OIDC permission. Each endpoint generates a temporary RSA recipient and receives one encrypted, single-use enrolment key. RSA-OAEP uses 3072-bit keys and SHA-256. We remove the enrolment files immediately and remove SSH private keys when fixture services stop. We never include plaintext credentials in artefacts or job output.
2228

2329
We pin the Tailscale CLI in `interop-client.py` and pin SSH host keys to endpoint identities. We disable route acceptance and DNS changes and verify the active preferences.

‎.github/actions/interoperability/action.yml‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -208,9 +208,6 @@ runs:
208208
id: outcome
209209
if: always()
210210
shell: bash
211-
run: |
212-
python3 -B -S - <<'PY'
213-
import os
214-
with open(os.environ['GITHUB_OUTPUT'], 'a') as output:
215-
output.write('blocked=' + os.environ.get('INTEROP_BLOCKED', 'false') + '\n')
216-
PY
211+
env:
212+
INTEROP_PLATFORM: ${{ inputs.platform }}
213+
run: python3 -B -S .github/scripts/interop-host.py outcome "$INTEROP_PLATFORM" --root "$RUNNER_TEMP/interoperability"

‎.github/scripts/interop-broker.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -256,12 +256,14 @@ def main():
256256
if args.phase == 'finish':
257257
write(root, 'result', passed=False, cleaned=True, roles=[])
258258
return 0
259-
api = Tailnet()
260259
if args.phase == 'broker':
261-
check_policy(api.policy()[0])
262260
recipients, keys = records('public', timeout=STARTUP_TIMEOUT), {}
263261
if len({record.get('public_key') for record in recipients.values()}) != len(ROLES):
264262
raise NetworkError('Endpoints share an enrolment recipient')
263+
check_ci()
264+
api = Tailnet()
265+
if args.phase == 'broker':
266+
check_policy(api.policy()[0])
265267
encrypted = {}
266268
save(root, 'keys', keys=keys, pending=None)
267269
for role in ROLES:

‎.github/scripts/interop-host.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ def main():
166166
parser = argparse.ArgumentParser()
167167
parser.add_argument('phase', choices=('install', 'recipient', 'connect', 'prepare', 'forward', 'quarantine', 'admit',
168168
'transfer1', 'wait1', 'next', 'transfer2', 'wait2', 'verify',
169-
'revoked', 'stop', 'result'))
169+
'revoked', 'stop', 'outcome', 'result'))
170170
parser.add_argument('platform', choices=(*ROLES, 'all'))
171171
parser.add_argument('--root', type=Path, required=True)
172172
args = parser.parse_args()
@@ -178,6 +178,15 @@ def main():
178178
if phase == 'stop':
179179
stop(root, role)
180180
return 0
181+
if phase == 'outcome':
182+
blocked = os.environ.get('INTEROP_BLOCKED', 'false')
183+
if blocked not in ('true', 'false'):
184+
raise NetworkError('Invalid endpoint outcome')
185+
with open(os.environ['GITHUB_OUTPUT'], 'a') as stream:
186+
stream.write('blocked=' + blocked + '\n')
187+
if blocked == 'true':
188+
raise NetworkError('Endpoint did not complete interoperability')
189+
return 0
181190
if phase == 'forward':
182191
forward(root, role)
183192
return 0

‎.github/scripts/interop_ci.py‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -201,20 +201,54 @@ def check_ci():
201201
'failure', 'cancelled', 'timed_out', 'action_required')]
202202
if failed:
203203
raise PeerUnavailable('Unavailable endpoints: ' + ', '.join(failed))
204+
return [job for job in listing if job.get('name', '').startswith('Interoperability / ')
205+
and job['name'] != 'Interoperability / Result']
206+
207+
208+
def started_endpoints(jobs):
209+
state = identity()
210+
started, names, job_ids = {}, set(), set()
211+
if len(jobs) > len(ROLES):
212+
raise NetworkError('Unexpected endpoint job count')
213+
for job in jobs:
214+
name, job_id = job['name'], job.get('id')
215+
if (name in names or type(job_id) is not int or job_id < 1 or job_id in job_ids
216+
or job.get('run_id') != int(state['run_id'])
217+
or job.get('run_attempt') != int(state['run_attempt'])):
218+
raise NetworkError('Invalid endpoint job identity')
219+
names.add(name)
220+
job_ids.add(job_id)
221+
if job.get('status') not in ('in_progress', 'completed'):
222+
continue
223+
if not isinstance(job.get('started_at'), str) or not re.fullmatch(TIMESTAMP, job['started_at']):
224+
raise NetworkError('Invalid endpoint job start')
225+
started[name] = job_id, job['started_at']
226+
return started
204227

205228

206229
def records(kind, roles=ROLES, run_id=None, timeout=600):
207230
state = identity()
208231
expected = {artifact_name(kind, role): role for role in roles}
209232
checked = [None]
233+
started = {}
210234
def active():
211235
now = time.monotonic()
212236
if checked[0] is not None and now - checked[0] < 30:
213237
return
214238
checked[0] = now
215-
check_ci()
239+
jobs = check_ci()
216240
if run_id is not None and kind != 'result' and api('/actions/runs/' + run_id).get('status') == 'completed':
217241
raise PeerUnavailable('Network controller has stopped')
242+
if kind in ('public', 'enrolment'):
243+
current = started_endpoints(jobs)
244+
if (any(name in started and started[name] != value for name, value in current.items())
245+
or len(set(started) | set(current)) > len(ROLES)):
246+
raise NetworkError('Changed endpoint job identity during startup')
247+
# Runner scheduling is progress only once per endpoint. Artefact
248+
# churn and retries cannot renew this startup stall window.
249+
advanced = bool(set(current) - set(started))
250+
started.update(current)
251+
return advanced
218252
return {role: validate(record, state) for role, (record, _) in
219253
download_records(run_id or state['run_id'], expected, timeout, progress=active).items()}
220254

‎.github/scripts/interop_network.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -131,8 +131,8 @@ def download_records(run_id, expected, timeout=600, progress=None):
131131
deadline = time.monotonic() + timeout
132132
cached, etag = None, None
133133
while time.monotonic() < deadline:
134-
if progress is not None:
135-
progress()
134+
if progress is not None and progress() is True:
135+
deadline = time.monotonic() + timeout
136136
try:
137137
body, headers = fetch(url, os.environ['GH_TOKEN'],
138138
headers={'If-None-Match': etag} if etag else {},

‎.github/scripts/test-interop.py‎

Lines changed: 170 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -435,11 +435,32 @@ def test_failed_endpoints_disable_the_controller_before_authority(self):
435435
self.assertEqual(output.read_text(), 'enabled=false\n')
436436
self.assertFalse((self.root / 'context.json').exists())
437437

438+
def test_startup_wait_precedes_network_authority(self):
439+
authority = Mock(side_effect=AssertionError('Startup wait obtained network authority'))
440+
recipients = Mock(side_effect=network.NetworkError('Readiness timed out'))
441+
with patch.object(sys, 'argv', ['interop-broker.py', 'broker', '--root', str(self.root)]), \
442+
patch.dict(broker['main'].__globals__, Tailnet=authority, records=recipients), \
443+
patch('sys.stderr', new_callable=io.StringIO):
444+
self.assertEqual(broker['main'](), 1)
445+
recipients.assert_called_once_with('public', timeout=ci.STARTUP_TIMEOUT)
446+
authority.assert_not_called()
447+
448+
def test_cancelled_startup_never_obtains_network_authority(self):
449+
authority = Mock(side_effect=AssertionError('Cancelled startup obtained network authority'))
450+
recipients = {role: dict(public_key=role) for role in ROLES}
451+
check = Mock(side_effect=[None, ci.NotSelected('CI finished during startup')])
452+
with patch.object(sys, 'argv', ['interop-broker.py', 'broker', '--root', str(self.root)]), \
453+
patch.dict(broker['main'].__globals__, Tailnet=authority, check_ci=check,
454+
records=Mock(return_value=recipients)), \
455+
patch('sys.stderr', new_callable=io.StringIO):
456+
self.assertEqual(broker['main'](), 1)
457+
authority.assert_not_called()
458+
438459
def test_cancelled_key_issuance_is_cleaned_before_delivery(self):
439460
api = Mock()
440461
api.policy.return_value = ({}, {})
441462
api.create_key.return_value = ('key1', 'tskey-auth-fixture')
442-
check = Mock(side_effect=[None, None, ci.NotSelected('CI finished')])
463+
check = Mock(side_effect=[None, None, None, ci.NotSelected('CI finished')])
443464
recipients = {role: {'public_key': role} for role in ROLES}
444465
with patch.dict(broker['main'].__globals__, Tailnet=lambda: api, check_ci=check,
445466
records=lambda *args, **kwargs: recipients, check_policy=lambda policy: None,
@@ -790,5 +811,153 @@ def test_cas_preserves_other_cohort(self):
790811
network.Tailnet().change(lambda value: value)
791812

792813

814+
class Startup(unittest.TestCase):
815+
def setUp(self):
816+
self.state = dict(run_id='123', run_attempt='2', source_sha='a' * 40)
817+
ci.bind(self.state)
818+
self.addCleanup(setattr, ci, 'bound', None)
819+
self.now = 0
820+
self.starts = [0, 0, 0, 1200, 1500, 1900, 2050]
821+
self.ready_at = 2200
822+
self.current = dict(run_attempt=2, status='in_progress')
823+
self.controller = dict(status='in_progress')
824+
self.record = self.state
825+
self.mutate_jobs = lambda jobs: jobs
826+
self.transient = False
827+
self.churn = False
828+
829+
def jobs(self):
830+
return self.mutate_jobs([
831+
dict(id=index + 1, name='Interoperability / ' + role, run_id=123, run_attempt=2,
832+
status='in_progress' if self.now >= start else 'queued', conclusion=None,
833+
started_at='2026-11-17T00:00:00Z' if self.now >= start else None)
834+
for index, (role, start) in enumerate(zip(ROLES, self.starts))])
835+
836+
def api(self, path):
837+
if path == '/actions/runs/123':
838+
return self.current
839+
if path == '/actions/runs/456':
840+
return self.controller
841+
if path == '/actions/runs/123/attempts/2/jobs?per_page=100':
842+
jobs = self.jobs()
843+
return dict(total_count=len(jobs), jobs=jobs)
844+
self.fail('Unexpected API request: ' + path)
845+
846+
def wait(self, kind):
847+
roles = (None,) if kind == 'enrolment' else ROLES
848+
names = {ci.artifact_name(kind, role): index + 1 for index, role in enumerate(roles)}
849+
def fetch(url, *args, **kwargs):
850+
if '/artifacts?' in url:
851+
if self.transient:
852+
raise network.NetworkError('Connection failed', retryable=True)
853+
items = [dict(id=number, name=name, expired=False) for name, number in names.items()]
854+
if self.now < self.ready_at:
855+
items = [dict(id=int(self.now) + 100, name='unrelated', expired=False)] if self.churn else []
856+
return json.dumps(dict(total_count=len(items), artifacts=items)).encode(), {}
857+
buffer = io.BytesIO()
858+
with zipfile.ZipFile(buffer, 'w') as archive:
859+
archive.writestr('record.json', json.dumps(self.record))
860+
return buffer.getvalue(), {}
861+
def sleep(seconds):
862+
self.now += seconds
863+
with patch.object(ci, 'api', side_effect=self.api), \
864+
patch.object(network, 'fetch', side_effect=fetch), \
865+
patch.dict(os.environ, GH_TOKEN='fixture'), \
866+
patch.object(time, 'monotonic', side_effect=lambda: self.now), \
867+
patch.object(time, 'sleep', side_effect=sleep):
868+
return ci.records(kind, roles, '456' if kind == 'enrolment' else None, ci.STARTUP_TIMEOUT)
869+
870+
def test_late_job_starts_renew_both_startup_waits(self):
871+
for kind in ('public', 'enrolment'):
872+
with self.subTest(kind=kind):
873+
self.now = 0
874+
records = self.wait(kind)
875+
self.assertEqual(len(records), 1 if kind == 'enrolment' else len(ROLES))
876+
self.assertGreaterEqual(self.now, self.ready_at)
877+
878+
def test_queued_jobs_cannot_extend_the_startup_wait(self):
879+
self.starts = [10000] * len(ROLES)
880+
self.mutate_jobs = lambda jobs: [dict(job, started_at='2026-11-17T00:00:00Z') for job in jobs]
881+
for kind in ('public', 'enrolment'):
882+
with self.subTest(kind=kind):
883+
self.now = 0
884+
with self.assertRaisesRegex(network.NetworkError, 'Timed out'):
885+
self.wait(kind)
886+
self.assertEqual(self.now, ci.STARTUP_TIMEOUT)
887+
888+
def test_each_job_start_renews_only_once(self):
889+
self.starts = [1700] * len(ROLES)
890+
self.ready_at = 10000
891+
with self.assertRaisesRegex(network.NetworkError, 'Timed out'):
892+
self.wait('public')
893+
self.assertGreater(self.now, ci.STARTUP_TIMEOUT)
894+
self.assertLessEqual(self.now, ci.STARTUP_TIMEOUT * 2)
895+
896+
def test_retries_and_unrelated_artifacts_cannot_renew_startup(self):
897+
self.starts = [0] * len(ROLES)
898+
for transient, churn in ((True, False), (False, True)):
899+
with self.subTest(transient=transient, churn=churn):
900+
self.now, self.transient, self.churn = 0, transient, churn
901+
with self.assertRaisesRegex(network.NetworkError, 'Timed out'):
902+
self.wait('public')
903+
self.assertEqual(self.now, ci.STARTUP_TIMEOUT)
904+
905+
def test_transfer_deadlines_do_not_follow_job_starts(self):
906+
with self.assertRaisesRegex(network.NetworkError, 'Timed out'):
907+
self.wait('ready')
908+
self.assertEqual(self.now, ci.STARTUP_TIMEOUT)
909+
910+
def test_delayed_readiness_still_rejects_stale_records(self):
911+
self.record = dict(self.state, run_attempt='1')
912+
with self.assertRaisesRegex(network.NetworkError, 'different CI run, attempt or revision'):
913+
self.wait('public')
914+
915+
def test_startup_preserves_liveness_checks(self):
916+
variants = [(dict(run_attempt=3), dict(status='in_progress'), None, ci.NotSelected),
917+
(dict(run_attempt=2, status='completed'), dict(status='in_progress'), None, ci.NotSelected),
918+
(dict(run_attempt=2, conclusion='cancelled'), dict(status='in_progress'), None, ci.NotSelected),
919+
(self.current, dict(status='completed'), None, ci.PeerUnavailable),
920+
(self.current, self.controller, 'failure', ci.PeerUnavailable)]
921+
for current, controller, conclusion, error in variants:
922+
with self.subTest(current=current, controller=controller, conclusion=conclusion):
923+
self.now, self.current, self.controller = 0, current, controller
924+
self.mutate_jobs = lambda jobs: [dict(job, conclusion=conclusion) for job in jobs]
925+
with self.assertRaises(error):
926+
self.wait('enrolment')
927+
self.assertEqual(self.now, 0)
928+
929+
def test_startup_rejects_ambiguous_job_progress(self):
930+
variants = [lambda jobs: jobs + [jobs[0]],
931+
lambda jobs: jobs + [dict(jobs[0], id=99, name='Interoperability / extra')],
932+
lambda jobs: [dict(job, run_id=999) for job in jobs],
933+
lambda jobs: [dict(job, run_attempt=1) for job in jobs],
934+
lambda jobs: [dict(job, id=True) for job in jobs],
935+
lambda jobs: [dict(job, id=1) for job in jobs],
936+
lambda jobs: [dict(job, started_at='invalid') for job in jobs]]
937+
self.starts = [0] * len(ROLES)
938+
for mutate in variants:
939+
with self.subTest(mutate=mutate):
940+
self.now, self.mutate_jobs = 0, mutate
941+
with self.assertRaisesRegex(network.NetworkError, 'endpoint job'):
942+
self.wait('public')
943+
self.assertEqual(self.now, 0)
944+
945+
def test_changed_job_identity_is_not_new_progress(self):
946+
self.starts = [0] * len(ROLES)
947+
self.mutate_jobs = lambda jobs: [dict(job, id=job['id'] + int(self.now)) for job in jobs]
948+
with self.assertRaisesRegex(network.NetworkError, 'endpoint job'):
949+
self.wait('public')
950+
self.assertEqual(self.now, 30)
951+
952+
def test_result_and_other_jobs_cannot_renew_startup(self):
953+
self.starts = [0] * len(ROLES)
954+
self.mutate_jobs = lambda jobs: jobs + [
955+
dict(jobs[0], id=100 + int(self.now), name=name)
956+
for name in ('Interoperability / Result', 'Other / endpoint')]
957+
with self.assertRaisesRegex(network.NetworkError, 'Timed out'):
958+
self.wait('public')
959+
self.assertEqual(self.now, ci.STARTUP_TIMEOUT)
960+
961+
793962
if __name__ == '__main__':
794963
unittest.main()

‎.github/scripts/test-transfers.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,18 @@ def test_local_failures_are_not_blocked_results(self):
323323
self.assertFalse(environment.exists())
324324
self.assertFalse((self.root / 'blocked.json').exists())
325325

326+
def test_endpoint_outcome_preserves_blocked_failure(self):
327+
output = self.root / 'outcome'
328+
for blocked, status in (('true', 1), ('false', 0)):
329+
with self.subTest(blocked=blocked), \
330+
patch.object(sys, 'argv', ['interop-host.py', 'outcome', 'linux', '--root', str(self.root)]), \
331+
patch.dict(os.environ, INTEROP_BLOCKED=blocked, GITHUB_OUTPUT=str(output)), \
332+
patch.dict(host['main'].__globals__, identity=lambda: self.fail('Outcome required CI identity')), \
333+
patch('sys.stderr', new_callable=io.StringIO):
334+
self.assertEqual(host['main'](), status)
335+
self.assertEqual(output.read_text(), 'blocked=' + blocked + '\n')
336+
output.unlink()
337+
326338
def test_failed_transfer_is_published_before_the_phase_barrier(self):
327339
ci.bind(self.state)
328340
with patch.object(sys, 'argv', ['interop-host.py', 'transfer2', 'linux', '--root', str(self.root)]), \

0 commit comments

Comments
 (0)