Skip to content

Commit 9c03e5d

Browse files
committed
ci: distinguish unavailable peers
1 parent 270cbe3 commit 9c03e5d

2 files changed

Lines changed: 33 additions & 11 deletions

File tree

‎.github/scripts/interop_ci.py‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@ class NotSelected(NetworkError):
1717
pass
1818

1919

20+
class PeerUnavailable(NetworkError):
21+
pass
22+
23+
2024
def api(path):
2125
cached = api_cache.get(path)
2226
try:
@@ -121,9 +125,11 @@ def active():
121125
for page in range(2, (count + 99) // 100 + 1):
122126
listing += api('/actions/runs/' + state['run_id'] + '/attempts/' + state['run_attempt']
123127
+ '/jobs?per_page=100&page=' + str(page))['jobs']
124-
if any(job.get('name', '').startswith('Interoperability / ') and job.get('conclusion') in (
125-
'failure', 'cancelled', 'timed_out', 'action_required') for job in listing):
126-
raise NetworkError('A native endpoint job failed')
128+
failed = [job['name'][len('Interoperability / '):] for job in listing
129+
if job.get('name', '').startswith('Interoperability / ') and job.get('conclusion') in (
130+
'failure', 'cancelled', 'timed_out', 'action_required')]
131+
if failed:
132+
raise PeerUnavailable('Unavailable endpoints: ' + ', '.join(failed))
127133
return {role: validate(record, state) for role, (record, _) in
128134
download_records(run_id or state['run_id'], expected, timeout, progress=active).items()}
129135

@@ -164,7 +170,7 @@ def controller():
164170
run = find_controller()
165171
if run is not None:
166172
if run.get('conclusion') in ('failure', 'cancelled', 'timed_out'):
167-
raise NetworkError('Network controller failed')
173+
raise PeerUnavailable('Network controller failed')
168174
return str(run['id'])
169175
time.sleep(5)
170176
raise NetworkError('Trusted controller did not start')

‎.github/scripts/test-interop.py‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -266,18 +266,34 @@ def verified(argv, **kwargs):
266266
with self.assertRaisesRegex(network.NetworkError, 'attested'):
267267
broker['recover'](self.root, None)
268268

269-
def test_failure_and_cancellation_abort_rendezvous(self):
270-
for current, jobs in ((dict(run_attempt=3), None), (dict(run_attempt=2, conclusion='cancelled'), None),
271-
(dict(run_attempt=2, status='completed'), None),
272-
(dict(run_attempt=2), dict(total_count=1, jobs=[dict(name='Interoperability / NetBSD', conclusion='failure')]))):
273-
responses = [current] if jobs is None else [current, jobs]
269+
def test_cancelled_or_superseded_ci_aborts_rendezvous(self):
270+
for current in (dict(run_attempt=3), dict(run_attempt=2, conclusion='cancelled'),
271+
dict(run_attempt=2, status='completed')):
274272
def downloading(*args, **kwargs):
275273
kwargs['progress']()
276274
self.fail('Failed CI did not abort')
277-
with self.subTest(current=current, jobs=jobs), patch.object(ci, 'api', side_effect=responses), \
275+
with self.subTest(current=current), patch.object(ci, 'api', return_value=current), \
278276
patch.object(ci, 'download_records', side_effect=downloading), patch.object(time, 'monotonic', return_value=100):
279-
with self.assertRaises(network.NetworkError):
277+
with self.assertRaises(network.NetworkError) as caught:
280278
ci.records('ready')
279+
self.assertNotIsInstance(caught.exception, ci.PeerUnavailable)
280+
281+
def test_failed_peer_blocks_rendezvous(self):
282+
def downloading(*args, **kwargs):
283+
kwargs['progress']()
284+
self.fail('Missing peer did not block the phase')
285+
for conclusion in ('failure', 'cancelled', 'timed_out', 'action_required'):
286+
jobs = dict(total_count=1, jobs=[dict(name='Interoperability / NetBSD', conclusion=conclusion)])
287+
with self.subTest(conclusion=conclusion), patch.object(ci, 'api', side_effect=[dict(run_attempt=2), jobs]), \
288+
patch.object(ci, 'download_records', side_effect=downloading), patch.object(time, 'monotonic', return_value=100), \
289+
self.assertRaisesRegex(ci.PeerUnavailable, 'NetBSD'):
290+
ci.records('ready')
291+
292+
def test_failed_controller_blocks_its_dependants(self):
293+
for conclusion in ('failure', 'cancelled', 'timed_out'):
294+
with self.subTest(conclusion=conclusion), patch.object(ci, 'find_controller', return_value=dict(conclusion=conclusion)), \
295+
self.assertRaisesRegex(ci.PeerUnavailable, 'Network controller failed'):
296+
ci.controller()
281297

282298
def test_invalid_recovery_lease_never_reaches_cleanup(self):
283299
run = dict(id=456, run_attempt=1, path='.github/workflows/interoperability.yml', head_branch='master',

0 commit comments

Comments
 (0)