Skip to content

Commit 2d86f54

Browse files
committed
ci: retry transient artifact downloads
1 parent 71ff225 commit 2d86f54

2 files changed

Lines changed: 44 additions & 13 deletions

File tree

‎.github/scripts/interop_network.py‎

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,10 @@
1515

1616

1717
class NetworkError(ValueError):
18-
def __init__(self, message, code=None):
18+
def __init__(self, message, code=None, retryable=False):
1919
super().__init__(message)
2020
self.code = code
21+
self.retryable = retryable
2122

2223

2324
class NoRedirect(urllib.request.HTTPRedirectHandler):
@@ -57,7 +58,7 @@ def fetch(url, token=None, data=None, headers=None, allowed_redirect=False, meth
5758
return fetch(target, operation=operation)
5859
raise NetworkError(operation + ': HTTP ' + str(error.code), error.code) from None
5960
except (OSError, urllib.error.URLError):
60-
raise NetworkError(operation + ': connection failed') from None
61+
raise NetworkError(operation + ': connection failed', retryable=True) from None
6162

6263

6364
def workflow_event():
@@ -131,23 +132,27 @@ def download_records(run_id, expected, timeout=600, progress=None):
131132
if progress is not None:
132133
progress()
133134
try:
134-
body, headers = fetch(url, os.environ['GH_TOKEN'], headers={'If-None-Match': etag} if etag else {},
135+
body, headers = fetch(url, os.environ['GH_TOKEN'],
136+
headers={'If-None-Match': etag} if etag else {},
135137
operation='List GitHub artifacts')
136-
except NetworkError as error:
137-
if error.code != 304 or cached is None:
138-
raise
139-
records = cached
140-
else:
141138
listing = decode(body)
142139
records = listing['artifacts']
143140
count = listing['total_count']
144141
if type(count) is not int or count < 0 or count > 1000:
145142
raise NetworkError('Artifact listing exceeds rendezvous limit')
146143
for page in range(2, (count + 99) // 100 + 1):
147-
body, _ = fetch(url + '&page=' + str(page), os.environ['GH_TOKEN'], operation='List GitHub artifacts')
144+
body, _ = fetch(url + '&page=' + str(page), os.environ['GH_TOKEN'],
145+
operation='List GitHub artifacts')
148146
records += decode(body)['artifacts']
149147
records = list({item['id']: item for item in records}.values())
150148
cached, etag = records, headers.get('ETag')
149+
except NetworkError as error:
150+
if error.retryable:
151+
time.sleep(5)
152+
continue
153+
if error.code != 304 or cached is None:
154+
raise
155+
records = cached
151156
found = {name: [item for item in records if item['name'] == name and not item['expired']]
152157
for name in expected}
153158
if any(len(items) > 1 for items in found.values()):
@@ -162,9 +167,10 @@ def download_records(run_id, expected, timeout=600, progress=None):
162167
try:
163168
data, _ = fetch(download, os.environ['GH_TOKEN'], allowed_redirect=True, operation='Download GitHub artifact')
164169
except NetworkError as error:
165-
if error.code != 404:
170+
if error.code != 404 and not error.retryable:
166171
raise
167-
# The listing can precede archive availability.
172+
# The listing can precede archive availability, and an
173+
# individual archive request can lose its connection.
168174
result = None
169175
break
170176
with zipfile.ZipFile(io.BytesIO(data)) as archive:

‎.github/scripts/test-interop.py‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -676,14 +676,15 @@ def test_artifact_archive_retry_and_duplicates(self):
676676
archive.writestr('ready.json', json.dumps(record))
677677
item = dict(name='ready', id=1, expired=False, archive_download_url='https://attacker.invalid')
678678
listing = json.dumps(dict(artifacts=[item], total_count=1)).encode()
679-
responses = [(listing, {'ETag': 'v1'}), network.NetworkError('Not ready', 404),
679+
responses = [(listing, {'ETag': 'v1'}), network.NetworkError('Connection failed', retryable=True),
680+
network.NetworkError('Not modified', 304), network.NetworkError('Not ready', 404),
680681
network.NetworkError('Not modified', 304), (buffer.getvalue(), {})]
681682
with patch.object(network, 'fetch', side_effect=responses) as fetch, patch.object(time, 'sleep'):
682683
value = network.download_records('123', {'ready': 'linux'})
683684
self.assertEqual(value['linux'][0], record)
684685
self.assertEqual(fetch.call_args_list[2][1]['headers'], {'If-None-Match': 'v1'})
685686
self.assertEqual(fetch.call_args[0][0], 'https://api.github.com/repos/RsyncProject/rsync/actions/artifacts/1/zip')
686-
for code in (401, 403, 500):
687+
for code in (None, 401, 403, 500):
687688
with self.subTest(code=code), patch.object(network, 'fetch', side_effect=[(listing, {}), network.NetworkError('Failed', code)]), patch.object(time, 'sleep') as sleep:
688689
with self.assertRaises(network.NetworkError):
689690
network.download_records('123', {'ready': 'linux'})
@@ -692,6 +693,20 @@ def test_artifact_archive_retry_and_duplicates(self):
692693
with patch.object(network, 'fetch', return_value=(duplicate, {})), self.assertRaisesRegex(network.NetworkError, 'Duplicate'):
693694
network.download_records('123', {'ready': 'linux'})
694695

696+
def test_artifact_listing_retries_connection_failures(self):
697+
record = dict(self.state, role='linux')
698+
buffer = io.BytesIO()
699+
with zipfile.ZipFile(buffer, 'w') as archive:
700+
archive.writestr('ready.json', json.dumps(record))
701+
item = dict(name='ready', id=1, expired=False)
702+
listing = json.dumps(dict(artifacts=[item], total_count=1)).encode()
703+
responses = [network.NetworkError('Connection failed', retryable=True),
704+
(listing, {}), (buffer.getvalue(), {})]
705+
with patch.object(network, 'fetch', side_effect=responses), patch.object(time, 'sleep') as sleep:
706+
value = network.download_records('123', {'ready': 'linux'})
707+
self.assertEqual(value['linux'][0], record)
708+
sleep.assert_called_once_with(5)
709+
695710
def test_redacted_http_errors(self):
696711
class Opener:
697712
def open(self, *args, **kwargs):
@@ -705,6 +720,16 @@ def open(self, *args, **kwargs):
705720
with self.assertRaisesRegex(network.NetworkError, 'Worker acl access probe: HTTP 403'):
706721
network.fetch('https://example.test', 'fixture-bearer', operation='Worker acl access probe')
707722

723+
def test_connection_errors_are_retryable(self):
724+
class Opener:
725+
def open(self, *args, **kwargs):
726+
raise urllib.error.URLError('fixture connection failure')
727+
with patch.object(network.urllib.request, 'build_opener', return_value=Opener()), \
728+
self.assertRaises(network.NetworkError) as caught:
729+
network.fetch('https://example.test', 'fixture-bearer')
730+
self.assertTrue(caught.exception.retryable)
731+
self.assertEqual(str(caught.exception), 'Network request: connection failed')
732+
708733
def test_artifact_redirect_drops_authorisation(self):
709734
requests = []
710735
class Response:

0 commit comments

Comments
 (0)