Skip to content

Commit 3723aec

Browse files
committed
ci: retry transient artifact server errors
1 parent 85ad31c commit 3723aec

2 files changed

Lines changed: 24 additions & 3 deletions

File tree

‎.github/scripts/interop_network.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212

1313
CLIENT_ID = 'TG2FEugs2S11CNTRL-kx3fW8WXMt11CNTRL'
1414
TAILNET_API = 'https://api.tailscale.com/api/v2/tailnet/-'
15+
RETRYABLE_HTTP = (408, 500, 502, 503, 504)
1516

1617

1718
class NetworkError(ValueError):
@@ -56,7 +57,8 @@ def fetch(url, token=None, data=None, headers=None, allowed_redirect=False, meth
5657
or parts.username or parts.password or parts.port not in (None, 443)):
5758
raise NetworkError('Unexpected artifact download host') from None
5859
return fetch(target, operation=operation)
59-
raise NetworkError(operation + ': HTTP ' + str(error.code), error.code) from None
60+
raise NetworkError(operation + ': HTTP ' + str(error.code), error.code,
61+
retryable=error.code in RETRYABLE_HTTP) from None
6062
except (OSError, urllib.error.URLError):
6163
raise NetworkError(operation + ': connection failed', retryable=True) from None
6264

‎.github/scripts/test-interop.py‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -677,14 +677,18 @@ def test_artifact_archive_retry_and_duplicates(self):
677677
item = dict(name='ready', id=1, expired=False, archive_download_url='https://attacker.invalid')
678678
listing = json.dumps(dict(artifacts=[item], total_count=1)).encode()
679679
responses = [(listing, {'ETag': 'v1'}), network.NetworkError('Connection failed', retryable=True),
680+
network.NetworkError('Not modified', 304),
681+
network.NetworkError('Service unavailable', 503, retryable=True),
680682
network.NetworkError('Not modified', 304), network.NetworkError('Not ready', 404),
681683
network.NetworkError('Not modified', 304), (buffer.getvalue(), {})]
682-
with patch.object(network, 'fetch', side_effect=responses) as fetch, patch.object(time, 'sleep'):
684+
with patch.object(network, 'fetch', side_effect=responses) as fetch, \
685+
patch.object(time, 'sleep') as sleep:
683686
value = network.download_records('123', {'ready': 'linux'})
684687
self.assertEqual(value['linux'][0], record)
685688
self.assertEqual(fetch.call_args_list[2][1]['headers'], {'If-None-Match': 'v1'})
686689
self.assertEqual(fetch.call_args[0][0], 'https://api.github.com/repos/RsyncProject/rsync/actions/artifacts/1/zip')
687-
for code in (None, 401, 403, 500):
690+
self.assertEqual(sleep.call_count, 3)
691+
for code in (None, 401, 403, 422):
688692
with self.subTest(code=code), patch.object(network, 'fetch', side_effect=[(listing, {}), network.NetworkError('Failed', code)]), patch.object(time, 'sleep') as sleep:
689693
with self.assertRaises(network.NetworkError):
690694
network.download_records('123', {'ready': 'linux'})
@@ -716,6 +720,7 @@ def open(self, *args, **kwargs):
716720
network.fetch('https://example.test', 'fixture-bearer')
717721
self.assertEqual(str(caught.exception), 'Network request: HTTP 403')
718722
self.assertEqual(caught.exception.code, 403)
723+
self.assertFalse(caught.exception.retryable)
719724
with patch.object(network.urllib.request, 'build_opener', return_value=Opener()):
720725
with self.assertRaisesRegex(network.NetworkError, 'Worker acl access probe: HTTP 403'):
721726
network.fetch('https://example.test', 'fixture-bearer', operation='Worker acl access probe')
@@ -730,6 +735,20 @@ def open(self, *args, **kwargs):
730735
self.assertTrue(caught.exception.retryable)
731736
self.assertEqual(str(caught.exception), 'Network request: connection failed')
732737

738+
def test_transient_http_errors_are_retryable(self):
739+
class Opener:
740+
def __init__(self, code):
741+
self.code = code
742+
def open(self, request, **kwargs):
743+
raise urllib.error.HTTPError(request.full_url, self.code, 'fixture', {}, io.BytesIO())
744+
for code in network.RETRYABLE_HTTP:
745+
with self.subTest(code=code), \
746+
patch.object(network.urllib.request, 'build_opener', return_value=Opener(code)), \
747+
self.assertRaises(network.NetworkError) as caught:
748+
network.fetch('https://example.test')
749+
self.assertEqual(caught.exception.code, code)
750+
self.assertTrue(caught.exception.retryable)
751+
733752
def test_artifact_redirect_drops_authorisation(self):
734753
requests = []
735754
class Response:

0 commit comments

Comments
 (0)