Fix: Prevent Orphaned Submissions When SQS Publish Fails

apps/jobs/views.py

@@ -409,11 +409,28 @@ def challenge_submission(request, challenge_id, challenge_phase_id):
 
         if serializer.is_valid():
             serializer.save()
-            response_data = serializer.data
             submission = serializer.instance
             message["submission_pk"] = submission.id
-            # publish message in the queue
-            publish_submission_message(message)
+
+            try:
+                publish_submission_message(message)
+            except Exception:
+                logger.exception(
+                    "SQS publish failed for submission %s in challenge %s, "
+                    "deleting submission",
+                    submission.pk,
+                    challenge_id,
+                )
+                submission.delete()
+                response_data = {
+                    "error": "Failed to process your submission. Please try again."
+                }
+                return Response(
+                    response_data,
+                    status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                )
+
+            response_data = serializer.data
             return Response(response_data, status=status.HTTP_201_CREATED)
         return Response(
             serializer.errors, status=status.HTTP_406_NOT_ACCEPTABLE

docker/dev/django/Dockerfile

@@ -7,8 +7,8 @@ ENV PYTHONUNBUFFERED=1 \
     PIP_DEFAULT_TIMEOUT=100
 
 # Install build dependencies only
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
+RUN DEBIAN_FRONTEND=noninteractive apt-get update --fix-missing && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends --fix-missing \
     build-essential \
     libpq-dev \
     libcurl4-openssl-dev \
@@ -20,7 +20,8 @@ RUN apt-get update && \
     libfreetype6-dev \
     liblcms2-dev \
     libwebp-dev \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/* \
+    && rm -rf /var/cache/apt/*
 
 WORKDIR /code
 
@@ -45,6 +46,8 @@ ENV PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1
 
 # Install runtime dependencies (apt-get handles multi-arch automatically)
+RUN DEBIAN_FRONTEND=noninteractive apt-get update --fix-missing && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends --fix-missing \
 # git: for running git commit from container (pre-commit in Linux env, avoids macOS/ARM issues)
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \

tests/unit/jobs/test_views.py

@@ -5,6 +5,7 @@
 from datetime import timedelta
 
 import boto3
+import botocore
 import mock
 import requests
 from allauth.account.models import EmailAddress
@@ -532,6 +533,142 @@ def test_challenge_submission_for_docker_based_challenges(self):
 
         self.assertEqual(response.status_code, status.HTTP_201_CREATED)
 
+    @mock.patch(
+        "jobs.views.publish_submission_message",
+        side_effect=Exception("SQS connection error"),
+    )
+    def test_challenge_submission_cleans_up_on_publish_failure(
+        self, mock_publish
+    ):
+        self.url = reverse_lazy(
+            "jobs:challenge_submission",
+            kwargs={
+                "challenge_id": self.challenge.pk,
+                "challenge_phase_id": self.challenge_phase.pk,
+            },
+        )
+
+        self.challenge.participant_teams.add(self.participant_team)
+        self.challenge.save()
+
+        submission_count_before = Submission.objects.count()
+
+        response = self.client.post(
+            self.url,
+            {"status": "submitting", "input_file": self.input_file},
+            format="multipart",
+        )
+        self.assertEqual(
+            response.status_code,
+            status.HTTP_500_INTERNAL_SERVER_ERROR,
+        )
+        self.assertIn("error", response.data)
+        # Orphaned submission must be deleted
+        self.assertEqual(Submission.objects.count(), submission_count_before)
+
+    @mock.patch(
+        "jobs.views.publish_submission_message",
+        side_effect=botocore.exceptions.EndpointConnectionError(
+            endpoint_url="https://sqs.us-east-1.amazonaws.com"
+        ),
+    )
+    def test_challenge_submission_handles_sqs_endpoint_failure(
+        self, mock_publish
+    ):
+        self.url = reverse_lazy(
+            "jobs:challenge_submission",
+            kwargs={
+                "challenge_id": self.challenge.pk,
+                "challenge_phase_id": self.challenge_phase.pk,
+            },
+        )
+
+        self.challenge.participant_teams.add(self.participant_team)
+        self.challenge.save()
+
+        submission_count_before = Submission.objects.count()
+
+        response = self.client.post(
+            self.url,
+            {"status": "submitting", "input_file": self.input_file},
+            format="multipart",
+        )
+        self.assertEqual(
+            response.status_code,
+            status.HTTP_500_INTERNAL_SERVER_ERROR,
+        )
+        self.assertEqual(Submission.objects.count(), submission_count_before)
+
+    @mock.patch(
+        "jobs.views.publish_submission_message",
+        side_effect=Exception("SQS send failed"),
+    )
+    def test_challenge_submission_preserves_quota_on_publish_failure(
+        self, mock_publish
+    ):
+        self.url = reverse_lazy(
+            "jobs:challenge_submission",
+            kwargs={
+                "challenge_id": self.challenge.pk,
+                "challenge_phase_id": self.challenge_phase.pk,
+            },
+        )
+
+        self.challenge.participant_teams.add(self.participant_team)
+        self.challenge.save()
+
+        # First attempt fails due to SQS
+        response = self.client.post(
+            self.url,
+            {"status": "submitting", "input_file": self.input_file},
+            format="multipart",
+        )
+        self.assertEqual(
+            response.status_code,
+            status.HTTP_500_INTERNAL_SERVER_ERROR,
+        )
+
+        # Participant's quota should not be consumed — retry must still be allowed
+        mock_publish.side_effect = None
+        mock_publish.return_value = None
+        retry_input = SimpleUploadedFile(
+            "retry_input.txt", b"file_content", content_type="text/plain"
+        )
+        response = self.client.post(
+            self.url,
+            {"status": "submitting", "input_file": retry_input},
+            format="multipart",
+        )
+        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+
+    @mock.patch("jobs.views.publish_submission_message")
+    def test_challenge_submission_returns_201_when_publish_succeeds(
+        self, mock_publish
+    ):
+        self.url = reverse_lazy(
+            "jobs:challenge_submission",
+            kwargs={
+                "challenge_id": self.challenge.pk,
+                "challenge_phase_id": self.challenge_phase.pk,
+            },
+        )
+
+        self.challenge.participant_teams.add(self.participant_team)
+        self.challenge.save()
+
+        submission_count_before = Submission.objects.count()
+
+        response = self.client.post(
+            self.url,
+            {"status": "submitting", "input_file": self.input_file},
+            format="multipart",
+        )
+        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+        self.assertEqual(
+            Submission.objects.count(), submission_count_before + 1
+        )
+        mock_publish.assert_called_once()
+
     def test_challenge_submission_when_file_url_is_none(self):
         self.url = reverse_lazy(
             "jobs:challenge_submission",