Return Unclear answers instead of raising exceptions when escalation fails

app/api/routes/image_queries.py

@@ -1,6 +1,6 @@
 import logging
 import random
-from typing import Literal, Optional
+from typing import Optional
 
 from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, Request, status
 from groundlight import Groundlight
@@ -14,7 +14,7 @@
     refresh_detector_metadata_if_needed,
 )
 from app.core.edge_inference import get_edge_inference_model_name
-from app.core.utils import create_iq, safe_call_sdk
+from app.core.utils import HUMAN_REVIEW_TYPE, create_iq, safe_call_sdk, safe_escalate_iq
 from app.metrics.iqactivity import record_iq_activity
 
 logger = logging.getLogger(__name__)
@@ -58,7 +58,7 @@ async def post_image_query(  # noqa: PLR0913, PLR0915, PLR0912
     image_bytes: bytes = Depends(validate_image_bytes),
     patience_time: Optional[float] = Query(None, ge=0),
     confidence_threshold: Optional[float] = Query(None, ge=0, le=1),
-    human_review: Optional[Literal["DEFAULT", "ALWAYS", "NEVER"]] = Query(None),
+    human_review: HUMAN_REVIEW_TYPE = Query(None),
     want_async: bool = Query(False),
     gl: Groundlight = Depends(get_groundlight_sdk_instance),
     app_state: AppState = Depends(get_app_state),
@@ -265,13 +265,25 @@ async def post_image_query(  # noqa: PLR0913, PLR0915, PLR0912
         raise AssertionError("Cloud escalation is disabled.")  # ...should never reach this point
 
     logger.debug(f"Submitting image query to cloud for {detector_id=}")
-    return safe_call_sdk(
-        gl.submit_image_query,
-        detector=detector_id,
-        image=image_bytes,
-        wait=0,  # wait on the client, not here
+    # return safe_call_sdk(
+    #     gl.submit_image_query,
+    #     detector=detector_id,
+    #     image=image_bytes,
+    #     wait=5,  # wait on the client, not here
+    #     patience_time=patience_time,
+    #     confidence_threshold=confidence_threshold,
+    #     human_review=human_review,
+    #     metadata={"edge_result": results},
+    #     want_async=True,
+    # )
+    return safe_escalate_iq(
+        gl=gl,
+        results=results,
+        detector_id=detector_id,
+        image_bytes=image_bytes,
         patience_time=patience_time,
         confidence_threshold=confidence_threshold,
         human_review=human_review,
-        metadata={"edge_result": results},
+        query=detector_metadata.query,
+        mode=detector_metadata.mode,
     )

app/core/utils.py

@@ -2,11 +2,12 @@
 import time
 from datetime import datetime, timezone
 from io import BytesIO
-from typing import Any, Callable
+from typing import Any, Callable, Literal, Optional
 
 import cachetools
 import ksuid
 from fastapi import HTTPException
+from groundlight import Groundlight
 from model import (
     ROI,
     BinaryClassificationResult,
@@ -26,6 +27,8 @@
 
 logger = logging.getLogger(__name__)
 
+HUMAN_REVIEW_TYPE = Optional[Literal["DEFAULT", "ALWAYS", "NEVER"]]
+
 
 def create_iq(  # noqa: PLR0913
     detector_id: str,
@@ -82,7 +85,7 @@ def _mode_to_result_and_type(
     based on the provided mode, confidence, and result value.
 
     :param mode: The mode of the detector.
-    :param mode_configuration:
+    :param mode_configuration: For counting only. A dict containing values for the max_count and class_name.
     :param confidence: The confidence of the predicted value.
     :param result_value: The predicted value.
 
@@ -138,6 +141,86 @@ def safe_call_sdk(api_method: Callable, **kwargs):
         raise ex
 
 
+def _mode_to_unclear_result(mode: ModeEnum):
+    source = Source.ALGORITHM  # TODO what should the source be?
+    if mode == ModeEnum.BINARY:
+        result_type = ResultTypeEnum.binary_classification
+        result = BinaryClassificationResult(
+            confidence=1.0,
+            source=source,
+            label=Label.UNCLEAR,
+        )
+    elif mode == ModeEnum.COUNT:
+        result_type = ResultTypeEnum.counting
+        result = CountingResult(
+            confidence=1.0,
+            source=source,
+            count=None,  # TODO double-check how to model a counting Unclear result. Also this doesn't work on current SDK version.
+            greater_than_max=False,
+        )
+    elif mode == ModeEnum.MULTI_CLASS:
+        raise NotImplementedError("Multiclass functionality is not yet implemented for the edge endpoint.")
+        # TODO add support for multiclass functionality.
+    else:
+        raise ValueError(f"Got unrecognized or unsupported detector mode: {mode}")
+
+    return result_type, result
+
+
+def safe_escalate_iq(
+    gl: Groundlight,
+    results: dict[str, Any],
+    detector_id: str,
+    image_bytes: bytes,
+    patience_time: float | None,
+    confidence_threshold: float,
+    human_review: HUMAN_REVIEW_TYPE,
+    query: str,
+    mode: ModeEnum,
+) -> ImageQuery:
+    """
+    This attempts to escalate an image query via the SDK. If it fails, it will catch the exception and return an
+    ImageQuery with an unclear result.
+    """
+    try:
+        iq_to_return = safe_call_sdk(
+            gl.submit_image_query,
+            detector=detector_id,
+            image=image_bytes,
+            wait=0,  # wait on the client, not here # TODO revert to 0
+            # want_async=True,
+            patience_time=patience_time,
+            confidence_threshold=confidence_threshold,
+            human_review=human_review,
+            metadata={"edge_result": results},
+        )
+        logger.info("I called the sdk and there was no exception")
+    except Exception as ex:
+        logger.info(f"I caught an exception! {ex=}")
+        result_type, result = _mode_to_unclear_result(mode)
+
+        if patience_time is None:
+            patience_time = constants.DEFAULT_PATIENCE_TIME
+
+        readable_exception_str = f"{ex.__class__.__name__}: {str(ex)}"
+
+        iq_to_return = ImageQuery(
+            metadata={"is_from_edge": True, "error_info": readable_exception_str},
+            id=prefixed_ksuid(prefix="iq_"),
+            type=ImageQueryTypeEnum.image_query,
+            created_at=datetime.now(timezone.utc),
+            query=query,
+            detector_id=detector_id,
+            result_type=result_type,
+            result=result,
+            patience_time=patience_time,
+            confidence_threshold=confidence_threshold,
+            rois=None,
+            text=None,
+        )
+    return iq_to_return
+
+
 def prefixed_ksuid(prefix: str | None = None) -> str:
     """Returns a unique identifier, with a bunch of nice properties.
     It's statistically guaranteed unique, about as strongly as UUIDv4 are.

test/api/test_utils.py

@@ -1,4 +1,7 @@
+from unittest.mock import MagicMock
+
 import pytest
+from groundlight import ImageQuery
 from model import (
     BinaryClassificationResult,
     CountingResult,
@@ -8,16 +11,154 @@
     Source,
 )
 
+from app.core.constants import DEFAULT_PATIENCE_TIME
 from app.core.utils import (
+    HUMAN_REVIEW_TYPE,
     ModelInfoBase,
     ModelInfoNoBinary,
     ModelInfoWithBinary,
     create_iq,
     parse_model_info,
     prefixed_ksuid,
+    safe_escalate_iq,
 )
 
 
+class TestSafeEscalateIQ:
+    def setup_method(self):
+        self.detector_id = "test"
+        self.image_bytes: bytes = None
+        self.patience_time = DEFAULT_PATIENCE_TIME
+        self.confidence_threshold = 0.9
+        self.human_review: HUMAN_REVIEW_TYPE = "NEVER"
+        self.query = "test query"
+        self.results_metadata = {"edge_result": {}}
+
+    def test_escalate_no_exception(self):
+        """Test escalating an IQ with no exception raised."""
+        mock_gl = MagicMock()
+        escalated_iq = safe_escalate_iq(
+            gl=mock_gl,
+            results={},
+            detector_id=self.detector_id,
+            image_bytes=self.image_bytes,
+            patience_time=self.patience_time,
+            confidence_threshold=self.confidence_threshold,
+            human_review=self.human_review,
+            query=self.query,
+            mode=ModeEnum.BINARY,
+        )
+        mock_gl.submit_image_query.assert_called_once()
+        mock_gl.submit_image_query.assert_called_with(
+            detector=self.detector_id,
+            image=self.image_bytes,
+            wait=0,
+            patience_time=self.patience_time,
+            confidence_threshold=self.confidence_threshold,
+            human_review=self.human_review,
+            metadata=self.results_metadata,
+        )
+
+        assert isinstance(escalated_iq, MagicMock)  # This indicates that the SDK result was returned
+
+    def test_escalate_binary_with_exception(self):
+        """Test escalating a binary IQ with a raised exception."""
+        mock_gl = MagicMock()
+        mock_gl.submit_image_query.side_effect = ValueError("Something went wrong while executing submit_image_query.")
+        escalated_iq = safe_escalate_iq(
+            gl=mock_gl,
+            results={},
+            detector_id=self.detector_id,
+            image_bytes=self.image_bytes,
+            patience_time=self.patience_time,
+            confidence_threshold=self.confidence_threshold,
+            human_review=self.human_review,
+            query=self.query,
+            mode=ModeEnum.BINARY,
+        )
+        mock_gl.submit_image_query.assert_called_once()
+        mock_gl.submit_image_query.assert_called_with(
+            detector=self.detector_id,
+            image=self.image_bytes,
+            wait=0,
+            patience_time=self.patience_time,
+            confidence_threshold=self.confidence_threshold,
+            human_review=self.human_review,
+            metadata=self.results_metadata,
+        )
+
+        assert isinstance(
+            escalated_iq, ImageQuery
+        )  # This indicates that we returned an ImageQuery constructed on the edge
+        assert isinstance(escalated_iq.result, BinaryClassificationResult)
+        assert escalated_iq.result.label == Label.UNCLEAR
+        assert escalated_iq.result.confidence == 1.0
+        assert escalated_iq.result.source == Source.ALGORITHM
+        assert (
+            escalated_iq.metadata.get("error_info")
+            == "ValueError: Something went wrong while executing submit_image_query."
+        )
+        assert escalated_iq.metadata.get("is_from_edge")
+
+    def test_escalate_count_with_exception(self):
+        """Test escalating a count IQ with a raised exception."""
+        mock_gl = MagicMock()
+        mock_gl.submit_image_query.side_effect = ValueError("Something went wrong while executing submit_image_query.")
+        escalated_iq = safe_escalate_iq(
+            gl=mock_gl,
+            results={},
+            detector_id=self.detector_id,
+            image_bytes=self.image_bytes,
+            patience_time=self.patience_time,
+            confidence_threshold=self.confidence_threshold,
+            human_review=self.human_review,
+            query=self.query,
+            mode=ModeEnum.COUNT,
+        )
+        mock_gl.submit_image_query.assert_called_once()
+        mock_gl.submit_image_query.assert_called_with(
+            detector=self.detector_id,
+            image=self.image_bytes,
+            wait=0,
+            patience_time=self.patience_time,
+            confidence_threshold=self.confidence_threshold,
+            human_review=self.human_review,
+            metadata=self.results_metadata,
+        )
+
+        assert isinstance(
+            escalated_iq, ImageQuery
+        )  # This indicates that we returned an ImageQuery constructed on the edge
+        assert isinstance(escalated_iq.result, CountingResult)
+        assert escalated_iq.result.count is None
+        assert escalated_iq.result.confidence == 1.0
+        assert escalated_iq.result.source == Source.ALGORITHM
+        assert not escalated_iq.result.greater_than_max
+        assert (
+            escalated_iq.metadata.get("error_info")
+            == "ValueError: Something went wrong while executing submit_image_query."
+        )
+        assert escalated_iq.metadata.get("is_from_edge")
+
+    def test_escalate_multiclass(self):
+        """Test escalating a multiclass IQ."""
+        mock_gl = MagicMock()
+        mock_gl.submit_image_query.side_effect = ValueError("Something went wrong while executing submit_image_query.")
+
+        with pytest.raises(NotImplementedError, match="Multiclass functionality is not yet implemented"):
+            safe_escalate_iq(
+                gl=mock_gl,
+                results={},
+                detector_id=self.detector_id,
+                image_bytes=self.image_bytes,
+                patience_time=self.patience_time,
+                confidence_threshold=self.confidence_threshold,
+                human_review=self.human_review,
+                query=self.query,
+                mode=ModeEnum.MULTI_CLASS,
+            )
+
+
 class TestCreateIQ:
     def setup_method(self):
         self.confidence_threshold = 0.75