From 6cc12151ae776007f5f5dddfd6bd15624ce84889 Mon Sep 17 00:00:00 2001 From: CoreyEWood Date: Tue, 8 Apr 2025 13:33:06 -0700 Subject: [PATCH 1/3] initial progress --- app/api/routes/image_queries.py | 19 ++++++-- app/core/utils.py | 79 ++++++++++++++++++++++++++++++++- 2 files changed, 92 insertions(+), 6 deletions(-) diff --git a/app/api/routes/image_queries.py b/app/api/routes/image_queries.py index 44fe484d2..9b5f97386 100644 --- a/app/api/routes/image_queries.py +++ b/app/api/routes/image_queries.py @@ -1,6 +1,6 @@ import logging import random -from typing import Literal, Optional +from typing import Optional from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, Request, status from groundlight import Groundlight @@ -14,7 +14,7 @@ refresh_detector_metadata_if_needed, ) from app.core.edge_inference import get_edge_inference_model_name -from app.core.utils import create_iq, safe_call_sdk +from app.core.utils import HUMAN_REVIEW_TYPE, create_iq, safe_call_sdk from app.metrics.iqactivity import record_iq_activity logger = logging.getLogger(__name__) @@ -58,7 +58,7 @@ async def post_image_query( # noqa: PLR0913, PLR0915, PLR0912 image_bytes: bytes = Depends(validate_image_bytes), patience_time: Optional[float] = Query(None, ge=0), confidence_threshold: Optional[float] = Query(None, ge=0, le=1), - human_review: Optional[Literal["DEFAULT", "ALWAYS", "NEVER"]] = Query(None), + human_review: HUMAN_REVIEW_TYPE = Query(None), want_async: bool = Query(False), gl: Groundlight = Depends(get_groundlight_sdk_instance), app_state: AppState = Depends(get_app_state), @@ -269,9 +269,20 @@ async def post_image_query( # noqa: PLR0913, PLR0915, PLR0912 gl.submit_image_query, detector=detector_id, image=image_bytes, - wait=0, # wait on the client, not here + wait=-1, # wait on the client, not here patience_time=patience_time, confidence_threshold=confidence_threshold, human_review=human_review, metadata={"edge_result": results}, ) + # return safe_escalate_iq( + # gl=gl, + # results=results, + # detector_id=detector_id, + # image_bytes=image_bytes, + # patience_time=patience_time, + # confidence_threshold=confidence_threshold, + # human_review=human_review, + # query=detector_metadata.query, + # mode=detector_metadata.mode, + # ) diff --git a/app/core/utils.py b/app/core/utils.py index 0f2e71fe3..2f8ef3041 100644 --- a/app/core/utils.py +++ b/app/core/utils.py @@ -2,11 +2,12 @@ import time from datetime import datetime, timezone from io import BytesIO -from typing import Any, Callable +from typing import Any, Callable, Literal, Optional import cachetools import ksuid from fastapi import HTTPException +from groundlight import Groundlight from model import ( ROI, BinaryClassificationResult, @@ -26,6 +27,8 @@ logger = logging.getLogger(__name__) +HUMAN_REVIEW_TYPE = Optional[Literal["DEFAULT", "ALWAYS", "NEVER"]] + def create_iq( # noqa: PLR0913 detector_id: str, @@ -82,7 +85,7 @@ def _mode_to_result_and_type( based on the provided mode, confidence, and result value. :param mode: The mode of the detector. - :param mode_configuration: + :param mode_configuration: For counting only. A dict containing values for the max_count and class_name. :param confidence: The confidence of the predicted value. :param result_value: The predicted value. @@ -138,6 +141,78 @@ def safe_call_sdk(api_method: Callable, **kwargs): raise ex +def _mode_to_unclear_result(mode: ModeEnum): + source = Source.ALGORITHM # TODO what should the source be? + if mode == ModeEnum.BINARY: + result_type = ResultTypeEnum.binary_classification + result = BinaryClassificationResult( + confidence=1.0, + source=source, + label=Label.UNCLEAR, + ) + elif mode == ModeEnum.COUNT: + result_type = ResultTypeEnum.counting + result = CountingResult( + confidence=1.0, + source=source, + count=None, # TODO double-check how to model a counting Unclear result + greater_than_max=False, + ) + elif mode == ModeEnum.MULTI_CLASS: + raise NotImplementedError("Multiclass functionality is not yet implemented for the edge endpoint.") + # TODO add support for multiclass functionality. + else: + raise ValueError(f"Got unrecognized or unsupported detector mode: {mode}") + + return result_type, result + + +def safe_escalate_iq( + gl: Groundlight, + results: dict[str, Any], + detector_id: str, + image_bytes: bytes, + patience_time: float, + confidence_threshold: float, + human_review: HUMAN_REVIEW_TYPE, + query: str, + mode: ModeEnum, +) -> ImageQuery: + """ + This attempts to escalate an image query via the SDK. If it fails, it will catch the exception and return an + ImageQuery with an unclear result. + """ + try: + iq_to_return = safe_call_sdk( + gl.submit_image_query, + detector=detector_id, + image=image_bytes, + wait=0, # wait on the client, not here + patience_time=patience_time, + confidence_threshold=confidence_threshold, + human_review=human_review, + metadata={"edge_result": results}, + ) + except Exception as ex: + result_type, result = _mode_to_unclear_result(mode) + + iq_to_return = ImageQuery( + metadata={"is_from_edge": True, "error_info": str(ex)}, + id=prefixed_ksuid(prefix="iq_"), + type=ImageQueryTypeEnum.image_query, + created_at=datetime.now(timezone.utc), + query=query, + detector_id=detector_id, + result_type=result_type, + result=result, + patience_time=patience_time, + confidence_threshold=confidence_threshold, + rois=None, + text=None, + ) + return iq_to_return + + def prefixed_ksuid(prefix: str | None = None) -> str: """Returns a unique identifier, with a bunch of nice properties. It's statistically guaranteed unique, about as strongly as UUIDv4 are. From e37b8342706ed40ce3ecd0f60bf7a25e5a3e4eda Mon Sep 17 00:00:00 2001 From: CoreyEWood Date: Tue, 8 Apr 2025 14:07:28 -0700 Subject: [PATCH 2/3] this worked in one specific case --- app/api/routes/image_queries.py | 37 +++++++++++++++++---------------- app/core/utils.py | 10 +++++++-- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/app/api/routes/image_queries.py b/app/api/routes/image_queries.py index 9b5f97386..346dc470e 100644 --- a/app/api/routes/image_queries.py +++ b/app/api/routes/image_queries.py @@ -14,7 +14,7 @@ refresh_detector_metadata_if_needed, ) from app.core.edge_inference import get_edge_inference_model_name -from app.core.utils import HUMAN_REVIEW_TYPE, create_iq, safe_call_sdk +from app.core.utils import HUMAN_REVIEW_TYPE, create_iq, safe_call_sdk, safe_escalate_iq from app.metrics.iqactivity import record_iq_activity logger = logging.getLogger(__name__) @@ -265,24 +265,25 @@ async def post_image_query( # noqa: PLR0913, PLR0915, PLR0912 raise AssertionError("Cloud escalation is disabled.") # ...should never reach this point logger.debug(f"Submitting image query to cloud for {detector_id=}") - return safe_call_sdk( - gl.submit_image_query, - detector=detector_id, - image=image_bytes, - wait=-1, # wait on the client, not here - patience_time=patience_time, - confidence_threshold=confidence_threshold, - human_review=human_review, - metadata={"edge_result": results}, - ) - # return safe_escalate_iq( - # gl=gl, - # results=results, - # detector_id=detector_id, - # image_bytes=image_bytes, + # return safe_call_sdk( + # gl.submit_image_query, + # detector=detector_id, + # image=image_bytes, + # wait=5, # wait on the client, not here # patience_time=patience_time, # confidence_threshold=confidence_threshold, # human_review=human_review, - # query=detector_metadata.query, - # mode=detector_metadata.mode, + # metadata={"edge_result": results}, + # want_async=True, # ) + return safe_escalate_iq( + gl=gl, + results=results, + detector_id=detector_id, + image_bytes=image_bytes, + patience_time=patience_time, + confidence_threshold=confidence_threshold, + human_review=human_review, + query=detector_metadata.query, + mode=detector_metadata.mode, + ) diff --git a/app/core/utils.py b/app/core/utils.py index 2f8ef3041..b8e85ae98 100644 --- a/app/core/utils.py +++ b/app/core/utils.py @@ -172,7 +172,7 @@ def safe_escalate_iq( results: dict[str, Any], detector_id: str, image_bytes: bytes, - patience_time: float, + patience_time: float | None, confidence_threshold: float, human_review: HUMAN_REVIEW_TYPE, query: str, @@ -187,15 +187,21 @@ def safe_escalate_iq( gl.submit_image_query, detector=detector_id, image=image_bytes, - wait=0, # wait on the client, not here + wait=5, # wait on the client, not here # TODO revert to 0 + want_async=True, patience_time=patience_time, confidence_threshold=confidence_threshold, human_review=human_review, metadata={"edge_result": results}, ) + logger.info("I called the sdk and there was no exception") except Exception as ex: + logger.info(f"I caught an exception! {ex=}") result_type, result = _mode_to_unclear_result(mode) + if patience_time is None: + patience_time = constants.DEFAULT_PATIENCE_TIME + iq_to_return = ImageQuery( metadata={"is_from_edge": True, "error_info": str(ex)}, id=prefixed_ksuid(prefix="iq_"), From aa3adfa2541193abd51127aa67375bdd22f20da2 Mon Sep 17 00:00:00 2001 From: CoreyEWood Date: Tue, 8 Apr 2025 14:55:35 -0700 Subject: [PATCH 3/3] adding tests --- app/core/utils.py | 10 +-- test/api/test_utils.py | 141 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 4 deletions(-) diff --git a/app/core/utils.py b/app/core/utils.py index b8e85ae98..c33f5921c 100644 --- a/app/core/utils.py +++ b/app/core/utils.py @@ -155,7 +155,7 @@ def _mode_to_unclear_result(mode: ModeEnum): result = CountingResult( confidence=1.0, source=source, - count=None, # TODO double-check how to model a counting Unclear result + count=None, # TODO double-check how to model a counting Unclear result. Also this doesn't work on current SDK version. greater_than_max=False, ) elif mode == ModeEnum.MULTI_CLASS: @@ -187,8 +187,8 @@ def safe_escalate_iq( gl.submit_image_query, detector=detector_id, image=image_bytes, - wait=5, # wait on the client, not here # TODO revert to 0 - want_async=True, + wait=0, # wait on the client, not here # TODO revert to 0 + # want_async=True, patience_time=patience_time, confidence_threshold=confidence_threshold, human_review=human_review, @@ -202,8 +202,10 @@ def safe_escalate_iq( if patience_time is None: patience_time = constants.DEFAULT_PATIENCE_TIME + readable_exception_str = f"{ex.__class__.__name__}: {str(ex)}" + iq_to_return = ImageQuery( - metadata={"is_from_edge": True, "error_info": str(ex)}, + metadata={"is_from_edge": True, "error_info": readable_exception_str}, id=prefixed_ksuid(prefix="iq_"), type=ImageQueryTypeEnum.image_query, created_at=datetime.now(timezone.utc), diff --git a/test/api/test_utils.py b/test/api/test_utils.py index 6ea0a0659..001a386be 100644 --- a/test/api/test_utils.py +++ b/test/api/test_utils.py @@ -1,4 +1,7 @@ +from unittest.mock import MagicMock + import pytest +from groundlight import ImageQuery from model import ( BinaryClassificationResult, CountingResult, @@ -8,16 +11,154 @@ Source, ) +from app.core.constants import DEFAULT_PATIENCE_TIME from app.core.utils import ( + HUMAN_REVIEW_TYPE, ModelInfoBase, ModelInfoNoBinary, ModelInfoWithBinary, create_iq, parse_model_info, prefixed_ksuid, + safe_escalate_iq, ) +class TestSafeEscalateIQ: + def setup_method(self): + self.detector_id = "test" + self.image_bytes: bytes = None + self.patience_time = DEFAULT_PATIENCE_TIME + self.confidence_threshold = 0.9 + self.human_review: HUMAN_REVIEW_TYPE = "NEVER" + self.query = "test query" + self.results_metadata = {"edge_result": {}} + + def test_escalate_no_exception(self): + """Test escalating an IQ with no exception raised.""" + mock_gl = MagicMock() + escalated_iq = safe_escalate_iq( + gl=mock_gl, + results={}, + detector_id=self.detector_id, + image_bytes=self.image_bytes, + patience_time=self.patience_time, + confidence_threshold=self.confidence_threshold, + human_review=self.human_review, + query=self.query, + mode=ModeEnum.BINARY, + ) + mock_gl.submit_image_query.assert_called_once() + mock_gl.submit_image_query.assert_called_with( + detector=self.detector_id, + image=self.image_bytes, + wait=0, + patience_time=self.patience_time, + confidence_threshold=self.confidence_threshold, + human_review=self.human_review, + metadata=self.results_metadata, + ) + + assert isinstance(escalated_iq, MagicMock) # This indicates that the SDK result was returned + + def test_escalate_binary_with_exception(self): + """Test escalating a binary IQ with a raised exception.""" + mock_gl = MagicMock() + mock_gl.submit_image_query.side_effect = ValueError("Something went wrong while executing submit_image_query.") + escalated_iq = safe_escalate_iq( + gl=mock_gl, + results={}, + detector_id=self.detector_id, + image_bytes=self.image_bytes, + patience_time=self.patience_time, + confidence_threshold=self.confidence_threshold, + human_review=self.human_review, + query=self.query, + mode=ModeEnum.BINARY, + ) + mock_gl.submit_image_query.assert_called_once() + mock_gl.submit_image_query.assert_called_with( + detector=self.detector_id, + image=self.image_bytes, + wait=0, + patience_time=self.patience_time, + confidence_threshold=self.confidence_threshold, + human_review=self.human_review, + metadata=self.results_metadata, + ) + + assert isinstance( + escalated_iq, ImageQuery + ) # This indicates that we returned an ImageQuery constructed on the edge + assert isinstance(escalated_iq.result, BinaryClassificationResult) + assert escalated_iq.result.label == Label.UNCLEAR + assert escalated_iq.result.confidence == 1.0 + assert escalated_iq.result.source == Source.ALGORITHM + assert ( + escalated_iq.metadata.get("error_info") + == "ValueError: Something went wrong while executing submit_image_query." + ) + assert escalated_iq.metadata.get("is_from_edge") + + def test_escalate_count_with_exception(self): + """Test escalating a count IQ with a raised exception.""" + mock_gl = MagicMock() + mock_gl.submit_image_query.side_effect = ValueError("Something went wrong while executing submit_image_query.") + escalated_iq = safe_escalate_iq( + gl=mock_gl, + results={}, + detector_id=self.detector_id, + image_bytes=self.image_bytes, + patience_time=self.patience_time, + confidence_threshold=self.confidence_threshold, + human_review=self.human_review, + query=self.query, + mode=ModeEnum.COUNT, + ) + mock_gl.submit_image_query.assert_called_once() + mock_gl.submit_image_query.assert_called_with( + detector=self.detector_id, + image=self.image_bytes, + wait=0, + patience_time=self.patience_time, + confidence_threshold=self.confidence_threshold, + human_review=self.human_review, + metadata=self.results_metadata, + ) + + assert isinstance( + escalated_iq, ImageQuery + ) # This indicates that we returned an ImageQuery constructed on the edge + assert isinstance(escalated_iq.result, CountingResult) + assert escalated_iq.result.count is None + assert escalated_iq.result.confidence == 1.0 + assert escalated_iq.result.source == Source.ALGORITHM + assert not escalated_iq.result.greater_than_max + assert ( + escalated_iq.metadata.get("error_info") + == "ValueError: Something went wrong while executing submit_image_query." + ) + assert escalated_iq.metadata.get("is_from_edge") + + def test_escalate_multiclass(self): + """Test escalating a multiclass IQ.""" + mock_gl = MagicMock() + mock_gl.submit_image_query.side_effect = ValueError("Something went wrong while executing submit_image_query.") + + with pytest.raises(NotImplementedError, match="Multiclass functionality is not yet implemented"): + safe_escalate_iq( + gl=mock_gl, + results={}, + detector_id=self.detector_id, + image_bytes=self.image_bytes, + patience_time=self.patience_time, + confidence_threshold=self.confidence_threshold, + human_review=self.human_review, + query=self.query, + mode=ModeEnum.MULTI_CLASS, + ) + + class TestCreateIQ: def setup_method(self): self.confidence_threshold = 0.75