From ba695d5e824e71ddfaa008713209d4fc736b608b Mon Sep 17 00:00:00 2001 From: CoreyEWood Date: Tue, 14 Jan 2025 20:43:54 +0000 Subject: [PATCH 01/10] small formatting --- test/setup_k3s_test_environment.sh | 1 - test/validate_setup_ee.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/test/setup_k3s_test_environment.sh b/test/setup_k3s_test_environment.sh index c51fe90f8..c36485e20 100755 --- a/test/setup_k3s_test_environment.sh +++ b/test/setup_k3s_test_environment.sh @@ -53,4 +53,3 @@ if ! kubectl rollout status deployment/edge-endpoint -n $DEPLOYMENT_NAMESPACE -- fi echo "Edge-endpoint pods have successfully rolled out." - diff --git a/test/validate_setup_ee.sh b/test/validate_setup_ee.sh index acf304afc..c99d4ba20 100755 --- a/test/validate_setup_ee.sh +++ b/test/validate_setup_ee.sh @@ -18,5 +18,5 @@ echo "Edge-endpoint pods have successfully rolled out in namespace $DEPLOYMENT_N echo "Deleting namespace $DEPLOYMENT_NAMESPACE..." kubectl delete namespace $DEPLOYMENT_NAMESPACE -echo "Deleting persistant volume..." +echo "Deleting persistent volume..." kubectl delete pv edge-endpoint-pv \ No newline at end of file From 13530990a65d581a8d916198294539f7eb208415 Mon Sep 17 00:00:00 2001 From: CoreyEWood Date: Tue, 14 Jan 2025 22:01:46 +0000 Subject: [PATCH 02/10] tests for local inference config behaviors --- pyproject.toml | 3 + test/api/test_image_queries_live.py | 228 ++++++++++++++++++++++++---- test/setup_k3s_test_environment.sh | 38 +++++ 3 files changed, 241 insertions(+), 28 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e2e117600..b129651c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,9 @@ trailing_comma_inline_array = true [tool.pytest.ini_options] testpaths = ["test"] +markers = [ + "live: marks tests that will be run against the live edge endpoint", +] [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/test/api/test_image_queries_live.py b/test/api/test_image_queries_live.py index 8dc9fe134..f1938b42e 100644 --- a/test/api/test_image_queries_live.py +++ b/test/api/test_image_queries_live.py @@ -4,8 +4,7 @@ import pytest import requests from fastapi import status -from groundlight import ApiException, Groundlight -from model import Detector +from groundlight import ApiException, Detector, Groundlight, ImageQuery from PIL import Image from app.core.utils import pil_image_to_bytes @@ -15,14 +14,32 @@ TEST_ENDPOINT = os.getenv("LIVE_TEST_ENDPOINT", "http://localhost:30101") MAX_WAIT_TIME_S = 60 -# Detector ID associated with the detector with parameters -# - name="edge_testing_det", +# Detectors for live testing. On the prod-biggies account. +# - name="live_edge_testing_1", # - query="Is there a dog in the image?", # - confidence_threshold=0.9 -DETECTOR_ID = "det_2SagpFUrs83cbMZsap5hZzRjZw4" +DETECTOR_ID_1 = "det_2raefZ74V0ojgbmM2UJzQCpFKyF" +# - name="live_edge_testing_2", +# - query="Is there a dog in the image?", +# - confidence_threshold=0.9 +DETECTOR_ID_2 = "det_2rdUY6SJOBJtuW5oqD3ExL1DjFn" +# - name="live_edge_testing_3", +# - query="Is there a dog in the image?", +# - confidence_threshold=0.9 +DETECTOR_ID_3 = "det_2rdUb0jljHCosfKGuTugVoo4eiY" +# - name="live_edge_testing_4", +# - query="Is there a dog in the image?", +# - confidence_threshold=0.9 +DETECTOR_ID_4 = "det_2rdVBErF53NWjVjhVdIrb6QJbRT" + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Fixtures +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -@pytest.mark.live @pytest.fixture(scope="module", autouse=True) def ensure_edge_endpoint_is_live_and_ready(): """Ensure that the edge-endpoint server is live and ready before running tests.""" @@ -49,33 +66,188 @@ def fixture_gl() -> Groundlight: @pytest.fixture -def detector(gl: Groundlight) -> Detector: - """Retrieve the detector using the Groundlight client.""" - return gl.get_detector(id=DETECTOR_ID) +def detector_default(gl: Groundlight) -> Detector: + """Retrieve the default detector using the Groundlight client.""" + return gl.get_detector(id=DETECTOR_ID_1) -@pytest.mark.live -def test_post_image_query_via_sdk(gl: Groundlight, detector: Detector): - """Test that submitting an image query using the edge server proceeds without failure.""" - image_bytes = pil_image_to_bytes(img=Image.open("test/assets/dog.jpeg")) - iq = gl.submit_image_query(detector=detector.id, image=image_bytes, wait=10.0) - assert iq is not None, "ImageQuery should not be None." +@pytest.fixture +def detector_edge_answers(gl: Groundlight) -> Detector: + """Retrieve the edge answers detector using the Groundlight client.""" + return gl.get_detector(id=DETECTOR_ID_2) + + +@pytest.fixture +def detector_no_cloud(gl: Groundlight) -> Detector: + """Retrieve the no cloud detector using the Groundlight client.""" + return gl.get_detector(id=DETECTOR_ID_3) + + +@pytest.fixture +def detector_disabled(gl: Groundlight) -> Detector: + """Retrieve the disabled detector using the Groundlight client.""" + return gl.get_detector(id=DETECTOR_ID_4) + + +@pytest.fixture +def image_bytes() -> bytes: + """Return the test image as bytes.""" + return pil_image_to_bytes(img=Image.open("test/assets/dog.jpeg")) + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Helpers +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +def answer_is_from_cloud(iq: ImageQuery) -> bool: + """Return True if the answer is from the cloud, False otherwise.""" + return not iq.metadata or not iq.metadata.get("is_from_edge", False) + + +def answer_is_from_edge(iq: ImageQuery) -> bool: + """Return True if the answer is from the edge, False otherwise.""" + return iq.metadata and iq.metadata.get("is_from_edge", False) + + +def was_escalated(gl: Groundlight, iq: ImageQuery, max_retries: int = 3, retry_delay: float = 1.0) -> bool: + """Return True if the answer was escalated to the cloud, False otherwise. + Retries up to max_retries times, waiting retry_delay seconds between retries, to account for the time it takes for + the cloud to process the image query. + + Args: + gl: Groundlight client + iq: ImageQuery to check + max_retries: Maximum number of retry attempts + retry_delay: Delay in seconds between retries + """ + for attempt in range(max_retries): + try: + gl.get_image_query(id=iq.id) + return True + except ApiException as e: + if e.status == status.HTTP_404_NOT_FOUND and attempt < max_retries - 1: + time.sleep(retry_delay) + continue + if e.status == status.HTTP_404_NOT_FOUND: + return False + raise + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Tests +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @pytest.mark.live -def test_post_image_query_via_sdk_want_async(gl: Groundlight, detector: Detector): - """Test that submitting an image query with want_async=True forwards directly to the cloud.""" - image_bytes = pil_image_to_bytes(img=Image.open("test/assets/dog.jpeg")) - iq = gl.ask_async(detector=detector.id, image=image_bytes) - assert iq is not None, "ImageQuery should not be None." - assert iq.id.startswith("iq_"), "ImageQuery id should start with 'iq_' because it was created on the cloud." - assert iq.result is None, "Result should be None because the query is still being processed." +class TestSubmittingToLocalInferenceConfigs: + """Tests for submitting image queries with different detector configurations.""" + + class TestDefaultConfig: + """Tests for default detector configuration behavior.""" + + def test_high_threshold_goes_to_cloud(self, gl: Groundlight, detector_default: Detector, image_bytes: bytes): + iq = gl.submit_image_query( + detector=detector_default.id, image=image_bytes, confidence_threshold=1, wait=0 + ) # TODO is this dependent on getting a fast cloud response? + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_cloud(iq), "Answer should be from the cloud." + + def test_low_threshold_comes_from_edge(self, gl: Groundlight, detector_default: Detector, image_bytes: bytes): + iq = gl.submit_image_query( + detector=detector_default.id, image=image_bytes, confidence_threshold=0.5, wait=0 + ) + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_edge(iq), "Answer should be from the edge." + + class TestEdgeAnswersConfig: + """Tests for edge_answers_with_escalation detector configuration.""" + + def test_high_threshold_comes_from_edge_and_escalated( + self, gl: Groundlight, detector_edge_answers: Detector, image_bytes: bytes + ): + iq = gl.submit_image_query( + detector=detector_edge_answers.id, image=image_bytes, confidence_threshold=1, wait=0 + ) + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_edge(iq), "Answer should be from the edge." + assert was_escalated(gl, iq), "Answer should be escalated." + + def test_low_threshold_comes_from_edge( + self, gl: Groundlight, detector_edge_answers: Detector, image_bytes: bytes + ): + iq = gl.submit_image_query( + detector=detector_edge_answers.id, image=image_bytes, confidence_threshold=0.5, wait=0 + ) + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_edge(iq), "Answer should be from the edge." + assert not was_escalated(gl, iq), "Answer should not be escalated." + + class TestNoCloudConfig: + """Tests for no_cloud detector configuration.""" + + def test_high_threshold_comes_from_edge_not_escalated(self, gl, detector_no_cloud, image_bytes): + iq = gl.submit_image_query(detector=detector_no_cloud.id, image=image_bytes, confidence_threshold=1, wait=0) + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_edge(iq), "Answer should be from the edge." + assert not was_escalated(gl, iq), "Answer should not be escalated." + + def test_low_threshold_comes_from_edge(self, gl, detector_no_cloud, image_bytes): + iq = gl.submit_image_query( + detector=detector_no_cloud.id, image=image_bytes, confidence_threshold=0.5, wait=0 + ) + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_edge(iq), "Answer should be from the edge." + assert not was_escalated(gl, iq), "Answer should not be escalated." + + class TestDisabledConfig: + """Tests for disabled detector configuration.""" + + def test_low_threshold_goes_to_cloud(self, gl: Groundlight, detector_disabled: Detector, image_bytes: bytes): + iq = gl.submit_image_query( + detector=detector_disabled.id, image=image_bytes, confidence_threshold=0.5, wait=0 + ) + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_cloud(iq), "Answer should be from the cloud." @pytest.mark.live -def test_post_image_query_via_sdk_with_metadata_throws_400(gl: Groundlight, detector: Detector): - """Test that submitting an image query with metadata raises a 400 error.""" - image_bytes = pil_image_to_bytes(img=Image.open("test/assets/dog.jpeg")) - with pytest.raises(ApiException) as exc_info: - gl.submit_image_query(detector=detector.id, image=image_bytes, wait=10.0, metadata={"foo": "bar"}) - assert exc_info.value.status == status.HTTP_400_BAD_REQUEST +class TestEdgeAnswerRequirements: + """Tests for edge-answer requirements and error conditions.""" + + @pytest.mark.parametrize("detector_fixture", ["detector_edge_answers", "detector_no_cloud"]) + def test_human_review_not_allowed(self, gl, request, detector_fixture, image_bytes): ... + + @pytest.mark.parametrize("detector_fixture", ["detector_edge_answers", "detector_no_cloud"]) + def test_want_async_not_allowed(self, gl, request, detector_fixture, image_bytes): ... + + @pytest.mark.parametrize("detector_fixture", ["detector_edge_answers", "detector_no_cloud"]) + def test_edge_inference_unavailable_errors(self, gl, request, detector_fixture, image_bytes): ... + + +# @pytest.mark.live +# def test_post_image_query_via_sdk(gl: Groundlight, detector: Detector, image_bytes: bytes): +# """Test that submitting an image query using the edge server proceeds without failure.""" +# iq = gl.submit_image_query(detector=detector.id, image=image_bytes, wait=10.0) +# assert iq is not None, "ImageQuery should not be None." + + +# @pytest.mark.live +# def test_post_image_query_via_sdk_want_async(gl: Groundlight, detector: Detector, image_bytes: bytes): +# """Test that submitting an image query with want_async=True forwards directly to the cloud.""" +# iq = gl.ask_async(detector=detector.id, image=image_bytes) +# assert iq is not None, "ImageQuery should not be None." +# assert iq.id.startswith("iq_"), "ImageQuery id should start with 'iq_' because it was created on the cloud." +# assert iq.result is None, "Result should be None because the query is still being processed." + + +# @pytest.mark.live +# def test_post_image_query_via_sdk_with_metadata_throws_400(gl: Groundlight, detector: Detector, image_bytes: bytes): +# """Test that submitting an image query with metadata raises a 400 error.""" +# with pytest.raises(ApiException) as exc_info: +# gl.submit_image_query(detector=detector.id, image=image_bytes, wait=10.0, metadata={"foo": "bar"}) +# assert exc_info.value.status == status.HTTP_400_BAD_REQUEST diff --git a/test/setup_k3s_test_environment.sh b/test/setup_k3s_test_environment.sh index c36485e20..67ed58248 100755 --- a/test/setup_k3s_test_environment.sh +++ b/test/setup_k3s_test_environment.sh @@ -40,6 +40,44 @@ echo "Building the Docker image..." export IMAGE_TAG=$(./deploy/bin/git-tag-name.sh) export INFERENCE_FLAVOR="CPU" + +EDGE_CONFIG=$(cat <<- EOM +global_config: + refresh_rate: 60 + +edge_inference_configs: + default: + enabled: true + always_return_edge_prediction: false + disable_cloud_escalation: false + + edge_answers_with_escalation: + enabled: true + always_return_edge_prediction: true + disable_cloud_escalation: false + min_time_between_escalations: 2.0 + + no_cloud: + enabled: true + always_return_edge_prediction: true + disable_cloud_escalation: true + + disabled: + enabled: false + +detectors: + - detector_id: "det_2raefZ74V0ojgbmM2UJzQCpFKyF" + edge_inference_config: "default" + - detector_id: "det_2rdUY6SJOBJtuW5oqD3ExL1DjFn" + edge_inference_config: "edge_answers_with_escalation" + - detector_id: "det_2rdUb0jljHCosfKGuTugVoo4eiY" + edge_inference_config: "no_cloud" + - detector_id: "det_2rdVBErF53NWjVjhVdIrb6QJbRT" + edge_inference_config: "disabled" +EOM +) +export EDGE_CONFIG + ./deploy/bin/setup-ee.sh From ee7328085907998040cffbef154ac88f71a5f656 Mon Sep 17 00:00:00 2001 From: CoreyEWood Date: Wed, 15 Jan 2025 21:13:49 +0000 Subject: [PATCH 03/10] progress on query params testing --- test/api/test_image_queries_live.py | 78 ++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 7 deletions(-) diff --git a/test/api/test_image_queries_live.py b/test/api/test_image_queries_live.py index f1938b42e..380f49436 100644 --- a/test/api/test_image_queries_live.py +++ b/test/api/test_image_queries_live.py @@ -216,17 +216,81 @@ def test_low_threshold_goes_to_cloud(self, gl: Groundlight, detector_disabled: D @pytest.mark.live -class TestEdgeAnswerRequirements: - """Tests for edge-answer requirements and error conditions.""" +class TestEdgeQueryParams: + """Testing behavior of submit_image_query parameters on edge.""" @pytest.mark.parametrize("detector_fixture", ["detector_edge_answers", "detector_no_cloud"]) - def test_human_review_not_allowed(self, gl, request, detector_fixture, image_bytes): ... + def test_human_review_not_allowed( + self, gl: Groundlight, request: pytest.FixtureRequest, detector_fixture: str, image_bytes: bytes + ): + """Test that human_review cannot be specified when edge answers are required.""" + detector = request.getfixturevalue(detector_fixture) + with pytest.raises(ApiException) as exc_info: + gl.submit_image_query(detector=detector.id, image=image_bytes, human_review="ALWAYS") + assert exc_info.value.status == status.HTTP_400_BAD_REQUEST @pytest.mark.parametrize("detector_fixture", ["detector_edge_answers", "detector_no_cloud"]) - def test_want_async_not_allowed(self, gl, request, detector_fixture, image_bytes): ... - - @pytest.mark.parametrize("detector_fixture", ["detector_edge_answers", "detector_no_cloud"]) - def test_edge_inference_unavailable_errors(self, gl, request, detector_fixture, image_bytes): ... + def test_want_async_not_allowed( + self, gl: Groundlight, request: pytest.FixtureRequest, detector_fixture: str, image_bytes: bytes + ): + """Test that want_async cannot be specified when edge answers are required.""" + detector = request.getfixturevalue(detector_fixture) + with pytest.raises(ApiException) as exc_info: + gl.submit_image_query(detector=detector.id, image=image_bytes, want_async=True, wait=0) + assert exc_info.value.status == status.HTTP_400_BAD_REQUEST + + def test_always_human_review_goes_to_cloud(self, gl: Groundlight, detector_default: Detector, image_bytes: bytes): + """Test that human_review=ALWAYS goes to the cloud even if the edge answer is sufficiently confident.""" + iq = gl.submit_image_query( + detector=detector_default.id, image=image_bytes, human_review="ALWAYS", confidence_threshold=0.5, wait=0 + ) + assert iq is not None + assert answer_is_from_cloud(iq), "Answer should be from the cloud." + + def test_want_async_goes_to_cloud(self, gl: Groundlight, detector_default: Detector, image_bytes: bytes): + """Test that want_async=True goes to the cloud even if the edge answer is sufficiently confident.""" + iq = gl.submit_image_query( + detector=detector_default.id, image=image_bytes, want_async=True, confidence_threshold=0.5, wait=0 + ) + assert iq is not None + assert answer_is_from_cloud(iq), "Answer should be from the cloud." + + def test_supported_params_dont_error(self, gl: Groundlight, detector_default: Detector, image_bytes: bytes): + """Test that supported parameters work without errors.""" + iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, wait=1.0) + assert iq is not None + + iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, patience_time=1.0) + assert iq is not None + + iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, confidence_threshold=0.8) + assert iq is not None + + iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, human_review="NEVER") + assert iq is not None + + iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, want_async=False) + assert iq is not None + + @pytest.mark.parametrize( + "unsupported_param", + [ + {"inspection_id": "insp_123"}, + {"metadata": {"test": "value"}}, + {"image_query_id": "iq_123"}, + ], + ) + def test_unsupported_params_raise_error( + self, + gl: Groundlight, + detector_default: Detector, + image_bytes: bytes, + unsupported_param: dict, + ): + """Test that unsupported parameters raise a 400 error.""" + with pytest.raises(ApiException) as exc_info: + gl.submit_image_query(detector=detector_default.id, image=image_bytes, **unsupported_param) + assert exc_info.value.status == status.HTTP_400_BAD_REQUEST # @pytest.mark.live From 8a54fb11d5b03937d824b7566d7fd5969b642a22 Mon Sep 17 00:00:00 2001 From: CoreyEWood Date: Wed, 19 Feb 2025 16:21:25 -0800 Subject: [PATCH 04/10] remove dynamic detector id from other integration tests (might re-add?) --- test/api/test_image_queries_live.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/api/test_image_queries_live.py b/test/api/test_image_queries_live.py index 5eee3b29e..5fb8b8493 100644 --- a/test/api/test_image_queries_live.py +++ b/test/api/test_image_queries_live.py @@ -32,9 +32,6 @@ # - confidence_threshold=0.9 DETECTOR_ID_4 = "det_2rdVBErF53NWjVjhVdIrb6QJbRT" -# we use a dynamically created detector for integration tests -DETECTOR_ID = os.getenv("DETECTOR_ID", "det_2SagpFUrs83cbMZsap5hZzRjZw4") - # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Fixtures From 9626501da6a28cd38c0c6c7ae98d6be13ab46aab Mon Sep 17 00:00:00 2001 From: CoreyEWood Date: Tue, 8 Apr 2025 22:51:12 +0000 Subject: [PATCH 05/10] move changed files to _2 versions --- test/api/test_image_queries_live.py | 297 +++--------------------- test/api/test_image_queries_live_2.py | 316 ++++++++++++++++++++++++++ test/setup_k3s_test_environment.sh | 37 --- test/setup_k3s_test_environment_2.sh | 93 ++++++++ 4 files changed, 439 insertions(+), 304 deletions(-) create mode 100644 test/api/test_image_queries_live_2.py create mode 100644 test/setup_k3s_test_environment_2.sh diff --git a/test/api/test_image_queries_live.py b/test/api/test_image_queries_live.py index 5fb8b8493..b43d1b0af 100644 --- a/test/api/test_image_queries_live.py +++ b/test/api/test_image_queries_live.py @@ -4,41 +4,25 @@ import pytest import requests from fastapi import status -from groundlight import ApiException, Detector, Groundlight, ImageQuery +from groundlight import ApiException, Groundlight +from model import Detector from PIL import Image from app.core.utils import pil_image_to_bytes - -# Tests in this file require a live edge-endpoint server and GL Api token in order to run. -# Not ideal for unit-testing. + @@ -15,16 +14,31 @@ TEST_ENDPOINT = os.getenv("LIVE_TEST_ENDPOINT", "http://localhost:30101") MAX_WAIT_TIME_S = 60 -# Detectors for live testing. On the prod-biggies account. -# - name="live_edge_testing_1", -# - query="Is there a dog in the image?", -# - confidence_threshold=0.9 -DETECTOR_ID_1 = "det_2raefZ74V0ojgbmM2UJzQCpFKyF" -# - name="live_edge_testing_2", -# - query="Is there a dog in the image?", -# - confidence_threshold=0.9 -DETECTOR_ID_2 = "det_2rdUY6SJOBJtuW5oqD3ExL1DjFn" -# - name="live_edge_testing_3", -# - query="Is there a dog in the image?", -# - confidence_threshold=0.9 -DETECTOR_ID_3 = "det_2rdUb0jljHCosfKGuTugVoo4eiY" -# - name="live_edge_testing_4", +# Detector ID associated with the detector with parameters +# - name="edge_testing_det", # - query="Is there a dog in the image?", # - confidence_threshold=0.9 -DETECTOR_ID_4 = "det_2rdVBErF53NWjVjhVdIrb6QJbRT" -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# Fixtures -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# we use a dynamically created detector for integration tests +DETECTOR_ID = os.getenv("DETECTOR_ID", "det_2SagpFUrs83cbMZsap5hZzRjZw4") +@pytest.mark.live @pytest.fixture(scope="module", autouse=True) def ensure_edge_endpoint_is_live_and_ready(): """Ensure that the edge-endpoint server is live and ready before running tests.""" @@ -56,8 +40,6 @@ def ensure_edge_endpoint_is_live_and_ready(): final_exception = e time.sleep(1) # wait for 1 second before retrying pytest.fail(f"Edge endpoint is not live and ready after polling for {MAX_WAIT_TIME_S} seconds. {final_exception=}") - - @pytest.fixture(name="gl") def fixture_gl() -> Groundlight: """Create a Groundlight client object.""" @@ -65,252 +47,33 @@ def fixture_gl() -> Groundlight: @pytest.fixture -def detector_default(gl: Groundlight) -> Detector: - """Retrieve the default detector using the Groundlight client.""" - return gl.get_detector(id=DETECTOR_ID_1) - - -@pytest.fixture -def detector_edge_answers(gl: Groundlight) -> Detector: - """Retrieve the edge answers detector using the Groundlight client.""" - return gl.get_detector(id=DETECTOR_ID_2) - - -@pytest.fixture -def detector_no_cloud(gl: Groundlight) -> Detector: - """Retrieve the no cloud detector using the Groundlight client.""" - return gl.get_detector(id=DETECTOR_ID_3) - - -@pytest.fixture -def detector_disabled(gl: Groundlight) -> Detector: - """Retrieve the disabled detector using the Groundlight client.""" - return gl.get_detector(id=DETECTOR_ID_4) - - -@pytest.fixture -def image_bytes() -> bytes: - """Return the test image as bytes.""" - return pil_image_to_bytes(img=Image.open("test/assets/dog.jpeg")) - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# Helpers -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -def answer_is_from_cloud(iq: ImageQuery) -> bool: - """Return True if the answer is from the cloud, False otherwise.""" - return not iq.metadata or not iq.metadata.get("is_from_edge", False) - - -def answer_is_from_edge(iq: ImageQuery) -> bool: - """Return True if the answer is from the edge, False otherwise.""" - return iq.metadata and iq.metadata.get("is_from_edge", False) - - -def was_escalated(gl: Groundlight, iq: ImageQuery, max_retries: int = 3, retry_delay: float = 1.0) -> bool: - """Return True if the answer was escalated to the cloud, False otherwise. - Retries up to max_retries times, waiting retry_delay seconds between retries, to account for the time it takes for - the cloud to process the image query. - - Args: - gl: Groundlight client - iq: ImageQuery to check - max_retries: Maximum number of retry attempts - retry_delay: Delay in seconds between retries - """ - for attempt in range(max_retries): - try: - gl.get_image_query(id=iq.id) - return True - except ApiException as e: - if e.status == status.HTTP_404_NOT_FOUND and attempt < max_retries - 1: - time.sleep(retry_delay) - continue - if e.status == status.HTTP_404_NOT_FOUND: - return False - raise - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# Tests -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +def detector(gl: Groundlight) -> Detector: + """Retrieve the detector using the Groundlight client.""" + return gl.get_detector(id=DETECTOR_ID) @pytest.mark.live -class TestSubmittingToLocalInferenceConfigs: - """Tests for submitting image queries with different detector configurations.""" - - class TestDefaultConfig: - """Tests for default detector configuration behavior.""" - - def test_high_threshold_goes_to_cloud(self, gl: Groundlight, detector_default: Detector, image_bytes: bytes): - iq = gl.submit_image_query( - detector=detector_default.id, image=image_bytes, confidence_threshold=1, wait=0 - ) # TODO is this dependent on getting a fast cloud response? - assert iq is not None, "ImageQuery should not be None." - assert answer_is_from_cloud(iq), "Answer should be from the cloud." - - def test_low_threshold_comes_from_edge(self, gl: Groundlight, detector_default: Detector, image_bytes: bytes): - iq = gl.submit_image_query( - detector=detector_default.id, image=image_bytes, confidence_threshold=0.5, wait=0 - ) - assert iq is not None, "ImageQuery should not be None." - assert answer_is_from_edge(iq), "Answer should be from the edge." - - class TestEdgeAnswersConfig: - """Tests for edge_answers_with_escalation detector configuration.""" - - def test_high_threshold_comes_from_edge_and_escalated( - self, gl: Groundlight, detector_edge_answers: Detector, image_bytes: bytes - ): - iq = gl.submit_image_query( - detector=detector_edge_answers.id, image=image_bytes, confidence_threshold=1, wait=0 - ) - assert iq is not None, "ImageQuery should not be None." - assert answer_is_from_edge(iq), "Answer should be from the edge." - assert was_escalated(gl, iq), "Answer should be escalated." - - def test_low_threshold_comes_from_edge( - self, gl: Groundlight, detector_edge_answers: Detector, image_bytes: bytes - ): - iq = gl.submit_image_query( - detector=detector_edge_answers.id, image=image_bytes, confidence_threshold=0.5, wait=0 - ) - assert iq is not None, "ImageQuery should not be None." - assert answer_is_from_edge(iq), "Answer should be from the edge." - assert not was_escalated(gl, iq), "Answer should not be escalated." - - class TestNoCloudConfig: - """Tests for no_cloud detector configuration.""" - - def test_high_threshold_comes_from_edge_not_escalated(self, gl, detector_no_cloud, image_bytes): - iq = gl.submit_image_query(detector=detector_no_cloud.id, image=image_bytes, confidence_threshold=1, wait=0) - assert iq is not None, "ImageQuery should not be None." - assert answer_is_from_edge(iq), "Answer should be from the edge." - assert not was_escalated(gl, iq), "Answer should not be escalated." - - def test_low_threshold_comes_from_edge(self, gl, detector_no_cloud, image_bytes): - iq = gl.submit_image_query( - detector=detector_no_cloud.id, image=image_bytes, confidence_threshold=0.5, wait=0 - ) - assert iq is not None, "ImageQuery should not be None." - assert answer_is_from_edge(iq), "Answer should be from the edge." - assert not was_escalated(gl, iq), "Answer should not be escalated." - - class TestDisabledConfig: - """Tests for disabled detector configuration.""" - - def test_low_threshold_goes_to_cloud(self, gl: Groundlight, detector_disabled: Detector, image_bytes: bytes): - iq = gl.submit_image_query( - detector=detector_disabled.id, image=image_bytes, confidence_threshold=0.5, wait=0 - ) - assert iq is not None, "ImageQuery should not be None." - assert answer_is_from_cloud(iq), "Answer should be from the cloud." +def test_post_image_query_via_sdk(gl: Groundlight, detector: Detector): + """Test that submitting an image query using the edge server proceeds without failure.""" + image_bytes = pil_image_to_bytes(img=Image.open("test/assets/dog.jpeg")) + iq = gl.submit_image_query(detector=detector.id, image=image_bytes, wait=10.0) + assert iq is not None, "ImageQuery should not be None." @pytest.mark.live -class TestEdgeQueryParams: - """Testing behavior of submit_image_query parameters on edge.""" +def test_post_image_query_via_sdk_want_async(gl: Groundlight, detector: Detector): + """Test that submitting an image query with want_async=True forwards directly to the cloud.""" + image_bytes = pil_image_to_bytes(img=Image.open("test/assets/dog.jpeg")) + iq = gl.ask_async(detector=detector.id, image=image_bytes) + assert iq is not None, "ImageQuery should not be None." + assert iq.id.startswith("iq_"), "ImageQuery id should start with 'iq_' because it was created on the cloud." + assert iq.result is None, "Result should be None because the query is still being processed." - @pytest.mark.parametrize("detector_fixture", ["detector_edge_answers", "detector_no_cloud"]) - def test_human_review_not_allowed( - self, gl: Groundlight, request: pytest.FixtureRequest, detector_fixture: str, image_bytes: bytes - ): - """Test that human_review cannot be specified when edge answers are required.""" - detector = request.getfixturevalue(detector_fixture) - with pytest.raises(ApiException) as exc_info: - gl.submit_image_query(detector=detector.id, image=image_bytes, human_review="ALWAYS") - assert exc_info.value.status == status.HTTP_400_BAD_REQUEST - @pytest.mark.parametrize("detector_fixture", ["detector_edge_answers", "detector_no_cloud"]) - def test_want_async_not_allowed( - self, gl: Groundlight, request: pytest.FixtureRequest, detector_fixture: str, image_bytes: bytes - ): - """Test that want_async cannot be specified when edge answers are required.""" - detector = request.getfixturevalue(detector_fixture) - with pytest.raises(ApiException) as exc_info: - gl.submit_image_query(detector=detector.id, image=image_bytes, want_async=True, wait=0) - assert exc_info.value.status == status.HTTP_400_BAD_REQUEST - - def test_always_human_review_goes_to_cloud(self, gl: Groundlight, detector_default: Detector, image_bytes: bytes): - """Test that human_review=ALWAYS goes to the cloud even if the edge answer is sufficiently confident.""" - iq = gl.submit_image_query( - detector=detector_default.id, image=image_bytes, human_review="ALWAYS", confidence_threshold=0.5, wait=0 - ) - assert iq is not None - assert answer_is_from_cloud(iq), "Answer should be from the cloud." - - def test_want_async_goes_to_cloud(self, gl: Groundlight, detector_default: Detector, image_bytes: bytes): - """Test that want_async=True goes to the cloud even if the edge answer is sufficiently confident.""" - iq = gl.submit_image_query( - detector=detector_default.id, image=image_bytes, want_async=True, confidence_threshold=0.5, wait=0 - ) - assert iq is not None - assert answer_is_from_cloud(iq), "Answer should be from the cloud." - - def test_supported_params_dont_error(self, gl: Groundlight, detector_default: Detector, image_bytes: bytes): - """Test that supported parameters work without errors.""" - iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, wait=1.0) - assert iq is not None - - iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, patience_time=1.0) - assert iq is not None - - iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, confidence_threshold=0.8) - assert iq is not None - - iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, human_review="NEVER") - assert iq is not None - - iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, want_async=False) - assert iq is not None - - @pytest.mark.parametrize( - "unsupported_param", - [ - {"inspection_id": "insp_123"}, - {"metadata": {"test": "value"}}, - {"image_query_id": "iq_123"}, - ], - ) - def test_unsupported_params_raise_error( - self, - gl: Groundlight, - detector_default: Detector, - image_bytes: bytes, - unsupported_param: dict, - ): - """Test that unsupported parameters raise a 400 error.""" - with pytest.raises(ApiException) as exc_info: - gl.submit_image_query(detector=detector_default.id, image=image_bytes, **unsupported_param) - assert exc_info.value.status == status.HTTP_400_BAD_REQUEST - - -# @pytest.mark.live -# def test_post_image_query_via_sdk(gl: Groundlight, detector: Detector, image_bytes: bytes): -# """Test that submitting an image query using the edge server proceeds without failure.""" -# iq = gl.submit_image_query(detector=detector.id, image=image_bytes, wait=10.0) -# assert iq is not None, "ImageQuery should not be None." - - -# @pytest.mark.live -# def test_post_image_query_via_sdk_want_async(gl: Groundlight, detector: Detector, image_bytes: bytes): -# """Test that submitting an image query with want_async=True forwards directly to the cloud.""" -# iq = gl.ask_async(detector=detector.id, image=image_bytes) -# assert iq is not None, "ImageQuery should not be None." -# assert iq.id.startswith("iq_"), "ImageQuery id should start with 'iq_' because it was created on the cloud." -# assert iq.result is None, "Result should be None because the query is still being processed." - - -# @pytest.mark.live -# def test_post_image_query_via_sdk_with_metadata_throws_400(gl: Groundlight, detector: Detector, image_bytes: bytes): -# """Test that submitting an image query with metadata raises a 400 error.""" -# with pytest.raises(ApiException) as exc_info: -# gl.submit_image_query(detector=detector.id, image=image_bytes, wait=10.0, metadata={"foo": "bar"}) -# assert exc_info.value.status == status.HTTP_400_BAD_REQUEST +@pytest.mark.live +def test_post_image_query_via_sdk_with_metadata_throws_400(gl: Groundlight, detector: Detector): + """Test that submitting an image query with metadata raises a 400 error.""" + image_bytes = pil_image_to_bytes(img=Image.open("test/assets/dog.jpeg")) + with pytest.raises(ApiException) as exc_info: + gl.submit_image_query(detector=detector.id, image=image_bytes, wait=10.0, metadata={"foo": "bar"}) + assert exc_info.value.status == status.HTTP_400_BAD_REQUEST diff --git a/test/api/test_image_queries_live_2.py b/test/api/test_image_queries_live_2.py new file mode 100644 index 000000000..5fb8b8493 --- /dev/null +++ b/test/api/test_image_queries_live_2.py @@ -0,0 +1,316 @@ +import os +import time + +import pytest +import requests +from fastapi import status +from groundlight import ApiException, Detector, Groundlight, ImageQuery +from PIL import Image + +from app.core.utils import pil_image_to_bytes + +# Tests in this file require a live edge-endpoint server and GL Api token in order to run. +# Not ideal for unit-testing. +TEST_ENDPOINT = os.getenv("LIVE_TEST_ENDPOINT", "http://localhost:30101") +MAX_WAIT_TIME_S = 60 + +# Detectors for live testing. On the prod-biggies account. +# - name="live_edge_testing_1", +# - query="Is there a dog in the image?", +# - confidence_threshold=0.9 +DETECTOR_ID_1 = "det_2raefZ74V0ojgbmM2UJzQCpFKyF" +# - name="live_edge_testing_2", +# - query="Is there a dog in the image?", +# - confidence_threshold=0.9 +DETECTOR_ID_2 = "det_2rdUY6SJOBJtuW5oqD3ExL1DjFn" +# - name="live_edge_testing_3", +# - query="Is there a dog in the image?", +# - confidence_threshold=0.9 +DETECTOR_ID_3 = "det_2rdUb0jljHCosfKGuTugVoo4eiY" +# - name="live_edge_testing_4", +# - query="Is there a dog in the image?", +# - confidence_threshold=0.9 +DETECTOR_ID_4 = "det_2rdVBErF53NWjVjhVdIrb6QJbRT" + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Fixtures +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +@pytest.fixture(scope="module", autouse=True) +def ensure_edge_endpoint_is_live_and_ready(): + """Ensure that the edge-endpoint server is live and ready before running tests.""" + start_time = time.time() + final_exception = None + while time.time() - start_time < MAX_WAIT_TIME_S: + try: + live_response = requests.get(TEST_ENDPOINT + "/health/live") + live_response.raise_for_status() + ready_response = requests.get(TEST_ENDPOINT + "/health/ready") + ready_response.raise_for_status() + if live_response.json().get("status") == "alive" and ready_response.json().get("status") == "ready": + return + except requests.RequestException as e: + final_exception = e + time.sleep(1) # wait for 1 second before retrying + pytest.fail(f"Edge endpoint is not live and ready after polling for {MAX_WAIT_TIME_S} seconds. {final_exception=}") + + +@pytest.fixture(name="gl") +def fixture_gl() -> Groundlight: + """Create a Groundlight client object.""" + return Groundlight(endpoint=TEST_ENDPOINT) + + +@pytest.fixture +def detector_default(gl: Groundlight) -> Detector: + """Retrieve the default detector using the Groundlight client.""" + return gl.get_detector(id=DETECTOR_ID_1) + + +@pytest.fixture +def detector_edge_answers(gl: Groundlight) -> Detector: + """Retrieve the edge answers detector using the Groundlight client.""" + return gl.get_detector(id=DETECTOR_ID_2) + + +@pytest.fixture +def detector_no_cloud(gl: Groundlight) -> Detector: + """Retrieve the no cloud detector using the Groundlight client.""" + return gl.get_detector(id=DETECTOR_ID_3) + + +@pytest.fixture +def detector_disabled(gl: Groundlight) -> Detector: + """Retrieve the disabled detector using the Groundlight client.""" + return gl.get_detector(id=DETECTOR_ID_4) + + +@pytest.fixture +def image_bytes() -> bytes: + """Return the test image as bytes.""" + return pil_image_to_bytes(img=Image.open("test/assets/dog.jpeg")) + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Helpers +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +def answer_is_from_cloud(iq: ImageQuery) -> bool: + """Return True if the answer is from the cloud, False otherwise.""" + return not iq.metadata or not iq.metadata.get("is_from_edge", False) + + +def answer_is_from_edge(iq: ImageQuery) -> bool: + """Return True if the answer is from the edge, False otherwise.""" + return iq.metadata and iq.metadata.get("is_from_edge", False) + + +def was_escalated(gl: Groundlight, iq: ImageQuery, max_retries: int = 3, retry_delay: float = 1.0) -> bool: + """Return True if the answer was escalated to the cloud, False otherwise. + Retries up to max_retries times, waiting retry_delay seconds between retries, to account for the time it takes for + the cloud to process the image query. + + Args: + gl: Groundlight client + iq: ImageQuery to check + max_retries: Maximum number of retry attempts + retry_delay: Delay in seconds between retries + """ + for attempt in range(max_retries): + try: + gl.get_image_query(id=iq.id) + return True + except ApiException as e: + if e.status == status.HTTP_404_NOT_FOUND and attempt < max_retries - 1: + time.sleep(retry_delay) + continue + if e.status == status.HTTP_404_NOT_FOUND: + return False + raise + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Tests +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +@pytest.mark.live +class TestSubmittingToLocalInferenceConfigs: + """Tests for submitting image queries with different detector configurations.""" + + class TestDefaultConfig: + """Tests for default detector configuration behavior.""" + + def test_high_threshold_goes_to_cloud(self, gl: Groundlight, detector_default: Detector, image_bytes: bytes): + iq = gl.submit_image_query( + detector=detector_default.id, image=image_bytes, confidence_threshold=1, wait=0 + ) # TODO is this dependent on getting a fast cloud response? + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_cloud(iq), "Answer should be from the cloud." + + def test_low_threshold_comes_from_edge(self, gl: Groundlight, detector_default: Detector, image_bytes: bytes): + iq = gl.submit_image_query( + detector=detector_default.id, image=image_bytes, confidence_threshold=0.5, wait=0 + ) + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_edge(iq), "Answer should be from the edge." + + class TestEdgeAnswersConfig: + """Tests for edge_answers_with_escalation detector configuration.""" + + def test_high_threshold_comes_from_edge_and_escalated( + self, gl: Groundlight, detector_edge_answers: Detector, image_bytes: bytes + ): + iq = gl.submit_image_query( + detector=detector_edge_answers.id, image=image_bytes, confidence_threshold=1, wait=0 + ) + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_edge(iq), "Answer should be from the edge." + assert was_escalated(gl, iq), "Answer should be escalated." + + def test_low_threshold_comes_from_edge( + self, gl: Groundlight, detector_edge_answers: Detector, image_bytes: bytes + ): + iq = gl.submit_image_query( + detector=detector_edge_answers.id, image=image_bytes, confidence_threshold=0.5, wait=0 + ) + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_edge(iq), "Answer should be from the edge." + assert not was_escalated(gl, iq), "Answer should not be escalated." + + class TestNoCloudConfig: + """Tests for no_cloud detector configuration.""" + + def test_high_threshold_comes_from_edge_not_escalated(self, gl, detector_no_cloud, image_bytes): + iq = gl.submit_image_query(detector=detector_no_cloud.id, image=image_bytes, confidence_threshold=1, wait=0) + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_edge(iq), "Answer should be from the edge." + assert not was_escalated(gl, iq), "Answer should not be escalated." + + def test_low_threshold_comes_from_edge(self, gl, detector_no_cloud, image_bytes): + iq = gl.submit_image_query( + detector=detector_no_cloud.id, image=image_bytes, confidence_threshold=0.5, wait=0 + ) + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_edge(iq), "Answer should be from the edge." + assert not was_escalated(gl, iq), "Answer should not be escalated." + + class TestDisabledConfig: + """Tests for disabled detector configuration.""" + + def test_low_threshold_goes_to_cloud(self, gl: Groundlight, detector_disabled: Detector, image_bytes: bytes): + iq = gl.submit_image_query( + detector=detector_disabled.id, image=image_bytes, confidence_threshold=0.5, wait=0 + ) + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_cloud(iq), "Answer should be from the cloud." + + +@pytest.mark.live +class TestEdgeQueryParams: + """Testing behavior of submit_image_query parameters on edge.""" + + @pytest.mark.parametrize("detector_fixture", ["detector_edge_answers", "detector_no_cloud"]) + def test_human_review_not_allowed( + self, gl: Groundlight, request: pytest.FixtureRequest, detector_fixture: str, image_bytes: bytes + ): + """Test that human_review cannot be specified when edge answers are required.""" + detector = request.getfixturevalue(detector_fixture) + with pytest.raises(ApiException) as exc_info: + gl.submit_image_query(detector=detector.id, image=image_bytes, human_review="ALWAYS") + assert exc_info.value.status == status.HTTP_400_BAD_REQUEST + + @pytest.mark.parametrize("detector_fixture", ["detector_edge_answers", "detector_no_cloud"]) + def test_want_async_not_allowed( + self, gl: Groundlight, request: pytest.FixtureRequest, detector_fixture: str, image_bytes: bytes + ): + """Test that want_async cannot be specified when edge answers are required.""" + detector = request.getfixturevalue(detector_fixture) + with pytest.raises(ApiException) as exc_info: + gl.submit_image_query(detector=detector.id, image=image_bytes, want_async=True, wait=0) + assert exc_info.value.status == status.HTTP_400_BAD_REQUEST + + def test_always_human_review_goes_to_cloud(self, gl: Groundlight, detector_default: Detector, image_bytes: bytes): + """Test that human_review=ALWAYS goes to the cloud even if the edge answer is sufficiently confident.""" + iq = gl.submit_image_query( + detector=detector_default.id, image=image_bytes, human_review="ALWAYS", confidence_threshold=0.5, wait=0 + ) + assert iq is not None + assert answer_is_from_cloud(iq), "Answer should be from the cloud." + + def test_want_async_goes_to_cloud(self, gl: Groundlight, detector_default: Detector, image_bytes: bytes): + """Test that want_async=True goes to the cloud even if the edge answer is sufficiently confident.""" + iq = gl.submit_image_query( + detector=detector_default.id, image=image_bytes, want_async=True, confidence_threshold=0.5, wait=0 + ) + assert iq is not None + assert answer_is_from_cloud(iq), "Answer should be from the cloud." + + def test_supported_params_dont_error(self, gl: Groundlight, detector_default: Detector, image_bytes: bytes): + """Test that supported parameters work without errors.""" + iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, wait=1.0) + assert iq is not None + + iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, patience_time=1.0) + assert iq is not None + + iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, confidence_threshold=0.8) + assert iq is not None + + iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, human_review="NEVER") + assert iq is not None + + iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, want_async=False) + assert iq is not None + + @pytest.mark.parametrize( + "unsupported_param", + [ + {"inspection_id": "insp_123"}, + {"metadata": {"test": "value"}}, + {"image_query_id": "iq_123"}, + ], + ) + def test_unsupported_params_raise_error( + self, + gl: Groundlight, + detector_default: Detector, + image_bytes: bytes, + unsupported_param: dict, + ): + """Test that unsupported parameters raise a 400 error.""" + with pytest.raises(ApiException) as exc_info: + gl.submit_image_query(detector=detector_default.id, image=image_bytes, **unsupported_param) + assert exc_info.value.status == status.HTTP_400_BAD_REQUEST + + +# @pytest.mark.live +# def test_post_image_query_via_sdk(gl: Groundlight, detector: Detector, image_bytes: bytes): +# """Test that submitting an image query using the edge server proceeds without failure.""" +# iq = gl.submit_image_query(detector=detector.id, image=image_bytes, wait=10.0) +# assert iq is not None, "ImageQuery should not be None." + + +# @pytest.mark.live +# def test_post_image_query_via_sdk_want_async(gl: Groundlight, detector: Detector, image_bytes: bytes): +# """Test that submitting an image query with want_async=True forwards directly to the cloud.""" +# iq = gl.ask_async(detector=detector.id, image=image_bytes) +# assert iq is not None, "ImageQuery should not be None." +# assert iq.id.startswith("iq_"), "ImageQuery id should start with 'iq_' because it was created on the cloud." +# assert iq.result is None, "Result should be None because the query is still being processed." + + +# @pytest.mark.live +# def test_post_image_query_via_sdk_with_metadata_throws_400(gl: Groundlight, detector: Detector, image_bytes: bytes): +# """Test that submitting an image query with metadata raises a 400 error.""" +# with pytest.raises(ApiException) as exc_info: +# gl.submit_image_query(detector=detector.id, image=image_bytes, wait=10.0, metadata={"foo": "bar"}) +# assert exc_info.value.status == status.HTTP_400_BAD_REQUEST diff --git a/test/setup_k3s_test_environment.sh b/test/setup_k3s_test_environment.sh index 67ed58248..10e411fd9 100755 --- a/test/setup_k3s_test_environment.sh +++ b/test/setup_k3s_test_environment.sh @@ -41,43 +41,6 @@ export IMAGE_TAG=$(./deploy/bin/git-tag-name.sh) export INFERENCE_FLAVOR="CPU" -EDGE_CONFIG=$(cat <<- EOM -global_config: - refresh_rate: 60 - -edge_inference_configs: - default: - enabled: true - always_return_edge_prediction: false - disable_cloud_escalation: false - - edge_answers_with_escalation: - enabled: true - always_return_edge_prediction: true - disable_cloud_escalation: false - min_time_between_escalations: 2.0 - - no_cloud: - enabled: true - always_return_edge_prediction: true - disable_cloud_escalation: true - - disabled: - enabled: false - -detectors: - - detector_id: "det_2raefZ74V0ojgbmM2UJzQCpFKyF" - edge_inference_config: "default" - - detector_id: "det_2rdUY6SJOBJtuW5oqD3ExL1DjFn" - edge_inference_config: "edge_answers_with_escalation" - - detector_id: "det_2rdUb0jljHCosfKGuTugVoo4eiY" - edge_inference_config: "no_cloud" - - detector_id: "det_2rdVBErF53NWjVjhVdIrb6QJbRT" - edge_inference_config: "disabled" -EOM -) -export EDGE_CONFIG - ./deploy/bin/setup-ee.sh diff --git a/test/setup_k3s_test_environment_2.sh b/test/setup_k3s_test_environment_2.sh new file mode 100644 index 000000000..67ed58248 --- /dev/null +++ b/test/setup_k3s_test_environment_2.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +# This script will setup the k3s testing environment. Once you've run them you can run the +# live tests, which will hit the API service that got setup +# Altogether, you can run everything with: +# > make test-with-k3s + +if [ -z "$GROUNDLIGHT_API_TOKEN" ]; then + echo "Error: GROUNDLIGHT_API_TOKEN environment variable is not set." + exit 1 +fi + +if ! command -v k3s &> /dev/null +then + echo "Error: you must have k3s setup" + exit 1 +fi +export PERSISTENT_VOLUME_NAME="test-with-k3s-pv" +export EDGE_ENDPOINT_PORT="30107" + +# # now we should delete the persistent volume before, in case it's in a bad state +# Check if the persistent volume exists +if kubectl get pv "$PERSISTENT_VOLUME_NAME" &> /dev/null; then + echo "Persistent volume $PERSISTENT_VOLUME_NAME exists. Deleting it..." + kubectl delete pv "$PERSISTENT_VOLUME_NAME" & + echo "Persistent volume $PERSISTENT_VOLUME_NAME deleted." +else + echo "Persistent volume $PERSISTENT_VOLUME_NAME does not exist. No action needed." +fi + + +export DEPLOYMENT_NAMESPACE="test-with-k3s" +if ! kubectl get namespace $DEPLOYMENT_NAMESPACE &> /dev/null; then + kubectl create namespace $DEPLOYMENT_NAMESPACE +fi + +# Build the Docker image and import it into k3s +echo "Building the Docker image..." +./deploy/bin/build-push-edge-endpoint-image.sh dev +export IMAGE_TAG=$(./deploy/bin/git-tag-name.sh) + +export INFERENCE_FLAVOR="CPU" + +EDGE_CONFIG=$(cat <<- EOM +global_config: + refresh_rate: 60 + +edge_inference_configs: + default: + enabled: true + always_return_edge_prediction: false + disable_cloud_escalation: false + + edge_answers_with_escalation: + enabled: true + always_return_edge_prediction: true + disable_cloud_escalation: false + min_time_between_escalations: 2.0 + + no_cloud: + enabled: true + always_return_edge_prediction: true + disable_cloud_escalation: true + + disabled: + enabled: false + +detectors: + - detector_id: "det_2raefZ74V0ojgbmM2UJzQCpFKyF" + edge_inference_config: "default" + - detector_id: "det_2rdUY6SJOBJtuW5oqD3ExL1DjFn" + edge_inference_config: "edge_answers_with_escalation" + - detector_id: "det_2rdUb0jljHCosfKGuTugVoo4eiY" + edge_inference_config: "no_cloud" + - detector_id: "det_2rdVBErF53NWjVjhVdIrb6QJbRT" + edge_inference_config: "disabled" +EOM +) +export EDGE_CONFIG + +./deploy/bin/setup-ee.sh + + +export LIVE_TEST_ENDPOINT="http://localhost:$EDGE_ENDPOINT_PORT" +echo "Waiting for edge-endpoint pods to rollout..." + + +if ! kubectl rollout status deployment/edge-endpoint -n $DEPLOYMENT_NAMESPACE --timeout=5m; then + echo "Error: edge-endpoint pods failed to rollout within the timeout period." + exit 1 +fi + +echo "Edge-endpoint pods have successfully rolled out." From 7e8fbde98ef9565396afa0050a132f502e57e4b4 Mon Sep 17 00:00:00 2001 From: CoreyEWood Date: Tue, 8 Apr 2025 22:52:10 +0000 Subject: [PATCH 06/10] undo other change --- test/api/test_image_queries_live.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/api/test_image_queries_live.py b/test/api/test_image_queries_live.py index b43d1b0af..d04bc9b52 100644 --- a/test/api/test_image_queries_live.py +++ b/test/api/test_image_queries_live.py @@ -9,7 +9,9 @@ from PIL import Image from app.core.utils import pil_image_to_bytes - @@ -15,16 +14,31 @@ + +# Tests in this file require a live edge-endpoint server and GL Api token in order to run. +# Not ideal for unit-testing. TEST_ENDPOINT = os.getenv("LIVE_TEST_ENDPOINT", "http://localhost:30101") MAX_WAIT_TIME_S = 60 @@ -40,6 +42,8 @@ def ensure_edge_endpoint_is_live_and_ready(): final_exception = e time.sleep(1) # wait for 1 second before retrying pytest.fail(f"Edge endpoint is not live and ready after polling for {MAX_WAIT_TIME_S} seconds. {final_exception=}") + + @pytest.fixture(name="gl") def fixture_gl() -> Groundlight: """Create a Groundlight client object.""" From e4fd5e82d63c6432c5f7e1de9af79c69d9b22cea Mon Sep 17 00:00:00 2001 From: Auto-format Bot Date: Tue, 8 Apr 2025 22:52:39 +0000 Subject: [PATCH 07/10] Automatically reformatting code with black and isort --- test/api/test_image_queries_live.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/api/test_image_queries_live.py b/test/api/test_image_queries_live.py index d04bc9b52..6afcee8db 100644 --- a/test/api/test_image_queries_live.py +++ b/test/api/test_image_queries_live.py @@ -10,7 +10,7 @@ from app.core.utils import pil_image_to_bytes -# Tests in this file require a live edge-endpoint server and GL Api token in order to run. +# Tests in this file require a live edge-endpoint server and GL Api token in order to run. # Not ideal for unit-testing. TEST_ENDPOINT = os.getenv("LIVE_TEST_ENDPOINT", "http://localhost:30101") MAX_WAIT_TIME_S = 60 From 0b816a7007865de25b32a37982ffb7e1264aa448 Mon Sep 17 00:00:00 2001 From: CoreyEWood Date: Tue, 8 Apr 2025 23:58:35 +0000 Subject: [PATCH 08/10] oops there's a third one (uncommitted progress from instance, should be pure progress as compared to 2) --- test/api/test_image_queries_live_3.py | 375 ++++++++++++++++++++++++++ 1 file changed, 375 insertions(+) create mode 100644 test/api/test_image_queries_live_3.py diff --git a/test/api/test_image_queries_live_3.py b/test/api/test_image_queries_live_3.py new file mode 100644 index 000000000..9c8f8f36d --- /dev/null +++ b/test/api/test_image_queries_live_3.py @@ -0,0 +1,375 @@ +import os +import time + +import pytest +import requests +from fastapi import status +from groundlight import ApiException, Detector, Groundlight, ImageQuery +from PIL import Image + +from app.core.utils import pil_image_to_bytes + +# Tests in this file require a live edge-endpoint server and GL Api token in order to run. +# Not ideal for unit-testing. +TEST_ENDPOINT = os.getenv("LIVE_TEST_ENDPOINT", "http://localhost:30101") +MAX_WAIT_TIME_S = 60 + +# Detectors for live testing. On the prod-biggies account. +# - name="live_edge_testing_1", +# - query="Is there a dog in the image?", +# - confidence_threshold=0.9 +DETECTOR_ID_1 = "det_2raefZ74V0ojgbmM2UJzQCpFKyF" +# - name="live_edge_testing_2", +# - query="Is there a dog in the image?", +# - confidence_threshold=0.9 +DETECTOR_ID_2 = "det_2rdUY6SJOBJtuW5oqD3ExL1DjFn" +# - name="live_edge_testing_3", +# - query="Is there a dog in the image?", +# - confidence_threshold=0.9 +DETECTOR_ID_3 = "det_2rdUb0jljHCosfKGuTugVoo4eiY" +# - name="live_edge_testing_4", +# - query="Is there a dog in the image?", +# - confidence_threshold=0.9 +DETECTOR_ID_4 = "det_2rdVBErF53NWjVjhVdIrb6QJbRT" + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Fixtures +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +@pytest.fixture(scope="module", autouse=True) +def ensure_edge_endpoint_is_live_and_ready(): + """Ensure that the edge-endpoint server is live and ready before running tests.""" + start_time = time.time() + final_exception = None + while time.time() - start_time < MAX_WAIT_TIME_S: + try: + live_response = requests.get(TEST_ENDPOINT + "/health/live") + live_response.raise_for_status() + ready_response = requests.get(TEST_ENDPOINT + "/health/ready") + ready_response.raise_for_status() + if live_response.json().get("status") == "alive" and ready_response.json().get("status") == "ready": + return + except requests.RequestException as e: + final_exception = e + time.sleep(1) # wait for 1 second before retrying + pytest.fail(f"Edge endpoint is not live and ready after polling for {MAX_WAIT_TIME_S} seconds. {final_exception=}") + + +@pytest.fixture(name="gl") +def fixture_gl() -> Groundlight: + """Create a Groundlight client object.""" + return Groundlight(endpoint=TEST_ENDPOINT) + + +@pytest.fixture +def detector_default(gl: Groundlight) -> Detector: + """Retrieve the default detector using the Groundlight client.""" + return gl.get_detector(id=DETECTOR_ID_1) + + +@pytest.fixture +def detector_edge_answers(gl: Groundlight) -> Detector: + """Retrieve the edge answers detector using the Groundlight client.""" + return gl.get_detector(id=DETECTOR_ID_2) + + +@pytest.fixture +def detector_no_cloud(gl: Groundlight) -> Detector: + """Retrieve the no cloud detector using the Groundlight client.""" + return gl.get_detector(id=DETECTOR_ID_3) + + +@pytest.fixture +def detector_disabled(gl: Groundlight) -> Detector: + """Retrieve the disabled detector using the Groundlight client.""" + return gl.get_detector(id=DETECTOR_ID_4) + + +@pytest.fixture +def image_bytes() -> bytes: + """Return the test image as bytes.""" + return pil_image_to_bytes(img=Image.open("test/assets/dog.jpeg")) + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Helpers +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +def answer_is_from_cloud(iq: ImageQuery) -> bool: + """Return True if the answer is from the cloud, False otherwise.""" + return not iq.metadata or not iq.metadata.get("is_from_edge", False) + + +def answer_is_from_edge(iq: ImageQuery) -> bool: + """Return True if the answer is from the edge, False otherwise.""" + return iq.metadata and iq.metadata.get("is_from_edge", False) + + +def was_escalated(gl: Groundlight, iq: ImageQuery, max_retries: int = 3, retry_delay: float = 1.0) -> bool: + """Return True if the answer was escalated to the cloud, False otherwise. + Retries up to max_retries times, waiting retry_delay seconds between retries, to account for the time it takes for + the cloud to process the image query. + + Args: + gl: Groundlight client + iq: ImageQuery to check + max_retries: Maximum number of retry attempts + retry_delay: Delay in seconds between retries + """ + for attempt in range(max_retries): + try: + gl.get_image_query(id=iq.id) + return True + except ApiException as e: + if e.status == status.HTTP_404_NOT_FOUND and attempt < max_retries - 1: + time.sleep(retry_delay) + continue + if e.status == status.HTTP_404_NOT_FOUND: + return False + raise + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Tests +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +@pytest.mark.live +class TestSubmittingToLocalInferenceConfigs: + """Tests for submitting image queries with different detector configurations.""" + + class TestDefaultConfig: + """Tests for default detector configuration behavior.""" + + def test_high_threshold_goes_to_cloud(self, gl: Groundlight, detector_default: Detector, image_bytes: bytes): + iq = gl.submit_image_query( + detector=detector_default.id, image=image_bytes, confidence_threshold=1, wait=0 + ) # TODO is this dependent on getting a fast cloud response? + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_cloud(iq), "Answer should be from the cloud." + + def test_low_threshold_comes_from_edge(self, gl: Groundlight, detector_default: Detector, image_bytes: bytes): + iq = gl.submit_image_query( + detector=detector_default.id, image=image_bytes, confidence_threshold=0.5, wait=0 + ) + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_edge(iq), "Answer should be from the edge." + + class TestEdgeAnswersConfig: + """Tests for edge_answers_with_escalation detector configuration.""" + + def test_high_threshold_comes_from_edge_and_escalated( + self, gl: Groundlight, detector_edge_answers: Detector, image_bytes: bytes + ): + iq = gl.submit_image_query( + detector=detector_edge_answers.id, image=image_bytes, confidence_threshold=1, wait=0 + ) + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_edge(iq), "Answer should be from the edge." + assert was_escalated(gl, iq), "Answer should be escalated." + + def test_low_threshold_comes_from_edge( + self, gl: Groundlight, detector_edge_answers: Detector, image_bytes: bytes + ): + iq = gl.submit_image_query( + detector=detector_edge_answers.id, image=image_bytes, confidence_threshold=0.5, wait=0 + ) + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_edge(iq), "Answer should be from the edge." + assert not was_escalated(gl, iq), "Answer should not be escalated." + + class TestNoCloudConfig: + """Tests for no_cloud detector configuration.""" + + def test_high_threshold_comes_from_edge_not_escalated(self, gl, detector_no_cloud, image_bytes): + iq = gl.submit_image_query(detector=detector_no_cloud.id, image=image_bytes, confidence_threshold=1, wait=0) + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_edge(iq), "Answer should be from the edge." + assert not was_escalated(gl, iq), "Answer should not be escalated." + + def test_low_threshold_comes_from_edge(self, gl, detector_no_cloud, image_bytes): + iq = gl.submit_image_query( + detector=detector_no_cloud.id, image=image_bytes, confidence_threshold=0.5, wait=0 + ) + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_edge(iq), "Answer should be from the edge." + assert not was_escalated(gl, iq), "Answer should not be escalated." + + class TestDisabledConfig: + """Tests for disabled detector configuration.""" + + def test_low_threshold_goes_to_cloud(self, gl: Groundlight, detector_disabled: Detector, image_bytes: bytes): + iq = gl.submit_image_query( + detector=detector_disabled.id, image=image_bytes, confidence_threshold=0.5, wait=0 + ) + assert iq is not None, "ImageQuery should not be None." + assert answer_is_from_cloud(iq), "Answer should be from the cloud." + + +@pytest.mark.live +class TestEdgeQueryParams: + """Testing behavior of submit_image_query parameters on edge.""" + + class TestWantAsync: + """Tests for want_async parameter behavior.""" + + @pytest.mark.parametrize("detector_fixture", ["detector_edge_answers", "detector_no_cloud"]) + def test_want_async_not_allowed_with_edge_answers( + self, gl: Groundlight, request: pytest.FixtureRequest, detector_fixture: str, image_bytes: bytes + ): + """Test that want_async cannot be specified when edge answers are required.""" + detector = request.getfixturevalue(detector_fixture) + with pytest.raises(ApiException) as exc_info: + gl.submit_image_query(detector=detector.id, image=image_bytes, want_async=True, wait=0) + assert exc_info.value.status == status.HTTP_400_BAD_REQUEST + + def test_want_async_goes_to_cloud(self, gl: Groundlight, detector_default: Detector, image_bytes: bytes): + """Test that want_async=True always goes to the cloud.""" + iq = gl.submit_image_query( + detector=detector_default.id, image=image_bytes, want_async=True, confidence_threshold=0.5, wait=0 + ) + assert iq is not None + assert answer_is_from_cloud(iq), "Answer should be from the cloud." + + def test_want_async_can_be_submitted_without_error( + self, gl: Groundlight, detector_default: Detector, image_bytes: bytes + ): + """Test that want_async can be submitted without error.""" + iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, want_async=False, wait=0) + assert iq is not None + + class TestHumanReview: + """Tests for human_review parameter behavior.""" + + @pytest.mark.parametrize("detector_fixture", ["detector_edge_answers", "detector_no_cloud"]) + def test_human_review_not_allowed_with_edge_answers( + self, gl: Groundlight, request: pytest.FixtureRequest, detector_fixture: str, image_bytes: bytes + ): + """Test that human_review cannot be specified when edge answers are required.""" + detector = request.getfixturevalue(detector_fixture) + with pytest.raises(ApiException) as exc_info: + gl.submit_image_query(detector=detector.id, image=image_bytes, human_review="ALWAYS") + assert exc_info.value.status == status.HTTP_400_BAD_REQUEST + + def test_always_human_review_goes_to_cloud( + self, gl: Groundlight, detector_default: Detector, image_bytes: bytes + ): + """Test that human_review=ALWAYS always goes to the cloud.""" + iq = gl.submit_image_query( + detector=detector_default.id, image=image_bytes, human_review="ALWAYS", confidence_threshold=0.5, wait=0 + ) + assert iq is not None + assert answer_is_from_cloud(iq), "Answer should be from the cloud." + + def test_human_review_can_be_submitted_without_error( + self, gl: Groundlight, detector_default: Detector, image_bytes: bytes + ): + """Test that human_review=(NEVER/DEFAULT) can be submitted without error.""" + iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, human_review="NEVER", wait=0) + assert iq is not None + + iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, human_review="DEFAULT", wait=0) + assert iq is not None + + class TestWait: + """Tests for wait parameter behavior.""" + + # TODO figure out what the functionality should be and test it + + def test_wait_can_be_submitted_without_error( + self, gl: Groundlight, detector_no_cloud: Detector, image_bytes: bytes + ): + """Test that zero and non-zero wait times can be submitted without error.""" + # Wait of 0 + wait_time = 0 + iq = gl.submit_image_query( + detector=detector_no_cloud.id, image=image_bytes, wait=wait_time, confidence_threshold=1 + ) + assert iq is not None + + # Non-zero wait + wait_time = 1.5 + start_time = time.time() + iq = gl.submit_image_query( + detector=detector_no_cloud.id, image=image_bytes, wait=wait_time, confidence_threshold=1 + ) + elapsed = time.time() - start_time + assert elapsed >= wait_time, f"Query took {elapsed:.2f}s but should have taken at least {wait_time}s" + assert iq is not None + + class TestPatienceTime: + """Tests for patience_time parameter behavior.""" + + # TODO figure out what the functionality should be and test it + + def test_patience_time_can_be_submitted_without_error( + self, gl: Groundlight, detector_no_cloud: Detector, image_bytes: bytes + ): + """Test that patience_time can be submitted without error.""" + iq = gl.submit_image_query(detector=detector_no_cloud.id, image=image_bytes, patience_time=1.0) + assert iq is not None + + class TestConfidenceThreshold: + """ + Tests for confidence_threshold parameter behavior. This is implicitly tested in other tests, so we just do a + simple check here. + """ + + def test_confidence_threshold_can_be_submitted_without_error( + self, gl: Groundlight, detector_no_cloud: Detector, image_bytes: bytes + ): + """Test that confidence_threshold can be submitted without error.""" + iq = gl.submit_image_query(detector=detector_no_cloud.id, image=image_bytes, confidence_threshold=0.8) + assert iq is not None + + @pytest.mark.parametrize( + "unsupported_param", + [ + {"inspection_id": "insp_123"}, + {"metadata": {"test": "value"}}, + {"image_query_id": "iq_123"}, + ], + ) + def test_unsupported_params_raise_error( + self, + gl: Groundlight, + detector_default: Detector, + image_bytes: bytes, + unsupported_param: dict, + ): + """Test that unsupported parameters raise a 400 error.""" + with pytest.raises(ApiException) as exc_info: + gl.submit_image_query(detector=detector_default.id, image=image_bytes, **unsupported_param) + assert exc_info.value.status == status.HTTP_400_BAD_REQUEST + + +# @pytest.mark.live +# def test_post_image_query_via_sdk(gl: Groundlight, detector: Detector, image_bytes: bytes): +# """Test that submitting an image query using the edge server proceeds without failure.""" +# iq = gl.submit_image_query(detector=detector.id, image=image_bytes, wait=10.0) +# assert iq is not None, "ImageQuery should not be None." + + +# @pytest.mark.live +# def test_post_image_query_via_sdk_want_async(gl: Groundlight, detector: Detector, image_bytes: bytes): +# """Test that submitting an image query with want_async=True forwards directly to the cloud.""" +# iq = gl.ask_async(detector=detector.id, image=image_bytes) +# assert iq is not None, "ImageQuery should not be None." +# assert iq.id.startswith("iq_"), "ImageQuery id should start with 'iq_' because it was created on the cloud." +# assert iq.result is None, "Result should be None because the query is still being processed." + + +# @pytest.mark.live +# def test_post_image_query_via_sdk_with_metadata_throws_400(gl: Groundlight, detector: Detector, image_bytes: bytes): +# """Test that submitting an image query with metadata raises a 400 error.""" +# with pytest.raises(ApiException) as exc_info: +# gl.submit_image_query(detector=detector.id, image=image_bytes, wait=10.0, metadata={"foo": "bar"}) +# assert exc_info.value.status == status.HTTP_400_BAD_REQUEST From 91faaab42cc77f6ad20d75371a30014d5cb841e0 Mon Sep 17 00:00:00 2001 From: CoreyEWood Date: Wed, 9 Apr 2025 00:24:36 +0000 Subject: [PATCH 09/10] patience_time triggers an error if it's less then 10 --- test/api/test_image_queries_live_2.py | 2 +- test/api/test_image_queries_live_3.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test/api/test_image_queries_live_2.py b/test/api/test_image_queries_live_2.py index 5fb8b8493..15adbe7fc 100644 --- a/test/api/test_image_queries_live_2.py +++ b/test/api/test_image_queries_live_2.py @@ -259,7 +259,7 @@ def test_supported_params_dont_error(self, gl: Groundlight, detector_default: De iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, wait=1.0) assert iq is not None - iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, patience_time=1.0) + iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, patience_time=10.0) assert iq is not None iq = gl.submit_image_query(detector=detector_default.id, image=image_bytes, confidence_threshold=0.8) diff --git a/test/api/test_image_queries_live_3.py b/test/api/test_image_queries_live_3.py index 9c8f8f36d..5408af489 100644 --- a/test/api/test_image_queries_live_3.py +++ b/test/api/test_image_queries_live_3.py @@ -189,13 +189,15 @@ def test_low_threshold_comes_from_edge( class TestNoCloudConfig: """Tests for no_cloud detector configuration.""" - def test_high_threshold_comes_from_edge_not_escalated(self, gl, detector_no_cloud, image_bytes): + def test_high_threshold_comes_from_edge_not_escalated( + self, gl: Groundlight, detector_no_cloud: Detector, image_bytes: bytes + ): iq = gl.submit_image_query(detector=detector_no_cloud.id, image=image_bytes, confidence_threshold=1, wait=0) assert iq is not None, "ImageQuery should not be None." assert answer_is_from_edge(iq), "Answer should be from the edge." assert not was_escalated(gl, iq), "Answer should not be escalated." - def test_low_threshold_comes_from_edge(self, gl, detector_no_cloud, image_bytes): + def test_low_threshold_comes_from_edge(self, gl: Groundlight, detector_no_cloud: Detector, image_bytes: bytes): iq = gl.submit_image_query( detector=detector_no_cloud.id, image=image_bytes, confidence_threshold=0.5, wait=0 ) @@ -314,7 +316,7 @@ def test_patience_time_can_be_submitted_without_error( self, gl: Groundlight, detector_no_cloud: Detector, image_bytes: bytes ): """Test that patience_time can be submitted without error.""" - iq = gl.submit_image_query(detector=detector_no_cloud.id, image=image_bytes, patience_time=1.0) + iq = gl.submit_image_query(detector=detector_no_cloud.id, image=image_bytes, patience_time=10.0) assert iq is not None class TestConfidenceThreshold: From d1322fc3d63db299ee33b40c8f88027a8089f84a Mon Sep 17 00:00:00 2001 From: CoreyEWood Date: Thu, 1 May 2025 20:52:56 +0000 Subject: [PATCH 10/10] detector names and new detectors --- test/api/test_image_queries_live_3.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/api/test_image_queries_live_3.py b/test/api/test_image_queries_live_3.py index 5408af489..cd9cbf467 100644 --- a/test/api/test_image_queries_live_3.py +++ b/test/api/test_image_queries_live_3.py @@ -14,23 +14,23 @@ TEST_ENDPOINT = os.getenv("LIVE_TEST_ENDPOINT", "http://localhost:30101") MAX_WAIT_TIME_S = 60 -# Detectors for live testing. On the prod-biggies account. +# Detectors for live testing. On the corey+edge-cicd account. # - name="live_edge_testing_1", # - query="Is there a dog in the image?", # - confidence_threshold=0.9 -DETECTOR_ID_1 = "det_2raefZ74V0ojgbmM2UJzQCpFKyF" +DEFAULT_DETECTOR_ID = "det_2vjZOWnQjmlewxTd7tc7YDWs3wo" # - name="live_edge_testing_2", # - query="Is there a dog in the image?", # - confidence_threshold=0.9 -DETECTOR_ID_2 = "det_2rdUY6SJOBJtuW5oqD3ExL1DjFn" +EDGE_ANSWERS_DETECTOR_ID = "det_2vjZQVkzWLimoAxqyjObJDX865p" # - name="live_edge_testing_3", # - query="Is there a dog in the image?", # - confidence_threshold=0.9 -DETECTOR_ID_3 = "det_2rdUb0jljHCosfKGuTugVoo4eiY" +NO_CLOUD_DETECTOR_ID = "det_2vjZSI4e6sKCyLTb51oJCvYHzRD" # - name="live_edge_testing_4", # - query="Is there a dog in the image?", # - confidence_threshold=0.9 -DETECTOR_ID_4 = "det_2rdVBErF53NWjVjhVdIrb6QJbRT" +DISABLED_DETECTOR_ID = "det_2vjZTtb9Mo5W6fkC6OfLIxYOdqv" # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -67,25 +67,25 @@ def fixture_gl() -> Groundlight: @pytest.fixture def detector_default(gl: Groundlight) -> Detector: """Retrieve the default detector using the Groundlight client.""" - return gl.get_detector(id=DETECTOR_ID_1) + return gl.get_detector(id=DEFAULT_DETECTOR_ID) @pytest.fixture def detector_edge_answers(gl: Groundlight) -> Detector: """Retrieve the edge answers detector using the Groundlight client.""" - return gl.get_detector(id=DETECTOR_ID_2) + return gl.get_detector(id=EDGE_ANSWERS_DETECTOR_ID) @pytest.fixture def detector_no_cloud(gl: Groundlight) -> Detector: """Retrieve the no cloud detector using the Groundlight client.""" - return gl.get_detector(id=DETECTOR_ID_3) + return gl.get_detector(id=NO_CLOUD_DETECTOR_ID) @pytest.fixture def detector_disabled(gl: Groundlight) -> Detector: """Retrieve the disabled detector using the Groundlight client.""" - return gl.get_detector(id=DETECTOR_ID_4) + return gl.get_detector(id=DISABLED_DETECTOR_ID) @pytest.fixture