feat(api): support stored request details for analyses

bases/ecoindex/backend/routers/__init__.py

@@ -8,7 +8,7 @@
 
 router = APIRouter()
 
-router.include_router(router=router_bff)
+router.include_router(router=router_bff, include_in_schema=False)
 router.include_router(router=router_ecoindex)
 router.include_router(router=router_compute)
 router.include_router(router=router_host)

bases/ecoindex/backend/routers/bff.py

@@ -11,21 +11,30 @@
 from fastapi.responses import RedirectResponse
 from sqlmodel.ext.asyncio.session import AsyncSession
 
-router = router = APIRouter(prefix="/{version}/ecoindexes", tags=["BFF"])
+router = APIRouter(
+    prefix="/{version}/ecoindexes",
+    tags=["BFF"],
+    deprecated=True,
+    include_in_schema=False,
+)
 
 
 @router.get(
     name="Get latest results",
     path="/latest",
     response_model=EcoindexSearchResults,
     response_description="Get latest results for a given url",
+    deprecated=True,
+    include_in_schema=False,
 )
 async def get_latest_results(
     response: Response,
     parameters: BffDepParameters,
     session: AsyncSession = Depends(get_session),
 ) -> EcoindexSearchResults:
     """
+    **Deprecated.** Use the Ecoindex BFF service instead.
+
     This returns the latest results for a given url. This feature is used by the Ecoindex
     browser extension. By default, the results are cached for 7 days.
 
@@ -49,6 +58,8 @@ async def get_latest_results(
     path="/latest/badge",
     response_description="Badge of the given url from [CDN V1](https://www.jsdelivr.com/package/gh/cnumr/ecoindex_badge)",
     responses={status.HTTP_404_NOT_FOUND: example_file_not_found},
+    deprecated=True,
+    include_in_schema=False,
 )
 async def get_badge_enpoint(
     parameters: BffDepParameters,
@@ -58,6 +69,8 @@ async def get_badge_enpoint(
     session: AsyncSession = Depends(get_session),
 ) -> Response:
     """
+    **Deprecated.** Use the Ecoindex BFF service instead.
+
     This returns the SVG badge of the given url. This feature is used by the Ecoindex
     badge. By default, the results are cached for 7 days.
 
@@ -79,12 +92,16 @@ async def get_badge_enpoint(
     name="Get latest results redirect",
     path="/latest/redirect",
     response_description="Redirect to the latest results for a given url",
+    deprecated=True,
+    include_in_schema=False,
 )
 async def get_latest_result_redirect(
     parameters: BffDepParameters,
     session: AsyncSession = Depends(get_session),
 ) -> RedirectResponse:
     """
+    **Deprecated.** Use the Ecoindex BFF service instead.
+
     This redirects to the latest results on the frontend website for the given url.
     This feature is used by the Ecoindex browser extension and badge.
 

bases/ecoindex/backend/routers/ecoindex.py

@@ -18,9 +18,15 @@
     get_count_analysis_db,
     get_ecoindex_result_by_id_db,
     get_ecoindex_result_list_db,
+    get_requests_by_analysis_id_db,
 )
 from ecoindex.models import example_ecoindex_not_found, example_file_not_found
 from ecoindex.models.enums import Version
+from ecoindex.models.scraper import (
+    RequestDetail,
+    RequestsDetailResponse,
+    aggregate_request_details,
+)
 from ecoindex.screenshot_storage import (
     get_screenshot_local_path,
     is_s3_screenshot_storage,
@@ -123,6 +129,54 @@ async def get_ecoindex_analysis_by_id(
     return ecoindex
 
 
+@router.get(
+    name="Get ecoindex analysis requests by id",
+    path="/{id}/requests",
+    response_model=RequestsDetailResponse | None,
+    response_description="Request details of the ecoindex analysis",
+    responses={status.HTTP_404_NOT_FOUND: example_ecoindex_not_found},
+    description=(
+        "This returns the detailed list of requests made by the page, "
+        "aggregated by category and by domain. Returns `null` when the "
+        "analysis exists but request details were not collected."
+    ),
+)
+async def get_ecoindex_analysis_requests_by_id(
+    id: IdParameter,
+    version: VersionParameter = Version.v1,
+    session: AsyncSession = Depends(get_session),
+) -> RequestsDetailResponse | None:
+    ecoindex = await get_ecoindex_result_by_id_db(
+        session=session, id=id, version=version
+    )
+
+    if not ecoindex:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Analysis {id} not found for version {version.value}",
+        )
+
+    request_rows = await get_requests_by_analysis_id_db(
+        session=session, analysis_id=id
+    )
+    if not request_rows:
+        return None
+
+    return aggregate_request_details(
+        [
+            RequestDetail(
+                id=row.id,
+                category=row.category,
+                domain=row.domain,
+                status=row.status,
+                url=row.url,
+                size=row.size,
+            )
+            for row in request_rows
+        ]
+    )
+
+
 @router.get(
     name="Get screenshot",
     path="/{id}/screenshot",

bases/ecoindex/backend/routers/tasks.py

@@ -8,7 +8,7 @@
 from ecoindex.backend.utils import check_quota
 from ecoindex.config.settings import Settings
 from ecoindex.database.engine import get_session
-from ecoindex.database.models import ApiEcoindexes
+from ecoindex.database.models import ApiEcoindexBatchItems
 from ecoindex.models import WebPage
 from ecoindex.models.enums import TaskStatus
 from ecoindex.models.response_examples import (
@@ -89,6 +89,15 @@ async def add_ecoindex_analysis_task(
             example={"X-My-Custom-Header": "MyValue"},
         ),
     ] = {},
+    include_requests_detail: Annotated[
+        bool,
+        Body(
+            description=(
+                "If true, store the detailed list of requests made by the page"
+            ),
+            example=False,
+        ),
+    ] = False,
     session: AsyncSession = Depends(get_session),
 ) -> str:
     if Settings().DAILY_LIMIT_PER_HOST:
@@ -141,6 +150,7 @@ async def add_ecoindex_analysis_task(
         width=web_page.width,
         height=web_page.height,
         custom_headers=headers,
+        include_requests_detail=include_requests_detail,
         **_enqueue_settings(),
     )
 
@@ -227,7 +237,7 @@ async def delete_ecoindex_analysis_task_by_id(
 )
 async def add_ecoindex_analysis_task_batch(
     results: Annotated[
-        ApiEcoindexes,
+        ApiEcoindexBatchItems,
         Body(
             default=...,
             title="List of ecoindex analysis results to save",

bases/ecoindex/worker/tasks.py

@@ -7,7 +7,7 @@
 from ecoindex.config.settings import Settings
 from ecoindex.database.engine import get_session
 from ecoindex.database.exceptions.quota import QuotaExceededException
-from ecoindex.database.models import ApiEcoindex
+from ecoindex.database.models import ApiEcoindexBatchItem
 from ecoindex.database.repositories.worker import save_ecoindex_result_db
 from ecoindex.exceptions.scraper import EcoindexScraperStatusException
 from ecoindex.exceptions.worker import (
@@ -18,6 +18,7 @@
 )
 from ecoindex.models import ScreenShot, WindowSize
 from ecoindex.models.enums import TaskStatus, Version
+from ecoindex.models.scraper import RequestDetail
 from ecoindex.models.tasks import QueueTaskError, QueueTaskResult
 from ecoindex.monitoring import capture_task_failure, init_sentry
 from ecoindex.scraper.scrap import EcoindexScraper
@@ -39,7 +40,11 @@ def _get_task_id() -> UUID:
 
 
 def ecoindex_task(
-    url: str, width: int, height: int, custom_headers: dict[str, str]
+    url: str,
+    width: int,
+    height: int,
+    custom_headers: dict[str, str],
+    include_requests_detail: bool = False,
 ) -> str:
     queue_task_result = run(
         async_ecoindex_task(
@@ -48,6 +53,7 @@ def ecoindex_task(
             width=width,
             height=height,
             custom_headers=custom_headers,
+            include_requests_detail=include_requests_detail,
         )
     )
 
@@ -60,6 +66,7 @@ async def async_ecoindex_task(
     width: int,
     height: int,
     custom_headers: dict[str, str],
+    include_requests_detail: bool = False,
 ) -> QueueTaskResult:
     try:
         settings = Settings()
@@ -76,7 +83,7 @@ async def async_ecoindex_task(
 
         await check_quota(session=session, host=urlparse(url=url).netloc)
 
-        ecoindex = await EcoindexScraper(
+        scraper = EcoindexScraper(
             url=url,
             window_size=WindowSize(height=height, width=width),
             wait_after_scroll=settings.WAIT_AFTER_SCROLL,
@@ -85,7 +92,16 @@ async def async_ecoindex_task(
             screenshot_gid=settings.SCREENSHOTS_GID,
             screenshot_uid=settings.SCREENSHOTS_UID,
             custom_headers=custom_headers,
-        ).get_page_analysis()
+        )
+        ecoindex = await scraper.get_page_analysis()
+        request_details = (
+            [
+                RequestDetail.from_request_item(item)
+                for item in await scraper.get_all_requests()
+            ]
+            if include_requests_detail
+            else None
+        )
 
         if screenshot:
             persist_screenshot(screenshot=screenshot, version=Version.v1.value)
@@ -94,6 +110,7 @@ async def async_ecoindex_task(
             session=session,
             id=task_id,
             ecoindex_result=ecoindex,
+            requests=request_details,
         )
 
         return QueueTaskResult(status=TaskStatus.SUCCESS, detail=db_result)
@@ -187,7 +204,7 @@ async def async_ecoindex_task(
 def ecoindex_batch_import_task(results: list[dict], source: str) -> str:
     queue_task_result = run(
         async_ecoindex_batch_import_task(
-            results=[ApiEcoindex.model_validate(result) for result in results],
+            results=[ApiEcoindexBatchItem.model_validate(result) for result in results],
             source=source,
         )
     )
@@ -196,7 +213,7 @@ def ecoindex_batch_import_task(results: list[dict], source: str) -> str:
 
 
 async def async_ecoindex_batch_import_task(
-    results: list[ApiEcoindex], source: str
+    results: list[ApiEcoindexBatchItem], source: str
 ) -> QueueTaskResult:
     try:
         session_generator = get_session()
@@ -208,6 +225,7 @@ async def async_ecoindex_batch_import_task(
                 id=result.id,  # type: ignore
                 ecoindex_result=result,
                 source=source,
+                requests=result.request_details,
             )
 
         return QueueTaskResult(status=TaskStatus.SUCCESS)

components/ecoindex/database/engine.py

@@ -1,6 +1,7 @@
 from typing import AsyncGenerator
 
 from ecoindex.config import Settings
+from ecoindex.database.models import ApiEcoindex, ApiEcoindexRequest  # noqa: F401
 from ecoindex.models.api import *  # noqa: F401, F403
 from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
 from sqlalchemy.pool import NullPool

components/ecoindex/database/models/__init__.py

@@ -1,7 +1,9 @@
-from uuid import UUID
+from uuid import UUID, uuid4
 
 from ecoindex.models.compute import Result
+from ecoindex.models.scraper import RequestDetail
 from pydantic import BaseModel
+from sqlalchemy import Column, Text
 from sqlmodel import Field, SQLModel
 
 
@@ -48,7 +50,62 @@ class ApiEcoindex(SQLModel, Result, table=True):  # type: ignore
     )
 
 
+class ApiEcoindexRequest(SQLModel, table=True):
+    id: UUID = Field(
+        default_factory=uuid4,
+        primary_key=True,
+        description="Request detail ID of type `UUID`",
+    )
+    analysis_id: UUID = Field(
+        default=...,
+        foreign_key="apiecoindex.id",
+        index=True,
+        description="ID of the related ecoindex analysis",
+    )
+    category: str = Field(
+        default=...,
+        title="Request category",
+        description="Category of the resource (html, css, javascript, image, ...)",
+    )
+    domain: str = Field(
+        default=...,
+        title="Request domain",
+        description="Domain that served the resource",
+    )
+    status: int = Field(
+        default=...,
+        title="HTTP status",
+        description="HTTP status code of the resource response",
+    )
+    url: str = Field(
+        default=...,
+        sa_column=Column(Text(), nullable=False),
+        title="Request URL",
+        description="URL of the resource without query parameters",
+    )
+    size: float = Field(
+        default=...,
+        title="Request size",
+        description="Transfer size of the resource in bytes",
+    )
+
+
+class ApiEcoindexBatchItem(Result):
+    id: UUID | None = None
+    host: str
+    version: int = 1
+    initial_ranking: int | None = None
+    initial_total_results: int | None = None
+    source: str | None = None
+    request_details: list[RequestDetail] | None = Field(
+        default=None,
+        title="Request details",
+        description="Optional list of requests made by the page",
+    )
+
+
 ApiEcoindexes = list[ApiEcoindex]
+ApiEcoindexBatchItems = list[ApiEcoindexBatchItem]
 
 
 class PageApiEcoindexes(BaseModel):