diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41b7fb4..b61dda6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,23 +43,12 @@ jobs: - name: Validate operations scripts run: | bash -n deploy/observability/alloy/validate.sh - bash -n deploy/observability/preflight-host.sh - bash -n deploy/observability/tests/host-preflight-test.sh - bash -n deploy/observability/tests/update-api-test.sh - bash -n deploy/observability/update-api.sh + bash -n deploy/observability/tests/recommendation-assets-test.sh test -x deploy/observability/alloy/validate.sh - test -x deploy/observability/preflight-host.sh - test -x deploy/observability/tests/host-preflight-test.sh - test -x deploy/observability/tests/update-api-test.sh - test -x deploy/observability/update-api.sh - - - name: Test host deployment preflight - run: deploy/observability/tests/host-preflight-test.sh - - - name: Test immutable deployment and rollback - run: deploy/observability/tests/update-api-test.sh + test -x deploy/observability/tests/recommendation-assets-test.sh - name: Validate Grafana resource sources run: | jq empty deploy/observability/grafana/mapleland-production-overview.json jq empty deploy/observability/grafana/alert-rules.json + deploy/observability/tests/recommendation-assets-test.sh diff --git a/.github/workflows/deploy-jar.yml b/.github/workflows/deploy-jar.yml deleted file mode 100644 index ad0ab00..0000000 --- a/.github/workflows/deploy-jar.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Deploy Spring Boot App to EC2 - -on: - workflow_dispatch: - -permissions: - contents: read - packages: write - -jobs: - build-and-deploy: - runs-on: ubuntu-latest - env: - IMAGE_NAME: "ghcr.io/team-maple/mls-be/mapleland-api" - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Restore FCM key file - run: | - mkdir -p src/main/resources/firebase - echo '${{ secrets.FIREBASE_KEY }}' > src/main/resources/firebase/maple-9f1a7-firebase-adminsdk-fbsvc-7c3b6fc032.json - - - name: Set up JDK 21 - uses: actions/setup-java@v4 - with: - java-version: '21' - distribution: 'corretto' - - - name: Grant execute permission for gradlew - run: chmod +x ./gradlew - - - name: Log in to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build image with Paketo buildpacks - run: ./gradlew bootBuildImage --imageName "${{ env.IMAGE_NAME }}:latest" - - - name: Push image - run: docker push "${{ env.IMAGE_NAME }}:latest" - - - name: Deploy on EC2 instance - uses: appleboy/ssh-action@v1.0.3 - with: - host: ${{ secrets.HOST }} - username: ${{ secrets.USERNAME }} - key: ${{ secrets.KEY }} - port: ${{ secrets.PORT }} - script: | - set -e - IMAGE="${{ env.IMAGE_NAME }}:latest" - CONTAINER_NAME="mapleland-api" - - echo "${{ secrets.GHCR_TOKEN }}" | docker login ghcr.io -u "${{ secrets.GHCR_USERNAME }}" --password-stdin - - docker pull "$IMAGE" - - docker ps -f name=$CONTAINER_NAME -q | xargs --no-run-if-empty docker container stop - docker ps -a -f name=$CONTAINER_NAME -q | xargs --no-run-if-empty docker container rm - docker run -d --name "$CONTAINER_NAME" --restart always --env-file ~/.env -p 8080:8080 "$IMAGE" - - docker image prune -f - - docker logout ghcr.io diff --git a/.github/workflows/deploy-oci.yml b/.github/workflows/deploy-oci.yml index ac4621f..35caeaf 100644 --- a/.github/workflows/deploy-oci.yml +++ b/.github/workflows/deploy-oci.yml @@ -10,91 +10,22 @@ concurrency: permissions: contents: read packages: write - id-token: write jobs: - host-preflight: - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - id-token: write - contents: read - outputs: - compose_override_sha256: ${{ steps.contract.outputs.compose_override_sha256 }} - preflight_sha256: ${{ steps.contract.outputs.preflight_sha256 }} - update_sha256: ${{ steps.contract.outputs.update_sha256 }} - - steps: - - name: Checkout deployment contract - uses: actions/checkout@v4 - - - name: Resolve reviewed host contract checksums - id: contract - shell: bash - run: | - set -euo pipefail - compose_override_sha256="$(sha256sum deploy/observability/docker-compose.override.example.yml | awk '{print $1}')" - preflight_sha256="$(sha256sum deploy/observability/preflight-host.sh | awk '{print $1}')" - update_sha256="$(sha256sum deploy/observability/update-api.sh | awk '{print $1}')" - [[ $compose_override_sha256 =~ ^[0-9a-f]{64}$ ]] - [[ $preflight_sha256 =~ ^[0-9a-f]{64}$ ]] - [[ $update_sha256 =~ ^[0-9a-f]{64}$ ]] - { - printf 'compose_override_sha256=%s\n' "$compose_override_sha256" - printf 'preflight_sha256=%s\n' "$preflight_sha256" - printf 'update_sha256=%s\n' "$update_sha256" - } >> "$GITHUB_OUTPUT" - - - name: Connect to Tailscale - uses: tailscale/github-action@v4 - with: - oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} - oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} - tags: tag:ci - - - name: Verify OCI host deployment contract - uses: appleboy/ssh-action@v1.2.5 - with: - host: oracle-cloud - username: ubuntu - key: ${{ secrets.ORACLE_SSH_KEY }} - script: preflight ${{ steps.contract.outputs.preflight_sha256 }} ${{ steps.contract.outputs.update_sha256 }} ${{ steps.contract.outputs.compose_override_sha256 }} - build-and-publish: - needs: host-preflight runs-on: ubuntu-24.04-arm - timeout-minutes: 20 - outputs: - image_ref: ${{ steps.published_image.outputs.image_ref }} env: IMAGE_NAME: ghcr.io/team-maple/mls-be/mapleland-api - IMAGE_TAG: ${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}-arm64 + IMAGE_TAG: latest-arm64 steps: - name: Checkout code uses: actions/checkout@v4 - name: Restore FCM key file - env: - FIREBASE_KEY: ${{ secrets.FIREBASE_KEY }} run: | - set -euo pipefail - umask 077 - firebase_dir=src/main/resources/firebase - firebase_key="${firebase_dir}/maple-9f1a7-firebase-adminsdk-fbsvc-7c3b6fc032.json" - mkdir -p "${firebase_dir}" - printf '%s' "$FIREBASE_KEY" \ - > "${firebase_key}" - test -s "${firebase_key}" - jq -e 'type == "object" and .type == "service_account" and - (.private_key | type == "string" and length > 0) and - (.client_email | type == "string" and length > 0)' \ - "${firebase_key}" >/dev/null - # Paketo launches as uid 1002, gid 1001 while application layers are - # owned by 1001:1001. Keep the secret group-readable only so the - # runtime process can traverse/read it without making it world-readable. - chmod 0750 "${firebase_dir}" - chmod 0640 "${firebase_key}" + mkdir -p src/main/resources/firebase + echo '${{ secrets.FIREBASE_KEY }}' > src/main/resources/firebase/maple-9f1a7-firebase-adminsdk-fbsvc-7c3b6fc032.json - name: Set up JDK 21 uses: actions/setup-java@v4 @@ -106,20 +37,7 @@ jobs: run: chmod +x ./gradlew - name: Build jar - shell: bash - run: | - set -euo pipefail - ./gradlew clean bootJar - jar_path=build/libs/api-0.0.1-SNAPSHOT.jar - firebase_dir_mode="$(zipinfo -l "${jar_path}" \ - 'BOOT-INF/classes/firebase/' | awk 'NR == 1 { print $1 }')" - firebase_key_mode="$(zipinfo -l "${jar_path}" \ - 'BOOT-INF/classes/firebase/maple-9f1a7-firebase-adminsdk-fbsvc-7c3b6fc032.json' \ - | awk 'NR == 1 { print $1 }')" - # ZIP directory entries use '-' in zipinfo's type column; the - # trailing slash still materializes this entry as a directory. - test "${firebase_dir_mode}" = '-rwxr-x---' - test "${firebase_key_mode}" = '-rw-r-----' + run: ./gradlew clean bootJar - name: Set up pack CLI uses: buildpacks/github-actions/setup-pack@v5.11.0 @@ -139,69 +57,14 @@ jobs: --platform linux/arm64 \ --publish - - name: Verify published image FCM permissions - shell: bash - run: | - set -euo pipefail - image_ref="${IMAGE_NAME}:${IMAGE_TAG}" - docker pull "${image_ref}" >/dev/null - runtime_user="$(docker image inspect --format '{{.Config.User}}' "${image_ref}")" - test "${runtime_user}" = '1002:1001' - - verification_container="fcm-permission-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" - cleanup() { - docker rm -f "${verification_container}" >/dev/null 2>&1 || true - } - trap cleanup EXIT - docker create --name "${verification_container}" "${image_ref}" >/dev/null - metadata="$(docker export "${verification_container}" \ - | tar --numeric-owner -tvf - \ - | awk ' - $NF == "workspace/BOOT-INF/classes/firebase/" { - directory_mode = $1; directory_owner = $2 - } - $NF == "workspace/BOOT-INF/classes/firebase/maple-9f1a7-firebase-adminsdk-fbsvc-7c3b6fc032.json" { - key_mode = $1; key_owner = $2 - } - END { - if (!directory_mode || !key_mode) exit 1 - print directory_mode, directory_owner, key_mode, key_owner - } - ')" - read -r directory_mode directory_owner key_mode key_owner <<< "${metadata}" - test "${directory_mode}" = 'drwxr-x---' - test "${directory_owner}" = '1001/1001' - test "${key_mode}" = '-rw-r-----' - test "${key_owner}" = '1001/1001' - - - name: Resolve published manifest digest - id: published_image - shell: bash - run: | - set -euo pipefail - image_tag="${IMAGE_NAME}:${IMAGE_TAG}" - digest="$(docker buildx imagetools inspect "$image_tag" \ - --format '{{json .Manifest}}' | jq -er '.digest')" - if [[ ! $digest =~ ^sha256:[0-9a-f]{64}$ ]]; then - echo "invalid published image digest" >&2 - exit 1 - fi - docker buildx imagetools inspect "${IMAGE_NAME}@${digest}" >/dev/null - printf 'image_ref=%s@%s\n' "$IMAGE_NAME" "$digest" >> "$GITHUB_OUTPUT" - deploy: - needs: - - host-preflight - - build-and-publish + needs: build-and-publish runs-on: ubuntu-latest - timeout-minutes: 15 - environment: - name: production permissions: - id-token: write contents: read env: - IMAGE_REF: ${{ needs['build-and-publish'].outputs.image_ref }} + IMAGE_NAME: ghcr.io/team-maple/mls-be/mapleland-api + IMAGE_TAG: latest-arm64 steps: - name: Connect to Tailscale @@ -217,4 +80,5 @@ jobs: host: oracle-cloud username: ubuntu key: ${{ secrets.ORACLE_SSH_KEY }} - script: deploy ${{ needs['host-preflight'].outputs.preflight_sha256 }} ${{ needs['host-preflight'].outputs.update_sha256 }} ${{ needs['host-preflight'].outputs.compose_override_sha256 }} ${{ env.IMAGE_REF }} + script: | + /opt/mapleland/update-api.sh diff --git a/build.gradle b/build.gradle index 654a930..b70f9b4 100644 --- a/build.gradle +++ b/build.gradle @@ -65,6 +65,9 @@ dependencies { testImplementation 'com.h2database:h2' testImplementation 'org.springframework.boot:spring-boot-starter-test' testImplementation 'org.springframework.security:spring-security-test' + testImplementation 'org.testcontainers:junit-jupiter' + testImplementation 'org.testcontainers:mysql' + testImplementation 'net.ttddyy:datasource-proxy:1.10.1' testRuntimeOnly 'org.junit.platform:junit-platform-launcher' } @@ -81,7 +84,26 @@ sourceSets { } } +// JGit bundled by gradle-git-properties 2.4.1 treats a linked-worktree `.git` +// pointer file as a directory. CI/release checkouts still generate git.properties; +// linked worktrees skip only that metadata task so the required clean test/bootJar +// validation can run from an isolated feature worktree. +if (file('.git').isFile()) { + def commonGitDir = providers.exec { + commandLine 'git', 'rev-parse', '--path-format=absolute', '--git-common-dir' + }.standardOutput.asText.get().trim() + gitProperties { + dotGitDirectory.set(file(commonGitDir)) + } + tasks.named('generateGitProperties') { + enabled = false + } +} tasks.named('test') { useJUnitPlatform() + // Testcontainers 1.20.6 defaults to an API older than Docker Engine 29's + // supported floor. API 1.40 is accepted by both current CI engines and + // Docker 29; callers can still override it with -Dapi.version. + systemProperty 'api.version', System.getProperty('api.version', '1.40') } diff --git a/deploy/observability/alloy/config.alloy b/deploy/observability/alloy/config.alloy index fd4d680..405775d 100644 --- a/deploy/observability/alloy/config.alloy +++ b/deploy/observability/alloy/config.alloy @@ -99,12 +99,13 @@ prometheus.scrape "application" { prometheus.relabel "application" { forward_to = [prometheus.remote_write.grafana_cloud.receiver] - // Preserve only the metrics needed for RED, JVM, process, and HikariCP - // views. Route-template `uri` remains; high-cardinality identifiers do not. + // Preserve only the metrics needed for RED, JVM, process, HikariCP, and the + // bounded recommendation outcome/result views. Route-template `uri` remains; + // high-cardinality identifiers do not. rule { action = "keep" source_labels = ["__name__"] - regex = "^(up|http_server_requests_seconds_(bucket|count|sum|max)|jvm_memory_(used|committed|max)_bytes|jvm_gc_pause_seconds_(bucket|count|sum|max)|jvm_threads_(live|daemon|peak|states)_threads|process_(cpu|memory|resident|virtual|uptime|start|files).*|system_cpu_(usage|count)|hikaricp_connections_(active|idle|pending|max|min|timeout_total|acquire_seconds_(bucket|count|sum|max)|creation_seconds_(bucket|count|sum|max)|usage_seconds_(bucket|count|sum|max)))$" + regex = "^(up|http_server_requests_seconds_(bucket|count|sum|max)|jvm_memory_(used|committed|max)_bytes|jvm_gc_pause_seconds_(bucket|count|sum|max)|jvm_threads_(live|daemon|peak|states)_threads|process_(cpu|memory|resident|virtual|uptime|start|files).*|system_cpu_(usage|count)|hikaricp_connections_(active|idle|pending|max|min|timeout_total|acquire_seconds_(bucket|count|sum|max)|creation_seconds_(bucket|count|sum|max)|usage_seconds_(bucket|count|sum|max))|mapleland_recommendation_requests_total|mapleland_recommendation_results_recommendations_(count|sum|max))$" } rule { @@ -224,28 +225,32 @@ loki.process "application" { stage.json { expressions = { - ecs_timestamp = "\"@timestamp\"", - level = "\"log.level\"", - logger = "\"log.logger\"", - thread = "\"process.thread.name\"", - process_pid = "\"process.pid\"", - service_name = "\"service.name\"", - deployment_environment = "\"service.environment\"", - service_version = "\"service.version\"", - event_action = "\"event.action\"", - event_outcome = "\"event.outcome\"", - event_category = "\"event.category\"", - http_request_method = "\"http.request.method\"", - http_response_status_code = "\"http.response.status_code\"", - http_route = "\"http.route\"", - error_type = "\"error.type\"", - error_message = "\"error.message\"", - trace_id = "\"trace.id\"", - span_id = "\"span.id\"", - request_id = "\"request.id\"", - mapleland_batch_type = "\"mapleland.batch.type\"", - mapleland_batch_record_count = "\"mapleland.batch.record_count\"", - mapleland_external_system = "\"mapleland.external.system\"", + ecs_timestamp = "\"@timestamp\"", + level = "\"log.level\"", + logger = "\"log.logger\"", + thread = "\"process.thread.name\"", + process_pid = "\"process.pid\"", + service_name = "\"service.name\"", + deployment_environment = "\"service.environment\"", + service_version = "\"service.version\"", + event_action = "\"event.action\"", + event_outcome = "\"event.outcome\"", + event_category = "\"event.category\"", + http_request_method = "\"http.request.method\"", + http_response_status_code = "\"http.response.status_code\"", + http_route = "\"http.route\"", + error_type = "\"error.type\"", + error_message = "\"error.message\"", + trace_id = "\"trace.id\"", + span_id = "\"span.id\"", + request_id = "\"request.id\"", + mapleland_batch_type = "\"mapleland.batch.type\"", + mapleland_batch_record_count = "\"mapleland.batch.record_count\"", + mapleland_external_system = "\"mapleland.external.system\"", + event_duration = "\"event.duration\"", + mapleland_recommendation_engine = "\"mapleland.recommendation.engine\"", + mapleland_api_version = "\"mapleland.api.version\"", + mapleland_result_count = "\"mapleland.result.count\"", } drop_malformed = false } @@ -291,24 +296,28 @@ loki.process "application" { // ECS and mapleland.* fields remain available through `| json` as well. stage.structured_metadata { values = { - logger = "logger", - thread = "thread", - process_pid = "process_pid", - service_version = "service_version", - event_action = "event_action", - event_outcome = "event_outcome", - event_category = "event_category", - http_request_method = "http_request_method", - http_response_status_code = "http_response_status_code", - http_route = "http_route", - error_type = "error_type", - error_message = "error_message", - trace_id = "trace_id", - span_id = "span_id", - request_id = "request_id", - mapleland_batch_type = "mapleland_batch_type", - mapleland_batch_record_count = "mapleland_batch_record_count", - mapleland_external_system = "mapleland_external_system", + logger = "logger", + thread = "thread", + process_pid = "process_pid", + service_version = "service_version", + event_action = "event_action", + event_outcome = "event_outcome", + event_category = "event_category", + http_request_method = "http_request_method", + http_response_status_code = "http_response_status_code", + http_route = "http_route", + error_type = "error_type", + error_message = "error_message", + trace_id = "trace_id", + span_id = "span_id", + request_id = "request_id", + mapleland_batch_type = "mapleland_batch_type", + mapleland_batch_record_count = "mapleland_batch_record_count", + mapleland_external_system = "mapleland_external_system", + event_duration = "event_duration", + mapleland_recommendation_engine = "mapleland_recommendation_engine", + mapleland_api_version = "mapleland_api_version", + mapleland_result_count = "mapleland_result_count", } } diff --git a/deploy/observability/docker-compose.override.example.yml b/deploy/observability/docker-compose.override.example.yml index dc35773..87ff670 100644 --- a/deploy/observability/docker-compose.override.example.yml +++ b/deploy/observability/docker-compose.override.example.yml @@ -1,6 +1,6 @@ # Install this reviewed fragment as -# /opt/mapleland/docker-compose.observability.yml. Deployment and preflight -# always render it after the existing base Compose file. The management +# /opt/mapleland/docker-compose.observability.yml. Deployment renders it after +# the existing base Compose file. The management # endpoint is reachable only from this VM; it is not attached to Traefik and # no firewall port is opened. services: @@ -10,6 +10,9 @@ services: MANAGEMENT_SERVER_PORT: "18080" MANAGEMENT_SCRAPE_TOKEN: "${MANAGEMENT_SCRAPE_TOKEN:?MANAGEMENT_SCRAPE_TOKEN must be set}" SERVICE_VERSION: "${SERVICE_VERSION:-unknown}" + RECOMMENDATION_V1_ENGINE: "${RECOMMENDATION_V1_ENGINE:-AURA}" + RECOMMENDATION_V2_ENABLED: "${RECOMMENDATION_V2_ENABLED:-false}" + RECOMMENDATION_QUERY_TIMEOUT_SECONDS: "${RECOMMENDATION_QUERY_TIMEOUT_SECONDS:-10}" ports: - "127.0.0.1:18080:18080" volumes: diff --git a/deploy/observability/grafana/mapleland-production-overview.json b/deploy/observability/grafana/mapleland-production-overview.json index 1bf83e5..f41d971 100644 --- a/deploy/observability/grafana/mapleland-production-overview.json +++ b/deploy/observability/grafana/mapleland-production-overview.json @@ -1478,6 +1478,319 @@ "title": "Filesystem available", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 18, + "panels": [], + "title": "Recommendations", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "Total v1/v2 endpoint request rate from the existing http.server.requests route-template metric, including validation and configuration failures.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 0, + "y": 29 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum by (uri) (rate(http_server_requests_seconds_count{service_name=\"mapleland-api\",deployment_environment=\"prod\",cloud_provider=\"oci\",instance=\"$instance\",uri=~\"/api/v[12]/maps/recommendations\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{uri}}", + "range": true, + "refId": "A" + } + ], + "title": "Recommendation request rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "p95 latency from the existing http.server.requests route-template histogram; no duplicate recommendation timer is used.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 5, + "y": 29 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.95, sum by (le, uri) (rate(http_server_requests_seconds_bucket{service_name=\"mapleland-api\",deployment_environment=\"prod\",cloud_provider=\"oci\",instance=\"$instance\",uri=~\"/api/v[12]/maps/recommendations\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "p95 {{uri}}", + "range": true, + "refId": "A" + } + ], + "title": "Recommendation p95 latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "HTTP 4xx/5xx rates plus bounded empty and unavailable processing outcomes for the recommendation routes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 10, + "y": 29 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "editorMode": "code", + "exemplar": true, + "expr": "sum by (api_version, outcome) (rate(mapleland_recommendation_requests_total{service_name=\"mapleland-api\",deployment_environment=\"prod\",cloud_provider=\"oci\",instance=\"$instance\",outcome=~\"empty|unavailable\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{api_version}} {{outcome}}", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "exemplar": true, + "expr": "sum by (uri, status) (rate(http_server_requests_seconds_count{service_name=\"mapleland-api\",deployment_environment=\"prod\",cloud_provider=\"oci\",instance=\"$instance\",uri=~\"/api/v[12]/maps/recommendations\",status=~\"4..|5..\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{uri}} HTTP {{status}}", + "range": true, + "refId": "B" + } + ], + "title": "Error / empty / unavailable rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "Engine/API pairs observed in the selected dashboard time range. ACTIVE means at least one request was recorded; this is traffic-observed state, not configuration truth.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "0": { + "color": "text", + "index": 0, + "text": "NO REQUESTS" + }, + "1": { + "color": "green", + "index": 1, + "text": "ACTIVE" + } + }, + "type": "value" + } + ], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 15, + "y": 29 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "sum by (api_version, engine) (increase(mapleland_recommendation_requests_total{service_name=\"mapleland-api\",deployment_environment=\"prod\",cloud_provider=\"oci\",instance=\"$instance\"}[$__range])) > bool 0", + "instant": true, + "legendFormat": "{{api_version}} / {{engine}}", + "range": false, + "refId": "A" + } + ], + "title": "Engine state (observed)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "grafanacloud-prom" + }, + "description": "Average result count for completed recommendation responses, including successful empty responses and excluding unavailable failures.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "decimals": 2, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 19, + "y": 29 + }, + "id": 23, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "sum by (api_version, engine) (rate(mapleland_recommendation_results_recommendations_sum{service_name=\"mapleland-api\",deployment_environment=\"prod\",cloud_provider=\"oci\",instance=\"$instance\"}[$__rate_interval])) / clamp_min(sum by (api_version, engine) (rate(mapleland_recommendation_results_recommendations_count{service_name=\"mapleland-api\",deployment_environment=\"prod\",cloud_provider=\"oci\",instance=\"$instance\"}[$__rate_interval])), 0.000000001)", + "instant": false, + "legendFormat": "{{api_version}} / {{engine}}", + "range": true, + "refId": "A" + } + ], + "title": "Average result count", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 24, + "panels": [], + "title": "Application logs", + "type": "row" + }, { "datasource": { "type": "loki", @@ -1488,7 +1801,7 @@ "h": 10, "w": 24, "x": 0, - "y": 28 + "y": 38 }, "id": 17, "options": { @@ -1559,6 +1872,6 @@ "timezone": "browser", "title": "Mapleland / Production Overview", "uid": "mapleland-production-overview", - "version": 1, + "version": 3, "weekStart": "" } diff --git a/deploy/observability/preflight-host.sh b/deploy/observability/preflight-host.sh deleted file mode 100755 index aafd00d..0000000 --- a/deploy/observability/preflight-host.sh +++ /dev/null @@ -1,358 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -readonly EXPECTED_REPOSITORY=ghcr.io/team-maple/mls-be/mapleland-api -readonly EXPECTED_SHA256_REGEX='^[0-9a-f]{64}$' -readonly EXPECTED_SERVICE_VERSION_REGEX='^[0-9a-f]{40}$' -readonly LEGACY_COMPOSE_ENV_REGEX='^[[:space:]]*-[[:space:]]*GRAFANA_CLOUD_(URL|USERNAME|PASSWORD)=' -readonly LEGACY_APP_ENV_REGEX='^GRAFANA_CLOUD_(URL|USERNAME|PASSWORD)=' - -fail() { - echo "host preflight failed: $*" >&2 - exit 1 -} - -if [[ ${MAPLELAND_PREFLIGHT_TEST_MODE:-0} == 1 ]]; then - (( EUID != 0 )) || fail 'test mode is forbidden for root' - readonly ROOT_PREFIX=${MAPLELAND_PREFLIGHT_TEST_ROOT:?test root is required} - [[ ${ROOT_PREFIX} == /* && ${ROOT_PREFIX} != / ]] \ - || fail 'test root must be a non-root absolute path' - readonly PREFLIGHT_FILE=${BASH_SOURCE[0]} -else - (( EUID == 0 )) || fail 'preflight-host.sh must run as root' - readonly ROOT_PREFIX='' - readonly PREFLIGHT_FILE=/opt/mapleland/preflight-host.sh -fi - -host_path() { - printf '%s%s' "${ROOT_PREFIX}" "$1" -} - -COMPOSE_FILE=$(host_path /opt/mapleland/docker-compose.yml) -readonly COMPOSE_FILE -COMPOSE_OVERRIDE_FILE=$(host_path /opt/mapleland/docker-compose.observability.yml) -readonly COMPOSE_OVERRIDE_FILE -APP_ENV_FILE=$(host_path /opt/mapleland/.env) -readonly APP_ENV_FILE -GHCR_ENV_FILE=$(host_path /opt/mapleland/ghcr.env) -readonly GHCR_ENV_FILE -UPDATE_FILE=$(host_path /opt/mapleland/update-api.sh) -readonly UPDATE_FILE -ALLOY_CONFIG_FILE=$(host_path /etc/alloy/config.alloy) -readonly ALLOY_CONFIG_FILE -ALLOY_ENV_FILE=$(host_path /etc/alloy/alloy.env) -readonly ALLOY_ENV_FILE -APP_LOG_DIR=$(host_path /var/log/mapleland-api) -readonly APP_LOG_DIR -APP_LOG_FILE=$(host_path /var/log/mapleland-api/mapleland-api.json) -readonly APP_LOG_FILE - -if [[ $# -ne 3 ]]; then - fail 'usage: preflight-host.sh ' -fi -readonly expected_preflight_sha=$1 -readonly expected_update_sha=$2 -readonly expected_compose_override_sha=$3 -[[ ${expected_preflight_sha} =~ ${EXPECTED_SHA256_REGEX} ]] \ - || fail 'expected preflight checksum must be 64 lowercase hex characters' -[[ ${expected_update_sha} =~ ${EXPECTED_SHA256_REGEX} ]] \ - || fail 'expected update checksum must be 64 lowercase hex characters' -[[ ${expected_compose_override_sha} =~ ${EXPECTED_SHA256_REGEX} ]] \ - || fail 'expected Compose override checksum must be 64 lowercase hex characters' - -for required_command in awk bash curl docker getfacl grep id jq runuser sha256sum ss stat systemctl; do - command -v "${required_command}" >/dev/null \ - || fail "required command is missing: ${required_command}" -done - -require_root_file() { - local path=$1 - local expected_mode=$2 - local label=$3 - - [[ -f ${path} && ! -L ${path} ]] \ - || fail "${label} must be a regular, non-symlink file" - [[ $(stat -c '%u' "${path}") == 0 && $(stat -c '%a' "${path}") == "${expected_mode}" ]] \ - || fail "${label} must be owned by root with mode 0${expected_mode}" -} - -require_root_file "${PREFLIGHT_FILE}" 755 preflight-host.sh -require_root_file "${UPDATE_FILE}" 755 update-api.sh -require_root_file "${COMPOSE_FILE}" 640 docker-compose.yml -require_root_file "${COMPOSE_OVERRIDE_FILE}" 640 docker-compose.observability.yml -require_root_file "${APP_ENV_FILE}" 600 .env -require_root_file "${GHCR_ENV_FILE}" 600 ghcr.env -require_root_file "${ALLOY_CONFIG_FILE}" 640 config.alloy -require_root_file "${ALLOY_ENV_FILE}" 600 alloy.env - -actual_preflight_sha=$(sha256sum "${PREFLIGHT_FILE}" | awk '{print $1}') -[[ ${actual_preflight_sha} == "${expected_preflight_sha}" ]] \ - || fail 'preflight-host.sh checksum mismatch; install the reviewed repository version before deployment' - -actual_update_sha=$(sha256sum "${UPDATE_FILE}" | awk '{print $1}') -[[ ${actual_update_sha} == "${expected_update_sha}" ]] \ - || fail 'update-api.sh checksum mismatch; active host script is stale or unreviewed' -bash -n "${UPDATE_FILE}" - -actual_compose_override_sha=$(sha256sum "${COMPOSE_OVERRIDE_FILE}" | awk '{print $1}') -[[ ${actual_compose_override_sha} == "${expected_compose_override_sha}" ]] \ - || fail 'docker-compose.observability.yml checksum mismatch; install the reviewed repository version' - -validate_ghcr_env() { - local line - local key - local value - local user_seen=false - local token_seen=false - - while IFS= read -r line || [[ -n ${line} ]]; do - case ${line} in - ''|'#'*) continue ;; - esac - [[ ${line} == *=* ]] || fail 'ghcr.env contains an invalid line' - key=${line%%=*} - value=${line#*=} - [[ -n ${value} && ${value} =~ ^[[:graph:]]+$ ]] \ - || fail 'ghcr.env values must be non-empty printable tokens' - case ${key} in - GHCR_USER) - [[ ${user_seen} == false ]] || fail 'ghcr.env contains duplicate GHCR_USER' - user_seen=true - ;; - GHCR_TOKEN) - [[ ${token_seen} == false ]] || fail 'ghcr.env contains duplicate GHCR_TOKEN' - token_seen=true - ;; - *) fail "ghcr.env contains an unexpected key: ${key}" ;; - esac - done < "${GHCR_ENV_FILE}" - - [[ ${user_seen} == true && ${token_seen} == true ]] \ - || fail 'ghcr.env must contain exactly one GHCR_USER and GHCR_TOKEN' -} - -validate_app_env() { - local line - local key - local value - local management_token_seen=false - local service_version_seen=false - local legacy_url_seen=false - local legacy_username_seen=false - local legacy_password_seen=false - app_legacy_count=0 - - while IFS= read -r line || [[ -n ${line} ]]; do - case ${line} in - ''|'#'*) continue ;; - esac - [[ ${line} == *=* ]] || continue - key=${line%%=*} - value=${line#*=} - case ${key} in - GRAFANA_CLOUD_URL) - [[ ${legacy_url_seen} == false ]] || fail '.env contains duplicate GRAFANA_CLOUD_URL' - legacy_url_seen=true - (( app_legacy_count += 1 )) - ;; - GRAFANA_CLOUD_USERNAME) - [[ ${legacy_username_seen} == false ]] || fail '.env contains duplicate GRAFANA_CLOUD_USERNAME' - legacy_username_seen=true - (( app_legacy_count += 1 )) - ;; - GRAFANA_CLOUD_PASSWORD) - [[ ${legacy_password_seen} == false ]] || fail '.env contains duplicate GRAFANA_CLOUD_PASSWORD' - legacy_password_seen=true - (( app_legacy_count += 1 )) - ;; - MANAGEMENT_SCRAPE_TOKEN) - [[ ${management_token_seen} == false ]] \ - || fail '.env contains duplicate MANAGEMENT_SCRAPE_TOKEN' - [[ -n ${value} && ${#value} -ge 32 && ${value} =~ ^[[:graph:]]+$ ]] \ - || fail '.env MANAGEMENT_SCRAPE_TOKEN must be at least 32 printable characters' - management_scrape_token=${value} - management_token_seen=true - ;; - SERVICE_VERSION) - [[ ${service_version_seen} == false ]] \ - || fail '.env contains duplicate SERVICE_VERSION' - [[ ${value} =~ ${EXPECTED_SERVICE_VERSION_REGEX} ]] \ - || fail '.env SERVICE_VERSION must be a full lowercase Git commit SHA' - service_version_seen=true - ;; - esac - done < "${APP_ENV_FILE}" - - [[ ${management_token_seen} == true && ${service_version_seen} == true ]] \ - || fail '.env must contain MANAGEMENT_SCRAPE_TOKEN and SERVICE_VERSION' -} - -management_scrape_token='' -validate_ghcr_env -validate_app_env -readonly management_scrape_token - -curl_management_prometheus() { - local escaped_token=${management_scrape_token//\\/\\\\} - escaped_token=${escaped_token//\"/\\\"} - printf 'header = "Authorization: Bearer %s"\n' "${escaped_token}" | - curl --config - "$@" http://127.0.0.1:18080/actuator/prometheus -} - -base_legacy_count=0 -for legacy_key in URL USERNAME PASSWORD; do - legacy_key_count=$(grep -Ec \ - "^[[:space:]]*-[[:space:]]*GRAFANA_CLOUD_${legacy_key}=" \ - "${COMPOSE_FILE}" || true) - (( legacy_key_count <= 1 )) \ - || fail "docker-compose.yml contains duplicate GRAFANA_CLOUD_${legacy_key}" - base_legacy_count=$((base_legacy_count + legacy_key_count)) -done -case "${base_legacy_count}:${app_legacy_count}" in - 0:0) legacy_rollback_contract=removed ;; - 3:3) legacy_rollback_contract=preserved ;; - *) fail 'legacy Grafana rollback environment must be either fully preserved or fully removed' ;; -esac -readonly legacy_rollback_contract - -# Compose otherwise auto-loads /opt/mapleland/.env. Use a root-only filtered -# copy so the first deployment can keep its exact legacy rollback inputs while -# proving that the new container model receives none of them. -sanitized_app_env=$(mktemp) -readonly sanitized_app_env -chmod 0600 "${sanitized_app_env}" -awk -v regex="${LEGACY_APP_ENV_REGEX}" '$0 !~ regex' \ - "${APP_ENV_FILE}" > "${sanitized_app_env}" -sanitized_compose_file=$(mktemp "$(dirname "${COMPOSE_FILE}")/.docker-compose.preflight.XXXXXX") -readonly sanitized_compose_file -chmod 0640 "${sanitized_compose_file}" -awk -v regex="${LEGACY_COMPOSE_ENV_REGEX}" '$0 !~ regex' \ - "${COMPOSE_FILE}" > "${sanitized_compose_file}" -trap 'rm -f "${sanitized_app_env}" "${sanitized_compose_file}"' EXIT - -readonly -a COMPOSE_ARGS=( - --env-file "${sanitized_app_env}" - -f "${sanitized_compose_file}" - -f "${COMPOSE_OVERRIDE_FILE}" -) -docker compose "${COMPOSE_ARGS[@]}" config --quiet -compose_json=$(docker compose "${COMPOSE_ARGS[@]}" config --format json) -if ! printf '%s' "${compose_json}" | jq -e --arg repository "${EXPECTED_REPOSITORY}:" ' - .services["mapleland-api"] as $app | - ($app | type == "object") and - (($app.image | type) == "string" and ($app.image | startswith($repository))) and - ($app.environment.MANAGEMENT_SERVER_ADDRESS == "0.0.0.0") and - (($app.environment.MANAGEMENT_SERVER_PORT | tostring) == "18080") and - (($app.environment.MANAGEMENT_SCRAPE_TOKEN | type) == "string") and - (($app.environment.MANAGEMENT_SCRAPE_TOKEN | length) >= 32) and - (($app.environment.SERVICE_VERSION | type) == "string") and - ($app.environment.SERVICE_VERSION | test("^[0-9a-f]{40}$")) and - ([ ($app.environment // {} | keys[]) | - select(startswith("GRAFANA_CLOUD_")) ] | length == 0) and - ([ $app.ports[]? | select((.target | tostring) == "18080") ] | length == 1) and - ([ $app.ports[]? | - select((.target | tostring) == "18080" and - (.published | tostring) == "18080" and - .host_ip == "127.0.0.1" and - (.protocol // "tcp") == "tcp") ] | length == 1) and - (any($app.volumes[]?; - .type == "bind" and - .source == "/var/log/mapleland-api" and - .target == "/workspace/logs" and - ((.bind.create_host_path // false) == false))) -' >/dev/null; then - compose_json='' - fail 'Compose observability contract is incomplete or exposes the management boundary' -fi -compose_sha=$(printf '%s' "${compose_json}" | sha256sum | awk '{print $1}') -compose_json='' - -[[ -d ${APP_LOG_DIR} && ! -L ${APP_LOG_DIR} ]] \ - || fail 'application log directory must be a non-symlink directory' -[[ $(stat -c '%u:%g %a' "${APP_LOG_DIR}") == '1002:1001 750' ]] \ - || fail 'application log directory must be owned by 1002:1001 with mode 0750' -acl=$(getfacl -cp "${APP_LOG_DIR}") -grep -Fx 'user:alloy:r-x' <<< "${acl}" >/dev/null \ - || fail 'application log directory is missing the Alloy access ACL' -grep -Fx 'default:user:alloy:r-x' <<< "${acl}" >/dev/null \ - || fail 'application log directory is missing the Alloy default ACL' -acl='' -runuser -u alloy -- test -r "${ALLOY_CONFIG_FILE}" \ - || fail 'Alloy cannot read its configuration' -if [[ -e ${APP_LOG_FILE} ]]; then - [[ -f ${APP_LOG_FILE} && ! -L ${APP_LOG_FILE} ]] \ - || fail 'active application log must be a regular, non-symlink file' - runuser -u alloy -- test -r "${APP_LOG_FILE}" \ - || fail 'Alloy cannot read the active application log' -fi - -if id -nG alloy | grep -Eq '(^| )(adm|systemd-journal)( |$)'; then - fail 'Alloy has an unnecessary broad log-reading group' -fi -systemctl is-active --quiet alloy.service \ - || fail 'Alloy service is not active' -[[ $(systemctl is-enabled alloy.service) == enabled ]] \ - || fail 'Alloy service is not enabled' -curl --fail --silent --show-error --max-time 5 \ - http://127.0.0.1:12345/-/ready >/dev/null \ - || fail 'Alloy readiness endpoint failed' - -validate_exact_loopback_listener() { - local port=$1 - local allow_absent=$2 - local sockets - sockets=$(ss -H -lnt "sport = :${port}") - if [[ -z ${sockets} ]]; then - [[ ${allow_absent} == true ]] || fail "port ${port} has no listener" - return 1 - fi - if ! printf '%s\n' "${sockets}" | awk -v expected="127.0.0.1:${port}" ' - $4 == expected { exact++ } - END { exit !(NR == 1 && exact == 1) } - '; then - if [[ ${port} == 18080 ]]; then - fail 'management port must be absent or bound only to 127.0.0.1' - fi - fail "port ${port} must have exactly one IPv4 loopback listener" - fi - return 0 -} - -validate_exact_loopback_listener 12345 false -management_listener=absent -if validate_exact_loopback_listener 18080 true; then - management_listener=loopback - curl_management_prometheus \ - --fail --silent --show-error --max-time 5 >/dev/null \ - || fail 'authenticated application Prometheus endpoint failed' -fi - -app_container_ids=$(docker compose "${COMPOSE_ARGS[@]}" ps -q mapleland-api) -[[ $(printf '%s\n' "${app_container_ids}" | grep -c .) -eq 1 ]] \ - || fail 'exactly one existing mapleland-api container is required' -readonly app_container_id=${app_container_ids} -app_state=$(docker inspect "${app_container_id}" --format '{{.State.Status}}') -[[ ${app_state} == running ]] || fail 'current mapleland-api container is not running' -current_image_id=$(docker inspect "${app_container_id}" --format '{{.Image}}') -[[ ${current_image_id} =~ ^sha256:[0-9a-f]{64}$ ]] \ - || fail 'current mapleland-api image ID is invalid' -docker image inspect "${current_image_id}" >/dev/null \ - || fail 'current mapleland-api image is unavailable for rollback' -curl --fail --silent --show-error --max-time 5 \ - http://127.0.0.1:8080/api/v1/jobs >/dev/null \ - || fail 'current public API smoke check failed' - -attestation=$(printf '%s\n' \ - "${actual_preflight_sha}" \ - "${actual_update_sha}" \ - "${actual_compose_override_sha}" \ - "${compose_sha}" \ - "${app_container_id}" \ - "${current_image_id}" \ - "${legacy_rollback_contract}" \ - "${management_listener}" | - sha256sum | awk '{print $1}') - -printf 'host_preflight=ok attestation=%s current_image=%s legacy_rollback_contract=%s management_listener=%s\n' \ - "${attestation}" "${current_image_id}" "${legacy_rollback_contract}" \ - "${management_listener}" diff --git a/deploy/observability/tests/host-preflight-test.sh b/deploy/observability/tests/host-preflight-test.sh deleted file mode 100755 index 2e4ffb9..0000000 --- a/deploy/observability/tests/host-preflight-test.sh +++ /dev/null @@ -1,266 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -# The fixture builders intentionally write literal shell expansions into mock -# executables; those values must expand only when each mock runs. - -TEST_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -readonly TEST_DIR -OBSERVABILITY_DIR=$(cd "${TEST_DIR}/.." && pwd) -readonly OBSERVABILITY_DIR -readonly PREFLIGHT_SCRIPT=${OBSERVABILITY_DIR}/preflight-host.sh -readonly UPDATE_SCRIPT=${OBSERVABILITY_DIR}/update-api.sh -readonly COMPOSE_OVERRIDE_SOURCE=${OBSERVABILITY_DIR}/docker-compose.override.example.yml -readonly FIXTURE_VERSION=f4bf228b934959be125a72540c91e43f003b7b6e - -fail() { - echo "FAIL: $*" >&2 - exit 1 -} - -assert_contains() { - local haystack=$1 - local needle=$2 - [[ ${haystack} == *"${needle}"* ]] \ - || fail "expected output to contain: ${needle}" -} - -assert_not_contains() { - local haystack=$1 - local needle=$2 - [[ ${haystack} != *"${needle}"* ]] \ - || fail "output exposed protected fixture value: ${needle}" -} - -TMP_DIR=$(mktemp -d) -readonly TMP_DIR -trap 'rm -rf "${TMP_DIR}"' EXIT -readonly ROOT_DIR=${TMP_DIR}/root -readonly FAKE_BIN=${TMP_DIR}/bin - -mkdir -p \ - "${ROOT_DIR}/opt/mapleland" \ - "${ROOT_DIR}/etc/alloy" \ - "${ROOT_DIR}/var/log/mapleland-api" \ - "${FAKE_BIN}" - -cp "${UPDATE_SCRIPT}" "${ROOT_DIR}/opt/mapleland/update-api.sh" -cp "${COMPOSE_OVERRIDE_SOURCE}" \ - "${ROOT_DIR}/opt/mapleland/docker-compose.observability.yml" -printf '%s\n' \ - 'GHCR_USER=fixture-user' \ - 'GHCR_TOKEN=fixture-ghcr-secret' \ - > "${ROOT_DIR}/opt/mapleland/ghcr.env" -printf '%s\n' \ - 'MANAGEMENT_SCRAPE_TOKEN=fixture-management-secret-0123456789' \ - "SERVICE_VERSION=${FIXTURE_VERSION}" \ - > "${ROOT_DIR}/opt/mapleland/.env" -printf '%s\n' 'services:' ' mapleland-api:' \ - ' image: ghcr.io/team-maple/mls-be/mapleland-api:latest-arm64' \ - > "${ROOT_DIR}/opt/mapleland/docker-compose.yml" -printf '%s\n' 'fixture alloy config' > "${ROOT_DIR}/etc/alloy/config.alloy" -printf '%s\n' 'fixture alloy env' > "${ROOT_DIR}/etc/alloy/alloy.env" -printf '%s\n' '{"message":"fixture"}' \ - > "${ROOT_DIR}/var/log/mapleland-api/mapleland-api.json" - -# shellcheck disable=SC2016 -printf '%s\n' \ - '#!/usr/bin/env bash' \ - 'set -euo pipefail' \ - 'format=${2:-}' \ - 'path=${3:-}' \ - 'case "${format}" in' \ - " '%u') printf '%s\\n' 0 ;;" \ - " '%g')" \ - " if [[ \${path} == */var/log/mapleland-api ]]; then printf '%s\\n' 1001; else printf '%s\\n' 0; fi ;;" \ - " '%a')" \ - " case \${path} in" \ - " */ghcr.env) printf '%s\\n' \"\${FAKE_GHCR_MODE:-600}\" ;;" \ - " */.env|*/alloy.env) printf '%s\\n' 600 ;;" \ - " */docker-compose.yml|*/docker-compose.observability.yml|*/config.alloy) printf '%s\\n' 640 ;;" \ - " */update-api.sh|*/preflight-host.sh) printf '%s\\n' 755 ;;" \ - " *) printf '%s\\n' 750 ;;" \ - ' esac ;;' \ - " '%u:%g %a')" \ - " if [[ \${path} == */var/log/mapleland-api ]]; then printf '%s\\n' '1002:1001 750'; else printf '%s\\n' '0:0 600'; fi ;;" \ - ' *) echo "unsupported fake stat format: ${format}" >&2; exit 2 ;;' \ - 'esac' \ - > "${FAKE_BIN}/stat" - -# shellcheck disable=SC2016 -printf '%s\n' \ - '#!/usr/bin/env bash' \ - 'set -euo pipefail' \ - 'if [[ ${1:-} == compose ]]; then' \ - ' shift' \ - ' while [[ ${1:-} == -f || ${1:-} == --env-file ]]; do shift 2; done' \ - ' case ${1:-} in' \ - ' config)' \ - ' if [[ ${2:-} == --quiet ]]; then exit 0; fi' \ - ' if [[ ${2:-} == --format && ${3:-} == json ]]; then' \ - " if [[ \${FAKE_LEGACY_GRAFANA_ENV:-0} == 1 ]]; then" \ - " printf '%s\\n' '{\"services\":{\"mapleland-api\":{\"image\":\"ghcr.io/team-maple/mls-be/mapleland-api:latest-arm64\",\"environment\":{\"MANAGEMENT_SERVER_ADDRESS\":\"0.0.0.0\",\"MANAGEMENT_SERVER_PORT\":\"18080\",\"MANAGEMENT_SCRAPE_TOKEN\":\"fixture-management-secret-0123456789\",\"SERVICE_VERSION\":\"f4bf228b934959be125a72540c91e43f003b7b6e\",\"GRAFANA_CLOUD_PASSWORD\":\"fixture-legacy-secret\"},\"ports\":[{\"host_ip\":\"127.0.0.1\",\"target\":18080,\"published\":\"18080\",\"protocol\":\"tcp\"}],\"volumes\":[{\"type\":\"bind\",\"source\":\"/var/log/mapleland-api\",\"target\":\"/workspace/logs\",\"bind\":{\"create_host_path\":false}}]}}}'" \ - " elif [[ \${FAKE_COMPOSE_VALID:-1} == 1 ]]; then" \ - " printf '%s\\n' '{\"services\":{\"mapleland-api\":{\"image\":\"ghcr.io/team-maple/mls-be/mapleland-api:latest-arm64\",\"environment\":{\"MANAGEMENT_SERVER_ADDRESS\":\"0.0.0.0\",\"MANAGEMENT_SERVER_PORT\":\"18080\",\"MANAGEMENT_SCRAPE_TOKEN\":\"fixture-management-secret-0123456789\",\"SERVICE_VERSION\":\"f4bf228b934959be125a72540c91e43f003b7b6e\"},\"ports\":[{\"host_ip\":\"127.0.0.1\",\"target\":18080,\"published\":\"18080\",\"protocol\":\"tcp\"}],\"volumes\":[{\"type\":\"bind\",\"source\":\"/var/log/mapleland-api\",\"target\":\"/workspace/logs\",\"bind\":{}}]}}}'" \ - " else" \ - " printf '%s\\n' '{\"services\":{\"mapleland-api\":{\"image\":\"ghcr.io/team-maple/mls-be/mapleland-api:latest-arm64\",\"environment\":{},\"ports\":[{\"host_ip\":\"0.0.0.0\",\"target\":18080,\"published\":\"18080\"}],\"volumes\":[]}}}'" \ - " fi" \ - " exit 0" \ - ' fi ;;' \ - ' ps) printf "%s\\n" fixture-container-id; exit 0 ;;' \ - ' esac' \ - 'elif [[ ${1:-} == inspect ]]; then' \ - ' case $* in' \ - ' *State.Status*) printf "%s\\n" running ;;' \ - ' *.Image*) printf "%s\\n" sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ;;' \ - ' *) exit 2 ;;' \ - ' esac' \ - ' exit 0' \ - 'elif [[ ${1:-} == image && ${2:-} == inspect ]]; then' \ - ' exit 0' \ - 'fi' \ - 'echo "unsupported fake docker invocation: $*" >&2' \ - 'exit 2' \ - > "${FAKE_BIN}/docker" - -# shellcheck disable=SC2016 -printf '%s\n' \ - '#!/usr/bin/env bash' \ - 'set -euo pipefail' \ - 'case ${1:-} in' \ - ' is-active) exit 0 ;;' \ - ' is-enabled) printf "%s\\n" enabled ;;' \ - ' *) exit 2 ;;' \ - 'esac' \ - > "${FAKE_BIN}/systemctl" - -printf '%s\n' '#!/usr/bin/env bash' 'exit 0' > "${FAKE_BIN}/curl" -printf '%s\n' '#!/usr/bin/env bash' 'exit 0' > "${FAKE_BIN}/runuser" -# shellcheck disable=SC2016 -printf '%s\n' \ - '#!/usr/bin/env bash' \ - 'if [[ ${1:-} == -nG && ${2:-} == alloy ]]; then printf "%s\\n" alloy; else /usr/bin/id "$@"; fi' \ - > "${FAKE_BIN}/id" - -# shellcheck disable=SC2016 -printf '%s\n' \ - '#!/usr/bin/env bash' \ - 'set -euo pipefail' \ - 'case $* in' \ - ' *:12345*) printf "%s\\n" "LISTEN 0 4096 127.0.0.1:12345 0.0.0.0:*" ;;' \ - ' *:18080*)' \ - ' if [[ ${FAKE_MANAGEMENT_LISTENER:-absent} == wildcard ]]; then' \ - ' printf "%s\\n" "LISTEN 0 4096 0.0.0.0:18080 0.0.0.0:*"' \ - ' elif [[ ${FAKE_MANAGEMENT_LISTENER:-absent} == loopback ]]; then' \ - ' printf "%s\\n" "LISTEN 0 4096 127.0.0.1:18080 0.0.0.0:*"' \ - ' fi ;;' \ - ' *) exit 2 ;;' \ - 'esac' \ - > "${FAKE_BIN}/ss" - -printf '%s\n' \ - '#!/usr/bin/env bash' \ - 'printf "%s\\n" "user:alloy:r-x" "default:user:alloy:r-x"' \ - > "${FAKE_BIN}/getfacl" - -printf '%s\n' \ - '#!/usr/bin/env bash' \ - 'if (($#)); then exec shasum -a 256 "$@"; else exec shasum -a 256; fi' \ - > "${FAKE_BIN}/sha256sum" - -chmod +x "${FAKE_BIN}"/* - -run_preflight() { - local expected_self_sha=$1 - local expected_update_sha=$2 - local expected_override_sha=$3 - shift 3 - env \ - PATH="${FAKE_BIN}:${PATH}" \ - MAPLELAND_PREFLIGHT_TEST_MODE=1 \ - MAPLELAND_PREFLIGHT_TEST_ROOT="${ROOT_DIR}" \ - "$@" \ - bash "${PREFLIGHT_SCRIPT}" \ - "${expected_self_sha}" \ - "${expected_update_sha}" \ - "${expected_override_sha}" -} - -SELF_SHA=$(shasum -a 256 "${PREFLIGHT_SCRIPT}" | awk '{print $1}') -readonly SELF_SHA -UPDATE_SHA=$(shasum -a 256 "${UPDATE_SCRIPT}" | awk '{print $1}') -readonly UPDATE_SHA -OVERRIDE_SHA=$(shasum -a 256 "${COMPOSE_OVERRIDE_SOURCE}" | awk '{print $1}') -readonly OVERRIDE_SHA - -success_output=$(run_preflight "${SELF_SHA}" "${UPDATE_SHA}" "${OVERRIDE_SHA}" 2>&1) \ - || fail "valid host contract should pass: ${success_output}" -assert_contains "${success_output}" 'host_preflight=ok' -assert_contains "${success_output}" 'management_listener=absent' -assert_not_contains "${success_output}" 'fixture-ghcr-secret' -assert_not_contains "${success_output}" 'fixture-management-secret' - -# Literal Compose interpolation expressions are intentional fixture data. -# shellcheck disable=SC2016 -printf '%s\n' \ - ' - GRAFANA_CLOUD_URL=${GRAFANA_CLOUD_URL}' \ - ' - GRAFANA_CLOUD_USERNAME=${GRAFANA_CLOUD_USERNAME}' \ - ' - GRAFANA_CLOUD_PASSWORD=${GRAFANA_CLOUD_PASSWORD}' \ - >> "${ROOT_DIR}/opt/mapleland/docker-compose.yml" -printf '%s\n' \ - 'GRAFANA_CLOUD_URL=https://example.invalid' \ - 'GRAFANA_CLOUD_USERNAME=fixture-user' \ - 'GRAFANA_CLOUD_PASSWORD=fixture-legacy-secret' \ - >> "${ROOT_DIR}/opt/mapleland/.env" -transition_output=$(run_preflight "${SELF_SHA}" "${UPDATE_SHA}" "${OVERRIDE_SHA}" 2>&1) \ - || fail "complete first-rollout rollback contract should pass: ${transition_output}" -assert_contains "${transition_output}" 'legacy_rollback_contract=preserved' -assert_not_contains "${transition_output}" 'fixture-legacy-secret' -for transition_file in \ - "${ROOT_DIR}/opt/mapleland/docker-compose.yml" \ - "${ROOT_DIR}/opt/mapleland/.env"; do - awk '!/GRAFANA_CLOUD_(URL|USERNAME|PASSWORD)/' \ - "${transition_file}" > "${transition_file}.clean" - mv "${transition_file}.clean" "${transition_file}" -done - -if mismatch_output=$(run_preflight "${SELF_SHA}" \ - '0000000000000000000000000000000000000000000000000000000000000000' \ - "${OVERRIDE_SHA}" \ - 2>&1); then - fail 'stale update-api.sh checksum should fail' -fi -assert_contains "${mismatch_output}" 'update-api.sh checksum mismatch' - -if override_output=$(run_preflight "${SELF_SHA}" "${UPDATE_SHA}" \ - '0000000000000000000000000000000000000000000000000000000000000000' \ - 2>&1); then - fail 'stale Compose observability override checksum should fail' -fi -assert_contains "${override_output}" 'docker-compose.observability.yml checksum mismatch' - -if mode_output=$(run_preflight "${SELF_SHA}" "${UPDATE_SHA}" "${OVERRIDE_SHA}" \ - FAKE_GHCR_MODE=644 2>&1); then - fail 'broad ghcr.env permissions should fail' -fi -assert_contains "${mode_output}" 'ghcr.env must be owned by root with mode 0600' - -if listener_output=$(run_preflight "${SELF_SHA}" "${UPDATE_SHA}" "${OVERRIDE_SHA}" \ - FAKE_MANAGEMENT_LISTENER=wildcard 2>&1); then - fail 'wildcard management listener should fail' -fi -assert_contains "${listener_output}" 'management port must be absent or bound only to 127.0.0.1' - -if compose_output=$(run_preflight "${SELF_SHA}" "${UPDATE_SHA}" "${OVERRIDE_SHA}" \ - FAKE_COMPOSE_VALID=0 2>&1); then - fail 'Compose without the observability boundary should fail' -fi -assert_contains "${compose_output}" 'Compose observability contract is incomplete' - -if legacy_output=$(run_preflight "${SELF_SHA}" "${UPDATE_SHA}" "${OVERRIDE_SHA}" \ - FAKE_LEGACY_GRAFANA_ENV=1 2>&1); then - fail 'legacy appender credentials in the app environment should fail' -fi -assert_contains "${legacy_output}" 'Compose observability contract is incomplete' -assert_not_contains "${legacy_output}" 'fixture-legacy-secret' - -echo 'host-preflight tests passed' diff --git a/deploy/observability/tests/recommendation-assets-test.sh b/deploy/observability/tests/recommendation-assets-test.sh new file mode 100755 index 0000000..1dc7898 --- /dev/null +++ b/deploy/observability/tests/recommendation-assets-test.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +observability_dir="$(cd -- "${script_dir}/.." && pwd)" +alloy_config="${observability_dir}/alloy/config.alloy" +dashboard="${observability_dir}/grafana/mapleland-production-overview.json" + +fail() { + printf '%s\n' "$1" >&2 + exit 1 +} + +jq empty "${dashboard}" + +grep -Fq 'mapleland_recommendation_requests_total' "${alloy_config}" \ + || fail 'recommendation request counter is missing from the Alloy application allowlist' +grep -Fq 'mapleland_recommendation_results_recommendations_(count|sum|max)' "${alloy_config}" \ + || fail 'recommendation result summary is missing from the Alloy application allowlist' + +for metadata_field in \ + event_duration \ + mapleland_recommendation_engine \ + mapleland_api_version \ + mapleland_result_count; do + field_occurrences="$(grep -c "${metadata_field}" "${alloy_config}")" + test "${field_occurrences}" -eq 2 \ + || fail "${metadata_field} must be extracted once and promoted to structured metadata once" +done + +label_keep_block="$(sed -n '/stage.label_keep {/,/^\t}/p' "${alloy_config}")" +for forbidden_label in \ + event_duration \ + mapleland_recommendation_engine \ + mapleland_api_version \ + mapleland_result_count; do + if grep -Fq "${forbidden_label}" <<<"${label_keep_block}"; then + fail "${forbidden_label} must not become an indexed Loki label" + fi +done + +jq -e ' + .uid == "mapleland-production-overview" + and ([.panels[].id] | index(18) < index(24)) + and ([.panels[] | select(.id == 18 and .type == "row" and .title == "Recommendations")] | length == 1) + and ([.panels[] | select(.id == 24 and .type == "row" and .title == "Application logs")] | length == 1) + and ([.panels[] | select(.id == 19 and .title == "Recommendation request rate")] | length == 1) + and ([.panels[] | select(.id == 20 and .title == "Recommendation p95 latency")] | length == 1) + and ([.panels[] | select(.id == 21 and .title == "Error / empty / unavailable rate")] | length == 1) + and ([.panels[] | select(.id == 22 and .title == "Engine state (observed)")] | length == 1) + and ([.panels[] | select(.id == 23 and .title == "Average result count")] | length == 1) +' "${dashboard}" >/dev/null || fail 'recommendation dashboard row or required panels are missing' + +jq -e ' + ([.panels[] | select(.id == 19) | .targets[].expr | contains("http_server_requests_seconds_count")] | all) + and ([.panels[] | select(.id == 19) | .targets[].expr | contains("/api/v[12]/maps/recommendations")] | all) + and ([.panels[] | select(.id == 20) | .targets[].expr | contains("http_server_requests_seconds_bucket")] | all) + and ([.panels[] | select(.id == 20) | .targets[].expr | contains("/api/v[12]/maps/recommendations")] | all) + and ([.panels[] | select(.id == 21) | .targets[].expr | contains("empty|unavailable")] | any) + and ([.panels[] | select(.id == 21) | .targets[].expr | contains("http_server_requests_seconds_count")] | any) + and ([.panels[] | select(.id == 21) | .targets[].expr | contains("4..|5..") ] | any) + and ([.panels[] | select(.id == 22) | .targets[].expr | contains("api_version, engine")] | all) + and ([.panels[] | select(.id == 23) | .targets[].expr | contains("mapleland_recommendation_results_recommendations_sum")] | all) + and ([.panels[] | select(.id == 23) | .targets[].expr | contains("mapleland_recommendation_results_recommendations_count")] | all) +' "${dashboard}" >/dev/null || fail 'recommendation dashboard queries do not match the metric contract' + +recommendation_queries="$(jq -r '.panels[] | select(.id >= 19 and .id <= 23) | .targets[].expr' "${dashboard}")" +if grep -Eiq '(job_?id|level|map_?id|member_?id|user_?id|raw_?uri|query_string)' \ + <<<"${recommendation_queries}"; then + fail 'recommendation dashboard query contains a prohibited high-cardinality or sensitive label' +fi + +printf '%s\n' 'Recommendation observability asset contract passed.' diff --git a/deploy/observability/tests/update-api-test.sh b/deploy/observability/tests/update-api-test.sh deleted file mode 100755 index d0d6e40..0000000 --- a/deploy/observability/tests/update-api-test.sh +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -TEST_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -readonly TEST_DIR -OBSERVABILITY_DIR=$(cd "${TEST_DIR}/.." && pwd) -readonly OBSERVABILITY_DIR -readonly UPDATE_SCRIPT=${OBSERVABILITY_DIR}/update-api.sh -readonly OVERRIDE_SOURCE=${OBSERVABILITY_DIR}/docker-compose.override.example.yml -readonly IMAGE_REF=ghcr.io/team-maple/mls-be/mapleland-api@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb - -fail() { - echo "update-api test failed: $*" >&2 - exit 1 -} - -assert_not_contains() { - local haystack=$1 - local needle=$2 - [[ ${haystack} != *"${needle}"* ]] \ - || fail "output unexpectedly contains: ${needle}" -} - -WORK_DIR=$(mktemp -d) -readonly WORK_DIR -trap 'rm -rf "${WORK_DIR}"' EXIT -readonly FAKE_BIN=${WORK_DIR}/bin -mkdir -p "${FAKE_BIN}" - -# The single-quoted bodies are written to executable mock scripts. -# shellcheck disable=SC2016 -printf '%s\n' \ - '#!/usr/bin/env bash' \ - 'set -euo pipefail' \ - 'case ${1:-} in' \ - ' -c) if [[ ${2:-} == %u ]]; then printf "%s\\n" 0; else case ${3##*/} in docker-compose.yml|docker-compose.observability.yml) printf "%s\\n" 640 ;; *) printf "%s\\n" 600 ;; esac; fi ;;' \ - ' -u) printf "%s\\n" 0 ;;' \ - ' *) exit 1 ;;' \ - 'esac' > "${FAKE_BIN}/stat" - -printf '%s\n' '#!/usr/bin/env bash' 'exit 0' > "${FAKE_BIN}/chown" -printf '%s\n' '#!/usr/bin/env bash' 'exit 0' > "${FAKE_BIN}/curl" - -# shellcheck disable=SC2016 -printf '%s\n' \ - '#!/usr/bin/env bash' \ - 'set -euo pipefail' \ - 'state_file=${FAKE_DOCKER_STATE:?}' \ - 'up_log=${FAKE_DOCKER_UP_LOG:?}' \ - 'case ${1:-} in' \ - ' login) cat >/dev/null; exit 0 ;;' \ - ' logout|pull) exit 0 ;;' \ - ' image)' \ - ' case ${2:-} in' \ - ' inspect)' \ - ' if [[ ${3:-} == --format || ${4:-} == --format ]]; then' \ - ' printf "%s\\n" sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' \ - ' fi' \ - ' exit 0 ;;' \ - ' tag) exit 0 ;;' \ - ' esac ;;' \ - ' compose)' \ - ' shift' \ - ' combined=false' \ - ' while [[ ${1:-} == -f || ${1:-} == --env-file ]]; do' \ - ' if [[ ${1:-} == -f && ${2:-} == *docker-compose.observability.yml ]]; then combined=true; fi' \ - ' shift 2' \ - ' done' \ - ' case ${1:-} in' \ - ' config)' \ - ' [[ ${2:-} == --images ]] && printf "%s\\n" ghcr.io/team-maple/mls-be/mapleland-api:latest-arm64' \ - ' exit 0 ;;' \ - ' ps)' \ - ' all=false' \ - ' shift' \ - ' for argument in "$@"; do [[ $argument == --all ]] && all=true; done' \ - ' if [[ $(cat "$state_file") == old ]]; then printf "%s\\n" old-container; elif [[ ${FAKE_NEW_STATE:-running} != exited || $all == true ]]; then printf "%s\\n" new-container; fi' \ - ' exit 0 ;;' \ - ' up)' \ - ' if [[ $combined == true ]]; then printf "%s\\n" combined >> "$up_log"; printf "%s\\n" new > "$state_file"; else printf "%s\\n" base >> "$up_log"; printf "%s\\n" old > "$state_file"; fi' \ - ' exit 0 ;;' \ - ' esac ;;' \ - ' inspect)' \ - ' target=${2:-}' \ - ' format=${4:-}' \ - ' case $format in' \ - ' *RestartCount*) if [[ $(cat "$state_file") == new ]]; then printf "%s\\n" "${FAKE_NEW_RESTART_COUNT:-0}"; else printf "%s\\n" 0; fi ;;' \ - ' *json*.State*) printf "%s\\n" "{\\"Status\\":\\"running\\",\\"ExitCode\\":0,\\"OOMKilled\\":false}" ;;' \ - ' *State.Status*) if [[ $(cat "$state_file") == new ]]; then printf "%s\\n" "${FAKE_NEW_STATE:-running}"; else printf "%s\\n" running; fi ;;' \ - ' *State.Health*) printf "%s\\n" healthy ;;' \ - ' *Config.Env*)' \ - ' if [[ $target == new-container && ${FAKE_NEW_HAS_LEGACY:-0} == 1 ]]; then printf "%s\\n" GRAFANA_CLOUD_PASSWORD=fixture-legacy-secret; else printf "%s\\n" SPRING_PROFILES_ACTIVE=prod; fi ;;' \ - ' *.Image*) [[ $target == old-container ]] && printf "%s\\n" sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa || printf "%s\\n" sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb ;;' \ - ' esac' \ - ' exit 0 ;;' \ - ' logs) printf "%s\\n" "fixture startup failure"; exit 0 ;;' \ - 'esac' \ - 'exit 1' > "${FAKE_BIN}/docker" - -chmod +x "${FAKE_BIN}"/* - -setup_fixture() { - local root=$1 - mkdir -p "${root}/opt/mapleland" "${root}/run" - cp "${OVERRIDE_SOURCE}" "${root}/opt/mapleland/docker-compose.observability.yml" - # Literal Compose interpolation expressions are intentional fixture data. - # shellcheck disable=SC2016 - printf '%s\n' \ - 'services:' \ - ' mapleland-api:' \ - ' image: ghcr.io/team-maple/mls-be/mapleland-api:latest-arm64' \ - ' environment:' \ - ' - GRAFANA_CLOUD_URL=${GRAFANA_CLOUD_URL}' \ - ' - GRAFANA_CLOUD_USERNAME=${GRAFANA_CLOUD_USERNAME}' \ - ' - GRAFANA_CLOUD_PASSWORD=${GRAFANA_CLOUD_PASSWORD}' \ - ' - SPRING_PROFILES_ACTIVE=prod' \ - > "${root}/opt/mapleland/docker-compose.yml" - printf '%s\n' \ - 'GRAFANA_CLOUD_URL=https://example.invalid' \ - 'GRAFANA_CLOUD_USERNAME=fixture-user' \ - 'GRAFANA_CLOUD_PASSWORD=fixture-legacy-secret' \ - 'MANAGEMENT_SCRAPE_TOKEN=fixture-management-secret-0123456789' \ - 'SERVICE_VERSION=f4bf228b934959be125a72540c91e43f003b7b6e' \ - > "${root}/opt/mapleland/.env" - printf '%s\n' \ - 'GHCR_USER=fixture-user' \ - 'GHCR_TOKEN=fixture-ghcr-secret' \ - > "${root}/opt/mapleland/ghcr.env" - printf '%s\n' old > "${root}/state" - : > "${root}/up.log" -} - -run_update() { - local root=$1 - shift - env \ - PATH="${FAKE_BIN}:${PATH}" \ - MAPLELAND_UPDATE_TEST_MODE=1 \ - MAPLELAND_UPDATE_TEST_ROOT="${root}" \ - FAKE_DOCKER_STATE="${root}/state" \ - FAKE_DOCKER_UP_LOG="${root}/up.log" \ - "$@" \ - "${BASH}" "${UPDATE_SCRIPT}" "${IMAGE_REF}" -} - -success_root=${WORK_DIR}/success -setup_fixture "${success_root}" -success_output=$(run_update "${success_root}" 2>&1) \ - || fail "healthy immutable deployment should succeed: ${success_output}" -[[ $(cat "${success_root}/state") == new ]] || fail 'new container was not selected' -[[ $(cat "${success_root}/up.log") == combined ]] \ - || fail 'deployment did not use the reviewed override' -if grep -Eq 'GRAFANA_CLOUD_(URL|USERNAME|PASSWORD)' \ - "${success_root}/opt/mapleland/docker-compose.yml" \ - "${success_root}/opt/mapleland/.env"; then - fail 'legacy appender environment was not removed after readiness' -fi -assert_not_contains "${success_output}" 'fixture-ghcr-secret' -assert_not_contains "${success_output}" 'fixture-legacy-secret' -assert_not_contains "${success_output}" 'fixture-management-secret' - -rollback_root=${WORK_DIR}/rollback -setup_fixture "${rollback_root}" -if rollback_output=$(run_update "${rollback_root}" FAKE_NEW_HAS_LEGACY=1 2>&1); then - fail 'candidate containing a legacy Grafana credential should fail' -fi -[[ $(cat "${rollback_root}/state") == old ]] || fail 'previous container was not restored' -[[ $(paste -sd, "${rollback_root}/up.log") == combined,base ]] \ - || fail 'first-rollout rollback did not use the preserved base contract' -grep -Eq 'GRAFANA_CLOUD_PASSWORD' "${rollback_root}/opt/mapleland/.env" \ - || fail 'failed deployment unexpectedly removed rollback credentials' -assert_not_contains "${rollback_output}" 'fixture-ghcr-secret' -assert_not_contains "${rollback_output}" 'fixture-legacy-secret' -assert_not_contains "${rollback_output}" 'fixture-management-secret' -[[ -f ${rollback_root}/var/log/mapleland-deploy/last-failure/state.json ]] \ - || fail 'failed candidate state was not preserved before rollback' -[[ $(cat "${rollback_root}/var/log/mapleland-deploy/last-failure/container.log") == \ - 'fixture startup failure' ]] \ - || fail 'failed candidate logs were not preserved before rollback' - -restart_root=${WORK_DIR}/restart -setup_fixture "${restart_root}" -if restart_output=$(run_update "${restart_root}" FAKE_NEW_RESTART_COUNT=1 2>&1); then - fail 'a candidate restart should fail the deployment' -fi -[[ $(cat "${restart_root}/state") == old ]] \ - || fail 'restarted candidate did not roll back to the previous image' -[[ ${restart_output} == *'candidate entered a restart loop (restart_count=1)'* ]] \ - || fail 'candidate restart was not diagnosed explicitly' -[[ -f ${restart_root}/var/log/mapleland-deploy/last-failure/container.log ]] \ - || fail 'restart-loop diagnostics were not preserved before rollback' -assert_not_contains "${restart_output}" 'fixture-management-secret' - -exited_root=${WORK_DIR}/exited -setup_fixture "${exited_root}" -if exited_output=$(run_update "${exited_root}" FAKE_NEW_STATE=exited 2>&1); then - fail 'an exited candidate should fail the deployment' -fi -[[ $(cat "${exited_root}/state") == old ]] \ - || fail 'exited candidate did not roll back to the previous image' -[[ ${exited_output} == *'candidate stopped before readiness (state=exited)'* ]] \ - || fail 'exited candidate was not diagnosed explicitly' -[[ -f ${exited_root}/var/log/mapleland-deploy/last-failure/container.log ]] \ - || fail 'exited candidate diagnostics were not preserved before rollback' -assert_not_contains "${exited_output}" 'fixture-management-secret' - -gateway_sudo=${WORK_DIR}/gateway-sudo -gateway_log=${WORK_DIR}/gateway.log -# The single-quoted body is written to the executable mock. -# shellcheck disable=SC2016 -printf '%s\n' \ - '#!/usr/bin/env bash' \ - 'set -euo pipefail' \ - 'printf "%s\\n" "$*" >> "${FAKE_GATEWAY_LOG:?}"' \ - > "${gateway_sudo}" -chmod +x "${gateway_sudo}" - -preflight_sha=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa -update_sha=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb -override_sha=cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc -: > "${gateway_log}" -env SSH_ORIGINAL_COMMAND="preflight ${preflight_sha} ${update_sha} ${override_sha}" \ - MAPLELAND_UPDATE_GATEWAY_TEST_MODE=1 \ - MAPLELAND_UPDATE_GATEWAY_SUDO_BIN="${gateway_sudo}" \ - FAKE_GATEWAY_LOG="${gateway_log}" \ - "${BASH}" "${UPDATE_SCRIPT}" -[[ $(cat "${gateway_log}") == \ - "-n /opt/mapleland/preflight-host.sh ${preflight_sha} ${update_sha} ${override_sha}" ]] \ - || fail 'gateway did not dispatch the exact preflight allowlist' - -: > "${gateway_log}" -env SSH_ORIGINAL_COMMAND="deploy ${preflight_sha} ${update_sha} ${override_sha} ${IMAGE_REF}" \ - MAPLELAND_UPDATE_GATEWAY_TEST_MODE=1 \ - MAPLELAND_UPDATE_GATEWAY_SUDO_BIN="${gateway_sudo}" \ - FAKE_GATEWAY_LOG="${gateway_log}" \ - "${BASH}" "${UPDATE_SCRIPT}" -expected_gateway_log=$(printf '%s\n%s' \ - "-n /opt/mapleland/preflight-host.sh ${preflight_sha} ${update_sha} ${override_sha}" \ - "-n /opt/mapleland/update-api.sh ${IMAGE_REF}") -[[ $(cat "${gateway_log}") == "${expected_gateway_log}" ]] \ - || fail 'gateway did not keep preflight and immutable deploy in the allowlist' - -: > "${gateway_log}" -if env SSH_ORIGINAL_COMMAND="preflight ${preflight_sha} ${update_sha} ${override_sha}"$'\nuname -a' \ - MAPLELAND_UPDATE_GATEWAY_TEST_MODE=1 \ - MAPLELAND_UPDATE_GATEWAY_SUDO_BIN="${gateway_sudo}" \ - FAKE_GATEWAY_LOG="${gateway_log}" \ - "${BASH}" "${UPDATE_SCRIPT}" >/dev/null 2>&1; then - fail 'gateway should reject multiline input' -fi -[[ ! -s ${gateway_log} ]] || fail 'rejected gateway input reached sudo' - -: > "${gateway_log}" -if printf 'preflight %s %s %s\n' \ - "${preflight_sha}" "${update_sha}" "${override_sha}" | - env SSH_ORIGINAL_COMMAND='uname -a' \ - MAPLELAND_UPDATE_GATEWAY_TEST_MODE=1 \ - MAPLELAND_UPDATE_GATEWAY_SUDO_BIN="${gateway_sudo}" \ - FAKE_GATEWAY_LOG="${gateway_log}" \ - "${BASH}" "${UPDATE_SCRIPT}" >/dev/null 2>&1; then - fail 'gateway should not accept an allowlisted command from stdin' -fi -[[ ! -s ${gateway_log} ]] || fail 'stdin bypass reached sudo' - -echo 'update-api tests passed' diff --git a/deploy/observability/update-api.sh b/deploy/observability/update-api.sh deleted file mode 100755 index 00b3d18..0000000 --- a/deploy/observability/update-api.sh +++ /dev/null @@ -1,503 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -readonly GATEWAY_SHA256_REGEX='[0-9a-f]{64}' -readonly GATEWAY_IMAGE_REGEX='ghcr[.]io/team-maple/mls-be/mapleland-api@sha256:[0-9a-f]{64}' - -dispatch_forced_command() { - local sudo_bin=$1 - local gateway_command - local gateway_input - local preflight_sha - local update_sha - local override_sha - local image_ref_from_gateway - - gateway_input=${SSH_ORIGINAL_COMMAND:-} - if [[ ${#gateway_input} -gt 512 ]]; then - echo 'CI deployment gateway command is too long' >&2 - exit 1 - fi - gateway_command=${gateway_input} - if [[ -z ${gateway_command} \ - || ${gateway_command} == *$'\n'* || ${gateway_command} == *$'\r'* ]]; then - echo 'invalid CI deployment gateway command' >&2 - exit 1 - fi - - if [[ ${gateway_command} =~ ^preflight\ (${GATEWAY_SHA256_REGEX})\ (${GATEWAY_SHA256_REGEX})\ (${GATEWAY_SHA256_REGEX})$ ]]; then - preflight_sha=${BASH_REMATCH[1]} - update_sha=${BASH_REMATCH[2]} - override_sha=${BASH_REMATCH[3]} - exec "${sudo_bin}" -n /opt/mapleland/preflight-host.sh \ - "${preflight_sha}" "${update_sha}" "${override_sha}" - fi - - if [[ ${gateway_command} =~ ^deploy\ (${GATEWAY_SHA256_REGEX})\ (${GATEWAY_SHA256_REGEX})\ (${GATEWAY_SHA256_REGEX})\ (${GATEWAY_IMAGE_REGEX})$ ]]; then - preflight_sha=${BASH_REMATCH[1]} - update_sha=${BASH_REMATCH[2]} - override_sha=${BASH_REMATCH[3]} - image_ref_from_gateway=${BASH_REMATCH[4]} - "${sudo_bin}" -n /opt/mapleland/preflight-host.sh \ - "${preflight_sha}" "${update_sha}" "${override_sha}" - exec "${sudo_bin}" -n /opt/mapleland/update-api.sh "${image_ref_from_gateway}" - fi - - echo 'CI deployment gateway command is not allowlisted' >&2 - exit 1 -} - -if [[ ${MAPLELAND_UPDATE_GATEWAY_TEST_MODE:-0} == 1 ]]; then - if (( EUID == 0 )); then - echo 'gateway test mode is forbidden for root' >&2 - exit 1 - fi - readonly gateway_sudo_bin=${MAPLELAND_UPDATE_GATEWAY_SUDO_BIN:?gateway test sudo path is required} - [[ ${gateway_sudo_bin} == /* && -x ${gateway_sudo_bin} ]] - dispatch_forced_command "${gateway_sudo_bin}" -elif [[ ${MAPLELAND_UPDATE_TEST_MODE:-0} == 1 ]]; then - if (( EUID == 0 )); then - echo 'test mode is forbidden for root' >&2 - exit 1 - fi - readonly ROOT_PREFIX=${MAPLELAND_UPDATE_TEST_ROOT:?test root is required} - if [[ ${ROOT_PREFIX} != /* || ${ROOT_PREFIX} == / ]]; then - echo 'test root must be a non-root absolute path' >&2 - exit 1 - fi - readonly READINESS_TIMEOUT_SECONDS=2 -elif (( EUID != 0 )); then - dispatch_forced_command /usr/bin/sudo -else - readonly ROOT_PREFIX='' - readonly READINESS_TIMEOUT_SECONDS=90 -fi - -host_path() { - printf '%s%s' "${ROOT_PREFIX}" "$1" -} - -COMPOSE_FILE=$(host_path /opt/mapleland/docker-compose.yml) -readonly COMPOSE_FILE -COMPOSE_OVERRIDE_FILE=$(host_path /opt/mapleland/docker-compose.observability.yml) -readonly COMPOSE_OVERRIDE_FILE -APP_ENV_FILE=$(host_path /opt/mapleland/.env) -readonly APP_ENV_FILE -GHCR_ENV_FILE=$(host_path /opt/mapleland/ghcr.env) -readonly GHCR_ENV_FILE -readonly EXPECTED_REPOSITORY=ghcr.io/team-maple/mls-be/mapleland-api -readonly EXPECTED_REPOSITORY_REGEX='ghcr[.]io/team-maple/mls-be/mapleland-api' -readonly LEGACY_COMPOSE_ENV_REGEX='^[[:space:]]*-[[:space:]]*GRAFANA_CLOUD_(URL|USERNAME|PASSWORD)=' -readonly LEGACY_APP_ENV_REGEX='^GRAFANA_CLOUD_(URL|USERNAME|PASSWORD)=' -readonly MANAGEMENT_PROMETHEUS_URL=http://127.0.0.1:18080/actuator/prometheus -readonly PUBLIC_SMOKE_URL=http://127.0.0.1:8080/api/v1/jobs -DEPLOY_DIAGNOSTIC_ROOT=$(host_path /var/log/mapleland-deploy) -readonly DEPLOY_DIAGNOSTIC_ROOT - -if [[ $# -ne 1 ]]; then - echo "usage: update-api.sh " >&2 - exit 1 -fi - -readonly image_ref=$1 -if [[ ! ${image_ref} =~ ^${EXPECTED_REPOSITORY_REGEX}@sha256:[0-9a-f]{64}$ ]]; then - echo "refusing mutable or unexpected image reference" >&2 - exit 1 -fi - -for compose_contract_file in "${COMPOSE_FILE}" "${COMPOSE_OVERRIDE_FILE}"; do - if [[ ! -f ${compose_contract_file} || -L ${compose_contract_file} ]]; then - echo "${compose_contract_file} must be a regular, non-symlink file" >&2 - exit 1 - fi - if [[ $(stat -c '%u' "${compose_contract_file}") != 0 ]] \ - || [[ $(stat -c '%a' "${compose_contract_file}") != 640 ]]; then - echo "${compose_contract_file} must be owned by root with mode 0640" >&2 - exit 1 - fi -done - -if [[ ! -f ${APP_ENV_FILE} || -L ${APP_ENV_FILE} ]]; then - echo "${APP_ENV_FILE} must be a regular, non-symlink file" >&2 - exit 1 -fi - -if [[ $(stat -c '%u' "${APP_ENV_FILE}") != 0 ]] \ - || [[ $(stat -c '%a' "${APP_ENV_FILE}") != 600 ]]; then - echo "${APP_ENV_FILE} must be owned by root with mode 0600" >&2 - exit 1 -fi - -management_scrape_token='' -management_token_count=0 -while IFS= read -r app_env_line || [[ -n ${app_env_line} ]]; do - if [[ ${app_env_line} == MANAGEMENT_SCRAPE_TOKEN=* ]]; then - management_scrape_token=${app_env_line#*=} - management_token_count=$((management_token_count + 1)) - fi -done < "${APP_ENV_FILE}" -if (( management_token_count != 1 )) \ - || [[ -z ${management_scrape_token} || ${#management_scrape_token} -lt 32 \ - || ! ${management_scrape_token} =~ ^[[:graph:]]+$ ]]; then - echo 'exactly one valid MANAGEMENT_SCRAPE_TOKEN is required' >&2 - exit 1 -fi -readonly management_scrape_token - -curl_management_prometheus() { - local escaped_token=${management_scrape_token//\\/\\\\} - escaped_token=${escaped_token//\"/\\\"} - printf 'header = "Authorization: Bearer %s"\n' "${escaped_token}" | - curl --config - "$@" "${MANAGEMENT_PROMETHEUS_URL}" -} - -base_legacy_count=0 -env_legacy_count=0 -for legacy_key in URL USERNAME PASSWORD; do - base_key_count=$(grep -Ec \ - "^[[:space:]]*-[[:space:]]*GRAFANA_CLOUD_${legacy_key}=" \ - "${COMPOSE_FILE}" || true) - env_key_count=$(grep -Ec "^GRAFANA_CLOUD_${legacy_key}=" \ - "${APP_ENV_FILE}" || true) - if (( base_key_count > 1 || env_key_count > 1 )); then - echo "duplicate GRAFANA_CLOUD_${legacy_key} rollback entry" >&2 - exit 1 - fi - base_legacy_count=$((base_legacy_count + base_key_count)) - env_legacy_count=$((env_legacy_count + env_key_count)) -done -case "${base_legacy_count}:${env_legacy_count}" in - 0:0) rollback_uses_legacy_contract=false ;; - 3:3) rollback_uses_legacy_contract=true ;; - *) - echo 'legacy Grafana rollback environment must be either fully preserved or fully removed' >&2 - exit 1 - ;; -esac -readonly rollback_uses_legacy_contract - -if [[ ! -f ${GHCR_ENV_FILE} || -L ${GHCR_ENV_FILE} ]]; then - echo "${GHCR_ENV_FILE} must be a regular, non-symlink file" >&2 - exit 1 -fi -if [[ $(stat -c '%u' "${GHCR_ENV_FILE}") != 0 ]] \ - || [[ $(stat -c '%a' "${GHCR_ENV_FILE}") != 600 ]]; then - echo "${GHCR_ENV_FILE} must be owned by root with mode 0600" >&2 - exit 1 -fi - -ghcr_user='' -ghcr_token='' -ghcr_user_seen=false -ghcr_token_seen=false -while IFS= read -r credential_line || [[ -n ${credential_line} ]]; do - case ${credential_line} in - ''|'#'*) - continue - ;; - esac - - if [[ ${credential_line} != *=* ]]; then - echo "invalid credential line in ${GHCR_ENV_FILE}" >&2 - exit 1 - fi - credential_key=${credential_line%%=*} - credential_value=${credential_line#*=} - if [[ -z ${credential_value} || ! ${credential_value} =~ ^[[:graph:]]+$ ]]; then - echo "credential values must be non-empty printable tokens" >&2 - exit 1 - fi - - case ${credential_key} in - GHCR_USER) - if [[ ${ghcr_user_seen} == true ]]; then - echo "duplicate GHCR_USER in ${GHCR_ENV_FILE}" >&2 - exit 1 - fi - ghcr_user=${credential_value} - ghcr_user_seen=true - ;; - GHCR_TOKEN) - if [[ ${ghcr_token_seen} == true ]]; then - echo "duplicate GHCR_TOKEN in ${GHCR_ENV_FILE}" >&2 - exit 1 - fi - ghcr_token=${credential_value} - ghcr_token_seen=true - ;; - *) - echo "unexpected credential key in ${GHCR_ENV_FILE}: ${credential_key}" >&2 - exit 1 - ;; - esac -done < "${GHCR_ENV_FILE}" - -if [[ ${ghcr_user_seen} != true || ${ghcr_token_seen} != true ]]; then - echo "GHCR_USER and GHCR_TOKEN are both required in ${GHCR_ENV_FILE}" >&2 - exit 1 -fi -credential_line='' -credential_value='' - -compose_image='' -rollback_tag='' -rollback_image_id='' -rollback_armed=false -compose_env_file='' -compose_candidate_file='' -COMPOSE_ARGS=() - -capture_failure_diagnostics() ( - local container_id=$1 - local diagnostic_dir=${DEPLOY_DIAGNOSTIC_ROOT}/last-failure - local diagnostic_tmp - - umask 077 - trap 'if [[ -n ${diagnostic_tmp:-} \ - && ${diagnostic_tmp} == "${DEPLOY_DIAGNOSTIC_ROOT}"/.last-failure.* \ - && -d ${diagnostic_tmp} && ! -L ${diagnostic_tmp} ]]; then \ - rm -rf -- "${diagnostic_tmp}"; \ - fi' EXIT - - if [[ -L ${DEPLOY_DIAGNOSTIC_ROOT} \ - || ( -e ${DEPLOY_DIAGNOSTIC_ROOT} && ! -d ${DEPLOY_DIAGNOSTIC_ROOT} ) ]]; then - echo 'deployment diagnostic root is not a safe directory' >&2 - return 1 - fi - mkdir -p "${DEPLOY_DIAGNOSTIC_ROOT}" \ - || { echo 'could not create deployment diagnostic root' >&2; return 1; } - chown root:root "${DEPLOY_DIAGNOSTIC_ROOT}" \ - || { echo 'could not own deployment diagnostic root' >&2; return 1; } - chmod 0700 "${DEPLOY_DIAGNOSTIC_ROOT}" \ - || { echo 'could not protect deployment diagnostic root' >&2; return 1; } - - if [[ -L ${diagnostic_dir} \ - || ( -e ${diagnostic_dir} && ! -d ${diagnostic_dir} ) ]]; then - echo 'deployment diagnostic target is not a safe directory' >&2 - return 1 - fi - - diagnostic_tmp=$(mktemp -d "${DEPLOY_DIAGNOSTIC_ROOT}/.last-failure.XXXXXX") \ - || { echo 'could not create temporary deployment diagnostics' >&2; return 1; } - chown root:root "${diagnostic_tmp}" \ - || { echo 'could not own temporary deployment diagnostics' >&2; return 1; } - chmod 0700 "${diagnostic_tmp}" \ - || { echo 'could not protect temporary deployment diagnostics' >&2; return 1; } - - if ! docker inspect "${container_id}" --format '{{json .State}}' \ - > "${diagnostic_tmp}/state.json"; then - printf '%s\n' '{"diagnostic":"container state unavailable"}' \ - > "${diagnostic_tmp}/state.json" \ - || { echo 'could not write fallback container state' >&2; return 1; } - fi - docker logs --timestamps --tail 500 "${container_id}" \ - > "${diagnostic_tmp}/container.log" 2>&1 || true - printf 'captured_at=%s\ncontainer_id=%s\n' \ - "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "${container_id}" \ - > "${diagnostic_tmp}/metadata" \ - || { echo 'could not write deployment diagnostic metadata' >&2; return 1; } - chmod 0600 \ - "${diagnostic_tmp}/state.json" \ - "${diagnostic_tmp}/container.log" \ - "${diagnostic_tmp}/metadata" \ - || { echo 'could not protect deployment diagnostic files' >&2; return 1; } - chown root:root \ - "${diagnostic_tmp}/state.json" \ - "${diagnostic_tmp}/container.log" \ - "${diagnostic_tmp}/metadata" \ - || { echo 'could not own deployment diagnostic files' >&2; return 1; } - - rm -rf -- "${diagnostic_dir}" \ - || { echo 'could not replace prior deployment diagnostics' >&2; return 1; } - mv "${diagnostic_tmp}" "${diagnostic_dir}" \ - || { echo 'could not publish deployment diagnostics' >&2; return 1; } - diagnostic_tmp='' - echo 'candidate diagnostics saved to /var/log/mapleland-deploy/last-failure' >&2 -) - -wait_for_readiness() { - local expected_image_id=$1 - local require_management_health=${2:-true} - local deadline=$((SECONDS + READINESS_TIMEOUT_SECONDS)) - local candidate_container_id='' - local observed_container_id - local candidate_image_id - local candidate_state - local candidate_health - local candidate_restart_count - local management_status - local public_status - - while (( SECONDS < deadline )); do - observed_container_id=$(docker compose "${COMPOSE_ARGS[@]}" \ - ps --all -q mapleland-api) - if [[ -n ${observed_container_id} ]]; then - candidate_container_id=${observed_container_id} - fi - if [[ -n ${candidate_container_id} ]]; then - candidate_restart_count=$(docker inspect "${candidate_container_id}" \ - --format '{{.RestartCount}}') - if [[ ${candidate_restart_count} =~ ^[0-9]+$ ]] \ - && (( candidate_restart_count > 0 )); then - capture_failure_diagnostics "${candidate_container_id}" || true - echo "candidate entered a restart loop (restart_count=${candidate_restart_count})" >&2 - return 1 - fi - candidate_image_id=$(docker inspect "${candidate_container_id}" --format '{{.Image}}') - candidate_state=$(docker inspect "${candidate_container_id}" --format '{{.State.Status}}') - if [[ ${candidate_state} == exited || ${candidate_state} == dead ]]; then - capture_failure_diagnostics "${candidate_container_id}" || true - echo "candidate stopped before readiness (state=${candidate_state})" >&2 - return 1 - fi - candidate_health=$(docker inspect "${candidate_container_id}" \ - --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}') - - if [[ ${candidate_image_id} == "${expected_image_id}" \ - && ${candidate_state} == running \ - && ( ${candidate_health} == none || ${candidate_health} == healthy ) ]]; then - if [[ ${require_management_health} == true ]] \ - && docker inspect "${candidate_container_id}" \ - --format '{{range .Config.Env}}{{println .}}{{end}}' | - grep -Eq '^GRAFANA_CLOUD_(URL|USERNAME|PASSWORD)='; then - sleep 2 - continue - fi - if [[ ${require_management_health} == false ]] \ - || curl_management_prometheus \ - --fail --silent --max-time 5 >/dev/null; then - if curl --fail --silent --max-time 5 "${PUBLIC_SMOKE_URL}" >/dev/null; then - return 0 - fi - fi - fi - fi - sleep 2 - done - - if [[ -n ${candidate_container_id} ]]; then - capture_failure_diagnostics "${candidate_container_id}" || true - fi - management_status=$(curl_management_prometheus \ - --silent --output /dev/null --max-time 5 --write-out '%{http_code}' || true) - public_status=$(curl --silent --output /dev/null --max-time 5 \ - --write-out '%{http_code}' "${PUBLIC_SMOKE_URL}" || true) - echo "mapleland-api did not become ready within ${READINESS_TIMEOUT_SECONDS}s" >&2 - printf 'readiness diagnostics: management_prometheus_status=%s public_smoke_status=%s\n' \ - "${management_status:-000}" "${public_status:-000}" >&2 - return 1 -} - -rollback_deployment() { - echo "deployment failed; rolling back to ${rollback_tag}" >&2 - docker image tag "${rollback_tag}" "${compose_image}" || return 1 - if [[ ${rollback_uses_legacy_contract} == true ]]; then - docker compose -f "${COMPOSE_FILE}" up \ - -d --no-deps --force-recreate --pull never mapleland-api || return 1 - else - docker compose "${COMPOSE_ARGS[@]}" up \ - -d --no-deps --force-recreate --pull never mapleland-api || return 1 - fi - wait_for_readiness "${rollback_image_id}" false || return 1 - echo "mapleland-api rollback completed with exact previous image" >&2 -} - -cleanup() { - local status=$? - trap - EXIT INT TERM - if (( status != 0 )) && [[ ${rollback_armed} == true ]]; then - if ! rollback_deployment; then - echo "automatic rollback failed; operator intervention is required" >&2 - status=1 - fi - fi - if [[ -n ${compose_env_file} ]]; then - rm -f "${compose_env_file}" - fi - if [[ -n ${compose_candidate_file} ]]; then - rm -f "${compose_candidate_file}" - fi - docker logout ghcr.io >/dev/null 2>&1 || true - exit "${status}" -} -trap cleanup EXIT -trap 'exit 130' INT -trap 'exit 143' TERM - -compose_env_file=$(mktemp "$(host_path /run)/mapleland-compose-env.XXXXXX") -chmod 0600 "${compose_env_file}" -awk -v regex="${LEGACY_APP_ENV_REGEX}" '$0 !~ regex' \ - "${APP_ENV_FILE}" > "${compose_env_file}" -compose_candidate_file=$(mktemp "$(host_path /opt/mapleland)/.docker-compose.deploy.XXXXXX") -chmod 0640 "${compose_candidate_file}" -awk -v regex="${LEGACY_COMPOSE_ENV_REGEX}" '$0 !~ regex' \ - "${COMPOSE_FILE}" > "${compose_candidate_file}" -chown root:root "${compose_env_file}" "${compose_candidate_file}" -COMPOSE_ARGS=( - --env-file "${compose_env_file}" - -f "${compose_candidate_file}" - -f "${COMPOSE_OVERRIDE_FILE}" -) -readonly -a COMPOSE_ARGS - -printf '%s' "${ghcr_token}" | docker login ghcr.io -u "${ghcr_user}" --password-stdin -ghcr_token='' -docker pull "${image_ref}" -expected_image_id=$(docker image inspect "${image_ref}" --format '{{.Id}}') - -mapfile -t configured_images < <(docker compose "${COMPOSE_ARGS[@]}" config --images) -for configured_image in "${configured_images[@]}"; do - if [[ ${configured_image} == "${EXPECTED_REPOSITORY}:"* ]]; then - if [[ -n ${compose_image} ]]; then - echo "multiple mapleland-api image references found in Compose configuration" >&2 - exit 1 - fi - compose_image=${configured_image} - fi -done -: "${compose_image:?mapleland-api image is missing from Compose configuration}" - -previous_container_id=$(docker compose "${COMPOSE_ARGS[@]}" ps -q mapleland-api) -if [[ -z ${previous_container_id} ]]; then - echo "a running mapleland-api container is required for safe rollback" >&2 - exit 1 -fi -rollback_image_id=$(docker inspect "${previous_container_id}" --format '{{.Image}}') -docker image inspect "${rollback_image_id}" >/dev/null -rollback_stamp=$(date -u +%Y%m%dT%H%M%S%N) -rollback_image_short=${rollback_image_id#sha256:} -rollback_tag="${EXPECTED_REPOSITORY}:rollback-${rollback_stamp}-${rollback_image_short:0:12}-$$" -docker image tag "${rollback_image_id}" "${rollback_tag}" -rollback_armed=true -echo "previous image preserved as ${rollback_tag}" - -# Keep the current Compose contract while making its local tag point at the exact -# digest-pinned image. --pull never prevents a concurrent registry resolution. -docker image tag "${image_ref}" "${compose_image}" -docker compose "${COMPOSE_ARGS[@]}" up \ - -d --no-deps --force-recreate --pull never mapleland-api - -wait_for_readiness "${expected_image_id}" true - -remove_legacy_appender_environment() { - if [[ ${base_legacy_count} == 0 ]]; then - rollback_armed=false - return 0 - fi - - # The new image is healthy and has no legacy credentials. Do not turn a - # subsequent local config-cleanup error into an unnecessary app rollback. - rollback_armed=false - if ! mv "${compose_env_file}" "${APP_ENV_FILE}"; then - return 1 - fi - compose_env_file='' - if ! mv "${compose_candidate_file}" "${COMPOSE_FILE}"; then - return 1 - fi - compose_candidate_file='' -} - -remove_legacy_appender_environment - -echo "mapleland-api is running the requested immutable image" diff --git a/docs/observability-rollout-checkpoint.md b/docs/observability-rollout-checkpoint.md index 9d350b4..d17a488 100644 --- a/docs/observability-rollout-checkpoint.md +++ b/docs/observability-rollout-checkpoint.md @@ -2,6 +2,54 @@ 이 문서는 Codex task, OAuth 또는 로컬 앱이 재시작돼도 운영 상태를 추측하거나 이미 끝난 작업을 반복하지 않기 위한 비밀값 없는 재개 기준이다. 외부 상태를 변경한 뒤에는 해당 operation의 URL·commit·checksum·검증 결과와 다음 안전 작업을 갱신한다. token, credential, 원본 환경 파일 내용은 기록하지 않는다. +## 2026-07-17 legacy OCI 배포 경로 복원과 운영 복구 + +- Owner가 성공 이력이 있는 [deploy run 28788801815](https://github.com/Team-Maple/MLS-BE/actions/runs/28788801815), commit `c49ee3255afb4ddfa0168ce91783fff368864a4d`의 legacy 배포 구조 복원을 명시적으로 승인했다. 동시 `latest-arm64` overwrite를 막는 `deploy-oci` concurrency와 실제로 쓰지 않는 `id-token: write` 제거만 적용했다. +- Workflow는 arm64 image를 `latest-arm64`로 build/publish한 뒤 Tailscale과 `appleboy/ssh-action@v1.2.5`로 접속해 host의 `/opt/mapleland/update-api.sh`를 인자 없이 실행한다. +- `host-preflight`, repository checksum, immutable digest 전달, image metadata/FCM permission gate, 별도 Environment 승인과 native OpenSSH fingerprint 검증은 사용하지 않는다. +- 새 immutable runner 구현과 전용 CI test는 저장소에서 제거했다. 초기 read-only 확인에서 active `/opt/mapleland/update-api.sh`가 legacy no-arg script이고 SHA-256이 `76650cef0cac9edf426bbc67203f4a967bb10a7b353128c6ac53aba65cc63b77`임을 확인했다. +- CI key의 forced command를 `command="/opt/mapleland/update-api.sh"`에서 `command="/usr/bin/sudo -n /opt/mapleland/update-api.sh"`로 원자 교체했다. Backup은 `/home/ubuntu/.ssh/authorized_keys.before-ci-sudo-20260716T235214Z`이며 active와 backup 모두 `ubuntu:ubuntu 0600`이다. Script는 `root:root 0755`, `.env`는 `root:root 0600`을 유지한다. +- Host에는 기존 `ubuntu ALL=(ALL) NOPASSWD: ALL`이 있어 sudoers는 변경하지 않았다. 초기 gateway 변경 뒤 `sudo -n -l`, root의 `.env` read, script `bash -n`, authorized_keys key parsing과 forced-command exact-count를 배포 실행 없이 검증했다. +- Legacy workflow는 host script를 root로 실행했지만 base Compose만 사용해 새 image에 `MANAGEMENT_SCRAPE_TOKEN`을 전달하지 못했고 container가 crash loop에 들어갔다. 이전 image `sha256:1b9b13b75debfe76ad755f618738bd63972a270b78d03f90d50133ee277fa3af` rollback도 당시 container에만 있던 Loki 설정이 재생성 과정에서 사라져 `URI with undefined scheme`으로 실패했다. +- 실패 image `sha256:3ee144df80b5110e4d72b723f570a53908f14cf9c036ae2fecb11cbbdba9569f`는 `failed-20260717T001115Z-3ee144df80b5` 태그로 보존했다. Owner 승인 복구에서 같은 image를 base와 `/opt/mapleland/docker-compose.observability.yml` 조합으로 재생성했고 exact image, `running`, restart count `0`, 공개 `/api/v1/jobs` 200과 Bearer 인증 `/actuator/prometheus` 200을 확인했다. +- Owner 승인 재발 방지 maintenance로 `/opt/mapleland/update-api.sh`의 pull/up 두 명령이 base와 observability override를 함께 사용하도록 변경했다. Root-only backup은 `/root/update-api.sh.before-observability-override-20260717T001948Z`, old SHA-256은 `76650cef0cac9edf426bbc67203f4a967bb10a7b353128c6ac53aba65cc63b77`, active SHA-256은 `436dae8156ec7115f823589aeb46b586b15e3259443bf9b1786d61fcec331b4d`다. Active는 `root:root 0755`이고 `bash -n`과 Compose render를 통과했으며 변경 후 script 자체는 실행하지 않았다. +- Host `.env`에는 `RECOMMENDATION_V1_ENGINE=AURA`, `RECOMMENDATION_V2_ENABLED=true`, `RECOMMENDATION_QUERY_TIMEOUT_SECONDS=10`이 있었지만 active override가 세 변수를 container에 전달하지 않아 v2가 code default `false`로 계속 503을 반환했다. Owner 승인 maintenance로 세 변수의 명시적 pass-through를 추가했다. +- 변경 전 override의 root-only backup은 `/root/docker-compose.observability.yml.before-recommendation-env-20260717T0115Z`이고 active override SHA-256은 `797e52dba26871e247d56be67aed34d4acd5ec26d92c184f49a3a9f9bd72a73e`다. Base와 candidate override를 실제 `.env`로 render한 뒤 `root:root 0640`으로 설치했다. +- Image pull 없이 API container만 exact image `sha256:3ee144df80b5110e4d72b723f570a53908f14cf9c036ae2fecb11cbbdba9569f`로 재생성했다. Container env의 세 설정값, `running`, restart count `0`, v1 recommendation 200, Bearer 인증 management scrape 200과 v2 recommendation 200을 확인했다. v2 smoke는 기본 limit 5에 따라 evidence score와 reasons를 가진 5개 item을 반환했다. +- 현재 v1은 Aura, v2는 MySQL, query timeout은 10초다. Schema/`EXPLAIN`, topology·SELECT 권한, reverse-proxy rate-limit과 Hikari saturation gate는 문서상 미완료이므로 v2 공개 활성화의 잔여 운영 위험으로 유지한다. 이번 maintenance에서는 v1 MySQL 전환, image pull, workflow dispatch와 merge를 수행하지 않았다. +- Legacy workflow에는 자동 복구 계약이 없다. 다음 dispatch의 수동 rollback baseline은 현재 정상 image `sha256:3ee144df80b5110e4d72b723f570a53908f14cf9c036ae2fecb11cbbdba9569f`이며 보존 tag와 base+observability override `--pull never --force-recreate` 절차를 사용한다. 다음 배포 전에도 exact current image와 tag 존재를 다시 확인한다. +- 이 복원은 검증된 운영 단순성을 우선한 명시적 위험 수용이다. `latest-arm64`의 가변성, third-party SSH action, workflow 내부 owner approval 부재, host script의 저장소 밖 drift는 잔여 위험이다. Concurrency는 겹치는 workflow run만 직렬화하며 mutable tag 자체를 immutable하게 만들지 않는다. +- 실패한 [deploy run 29540196835](https://github.com/Team-Maple/MLS-BE/actions/runs/29540196835)는 deploy 전에 추가 image 검증 script의 Bash `case` 문법 오류로 끝났고 host 변경은 시작되지 않았다. +- Merge와 운영 workflow dispatch는 owner의 별도 명시적 승인 전에는 수행하지 않는다. + +## 폐기된 2026-07-17 immutable 배포 단순화 계획 + +아래 계획은 구현됐지만 운영 적용 전에 폐기됐다. 현재 배포 절차로 사용하지 않는다. + +- 추천 branch의 [deploy run 29531165131](https://github.com/Team-Maple/MLS-BE/actions/runs/29531165131)은 host의 기존 `preflight-host.sh` checksum과 branch의 변경본이 달라 `host-preflight`에서 실패했다. Build, image publish, container pull/recreate와 운영 설정 변경은 시작되지 않았다. +- 파일 checksum을 확인하는 host script 자체가 새 checksum 계약을 알아야 하는 순환 bootstrap이 원인이었다. 애플리케이션 상태나 추천 코드 문제가 아니다. +- PR #35에서는 routine workflow를 `build-and-publish -> production approval -> deploy `로 줄이고, `preflight-host.sh`와 세 checksum 전달을 제거한다. Main-only `production-build` job은 image를 publish하되 deployment record를 만들지 않고, 별도 최소권한 deploy job 하나만 `production` 승인을 받는다. Runner에는 digest allowlist, 동시 배포 lock, exact previous-image 보존, bounded public·management smoke, 자동 rollback과 root-only 실패 진단만 유지한다. +- Build가 publish tag를 한 번 digest로 해소한 뒤 FCM permission, OCI revision, run-image와 deploy output을 같은 digest로 검증하도록 바꿨다. Gradle 8.13 distribution, arm64 build runner의 pack 0.40.0 archive와 x64 deploy runner의 Tailscale 1.94.2 archive는 architecture별 공식 SHA-256으로 고정했다. Production SSH는 release binary를 다시 내려받는 third-party action 대신 runner 기본 OpenSSH와 reviewed host-key fingerprint를 사용한다. +- 당시 host의 forced-command runner는 이전 command 문법을 사용했고 active Compose override에는 recommendation 환경 전달이 없었다. 이 immutable 전환 계획은 폐기됐으며, 이후 owner 승인 legacy maintenance에서 forced command, host script와 active override를 별도로 수정했다. +- Workflow가 읽는 `FIREBASE_KEY`, `TS_OAUTH_CLIENT_ID`, `TS_OAUTH_SECRET`, `ORACLE_SSH_KEY`는 현재 repository secret이고 Environment secret은 비어 있다. 첫 실행 전에 owner가 Firebase key를 새 `production-build`, Tailscale/SSH credential을 `production` scope로 재발급·이전하고 repository 사본을 제거해야 한다. 이 secret 변경은 아직 수행하지 않았다. +- `production` Environment variable `ORACLE_SSH_FINGERPRINT`도 아직 없다. Owner가 host console에서 확인한 SHA256 host-key fingerprint를 등록하기 전에는 deploy job이 실패하도록 고정했다. +- 삭제되는 legacy EC2 workflow가 참조하던 `HOST`, `USERNAME`, `KEY`, `PORT`, `GHCR_TOKEN`, `GHCR_USERNAME` repository secret도 남아 있다. 다른 consumer가 없는지 확인한 뒤 credential revoke/rotation과 secret 제거가 필요하며 아직 수행하지 않았다. +- `production` Environment는 owner required reviewer와 self-review 허용 상태지만 admin bypass가 가능하고 branch policy에 오래된 `feature/observability-phase-1`가 남아 있다. `production-build` Environment도 아직 없다. 첫 실행 전에 owner 승인으로 두 Environment를 `main`만 허용하고 production admin bypass를 비활성화해야 한다. 이 설정 변경은 아직 수행하지 않았다. +- 기존 mutable `:latest` EC2 workflow는 2026-04-07 이후 실행 이력이 없고 OCI 승인·rollback 경계를 우회하므로 PR #35에서 제거한다. Merge 전까지 GitHub 기본 branch에는 그대로 존재한다. +- Legacy workflow의 최근 10회 이력은 2026-01-27~2026-04-07이고 마지막 run 24086529960은 성공이다. Public API DNS는 Cloudflare proxy 주소만 보여 EC2 origin/DR 의존성 부재를 입증하지 못했다. Owner의 DNS/LB/failover 운영 inventory 확인 전에는 workflow 삭제와 legacy credential 폐기를 승인 완료로 보지 않는다. +- `production` Environment의 현재 branch policy에는 recommendation branch가 없다. Merge·runner 전환·운영 배포는 각각 owner의 명시적 승인 전에는 수행하지 않는다. +- 기존 Firebase key는 build 과정에서 bootJar/image layer에 포함돼 GHCR read 권한자가 읽을 수 있는 잔여 위험이 있다. 이번 배포 단순화 범위에서 runtime secret mount로 바꾸지 않았으며, 첫 배포 전에 owner risk acceptance 또는 별도 mount 전환·key rotation이 필요하다. + +## 2026-07-16 추천 dashboard 사전 갱신 + +- 구현 작업은 [Issue #34](https://github.com/Team-Maple/MLS-BE/issues/34), [draft PR #35](https://github.com/Team-Maple/MLS-BE/pull/35)의 branch `feature/mysql-evidence-recommendations`에 있다. 구현/계약/관측 커밋은 각각 `79d2dff`, `037ad30`, `49bec1e`이며 애플리케이션 merge·배포와 v1 MySQL 전환은 수행하지 않았다. +- 기존 [Mapleland / Production Overview](https://mungmnb777.grafana.net/d/mapleland-production-overview/mapleland-production-overview?orgId=1&from=now-6h&to=now&timezone=browser&var-instance=mapleland-oci-1&refresh=1m)를 고정 UID `mapleland-production-overview`로 먼저 읽은 뒤 같은 UID를 `Import (Overwrite)`해 create-or-update했다. 기존 `Dashboards` folder와 datasource를 유지했다. +- live dashboard는 version `3`, panel `24`개다. version history에서 version 3이 `2026-07-16 23:21:43 KST`의 Latest이고 직전 version 2와 version 1은 restore 가능한 상태임을 확인했다. +- recommendation row와 `Recommendation request rate`, `Recommendation p95 latency`, `Error / empty / unavailable rate`, `Engine state (observed)`, `Average result count` panel, 그리고 기존 log panel을 구분하는 `Application logs` row가 각각 한 번 렌더링됨을 확인했다. +- request rate와 HTTP error는 `http_server_requests_seconds_count`, p95는 기존 `http_server_requests_seconds_bucket`의 v1/v2 route template을 사용한다. 따라서 validation/config parsing/404/5xx를 포함한 실제 endpoint traffic을 보고 custom counter는 scorer의 empty/unavailable outcome만 보완한다. live 화면에서 panel 다섯 개가 load error 없이 렌더링됐지만 애플리케이션/Alloy 변경이 아직 배포되지 않았고 추천 route traffic 표본도 없으므로 live series 존재를 완료 조건으로 주장하지 않는다. 배포 후 v2 안전 smoke에서 data와 label을 다시 확인한다. +- alert rule, threshold, contact point, notification policy, datasource, unrelated panel은 수정하거나 삭제하지 않았다. source dashboard JSON도 version 3과 동일한 24개 panel로 갱신했다. +- mapledb index/preflight 변경은 별도 [mapleland draft PR #132](https://github.com/mungmnb777/mapleland/pull/132)의 `d59d4aa`, `d7cc047`에 있으며 production DDL은 실행하지 않았다. + ## 2026-07-16 최종 운영 적용 상태 - Issue: [#32](https://github.com/Team-Maple/MLS-BE/issues/32) @@ -67,14 +115,14 @@ ## 다음 안전 작업 -1. PR #33의 최종 CI와 리뷰를 확인한 뒤 별도 요청이 있을 때만 merge한다. +1. PR #35의 최종 CI와 리뷰를 확인하되 owner의 별도 요청 전에는 merge 또는 legacy OCI workflow dispatch를 수행하지 않는다. 2. 반복됐던 Hikari connection validation warning의 발생량과 DB/server timeout·`maxLifetime` 정합성을 별도 후속 작업으로 조사한다. 3. 인증 재발급 경로의 NullPointerException 재현 조건과 null contract를 별도 후속 작업으로 조사한다. 4. 운영 traffic이 늘면 active series, Loki 일일 환산량, Alloy RSS/CPU와 alert noise를 같은 30분 gate로 다시 측정한다. ## 재개 규칙 -- GitHub Actions는 `host-preflight -> build-and-publish -> production approval -> host-preflight recheck -> deploy` 순서를 벗어나지 않는다. +- Routine GitHub Actions는 run `28788801815`와 같은 `build latest-arm64 -> Tailscale SSH -> /opt/mapleland/update-api.sh` 순서를 사용한다. Workflow 내부 owner approval과 immutable digest 보장은 없으므로 dispatch 자체를 owner의 명시적 승인으로 취급하고 종료 직후 공개 API와 dashboard를 수동 확인한다. - manual OCI 작업은 1Password 앱 잠금 상태를 추측하지 않는다. 정확한 SSH alias `OracleCloud`를 사용하고 명시적 `SSH_AUTH_SOCK`으로 `ssh-add -l`을 확인한다. 서명 승인 상태가 불명확하면 TTY에서 `ssh-add -T ~/.ssh/1Password/SHA256_QoL9bUNkoXz+boL+ozfL1CHCMCaDSZWulM8S2cVMTWs.pub`를 먼저 실행해 1Password 승인을 받은 뒤 `BatchMode` SSH 연결을 검증한다. `oracle-cloud`처럼 다른 대소문자의 host를 사용해 identity 규칙을 우회하지 않는다. - routine deploy는 GitHub Actions의 `ORACLE_SSH_KEY`를 사용한다. 1Password SSH agent는 manual host preparation과 break-glass 확인에만 사용한다. - Grafana Cloud MCP는 한 Codex task만 사용한다. 병렬 agent를 시작한 상태에서 Grafana MCP를 초기화하거나 OAuth 재로그인을 반복하지 않는다. diff --git a/docs/observability-runbook.md b/docs/observability-runbook.md index 52467d7..321770e 100644 --- a/docs/observability-runbook.md +++ b/docs/observability-runbook.md @@ -53,7 +53,8 @@ Actuator exposure는 `health`, `info`, `prometheus`만 허용한다. `env`, `con - `service.name=mapleland-api` - `service.environment=prod` -- `service.version=${SERVICE_VERSION:-unknown}`; 배포 시 Git SHA 또는 release tag를 주입한다. +- `service.version=${SERVICE_VERSION:-unknown}`; legacy deploy workflow는 source revision을 + 주입하지 않으므로 host 환경에 값이 없으면 `unknown`이다. - active file: `/var/log/mapleland-api/mapleland-api.json` - rolled file: `mapleland-api.YYYY-MM-DD.N.json` - rotation: 10 MiB/file, 14일, 총 250 MiB 상한 @@ -83,9 +84,13 @@ Alloy scrape interval은 host/application/self 모두 60초, timeout은 10초다 - `jvm_memory_*`, `jvm_gc_pause_*`, `jvm_threads_*` - `process_*`, `system_cpu_*` - `hikaricp_connections_*` +- `mapleland_recommendation_requests_total` (`engine`, `api_version`, `outcome`만 사용) +- `mapleland_recommendation_results_recommendations_{count,sum,max}` (`engine`, `api_version`만 사용) HTTP latency bucket은 50 ms, 100 ms, 250 ms, 500 ms, 1 s, 2 s, 5 s, 10 s의 고정 SLO 경계만 추가한다. URI는 Spring MVC route template tag를 그대로 사용하며 raw URI/query/user tag는 추가하지 않는다. +추천 처리 로그의 `event.duration`, `mapleland.recommendation.engine`, `mapleland.api.version`, `mapleland.result.count`는 structured metadata로만 전달한다. Job/level/map/member와 원본 URI/query는 metric label이나 Loki indexed label로 만들지 않는다. + Host collector는 CPU, load, memory, filesystem, 최소 network, uname만 사용한다. Docker/loopback/VPN의 임시 network와 pseudo/container filesystem을 제외하고, remote write 전에 dashboard에 필요한 metric name allowlist를 다시 적용한다. 단일 고정 `instance=mapleland-oci-1`은 향후 host별 구분을 위한 bounded label이다. ## Grafana Alloy 고정 설치 @@ -439,178 +444,306 @@ sudo stat -c '%U:%G %a %n' /etc/alloy /etc/alloy/config.alloy \ /etc/alloy/alloy.env /var/lib/alloy ``` -## 애플리케이션 적용 순서 - -이 절은 단일 컨테이너 재생성을 포함하므로 실행 직전 승인이 필요하다. - -재시작 전에 현재 container image ID를 별도 local tag로 고정하고 Compose, `.env`, 기존 `update-api.sh`를 root-only 경로에 백업한다. Registry tag는 pull 뒤 다른 digest를 가리킬 수 있으므로 digest 또는 그 image ID에 붙인 rollback tag 없이 `docker compose up`만 실행하는 것은 롤백이 아니다. Build job은 run-attempt별 tag로 publish한 직후 registry manifest descriptor의 digest를 읽고 deploy job에는 `ghcr.io/team-maple/mls-be/mapleland-api@sha256:...`만 전달한다. 저장소의 `deploy/observability/update-api.sh`도 digest ref 외에는 거부하고, 매 배포 직전 image를 별도 rollback tag로 고정한 뒤 `--pull never`로 재생성한다. 새 image의 exact ID, management health와 public smoke가 90초 안에 모두 통과하지 않으면 exact 이전 image로 자동 복구하고 workflow를 실패시킨다. +## 애플리케이션 배포 + +현재 운영 애플리케이션 배포 진입점은 `.github/workflows/deploy-oci.yml` 하나다. Workflow는 +검증된 [run 28788801815](https://github.com/Team-Maple/MLS-BE/actions/runs/28788801815)의 +commit `c49ee3255afb4ddfa0168ce91783fff368864a4d`와 같은 계약을 사용한다. +동시에 실행한 두 run이 같은 mutable tag를 덮어쓰지 않도록 `deploy-oci` concurrency만 유지하고, +OAuth secret을 사용하는 Tailscale 경로에 불필요한 OIDC 권한은 부여하지 않는다. + +1. arm64 image를 `ghcr.io/team-maple/mls-be/mapleland-api:latest-arm64`로 build/publish한다. +2. Tailscale에 연결한다. +3. `appleboy/ssh-action@v1.2.5`로 `oracle-cloud`에 접속한다. +4. host에 이미 설치된 `/opt/mapleland/update-api.sh`를 인자 없이 실행한다. + +Repository의 host preflight와 update runner를 동기화하거나 설치하지 않는다. Routine workflow는 +가변 tag, repository secret, third-party SSH action과 host-local script에 의존한다. Workflow +내부의 별도 owner approval, immutable digest, host-key fingerprint와 자동 rollback 검증은 없다. +운영 dispatch는 owner의 명시적 승인 뒤에만 수행하고, Actions 종료 후 기존 dashboard와 공개 +API를 수동 확인한다. Host script 변경은 이 runbook의 routine deploy 범위가 아니다. + +Owner 승인 maintenance에서 active host script가 legacy no-arg 계약이고 SHA-256이 +`436dae8156ec7115f823589aeb46b586b15e3259443bf9b1786d61fcec331b4d`임을 확인했다. Script의 +pull/up은 base와 `/opt/mapleland/docker-compose.observability.yml`을 항상 함께 사용한다. CI key의 +forced command는 `command="/usr/bin/sudo -n /opt/mapleland/update-api.sh"`이며, script와 `.env`는 +각각 `root:root 0755`, `root:root 0600`을 유지한다. Host의 기존 broad `NOPASSWD: ALL`은 이번 +maintenance에서 변경하지 않았다. Override는 management 설정과 세 `RECOMMENDATION_*` 설정을 +명시적으로 container에 전달한다. 상세 backup, checksum과 현재 활성값 검증 결과는 checkpoint를 따른다. +또한 첫 dispatch 전에 exact previous image의 존재와 현재 host script에 맞는 수동 rollback 명령을 +read-only로 확인해 checkpoint에 기록한다. 폐기된 immutable runner의 `rollback_tag` 절차를 +legacy workflow에 사용하지 않는다. + +## 폐기된 immutable 배포 계약 기록 + +아래 내용은 PR #35에서 검토했지만 운영 적용 전에 폐기된 설계 기록이다. 현재 배포 절차로 +실행하지 않는다. + +Routine deploy는 다음 다섯 단계만 수행한다. + +1. arm64 image를 build하고 GHCR에 publish한다. +2. registry manifest digest를 확정해 + `ghcr.io/team-maple/mls-be/mapleland-api@sha256:<64-hex>` 형태로 고정한다. +3. `production` Environment owner 승인을 한 번 받는다. +4. 별도 최소권한 runner가 Tailscale SSH forced command에 + `deploy `를 전달한다. +5. host runner가 직전 image를 보존하고 candidate를 재생성한 뒤 공개·management smoke를 + 확인한다. + +Publish 직후 tag는 정확히 한 번만 digest로 해소한다. 이후 권한·revision·run-image 검증과 +deploy output은 모두 같은 digest reference를 사용하므로 tag를 다시 조회하지 않는다. +Gradle distribution, arm64 build runner의 pack CLI archive와 x64 deploy runner의 Tailscale +archive에는 architecture별 reviewed SHA-256을 명시한다. +SSH는 별도 downloader action 대신 GitHub runner의 OpenSSH를 사용하며, `ssh-keyscan` 결과 중 +owner가 `production` Environment에 등록한 fingerprint와 정확히 일치하는 host key만 ephemeral +`known_hosts`에 넣는다. + +Host runner는 배포마다 저장소 파일 checksum, Alloy 상태, 전체 Compose 구조, 추천 설정이나 +legacy Grafana migration을 감사하지 않는다. 이런 검사는 host provisioning 또는 별도 운영 +점검의 책임이다. Root Docker 입력 변조를 막기 위한 네 실행 파일의 regular-file·owner·mode와 +management token 형식만 직접 신뢰 경계로 확인한다. Runner의 안전 경계는 다음과 같다. + +- 정확한 GHCR repository의 immutable digest만 허용한다. +- non-blocking host lock으로 동시 배포를 거부한다. +- 기존 running container의 exact image ID를 local rollback tag로 보존한다. +- candidate는 local Compose tag에 digest image를 연결한 뒤 `--pull never`로 재생성한다. +- exact candidate image ID, service version, restart count `0`, container state/health, 공개 `/api/v1/jobs`, + loopback management `/actuator/info`, secret-safe authenticated `/actuator/prometheus` exact + 200을 90초 polling deadline으로 확인한다. 개별 command의 bounded timeout 때문에 실제 + wall-clock은 이 deadline보다 길 수 있다. +- restart, exit, timeout이면 같은 Compose 계약으로 직전 image를 자동 복구한다. +- image의 OCI `org.opencontainers.image.revision` label을 candidate의 `SERVICE_VERSION`으로 + 주입하고, 자동 복구 시에는 이전 container의 값을 복원한다. +- 단계와 결과는 `deployment phase= outcome=` 한 줄 로그로 남긴다. +- candidate/rollback의 raw log는 Actions에 출력하지 않고 host의 root-only + `/var/log/mapleland-deploy/last-candidate-failure/`와 + `/var/log/mapleland-deploy/last-rollback-failure/`에 분리해 보존한다. + +Recommendation 환경값을 생략하면 application과 Compose가 함께 +`AURA`, `false`, `10`을 기본값으로 사용한다. 운영 전환은 필요할 때 root-only +`/opt/mapleland/.env`에서 명시적으로 바꾼다. Runner는 값을 해석하거나 후보 container와 +비교하지 않는다. + +설정 파일 변경은 routine image deploy와 별도인 owner 승인 host maintenance다. `.env`를 +backup하고 원자적으로 교체한 뒤 Compose rendering을 확인하고, 같은 workflow로 container를 +재생성해야 실행 중인 process에 반영된다. Runner의 image rollback은 현재 설정을 그대로 +재사용하므로 설정 오류로 `rollback_failed`가 나면 먼저 `.env` backup을 복원한 뒤 승인된 +immutable image를 다시 실행한다. + +### 폐기됨: 한 번만 수행하는 runner와 Compose override 전환 + +현재 host에 checksum 세 개를 요구하는 이전 runner가 설치돼 있다면 새 workflow 명령을 +거부한다. 이 전환은 routine application deploy가 아니라 별도 host maintenance다. Owner +승인 change window에서 merged `main`의 reviewed `update-api.sh`와 recommendation 환경 +변수를 전달하는 Compose override를 함께 원자적으로 설치한다. Gateway allowlist는 저장소의 +runner 계약 테스트가 검증한다. 이 작업을 하기 전에는 새 workflow를 dispatch하지 않는다. ```bash -cd /opt/mapleland -stamp="$(date -u +%Y%m%dT%H%M%SZ)" -compose_backup="/root/docker-compose.yml.pre-observability-${stamp}" -env_backup="/root/mapleland.env.pre-observability-${stamp}" -update_script_backup="/root/update-api.sh.pre-observability-${stamp}" -previous_image_id="$(sudo docker inspect --format '{{.Image}}' mapleland-mapleland-api-1)" -rollback_tag="ghcr.io/team-maple/mls-be/mapleland-api:rollback-${stamp}" -image_name="ghcr.io/team-maple/mls-be/mapleland-api" -compose_image="$(sudo docker compose config --images | - grep -E '^ghcr[.]io/team-maple/mls-be/mapleland-api:')" -test -n "$compose_image" -test "$(printf '%s\n' "$compose_image" | grep -c .)" -eq 1 -expected_commit='' -staged_update_script='/update-api.sh' -staged_preflight_script='/preflight-host.sh' -staged_compose_override='/docker-compose.override.example.yml' -expected_update_script_sha256='' -expected_preflight_script_sha256='' -expected_compose_override_sha256='' -printf '%s\n' "$expected_commit" | grep -Eq '^[0-9a-f]{40}$' -printf '%s\n' "$expected_update_script_sha256" | grep -Eq '^[0-9a-f]{64}$' -printf '%s\n' "$expected_preflight_script_sha256" | grep -Eq '^[0-9a-f]{64}$' -printf '%s\n' "$expected_compose_override_sha256" | grep -Eq '^[0-9a-f]{64}$' -sudo test -f "$staged_update_script" -sudo test -f "$staged_preflight_script" -sudo test -f "$staged_compose_override" -sudo bash -n "$staged_update_script" -sudo bash -n "$staged_preflight_script" -printf '%s %s\n' "$expected_update_script_sha256" \ - "$staged_update_script" | sudo sha256sum -c - -printf '%s %s\n' "$expected_preflight_script_sha256" \ - "$staged_preflight_script" | sudo sha256sum -c - -printf '%s %s\n' "$expected_compose_override_sha256" \ - "$staged_compose_override" | sudo sha256sum -c - - -sudo cp -a docker-compose.yml "$compose_backup" -sudo cp -a .env "$env_backup" -sudo cp -a /opt/mapleland/update-api.sh "$update_script_backup" -sudo docker image tag "$previous_image_id" "$rollback_tag" -sudo sh -c "umask 077; printf \ - 'COMPOSE_BACKUP=%s\nENV_BACKUP=%s\nUPDATE_SCRIPT_BACKUP=%s\nROLLBACK_TAG=%s\nROLLBACK_COMPOSE_IMAGE=%s\nEXPECTED_COMMIT=%s\n' \ - '$compose_backup' '$env_backup' '$update_script_backup' \ - '$rollback_tag' '$compose_image' \ - '$expected_commit' \ - > /root/mapleland-observability-rollback.env" -sudo docker image inspect "$rollback_tag" >/dev/null -sudo install -o root -g root -m 0755 "$staged_update_script" \ - /opt/mapleland/update-api.sh -sudo install -o root -g root -m 0755 "$staged_preflight_script" \ - /opt/mapleland/preflight-host.sh -sudo install -o root -g root -m 0640 "$staged_compose_override" \ +set -euo pipefail +reviewed_update_script='' +reviewed_compose_override='' +cutover_stamp="$(date -u +%Y%m%dT%H%M%SZ)" +sudo bash -n "$reviewed_update_script" +sudo test ! -L /opt/mapleland/docker-compose.yml +sudo test ! -L /opt/mapleland/docker-compose.observability.yml +sudo test ! -L /opt/mapleland/.env +sudo test ! -L /opt/mapleland/ghcr.env +sudo test ! -L /var/log/mapleland-api +command -v flock timeout docker jq +sudo docker compose version +sudo cp -a /opt/mapleland/update-api.sh \ + "/root/update-api.sh.before-simple-runner-${cutover_stamp}" +sudo cp -a /opt/mapleland/docker-compose.observability.yml \ + "/root/docker-compose.observability.yml.before-simple-runner-${cutover_stamp}" +sudo install -o root -g root -m 0755 "$reviewed_update_script" \ + /opt/mapleland/.update-api.sh.next +sudo install -o root -g root -m 0640 "$reviewed_compose_override" \ + /opt/mapleland/.docker-compose.observability.yml.next +sudo bash -n /opt/mapleland/.update-api.sh.next +sudo docker compose --env-file /opt/mapleland/.env \ + -f /opt/mapleland/docker-compose.yml \ + -f /opt/mapleland/.docker-compose.observability.yml.next \ + config --format json | jq -e ' + .services["mapleland-api"].environment.RECOMMENDATION_V1_ENGINE == "AURA" and + .services["mapleland-api"].environment.RECOMMENDATION_V2_ENABLED == "false" and + .services["mapleland-api"].environment.RECOMMENDATION_QUERY_TIMEOUT_SECONDS == "10" and + (.services["mapleland-api"].environment.MANAGEMENT_SCRAPE_TOKEN | type == "string" and length >= 32) + ' >/dev/null +sudo mv -f /opt/mapleland/.docker-compose.observability.yml.next \ /opt/mapleland/docker-compose.observability.yml -if ! sudo test -e /opt/mapleland/ghcr.env; then - sudo install -o root -g root -m 0600 /dev/null /opt/mapleland/ghcr.env +sudo mv -f /opt/mapleland/.update-api.sh.next /opt/mapleland/update-api.sh +sudo bash -n /opt/mapleland/update-api.sh +test "$(sudo stat -c '%U:%G:%a' /opt/mapleland/docker-compose.yml)" = root:root:640 +test "$(sudo stat -c '%U:%G:%a' /opt/mapleland/docker-compose.observability.yml)" = root:root:640 +test "$(sudo stat -c '%U:%G:%a' /opt/mapleland/.env)" = root:root:600 +test "$(sudo stat -c '%U:%G:%a' /opt/mapleland/ghcr.env)" = root:root:600 +test "$(sudo stat -c '%U:%G:%a' /opt/mapleland/update-api.sh)" = root:root:755 +test "$(sudo stat -c '%u:%g:%a' /var/log/mapleland-api)" = 1002:1001:750 +if sudo test -e /var/log/mapleland-deploy; then + sudo test ! -L /var/log/mapleland-deploy + test "$(sudo stat -c '%U:%G:%a' /var/log/mapleland-deploy)" = root:root:700 fi -# Populate only GHCR_USER and a read:packages GHCR_TOKEN through the approved -# secret channel. Do not paste either value into shell history or this runbook. -sudoedit /opt/mapleland/ghcr.env +sudo stat -c '%U:%G %a %n' \ + /opt/mapleland/docker-compose.yml \ + /opt/mapleland/docker-compose.observability.yml \ + /opt/mapleland/.env /opt/mapleland/ghcr.env \ + /opt/mapleland/update-api.sh +sudo sha256sum /opt/mapleland/update-api.sh \ + /opt/mapleland/docker-compose.observability.yml ``` -1. `/var/log/mapleland-api`의 owner `1002:1001`, mode와 기존/default Alloy ACL을 확인한다. -2. root-only `/opt/mapleland/.env`와 `/etc/alloy/alloy.env`에 동일한 `MANAGEMENT_SCRAPE_TOKEN`을 안전하게 입력한다. `/opt/mapleland/.env`는 `root:root 0600`으로 낮춘다. -3. 기존 Compose `.env`를 shell로 `source`하지 않는다. `ghcr.env.example`의 두 key만 포함하는 `/opt/mapleland/ghcr.env`를 별도로 만들고 `root:root 0600`, non-symlink인지 확인한다. `update-api.sh`는 이 파일을 실행하지 않고 exact key/value parser로만 읽는다. -4. `docker-compose.override.example.yml`을 checksum 검증 후 `/opt/mapleland/docker-compose.observability.yml`로 설치한다. 첫 전환 전에는 base Compose와 `.env`의 `GRAFANA_CLOUD_URL`, `GRAFANA_CLOUD_USERNAME`, `GRAFANA_CLOUD_PASSWORD` 세 legacy entry를 제거하지 않는다. 새 container가 실패하면 이전 Loki4j image가 exact rollback 입력으로 사용할 수 있어야 하기 때문이다. `update-api.sh`와 preflight는 이 세 entry를 제외한 root-only 임시 base와 `--env-file`을 사용해 새 container model을 render한다. App service의 기존 repository tag 계약은 `update-api.sh`가 digest-pinned image ID로 local retag하므로 임의 변경하지 않는다. -5. `SERVICE_VERSION`을 위 `expected_commit`으로 설정한다. Publish workflow run, commit과 build job이 확정한 manifest digest를 운영 기록에 남긴다. -6. 새 image의 readiness, management health, public smoke와 실제 container environment의 legacy key 부재가 모두 확인된 뒤에만 `update-api.sh`가 base Compose와 `.env`의 세 entry를 같은 directory의 root-only candidate로 만들고 render validation 후 atomic rename한다. 실패 전에는 원본을 보존하고, 성공 뒤에는 root-only backup에만 남긴다. Alloy write secret과 이 legacy rollback 값은 서로 다른 파일과 목적을 유지한다. -7. `18080`이 비어 있는지 다시 확인한다. public/Traefik routing이나 firewall rule은 추가하지 않는다. 저장소에서 계산한 두 script와 override checksum으로 `/opt/mapleland/preflight-host.sh`를 실행해 sanitized Compose render, legacy rollback entry의 `3:3` 또는 `0:0` 완전성, root-only env, active deployment script, log ACL, Alloy readiness/loopback listener, 현재 app container와 rollback image, public smoke를 읽기 전용으로 검증한다. -8. 승인 후 workflow를 dispatch한다. Workflow 순서는 `host-preflight -> build-and-publish -> production Environment approval -> host-preflight recheck -> deploy`다. 첫 preflight가 실패하면 image를 만들지 않으며, 두 번째 preflight와 `update-api.sh`는 forced-command gateway의 한 `deploy` 요청에서 연속 실행된다. Build job은 manifest digest를 `image_ref` output으로 넘기고 gateway가 세 checksum과 `@sha256:` 전체를 allowlist로 검증한다. Root update script가 digest pull, per-deploy rollback tag, local retag, `--pull never`, legacy environment 부재, exact image ID, bounded readiness/smoke를 모두 성공해야 한다. 첫 전환 실패 시 base Compose와 원래 `.env`로 이전 Loki4j image를 복구하고, 성공 시에만 legacy entry를 제거한다. 한 번의 전환으로 Loki4j를 제거하고 Alloy file tail을 시작해 중복 전송 구간을 만들지 않는다. -9. 실행 container의 image ID와 `SERVICE_VERSION`이 기대값과 같은지 먼저 확인하고, management health, unauthenticated scrape 거부, 공용 health/proxy, 대표 API, local ECS file, Alloy authenticated scrape, Grafana arrival를 순서대로 확인한다. - -첫 전환 동안 preflight와 update script는 기본 `.env` 자동 로드를 사용하지 않고 legacy 세 entry만 제외한 mode `0640` 임시 base와 mode `0600` 임시 `--env-file`을 같은 project directory에서 사용한다. 따라서 원본 rollback 값은 보존하면서도 새 container에는 전달하지 않는다. 명시적 env file의 project environment 동작은 [Docker Compose `--env-file` 규칙](https://docs.docker.com/compose/how-tos/environment-variables/variable-interpolation/#local-env-file-versus-project-directory-env-file)을 따른다. 실제 container의 `.Config.Env`도 readiness gate에서 값 출력 없이 재검증한다. 성공한 후보 파일만 active base와 `.env`로 승격된다. - -`sudo`는 Docker 접근을 위한 것이 아니라 root-only deployment state를 위한 것이다. Script는 mode `0600`인 `.env`/`ghcr.env`를 읽고 mode `0640`인 Compose 계약을 검증·교체하므로 EUID 0이 아니면 중단한다. Workflow가 상승시키는 범위는 checksum이 preflight에서 검증된 `/opt/mapleland/update-api.sh` 실행 한 번이며, 일반 사용자에게 secret read나 Compose write 권한을 부여하지 않는다. - -GitHub CI key의 `authorized_keys` 항목은 `command="/opt/mapleland/update-api.sh"`, `no-port-forwarding`, `no-agent-forwarding`, `no-X11-forwarding`, `no-pty`를 유지한다. 따라서 SSH client가 보낸 command는 직접 실행되지 않고 OpenSSH가 강제 entrypoint의 `SSH_ORIGINAL_COMMAND`에 보존한다. Non-root entrypoint는 newline 없는 단일 line 전체를 정규식으로 검증하며 `preflight <3 checksums>` 또는 `deploy <3 checksums> `만 허용한다. 표준입력, shell `eval`, tag image, 임의 command와 추가 argument는 실행 경계로 사용하지 않는다. Allowlist를 통과한 경우에만 `/usr/bin/sudo -n`으로 고정된 preflight/update script를 호출하고, root로 재진입한 update script가 실제 배포를 수행한다. - -운영 Docker Compose 2.38.2는 long bind syntax의 `create_host_path: false`를 적용하면서도 `config --format json`에서는 false 기본값을 `null`로 생략한다. Preflight는 active override의 SHA-256을 reviewed source와 먼저 대조하므로 해당 source가 `create_host_path: false`임을 바이트 단위로 고정하고, rendered model에서는 bind type/source/target과 `create_host_path`가 `true`가 아님을 확인한다. `true`, 다른 source/target 또는 checksum 불일치는 모두 fail closed다. +GitHub CI key의 `authorized_keys` forced command는 계속 +`command="/opt/mapleland/update-api.sh"`를 사용한다. `no-port-forwarding`, +`no-agent-forwarding`, `no-X11-forwarding`, `no-pty`도 유지한다. 새 runner가 +`deploy `만 허용하므로 `preflight`, tag image, 추가 인자, multiline, +stdin 우회나 임의 command는 실행되지 않는다. 전환 확인 후 사용하지 않는 +`/opt/mapleland/preflight-host.sh`는 별도 승인 아래 제거할 수 있다. + +Runner는 Ubuntu의 `flock`과 GNU `timeout`, 다음 기존 파일을 실행 입력으로 사용한다. + +- `/opt/mapleland/docker-compose.yml` +- `/opt/mapleland/docker-compose.observability.yml` +- `/opt/mapleland/.env` +- `/opt/mapleland/ghcr.env`의 `GHCR_USER`, `GHCR_TOKEN` + +`ghcr.env`는 shell로 source하지 않는다. Runner는 두 값만 읽어 임시 +`DOCKER_CONFIG`로 `docker login --password-stdin`을 수행하고 종료할 때 임시 +credential directory를 제거한다. Secret 값은 workflow 출력, Issue 또는 PR에 기록하지 +않는다. + +Routine runner를 실행기 역할로 유지하는 대신 host privilege 경계는 월 1회와 host +maintenance 직후 별도 audit로 확인한다. Base/override/`.env`/`ghcr.env`가 regular file인지, +owner/mode가 각각 cutover 계약과 같은지, forced command와 forwarding 제한이 유지되는지, +log directory owner/mode와 effective Compose의 안전 추천 기본값을 확인한다. Drift가 있으면 +routine workflow를 실행하지 않고 owner 승인 maintenance로 복구한다. 이 audit은 application +deploy마다 반복하지 않는다. + +Workflow가 사용하는 `FIREBASE_KEY`는 첫 실행 전에 owner가 `production-build` Environment +secret으로, `TS_OAUTH_CLIENT_ID`, `TS_OAUTH_SECRET`, `ORACLE_SSH_KEY`는 `production` +Environment secret으로 재발급·이전하고 같은 이름의 repository/organization secret 사본을 +제거한다. 그래야 선택 branch가 Environment 승인을 우회해 운영 credential을 읽을 수 없다. +Secret 이전과 rotation은 아직 수행하지 않았고, 완료 전에는 workflow를 dispatch하지 않는다. +검증한 Oracle host의 `SHA256:` public-key fingerprint는 `production` Environment variable +`ORACLE_SSH_FINGERPRINT`로 등록한다. Workflow는 비어 있거나 형식이 다른 값과 host key +불일치를 모두 배포 전에 거부한다. +삭제되는 legacy EC2 workflow가 사용하던 `HOST`, `USERNAME`, `KEY`, `PORT`, +`GHCR_TOKEN`, `GHCR_USERNAME` repository secret도 다른 consumer가 없는지 read-only로 확인한 +뒤 credential을 revoke/rotate하고 제거한다. 현재 workflow는 이 이름들을 참조하지 않는다. + +같은 change window에서 `production-build`와 `production` branch policy는 모두 `main`만 +허용한다. `production-build`에는 reviewer를 두지 않고, `production` required reviewer는 +owner로 유지한다. Owner가 직접 시작한 run을 명시적으로 승인할 수 있도록 self-review는 +허용하되, 승인 단계를 건너뛰지 못하도록 administrator bypass는 비활성화한다. 현재 +`production-build`는 아직 생성하지 않았고, `production`에 남은 +`feature/observability-phase-1` 허용과 admin bypass도 아직 제거하지 않았다. + +현재 build는 기존 계약대로 Firebase service-account JSON을 bootJar와 image layer에 포함한다. +따라서 GHCR image read 권한은 사실상 이 key의 read 권한이기도 하다. 이 PR에서 FCM runtime +secret mount 전환까지 함께 수행하지 않으며, 첫 배포 전 owner가 이 잔여 위험을 명시적으로 +수용하거나 별도 보안 변경으로 runtime mount 전환과 key rotation을 완료해야 한다. + +### 폐기됨: 실행과 감시 + +정상적인 workflow 흐름은 다음과 같다. -```bash -sudo chown root:root /opt/mapleland/.env -sudo chmod 0600 /opt/mapleland/.env -sudo chown root:root /opt/mapleland/docker-compose.yml -sudo chmod 0640 /opt/mapleland/docker-compose.yml -test "$(sudo stat -c '%U:%G %a' /opt/mapleland/docker-compose.yml)" = 'root:root 640' -test "$(sudo stat -c '%U:%G %a' /opt/mapleland/docker-compose.observability.yml)" = 'root:root 640' -sudo test -f /opt/mapleland/ghcr.env -sudo test ! -L /opt/mapleland/ghcr.env -test "$(sudo stat -c '%U:%G %a' /opt/mapleland/ghcr.env)" = 'root:root 600' -test "$(sudo awk -F= ' - /^(GHCR_USER|GHCR_TOKEN)=/ { seen[$1]++ } - END { print (seen["GHCR_USER"] == 1 && seen["GHCR_TOKEN"] == 1 && length(seen) == 2) ? "ok" : "bad" } -' /opt/mapleland/ghcr.env)" = ok -test -d /var/log/mapleland-api -if ss -lntH 'sport = :18080' | grep -q .; then - echo 'management port 18080 is already in use' >&2 - exit 1 -fi -cd /opt/mapleland -effective_images="$(sudo docker compose -f docker-compose.yml config --images)" -effective_app_image="$(printf '%s\n' "$effective_images" | - grep -E '^ghcr[.]io/team-maple/mls-be/mapleland-api:')" -test -n "$effective_app_image" -test "$effective_app_image" = "$compose_image" -sudo /opt/mapleland/preflight-host.sh \ - "$expected_preflight_script_sha256" \ - "$expected_update_script_sha256" \ - "$expected_compose_override_sha256" +```text +build-and-publish -> production approval -> deploy immutable digest ``` -Preflight는 secret 값이나 rendered Compose를 출력하지 않고 non-secret checksum, current image ID와 management listener 상태로 만든 attestation만 출력한다. checksum mismatch, broad env permission, wildcard listener, Compose boundary 누락, Alloy/readiness/ACL 또는 public smoke 실패가 있으면 원인을 해결하기 전 workflow를 재실행하지 않는다. 특히 active script가 저장소와 다르면 legacy script를 실행해 보는 방식으로 확인하지 않는다. - -배포 readiness는 전체 `/actuator/health` 집계를 사용하지 않는다. 이 집계는 선택적 Neo4j 같은 외부 dependency 하나가 `DOWN`이어도 503이 되어, JVM·관리 서버·핵심 MySQL API가 정상인 후보를 불필요하게 롤백할 수 있다. 대신 Bearer 인증된 Prometheus endpoint로 실제 Alloy scrape 경계를 확인하고 `/api/v1/jobs`로 공개 서버와 핵심 DB 경계를 함께 확인한다. Token은 curl config stdin으로만 전달하며 argv와 로그에 남기지 않는다. - -후보가 한 번이라도 자동 재시작되면 startup crash로 간주해 90초를 모두 기다리지 않고 롤백한다. 롤백 전에 해당 컨테이너의 `.State`와 timestamp가 포함된 최근 Docker 로그 500줄을 `/var/log/mapleland-deploy/last-failure/`에 보존한다. 디렉터리는 root 전용 `0700`, 파일은 `0600`이며 같은 경로를 교체하므로 실패마다 무한히 누적되지 않는다. 이 로그는 운영 로그 정책 적용 전 startup 출력이므로 credential이나 식별자가 포함됐다고 가정하고 CI·Issue·PR에 원문을 붙이지 않는다. 원인 분석 후 `sudo rm -rf /var/log/mapleland-deploy/last-failure`로 제거한다. - -Paketo image의 application layer는 `1001:1001`, runtime process는 `1002:1001`이다. GitHub Actions가 복원하는 Firebase resource가 `0700/0600`이면 memory calculator가 `/workspace/BOOT-INF/classes/firebase`를 순회하지 못해 JVM 실행 전에 종료된다. Restore 단계는 비어 있지 않은 service-account JSON인지 출력 없이 확인하고 directory `0750`, key file `0640`을 적용한다. bootJar의 Unix mode를 publish 전에 검사하고, shell이 없는 tiny image는 시작하지 않은 container의 exported metadata에서 실제 `Config.User`, directory/file owner와 mode를 다시 확인한다. 따라서 mutable builder의 UID/GID가 바뀌어도 deploy job 전에 fail closed한다. Key는 world-readable하지 않고 값·내용·checksum을 workflow 출력에 기록하지 않는다. +Actions log에서는 마지막 `phase`와 `outcome`으로 중단 위치를 바로 확인한다. 정상 +순서는 `lock`, `authenticate`, `pull`, `preserve`, `recreate`, `readiness`, +최종 `deployment outcome=success`다. 실패 후 자동 복구가 성공하면 workflow는 실패 +상태를 유지하면서 `deployment outcome=rolled_back`을 남긴다. 자동 복구도 실패하면 +`deployment outcome=rollback_failed operator_action=required`가 남는다. -승인 요청에는 다음 값을 채운다. +`deployment outcome=success`는 application과 loopback management listener의 readiness를 +뜻하며 Grafana ingest 성공을 가장하지 않는다. 그래서 마지막 로그에는 +`telemetry_verification=external`을 함께 남긴다. 승인자는 workflow 종료 뒤 5분 안에 기존 +application scrape-down alert와 production dashboard의 scrape, HTTP, recommendation panel을 +확인한다. Series가 없거나 scrape alert가 Pending/Firing이면 initial rollout을 완료로 표시하지 +않고 Alloy/credential 경계를 조사한다. Runner 안에 Grafana credential이나 중복 timer를 넣지 +않는다. -- 예상 중단: 기존 기동 17.742초를 기준으로 정상 20~40초의 단일 컨테이너 교체. readiness hard timeout은 90초이며 실패 시 이전 image 자동 복구까지 최악 약 180초 -- 변경 port: 컨테이너 `18080`, 호스트 `127.0.0.1:18080`만 추가; 공개 port 변화 없음 -- artifact: full commit SHA, run-attempt별 publish tag, build job이 확정한 `repository@sha256:`, 실행 전후 exact image ID -- management readiness: Bearer 인증된 `http://127.0.0.1:18080/actuator/prometheus`가 성공하고 Prometheus text를 반환 -- public smoke: `curl -fsS http://127.0.0.1:8080/api/v1/jobs` -- rollback: `/root/mapleland-observability-rollback.env`에 기록된 exact local image tag와 Compose/`.env`/script backup으로 아래 명령 실행 -- Grafana 확인: 아래 PromQL/LogQL 및 Production Overview panel - -GitHub `production` Environment required reviewer 승인을 받은 뒤에만 deploy job이 시작된다. 아래는 workflow가 host에서 preflight를 다시 통과한 직후 호출하는 명령과 사후 검증의 등가 표현이다. `image_ref`는 build job output에서 복사한 digest ref여야 하며 tag를 입력하면 script가 거부한다. - -`production` Environment는 required reviewer `mungmnb777`와 custom branch policy를 사용한다. 허용 branch는 `main`, `feature/observability-phase-1` 두 개뿐이며 이번 rollout 종료 후 feature branch policy만 제거한다. Environment 자체나 기존 deployment history는 삭제하지 않는다. +아래 명령은 workflow가 승인 뒤 host에서 수행하는 것과 같은 수동 표현이다. Break-glass +진단 외에는 workflow를 사용한다. ```bash -image_ref='ghcr.io/team-maple/mls-be/mapleland-api@sha256:' +image_ref='ghcr.io/team-maple/mls-be/mapleland-api@sha256:<64-hex-digest>' printf '%s\n' "$image_ref" | grep -Eq \ '^ghcr[.]io/team-maple/mls-be/mapleland-api@sha256:[0-9a-f]{64}$' sudo /opt/mapleland/update-api.sh "$image_ref" -expected_image_id="$(sudo docker image inspect --format '{{.Id}}' "$image_ref")" -actual_image_id="$(sudo docker inspect --format '{{.Image}}' mapleland-mapleland-api-1)" -test "$actual_image_id" = "$expected_image_id" -sudo docker inspect mapleland-mapleland-api-1 | - jq -e --arg expected "SERVICE_VERSION=${expected_commit}" \ - '.[0].Config.Env | index($expected) != null' >/dev/null ``` -`update-api.sh` 내부 readiness, legacy environment 부재 또는 smoke가 실패하면 배포마다 생성한 exact previous-image tag로 먼저 자동 복구되고 workflow는 실패한다. 첫 Observability 전환에서는 성공 전까지 보존한 base Compose와 `.env`를 사용하므로 이전 Loki4j image의 환경도 함께 복구된다. 전환 완료 뒤의 배포는 sanitized env와 observability override로 직전 image를 복구한다. 자동 rollback이 public smoke를 회복하지 못하거나 이후 behavioral 검증이 실패하면 다음 full rollback을 즉시 실행한다. +실패 진단은 원문을 외부로 복사하지 말고 host에서 확인한다. ```bash -sudo sh -eu -c ' - . /root/mapleland-observability-rollback.env - cp -a "$COMPOSE_BACKUP" /opt/mapleland/docker-compose.yml - cp -a "$ENV_BACKUP" /opt/mapleland/.env - cp -a "$UPDATE_SCRIPT_BACKUP" /opt/mapleland/update-api.sh - if test -f /opt/mapleland/docker-compose.observability.yml; then - mv /opt/mapleland/docker-compose.observability.yml \ - "/root/docker-compose.observability.yml.disabled-${EXPECTED_COMMIT}" - fi - docker image tag "$ROLLBACK_TAG" "$ROLLBACK_COMPOSE_IMAGE" - expected_rollback_image_id="$(docker image inspect --format "{{.Id}}" "$ROLLBACK_TAG")" - cd /opt/mapleland - docker compose up -d --no-deps --force-recreate --pull never mapleland-api - actual_rollback_image_id="$(docker inspect --format "{{.Image}}" mapleland-mapleland-api-1)" - test "$actual_rollback_image_id" = "$expected_rollback_image_id" -' +failure_role='candidate' # 또는 rollback +sudo cat "/var/log/mapleland-deploy/last-${failure_role}-failure/metadata" +sudo jq . "/var/log/mapleland-deploy/last-${failure_role}-failure/state.json" +sudo less "/var/log/mapleland-deploy/last-${failure_role}-failure/container.log" ``` +자동 rollback은 배포 직전에 로그로 남긴 `rollback_tag`와 exact previous image ID를 +사용한다. `rolled_back`이면 추가 재생성을 하지 말고 원인을 수정한 뒤 새 workflow를 +실행한다. `rollback_failed`일 때만 로그의 rollback tag를 사용해 같은 base/override +Compose 계약으로 복구하고 공개·management smoke를 다시 확인한다. + +```bash +set -euo pipefail +cd /opt/mapleland +rollback_tag='' +rollback_service_version='' +expected_image_id="$(sudo docker image inspect "$rollback_tag" --format '{{.Id}}')" +compose_image="$(sudo docker compose --env-file .env \ + -f docker-compose.yml -f docker-compose.observability.yml \ + config --images mapleland-api)" +test -n "$compose_image" +test "${compose_image#ghcr.io/team-maple/mls-be/mapleland-api:}" != "$compose_image" +printf '%s\n' "$rollback_service_version" | grep -Eq '^[0-9a-f]{40}$' +sudo docker image tag "$rollback_tag" "$compose_image" +sudo env SERVICE_VERSION="$rollback_service_version" docker compose --env-file .env \ + -f docker-compose.yml -f docker-compose.observability.yml \ + up -d --no-deps --force-recreate --pull never mapleland-api +container_id="$(sudo docker compose --env-file .env \ + -f docker-compose.yml -f docker-compose.observability.yml ps -q mapleland-api)" +test "$(sudo docker inspect "$container_id" --format '{{.Image}}')" = "$expected_image_id" +test "$(sudo docker inspect "$container_id" --format '{{.State.Status}}')" = running +test "$(sudo docker inspect "$container_id" --format '{{.RestartCount}}')" = 0 +health="$(sudo docker inspect "$container_id" \ + --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}')" +test "$health" = none || test "$health" = healthy +sudo docker inspect "$container_id" --format '{{range .Config.Env}}{{println .}}{{end}}' | + grep -Fx "SERVICE_VERSION=$rollback_service_version" +test "$(curl --silent --output /dev/null --max-time 5 --write-out '%{http_code}' \ + http://127.0.0.1:8080/api/v1/jobs)" = 200 +test "$(curl --silent --output /dev/null --max-time 5 --write-out '%{http_code}' \ + http://127.0.0.1:18080/actuator/info)" = 200 +management_token_line="$(sudo grep -E '^MANAGEMENT_SCRAPE_TOKEN=' .env)" +test "${management_token_line#MANAGEMENT_SCRAPE_TOKEN=}" != "$management_token_line" +test "$management_token_line" != *$'\n'* +management_scrape_token=${management_token_line#*=} +test "${#management_scrape_token}" -ge 32 +escaped_token=${management_scrape_token//\\/\\\\} +escaped_token=${escaped_token//\"/\\\"} +prometheus_status="$(printf 'header = "Authorization: Bearer %s"\n' "$escaped_token" | + curl --config - --silent --output /dev/null --max-time 5 \ + --write-out '%{http_code}' http://127.0.0.1:18080/actuator/prometheus)" +test "$prometheus_status" = 200 +management_scrape_token='' +management_token_line='' +escaped_token='' +``` + +Runner는 incident 직전 image를 확실히 남기기 위해 rollback tag를 자동 삭제하지 않는다. +따라서 정기 운영 점검에서 host disk alert와 `docker system df`를 확인하고, 현재 container와 +최근 승인 rollback image를 제외한 오래된 image 정리는 별도 owner 승인 maintenance로 +수행한다. Routine deploy 안에서는 `docker image prune`을 실행하지 않는다. + +승인 기록에는 full commit SHA, workflow run URL, manifest digest, 실행 전후 image ID, +결과와 rollback 여부를 남긴다. Grafana에서는 같은 시간대의 recommendation route +request rate, p95 latency, error/empty outcome과 engine 상태를 확인한다. 운영 traffic을 +이용한 부하 테스트는 수행하지 않는다. + ## 자동 검증 다음 두 test는 stack trace 정책의 서로 다른 경계를 고정한다. `EcsStructuredLoggingTest`는 `SafeExceptionLog`의 원본 exception message·식별자·인증정보 redaction, cause/root application frame 보존, 16 KiB 상한과 한 event당 한 JSON line을 검증한다. `ProdFileLoggingForkIntegrationTest`는 별도 JVM의 실제 `prod` profile과 실제 rolling file appender에서도 동일한 계약이 유지되고 모든 물리 line이 유효한 ECS JSON인지 검증한다. @@ -786,6 +919,7 @@ increase(loki_process_custom_ecs_log_lines_total{job="alloy"}[30m]) - dashboard UID/title: `mapleland-production-overview` / `Mapleland / Production Overview` - datasource: `grafanacloud-prom`, `grafanacloud-logs` - source: `deploy/observability/grafana/` +- recommendation row: `http.server.requests` 기반 v1/v2 전체 request rate와 p95/HTTP error, custom empty/unavailable outcome, 요청이 관찰된 engine, 평균 result count. 새 alert threshold는 baseline 없이 추가하지 않는다. Alert 정책: diff --git a/docs/recommendation-evidence-rollout.md b/docs/recommendation-evidence-rollout.md new file mode 100644 index 0000000..7fbc19c --- /dev/null +++ b/docs/recommendation-evidence-rollout.md @@ -0,0 +1,325 @@ +# MySQL 근거 기반 사냥터 추천 롤아웃 가이드 + +이 문서는 Issue [#34](https://github.com/Team-Maple/MLS-BE/issues/34)의 구현을 검증하고 단계적으로 운영에 적용하기 위한 개발자·운영자용 체크리스트다. 공개 API 계약과 운영 DB read path를 함께 변경하는 고위험 작업이므로 코드 리뷰와 위험 리뷰를 모두 통과해야 한다. 이 문서 작성 시점에는 구현 branch를 merge하거나 운영에 배포하지 않았으며, 아래 단계는 owner의 명시적 승인 없이는 실행하지 않는다. + +## 관련 작업과 범위 + +- MLS-BE 구현 Issue: [Team-Maple/MLS-BE#34](https://github.com/Team-Maple/MLS-BE/issues/34) +- upstream comparator 오류: [mungmnb777/mapleland#130](https://github.com/mungmnb777/mapleland/issues/130) +- mapledb scoring index: [mungmnb777/mapleland#131](https://github.com/mungmnb777/mapleland/issues/131), [draft PR #132](https://github.com/mungmnb777/mapleland/pull/132) +- 최초 운영 기본값: `RECOMMENDATION_V1_ENGINE=AURA` +- v2 엔진: MySQL 고정, 최초 운영 노출은 `RECOMMENDATION_V2_ENABLED=false` +- scorer/transaction timeout: `RECOMMENDATION_QUERY_TIMEOUT_SECONDS=10` (허용 1~60초) +- 기본 `limit`: 5, 허용 범위: 1~20, 최종 정렬 뒤 적용 + +MLS-BE는 mapleland의 HTTP test endpoint를 호출하지 않는다. 같은 MySQL의 승인된 추천 테이블을 query-only로 직접 읽고 MLS-BE 안에서 점수를 계산한다. importer, crawler, admin UI/API, alias writer, resolve-pending, FAMILY 관리 기능은 범위 밖이다. FAMILY alias fan-out은 reviewed claim에 이미 materialize되어 있으므로 요청 시 alias를 다시 해소하지 않는다. AuraDB dependency, 설정, keep-alive, secret 제거도 안정화 이후 별도 Issue/PR에서 처리한다. + +## upstream 근거와 adaptation + +승인된 기준 구현은 mapleland commit `bfa2af8e9135b53b51a0891a9d5187a21b74d2af`다. 조사한 upstream 작업 checkout의 HEAD는 `79f7a036d879e7ddd6254dcedab86d89f73a438e`로 기준보다 6 commits 앞서 있었지만, 이번 작업에서 참조한 추천 service/port/adapter/domain/test, ADR, admin schema에는 두 commit 사이의 차이가 없었다. 별도 schema index 작업은 최신 `main` `353dc4a98ecaff4e9a1e211369b744518f92cac6`를 기준으로 추적한다. + +MLS-BE adaptation은 다음 차이를 의도적으로 둔다. + +- mapleland의 hexagonal package를 복제하지 않고 MLS-BE의 기존 `application`/`domain`/`repository` 계층과 DTO `toDto` 정적 팩터리 관례를 따른다. +- 엔진 교체 경계는 `MapRecommendationRepository`와 `MapRecommendationEngineRouter`로 유지하되 별도 `port`/`adapter` package는 만들지 않는다. +- Job lineage를 MySQL 8 recursive CTE 한 번으로 읽는다. +- claim별 patch count, 조상별 claim, claim별 reason 쿼리를 하나의 scoring 쿼리로 합친다. +- map과 로그인 사용자의 bookmark는 각각 bulk 조회한다. +- upstream의 고정 `.limit(3)`을 가져오지 않고 API `limit` 계약을 적용한다. +- upstream의 chained `.reversed()` comparator를 가져오지 않고 각 정렬 키의 방향을 명시한다. +- v1과 v2 DTO를 분리해 v1 strict decoder 계약을 보존한다. +- 요청 단위 Aura/MySQL dual-read와 MySQL 오류 시 자동 Aura fallback을 하지 않는다. + +## 점수 계약 + +각 deduplicated evidence `i`의 기여도는 다음과 같다. + +```text +levelMatch_i = requestedLevel이 유효 레벨 범위 안이면 1, 아니면 0 +freshnessWeight_i = round(max(0.1, 1.0 - 0.05 * 이후 PATCH_NOTE 수), 3) +polaritySign_i = positive이면 +1, negative이면 -1 +contribution_i = polaritySign_i * freshnessWeight_i * levelMatch_i +mapScore = 같은 mapId에 속한 contribution_i의 합 +``` + +유효 레벨 범위는 양쪽 bound가 있으면 inclusive `[levelMin, levelMax]`다. `levelMin`만 있으면 `levelMax = levelMin + 10`, `levelMax`만 있으면 `levelMin = levelMax - 10`으로 본다. 양쪽이 모두 없으면 점수화하지 않는다. + +다음 예시는 domain test와 MySQL 8 integration fixture 양쪽에 고정한다. + +```text +positive, 이후 patch 0회: +1.00 +positive, 이후 patch 2회: +0.90 +negative, 이후 patch 1회: -0.95 +최종 mapScore: 0.95 +``` + +점수에 `confidence_score`, 조회 수, 좋아요/싫어요, 댓글 수, 요청 Job과 조상 Job 간 별도 가중치, AuraDB의 `levelHits`/`jobHits`, 임의 정규화를 추가하지 않는다. 반환하는 `score`는 선택 엔진의 실제 점수이며, MySQL 선택 시 위 evidence net score다. + +### 조회와 후보 불변식 + +- `review_status = 'APPROVED'`인 근거만 사용한다. +- 요청 Job 자신과 조상 Job의 근거만 사용하고 자손·무관 Job을 제외한다. +- 정확한 요청 Job과 조상 Job의 contribution 가중치는 같다. +- 같은 `extracted_claim_id + final_map_id`가 lineage 여러 단계에 있으면 한 번만 합산한다. +- 같은 extracted claim이라도 서로 다른 `final_map_id`는 별도 후보로 유지한다. +- positive와 negative를 모두 합산한다. +- 레벨이 일치하는 positive 근거가 하나 이상이고 최종 `mapScore > 0`인 맵만 반환한다. +- 최종 점수는 유한한 `double`이어야 하며 NaN/Infinity이면 실패 처리한다. + +최종 정렬은 다음 키를 순서대로 적용한다. + +1. `mapScore` 내림차순 +2. `freshnessSum` 내림차순. 여기서 `freshnessSum`은 합산된 contribution 절댓값의 합이다. +3. 가장 높은 contribution을 만든 근거의 `publishedAt` 내림차순 +4. `mapId` 오름차순 + +동일한 최고 contribution을 만든 근거가 여러 개면 더 최신 `publishedAt`을 대표 근거로 선택한다. 높은 `mapScore`가 먼저 오는 regression test로 upstream Issue #130의 comparator 오류가 재발하지 않게 한다. + +## 추천 이유 계약 + +reason facet도 해당 evidence의 signed contribution을 같은 방식으로 합산한다. 축별 누적 weight가 0보다 큰 facet 중 가장 큰 하나만 반환하고, 동률이면 아래 선언 순서의 priority를 사용한다. + +| axis | 허용 value와 priority | +| --- | --- | +| `reward` | `xp`, `meso`, `loot` | +| `play_style` | `solo`, `party`, `party_quest` | +| `operability` | `fatigue`, `mobility`, `budget` | + +응답 순서는 `reward`, `play_style`, `operability`다. reason text, excerpt, claim 원문, 작성자 정보는 SQL select와 공개 DTO에 포함하지 않는다. + +## 공개 API 계약 + +### v1 + +`GET /api/v1/maps/recommendations?level={1..200}&jobId={id}&limit={1..20}` + +outer wrapper, validation, status/error 계약, 익명·로그인 접근 정책을 유지한다. item의 JSON key는 아래 다섯 개뿐이며 `reasons`, `facets`, `hasRecommendation` 같은 additive field도 허용하지 않는다. 추천이 없으면 `data: []`다. + +```json +{ + "success": true, + "code": "SUCCESS", + "message": null, + "data": [ + { + "mapId": 100000000, + "score": 0.95, + "iconUrl": "https://...", + "nameKr": "헤네시스", + "bookmarkId": null + } + ] +} +``` + +`RECOMMENDATION_V1_ENGINE=AURA|MYSQL`로 선택한다. 최초 production 배포에서는 `AURA`를 유지한다. owner가 DB preflight와 v2 결과를 승인해 `MYSQL`로 바꾼 뒤에는 `score` 의미가 Aura hit score에서 evidence net score로 바뀌지만 JSON shape은 그대로다. + +### v2 + +`GET /api/v2/maps/recommendations?level={1..200}&jobId={id}&limit={1..20}` + +parameter, limit, 선택적 인증, bookmark, outer wrapper는 v1과 같다. 별도 v2 DTO가 `reasons`만 추가하며 이 값은 항상 배열이고 이유가 없으면 `[]`다. + +v2는 MySQL 고정이므로 별도 kill switch인 `RECOMMENDATION_V2_ENABLED`를 둔다. production 기본값은 `false`이며 topology/schema/실행 계획과 외부 rate-limit gate를 확인한 뒤 owner 승인으로만 `true`로 전환한다. 비활성 상태는 DB를 읽지 않고 기존 recommendation unavailable 503을 반환한다. + +```json +{ + "success": true, + "code": "SUCCESS", + "message": null, + "data": [ + { + "mapId": 100000000, + "score": 0.95, + "iconUrl": "https://...", + "nameKr": "헤네시스", + "bookmarkId": null, + "reasons": [ + { + "axis": "reward", + "value": "xp" + } + ] + } + ] +} +``` + +존재하지 않는 Job은 기존 not-found 계약을 사용한다. 선택 엔진이나 추천 schema를 사용할 수 없을 때 애플리케이션 시작 자체를 실패시키지 않고 해당 endpoint에서 기존 `MAP_RECOMMENDATION_UNAVAILABLE` 계열 503을 반환한다. + +## DB read-only preflight 증거 + +2026-07-16에 접근 가능한 mapleland DB를 read-only transaction으로 확인한 결과다. credential, endpoint, 원본 환경 파일 값은 기록하지 않았다. + +| 항목 | 확인 결과 | +| --- | --- | +| MySQL | 8.0.42 Community | +| schema | `mapledb` | +| 추천/Job/Map table과 필수 column | 존재 | +| `APPROVED` reviewed claim | 3,010 rows | +| 전체 reason | 11,955 rows | +| `APPROVED` claim에 연결된 reason | 4,817 rows | +| `PATCH_NOTE` | 95 rows | +| canonical map orphan | 0 | +| canonical job orphan | 0 | +| Job parent cycle | 0 | +| 관찰된 최대 Job lineage depth | 3 | +| 중복 `extracted_claim_id + final_map_id` group/surplus | 0 / 0 | +| 양쪽 level bound가 모두 null인 `APPROVED` claim | 0 | + +현재 index가 없는 상태의 `EXPLAIN`에서는 `recommendation_reviewed_claims`가 `ALL`, key 없음, 약 7,649 rows였고 `alrim`이 `ALL`, key 없음, 약 464 rows였다. 이 결과가 schema Issue #131의 근거다. + +### 운영 topology blocker + +mapleland가 접속한 `mapledb`에 MLS canonical table과 추천 table이 함께 있다는 사실은 확인했다. 그러나 현재 접근 권한으로 MLS-BE production의 `${DB_URL}`이 같은 MySQL endpoint와 schema를 가리키는지는 독립적으로 입증하지 못했다. 로컬 runtime credential이 없었고 manual OCI SSH identity도 사용할 수 없었다. + +따라서 다음을 secret-safe/read-only 방식으로 확인하기 전에는 v2 운영 검증과 v1 MySQL 전환을 진행하지 않는다. + +- MLS-BE와 mapleland의 DB host/port/schema가 동일한지 값을 출력하지 않고 equality만 확인 +- 운영 연결 사용자에게 필수 table/column과 필요한 read 권한이 있는지 확인 +- 위 row count, orphan, cycle, duplicate 검사를 운영 endpoint에서 재실행 +- 실제 scoring query의 `EXPLAIN FORMAT=JSON`과 선택 index 확인 +- `APPROVED` polarity 및 reason axis/value allowlist 위반이 0인지 확인 + +topology가 다르면 credential, cross-schema 권한, 복제 table을 임의로 만들지 않고 blocker로 owner에게 보고한다. + +## index와 schema ownership + +필요한 index는 mapledb owner인 mapleland의 Issue #131에서 forward/rollback/preflight SQL로 관리한다. + +```sql +CREATE INDEX idx_recommendation_reviewed_claims_scoring + ON recommendation_reviewed_claims (review_status, final_job_id); + +CREATE INDEX idx_alrim_type_date + ON alrim (type, date); +``` + +Testcontainers fixture에는 두 index를 넣어 실행 계획에서 사용 가능성을 검증했다. `EXPLAIN FORMAT=JSON`은 reviewed-claim index를 possible key로 인식했고 patch index는 실제 선택했다. 다만 synthetic fixture가 100% `APPROVED`이고 모든 row가 요청 lineage에 일치하도록 구성돼 reviewed claim은 full scan이 더 저렴하다고 판단됐다. 이 결과를 production index 선택 증거로 확대 해석하지 않는다. 실제 production DDL은 실행하지 않았고 Hibernate `ddl-auto`에도 맡기지 않는다. 별도 schema PR 검토, owner 승인, production preflight와 change window를 거친 뒤 forward SQL을 실행한다. rollback SQL은 index 이름·존재 여부를 preflight한 뒤에만 사용한다. + +schema PR preflight는 reason table/column/index, polarity·facet allowlist, MLS-BE의 consolidated lineage/dedup/patch/reason query 전체 `EXPLAIN FORMAT=TREE`를 검사하도록 보강했다. 이 EXPLAIN이 성공하면 해당 session의 필수 table SELECT 권한도 함께 확인된다. 다만 MLS-BE production credential로 같은 결과를 얻는 gate는 topology blocker가 해소될 때까지 남아 있다. + +## query budget과 성능 증거 + +한 요청의 고정 query path는 다음과 같다. + +| 단계 | 최대 round trip | 조건 | +| --- | ---: | --- | +| Job 존재 확인 | 1 | 모든 정상 validation 요청 | +| lineage/evidence/patch/reason scoring | 1 | MySQL 엔진 선택 시 | +| canonical map bulk 조회 | 1 | 후보가 있을 때 | +| bookmark bulk 조회 | 1 | 후보가 있고 로그인했을 때 | + +따라서 MySQL 추천은 후보·evidence 수와 무관하게 로그인 결과 요청 최대 4회, 익명 결과 요청 최대 3회다. 빈 후보는 enrichment 쿼리를 생략한다. repository integration query counter는 scorer가 요청당 정확히 1회임을 cold 1회와 warm 40회 모두 확인했고, enrichment unit test는 map과 bookmark bulk 조회가 각각 한 번만 호출되고 scorer 정렬 순서를 보존함을 확인했다. 단일 proxy가 controller 전체 DB 호출을 합산하는 end-to-end 계측은 아직 별도 증거가 없으므로 위 전체 횟수는 각 검증된 경계를 합친 query budget이다. + +MySQL scoring JDBC statement와 이를 감싸는 read-only transaction에는 같은 configurable 10초 timeout을 적용한다. Testcontainers의 `SELECT SLEEP(3)`를 1초 statement timeout으로 취소하는 계약 테스트를 포함한다. 이 값은 connection-pool 획득 전 대기까지 포함한 endpoint 전체 wall-clock 제한은 아니므로, 운영 전환 전에 기존 Hikari pool saturation과 HTTP latency도 함께 확인한다. 기존 Aura v1의 relational transaction에는 이 새 timeout을 적용하지 않아 slow Aura 호출 뒤 enrichment가 임의로 503이 되는 호환 회귀를 막는다. v2 default-off kill switch는 긴급 MySQL traffic drain에 사용한다. reverse proxy의 recommendation 전용 rate-limit 설정은 현재 저장소/접근 범위에서 입증하지 못했으므로 이를 확인하기 전에는 v2를 운영 공개하지 않는다. 측정 없이 cache를 추가하지 않았다. + +로컬 Testcontainers `mysql:8.4`의 대표 fixture 측정값은 다음과 같다. + +| evidence rows | candidate maps | final results | scorer queries/request | cold | warm p50 | warm p95 | warm p99 | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 3,000 | 100 | 20 | 1 | 90.896 ms | 33.618 ms | 60.479 ms | 70.781 ms | + +이 수치는 개발 machine의 단일 container 측정이며 production SLO나 alert threshold가 아니다. MySQL 8 recursive CTE/window 실행, 결과 수와 고정 query count를 검증한 characterization evidence다. 같은 fixture의 tabular plan summary는 다음과 같았다. + +```text +:ALL:null, reason:ref:uk_recommendation_reviewed_claim_reasons, +:ref:, :ALL:null, +patch:ref:idx_alrim_type_date, rc:ALL:null, ec:eq_ref:PRIMARY, +canonical_map:eq_ref:PRIMARY, :ref:, +j:const:PRIMARY, lineage:ALL:null, parent:ALL:null +``` + +`idx_alrim_type_date`는 선택됐지만 `idx_recommendation_reviewed_claims_scoring`은 JSON의 possible key로만 나타났고, 위 fixture 특성 때문에 optimizer가 `rc:ALL`을 선택했다. production에서 DDL을 승인·적용한 뒤 실제 분포로 다시 실행한 `EXPLAIN`은 아직 없다. 운영 latency와 Aura baseline 비교는 production topology 확인과 안전한 v2 smoke 뒤 기존 `http.server.requests` 표본으로 수행한다. 표본이 부족하면 부족하다고 기록하며 운영 traffic을 부하 테스트에 사용하지 않는다. + +## 관측성과 민감정보 경계 + +기존 `http.server.requests`의 route-template metric으로 v1/v2 request rate, latency, HTTP error를 본다. 같은 latency를 재는 custom timer는 추가하지 않는다. custom metric은 추천 outcome/engine/api version과 result count만 기록한다. + +- request counter label: `engine`, `api_version`, `outcome` +- result summary label: `engine`, `api_version` +- 금지 label: jobId, level, mapId, member ID, raw URI/query +- 허용 structured log field: `event.action`, `event.outcome`, recommendation engine, API version, duration, result count +- 금지 log: member ID, 요청 level/job, query string, claim/reason 원문, credential +- 예외는 기존 `SafeExceptionLog` 정책을 사용한다. + +versioned dashboard JSON과 live Grafana dashboard UID `mapleland-production-overview`는 recommendation row의 실제 v1/v2 HTTP request rate, 기존 HTTP p95, HTTP error와 custom empty/unavailable outcome, 관찰된 engine, 평균 result count 쿼리를 포함하도록 함께 갱신했다. total request/error에는 `http.server.requests` route/status를 사용하므로 config parsing, validation, 404도 빠지지 않는다. custom counter는 scorer 처리 outcome 의미로만 사용한다. panel 24개를 유지하며 기존 UID와 `Dashboards` folder를 create-or-update했다. live version history에서 source와 같은 version `3`이 `2026-07-16 23:21:43 KST`의 Latest이고 version 2/1이 restore 가능한 상태임을 확인했다. 애플리케이션/Alloy 변경이 아직 배포되지 않았고 route traffic 표본도 없으므로 pre-deployment panel 렌더링은 live metric series 검증이 아니다. folder/datasource/contact point/notification policy/unrelated alert는 변경하지 않았고 baseline 없이 alert threshold를 추가하지 않았다. + +## 초기 image 배포 게이트 + +초기 배포는 v1 `AURA`, v2 `false`, query timeout `10`의 안전값으로 MySQL traffic을 만들지 +않는다. 다음 항목이 모두 충족돼야 이 initial image 배포 승인을 요청할 수 있다. + +- [x] Issue #34와 draft PR #35에 v1/v2 계약, 점수 공식, DB 전제, rollout/rollback이 기록됨 +- [x] legacy OCI workflow 복원 변경을 검토하고 `c49ee3255afb4ddfa0168ce91783fff368864a4d`의 구조 및 concurrency/least-privilege adaptation을 재검증함 +- [ ] 최신 `./gradlew clean test` 전체 실행이 성공함. 현재 외부 실사이트 crawler 3건의 timeout 재확인과 CI가 남아 있음 +- [ ] CI가 성공하고 unresolved required review가 없음 +- [ ] legacy workflow가 사용하는 repository secret `FIREBASE_KEY`, `TS_OAUTH_CLIENT_ID`, `TS_OAUTH_SECRET`, `ORACLE_SSH_KEY`의 존재와 접근 범위를 owner가 확인함 +- [ ] Firebase key가 image layer에 포함되는 기존 잔여 위험을 owner가 명시적으로 수용하거나 runtime secret mount 전환과 key rotation을 완료함 +- [ ] 삭제 대상 legacy EC2 경로가 DNS/LB/failover/DR에서 쓰이지 않음을 운영 inventory로 확인하고 HOST/USERNAME/KEY/PORT/GHCR credential revoke·rotation·secret 제거를 완료함 +- [x] host의 기존 `/opt/mapleland/update-api.sh`가 legacy no-arg 계약이고 CI forced command가 이를 `sudo -n`으로 실행하도록 owner 승인 maintenance와 read-only 검증을 완료함 +- [x] host `update-api.sh`가 base와 observability override를 함께 사용하며 deployed image에서 공개 API와 인증 management scrape가 200이고 restart count 0임을 확인함 +- [x] exact current image와 base+observability override를 사용하는 수동 rollback 절차를 운영 복구에서 검증하고 checkpoint에 기록함 +- [x] Grafana versioned JSON과 live dashboard UID/version이 일치하고 새 panel query가 오류 없이 완료됨. 실제 series 검증은 배포 후 smoke gate로 남음 +- [ ] merge와 운영 배포에 대한 owner의 명시적 승인이 있음 + +## MySQL 활성화 게이트 + +다음 항목은 initial AURA/default-off image merge를 막지 않지만, v2를 켜거나 v1을 MySQL로 +전환하기 전에는 모두 P1 blocker로 정의했다. 2026-07-17 owner 승인 운영 maintenance에서 v2가 +먼저 활성화됐으므로, 미완료 항목은 현재 공개 v2의 잔여 운영 위험이자 v1 MySQL 전환 blocker다. + +- [ ] mapleland schema PR의 forward/rollback/preflight SQL과 `EXPLAIN` 근거가 검토됨 +- [ ] MLS-BE production DB topology와 SELECT 권한 blocker가 해소됨 +- [ ] production reverse proxy의 recommendation route rate-limit이 확인되고 v2 공개량이 승인됨 +- [ ] Hikari connection 획득 대기 상한과 pool saturation 대응을 결정하고 운영 metric으로 확인함 +- [ ] production DDL이 필요하면 owner가 별도로 승인하고 적용 결과·rollback 기준을 기록함 +- [ ] initial image 뒤 Grafana scrape와 recommendation metric live series를 확인함 + +최소 로컬 검증 명령은 다음과 같다. + +```bash +./gradlew clean test +./gradlew bootJar +jq empty deploy/observability/grafana/mapleland-production-overview.json +jq empty deploy/observability/grafana/alert-rules.json +bash deploy/observability/alloy/validate.sh +``` + +2026-07-16 최종 로컬 트리에서 `./gradlew clean test`는 88 tests, failure/error/skip 0으로 성공했고 `./gradlew bootJar`, 두 Grafana JSON `jq empty`, Alloy 공식 container validation, recommendation asset test, shell syntax·executable 검증도 성공했다. + +2026-07-17 legacy OCI workflow 복원 전 `bootJar`, 추천/설정 바인딩 테스트군, recommendation asset test, 두 Grafana JSON과 Alloy validation은 성공했다. `./gradlew clean test`는 91개 중 88개가 통과했고, 운영 외부 사이트를 직접 호출하는 기존 `NoticeApiTest` 세 건만 `SocketTimeoutException`으로 실패했다. 이 테스트를 우회하거나 배포 변경에 unrelated한 crawler 코드를 수정하지 않으며 GitHub CI에서 재확인한다. + +## 권장 rollout + +1. 구현 PR을 merge하지 않은 상태에서 전체 test, 독립 코드 리뷰, 위험 리뷰, CI를 완료한다. +2. Legacy workflow가 사용하는 repository secret, host의 기존 `/opt/mapleland/update-api.sh`, exact previous image와 수동 rollback 명령을 owner가 read-only로 확인한다. +3. Owner가 workflow dispatch를 명시적으로 승인하면 `latest-arm64` image를 build/publish하고 기존 Tailscale SSH 경로로 host script를 실행한다. 이 단계에서 v1은 기존 Aura 동작을 유지하고 v2는 DB를 읽지 않는 503 kill-switch 상태다. +4. Workflow 자체에는 digest pinning, host-key fingerprint와 자동 smoke/rollback 계약이 없으므로 Actions 종료 직후 공개 `/api/v1/jobs`, 기존 dashboard와 application scrape를 수동 확인한다. +5. 기존 dashboard에서 application scrape와 recommendation panel의 배포 후 상태를 확인한다. 이 확인 전에는 initial rollout을 완료로 표시하지 않는다. +6. mapleland schema PR, topology equality, 운영 SELECT 권한, full query plan, reverse-proxy rate-limit, Hikari connection 획득 대기 상한과 pool saturation 대응을 검토한다. DDL이 필요하면 owner 승인 뒤 forward SQL만 실행하고 결과를 기록한다. +7. MySQL 활성화 게이트를 모두 통과한 뒤 owner 승인 host maintenance로 `.env` backup을 남기고 `RECOMMENDATION_V2_ENABLED=true`를 원자적으로 적용한다. Compose rendering을 확인한 뒤 같은 workflow로 재생성해야 running container에 반영된다. v2 익명/로그인 smoke를 소량 수행해 exact contract, bookmark, score/reasons, empty, invalid input, missing Job, 503을 확인한다. 운영 부하 테스트는 하지 않는다. +8. 기존 dashboard UID에서 v1/v2 rate, p95, error/empty, engine, result count를 관찰한다. 충분한 표본이 생길 때까지 임의 threshold를 만들지 않는다. +9. v2 결과, latency와 오류율을 owner가 승인한 뒤 같은 backup·원자 교체·Compose rendering·workflow 재생성 절차로 `RECOMMENDATION_V1_ENGINE=MYSQL`을 적용하고 v1 exact five-key contract와 evidence score semantic change를 다시 smoke한다. +10. 안정화 기간 뒤 Aura dependency/config/keep-alive/secret 제거를 별도 Issue/PR로 진행한다. + +## rollback + +문제가 생기면 request 단위 fallback을 추가하지 않는다. 먼저 root-only 운영 설정의 승인된 backup을 복원하거나 `RECOMMENDATION_V1_ENGINE=AURA`, `RECOMMENDATION_V2_ENABLED=false`를 원자적으로 적용하고 MySQL recommendation traffic을 drain한다. `.env` 편집만으로는 실행 중 process가 바뀌지 않는다. Legacy workflow에는 자동 rollback과 이전 digest 보존 계약이 없으므로 구현 image 회귀 시 host가 보유한 승인된 이전 image를 owner의 break-glass 절차로 재실행해야 한다. 정확한 이전 image가 확인되지 않으면 새 workflow를 반복 실행하지 않는다. v2를 Aura로 의미 변경하지 않는다. + +index rollback은 애플리케이션 rollback과 분리한다. scoring traffic drain과 실행 계획을 확인하고 owner가 승인한 경우에만 schema Issue #131의 rollback SQL을 사용한다. topology 차이를 credential 추가, cross-schema grant, 임시 복제 table로 우회하지 않는다. + +## 현재 미완료 또는 남은 위험 + +- MLS-BE production DB와 mapleland DB의 endpoint/schema equality가 확인되지 않았다. +- production reverse proxy의 recommendation 전용 rate-limit은 확인되지 않았지만 v2는 현재 활성화 상태다. +- Hikari connection 획득 대기 상한과 pool saturation 대응은 결정되지 않아 statement timeout만으로 endpoint 전체 wall-clock을 보장하지 않는다. +- deploy credential은 legacy workflow 요구대로 repository scope에 있으며 Environment 승인으로 격리되지 않는다. +- 삭제되는 legacy workflow의 SSH/GHCR repository secret revoke·제거도 아직 수행하지 않았다. +- Legacy workflow는 mutable `latest-arm64`, third-party SSH action과 host-local script에 의존하고 자동 rollback을 보장하지 않는다. +- Firebase service-account key는 기존 방식대로 image layer에 포함되므로 GHCR package read 권한을 key 접근 권한으로 취급해야 한다. Runtime secret mount 전환과 key rotation은 별도 보안 작업이다. +- production index DDL은 실행하지 않았다. +- live Grafana dashboard layout/version 갱신과 애플리케이션 배포는 완료했지만 recommendation metric의 live series 검증은 남아 있다. +- 로컬 전체 test는 기존 외부 실사이트 crawler 3건의 timeout 때문에 91개 중 88개 성공 상태이며 CI/required review의 최종 결과도 확인해야 한다. 추천/설정 바인딩 테스트군과 패키징은 성공했다. +- Testcontainers 수치는 local characterization이며 production latency 표본이 아니다. +- 이 문서 작성 시점에는 feature branch image와 v2 MySQL 경로가 운영에 배포됐지만 구현 merge, v1 MySQL 설정 전환과 Aura 제거는 수행하지 않았다. diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 37f853b..c99f974 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-bin.zip +distributionSha256Sum=20f1b1176237254a6fc204d8434196fa11a4cfb387567519c61556e8710aed78 networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/src/main/java/com/maple/api/common/config/RecommendationConfig.java b/src/main/java/com/maple/api/common/config/RecommendationConfig.java new file mode 100644 index 0000000..da693da --- /dev/null +++ b/src/main/java/com/maple/api/common/config/RecommendationConfig.java @@ -0,0 +1,32 @@ +package com.maple.api.common.config; + +import com.maple.api.map.domain.RecommendationScoringService; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; + +import javax.sql.DataSource; + +@Configuration +@EnableConfigurationProperties(RecommendationProperties.class) +public class RecommendationConfig { + + @Bean + public RecommendationScoringService recommendationScoringService() { + return new RecommendationScoringService(); + } + + @Bean + @Qualifier("recommendationJdbcTemplate") + public NamedParameterJdbcTemplate recommendationJdbcTemplate( + DataSource dataSource, + RecommendationProperties properties + ) { + JdbcTemplate jdbcTemplate = new JdbcTemplate(dataSource); + jdbcTemplate.setQueryTimeout(properties.getQueryTimeoutSeconds()); + return new NamedParameterJdbcTemplate(jdbcTemplate); + } +} diff --git a/src/main/java/com/maple/api/common/config/RecommendationProperties.java b/src/main/java/com/maple/api/common/config/RecommendationProperties.java new file mode 100644 index 0000000..7327724 --- /dev/null +++ b/src/main/java/com/maple/api/common/config/RecommendationProperties.java @@ -0,0 +1,24 @@ +package com.maple.api.common.config; + +import com.maple.api.map.domain.RecommendationEngineType; +import lombok.Getter; +import lombok.Setter; +import jakarta.validation.constraints.Max; +import jakarta.validation.constraints.Min; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +@Getter +@Setter +@Validated +@ConfigurationProperties(prefix = "recommendation") +public class RecommendationProperties { + + private RecommendationEngineType v1Engine = RecommendationEngineType.AURA; + + private boolean v2Enabled = false; + + @Min(1) + @Max(60) + private int queryTimeoutSeconds = 10; +} diff --git a/src/main/java/com/maple/api/common/presentation/config/RecommendationOpenApiConfig.java b/src/main/java/com/maple/api/common/presentation/config/RecommendationOpenApiConfig.java new file mode 100644 index 0000000..f173189 --- /dev/null +++ b/src/main/java/com/maple/api/common/presentation/config/RecommendationOpenApiConfig.java @@ -0,0 +1,38 @@ +package com.maple.api.common.presentation.config; + +import io.swagger.v3.oas.models.Operation; +import io.swagger.v3.oas.models.PathItem; +import io.swagger.v3.oas.models.security.SecurityRequirement; +import org.springdoc.core.customizers.OpenApiCustomizer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.List; + +@Configuration +public class RecommendationOpenApiConfig { + + private static final String AUTHORIZATION_SCHEME = "Authorization"; + + @Bean + public OpenApiCustomizer recommendationOptionalAuthentication() { + return openApi -> { + setOptionalAuthentication(openApi.getPaths().get("/api/v1/maps/recommendations")); + setOptionalAuthentication(openApi.getPaths().get("/api/v2/maps/recommendations")); + }; + } + + private void setOptionalAuthentication(PathItem pathItem) { + if (pathItem == null) { + return; + } + Operation operation = pathItem.getGet(); + if (operation == null) { + return; + } + operation.setSecurity(List.of( + new SecurityRequirement(), + new SecurityRequirement().addList(AUTHORIZATION_SCHEME) + )); + } +} diff --git a/src/main/java/com/maple/api/common/presentation/config/SecurityConfig.java b/src/main/java/com/maple/api/common/presentation/config/SecurityConfig.java index 03c4e25..c48918f 100644 --- a/src/main/java/com/maple/api/common/presentation/config/SecurityConfig.java +++ b/src/main/java/com/maple/api/common/presentation/config/SecurityConfig.java @@ -57,6 +57,7 @@ protected SecurityFilterChain filterChain(HttpSecurity http) throws Exception { .requestMatchers("/api/v1/items/**").permitAll() .requestMatchers("/api/v1/monsters/**").permitAll() .requestMatchers("/api/v1/maps/**").permitAll() + .requestMatchers("/api/v2/maps/**").permitAll() .requestMatchers("/api/v1/npcs/**").permitAll() .requestMatchers("/api/v1/quests/**").permitAll() .requestMatchers("/api/v1/categories/**").permitAll() diff --git a/src/main/java/com/maple/api/map/application/MapRecommendationEngineRouter.java b/src/main/java/com/maple/api/map/application/MapRecommendationEngineRouter.java new file mode 100644 index 0000000..dcb3bd4 --- /dev/null +++ b/src/main/java/com/maple/api/map/application/MapRecommendationEngineRouter.java @@ -0,0 +1,32 @@ +package com.maple.api.map.application; + +import com.maple.api.map.domain.RecommendationEngineType; +import com.maple.api.map.repository.MapRecommendationRepository; +import org.springframework.stereotype.Component; + +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +@Component +public class MapRecommendationEngineRouter { + + private final Map repositories; + + public MapRecommendationEngineRouter(List repositories) { + EnumMap registered = + new EnumMap<>(RecommendationEngineType.class); + for (MapRecommendationRepository repository : repositories) { + MapRecommendationRepository previous = registered.put(repository.engineType(), repository); + if (previous != null) { + throw new IllegalStateException("Duplicate recommendation engine: " + repository.engineType()); + } + } + this.repositories = Map.copyOf(registered); + } + + public Optional find(RecommendationEngineType engineType) { + return Optional.ofNullable(repositories.get(engineType)); + } +} diff --git a/src/main/java/com/maple/api/map/application/MapRecommendationEnrichmentService.java b/src/main/java/com/maple/api/map/application/MapRecommendationEnrichmentService.java new file mode 100644 index 0000000..e78e094 --- /dev/null +++ b/src/main/java/com/maple/api/map/application/MapRecommendationEnrichmentService.java @@ -0,0 +1,60 @@ +package com.maple.api.map.application; + +import com.maple.api.bookmark.application.BookmarkFlagService; +import com.maple.api.bookmark.domain.BookmarkType; +import com.maple.api.map.application.dto.MapRecommendationResultDto; +import com.maple.api.map.domain.Map; +import com.maple.api.map.domain.RecommendationCandidate; +import com.maple.api.map.repository.MapRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.util.LinkedHashSet; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Service +@RequiredArgsConstructor +public class MapRecommendationEnrichmentService { + + private final MapRepository mapRepository; + private final BookmarkFlagService bookmarkFlagService; + + public List enrich( + String memberId, + List candidates + ) { + if (candidates.isEmpty()) { + return List.of(); + } + + List mapIds = candidates.stream() + .map(RecommendationCandidate::mapId) + .map(Math::toIntExact) + .collect(Collectors.collectingAndThen( + Collectors.toCollection(LinkedHashSet::new), + List::copyOf + )); + + var mapsById = mapRepository.findByMapIdIn(mapIds).stream() + .collect(Collectors.toMap(Map::getMapId, Function.identity())); + + var bookmarkIds = bookmarkFlagService.findBookmarkIds(memberId, BookmarkType.MAP, mapIds); + + return candidates.stream() + .map(candidate -> { + int mapId = Math.toIntExact(candidate.mapId()); + Map map = mapsById.get(mapId); + return new MapRecommendationResultDto( + mapId, + candidate.score(), + map != null ? map.getIconUrl() : null, + map != null ? map.getNameKr() : null, + bookmarkIds.get(mapId), + candidate.reasons() + ); + }) + .toList(); + } +} diff --git a/src/main/java/com/maple/api/map/application/MapRecommendationObservability.java b/src/main/java/com/maple/api/map/application/MapRecommendationObservability.java new file mode 100644 index 0000000..7a42a56 --- /dev/null +++ b/src/main/java/com/maple/api/map/application/MapRecommendationObservability.java @@ -0,0 +1,83 @@ +package com.maple.api.map.application; + +import com.maple.api.common.logging.SafeExceptionLog; +import com.maple.api.map.domain.RecommendationEngineType; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.DistributionSummary; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class MapRecommendationObservability { + + static final String REQUESTS_METRIC = "mapleland.recommendation.requests"; + static final String RESULTS_METRIC = "mapleland.recommendation.results"; + + private final MeterRegistry meterRegistry; + + public MapRecommendationObservability(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } + + public void completed( + RecommendationEngineType engine, + String apiVersion, + int resultCount, + long durationNanos + ) { + String outcome = resultCount == 0 ? "empty" : "success"; + counter(engine, apiVersion, outcome).increment(); + resultSummary(engine, apiVersion).record(resultCount); + + log.atInfo() + .addKeyValue("event.action", "recommendation.query") + .addKeyValue("event.outcome", "success") + .addKeyValue("mapleland.recommendation.engine", engine.metricValue()) + .addKeyValue("mapleland.api.version", apiVersion) + .addKeyValue("event.duration", durationNanos) + .addKeyValue("mapleland.result.count", resultCount) + .log("Recommendation query completed"); + } + + public void unavailable( + RecommendationEngineType engine, + String apiVersion, + long durationNanos, + Throwable exception + ) { + counter(engine, apiVersion, "unavailable").increment(); + + SafeExceptionLog.addException(log.atWarn(), exception) + .addKeyValue("event.action", "recommendation.query") + .addKeyValue("event.outcome", "failure") + .addKeyValue("mapleland.recommendation.engine", engine.metricValue()) + .addKeyValue("mapleland.api.version", apiVersion) + .addKeyValue("event.duration", durationNanos) + .addKeyValue("mapleland.result.count", 0) + .log("Recommendation query unavailable"); + } + + public void disabled(RecommendationEngineType engine, String apiVersion) { + counter(engine, apiVersion, "unavailable").increment(); + } + + private Counter counter(RecommendationEngineType engine, String apiVersion, String outcome) { + return Counter.builder(REQUESTS_METRIC) + .description("Recommendation endpoint outcomes by selected engine and API version") + .tag("engine", engine.metricValue()) + .tag("api_version", apiVersion) + .tag("outcome", outcome) + .register(meterRegistry); + } + + private DistributionSummary resultSummary(RecommendationEngineType engine, String apiVersion) { + return DistributionSummary.builder(RESULTS_METRIC) + .description("Number of recommendation items returned per successful request") + .baseUnit("recommendations") + .tag("engine", engine.metricValue()) + .tag("api_version", apiVersion) + .register(meterRegistry); + } +} diff --git a/src/main/java/com/maple/api/map/application/MapRecommendationQueryExecutor.java b/src/main/java/com/maple/api/map/application/MapRecommendationQueryExecutor.java new file mode 100644 index 0000000..0afe8a2 --- /dev/null +++ b/src/main/java/com/maple/api/map/application/MapRecommendationQueryExecutor.java @@ -0,0 +1,73 @@ +package com.maple.api.map.application; + +import com.maple.api.common.presentation.exception.ApiException; +import com.maple.api.map.application.dto.MapRecommendationResultDto; +import com.maple.api.job.exception.JobException; +import com.maple.api.job.repository.JobRepository; +import com.maple.api.common.config.RecommendationProperties; +import com.maple.api.map.domain.RecommendationCandidate; +import com.maple.api.map.domain.RecommendationEngineType; +import com.maple.api.map.repository.MapRecommendationRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; + +import java.util.List; +import java.util.Objects; + +/** + * Programmatic transaction boundary invoked from the non-transactional API service. Transaction + * start and datasource failures stay inside the caller's 503 boundary. The bounded timeout is + * applied only to MySQL evidence reads; Aura keeps the legacy no-timeout relational transaction. + */ +@Service +@RequiredArgsConstructor +public class MapRecommendationQueryExecutor { + + private final JobRepository jobRepository; + private final MapRecommendationEngineRouter engineRouter; + private final MapRecommendationEnrichmentService enrichmentService; + private final PlatformTransactionManager transactionManager; + private final RecommendationProperties recommendationProperties; + + public List execute( + String memberId, + int level, + int jobId, + int limit, + RecommendationEngineType engine + ) { + TransactionTemplate transaction = new TransactionTemplate(transactionManager); + transaction.setReadOnly(true); + if (engine == RecommendationEngineType.MYSQL) { + transaction.setTimeout(recommendationProperties.getQueryTimeoutSeconds()); + } + return Objects.requireNonNull(transaction.execute(status -> executeWithinTransaction( + memberId, + level, + jobId, + limit, + engine + ))); + } + + private List executeWithinTransaction( + String memberId, + int level, + int jobId, + int limit, + RecommendationEngineType engine + ) { + if (!jobRepository.existsById(jobId)) { + throw ApiException.of(JobException.JOB_NOT_FOUND); + } + + MapRecommendationRepository repository = engineRouter.find(engine) + .orElseThrow(() -> new IllegalStateException( + "Recommendation engine is not configured: " + engine + )); + List candidates = repository.findRecommendations(level, jobId, limit); + return enrichmentService.enrich(memberId, candidates); + } +} diff --git a/src/main/java/com/maple/api/map/application/MapRecommendationService.java b/src/main/java/com/maple/api/map/application/MapRecommendationService.java new file mode 100644 index 0000000..0622ef7 --- /dev/null +++ b/src/main/java/com/maple/api/map/application/MapRecommendationService.java @@ -0,0 +1,84 @@ +package com.maple.api.map.application; + +import com.maple.api.common.presentation.exception.ApiException; +import com.maple.api.map.application.dto.MapRecommendationDto; +import com.maple.api.map.application.dto.MapRecommendationResultDto; +import com.maple.api.map.application.dto.MapRecommendationV2Dto; +import com.maple.api.map.exception.MapException; +import com.maple.api.common.config.RecommendationProperties; +import com.maple.api.map.domain.RecommendationEngineType; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +@RequiredArgsConstructor +public class MapRecommendationService { + + static final int DEFAULT_LIMIT = 5; + + private final RecommendationProperties recommendationProperties; + private final MapRecommendationQueryExecutor queryExecutor; + private final MapRecommendationObservability observability; + + public List recommendV1( + String memberId, + int level, + int jobId, + Integer limit + ) { + RecommendationEngineType engine = recommendationProperties.getV1Engine(); + return recommend(memberId, level, jobId, limit, engine, "v1").stream() + .map(MapRecommendationDto::toDto) + .toList(); + } + + public List recommendV2( + String memberId, + int level, + int jobId, + Integer limit + ) { + if (!recommendationProperties.isV2Enabled()) { + observability.disabled(RecommendationEngineType.MYSQL, "v2"); + throw ApiException.of(MapException.MAP_RECOMMENDATION_UNAVAILABLE); + } + return recommend(memberId, level, jobId, limit, RecommendationEngineType.MYSQL, "v2").stream() + .map(MapRecommendationV2Dto::toDto) + .toList(); + } + + private List recommend( + String memberId, + int level, + int jobId, + Integer limit, + RecommendationEngineType engine, + String apiVersion + ) { + int effectiveLimit = limit == null ? DEFAULT_LIMIT : limit; + long startedAt = System.nanoTime(); + + try { + List enriched = queryExecutor.execute( + memberId, + level, + jobId, + effectiveLimit, + engine + ); + observability.completed(engine, apiVersion, enriched.size(), elapsedSince(startedAt)); + return enriched; + } catch (ApiException exception) { + throw exception; + } catch (RuntimeException exception) { + observability.unavailable(engine, apiVersion, elapsedSince(startedAt), exception); + throw ApiException.of(MapException.MAP_RECOMMENDATION_UNAVAILABLE, exception); + } + } + + private long elapsedSince(long startedAt) { + return Math.max(0L, System.nanoTime() - startedAt); + } +} diff --git a/src/main/java/com/maple/api/map/application/MapService.java b/src/main/java/com/maple/api/map/application/MapService.java index 5503db3..0045b46 100644 --- a/src/main/java/com/maple/api/map/application/MapService.java +++ b/src/main/java/com/maple/api/map/application/MapService.java @@ -3,8 +3,6 @@ import com.maple.api.bookmark.application.BookmarkFlagService; import com.maple.api.bookmark.domain.BookmarkType; import com.maple.api.common.presentation.exception.ApiException; -import com.maple.api.job.exception.JobException; -import com.maple.api.job.repository.JobRepository; import com.maple.api.map.application.dto.*; import com.maple.api.map.domain.Map; import com.maple.api.map.exception.MapException; @@ -17,9 +15,6 @@ import org.springframework.transaction.annotation.Transactional; import java.util.List; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; @Service @RequiredArgsConstructor @@ -31,8 +26,6 @@ public class MapService { private final MapMonsterQueryDslRepository mapMonsterQueryDslRepository; private final MapNpcQueryDslRepository mapNpcQueryDslRepository; private final BookmarkFlagService bookmarkFlagService; - private final Optional mapRecommendationRepository; - private final JobRepository jobRepository; @Transactional(readOnly = true) public Page searchMaps(String memberId, MapSearchRequestDto request, Pageable pageable) { @@ -74,45 +67,4 @@ public long countMapsByKeyword(String keyword) { return mapQueryDslRepository.countMapsByKeyword(keyword); } - @Transactional(readOnly = true) - public List recommendMaps(String memberId, int level, int jobId, Integer limit) { - validateJobExists(jobId); - - MapRecommendationRepository repository = mapRecommendationRepository - .orElseThrow(() -> ApiException.of(MapException.MAP_RECOMMENDATION_UNAVAILABLE)); - - int sanitizedLimit = limit == null || limit <= 0 ? 5 : limit; - - List recommendations = repository.findRecommendedMaps(level, jobId, sanitizedLimit); - if (recommendations.isEmpty()) { - return recommendations; - } - - List mapIds = recommendations.stream() - .map(MapRecommendationDto::mapId) - .toList(); - - var mapsById = mapRepository.findByMapIdIn(mapIds).stream() - .collect(Collectors.toMap(Map::getMapId, Function.identity())); - var bookmarkIds = bookmarkFlagService.findBookmarkIds(memberId, BookmarkType.MAP, mapIds); - - return recommendations.stream() - .map(recommendation -> { - Map map = mapsById.get(recommendation.mapId()); - return new MapRecommendationDto( - recommendation.mapId(), - recommendation.score(), - map != null ? map.getIconUrl() : null, - map != null ? map.getNameKr() : null, - bookmarkIds.get(recommendation.mapId()) - ); - }) - .toList(); - } - - private void validateJobExists(int jobId) { - if (!jobRepository.existsById(jobId)) { - throw ApiException.of(JobException.JOB_NOT_FOUND); - } - } } diff --git a/src/main/java/com/maple/api/map/application/command/AuraDbKeepAliveBatch.java b/src/main/java/com/maple/api/map/application/command/AuraDbKeepAliveBatch.java index 09b2452..a57e7c7 100644 --- a/src/main/java/com/maple/api/map/application/command/AuraDbKeepAliveBatch.java +++ b/src/main/java/com/maple/api/map/application/command/AuraDbKeepAliveBatch.java @@ -1,9 +1,10 @@ package com.maple.api.map.application.command; import com.maple.api.common.logging.SafeExceptionLog; -import com.maple.api.map.repository.MapRecommendationRepository; +import com.maple.api.map.repository.AuraMapRecommendationRepository; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @@ -11,17 +12,18 @@ @Slf4j @Component @RequiredArgsConstructor +@ConditionalOnBean(AuraMapRecommendationRepository.class) @ConditionalOnProperty(name = "batch.auradb-keep-alive.enabled", havingValue = "true") public class AuraDbKeepAliveBatch { - private final MapRecommendationRepository mapRecommendationRepository; + private final AuraMapRecommendationRepository auraMapRecommendationRepository; // AuraDB Free 티어 미사용 일시중지 방지용 keep-alive // 매일 04:00, 16:00 (KST) @Scheduled(cron = "0 0 4,16 * * *", zone = "Asia/Seoul") public void keepAlive() { try { - mapRecommendationRepository.ping(); + auraMapRecommendationRepository.ping(); log.atInfo() .addKeyValue("event.action", "external.keep-alive") .addKeyValue("event.outcome", "success") diff --git a/src/main/java/com/maple/api/map/application/dto/MapRecommendationDto.java b/src/main/java/com/maple/api/map/application/dto/MapRecommendationDto.java index bab6bf7..3236638 100644 --- a/src/main/java/com/maple/api/map/application/dto/MapRecommendationDto.java +++ b/src/main/java/com/maple/api/map/application/dto/MapRecommendationDto.java @@ -7,7 +7,10 @@ public record MapRecommendationDto( @Schema(description = "추천 맵 ID", example = "100000000") Integer mapId, - @Schema(description = "레벨/직업 가중치를 반영한 최종 점수", example = "8.0") + @Schema( + description = "선택 엔진의 실제 추천 점수. MySQL 엔진에서는 APPROVED 근거의 signed freshness contribution을 합산한 evidence net score", + example = "0.95" + ) double score, @Schema(description = "맵 아이콘 URL", example = "https://maplestory.io/api/gms/62/map/100000000/icon?resize=2") @@ -22,4 +25,14 @@ public record MapRecommendationDto( public MapRecommendationDto(Integer mapId, double score) { this(mapId, score, null, null, null); } + + public static MapRecommendationDto toDto(MapRecommendationResultDto result) { + return new MapRecommendationDto( + result.mapId(), + result.score(), + result.iconUrl(), + result.nameKr(), + result.bookmarkId() + ); + } } diff --git a/src/main/java/com/maple/api/map/application/dto/MapRecommendationReasonDto.java b/src/main/java/com/maple/api/map/application/dto/MapRecommendationReasonDto.java new file mode 100644 index 0000000..8077aa0 --- /dev/null +++ b/src/main/java/com/maple/api/map/application/dto/MapRecommendationReasonDto.java @@ -0,0 +1,25 @@ +package com.maple.api.map.application.dto; + +import com.maple.api.map.domain.RecommendationReason; +import io.swagger.v3.oas.annotations.media.Schema; + +@Schema(description = "추천 근거 코드. axis/value 조합은 문서화된 고정 코드만 사용합니다.") +public record MapRecommendationReasonDto( + @Schema( + description = "추천 이유 축", + allowableValues = {"reward", "play_style", "operability"}, + example = "reward" + ) + String axis, + + @Schema( + description = "축별 이유 코드: reward=xp|meso|loot, play_style=solo|party|party_quest, operability=fatigue|mobility|budget", + allowableValues = {"xp", "meso", "loot", "solo", "party", "party_quest", "fatigue", "mobility", "budget"}, + example = "xp" + ) + String value +) { + public static MapRecommendationReasonDto toDto(RecommendationReason reason) { + return new MapRecommendationReasonDto(reason.axis(), reason.value()); + } +} diff --git a/src/main/java/com/maple/api/map/application/dto/MapRecommendationResultDto.java b/src/main/java/com/maple/api/map/application/dto/MapRecommendationResultDto.java new file mode 100644 index 0000000..65c3d20 --- /dev/null +++ b/src/main/java/com/maple/api/map/application/dto/MapRecommendationResultDto.java @@ -0,0 +1,22 @@ +package com.maple.api.map.application.dto; + +import com.maple.api.map.domain.RecommendationReason; + +import java.util.List; +import java.util.Objects; + +public record MapRecommendationResultDto( + Integer mapId, + double score, + String iconUrl, + String nameKr, + Integer bookmarkId, + List reasons +) { + public MapRecommendationResultDto { + if (!Double.isFinite(score)) { + throw new IllegalArgumentException("score must be finite"); + } + reasons = List.copyOf(Objects.requireNonNull(reasons, "reasons must not be null")); + } +} diff --git a/src/main/java/com/maple/api/map/application/dto/MapRecommendationV2Dto.java b/src/main/java/com/maple/api/map/application/dto/MapRecommendationV2Dto.java new file mode 100644 index 0000000..fb3b996 --- /dev/null +++ b/src/main/java/com/maple/api/map/application/dto/MapRecommendationV2Dto.java @@ -0,0 +1,50 @@ +package com.maple.api.map.application.dto; + +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Schema; + +import java.util.List; + +@Schema(description = "근거 코드가 포함된 v2 사냥터 추천 결과") +public record MapRecommendationV2Dto( + @Schema(description = "추천 맵 ID", example = "100000000") + Integer mapId, + + @Schema( + description = "APPROVED 근거의 signed freshness contribution을 합산한 evidence net score", + example = "0.95" + ) + double score, + + @Schema(description = "맵 아이콘 URL", example = "https://maplestory.io/api/gms/62/map/100000000/icon?resize=2") + String iconUrl, + + @Schema(description = "한국어 맵 이름", example = "헤네시스") + String nameKr, + + @Schema(description = "로그인 사용자가 생성한 북마크 ID (없으면 null)", example = "123") + Integer bookmarkId, + + @ArraySchema( + arraySchema = @Schema(description = "축별 최대 하나인 안정적인 추천 이유 코드. 이유가 없으면 빈 배열"), + schema = @Schema(implementation = MapRecommendationReasonDto.class) + ) + List reasons +) { + public MapRecommendationV2Dto { + reasons = reasons == null ? List.of() : List.copyOf(reasons); + } + + public static MapRecommendationV2Dto toDto(MapRecommendationResultDto result) { + return new MapRecommendationV2Dto( + result.mapId(), + result.score(), + result.iconUrl(), + result.nameKr(), + result.bookmarkId(), + result.reasons().stream() + .map(MapRecommendationReasonDto::toDto) + .toList() + ); + } +} diff --git a/src/main/java/com/maple/api/map/domain/Polarity.java b/src/main/java/com/maple/api/map/domain/Polarity.java new file mode 100644 index 0000000..87fc6fc --- /dev/null +++ b/src/main/java/com/maple/api/map/domain/Polarity.java @@ -0,0 +1,31 @@ +package com.maple.api.map.domain; + +import java.util.Arrays; + +public enum Polarity { + POSITIVE("positive", 1), + NEGATIVE("negative", -1); + + private final String dbValue; + private final int sign; + + Polarity(String dbValue, int sign) { + this.dbValue = dbValue; + this.sign = sign; + } + + public String dbValue() { + return dbValue; + } + + public int sign() { + return sign; + } + + public static Polarity from(String value) { + return Arrays.stream(values()) + .filter(candidate -> candidate.dbValue.equalsIgnoreCase(value)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("Unsupported polarity: " + value)); + } +} diff --git a/src/main/java/com/maple/api/map/domain/RecommendationCandidate.java b/src/main/java/com/maple/api/map/domain/RecommendationCandidate.java new file mode 100644 index 0000000..0805069 --- /dev/null +++ b/src/main/java/com/maple/api/map/domain/RecommendationCandidate.java @@ -0,0 +1,14 @@ +package com.maple.api.map.domain; + +import java.util.List; +import java.util.Objects; + +public record RecommendationCandidate(long mapId, double score, List reasons) { + + public RecommendationCandidate { + if (!Double.isFinite(score)) { + throw new IllegalArgumentException("score must be finite"); + } + reasons = List.copyOf(Objects.requireNonNull(reasons, "reasons must not be null")); + } +} diff --git a/src/main/java/com/maple/api/map/domain/RecommendationEngineType.java b/src/main/java/com/maple/api/map/domain/RecommendationEngineType.java new file mode 100644 index 0000000..c34ff95 --- /dev/null +++ b/src/main/java/com/maple/api/map/domain/RecommendationEngineType.java @@ -0,0 +1,12 @@ +package com.maple.api.map.domain; + +import java.util.Locale; + +public enum RecommendationEngineType { + AURA, + MYSQL; + + public String metricValue() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/src/main/java/com/maple/api/map/domain/RecommendationEvidence.java b/src/main/java/com/maple/api/map/domain/RecommendationEvidence.java new file mode 100644 index 0000000..95dd947 --- /dev/null +++ b/src/main/java/com/maple/api/map/domain/RecommendationEvidence.java @@ -0,0 +1,35 @@ +package com.maple.api.map.domain; + +import java.time.LocalDateTime; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Objects; + +/** + * Immutable projection of an APPROVED reviewed claim returned for the requested job lineage. + * Approval and lineage filtering belong to the recommendation repository; this type deliberately carries + * no mutable review state. + */ +public record RecommendationEvidence( + long extractedClaimId, + long reviewedClaimId, + long mapId, + long jobId, + Integer levelMin, + Integer levelMax, + Polarity polarity, + LocalDateTime publishedAt, + int patchCount, + List reasonFacets +) { + + public RecommendationEvidence { + Objects.requireNonNull(polarity, "polarity must not be null"); + Objects.requireNonNull(publishedAt, "publishedAt must not be null"); + Objects.requireNonNull(reasonFacets, "reasonFacets must not be null"); + if (patchCount < 0) { + throw new IllegalArgumentException("patchCount must not be negative"); + } + reasonFacets = List.copyOf(new LinkedHashSet<>(reasonFacets)); + } +} diff --git a/src/main/java/com/maple/api/map/domain/RecommendationFacet.java b/src/main/java/com/maple/api/map/domain/RecommendationFacet.java new file mode 100644 index 0000000..92888e2 --- /dev/null +++ b/src/main/java/com/maple/api/map/domain/RecommendationFacet.java @@ -0,0 +1,57 @@ +package com.maple.api.map.domain; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +public enum RecommendationFacet { + REWARD_XP("reward", "xp", 0), + REWARD_MESO("reward", "meso", 1), + REWARD_LOOT("reward", "loot", 2), + PLAY_STYLE_SOLO("play_style", "solo", 0), + PLAY_STYLE_PARTY("play_style", "party", 1), + PLAY_STYLE_PARTY_QUEST("play_style", "party_quest", 2), + OPERABILITY_FATIGUE("operability", "fatigue", 0), + OPERABILITY_MOBILITY("operability", "mobility", 1), + OPERABILITY_BUDGET("operability", "budget", 2); + + private static final Comparator PRIORITY_ORDER = + Comparator.comparingInt(RecommendationFacet::priority); + + private final String axis; + private final String value; + private final int priority; + + RecommendationFacet(String axis, String value, int priority) { + this.axis = axis; + this.value = value; + this.priority = priority; + } + + public String axis() { + return axis; + } + + public String value() { + return value; + } + + public int priority() { + return priority; + } + + public static RecommendationFacet from(String axis, String value) { + return Arrays.stream(values()) + .filter(candidate -> candidate.axis.equalsIgnoreCase(axis) + && candidate.value.equalsIgnoreCase(value)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("Unsupported facet: " + axis + ':' + value)); + } + + public static List forAxis(String axis) { + return Arrays.stream(values()) + .filter(candidate -> candidate.axis.equalsIgnoreCase(axis)) + .sorted(PRIORITY_ORDER) + .toList(); + } +} diff --git a/src/main/java/com/maple/api/map/domain/RecommendationReason.java b/src/main/java/com/maple/api/map/domain/RecommendationReason.java new file mode 100644 index 0000000..c5aa896 --- /dev/null +++ b/src/main/java/com/maple/api/map/domain/RecommendationReason.java @@ -0,0 +1,11 @@ +package com.maple.api.map.domain; + +import java.util.Objects; + +public record RecommendationReason(String axis, String value) { + + public RecommendationReason { + Objects.requireNonNull(axis, "axis must not be null"); + Objects.requireNonNull(value, "value must not be null"); + } +} diff --git a/src/main/java/com/maple/api/map/domain/RecommendationScoringService.java b/src/main/java/com/maple/api/map/domain/RecommendationScoringService.java new file mode 100644 index 0000000..6f48445 --- /dev/null +++ b/src/main/java/com/maple/api/map/domain/RecommendationScoringService.java @@ -0,0 +1,195 @@ +package com.maple.api.map.domain; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.LocalDateTime; +import java.util.Comparator; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Pure scoring policy for repository evidence. Inputs must already be restricted to APPROVED + * evidence for the requested job and its ancestors. + */ +public final class RecommendationScoringService { + + private static final int MIN_LIMIT = 1; + private static final int MAX_LIMIT = 20; + private static final BigDecimal PATCH_DECAY = new BigDecimal("0.05"); + private static final BigDecimal MIN_FRESHNESS = new BigDecimal("0.1"); + private static final List REASON_AXIS_ORDER = List.of("reward", "play_style", "operability"); + + private static final Comparator CANDIDATE_ORDER = + Comparator.comparing(CandidateAggregate::netScore, Comparator.reverseOrder()) + .thenComparing(CandidateAggregate::freshnessSum, Comparator.reverseOrder()) + .thenComparing(CandidateAggregate::representativePublishedAt, Comparator.reverseOrder()) + .thenComparingLong(CandidateAggregate::mapId); + + public List score( + int requestedLevel, + List evidence, + int limit + ) { + validateLimit(limit); + Objects.requireNonNull(evidence, "evidence must not be null"); + + Map deduplicated = deduplicate(evidence); + Map candidatesByMap = new HashMap<>(); + + for (RecommendationEvidence row : deduplicated.values()) { + if (!matchesLevel(requestedLevel, row.levelMin(), row.levelMax())) { + continue; + } + + BigDecimal contribution = freshnessWeight(row.patchCount()) + .multiply(BigDecimal.valueOf(row.polarity().sign())); + CandidateAggregate aggregate = candidatesByMap.computeIfAbsent( + row.mapId(), + CandidateAggregate::new + ); + aggregate.add(row, contribution); + } + + return candidatesByMap.values().stream() + .filter(CandidateAggregate::hasPositiveEvidence) + .filter(candidate -> candidate.netScore().signum() > 0) + .sorted(CANDIDATE_ORDER) + .limit(limit) + .map(CandidateAggregate::toCandidate) + .toList(); + } + + private Map deduplicate(List evidence) { + Map deduplicated = new LinkedHashMap<>(); + for (RecommendationEvidence row : evidence) { + Objects.requireNonNull(row, "evidence row must not be null"); + EvidenceKey key = new EvidenceKey(row.extractedClaimId(), row.mapId()); + deduplicated.merge( + key, + row, + (current, candidate) -> candidate.reviewedClaimId() < current.reviewedClaimId() + ? candidate + : current + ); + } + return deduplicated; + } + + private boolean matchesLevel(int requestedLevel, Integer levelMin, Integer levelMax) { + if (levelMin == null && levelMax == null) { + return false; + } + + long effectiveMin = levelMin != null ? levelMin : (long) levelMax - 10L; + long effectiveMax = levelMax != null ? levelMax : (long) levelMin + 10L; + return requestedLevel >= effectiveMin && requestedLevel <= effectiveMax; + } + + private BigDecimal freshnessWeight(int patchCount) { + BigDecimal decayed = BigDecimal.ONE.subtract(PATCH_DECAY.multiply(BigDecimal.valueOf(patchCount))); + return decayed.max(MIN_FRESHNESS).setScale(3, RoundingMode.HALF_UP); + } + + private void validateLimit(int limit) { + if (limit < MIN_LIMIT || limit > MAX_LIMIT) { + throw new IllegalArgumentException("limit must be between 1 and 20"); + } + } + + private record EvidenceKey(long extractedClaimId, long mapId) { + } + + private static final class CandidateAggregate { + private final long mapId; + private final Map facetWeights = + new EnumMap<>(RecommendationFacet.class); + private BigDecimal netScore = BigDecimal.ZERO; + private BigDecimal freshnessSum = BigDecimal.ZERO; + private boolean hasPositiveEvidence; + private BigDecimal representativeContribution; + private LocalDateTime representativePublishedAt; + + private CandidateAggregate(long mapId) { + this.mapId = mapId; + } + + private void add(RecommendationEvidence evidence, BigDecimal contribution) { + netScore = netScore.add(contribution); + freshnessSum = freshnessSum.add(contribution.abs()); + if (evidence.polarity() == Polarity.POSITIVE) { + hasPositiveEvidence = true; + } + + if (isBetterRepresentative(contribution, evidence.publishedAt())) { + representativeContribution = contribution; + representativePublishedAt = evidence.publishedAt(); + } + + for (RecommendationFacet facet : evidence.reasonFacets()) { + facetWeights.merge(facet, contribution, BigDecimal::add); + } + } + + private boolean isBetterRepresentative(BigDecimal contribution, LocalDateTime publishedAt) { + if (representativeContribution == null) { + return true; + } + int contributionOrder = contribution.compareTo(representativeContribution); + return contributionOrder > 0 + || contributionOrder == 0 && publishedAt.isAfter(representativePublishedAt); + } + + private boolean hasPositiveEvidence() { + return hasPositiveEvidence; + } + + private long mapId() { + return mapId; + } + + private BigDecimal netScore() { + return netScore; + } + + private BigDecimal freshnessSum() { + return freshnessSum; + } + + private LocalDateTime representativePublishedAt() { + return representativePublishedAt; + } + + private RecommendationCandidate toCandidate() { + double score = netScore.doubleValue(); + if (!Double.isFinite(score)) { + throw new IllegalStateException("Recommendation score is not finite for map " + mapId); + } + return new RecommendationCandidate(mapId, score, reasons()); + } + + private List reasons() { + return REASON_AXIS_ORDER.stream() + .map(this::bestFacetForAxis) + .filter(Objects::nonNull) + .map(facet -> new RecommendationReason(facet.axis(), facet.value())) + .toList(); + } + + private RecommendationFacet bestFacetForAxis(String axis) { + RecommendationFacet bestFacet = null; + BigDecimal bestWeight = null; + for (RecommendationFacet candidate : RecommendationFacet.forAxis(axis)) { + BigDecimal weight = facetWeights.getOrDefault(candidate, BigDecimal.ZERO); + if (bestWeight == null || weight.compareTo(bestWeight) > 0) { + bestFacet = candidate; + bestWeight = weight; + } + } + return bestWeight != null && bestWeight.signum() > 0 ? bestFacet : null; + } + } +} diff --git a/src/main/java/com/maple/api/map/presentation/restapi/MapController.java b/src/main/java/com/maple/api/map/presentation/restapi/MapController.java index 3d24254..0c57030 100644 --- a/src/main/java/com/maple/api/map/presentation/restapi/MapController.java +++ b/src/main/java/com/maple/api/map/presentation/restapi/MapController.java @@ -5,8 +5,10 @@ import com.maple.api.common.presentation.restapi.ResponseTemplate; import com.maple.api.map.application.MapService; import com.maple.api.map.application.dto.*; +import com.maple.api.map.application.MapRecommendationService; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.tags.Tag; @@ -37,6 +39,7 @@ public class MapController { private final MapService mapService; + private final MapRecommendationService mapRecommendationService; @GetMapping @Operation( @@ -128,21 +131,36 @@ public ResponseEntity>> getMapNpcs(@PathVariabl @GetMapping("/recommendations") @Operation( summary = "사냥터 추천", - description = "레벨과 직업을 이용해 사냥터를 추천합니다." + description = "레벨과 직업을 이용해 사냥터를 추천합니다. 응답 score는 선택 엔진의 실제 점수이며, " + + "MySQL 엔진에서는 APPROVED 근거의 signed freshness contribution 합인 evidence net score입니다. " + + "모바일 strict decoder 호환을 위해 v1 item은 mapId, score, iconUrl, nameKr, bookmarkId만 반환합니다." ) @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "사냥터 추천 결과 조회 성공") + @ApiResponse(responseCode = "200", description = "사냥터 추천 결과 조회 성공"), + @ApiResponse(responseCode = "400", description = "레벨 또는 limit validation 실패"), + @ApiResponse(responseCode = "404", description = "존재하지 않는 직업"), + @ApiResponse(responseCode = "503", description = "선택된 추천 엔진 또는 추천 schema를 사용할 수 없음") }) public ResponseEntity>> recommendMaps( @Parameter(description = "요청 캐릭터 레벨", example = "100") @RequestParam @Min(1) @Max(200) int level, @Parameter(description = "요청 직업 ID", example = "100") @RequestParam int jobId, - @Parameter(description = "반환할 추천 개수 (최대 20)", example = "5") - @RequestParam(required = false) @Min(1) @Max(20) Integer limit, + @Parameter( + description = "최종 정렬 뒤 적용할 반환 개수", + example = "5", + schema = @Schema( + type = "integer", + format = "int32", + defaultValue = "5", + minimum = "1", + maximum = "20" + ) + ) + @RequestParam(defaultValue = "5") @Min(1) @Max(20) Integer limit, @AuthenticationPrincipal PrincipalDetails principalDetails) { String memberId = principalDetails != null ? principalDetails.getProviderId() : null; - List recommendations = mapService.recommendMaps(memberId, level, jobId, limit); + List recommendations = mapRecommendationService.recommendV1(memberId, level, jobId, limit); return ResponseEntity.ok(ResponseTemplate.success(recommendations)); } } diff --git a/src/main/java/com/maple/api/map/presentation/restapi/MapV2Controller.java b/src/main/java/com/maple/api/map/presentation/restapi/MapV2Controller.java new file mode 100644 index 0000000..3e977d0 --- /dev/null +++ b/src/main/java/com/maple/api/map/presentation/restapi/MapV2Controller.java @@ -0,0 +1,72 @@ +package com.maple.api.map.presentation.restapi; + +import com.maple.api.auth.domain.PrincipalDetails; +import com.maple.api.common.presentation.restapi.ResponseTemplate; +import com.maple.api.map.application.dto.MapRecommendationV2Dto; +import com.maple.api.map.application.MapRecommendationService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; +import jakarta.validation.constraints.Max; +import jakarta.validation.constraints.Min; +import lombok.RequiredArgsConstructor; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@RestController +@RequestMapping("/api/v2/maps") +@RequiredArgsConstructor +@Tag(name = "Map V2", description = "추천 근거 코드를 포함한 v2 맵 API") +@Validated +public class MapV2Controller { + + private final MapRecommendationService mapRecommendationService; + + @GetMapping("/recommendations") + @Operation( + summary = "근거 기반 사냥터 추천", + description = "MySQL의 APPROVED 추천 근거를 요청 Job lineage에 맞춰 합산합니다. " + + "reasons는 reward, play_style, operability 순서이며 축별 최대 하나의 안정적인 axis/value 코드만 반환합니다." + ) + @ApiResponses(value = { + @ApiResponse(responseCode = "200", description = "사냥터 추천 결과 조회 성공"), + @ApiResponse(responseCode = "400", description = "레벨 또는 limit validation 실패"), + @ApiResponse(responseCode = "404", description = "존재하지 않는 직업"), + @ApiResponse( + responseCode = "503", + description = "v2가 비활성 상태이거나 MySQL 추천 인프라/schema/시간 제한을 사용할 수 없음" + ) + }) + public ResponseEntity>> recommendMaps( + @Parameter(description = "요청 캐릭터 레벨", example = "100") + @RequestParam @Min(1) @Max(200) int level, + @Parameter(description = "요청 직업 ID", example = "100") + @RequestParam int jobId, + @Parameter( + description = "최종 정렬 뒤 적용할 반환 개수", + example = "5", + schema = @Schema( + type = "integer", + format = "int32", + defaultValue = "5", + minimum = "1", + maximum = "20" + ) + ) + @RequestParam(defaultValue = "5") @Min(1) @Max(20) Integer limit, + @AuthenticationPrincipal PrincipalDetails principalDetails) { + String memberId = principalDetails != null ? principalDetails.getProviderId() : null; + List recommendations = mapRecommendationService.recommendV2(memberId, level, jobId, limit); + return ResponseEntity.ok(ResponseTemplate.success(recommendations)); + } +} diff --git a/src/main/java/com/maple/api/map/repository/AuraMapRecommendationRepository.java b/src/main/java/com/maple/api/map/repository/AuraMapRecommendationRepository.java new file mode 100644 index 0000000..3c63db2 --- /dev/null +++ b/src/main/java/com/maple/api/map/repository/AuraMapRecommendationRepository.java @@ -0,0 +1,80 @@ +package com.maple.api.map.repository; + +import com.maple.api.map.domain.RecommendationCandidate; +import com.maple.api.map.domain.RecommendationEngineType; +import lombok.RequiredArgsConstructor; +import org.neo4j.driver.Driver; +import org.neo4j.driver.Record; +import org.neo4j.driver.Result; +import org.neo4j.driver.Session; +import org.neo4j.driver.SessionConfig; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.stereotype.Repository; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Repository +@ConditionalOnBean(Driver.class) +@RequiredArgsConstructor +public class AuraMapRecommendationRepository implements MapRecommendationRepository { + + private static final String RECOMMEND_QUERY = """ + MATCH (lvl:Level {value: $level})-[rl:RECO {dim:'LEVEL'}]->(m:Map) + OPTIONAL MATCH (j:Job)-[rj:RECO {dim:'JOB'}]->(m) + WHERE j.job_id IN [$jobId, 0] + WITH m, + coalesce(rl.hit_count_level, 0) AS levelHits, + coalesce(sum( + CASE + WHEN j.job_id = $jobId THEN coalesce(rj.hit_count_job, 0) + ELSE coalesce(rj.hit_count_job, 0) * 0.5 + END + ), 0) AS jobHits + WITH m, levelHits, jobHits, + levelHits * 0.8 + jobHits * 0.2 AS score + RETURN m.map_id AS mapId, + score + ORDER BY score DESC + LIMIT $limit + """; + + private final Driver auraDbDriver; + + @Override + public RecommendationEngineType engineType() { + return RecommendationEngineType.AURA; + } + + @Override + public List findRecommendations(int level, int jobId, int limit) { + try (Session session = auraDbDriver.session(SessionConfig.defaultConfig())) { + Map params = new HashMap<>(); + params.put("level", level); + params.put("jobId", jobId); + params.put("limit", limit); + + return session.executeRead(tx -> { + Result result = tx.run(RECOMMEND_QUERY, params); + List recommendations = new ArrayList<>(); + while (result.hasNext()) { + Record record = result.next(); + recommendations.add(new RecommendationCandidate( + record.get("mapId").asLong(), + record.get("score").asDouble(), + List.of() + )); + } + return recommendations; + }); + } + } + + public void ping() { + try (Session session = auraDbDriver.session(SessionConfig.defaultConfig())) { + session.run("RETURN 1").consume(); + } + } +} diff --git a/src/main/java/com/maple/api/map/repository/MapRecommendationRepository.java b/src/main/java/com/maple/api/map/repository/MapRecommendationRepository.java index 63eea8c..26f3450 100644 --- a/src/main/java/com/maple/api/map/repository/MapRecommendationRepository.java +++ b/src/main/java/com/maple/api/map/repository/MapRecommendationRepository.java @@ -1,72 +1,13 @@ package com.maple.api.map.repository; -import com.maple.api.map.application.dto.MapRecommendationDto; -import lombok.RequiredArgsConstructor; -import org.neo4j.driver.*; -import org.neo4j.driver.Record; -import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; -import org.springframework.stereotype.Repository; +import com.maple.api.map.domain.RecommendationCandidate; +import com.maple.api.map.domain.RecommendationEngineType; -import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; -@Repository -@ConditionalOnBean(Driver.class) -@RequiredArgsConstructor -public class MapRecommendationRepository { +public interface MapRecommendationRepository { - private static final String RECOMMEND_QUERY = """ - MATCH (lvl:Level {value: $level})-[rl:RECO {dim:'LEVEL'}]->(m:Map) - OPTIONAL MATCH (j:Job)-[rj:RECO {dim:'JOB'}]->(m) - WHERE j.job_id IN [$jobId, 0] - WITH m, - coalesce(rl.hit_count_level, 0) AS levelHits, - coalesce(sum( - CASE - WHEN j.job_id = $jobId THEN coalesce(rj.hit_count_job, 0) - ELSE coalesce(rj.hit_count_job, 0) * 0.5 - END - ), 0) AS jobHits - WITH m, levelHits, jobHits, - levelHits * 0.8 + jobHits * 0.2 AS score - RETURN m.map_id AS mapId, - score - ORDER BY score DESC - LIMIT $limit - """; + RecommendationEngineType engineType(); - private final Driver auraDbDriver; - - public List findRecommendedMaps(int level, int jobId, int limit) { - SessionConfig sessionConfig = SessionConfig.defaultConfig(); - - try (Session session = auraDbDriver.session(sessionConfig)) { - Map params = new HashMap<>(); - params.put("level", level); - params.put("jobId", jobId); - params.put("limit", limit); - - return session.executeRead(tx -> { - Result result = tx.run(RECOMMEND_QUERY, params); - - List recommendations = new ArrayList<>(); - while (result.hasNext()) { - Record record = result.next(); - recommendations.add(new MapRecommendationDto( - record.get("mapId").asInt(), - record.get("score").asDouble() - )); - } - return recommendations; - }); - } - } - - public void ping() { - try (Session session = auraDbDriver.session(SessionConfig.defaultConfig())) { - session.run("RETURN 1").consume(); - } - } + List findRecommendations(int level, int jobId, int limit); } diff --git a/src/main/java/com/maple/api/map/repository/MySqlMapRecommendationRepository.java b/src/main/java/com/maple/api/map/repository/MySqlMapRecommendationRepository.java new file mode 100644 index 0000000..52b5c33 --- /dev/null +++ b/src/main/java/com/maple/api/map/repository/MySqlMapRecommendationRepository.java @@ -0,0 +1,241 @@ +package com.maple.api.map.repository; + +import com.maple.api.map.domain.Polarity; +import com.maple.api.map.domain.RecommendationCandidate; +import com.maple.api.map.domain.RecommendationEvidence; +import com.maple.api.map.domain.RecommendationFacet; +import com.maple.api.map.domain.RecommendationScoringService; +import com.maple.api.map.domain.RecommendationEngineType; +import org.springframework.dao.DataAccessException; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.jdbc.core.ResultSetExtractor; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.stereotype.Repository; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +@Repository +public class MySqlMapRecommendationRepository implements MapRecommendationRepository { + + static final String SCORING_QUERY = """ + WITH RECURSIVE job_lineage (job_id, parent_job_id, lineage_path) AS ( + SELECT j.job_id, + j.parent_job_id, + CAST(CONCAT(',', j.job_id, ',') AS CHAR(2048)) AS lineage_path + FROM jobs j + WHERE j.job_id = :jobId + + UNION ALL + + SELECT parent.job_id, + parent.parent_job_id, + CAST(CONCAT(lineage.lineage_path, parent.job_id, ',') AS CHAR(2048)) + FROM jobs parent + JOIN job_lineage lineage ON parent.job_id = lineage.parent_job_id + WHERE LOCATE(CONCAT(',', parent.job_id, ','), lineage.lineage_path) = 0 + ), + ranked_claims AS ( + SELECT rc.reviewed_claim_id, + rc.extracted_claim_id, + rc.final_map_id, + rc.final_job_id, + rc.final_level_min, + rc.final_level_max, + rc.final_polarity, + ec.source_published_at, + ROW_NUMBER() OVER ( + PARTITION BY rc.extracted_claim_id, rc.final_map_id + ORDER BY rc.reviewed_claim_id + ) AS evidence_rank + FROM recommendation_reviewed_claims rc + JOIN recommendation_extracted_claims ec + ON ec.extracted_claim_id = rc.extracted_claim_id + JOIN job_lineage lineage + ON lineage.job_id = rc.final_job_id + JOIN maps canonical_map + ON canonical_map.map_id = rc.final_map_id + WHERE rc.review_status = 'APPROVED' + AND rc.final_map_id IS NOT NULL + AND rc.final_job_id IS NOT NULL + AND rc.final_polarity IS NOT NULL + ), + deduplicated_claims AS ( + SELECT reviewed_claim_id, + extracted_claim_id, + final_map_id, + final_job_id, + final_level_min, + final_level_max, + final_polarity, + source_published_at + FROM ranked_claims + WHERE evidence_rank = 1 + ), + patch_counts AS ( + SELECT claim.reviewed_claim_id, + COUNT(patch.id) AS patch_count + FROM deduplicated_claims claim + LEFT JOIN alrim patch + ON patch.type = 'PATCH_NOTE' + AND patch.date > claim.source_published_at + GROUP BY claim.reviewed_claim_id + ) + SELECT claim.reviewed_claim_id, + claim.extracted_claim_id, + claim.final_map_id, + claim.final_job_id, + claim.final_level_min, + claim.final_level_max, + claim.final_polarity, + claim.source_published_at, + patch_count.patch_count, + reason.reason_axis, + reason.reason_value + FROM deduplicated_claims claim + JOIN patch_counts patch_count + ON patch_count.reviewed_claim_id = claim.reviewed_claim_id + LEFT JOIN recommendation_reviewed_claim_reasons reason + ON reason.reviewed_claim_id = claim.reviewed_claim_id + ORDER BY claim.reviewed_claim_id, + reason.display_order, + reason.reviewed_claim_reason_id + """; + + private final NamedParameterJdbcTemplate jdbcTemplate; + private final RecommendationScoringService scoringService; + + public MySqlMapRecommendationRepository( + @Qualifier("recommendationJdbcTemplate") NamedParameterJdbcTemplate jdbcTemplate, + RecommendationScoringService scoringService + ) { + this.jdbcTemplate = jdbcTemplate; + this.scoringService = scoringService; + } + + @Override + public RecommendationEngineType engineType() { + return RecommendationEngineType.MYSQL; + } + + @Override + public List findRecommendations(int level, int jobId, int limit) { + List evidence = loadEvidence(jobId); + return scoringService.score(level, evidence, limit); + } + + List loadEvidence(int jobId) throws DataAccessException { + return jdbcTemplate.query( + SCORING_QUERY, + Map.of("jobId", jobId), + (ResultSetExtractor>) this::extractEvidence + ); + } + + private List extractEvidence(ResultSet resultSet) throws SQLException { + Map byReviewedClaim = new LinkedHashMap<>(); + while (resultSet.next()) { + long reviewedClaimId = resultSet.getLong("reviewed_claim_id"); + EvidenceAccumulator accumulator = byReviewedClaim.computeIfAbsent( + reviewedClaimId, + ignored -> EvidenceAccumulator.from(resultSet) + ); + String reasonAxis = resultSet.getString("reason_axis"); + String reasonValue = resultSet.getString("reason_value"); + if (reasonAxis != null && reasonValue != null) { + accumulator.addReason(RecommendationFacet.from(reasonAxis, reasonValue)); + } + } + return byReviewedClaim.values().stream() + .map(EvidenceAccumulator::toEvidence) + .toList(); + } + + private static Integer nullableInteger(ResultSet resultSet, String column) throws SQLException { + int value = resultSet.getInt(column); + return resultSet.wasNull() ? null : value; + } + + private static final class EvidenceAccumulator { + private final long extractedClaimId; + private final long reviewedClaimId; + private final long mapId; + private final long jobId; + private final Integer levelMin; + private final Integer levelMax; + private final Polarity polarity; + private final LocalDateTime publishedAt; + private final int patchCount; + private final List reasons = new ArrayList<>(); + + private EvidenceAccumulator( + long extractedClaimId, + long reviewedClaimId, + long mapId, + long jobId, + Integer levelMin, + Integer levelMax, + Polarity polarity, + LocalDateTime publishedAt, + int patchCount + ) { + this.extractedClaimId = extractedClaimId; + this.reviewedClaimId = reviewedClaimId; + this.mapId = mapId; + this.jobId = jobId; + this.levelMin = levelMin; + this.levelMax = levelMax; + this.polarity = polarity; + this.publishedAt = publishedAt; + this.patchCount = patchCount; + } + + private static EvidenceAccumulator from(ResultSet resultSet) { + try { + return new EvidenceAccumulator( + resultSet.getLong("extracted_claim_id"), + resultSet.getLong("reviewed_claim_id"), + resultSet.getLong("final_map_id"), + resultSet.getLong("final_job_id"), + nullableInteger(resultSet, "final_level_min"), + nullableInteger(resultSet, "final_level_max"), + Polarity.from(resultSet.getString("final_polarity")), + resultSet.getTimestamp("source_published_at").toLocalDateTime(), + Math.toIntExact(resultSet.getLong("patch_count")) + ); + } catch (SQLException exception) { + throw new EvidenceMappingException(exception); + } + } + + private void addReason(RecommendationFacet reason) { + reasons.add(reason); + } + + private RecommendationEvidence toEvidence() { + return new RecommendationEvidence( + extractedClaimId, + reviewedClaimId, + mapId, + jobId, + levelMin, + levelMax, + polarity, + publishedAt, + patchCount, + reasons + ); + } + } + + private static final class EvidenceMappingException extends RuntimeException { + private EvidenceMappingException(SQLException cause) { + super(cause); + } + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 42b73be..5bedbc6 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -53,6 +53,13 @@ auradb: username: ${AURADB_USERNAME:} password: ${AURADB_PASSWORD:} database: ${AURADB_DATABASE:neo4j} + +recommendation: + # Keep the public v1 route on Aura until the MySQL schema preflight and v2 gate are approved. + v1-engine: ${RECOMMENDATION_V1_ENGINE:AURA} + # Keep the unauthenticated MySQL route dark until topology, schema, and rate-limit gates pass. + v2-enabled: ${RECOMMENDATION_V2_ENABLED:false} + query-timeout-seconds: ${RECOMMENDATION_QUERY_TIMEOUT_SECONDS:10} --- spring: config: diff --git a/src/test/java/com/maple/api/common/config/RecommendationPropertiesBindingTest.java b/src/test/java/com/maple/api/common/config/RecommendationPropertiesBindingTest.java new file mode 100644 index 0000000..f079648 --- /dev/null +++ b/src/test/java/com/maple/api/common/config/RecommendationPropertiesBindingTest.java @@ -0,0 +1,42 @@ +package com.maple.api.common.config; + +import com.maple.api.map.domain.RecommendationEngineType; +import org.junit.jupiter.api.Test; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.annotation.Configuration; + +import static org.assertj.core.api.Assertions.assertThat; + +class RecommendationPropertiesBindingTest { + + private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() + .withUserConfiguration(TestConfiguration.class); + + @Test + void bindsSupportedEngineCaseInsensitively() { + contextRunner + .withPropertyValues("recommendation.v1-engine=mysql") + .run(context -> { + assertThat(context).hasNotFailed(); + assertThat(context.getBean(RecommendationProperties.class).getV1Engine()) + .isEqualTo(RecommendationEngineType.MYSQL); + }); + } + + @Test + void rejectsUnsupportedEngineDuringConfigurationBinding() { + contextRunner + .withPropertyValues("recommendation.v1-engine=not-an-engine") + .run(context -> { + assertThat(context).hasFailed(); + assertThat(context.getStartupFailure()) + .hasStackTraceContaining("recommendation.v1-engine"); + }); + } + + @Configuration(proxyBeanMethods = false) + @EnableConfigurationProperties(RecommendationProperties.class) + static class TestConfiguration { + } +} diff --git a/src/test/java/com/maple/api/map/application/MapRecommendationEnrichmentServiceTest.java b/src/test/java/com/maple/api/map/application/MapRecommendationEnrichmentServiceTest.java new file mode 100644 index 0000000..f64cdce --- /dev/null +++ b/src/test/java/com/maple/api/map/application/MapRecommendationEnrichmentServiceTest.java @@ -0,0 +1,77 @@ +package com.maple.api.map.application; + +import com.maple.api.bookmark.application.BookmarkFlagService; +import com.maple.api.bookmark.domain.BookmarkType; +import com.maple.api.map.application.dto.MapRecommendationResultDto; +import com.maple.api.map.domain.Map; +import com.maple.api.map.domain.RecommendationCandidate; +import com.maple.api.map.repository.MapRepository; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +class MapRecommendationEnrichmentServiceTest { + + private final MapRepository mapRepository = mock(MapRepository.class); + private final BookmarkFlagService bookmarkFlagService = mock(BookmarkFlagService.class); + private final MapRecommendationEnrichmentService service = + new MapRecommendationEnrichmentService(mapRepository, bookmarkFlagService); + + @Test + void bulkLoadsMapsAndBookmarksOnceAndPreservesScoreOrder() { + List candidates = List.of( + new RecommendationCandidate(200L, 2.0d, List.of()), + new RecommendationCandidate(100L, 1.0d, List.of()) + ); + when(mapRepository.findByMapIdIn(List.of(200, 100))).thenReturn(List.of( + map(100, "low"), + map(200, "high") + )); + when(bookmarkFlagService.findBookmarkIds("member", BookmarkType.MAP, List.of(200, 100))) + .thenReturn(java.util.Map.of(100, 10)); + + var result = service.enrich("member", candidates); + + assertThat(result).extracting(MapRecommendationResultDto::mapId) + .containsExactly(200, 100); + assertThat(result).extracting(MapRecommendationResultDto::bookmarkId) + .containsExactly(null, 10); + verify(mapRepository).findByMapIdIn(List.of(200, 100)); + verify(bookmarkFlagService).findBookmarkIds("member", BookmarkType.MAP, List.of(200, 100)); + } + + @Test + void emptyCandidatesSkipAllEnrichmentQueries() { + assertThat(service.enrich("member", List.of())).isEmpty(); + verifyNoInteractions(mapRepository, bookmarkFlagService); + } + + @Test + void auraCandidateWithoutCanonicalMapKeepsLegacyNullableEnrichmentContract() { + List candidates = List.of( + new RecommendationCandidate(100L, 1.0d, List.of()) + ); + when(mapRepository.findByMapIdIn(List.of(100))).thenReturn(List.of()); + when(bookmarkFlagService.findBookmarkIds("member", BookmarkType.MAP, List.of(100))) + .thenReturn(java.util.Map.of(100, 17)); + + assertThat(service.enrich("member", candidates)).singleElement().satisfies(candidate -> { + assertThat(candidate.mapId()).isEqualTo(100); + assertThat(candidate.score()).isEqualTo(1.0d); + assertThat(candidate.iconUrl()).isNull(); + assertThat(candidate.nameKr()).isNull(); + assertThat(candidate.bookmarkId()).isEqualTo(17); + }); + verify(bookmarkFlagService).findBookmarkIds("member", BookmarkType.MAP, List.of(100)); + } + + private Map map(int mapId, String name) { + return new Map(mapId, name, null, null, null, null, null, "icon-" + mapId); + } +} diff --git a/src/test/java/com/maple/api/map/application/MapRecommendationObservabilityTest.java b/src/test/java/com/maple/api/map/application/MapRecommendationObservabilityTest.java new file mode 100644 index 0000000..0353abf --- /dev/null +++ b/src/test/java/com/maple/api/map/application/MapRecommendationObservabilityTest.java @@ -0,0 +1,53 @@ +package com.maple.api.map.application; + +import com.maple.api.map.domain.RecommendationEngineType; +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.junit.jupiter.api.Test; + +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; + +class MapRecommendationObservabilityTest { + + @Test + void exposesOnlyBoundedEngineApiVersionAndOutcomeTags() { + SimpleMeterRegistry registry = new SimpleMeterRegistry(); + MapRecommendationObservability observability = new MapRecommendationObservability(registry); + + observability.completed(RecommendationEngineType.MYSQL, "v2", 0, 100L); + observability.completed(RecommendationEngineType.MYSQL, "v2", 3, 200L); + observability.disabled(RecommendationEngineType.MYSQL, "v2"); + observability.unavailable(RecommendationEngineType.AURA, "v1", 300L, new RuntimeException("down")); + + assertThat(registry.get(MapRecommendationObservability.REQUESTS_METRIC) + .tags("engine", "mysql", "api_version", "v2", "outcome", "empty") + .counter().count()).isEqualTo(1.0d); + assertThat(registry.get(MapRecommendationObservability.REQUESTS_METRIC) + .tags("engine", "mysql", "api_version", "v2", "outcome", "success") + .counter().count()).isEqualTo(1.0d); + assertThat(registry.get(MapRecommendationObservability.REQUESTS_METRIC) + .tags("engine", "mysql", "api_version", "v2", "outcome", "unavailable") + .counter().count()).isEqualTo(1.0d); + assertThat(registry.get(MapRecommendationObservability.REQUESTS_METRIC) + .tags("engine", "aura", "api_version", "v1", "outcome", "unavailable") + .counter().count()).isEqualTo(1.0d); + assertThat(registry.get(MapRecommendationObservability.RESULTS_METRIC) + .tags("engine", "mysql", "api_version", "v2") + .summary().count()).isEqualTo(2L); + assertThat(registry.get(MapRecommendationObservability.RESULTS_METRIC) + .tags("engine", "mysql", "api_version", "v2") + .summary().totalAmount()).isEqualTo(3.0d); + + Set allowedTags = Set.of("engine", "api_version", "outcome"); + assertThat(registry.getMeters()) + .flatMap(meter -> meter.getId().getTags()) + .extracting(io.micrometer.core.instrument.Tag::getKey) + .allMatch(allowedTags::contains); + assertThat(registry.getMeters()) + .extracting(Meter::getId) + .extracting(Meter.Id::getName) + .allMatch(name -> name.startsWith("mapleland.recommendation.")); + } +} diff --git a/src/test/java/com/maple/api/map/application/MapRecommendationServiceTest.java b/src/test/java/com/maple/api/map/application/MapRecommendationServiceTest.java new file mode 100644 index 0000000..59c3f46 --- /dev/null +++ b/src/test/java/com/maple/api/map/application/MapRecommendationServiceTest.java @@ -0,0 +1,224 @@ +package com.maple.api.map.application; + +import com.maple.api.common.presentation.exception.ApiException; +import com.maple.api.map.application.dto.MapRecommendationResultDto; +import com.maple.api.job.exception.JobException; +import com.maple.api.job.repository.JobRepository; +import com.maple.api.map.exception.MapException; +import com.maple.api.common.config.RecommendationProperties; +import com.maple.api.map.domain.RecommendationCandidate; +import com.maple.api.map.domain.RecommendationReason; +import com.maple.api.map.repository.MapRecommendationRepository; +import com.maple.api.map.domain.RecommendationEngineType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.transaction.CannotCreateTransactionException; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.TransactionStatus; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +class MapRecommendationServiceTest { + + private JobRepository jobRepository; + private MapRecommendationRepository aura; + private MapRecommendationRepository mysql; + private MapRecommendationEnrichmentService enrichmentService; + private MapRecommendationObservability observability; + private RecommendationProperties properties; + private PlatformTransactionManager transactionManager; + private TransactionStatus transactionStatus; + private MapRecommendationService service; + + @BeforeEach + void setUp() { + jobRepository = mock(JobRepository.class); + aura = engine(RecommendationEngineType.AURA); + mysql = engine(RecommendationEngineType.MYSQL); + enrichmentService = mock(MapRecommendationEnrichmentService.class); + observability = mock(MapRecommendationObservability.class); + properties = new RecommendationProperties(); + properties.setV2Enabled(true); + transactionManager = mock(PlatformTransactionManager.class); + transactionStatus = mock(TransactionStatus.class); + when(transactionManager.getTransaction(any(TransactionDefinition.class))) + .thenReturn(transactionStatus); + MapRecommendationQueryExecutor queryExecutor = new MapRecommendationQueryExecutor( + jobRepository, + new MapRecommendationEngineRouter(List.of(aura, mysql)), + enrichmentService, + transactionManager, + properties + ); + service = new MapRecommendationService( + properties, + queryExecutor, + observability + ); + clearInvocations(aura, mysql); + } + + @Test + void v1UsesConfiguredAuraAndDefaultLimitFive() { + when(jobRepository.existsById(110)).thenReturn(true); + RecommendationCandidate candidate = new RecommendationCandidate(100L, 0.95d, List.of()); + when(aura.findRecommendations(45, 110, 5)).thenReturn(List.of(candidate)); + when(enrichmentService.enrich("member", List.of(candidate))).thenReturn(List.of( + new MapRecommendationResultDto(100, 0.95d, "icon", "map", 7, List.of()) + )); + + var result = service.recommendV1("member", 45, 110, null); + + assertThat(result).singleElement().satisfies(item -> { + assertThat(item.mapId()).isEqualTo(100); + assertThat(item.score()).isEqualTo(0.95d); + assertThat(item.bookmarkId()).isEqualTo(7); + }); + verify(aura).findRecommendations(45, 110, 5); + verify(transactionManager).getTransaction(org.mockito.ArgumentMatchers.argThat(definition -> + definition.isReadOnly() + && definition.getTimeout() == TransactionDefinition.TIMEOUT_DEFAULT)); + verify(observability).completed( + org.mockito.ArgumentMatchers.eq(RecommendationEngineType.AURA), + org.mockito.ArgumentMatchers.eq("v1"), + org.mockito.ArgumentMatchers.eq(1), + anyLong() + ); + } + + @Test + void v1CanSelectMysqlWithoutDualReadOrFallback() { + properties.setV1Engine(RecommendationEngineType.MYSQL); + when(jobRepository.existsById(110)).thenReturn(true); + when(mysql.findRecommendations(45, 110, 20)).thenReturn(List.of()); + when(enrichmentService.enrich(null, List.of())).thenReturn(List.of()); + + assertThat(service.recommendV1(null, 45, 110, 20)).isEmpty(); + + verify(mysql).findRecommendations(45, 110, 20); + verify(transactionManager).getTransaction(org.mockito.ArgumentMatchers.argThat(definition -> + definition.isReadOnly() + && definition.getTimeout() == properties.getQueryTimeoutSeconds())); + verifyNoInteractions(aura); + } + + @Test + void v2AlwaysUsesMysqlAndMapsNonNullReasons() { + when(jobRepository.existsById(110)).thenReturn(true); + RecommendationCandidate candidate = new RecommendationCandidate( + 100L, + 0.95d, + List.of(new RecommendationReason("reward", "xp")) + ); + when(mysql.findRecommendations(45, 110, 1)).thenReturn(List.of(candidate)); + when(enrichmentService.enrich(null, List.of(candidate))).thenReturn(List.of( + new MapRecommendationResultDto( + 100, + 0.95d, + "icon", + "map", + null, + List.of(new RecommendationReason("reward", "xp")) + ) + )); + + var result = service.recommendV2(null, 45, 110, 1); + + assertThat(result).singleElement().satisfies(item -> { + assertThat(item.reasons()).isNotNull(); + assertThat(item.reasons()).singleElement().satisfies(reason -> { + assertThat(reason.axis()).isEqualTo("reward"); + assertThat(reason.value()).isEqualTo("xp"); + }); + }); + verifyNoInteractions(aura); + verify(transactionManager, atLeastOnce()).getTransaction( + org.mockito.ArgumentMatchers.argThat(definition -> + definition.getTimeout() == properties.getQueryTimeoutSeconds())); + } + + @Test + void missingJobKeepsExistingNotFoundContract() { + when(jobRepository.existsById(999)).thenReturn(false); + + assertThatThrownBy(() -> service.recommendV2(null, 45, 999, null)) + .isInstanceOfSatisfying(ApiException.class, exception -> + assertThat(exception.getExceptionCode()).isEqualTo(JobException.JOB_NOT_FOUND)); + + verifyNoInteractions(mysql, enrichmentService, observability); + } + + @Test + void selectedEngineFailureBecomesRecommendationUnavailableWithoutFallback() { + when(jobRepository.existsById(110)).thenReturn(true); + RuntimeException databaseFailure = new RuntimeException("schema missing"); + when(mysql.findRecommendations(45, 110, 5)).thenThrow(databaseFailure); + + assertThatThrownBy(() -> service.recommendV2(null, 45, 110, null)) + .isInstanceOfSatisfying(ApiException.class, exception -> { + assertThat(exception.getExceptionCode()).isEqualTo(MapException.MAP_RECOMMENDATION_UNAVAILABLE); + assertThat(exception.getCause()).isSameAs(databaseFailure); + }); + + verify(observability).unavailable( + org.mockito.ArgumentMatchers.eq(RecommendationEngineType.MYSQL), + org.mockito.ArgumentMatchers.eq("v2"), + anyLong(), + org.mockito.ArgumentMatchers.same(databaseFailure) + ); + verifyNoInteractions(aura, enrichmentService); + } + + @Test + void v2KillSwitchReturnsUnavailableWithoutTouchingDatabaseOrEngine() { + properties.setV2Enabled(false); + + assertThatThrownBy(() -> service.recommendV2(null, 45, 110, null)) + .isInstanceOfSatisfying(ApiException.class, exception -> + assertThat(exception.getExceptionCode()) + .isEqualTo(MapException.MAP_RECOMMENDATION_UNAVAILABLE)); + + verify(observability).disabled(RecommendationEngineType.MYSQL, "v2"); + verifyNoInteractions(jobRepository, aura, mysql, enrichmentService); + } + + @Test + void transactionStartFailureBecomesUnavailableAndIsObserved() { + CannotCreateTransactionException transactionFailure = + new CannotCreateTransactionException("database unavailable"); + when(transactionManager.getTransaction(any(TransactionDefinition.class))) + .thenThrow(transactionFailure); + + assertThatThrownBy(() -> service.recommendV2(null, 45, 110, null)) + .isInstanceOfSatisfying(ApiException.class, exception -> { + assertThat(exception.getExceptionCode()) + .isEqualTo(MapException.MAP_RECOMMENDATION_UNAVAILABLE); + assertThat(exception.getCause()).isSameAs(transactionFailure); + }); + + verify(observability).unavailable( + org.mockito.ArgumentMatchers.eq(RecommendationEngineType.MYSQL), + org.mockito.ArgumentMatchers.eq("v2"), + anyLong(), + org.mockito.ArgumentMatchers.same(transactionFailure) + ); + } + + private MapRecommendationRepository engine(RecommendationEngineType type) { + MapRecommendationRepository repository = mock(MapRecommendationRepository.class); + when(repository.engineType()).thenReturn(type); + return repository; + } +} diff --git a/src/test/java/com/maple/api/map/application/command/AuraDbKeepAliveBatchContextTest.java b/src/test/java/com/maple/api/map/application/command/AuraDbKeepAliveBatchContextTest.java new file mode 100644 index 0000000..4176690 --- /dev/null +++ b/src/test/java/com/maple/api/map/application/command/AuraDbKeepAliveBatchContextTest.java @@ -0,0 +1,43 @@ +package com.maple.api.map.application.command; + +import com.maple.api.map.repository.AuraMapRecommendationRepository; +import org.junit.jupiter.api.Test; +import org.neo4j.driver.Driver; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +class AuraDbKeepAliveBatchContextTest { + + private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() + .withPropertyValues("batch.auradb-keep-alive.enabled=true") + .withUserConfiguration(AuraComponents.class); + + @Test + void createsRepositoryAndKeepAliveWhenDriverIsAvailable() { + contextRunner + .withBean(Driver.class, () -> mock(Driver.class)) + .run(context -> { + assertThat(context).hasNotFailed(); + assertThat(context).hasSingleBean(AuraMapRecommendationRepository.class); + assertThat(context).hasSingleBean(AuraDbKeepAliveBatch.class); + }); + } + + @Test + void startsWithoutAuraBeansWhenDriverIsUnavailable() { + contextRunner.run(context -> { + assertThat(context).hasNotFailed(); + assertThat(context).doesNotHaveBean(AuraMapRecommendationRepository.class); + assertThat(context).doesNotHaveBean(AuraDbKeepAliveBatch.class); + }); + } + + @Configuration(proxyBeanMethods = false) + @Import({AuraMapRecommendationRepository.class, AuraDbKeepAliveBatch.class}) + static class AuraComponents { + } +} diff --git a/src/test/java/com/maple/api/map/domain/RecommendationScoringServiceTest.java b/src/test/java/com/maple/api/map/domain/RecommendationScoringServiceTest.java new file mode 100644 index 0000000..c8101a1 --- /dev/null +++ b/src/test/java/com/maple/api/map/domain/RecommendationScoringServiceTest.java @@ -0,0 +1,278 @@ +package com.maple.api.map.domain; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.within; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; + +class RecommendationScoringServiceTest { + + private static final LocalDateTime PUBLISHED_AT = LocalDateTime.of(2025, 1, 1, 12, 0); + + private final RecommendationScoringService scoringService = new RecommendationScoringService(); + + @Test + void sumsPositiveAndNegativeEvidenceWithPatchFreshness() { + List evidence = List.of( + evidence(1, 1, 100, 110, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT), + evidence(2, 2, 100, 110, 40, 50, Polarity.POSITIVE, 2, PUBLISHED_AT), + evidence(3, 3, 100, 110, 40, 50, Polarity.NEGATIVE, 1, PUBLISHED_AT) + ); + + List result = scoringService.score(45, evidence, 5); + + assertThat(result).singleElement().satisfies(candidate -> { + assertThat(candidate.mapId()).isEqualTo(100); + assertThat(candidate.score()).isCloseTo(0.95d, within(0.000_000_1d)); + }); + } + + @Test + void matchesInclusiveLevelRangesAndInfersMissingBoundByTenLevels() { + List evidence = List.of( + evidence(1, 1, 100, 110, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT), + evidence(2, 2, 100, 110, 50, null, Polarity.POSITIVE, 1, PUBLISHED_AT), + evidence(3, 3, 100, 110, null, 50, Polarity.POSITIVE, 2, PUBLISHED_AT), + evidence(4, 4, 100, 110, 51, null, Polarity.POSITIVE, 0, PUBLISHED_AT), + evidence(5, 5, 100, 110, null, 49, Polarity.POSITIVE, 0, PUBLISHED_AT), + evidence(6, 6, 100, 110, null, null, Polarity.POSITIVE, 0, PUBLISHED_AT) + ); + + List result = scoringService.score(50, evidence, 5); + + assertThat(result).singleElement() + .extracting(RecommendationCandidate::score) + .isEqualTo(2.85d); + } + + @Test + void floorsFreshnessAtPointOneAndAlwaysReturnsFiniteScores() { + List evidence = List.of( + evidence(1, 1, 100, 110, 1, 200, Polarity.POSITIVE, 18, PUBLISHED_AT), + evidence(2, 2, 100, 110, 1, 200, Polarity.POSITIVE, 19, PUBLISHED_AT), + evidence(3, 3, 100, 110, 1, 200, Polarity.POSITIVE, Integer.MAX_VALUE, PUBLISHED_AT) + ); + + RecommendationCandidate candidate = scoringService.score(100, evidence, 5).getFirst(); + + assertThat(candidate.score()).isEqualTo(0.3d); + assertThat(Double.isFinite(candidate.score())).isTrue(); + } + + @Test + void deduplicatesLineageFanoutByExtractedClaimAndMapUsingLowestReviewId() { + List evidence = List.of( + evidence(10, 20, 100, 112, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT), + evidence(10, 10, 100, 110, 40, 50, Polarity.POSITIVE, 2, PUBLISHED_AT), + evidence(10, 30, 101, 110, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT) + ); + + List result = scoringService.score(45, evidence, 5); + + assertThat(result) + .extracting(RecommendationCandidate::mapId, RecommendationCandidate::score) + .containsExactly( + org.assertj.core.groups.Tuple.tuple(101L, 1.0d), + org.assertj.core.groups.Tuple.tuple(100L, 0.9d) + ); + } + + @Test + void requiresMatchedPositiveEvidenceAndPositiveNetScore() { + List evidence = List.of( + evidence(1, 1, 100, 110, 40, 50, Polarity.NEGATIVE, 0, PUBLISHED_AT), + evidence(2, 2, 101, 110, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT), + evidence(3, 3, 101, 110, 40, 50, Polarity.NEGATIVE, 0, PUBLISHED_AT), + evidence(4, 4, 102, 110, 40, 50, Polarity.POSITIVE, 2, PUBLISHED_AT), + evidence(5, 5, 102, 110, 40, 50, Polarity.NEGATIVE, 0, PUBLISHED_AT), + evidence(6, 6, 103, 110, 40, 50, Polarity.POSITIVE, 19, PUBLISHED_AT) + ); + + List result = scoringService.score(45, evidence, 5); + + assertThat(result).singleElement() + .extracting(RecommendationCandidate::mapId, RecommendationCandidate::score) + .containsExactly(103L, 0.1d); + } + + @Test + void aggregatesSignedFacetWeightsOmitsNonPositiveAxesAndUsesStableAxisOrder() { + List evidence = List.of( + evidence(1, 1, 100, 110, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT, + RecommendationFacet.REWARD_XP, + RecommendationFacet.PLAY_STYLE_SOLO, + RecommendationFacet.OPERABILITY_FATIGUE), + evidence(2, 2, 100, 110, 40, 50, Polarity.NEGATIVE, 0, PUBLISHED_AT, + RecommendationFacet.REWARD_XP, + RecommendationFacet.OPERABILITY_FATIGUE), + evidence(3, 3, 100, 110, 40, 50, Polarity.POSITIVE, 2, PUBLISHED_AT, + RecommendationFacet.REWARD_MESO, + RecommendationFacet.PLAY_STYLE_PARTY) + ); + + RecommendationCandidate candidate = scoringService.score(45, evidence, 5).getFirst(); + + assertThat(candidate.reasons()).containsExactly( + new RecommendationReason("reward", "meso"), + new RecommendationReason("play_style", "solo") + ); + } + + @Test + void usesFacetPriorityWhenAccumulatedWeightsTie() { + List evidence = List.of( + evidence(1, 1, 100, 110, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT, + RecommendationFacet.REWARD_XP, + RecommendationFacet.PLAY_STYLE_PARTY, + RecommendationFacet.OPERABILITY_BUDGET), + evidence(2, 2, 100, 110, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT, + RecommendationFacet.REWARD_MESO, + RecommendationFacet.PLAY_STYLE_SOLO, + RecommendationFacet.OPERABILITY_MOBILITY) + ); + + RecommendationCandidate candidate = scoringService.score(45, evidence, 5).getFirst(); + + assertThat(candidate.reasons()).containsExactly( + new RecommendationReason("reward", "xp"), + new RecommendationReason("play_style", "solo"), + new RecommendationReason("operability", "mobility") + ); + } + + @Test + void sortsHigherScoreFirstEvenWhenLowerScoreHasHigherFreshnessSum() { + List evidence = List.of( + evidence(1, 1, 100, 110, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT), + evidence(2, 2, 101, 110, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT.plusDays(1)), + evidence(3, 3, 101, 110, 40, 50, Polarity.NEGATIVE, 2, PUBLISHED_AT.plusDays(1)) + ); + + List result = scoringService.score(45, evidence, 5); + + assertThat(result).extracting(RecommendationCandidate::mapId).containsExactly(100L, 101L); + assertThat(result).extracting(RecommendationCandidate::score).containsExactly(1.0d, 0.1d); + } + + @Test + void breaksEqualScoreByHigherFreshnessSum() { + List evidence = List.of( + evidence(1, 1, 100, 110, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT), + evidence(2, 2, 101, 110, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT), + evidence(3, 3, 101, 110, 40, 50, Polarity.POSITIVE, 2, PUBLISHED_AT), + evidence(4, 4, 101, 110, 40, 50, Polarity.NEGATIVE, 2, PUBLISHED_AT) + ); + + List result = scoringService.score(45, evidence, 5); + + assertThat(result).extracting(RecommendationCandidate::mapId).containsExactly(101L, 100L); + } + + @Test + void representativeUsesNewestEvidenceWhenHighestContributionsTie() { + List evidence = List.of( + evidence(1, 1, 100, 110, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT), + evidence(2, 2, 100, 110, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT.plusDays(3)), + evidence(3, 3, 101, 110, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT), + evidence(4, 4, 101, 110, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT.plusDays(2)) + ); + + List result = scoringService.score(45, evidence, 5); + + assertThat(result).extracting(RecommendationCandidate::mapId).containsExactly(100L, 101L); + } + + @Test + void representativeDateComesFromHighestContributionRatherThanNewestEvidence() { + List evidence = List.of( + evidence(1, 1, 100, 110, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT), + evidence(2, 2, 100, 110, 40, 50, Polarity.POSITIVE, 2, PUBLISHED_AT.plusDays(10)), + evidence(3, 3, 101, 110, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT.plusDays(1)), + evidence(4, 4, 101, 110, 40, 50, Polarity.POSITIVE, 2, PUBLISHED_AT.minusDays(1)) + ); + + List result = scoringService.score(45, evidence, 5); + + assertThat(result).extracting(RecommendationCandidate::mapId).containsExactly(101L, 100L); + } + + @Test + void breaksRemainingTiesByMapIdAscending() { + List evidence = List.of( + evidence(1, 1, 200, 110, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT), + evidence(2, 2, 100, 110, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT) + ); + + List result = scoringService.score(45, evidence, 5); + + assertThat(result).extracting(RecommendationCandidate::mapId).containsExactly(100L, 200L); + } + + @Test + void appliesLimitOnlyAfterFinalSortingAndAcceptsContractBounds() { + List evidence = List.of( + evidence(1, 1, 100, 110, 40, 50, Polarity.POSITIVE, 2, PUBLISHED_AT), + evidence(2, 2, 101, 110, 40, 50, Polarity.POSITIVE, 1, PUBLISHED_AT), + evidence(3, 3, 102, 110, 40, 50, Polarity.POSITIVE, 0, PUBLISHED_AT) + ); + + assertThat(scoringService.score(45, evidence, 1)) + .extracting(RecommendationCandidate::mapId) + .containsExactly(102L); + assertThat(scoringService.score(45, evidence, 20)).hasSize(3); + assertThatThrownBy(() -> scoringService.score(45, evidence, 0)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("limit"); + assertThatThrownBy(() -> scoringService.score(45, evidence, 21)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("limit"); + } + + @Test + void returnedCollectionsAreImmutableSnapshots() { + ArrayList mutableFacets = new ArrayList<>(); + mutableFacets.add(RecommendationFacet.REWARD_XP); + RecommendationEvidence row = new RecommendationEvidence( + 1, 1, 100, 110, 40, 50, Polarity.POSITIVE, PUBLISHED_AT, 0, mutableFacets + ); + mutableFacets.clear(); + + List result = scoringService.score(45, List.of(row), 5); + + assertThat(result.getFirst().reasons()).containsExactly(new RecommendationReason("reward", "xp")); + assertThatThrownBy(() -> result.add(result.getFirst())) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> result.getFirst().reasons().clear()) + .isInstanceOf(UnsupportedOperationException.class); + } + + private RecommendationEvidence evidence( + long extractedClaimId, + long reviewedClaimId, + long mapId, + long jobId, + Integer levelMin, + Integer levelMax, + Polarity polarity, + int patchCount, + LocalDateTime publishedAt, + RecommendationFacet... facets + ) { + return new RecommendationEvidence( + extractedClaimId, + reviewedClaimId, + mapId, + jobId, + levelMin, + levelMax, + polarity, + publishedAt, + patchCount, + List.of(facets) + ); + } +} diff --git a/src/test/java/com/maple/api/map/presentation/MapRecommendationControllerContractTest.java b/src/test/java/com/maple/api/map/presentation/MapRecommendationControllerContractTest.java new file mode 100644 index 0000000..f4a5678 --- /dev/null +++ b/src/test/java/com/maple/api/map/presentation/MapRecommendationControllerContractTest.java @@ -0,0 +1,181 @@ +package com.maple.api.map.presentation; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.maple.api.auth.domain.PrincipalDetails; +import com.maple.api.common.presentation.exception.ApiException; +import com.maple.api.job.exception.JobException; +import com.maple.api.map.application.dto.MapRecommendationDto; +import com.maple.api.map.application.dto.MapRecommendationReasonDto; +import com.maple.api.map.application.dto.MapRecommendationV2Dto; +import com.maple.api.map.exception.MapException; +import com.maple.api.map.application.MapRecommendationService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.bean.override.mockito.MockitoBean; +import org.springframework.test.web.servlet.MockMvc; + +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@ActiveProfiles("test") +@AutoConfigureMockMvc +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +class MapRecommendationControllerContractTest { + + @Autowired + private MockMvc mockMvc; + + @Autowired + private ObjectMapper objectMapper; + + @MockitoBean + private MapRecommendationService recommendationService; + + @BeforeEach + void resetMock() { + reset(recommendationService); + } + + @Test + void v1KeepsExactOuterAndItemKeysWithoutReasonsForAnonymousCaller() throws Exception { + when(recommendationService.recommendV1(null, 45, 110, 5)).thenReturn(List.of( + new MapRecommendationDto(100000000, 0.95d, "https://icon", "헤네시스", null) + )); + + String json = mockMvc.perform(get("/api/v1/maps/recommendations") + .param("level", "45") + .param("jobId", "110") + .accept(MediaType.APPLICATION_JSON)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.success").value(true)) + .andExpect(jsonPath("$.code").value("SUCCESS")) + .andExpect(jsonPath("$.message").isEmpty()) + .andExpect(jsonPath("$.data[0].bookmarkId").isEmpty()) + .andExpect(jsonPath("$.data[0].reasons").doesNotExist()) + .andReturn().getResponse().getContentAsString(); + + JsonNode root = objectMapper.readTree(json); + assertThat(fieldNames(root)).containsExactlyInAnyOrder("success", "code", "message", "data"); + assertThat(root.get("message").isNull()).isTrue(); + assertThat(fieldNames(root.get("data").get(0))).containsExactlyInAnyOrder( + "mapId", "score", "iconUrl", "nameKr", "bookmarkId" + ); + verify(recommendationService).recommendV1(null, 45, 110, 5); + } + + @Test + void v1PassesLoggedInMemberAndReturnsBookmark() throws Exception { + when(recommendationService.recommendV1("member-1", 45, 110, 5)).thenReturn(List.of( + new MapRecommendationDto(100, 1.0d, "icon", "map", 77) + )); + + mockMvc.perform(get("/api/v1/maps/recommendations") + .param("level", "45") + .param("jobId", "110") + .param("limit", "5") + .with(user(new PrincipalDetails("member-1")))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.data[0].bookmarkId").value(77)); + + verify(recommendationService).recommendV1("member-1", 45, 110, 5); + } + + @Test + void v1SupportsContractLimitBoundsAndRejectsInvalidLimits() throws Exception { + when(recommendationService.recommendV1(null, 45, 110, 1)).thenReturn(List.of()); + when(recommendationService.recommendV1(null, 45, 110, 20)).thenReturn(List.of()); + + mockMvc.perform(get("/api/v1/maps/recommendations") + .param("level", "45").param("jobId", "110").param("limit", "1")) + .andExpect(status().isOk()); + mockMvc.perform(get("/api/v1/maps/recommendations") + .param("level", "45").param("jobId", "110").param("limit", "20")) + .andExpect(status().isOk()); + mockMvc.perform(get("/api/v1/maps/recommendations") + .param("level", "45").param("jobId", "110").param("limit", "0")) + .andExpect(status().isBadRequest()); + mockMvc.perform(get("/api/v1/maps/recommendations") + .param("level", "45").param("jobId", "110").param("limit", "21")) + .andExpect(status().isBadRequest()); + } + + @Test + void v1KeepsEmptyNotFoundAndUnavailableContracts() throws Exception { + when(recommendationService.recommendV1(null, 45, 110, 5)).thenReturn(List.of()); + mockMvc.perform(get("/api/v1/maps/recommendations") + .param("level", "45").param("jobId", "110")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.data").isArray()) + .andExpect(jsonPath("$.data").isEmpty()); + + when(recommendationService.recommendV1(null, 45, 999, 5)) + .thenThrow(ApiException.of(JobException.JOB_NOT_FOUND)); + mockMvc.perform(get("/api/v1/maps/recommendations") + .param("level", "45").param("jobId", "999")) + .andExpect(status().isNotFound()) + .andExpect(jsonPath("$.message").value(JobException.JOB_NOT_FOUND.getMessage())); + + when(recommendationService.recommendV1(null, 45, 500, 5)) + .thenThrow(ApiException.of(MapException.MAP_RECOMMENDATION_UNAVAILABLE)); + mockMvc.perform(get("/api/v1/maps/recommendations") + .param("level", "45").param("jobId", "500")) + .andExpect(status().isServiceUnavailable()) + .andExpect(jsonPath("$.message").value(MapException.MAP_RECOMMENDATION_UNAVAILABLE.getMessage())); + } + + @Test + void v2IsAnonymousAndReturnsSeparateDtoWithNonNullReasonsArrays() throws Exception { + when(recommendationService.recommendV2(null, 45, 110, 5)).thenReturn(List.of( + new MapRecommendationV2Dto( + 100, + 0.95d, + "icon", + "map", + null, + List.of(new MapRecommendationReasonDto("reward", "xp")) + ), + new MapRecommendationV2Dto(101, 0.9d, "icon2", "map2", null, List.of()) + )); + + String json = mockMvc.perform(get("/api/v2/maps/recommendations") + .param("level", "45") + .param("jobId", "110")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.data[0].reasons").isArray()) + .andExpect(jsonPath("$.data[0].reasons[0].axis").value("reward")) + .andExpect(jsonPath("$.data[0].reasons[0].value").value("xp")) + .andExpect(jsonPath("$.data[1].reasons").isArray()) + .andExpect(jsonPath("$.data[1].reasons").isEmpty()) + .andReturn().getResponse().getContentAsString(); + + JsonNode first = objectMapper.readTree(json).get("data").get(0); + assertThat(fieldNames(first)).containsExactlyInAnyOrder( + "mapId", "score", "iconUrl", "nameKr", "bookmarkId", "reasons" + ); + assertThat(fieldNames(first.get("reasons").get(0))).containsExactlyInAnyOrder("axis", "value"); + } + + private Set fieldNames(JsonNode node) { + Set names = new LinkedHashSet<>(); + Iterator iterator = node.fieldNames(); + iterator.forEachRemaining(names::add); + return names; + } +} diff --git a/src/test/java/com/maple/api/map/presentation/MapRecommendationOpenApiTest.java b/src/test/java/com/maple/api/map/presentation/MapRecommendationOpenApiTest.java new file mode 100644 index 0000000..3d90540 --- /dev/null +++ b/src/test/java/com/maple/api/map/presentation/MapRecommendationOpenApiTest.java @@ -0,0 +1,130 @@ +package com.maple.api.map.presentation; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.maple.api.map.application.MapRecommendationService; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.bean.override.mockito.MockitoBean; +import org.springframework.test.web.servlet.MockMvc; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@ActiveProfiles("test") +@AutoConfigureMockMvc +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +class MapRecommendationOpenApiTest { + + private static final String V1_SCHEMA = "com.maple.api.map.application.dto.MapRecommendationDto"; + private static final String V2_SCHEMA = "com.maple.api.map.application.dto.MapRecommendationV2Dto"; + private static final String REASON_SCHEMA = "com.maple.api.map.application.dto.MapRecommendationReasonDto"; + + @Autowired + private MockMvc mockMvc; + + @Autowired + private ObjectMapper objectMapper; + + @MockitoBean + private MapRecommendationService recommendationService; + + @Test + void documentsV1ExactSchemaAndEvidenceScoreSemanticChange() throws Exception { + JsonNode document = openApi(); + JsonNode operation = document.at("/paths/~1api~1v1~1maps~1recommendations/get"); + + assertThat(operation.isMissingNode()).isFalse(); + assertThat(operation.at("/responses/503").isMissingNode()).isFalse(); + assertLimitContract(operation); + assertOptionalAuthentication(operation); + + JsonNode schema = document.path("components").path("schemas").path(V1_SCHEMA); + assertThat(fieldNames(schema.path("properties"))).containsExactlyInAnyOrder( + "mapId", "score", "iconUrl", "nameKr", "bookmarkId" + ); + assertThat(schema.at("/properties/score/description").asText()) + .contains("evidence net score") + .contains("APPROVED"); + assertThat(schema.path("properties").has("reasons")).isFalse(); + } + + @Test + void documentsV2ReasonsAsSeparateStableCodeSchema() throws Exception { + JsonNode document = openApi(); + JsonNode operation = document.at("/paths/~1api~1v2~1maps~1recommendations/get"); + + assertThat(operation.isMissingNode()).isFalse(); + assertThat(operation.at("/responses/503").isMissingNode()).isFalse(); + assertLimitContract(operation); + assertOptionalAuthentication(operation); + + JsonNode v2Schema = document.path("components").path("schemas").path(V2_SCHEMA); + assertThat(fieldNames(v2Schema.path("properties"))).containsExactlyInAnyOrder( + "mapId", "score", "iconUrl", "nameKr", "bookmarkId", "reasons" + ); + assertThat(v2Schema.at("/properties/reasons/type").asText()).isEqualTo("array"); + assertThat(v2Schema.at("/properties/reasons/items/$ref").asText()).endsWith(REASON_SCHEMA); + + JsonNode reasonSchema = document.path("components").path("schemas").path(REASON_SCHEMA); + assertThat(strings(reasonSchema.at("/properties/axis/enum"))) + .containsExactly("reward", "play_style", "operability"); + assertThat(strings(reasonSchema.at("/properties/value/enum"))) + .containsExactly("xp", "meso", "loot", "solo", "party", "party_quest", "fatigue", "mobility", "budget"); + } + + private JsonNode openApi() throws Exception { + String json = mockMvc.perform(get("/v3/api-docs")) + .andExpect(status().isOk()) + .andReturn().getResponse().getContentAsString(); + return objectMapper.readTree(json); + } + + private void assertLimitContract(JsonNode operation) { + JsonNode limit = findParameter(operation, "limit"); + assertThat(limit.path("required").asBoolean()).isFalse(); + assertThat(limit.at("/schema/default").asInt()).isEqualTo(5); + assertThat(limit.at("/schema/minimum").asInt()).isEqualTo(1); + assertThat(limit.at("/schema/maximum").asInt()).isEqualTo(20); + } + + private void assertOptionalAuthentication(JsonNode operation) { + JsonNode security = operation.path("security"); + assertThat(security.isArray()).isTrue(); + assertThat(security).anySatisfy(requirement -> assertThat(requirement.isEmpty()).isTrue()); + assertThat(security).anySatisfy(requirement -> + assertThat(requirement.has("Authorization")).isTrue()); + } + + private JsonNode findParameter(JsonNode operation, String name) { + for (JsonNode parameter : operation.path("parameters")) { + if (name.equals(parameter.path("name").asText())) { + return parameter; + } + } + throw new AssertionError("Missing OpenAPI parameter: " + name); + } + + private Set fieldNames(JsonNode node) { + Set names = new LinkedHashSet<>(); + Iterator iterator = node.fieldNames(); + iterator.forEachRemaining(names::add); + return names; + } + + private List strings(JsonNode array) { + List values = new ArrayList<>(); + array.forEach(value -> values.add(value.asText())); + return values; + } +} diff --git a/src/test/java/com/maple/api/map/presentation/RecommendationInfrastructureUnavailableIntegrationTest.java b/src/test/java/com/maple/api/map/presentation/RecommendationInfrastructureUnavailableIntegrationTest.java new file mode 100644 index 0000000..bc5ebf4 --- /dev/null +++ b/src/test/java/com/maple/api/map/presentation/RecommendationInfrastructureUnavailableIntegrationTest.java @@ -0,0 +1,54 @@ +package com.maple.api.map.presentation; + +import com.maple.api.map.exception.MapException; +import com.maple.api.map.application.MapRecommendationQueryExecutor; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.bean.override.mockito.MockitoBean; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.transaction.CannotCreateTransactionException; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@ActiveProfiles("test") +@AutoConfigureMockMvc +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +class RecommendationInfrastructureUnavailableIntegrationTest { + + @Autowired + private MockMvc mockMvc; + + @MockitoBean + private MapRecommendationQueryExecutor queryExecutor; + + @BeforeEach + void failAtTransactionalProxyBoundary() { + when(queryExecutor.execute(isNull(), anyInt(), anyInt(), anyInt(), any())) + .thenThrow(new CannotCreateTransactionException("database unavailable")); + } + + @Test + void transactionStartFailureReturnsRecommendation503ForV1AndV2() throws Exception { + assertUnavailable("/api/v1/maps/recommendations"); + assertUnavailable("/api/v2/maps/recommendations"); + } + + private void assertUnavailable(String path) throws Exception { + mockMvc.perform(get(path) + .param("level", "50") + .param("jobId", "111")) + .andExpect(status().isServiceUnavailable()) + .andExpect(jsonPath("$.message") + .value(MapException.MAP_RECOMMENDATION_UNAVAILABLE.getMessage())); + } +} diff --git a/src/test/java/com/maple/api/map/presentation/RecommendationJobLookupUnavailableIntegrationTest.java b/src/test/java/com/maple/api/map/presentation/RecommendationJobLookupUnavailableIntegrationTest.java new file mode 100644 index 0000000..20b67c0 --- /dev/null +++ b/src/test/java/com/maple/api/map/presentation/RecommendationJobLookupUnavailableIntegrationTest.java @@ -0,0 +1,52 @@ +package com.maple.api.map.presentation; + +import com.maple.api.job.repository.JobRepository; +import com.maple.api.map.exception.MapException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.dao.DataAccessResourceFailureException; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.bean.override.mockito.MockitoBean; +import org.springframework.test.web.servlet.MockMvc; + +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@ActiveProfiles("test") +@AutoConfigureMockMvc +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +class RecommendationJobLookupUnavailableIntegrationTest { + + @Autowired + private MockMvc mockMvc; + + @MockitoBean + private JobRepository jobRepository; + + @BeforeEach + void failJobLookup() { + when(jobRepository.existsById(anyInt())) + .thenThrow(new DataAccessResourceFailureException("datasource unavailable")); + } + + @Test + void jobDatasourceFailureReturnsRecommendation503ForV1AndV2() throws Exception { + for (String path : new String[]{ + "/api/v1/maps/recommendations", + "/api/v2/maps/recommendations" + }) { + mockMvc.perform(get(path) + .param("level", "50") + .param("jobId", "110")) + .andExpect(status().isServiceUnavailable()) + .andExpect(jsonPath("$.message") + .value(MapException.MAP_RECOMMENDATION_UNAVAILABLE.getMessage())); + } + } +} diff --git a/src/test/java/com/maple/api/map/presentation/RecommendationSchemaUnavailableIntegrationTest.java b/src/test/java/com/maple/api/map/presentation/RecommendationSchemaUnavailableIntegrationTest.java new file mode 100644 index 0000000..211651f --- /dev/null +++ b/src/test/java/com/maple/api/map/presentation/RecommendationSchemaUnavailableIntegrationTest.java @@ -0,0 +1,62 @@ +package com.maple.api.map.presentation; + +import com.maple.api.map.exception.MapException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.web.servlet.MockMvc; + +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@ActiveProfiles("test") +@AutoConfigureMockMvc +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = { + "spring.datasource.url=jdbc:h2:mem:recommendation-unavailable;DB_CLOSE_DELAY=-1;MODE=MYSQL", + "recommendation.v1-engine=MYSQL" + } +) +class RecommendationSchemaUnavailableIntegrationTest { + + @Autowired + private MockMvc mockMvc; + + @Autowired + private JdbcTemplate jdbcTemplate; + + @BeforeEach + void insertExistingJobWithoutRecommendationSchema() { + jdbcTemplate.update(""" + INSERT INTO jobs(job_id, job_name, job_level, parent_job_id, disabled) + VALUES (111, 'Crusader', 3, NULL, FALSE) + """); + } + + @Test + void selectedMysqlEngineReturns503WhileApplicationAndOtherEndpointsRemainAvailable() throws Exception { + mockMvc.perform(get("/api/v2/maps/recommendations") + .param("level", "50") + .param("jobId", "111")) + .andExpect(status().isServiceUnavailable()) + .andExpect(jsonPath("$.message") + .value(MapException.MAP_RECOMMENDATION_UNAVAILABLE.getMessage())); + + mockMvc.perform(get("/api/v1/maps/recommendations") + .param("level", "50") + .param("jobId", "111")) + .andExpect(status().isServiceUnavailable()) + .andExpect(jsonPath("$.message") + .value(MapException.MAP_RECOMMENDATION_UNAVAILABLE.getMessage())); + + mockMvc.perform(get("/api/v1/jobs")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.success").value(true)); + } +} diff --git a/src/test/java/com/maple/api/map/repository/MySqlMapRecommendationRepositoryIntegrationTest.java b/src/test/java/com/maple/api/map/repository/MySqlMapRecommendationRepositoryIntegrationTest.java new file mode 100644 index 0000000..98d9dcf --- /dev/null +++ b/src/test/java/com/maple/api/map/repository/MySqlMapRecommendationRepositoryIntegrationTest.java @@ -0,0 +1,395 @@ +package com.maple.api.map.repository; + +import com.maple.api.map.domain.RecommendationCandidate; +import com.maple.api.map.domain.RecommendationReason; +import com.maple.api.map.domain.RecommendationScoringService; +import com.maple.api.common.config.RecommendationConfig; +import com.maple.api.common.config.RecommendationProperties; +import net.ttddyy.dsproxy.QueryCountHolder; +import net.ttddyy.dsproxy.support.ProxyDataSourceBuilder; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.springframework.core.io.ClassPathResource; +import org.springframework.dao.DataAccessException; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.jdbc.datasource.DriverManagerDataSource; +import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; +import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@Testcontainers +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +class MySqlMapRecommendationRepositoryIntegrationTest { + + private static final int PERFORMANCE_EVIDENCE_COUNT = 3_000; + private static final int PERFORMANCE_CANDIDATE_COUNT = 100; + + @Container + private static final MySQLContainer MYSQL = new MySQLContainer<>(DockerImageName.parse("mysql:8.4")) + .withDatabaseName("mapledb") + .withUsername("maple") + .withPassword("maple"); + + private static DataSource rawDataSource; + private static JdbcTemplate rawJdbc; + private static MySqlMapRecommendationRepository repository; + + @BeforeAll + static void setUpDatabase() throws Exception { + DriverManagerDataSource mysqlDataSource = new DriverManagerDataSource(); + mysqlDataSource.setDriverClassName(MYSQL.getDriverClassName()); + mysqlDataSource.setUrl(MYSQL.getJdbcUrl()); + mysqlDataSource.setUsername(MYSQL.getUsername()); + mysqlDataSource.setPassword(MYSQL.getPassword()); + rawDataSource = mysqlDataSource; + rawJdbc = new JdbcTemplate(rawDataSource); + + new ResourceDatabasePopulator( + new ClassPathResource("sql/recommendation-mysql8-schema.sql") + ).execute(rawDataSource); + seedContractFixture(); + seedPerformanceFixture(); + + DataSource countedDataSource = ProxyDataSourceBuilder + .create("recommendation-mysql8", rawDataSource) + .countQuery() + .build(); + repository = new MySqlMapRecommendationRepository( + new NamedParameterJdbcTemplate(countedDataSource), + new RecommendationScoringService() + ); + } + + @Test + @Order(1) + void executesMysql8CteAndWindowWithOneRoundTripAndRecordsRepresentativeLatency() { + assertThat(rawJdbc.queryForObject(""" + SELECT COUNT(*) + FROM recommendation_reviewed_claims + WHERE reviewed_claim_id >= 10000 + """, Integer.class)).isEqualTo(PERFORMANCE_EVIDENCE_COUNT); + assertThat(rawJdbc.queryForObject(""" + SELECT COUNT(DISTINCT final_map_id) + FROM recommendation_reviewed_claims + WHERE reviewed_claim_id >= 10000 + """, Integer.class)).isEqualTo(PERFORMANCE_CANDIDATE_COUNT); + + QueryCountHolder.clear(); + long coldStartedAt = System.nanoTime(); + List coldResult = repository.findRecommendations(50, 911, 20); + double coldMillis = elapsedMillis(coldStartedAt); + + assertThat(coldResult).hasSize(20); + assertThat(QueryCountHolder.getGrandTotal().getTotal()).isEqualTo(1); + + List warmMillis = new ArrayList<>(); + for (int iteration = 0; iteration < 40; iteration++) { + QueryCountHolder.clear(); + long startedAt = System.nanoTime(); + List result = repository.findRecommendations(50, 911, 20); + warmMillis.add(elapsedMillis(startedAt)); + assertThat(result).hasSize(20); + assertThat(QueryCountHolder.getGrandTotal().getTotal()).isEqualTo(1); + } + + String executableQuery = MySqlMapRecommendationRepository.SCORING_QUERY + .replace(":jobId", "911"); + String jsonPlan = rawJdbc.queryForObject("EXPLAIN FORMAT=JSON " + executableQuery, String.class); + assertThat(jsonPlan) + .contains("recommendation_reviewed_claims") + .contains("alrim") + .contains("idx_recommendation_reviewed_claims_scoring") + .contains("idx_alrim_type_date"); + + String planSummary = rawJdbc.queryForList("EXPLAIN " + executableQuery).stream() + .map(row -> "%s:%s:%s".formatted(row.get("table"), row.get("type"), row.get("key"))) + .collect(Collectors.joining(",")); + + warmMillis.sort(Comparator.naturalOrder()); + System.out.printf( + "RECOMMENDATION_PERF mysql=%s evidence_rows=%d candidate_maps=%d final_results=%d " + + "queries_per_request=1 cold_ms=%.3f warm_p50_ms=%.3f warm_p95_ms=%.3f " + + "warm_p99_ms=%.3f plan=%s%n", + MYSQL.getContainerInfo().getConfig().getImage(), + PERFORMANCE_EVIDENCE_COUNT, + PERFORMANCE_CANDIDATE_COUNT, + coldResult.size(), + coldMillis, + percentile(warmMillis, 50), + percentile(warmMillis, 95), + percentile(warmMillis, 99), + planSummary + ); + } + + @Test + @Order(2) + void enforcesApprovalLineageDedupMapIdentityPolarityAndFacetContracts() { + QueryCountHolder.clear(); + + List result = repository.findRecommendations(50, 111, 20); + + assertThat(QueryCountHolder.getGrandTotal().getTotal()).isEqualTo(1); + Map byMap = result.stream() + .collect(Collectors.toMap(RecommendationCandidate::mapId, Function.identity())); + + assertThat(byMap.get(1000L).score()).isEqualTo(0.95d); + assertThat(byMap.get(1000L).reasons()).containsExactly( + new RecommendationReason("reward", "xp"), + new RecommendationReason("play_style", "solo"), + new RecommendationReason("operability", "budget") + ); + + assertThat(byMap.get(1004L).score()).isEqualTo(1.0d); + assertThat(byMap.get(1004L).reasons()) + .containsExactly(new RecommendationReason("reward", "xp")); + assertThat(byMap).containsKeys(1005L, 1006L, 1007L, 1008L); + assertThat(byMap).doesNotContainKeys( + 1001L, + 1002L, + 1003L, + 1009L, + 1010L, + 1011L, + 1999L + ); + + assertThat(repository.loadEvidence(111)) + .filteredOn(row -> row.extractedClaimId() == 7L && row.mapId() == 1004L) + .singleElement() + .satisfies(row -> assertThat(row.jobId()).isEqualTo(111L)); + } + + @Test + @Order(3) + void configuredJdbcStatementTimeoutCancelsSlowMysqlQuery() { + RecommendationProperties properties = new RecommendationProperties(); + properties.setQueryTimeoutSeconds(1); + NamedParameterJdbcTemplate timeoutTemplate = new RecommendationConfig() + .recommendationJdbcTemplate(rawDataSource, properties); + + assertThatThrownBy(() -> timeoutTemplate.queryForObject( + "SELECT SLEEP(3)", + Map.of(), + Integer.class + )).isInstanceOf(DataAccessException.class); + } + + private static void seedContractFixture() { + rawJdbc.update(""" + INSERT INTO jobs(job_id, parent_job_id) VALUES + (100, NULL), + (110, 100), + (111, 110), + (112, 111), + (200, NULL), + (900, NULL), + (910, 900), + (911, 910) + """); + rawJdbc.update(""" + INSERT INTO maps(map_id) VALUES + (1000), (1001), (1002), (1003), (1004), (1005), + (1006), (1007), (1008), (1009), (1010), (1011) + """); + rawJdbc.update(""" + INSERT INTO alrim(type, date) VALUES + ('PATCH_NOTE', '2026-01-10 00:00:00'), + ('PATCH_NOTE', '2026-02-10 00:00:00'), + ('NOTICE', '2026-03-10 00:00:00') + """); + + claim(1, "2026-03-01 00:00:00"); + reviewed(1, 1, "APPROVED", 1000, 111, 40, 60, "POSITIVE"); + reason(1, "reward", "xp", 0); + reason(1, "play_style", "solo", 1); + + claim(2, "2026-01-01 00:00:00"); + reviewed(2, 2, "APPROVED", 1000, 110, 40, 60, "POSITIVE"); + reason(2, "reward", "xp", 0); + reason(2, "play_style", "party", 1); + reason(2, "operability", "budget", 2); + + claim(3, "2026-02-01 00:00:00"); + reviewed(3, 3, "APPROVED", 1000, 100, 40, 60, "NEGATIVE"); + reason(3, "reward", "meso", 0); + reason(3, "operability", "fatigue", 1); + + claim(4, "2026-03-01 00:00:00"); + reviewed(4, 4, "PENDING", 1001, 111, 40, 60, "POSITIVE"); + + claim(5, "2026-03-01 00:00:00"); + reviewed(5, 5, "APPROVED", 1002, 112, 40, 60, "POSITIVE"); + + claim(6, "2026-03-01 00:00:00"); + reviewed(6, 6, "APPROVED", 1003, 200, 40, 60, "POSITIVE"); + + claim(7, "2026-03-01 00:00:00"); + reviewed(7, 7, "APPROVED", 1004, 111, 40, 60, "POSITIVE"); + reviewed(8, 7, "APPROVED", 1004, 110, 40, 60, "POSITIVE"); + reason(7, "reward", "xp", 0); + reason(8, "reward", "meso", 0); + + claim(8, "2026-03-01 00:00:00"); + reviewed(9, 8, "APPROVED", 1005, 111, 40, 60, "POSITIVE"); + reviewed(10, 8, "APPROVED", 1006, 110, 40, 60, "POSITIVE"); + + claim(9, "2026-03-01 00:00:00"); + reviewed(11, 9, "APPROVED", 1007, 111, 50, null, "POSITIVE"); + + claim(10, "2026-03-01 00:00:00"); + reviewed(12, 10, "APPROVED", 1008, 111, null, 50, "POSITIVE"); + + claim(11, "2026-03-01 00:00:00"); + reviewed(13, 11, "APPROVED", 1011, 111, null, null, "POSITIVE"); + + claim(12, "2026-03-01 00:00:00"); + reviewed(14, 12, "APPROVED", 1009, 111, 40, 60, "NEGATIVE"); + + claim(13, "2026-03-01 00:00:00"); + reviewed(15, 13, "APPROVED", 1010, 111, 40, 60, "POSITIVE"); + claim(14, "2026-03-01 00:00:00"); + reviewed(16, 14, "APPROVED", 1010, 111, 40, 60, "NEGATIVE"); + + claim(15, "2026-03-01 00:00:00"); + reviewed(17, 15, "APPROVED", 1999, 111, 40, 60, "POSITIVE"); + } + + private static void seedPerformanceFixture() throws Exception { + try (Connection connection = rawDataSource.getConnection()) { + connection.setAutoCommit(false); + try (PreparedStatement map = connection.prepareStatement("INSERT INTO maps(map_id) VALUES (?)"); + PreparedStatement extracted = connection.prepareStatement(""" + INSERT INTO recommendation_extracted_claims( + extracted_claim_id, source_published_at + ) VALUES (?, ?) + """); + PreparedStatement reviewed = connection.prepareStatement(""" + INSERT INTO recommendation_reviewed_claims( + reviewed_claim_id, extracted_claim_id, review_status, + final_map_id, final_job_id, final_level_min, final_level_max, final_polarity + ) VALUES (?, ?, 'APPROVED', ?, ?, 40, 60, ?) + """); + PreparedStatement reason = connection.prepareStatement(""" + INSERT INTO recommendation_reviewed_claim_reasons( + reviewed_claim_id, reason_axis, reason_value, display_order + ) VALUES (?, 'reward', 'xp', 0) + """)) { + for (int mapOffset = 0; mapOffset < PERFORMANCE_CANDIDATE_COUNT; mapOffset++) { + map.setLong(1, 2000L + mapOffset); + map.addBatch(); + } + map.executeBatch(); + + Timestamp publishedAt = Timestamp.valueOf("2026-01-01 00:00:00"); + for (int index = 0; index < PERFORMANCE_EVIDENCE_COUNT; index++) { + long id = 10_000L + index; + extracted.setLong(1, id); + extracted.setTimestamp(2, publishedAt); + extracted.addBatch(); + + reviewed.setLong(1, id); + reviewed.setLong(2, id); + reviewed.setLong(3, 2000L + index % PERFORMANCE_CANDIDATE_COUNT); + reviewed.setLong(4, 900L + switch (index % 3) { + case 0 -> 0L; + case 1 -> 10L; + default -> 11L; + }); + reviewed.setString(5, (index / PERFORMANCE_CANDIDATE_COUNT) % 4 == 0 + ? "NEGATIVE" + : "POSITIVE"); + reviewed.addBatch(); + + reason.setLong(1, id); + reason.addBatch(); + + if ((index + 1) % 500 == 0) { + extracted.executeBatch(); + reviewed.executeBatch(); + reason.executeBatch(); + } + } + connection.commit(); + } + } + } + + private static void claim(long extractedClaimId, String publishedAt) { + rawJdbc.update( + "INSERT INTO recommendation_extracted_claims(extracted_claim_id, source_published_at) VALUES (?, ?)", + extractedClaimId, + Timestamp.valueOf(publishedAt) + ); + } + + private static void reviewed( + long reviewedClaimId, + long extractedClaimId, + String status, + long mapId, + long jobId, + Integer levelMin, + Integer levelMax, + String polarity + ) { + rawJdbc.update(""" + INSERT INTO recommendation_reviewed_claims( + reviewed_claim_id, extracted_claim_id, review_status, + final_map_id, final_job_id, final_level_min, final_level_max, final_polarity + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + reviewedClaimId, + extractedClaimId, + status, + mapId, + jobId, + levelMin, + levelMax, + polarity + ); + } + + private static void reason(long reviewedClaimId, String axis, String value, int displayOrder) { + rawJdbc.update(""" + INSERT INTO recommendation_reviewed_claim_reasons( + reviewed_claim_id, reason_axis, reason_value, display_order + ) VALUES (?, ?, ?, ?) + """, + reviewedClaimId, + axis, + value, + displayOrder + ); + } + + private static double elapsedMillis(long startedAt) { + return (System.nanoTime() - startedAt) / 1_000_000.0d; + } + + private static double percentile(List sortedValues, int percentile) { + int index = Math.max(0, (int) Math.ceil(percentile / 100.0d * sortedValues.size()) - 1); + return sortedValues.get(index); + } +} diff --git a/src/test/java/com/maple/api/observability/ObservabilityHttpIntegrationTest.java b/src/test/java/com/maple/api/observability/ObservabilityHttpIntegrationTest.java index 8ff7452..08138ea 100644 --- a/src/test/java/com/maple/api/observability/ObservabilityHttpIntegrationTest.java +++ b/src/test/java/com/maple/api/observability/ObservabilityHttpIntegrationTest.java @@ -1,5 +1,7 @@ package com.maple.api.observability; +import com.maple.api.map.application.MapRecommendationObservability; +import com.maple.api.map.domain.RecommendationEngineType; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -45,6 +47,9 @@ class ObservabilityHttpIntegrationTest { @Autowired private TestRestTemplate http; + @Autowired + private MapRecommendationObservability recommendationObservability; + @BeforeAll static void useLoopbackForHttpClient() { System.setProperty("java.net.preferIPv4Stack", "true"); @@ -99,6 +104,22 @@ void httpRequestMetricsUseTheNormalizedSpringRoute() { .contains("uri=\"/api/v1/jobs\""); } + @Test + void recommendationMetricsExposeOnlyTheAllowlistedPrometheusFamilies() { + recommendationObservability.completed(RecommendationEngineType.MYSQL, "v2", 3, 1_000L); + + String metrics = getWithBearer( + managementPort, "/actuator/prometheus", SCRAPE_TOKEN).getBody(); + assertThat(metrics) + .contains("mapleland_recommendation_requests_total") + .contains("mapleland_recommendation_results_recommendations_count") + .contains("mapleland_recommendation_results_recommendations_sum") + .contains("mapleland_recommendation_results_recommendations_max") + .contains("api_version=\"v2\"") + .contains("engine=\"mysql\"") + .contains("outcome=\"success\""); + } + @Test void publicPortDoesNotExposeManagementEndpointsOrHealthDetails() { assertThat(get(applicationPort, "/actuator/env").getStatusCode()).isEqualTo(HttpStatus.NOT_FOUND); diff --git a/src/test/resources/application-test.yml b/src/test/resources/application-test.yml index f4550a1..7a85e9b 100644 --- a/src/test/resources/application-test.yml +++ b/src/test/resources/application-test.yml @@ -11,3 +11,5 @@ spring: h2: console: enabled: true +recommendation: + v2-enabled: true diff --git a/src/test/resources/sql/recommendation-mysql8-schema.sql b/src/test/resources/sql/recommendation-mysql8-schema.sql new file mode 100644 index 0000000..128fa8b --- /dev/null +++ b/src/test/resources/sql/recommendation-mysql8-schema.sql @@ -0,0 +1,50 @@ +CREATE TABLE jobs ( + job_id BIGINT NOT NULL PRIMARY KEY, + parent_job_id BIGINT NULL +); + +CREATE TABLE maps ( + map_id BIGINT NOT NULL PRIMARY KEY +); + +CREATE TABLE alrim ( + id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, + type VARCHAR(31) NOT NULL, + date TIMESTAMP NOT NULL, + INDEX idx_alrim_type_date (type, date) +); + +CREATE TABLE recommendation_extracted_claims ( + extracted_claim_id BIGINT NOT NULL PRIMARY KEY, + source_published_at TIMESTAMP NOT NULL +); + +CREATE TABLE recommendation_reviewed_claims ( + reviewed_claim_id BIGINT NOT NULL PRIMARY KEY, + extracted_claim_id BIGINT NOT NULL, + review_status VARCHAR(20) NOT NULL, + final_map_id BIGINT NULL, + final_job_id BIGINT NULL, + final_level_min INT NULL, + final_level_max INT NULL, + final_polarity VARCHAR(20) NULL, + UNIQUE KEY uk_recommendation_reviewed_claims_extracted ( + extracted_claim_id, + final_map_id, + final_job_id + ), + INDEX idx_recommendation_reviewed_claims_scoring (review_status, final_job_id) +); + +CREATE TABLE recommendation_reviewed_claim_reasons ( + reviewed_claim_reason_id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, + reviewed_claim_id BIGINT NOT NULL, + reason_axis VARCHAR(30) NOT NULL, + reason_value VARCHAR(30) NOT NULL, + display_order INT NULL, + UNIQUE KEY uk_recommendation_reviewed_claim_reasons ( + reviewed_claim_id, + reason_axis, + reason_value + ) +);