feat: 운영 서비스 1차 Observability 구축

.github/workflows/ci.yml

@@ -0,0 +1,65 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test-and-package:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v6
+
+      - name: Set up JDK 21
+        uses: actions/setup-java@v5
+        with:
+          distribution: temurin
+          java-version: '21'
+
+      - name: Set up Gradle
+        uses: gradle/actions/setup-gradle@v6
+        with:
+          cache-provider: basic
+          validate-wrappers: true
+
+      - name: Test and package
+        run: ./gradlew clean test bootJar --no-daemon
+
+      - name: Validate Grafana Alloy configuration
+        run: deploy/observability/alloy/validate.sh
+
+      - name: Validate operations scripts
+        run: |
+          bash -n deploy/observability/alloy/validate.sh
+          bash -n deploy/observability/preflight-host.sh
+          bash -n deploy/observability/tests/host-preflight-test.sh
+          bash -n deploy/observability/tests/update-api-test.sh
+          bash -n deploy/observability/update-api.sh
+          test -x deploy/observability/alloy/validate.sh
+          test -x deploy/observability/preflight-host.sh
+          test -x deploy/observability/tests/host-preflight-test.sh
+          test -x deploy/observability/tests/update-api-test.sh
+          test -x deploy/observability/update-api.sh
+
+      - name: Test host deployment preflight
+        run: deploy/observability/tests/host-preflight-test.sh
+
+      - name: Test immutable deployment and rollback
+        run: deploy/observability/tests/update-api-test.sh
+
+      - name: Validate Grafana resource sources
+        run: |
+          jq empty deploy/observability/grafana/mapleland-production-overview.json
+          jq empty deploy/observability/grafana/alert-rules.json

.github/workflows/deploy-oci.yml

@@ -3,26 +3,98 @@ name: Deploy Spring Boot App to Oracle Cloud
 on:
   workflow_dispatch:
 
+concurrency:
+  group: deploy-oci
+  cancel-in-progress: false
+
 permissions:
   contents: read
   packages: write
   id-token: write
 
 jobs:
+  host-preflight:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      id-token: write
+      contents: read
+    outputs:
+      compose_override_sha256: ${{ steps.contract.outputs.compose_override_sha256 }}
+      preflight_sha256: ${{ steps.contract.outputs.preflight_sha256 }}
+      update_sha256: ${{ steps.contract.outputs.update_sha256 }}
+
+    steps:
+      - name: Checkout deployment contract
+        uses: actions/checkout@v4
+
+      - name: Resolve reviewed host contract checksums
+        id: contract
+        shell: bash
+        run: |
+          set -euo pipefail
+          compose_override_sha256="$(sha256sum deploy/observability/docker-compose.override.example.yml | awk '{print $1}')"
+          preflight_sha256="$(sha256sum deploy/observability/preflight-host.sh | awk '{print $1}')"
+          update_sha256="$(sha256sum deploy/observability/update-api.sh | awk '{print $1}')"
+          [[ $compose_override_sha256 =~ ^[0-9a-f]{64}$ ]]
+          [[ $preflight_sha256 =~ ^[0-9a-f]{64}$ ]]
+          [[ $update_sha256 =~ ^[0-9a-f]{64}$ ]]
+          {
+            printf 'compose_override_sha256=%s\n' "$compose_override_sha256"
+            printf 'preflight_sha256=%s\n' "$preflight_sha256"
+            printf 'update_sha256=%s\n' "$update_sha256"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Connect to Tailscale
+        uses: tailscale/github-action@v4
+        with:
+          oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
+          oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
+          tags: tag:ci
+
+      - name: Verify OCI host deployment contract
+        uses: appleboy/ssh-action@v1.2.5
+        with:
+          host: oracle-cloud
+          username: ubuntu
+          key: ${{ secrets.ORACLE_SSH_KEY }}
+          script: preflight ${{ steps.contract.outputs.preflight_sha256 }} ${{ steps.contract.outputs.update_sha256 }} ${{ steps.contract.outputs.compose_override_sha256 }}
+
   build-and-publish:
+    needs: host-preflight
     runs-on: ubuntu-24.04-arm
+    timeout-minutes: 20
+    outputs:
+      image_ref: ${{ steps.published_image.outputs.image_ref }}
     env:
       IMAGE_NAME: ghcr.io/team-maple/mls-be/mapleland-api
-      IMAGE_TAG: latest-arm64
+      IMAGE_TAG: ${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}-arm64
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
       - name: Restore FCM key file
+        env:
+          FIREBASE_KEY: ${{ secrets.FIREBASE_KEY }}
         run: |
-          mkdir -p src/main/resources/firebase
-          echo '${{ secrets.FIREBASE_KEY }}' > src/main/resources/firebase/maple-9f1a7-firebase-adminsdk-fbsvc-7c3b6fc032.json
+          set -euo pipefail
+          umask 077
+          firebase_dir=src/main/resources/firebase
+          firebase_key="${firebase_dir}/maple-9f1a7-firebase-adminsdk-fbsvc-7c3b6fc032.json"
+          mkdir -p "${firebase_dir}"
+          printf '%s' "$FIREBASE_KEY" \
+            > "${firebase_key}"
+          test -s "${firebase_key}"
+          jq -e 'type == "object" and .type == "service_account" and
+            (.private_key | type == "string" and length > 0) and
+            (.client_email | type == "string" and length > 0)' \
+            "${firebase_key}" >/dev/null
+          # Paketo launches as uid 1002, gid 1001 while application layers are
+          # owned by 1001:1001. Keep the secret group-readable only so the
+          # runtime process can traverse/read it without making it world-readable.
+          chmod 0750 "${firebase_dir}"
+          chmod 0640 "${firebase_key}"
 
       - name: Set up JDK 21
         uses: actions/setup-java@v4
@@ -34,7 +106,20 @@ jobs:
         run: chmod +x ./gradlew
 
       - name: Build jar
-        run: ./gradlew clean bootJar
+        shell: bash
+        run: |
+          set -euo pipefail
+          ./gradlew clean bootJar
+          jar_path=build/libs/api-0.0.1-SNAPSHOT.jar
+          firebase_dir_mode="$(zipinfo -l "${jar_path}" \
+            'BOOT-INF/classes/firebase/' | awk 'NR == 1 { print $1 }')"
+          firebase_key_mode="$(zipinfo -l "${jar_path}" \
+            'BOOT-INF/classes/firebase/maple-9f1a7-firebase-adminsdk-fbsvc-7c3b6fc032.json' \
+            | awk 'NR == 1 { print $1 }')"
+          # ZIP directory entries use '-' in zipinfo's type column; the
+          # trailing slash still materializes this entry as a directory.
+          test "${firebase_dir_mode}" = '-rwxr-x---'
+          test "${firebase_key_mode}" = '-rw-r-----'
 
       - name: Set up pack CLI
         uses: buildpacks/github-actions/setup-pack@v5.11.0
@@ -54,15 +139,69 @@ jobs:
             --platform linux/arm64 \
             --publish
 
+      - name: Verify published image FCM permissions
+        shell: bash
+        run: |
+          set -euo pipefail
+          image_ref="${IMAGE_NAME}:${IMAGE_TAG}"
+          docker pull "${image_ref}" >/dev/null
+          runtime_user="$(docker image inspect --format '{{.Config.User}}' "${image_ref}")"
+          test "${runtime_user}" = '1002:1001'
+
+          verification_container="fcm-permission-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          cleanup() {
+            docker rm -f "${verification_container}" >/dev/null 2>&1 || true
+          }
+          trap cleanup EXIT
+          docker create --name "${verification_container}" "${image_ref}" >/dev/null
+          metadata="$(docker export "${verification_container}" \
+            | tar --numeric-owner -tvf - \
+            | awk '
+                $NF == "workspace/BOOT-INF/classes/firebase/" {
+                  directory_mode = $1; directory_owner = $2
+                }
+                $NF == "workspace/BOOT-INF/classes/firebase/maple-9f1a7-firebase-adminsdk-fbsvc-7c3b6fc032.json" {
+                  key_mode = $1; key_owner = $2
+                }
+                END {
+                  if (!directory_mode || !key_mode) exit 1
+                  print directory_mode, directory_owner, key_mode, key_owner
+                }
+              ')"
+          read -r directory_mode directory_owner key_mode key_owner <<< "${metadata}"
+          test "${directory_mode}" = 'drwxr-x---'
+          test "${directory_owner}" = '1001/1001'
+          test "${key_mode}" = '-rw-r-----'
+          test "${key_owner}" = '1001/1001'
+
+      - name: Resolve published manifest digest
+        id: published_image
+        shell: bash
+        run: |
+          set -euo pipefail
+          image_tag="${IMAGE_NAME}:${IMAGE_TAG}"
+          digest="$(docker buildx imagetools inspect "$image_tag" \
+            --format '{{json .Manifest}}' | jq -er '.digest')"
+          if [[ ! $digest =~ ^sha256:[0-9a-f]{64}$ ]]; then
+            echo "invalid published image digest" >&2
+            exit 1
+          fi
+          docker buildx imagetools inspect "${IMAGE_NAME}@${digest}" >/dev/null
+          printf 'image_ref=%s@%s\n' "$IMAGE_NAME" "$digest" >> "$GITHUB_OUTPUT"
+
   deploy:
-    needs: build-and-publish
+    needs:
+      - host-preflight
+      - build-and-publish
     runs-on: ubuntu-latest
+    timeout-minutes: 15
+    environment:
+      name: production
     permissions:
       id-token: write
       contents: read
     env:
-      IMAGE_NAME: ghcr.io/team-maple/mls-be/mapleland-api
-      IMAGE_TAG: latest-arm64
+      IMAGE_REF: ${{ needs['build-and-publish'].outputs.image_ref }}
 
     steps:
       - name: Connect to Tailscale
@@ -78,5 +217,4 @@ jobs:
           host: oracle-cloud
           username: ubuntu
           key: ${{ secrets.ORACLE_SSH_KEY }}
-          script: |
-            /opt/mapleland/update-api.sh
+          script: deploy ${{ needs['host-preflight'].outputs.preflight_sha256 }} ${{ needs['host-preflight'].outputs.update_sha256 }} ${{ needs['host-preflight'].outputs.compose_override_sha256 }} ${{ env.IMAGE_REF }}

.gitignore

@@ -39,4 +39,11 @@ out/
 ### Jenv ###
 .java-version
 
-resouces/firebase
+### Runtime secrets ###
+.env
+.env.*
+*.env
+!.env.example
+!*.env.example
+
+src/main/resources/firebase/

build.gradle

@@ -31,6 +31,7 @@ dependencies {
   implementation 'org.springframework.boot:spring-boot-starter-web'
   implementation 'org.springframework.boot:spring-boot-starter-actuator'
   implementation 'org.springframework.boot:spring-boot-starter-validation'
+  runtimeOnly 'io.micrometer:micrometer-registry-prometheus'
 
   implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.7.0'
 
@@ -52,9 +53,6 @@ dependencies {
   // JSoup
   implementation 'org.jsoup:jsoup:1.17.2'
 
-  // logging
-  implementation 'com.github.loki4j:loki-logback-appender:2.0.1'
-
   // QueryDSL
   implementation 'com.querydsl:querydsl-jpa:5.1.0:jakarta'
   annotationProcessor "com.querydsl:querydsl-apt:5.1.0:jakarta"

deploy/observability/alloy/alloy.env.example

@@ -0,0 +1,17 @@
+# Install as /etc/alloy/alloy.env, owned by root:root with mode 0600.
+# Do not commit the populated file. Use two minimum-scope access-policy tokens:
+# metrics:write for metrics and logs:write for logs.
+
+ALLOY_HOST_NAME=mapleland-oci-1
+
+# Set this to the same long, random value injected into the Spring container.
+# It protects the management scrape even from peers on shared Docker networks.
+MANAGEMENT_SCRAPE_TOKEN=replace-with-long-random-management-scrape-token
+
+GRAFANA_CLOUD_METRICS_URL=https://prometheus-prod-49-prod-ap-northeast-0.grafana.net/api/prom/push
+GRAFANA_CLOUD_METRICS_USERNAME=replace-with-metrics-instance-id
+GRAFANA_CLOUD_METRICS_API_TOKEN=replace-with-metrics-write-token
+
+GRAFANA_CLOUD_LOGS_URL=https://logs-prod-030.grafana.net/loki/api/v1/push
+GRAFANA_CLOUD_LOGS_USERNAME=replace-with-logs-instance-id
+GRAFANA_CLOUD_LOGS_API_TOKEN=replace-with-logs-write-token