implemented exit-gate

.github/workflows/build-and-unit-test.yml

@@ -1,4 +1,4 @@
-# Copyright © 2022 Cask Data, Inc.
+# Copyright © 2025 Cask Data, Inc.
 #  Licensed under the Apache License, Version 2.0 (the "License"); you may not
 #  use this file except in compliance with the License. You may obtain a copy of
 #  the License at
@@ -85,10 +85,10 @@ jobs:
         if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }}
         with:
           secrets: |-
-            CDAP_OSSRH_USERNAME:cdapio-github-builds/CDAP_OSSRH_USERNAME
-            CDAP_OSSRH_PASSWORD:cdapio-github-builds/CDAP_OSSRH_PASSWORD
-            CDAP_GPG_PASSPHRASE:cdapio-github-builds/CDAP_GPG_PASSPHRASE
             CDAP_GPG_PRIVATE_KEY:cdapio-github-builds/CDAP_GPG_PRIVATE_KEY
+            CDAP_GPG_PASSPHRASE:cdapio-github-builds/CDAP_GPG_PASSPHRASE
+            ARTIFACT_BUCKET:cdapio-github-builds/artifacts_bucket
+            SECURE_PUBLISH_BUCKET:cdapio-github-builds/publish_bucket
 
       - name: Recursively Checkout Repository
         uses: actions/checkout@v4
@@ -160,8 +160,8 @@ jobs:
           retry_on: error
           on_retry_command: echo "Build Standalone failed in this attempt, retrying ..."
           command: |
-              cd cdap-build
-              MAVEN_OPTS="-Xmx12G" mvn -e -T2 clean package -Dgpg.skip -DskipTests -Ddocker.skip=true -nsu -am -amd -P templates,dist,release -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30
+            cd cdap-build
+            MAVEN_OPTS="-Xmx12G" mvn -e -T2 clean package -Dgpg.skip -DskipTests -Ddocker.skip=true -nsu -am -amd -P templates,dist,release -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30
 
       - name: Find Build Version
         working-directory: cdap-build/cdap
@@ -189,65 +189,38 @@ jobs:
 
       - name: Set up GPG conf
         if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }}
-        run: |
-          echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
-          echo "allow-loopback-pinentry" >> ~/.gnupg/gpg-agent.conf
 
       - name: Import GPG key
         if: ${{ matrix.branch == 'develop' || startsWith(matrix.branch, 'release/') }}
         run: |
-          echo "$GPG_PRIVATE_KEY" > private.key
-          gpg --import --batch private.key
+            echo "$GPG_PRIVATE_KEY" > private.key
+            gpg --import --batch private.key
         env:
           GPG_PRIVATE_KEY: ${{ steps.secrets.outputs.CDAP_GPG_PRIVATE_KEY }}
 
-      - name: Maven Deploy
+      - name: Submit Build to GCB
+        id: gcb
+        working-directory: cdap-build
         run: |
-          cd cdap-build
-          if [[ (${{ matrix.branch }} == "develop") || (${{ matrix.branch }} == release/*) ]];
-          then
-            retry_count=0
-            failed_module=""
-            while [ $retry_count -lt 3 ]; do
-              if [ -n "$failed_module" ];
-              then
-                build_output=$(mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$CDAP_GPG_PASSPHRASE -rf :"$failed_module" 2>&1 || true)
-              else
-                build_output=$(mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$CDAP_GPG_PASSPHRASE 2>&1 || true)
-              fi
-              echo "$build_output"
-              if [ $(echo "$build_output" | grep -c "BUILD FAILURE") -gt 0 ];
-              then
-                echo "[WARNING] Deployment failed, retrying..."
-                failed_module=$(echo "$build_output" | grep -e 'mvn <args> -rf' | sed -n 's/.*mvn <args> -rf ://p')
-                echo "[INFO] FAILED MODULE = $failed_module"
-                retry_count=$((retry_count + 1))
-              else
-                echo "[INFO] Deployment successful"
-                break
-              fi
-            done
-            if [ $retry_count -ge 3 ];
-            then
-              echo "[ERROR] Max retries reached..., deployment failed"
-              exit 1
-            fi
-          else
-            mvn verify -B -V -T2 -DskipTests -Dgpg.skip -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true
+          BUILD_OUTPUT=$(gcloud builds submit . \
+          --config=cloudbuild-release.yaml \
+          --project='cdapio-github-builds' \
+          --substitutions="_SECURE_PUBLISH_BUCKET_NAME=${{ SECURE_PUBLISH_BUCKET }}, GPG_PASSPHRASE=${{CDAP_GPG_PASSPHRASE}})
+          echo "$BUILD_OUTPUT"
+          BUILD_ID=$(echo "$BUILD_OUTPUT" | grep -oP 'ID: \K[a-f0-9-]+' | head -n 1)
+          if [ -z "$BUILD_ID" ]; then
+          echo "Failed to extract BUILD_ID from gcloud output." >&2
+          # Fallback: Get the last build submitted by this service account.
+          # This assumes no other concurrent builds are started by this SA.
+          BUILD_ID=$(gcloud builds list --project='cdapio-github-builds' --limit=1 --format='value(ID)' --filter="buildTriggerId='' AND status!=WORKING AND status!=QUEUED" --sort-by=~CREATE_TIME)
+          if [ -z "$BUILD_ID" ]; then
+          echo "Fallback failed to get BUILD_ID." >&2
+          exit 1
           fi
-        env:
-          CDAP_OSSRH_USERNAME: ${{ steps.secrets.outputs.CDAP_OSSRH_USERNAME }}
-          CDAP_OSSRH_PASSWORD: ${{ steps.secrets.outputs.CDAP_OSSRH_PASSWORD }}
-          CDAP_GPG_PASSPHRASE: ${{ steps.secrets.outputs.CDAP_GPG_PASSPHRASE }}
-          MAVEN_OPTS: "-Xmx12G"
-
-      - name: Build DEB Bundle
-        working-directory: cdap-build/cdap
-        run: |
-          mkdir -p cdap-distributions/target/deb-bundle-tmp
-          cd cdap-distributions/target/deb-bundle-tmp
-          cp ../../../*/target/*.deb .
-          tar zcf ../cdap-distributed-deb-bundle-${{env.CDAP_VERSION}}.tgz *.deb
+          echo "Fallback BUILD_ID: $BUILD_ID"
+          fi
+          echo "build_id=$BUILD_ID" >> $GITHUB_OUTPUT
+          echo "GCB Build ID: $BUILD_ID"
 
       - name: Set Up Tag
         working-directory: cdap-build

.github/workflows/cloudbuild.yaml

@@ -0,0 +1,118 @@
+# Copyright © 2025 Cask Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+steps:
+  - name: 'gcr.io/cloud-builders/mvn:3.8-jdk-8'
+    id: mvn-deploy
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        if [[ ("$BRANCH" == "develop") || ("$BRANCH" == release/*) ]];
+        then
+          echo "--- Running 'mvn deploy' with retries for branch: $BRANCH ---"
+          retry_count=0
+          failed_module=""
+          while [ $retry_count -lt 3 ]; do
+            MVN_CMD="mvn deploy -B -V -DskipTests -DskipLocalStaging=true -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dgpg.passphrase=$GPG_PASSPHRASE"
+
+            if [ -n "$failed_module" ]; then
+              MVN_CMD="$MVN_CMD -rf :$failed_module"
+              echo "[INFO] Retrying build from module: $failed_module"
+            fi
+
+            echo "Executing Maven command attempt $((retry_count + 1)): $MVN_CMD"
+            # Execute the Maven command, redirecting stderr to stdout, and continuing on failure
+            build_output=$(eval "$MVN_CMD" 2>&1 || true)
+            echo "$build_output"
+
+            if echo "$build_output" | grep -q "BUILD FAILURE"; then
+              echo "[WARNING] Deployment failed, attempt $((retry_count + 1)) of 3."
+              # Extract the failed module suggested by Maven's output for resuming
+              failed_module=$(echo "$build_output" | grep -e 'mvn <args> -rf' | sed -n 's/.*mvn <args> -rf ://p' | head -n 1)
+
+              if [ -z "$failed_module" ]; then
+                echo "[WARNING] Could not extract specific failed module. Retrying full build."
+              else
+                echo "[INFO] Found failed module: $failed_module"
+              fi
+              retry_count=$((retry_count + 1))
+            else
+              echo "[INFO] Deployment successful"
+              break
+            fi
+          done
+
+          if [ $retry_count -ge 3 ]; then
+            echo "[ERROR] Max retries reached..., deployment failed"
+            exit 1
+          fi
+        else
+          echo "--- Running 'mvn verify' for branch: $BRANCH ---"
+          mvn verify -B -V -T2 -DskipTests -Dgpg.skip -Ddocker.skip=true -P templates,dist,release,rpm-prepare,rpm,deb-prepare,deb,tgz,unit-tests -Dadditional.artifacts.dir=$(pwd)/app-artifacts -Dsecurity.extensions.dir=$(pwd)/security-extensions -Dmaven.wagon.http.retryHandler.count=5 -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true
+        fi
+    secretEnv:
+        - 'GPG_PASSPHRASE'
+
+  - name: 'bash'
+    id: stage-artifacts
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        set -e
+        GCS_MANIFEST_DIR="gs://oss-exit-gate-prod-projects-bucket/cloud-data-fusion/mavencentral/manifests/"
+        MANIFEST_FILE="/workspace/exit_gate_manifest.textproto"
+        MANIFEST_FILENAME="release_$$(date -u +%Y%m%d%H%M%S).textproto"
+        echo "Uploading manifest to $${GCS_MANIFEST_DIR}$${MANIFEST_FILENAME}"
+        gsutil cp "$${MANIFEST_FILE}" "$${GCS_MANIFEST_DIR}$${MANIFEST_FILENAME}"
+        echo "Manifest uploaded successfully. OSS Exit Gate process should now be triggered."
+    waitFor: ["mvn-deploy"]
+
+  # 5. Create Exit Gate Manifest (ONLY for RELEASE)
+  - name: 'bash'
+    id: create-exit-gate-manifest
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        set -e
+        MANIFEST_FILE="/workspace/exit_gate_manifest.textproto"
+        echo '' >> "$${MANIFEST_FILE}"
+        echo "Created manifest file: $${MANIFEST_FILE}"
+    waitFor: ['stage-artifact']
+
+  # 6. Upload Exit Gate Manifest (ONLY for RELEASE)
+  - name: 'gcr.io/cloud-builders/gsutil'
+    id: upload-exit-gate-manifest
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+          set -e
+          GCS_MANIFEST_DIR="gs://oss-exit-gate-prod-projects-bucket/cloud-data-fusion/mavencentral/manifests/"
+          MANIFEST_FILE="/workspace/exit_gate_manifest.textproto"
+          MANIFEST_FILENAME="release_$$(date -u +%Y%m%d%H%M%S).textproto"
+          echo "Uploading manifest to $${GCS_MANIFEST_DIR}$${MANIFEST_FILENAME}"
+          gsutil cp "$${MANIFEST_FILE}" "$${GCS_MANIFEST_DIR}$${MANIFEST_FILENAME}"
+          echo "Manifest uploaded successfully. OSS Exit Gate process should now be triggered."
+        else
+          echo "Skipping Exit Gate manifest upload for SNAPSHOT build."
+        fi
+    waitFor: ['create-exit-gate-manifest']
+
+options:
+  requestedVerifyOption: VERIFIED
+  machineType: 'E2_HIGHCPU_32'
+

pom.xml

@@ -53,6 +53,30 @@
     <tag>HEAD</tag>
   </scm>
 
+  <distributionManagement>
+    <snapshotRepository>
+      <id>artifact-registry</id>
+      <url>artifactregistry://us-maven.pkg.dev/oss-exit-gate-prod/cloud-data-fusion--mavencentral</url>
+    </snapshotRepository>
+    <repository>
+      <id>artifact-registry</id>
+      <url>artifactregistry://us-maven.pkg.dev/oss-exit-gate-prod/cloud-data-fusion--mavencentral</url>
+    </repository>
+  </distributionManagement>
+
+  <repositories>
+    <repository>
+      <id>artifact-registry</id>
+      <url>artifactregistry://us-maven.pkg.dev/oss-exit-gate-prod/cloud-data-fusion--mavencentral</url>
+      <releases>
+        <enabled>true</enabled>
+      </releases>
+      <snapshots>
+        <enabled>true</enabled>
+      </snapshots>
+    </repository>
+  </repositories>
+
   <modules>
     <module>app-artifacts/bigquery-delta-plugins</module>
     <module>app-artifacts/database-delta-plugins</module>
@@ -72,6 +96,16 @@
     <module>cdap</module>
   </modules>
 
+  <build>
+    <extensions>
+      <extension>
+        <groupId>com.google.cloud.artifactregistry</groupId>
+        <artifactId>artifactregistry-maven-wagon</artifactId>
+        <version>2.2.5</version>
+      </extension>
+    </extensions>
+  </build>
+
   <profiles>
     <profile>
       <id>release</id>