Merge pull request #4028 from SwiftPackageIndex/mitigate-gitlab-500s-part-3

Added logging to determine critical error source

Author: finestructure

Sources/App/Commands/TriggerBuilds.swift

@@ -90,14 +90,14 @@ struct TriggerBuildsCommand: AsyncCommand {
         do {
             try await triggerBuilds(on: context.application.db, mode: mode)
         } catch {
-            logger.critical("\(error)")
+            logger.critical("triggerBuilds.run: \(error)")
         }
 
         do {
             try await AppMetrics.push(client: context.application.client,
                                       jobName: "trigger-builds")
         } catch {
-            logger.warning("\(error)")
+            logger.warning("triggerBuilds.run: \(error)")
         }
     }
 
@@ -201,6 +201,8 @@ func triggerBuilds(on database: Database,
     // See https://github.com/SwiftPackageIndex/SwiftPackageIndex-Server/issues/4024 for details.
     let runningJobs = (try? await runningJobsTask) ?? 0
 
+    logger.info("Jobs pending: \(pendingJobs), running: \(runningJobs)")
+
     AppMetrics.buildPendingJobsCount?.set(pendingJobs)
     AppMetrics.buildRunningJobsCount?.set(runningJobs)
 
@@ -235,7 +237,12 @@ func triggerBuilds(on database: Database,
                 let triggeredJobCount = triggers.reduce(0) { $0 + $1.buildPairs.count }
                 await newJobs.withValue { $0 += triggeredJobCount }
 
-                try await triggerBuildsUnchecked(on: database, triggers: triggers)
+                do {
+                    try await triggerBuildsUnchecked(on: database, triggers: triggers)
+                } catch {
+                    logger.error("triggerBuildsUnchecked failed: \(error)")
+                    throw error
+                }
             }
         }
     }

Sources/App/Core/Gitlab.swift

@@ -181,11 +181,17 @@ extension Gitlab.Builder {
                                maxPageCount: Int = 5) async throws -> Int {
         let count = try await fetchJobs(status: status, page: page, pageSize: pageSize).count
         if count == pageSize && page < maxPageCount {
-            let statusCount = try await getStatusCount(status: status,
-                                                       page: page + 1,
-                                                       pageSize: pageSize,
-                                                       maxPageCount: maxPageCount)
-            return count + statusCount
+            do {
+                let statusCount = try await getStatusCount(status: status,
+                                                           page: page + 1,
+                                                           pageSize: pageSize,
+                                                           maxPageCount: maxPageCount)
+                return count + statusCount
+            } catch {
+                @Dependency(\.logger) var logger
+                logger.error("Gitlab.Builder.getStatusCount failed: \(error)")
+                throw error
+            }
         } else {
             return count
         }