feat(api-service): update logger level runtime

apps/api/src/app/shared/shared.module.ts

@@ -16,6 +16,7 @@ import {
   GetDecryptedSecretKey,
   InvalidateCacheService,
   LoggerModule,
+  LogLevelService,
   QueuesModule,
   RequestLogRepository,
   StepRunRepository,
@@ -139,6 +140,7 @@ const PROVIDERS = [
   DalServiceHealthIndicator,
   featureFlagsService,
   InvalidateCacheService,
+  LogLevelService,
   storageService,
   ...DAL_MODELS,
   CreateExecutionDetails,

apps/api/src/config/env.validators.ts

@@ -73,16 +73,27 @@ export const envValidators = {
       CLICK_HOUSE_PASSWORD: str({ default: '' }),
     }),
 
-  // Feature Flags
-  ...Object.keys(FeatureFlagsKeysEnum).reduce(
-    (acc, key) => {
-      return {
-        ...acc,
-        [key as FeatureFlagsKeysEnum]: bool({ default: false }),
-      };
-    },
-    {} as Record<FeatureFlagsKeysEnum, ValidatorSpec<boolean>>
-  ),
+  // Feature Flags - Boolean (start with IS_)
+  ...Object.keys(FeatureFlagsKeysEnum)
+    .filter((key) => key.startsWith('IS_'))
+    .reduce<Record<string, ValidatorSpec<boolean>>>((acc, key) => {
+      acc[key] = bool({ default: false });
+
+      return acc;
+    }, {}),
+
+  // Feature Flags - Numeric (end with _NUMBER)
+  ...Object.keys(FeatureFlagsKeysEnum)
+    .filter((key) => key.endsWith('_NUMBER'))
+    .reduce<Record<string, ValidatorSpec<number | undefined>>>((acc, key) => {
+      acc[key] = num({ default: undefined });
+
+      return acc;
+    }, {}),
+
+  // Feature Flags - String
+  CF_SCHEDULER_MODE_STR: str({ choices: ['off', 'shadow', 'live', 'complete'], default: 'off' }),
+  LOG_LEVEL_STR: str({ choices: ['trace', 'debug', 'info', 'warn', 'error', 'fatal', 'none'], default: undefined }),
 
   // Azure validators
   ...(processEnv.STORAGE_SERVICE === 'AZURE' && {

apps/webhook/src/shared/shared.module.ts

@@ -1,5 +1,5 @@
 import { Module } from '@nestjs/common';
-import { AnalyticsService } from '@novu/application-generic';
+import { AnalyticsService, featureFlagsService, LogLevelService } from '@novu/application-generic';
 import { DalService, ExecutionDetailsRepository, IntegrationRepository, MessageRepository } from '@novu/dal';
 
 const DAL_MODELS = [ExecutionDetailsRepository, MessageRepository, IntegrationRepository];
@@ -26,6 +26,8 @@ const PROVIDERS = [
       return analyticsService;
     },
   },
+  featureFlagsService,
+  LogLevelService,
 ];
 
 @Module({

apps/worker/src/app/shared/shared.module.ts

@@ -19,6 +19,7 @@ import {
   GetTenant,
   InvalidateCacheService,
   LoggerModule,
+  LogLevelService,
   MetricsModule,
   ProcessTenant,
   QueuesModule,
@@ -110,6 +111,7 @@ const PROVIDERS = [
   DigestFilterSteps,
   featureFlagsService,
   InvalidateCacheService,
+  LogLevelService,
   StorageHelperService,
   storageService,
   UpdateSubscriber,

apps/ws/src/shared/shared.module.ts

@@ -3,6 +3,8 @@ import { JwtModule } from '@nestjs/jwt';
 import {
   AnalyticsService,
   DalServiceHealthIndicator,
+  featureFlagsService,
+  LogLevelService,
   QueuesModule,
   WebSocketsInMemoryProviderService,
 } from '@novu/application-generic';
@@ -37,6 +39,8 @@ const PROVIDERS = [
   analyticsService,
   dalService,
   DalServiceHealthIndicator,
+  featureFlagsService,
+  LogLevelService,
   SubscriberOnlineService,
   WebSocketsInMemoryProviderService,
   ...DAL_MODELS,

libs/application-generic/src/logging/index.ts

@@ -10,7 +10,7 @@ export function getErrorInterceptor(): NestInterceptor {
   return new LoggerErrorInterceptor();
 }
 
-const loggingLevelSet = {
+export const loggingLevelSet = {
   trace: 10,
   debug: 20,
   info: 30,
@@ -19,7 +19,7 @@ const loggingLevelSet = {
   fatal: 60,
   none: 70,
 };
-const loggingLevelArr = Object.keys(loggingLevelSet);
+export const loggingLevelArr = Object.keys(loggingLevelSet);
 
 export function getLogLevel() {
   let logLevel = null;
@@ -76,7 +76,7 @@ export function createNestLoggingModuleOptions(settings: {
   return {
     exclude: [
       { path: '*/health-check', method: RequestMethod.GET },
-      { path: '/v1/internal/subscriber-online-state', method: RequestMethod.POST }
+      { path: '/v1/internal/subscriber-online-state', method: RequestMethod.POST },
     ],
     assignResponse: true,
     pinoHttp: {

libs/application-generic/src/services/index.ts

@@ -20,6 +20,7 @@ export * from './content.service';
 export * from './cron';
 export * from './feature-flags';
 export * from './in-memory-provider';
+export * from './log-level';
 export {
   MessageInteractionResult,
   MessageInteractionService,

libs/application-generic/src/services/log-level/index.ts

@@ -0,0 +1 @@
+export * from './log-level.service';

libs/application-generic/src/services/log-level/log-level.service.ts

@@ -0,0 +1,102 @@
+import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
+import { FeatureFlagsKeysEnum } from '@novu/shared';
+import { PinoLogger } from 'nestjs-pino';
+import { getLogLevel, loggingLevelArr } from '../../logging';
+import { FeatureFlagsService } from '../feature-flags';
+
+const LOG_CONTEXT = 'LogLevelService';
+const DEFAULT_POLLING_INTERVAL_MS = 60_000; // one minute
+
+@Injectable()
+export class LogLevelService implements OnModuleInit, OnModuleDestroy {
+  private pollingInterval: NodeJS.Timeout | null = null;
+  private currentLogLevel: string;
+  private readonly pollingIntervalMs: number;
+
+  constructor(
+    private featureFlagsService: FeatureFlagsService,
+    private logger: PinoLogger
+  ) {
+    this.pollingIntervalMs = Number(process.env.LOG_LEVEL_POLLING_INTERVAL_MS) || DEFAULT_POLLING_INTERVAL_MS;
+    this.currentLogLevel = getLogLevel();
+  }
+
+  async onModuleInit(): Promise<void> {
+    await this.updateLogLevel();
+
+    this.pollingInterval = setInterval(async () => {
+      try {
+        await this.updateLogLevel();
+      } catch (error) {
+        Logger.error(`Failed to update log level: ${(error as Error).message}`, (error as Error).stack, LOG_CONTEXT);
+      }
+    }, this.pollingIntervalMs);
+
+    Logger.log(`Log level polling started with interval of ${this.pollingIntervalMs}ms`, LOG_CONTEXT);
+  }
+
+  onModuleDestroy(): void {
+    if (this.pollingInterval) {
+      clearInterval(this.pollingInterval);
+      this.pollingInterval = null;
+      Logger.log('Log level polling stopped', LOG_CONTEXT);
+    }
+  }
+
+  private async updateLogLevel(): Promise<void> {
+    const logLevelFromFlag = await this.getLogLevelFromFeatureFlag();
+
+    const newLogLevel = logLevelFromFlag || this.getFallbackLogLevel();
+
+    if (!this.isValidLogLevel(newLogLevel)) {
+      Logger.warn(
+        `Invalid log level "${newLogLevel}". Valid levels: ${loggingLevelArr.join(', ')}. Keeping current level: ${this.currentLogLevel}`,
+        LOG_CONTEXT
+      );
+
+      return;
+    }
+
+    if (newLogLevel !== this.currentLogLevel) {
+      this.setLogLevel(newLogLevel);
+      Logger.log(`Log level changed from "${this.currentLogLevel}" to "${newLogLevel}"`, LOG_CONTEXT);
+      this.currentLogLevel = newLogLevel;
+    }
+  }
+
+  private async getLogLevelFromFeatureFlag(): Promise<string | undefined> {
+    try {
+      const flagValue = await this.featureFlagsService.getFlag<string | undefined>({
+        key: FeatureFlagsKeysEnum.LOG_LEVEL_STR,
+        defaultValue: undefined,
+        user: { _id: 'system' },
+      });
+
+      if (flagValue && flagValue !== 'undefined') {
+        return flagValue;
+      }
+
+      return undefined;
+    } catch (error) {
+      Logger.warn(`Failed to get log level from feature flag: ${(error as Error).message}`, LOG_CONTEXT);
+
+      return undefined;
+    }
+  }
+
+  private getFallbackLogLevel(): string {
+    return process.env.LOG_LEVEL || process.env.LOGGING_LEVEL || 'info';
+  }
+
+  private isValidLogLevel(level: string): boolean {
+    return loggingLevelArr.includes(level);
+  }
+
+  private setLogLevel(level: string): void {
+    if ((this.logger as any).pinoLogger) {
+      (this.logger as any).pinoLogger.level = level;
+    } else if ((this.logger as any).logger) {
+      (this.logger as any).logger.level = level;
+    }
+  }
+}

libs/application-generic/src/services/queues/standard-queue.service.ts

@@ -46,7 +46,7 @@ export class StandardQueueService extends QueueBaseService {
     }
 
     const schedulerMode = await this.featureFlagsService.getFlag<string>({
-      key: FeatureFlagsKeysEnum.CF_SCHEDULER_MODE,
+      key: FeatureFlagsKeysEnum.CF_SCHEDULER_MODE_STR,
       defaultValue: CloudflareSchedulerMode.OFF,
       organization: { _id: data.data._organizationId, apiServiceLevel: organization.apiServiceLevel },
       environment: { _id: data.data._environmentId },

packages/shared/src/types/feature-flags.ts

@@ -86,9 +86,12 @@ export enum FeatureFlagsKeysEnum {
   IS_BILLING_USAGE_CLICKHOUSE_ENABLED = 'IS_BILLING_USAGE_CLICKHOUSE_ENABLED',
   IS_BILLING_USAGE_CLICKHOUSE_SHADOW_ENABLED = 'IS_BILLING_USAGE_CLICKHOUSE_SHADOW_ENABLED',
   IS_BILLING_USAGE_DETAILED_DIAGNOSTICS_ENABLED = 'IS_BILLING_USAGE_DETAILED_DIAGNOSTICS_ENABLED',
+  IS_ANALYTICS_PAGE_ENABLED = 'IS_ANALYTICS_PAGE_ENABLED',
+  IS_LEGACY_SELECTOR_BUTTON_VISIBLE = 'IS_LEGACY_SELECTOR_BUTTON_VISIBLE',
 
   // String flags
-  CF_SCHEDULER_MODE = 'CF_SCHEDULER_MODE', // Values: "off" | "shadow" | "live" | "complete"
+  CF_SCHEDULER_MODE_STR = 'CF_SCHEDULER_MODE_STR', // Values: "off" | "shadow" | "live" | "complete"
+  LOG_LEVEL_STR = 'LOG_LEVEL_STR', // Values: "trace" | "debug" | "info" | "warn" | "error" | "fatal" | "none"
 
   // Numeric flags
   MAX_WORKFLOW_LIMIT_NUMBER = 'MAX_WORKFLOW_LIMIT_NUMBER',
@@ -98,8 +101,6 @@ export enum FeatureFlagsKeysEnum {
   LOG_EXPIRATION_DAYS_NUMBER = 'LOG_EXPIRATION_DAYS_NUMBER',
   MAX_DATE_ANALYTICS_ENABLED_NUMBER = 'MAX_DATE_ANALYTICS_ENABLED_NUMBER',
   MAX_ENVIRONMENT_COUNT = 'MAX_ENVIRONMENT_COUNT',
-  IS_ANALYTICS_PAGE_ENABLED = 'IS_ANALYTICS_PAGE_ENABLED',
-  IS_LEGACY_SELECTOR_BUTTON_VISIBLE = 'IS_LEGACY_SELECTOR_BUTTON_VISIBLE',
 }
 
 export enum CloudflareSchedulerMode {