diff --git a/apps/api/src/app/shared/shared.module.ts b/apps/api/src/app/shared/shared.module.ts index 628e6ecc208..b87ed39d700 100644 --- a/apps/api/src/app/shared/shared.module.ts +++ b/apps/api/src/app/shared/shared.module.ts @@ -16,6 +16,7 @@ import { GetDecryptedSecretKey, InvalidateCacheService, LoggerModule, + LogLevelService, QueuesModule, RequestLogRepository, StepRunRepository, @@ -139,6 +140,7 @@ const PROVIDERS = [ DalServiceHealthIndicator, featureFlagsService, InvalidateCacheService, + LogLevelService, storageService, ...DAL_MODELS, CreateExecutionDetails, diff --git a/apps/api/src/config/env.validators.ts b/apps/api/src/config/env.validators.ts index c1332ac54ca..4f922ba93b4 100644 --- a/apps/api/src/config/env.validators.ts +++ b/apps/api/src/config/env.validators.ts @@ -73,16 +73,27 @@ export const envValidators = { CLICK_HOUSE_PASSWORD: str({ default: '' }), }), - // Feature Flags - ...Object.keys(FeatureFlagsKeysEnum).reduce( - (acc, key) => { - return { - ...acc, - [key as FeatureFlagsKeysEnum]: bool({ default: false }), - }; - }, - {} as Record> - ), + // Feature Flags - Boolean (start with IS_) + ...Object.keys(FeatureFlagsKeysEnum) + .filter((key) => key.startsWith('IS_')) + .reduce>>((acc, key) => { + acc[key] = bool({ default: false }); + + return acc; + }, {}), + + // Feature Flags - Numeric (end with _NUMBER) + ...Object.keys(FeatureFlagsKeysEnum) + .filter((key) => key.endsWith('_NUMBER')) + .reduce>>((acc, key) => { + acc[key] = num({ default: undefined }); + + return acc; + }, {}), + + // Feature Flags - String + CF_SCHEDULER_MODE_STR: str({ choices: ['off', 'shadow', 'live', 'complete'], default: 'off' }), + LOG_LEVEL_STR: str({ choices: ['trace', 'debug', 'info', 'warn', 'error', 'fatal', 'none'], default: undefined }), // Azure validators ...(processEnv.STORAGE_SERVICE === 'AZURE' && { diff --git a/apps/webhook/src/shared/shared.module.ts b/apps/webhook/src/shared/shared.module.ts index cdcae30f3be..a44fb65f212 100644 --- a/apps/webhook/src/shared/shared.module.ts +++ b/apps/webhook/src/shared/shared.module.ts @@ -1,5 +1,5 @@ import { Module } from '@nestjs/common'; -import { AnalyticsService } from '@novu/application-generic'; +import { AnalyticsService, featureFlagsService, LogLevelService } from '@novu/application-generic'; import { DalService, ExecutionDetailsRepository, IntegrationRepository, MessageRepository } from '@novu/dal'; const DAL_MODELS = [ExecutionDetailsRepository, MessageRepository, IntegrationRepository]; @@ -26,6 +26,8 @@ const PROVIDERS = [ return analyticsService; }, }, + featureFlagsService, + LogLevelService, ]; @Module({ diff --git a/apps/worker/src/app/shared/shared.module.ts b/apps/worker/src/app/shared/shared.module.ts index 8b4318806fa..58dd8493624 100644 --- a/apps/worker/src/app/shared/shared.module.ts +++ b/apps/worker/src/app/shared/shared.module.ts @@ -19,6 +19,7 @@ import { GetTenant, InvalidateCacheService, LoggerModule, + LogLevelService, MetricsModule, ProcessTenant, QueuesModule, @@ -110,6 +111,7 @@ const PROVIDERS = [ DigestFilterSteps, featureFlagsService, InvalidateCacheService, + LogLevelService, StorageHelperService, storageService, UpdateSubscriber, diff --git a/apps/ws/src/shared/shared.module.ts b/apps/ws/src/shared/shared.module.ts index 75252887465..a6c8d3c89ce 100644 --- a/apps/ws/src/shared/shared.module.ts +++ b/apps/ws/src/shared/shared.module.ts @@ -3,6 +3,8 @@ import { JwtModule } from '@nestjs/jwt'; import { AnalyticsService, DalServiceHealthIndicator, + featureFlagsService, + LogLevelService, QueuesModule, WebSocketsInMemoryProviderService, } from '@novu/application-generic'; @@ -37,6 +39,8 @@ const PROVIDERS = [ analyticsService, dalService, DalServiceHealthIndicator, + featureFlagsService, + LogLevelService, SubscriberOnlineService, WebSocketsInMemoryProviderService, ...DAL_MODELS, diff --git a/libs/application-generic/src/logging/index.ts b/libs/application-generic/src/logging/index.ts index 34fe1a7248b..500ce34d198 100644 --- a/libs/application-generic/src/logging/index.ts +++ b/libs/application-generic/src/logging/index.ts @@ -10,7 +10,7 @@ export function getErrorInterceptor(): NestInterceptor { return new LoggerErrorInterceptor(); } -const loggingLevelSet = { +export const loggingLevelSet = { trace: 10, debug: 20, info: 30, @@ -19,7 +19,7 @@ const loggingLevelSet = { fatal: 60, none: 70, }; -const loggingLevelArr = Object.keys(loggingLevelSet); +export const loggingLevelArr = Object.keys(loggingLevelSet); export function getLogLevel() { let logLevel = null; @@ -76,7 +76,7 @@ export function createNestLoggingModuleOptions(settings: { return { exclude: [ { path: '*/health-check', method: RequestMethod.GET }, - { path: '/v1/internal/subscriber-online-state', method: RequestMethod.POST } + { path: '/v1/internal/subscriber-online-state', method: RequestMethod.POST }, ], assignResponse: true, pinoHttp: { diff --git a/libs/application-generic/src/services/index.ts b/libs/application-generic/src/services/index.ts index efed94abcd7..0329131453d 100644 --- a/libs/application-generic/src/services/index.ts +++ b/libs/application-generic/src/services/index.ts @@ -20,6 +20,7 @@ export * from './content.service'; export * from './cron'; export * from './feature-flags'; export * from './in-memory-provider'; +export * from './log-level'; export { MessageInteractionResult, MessageInteractionService, diff --git a/libs/application-generic/src/services/log-level/index.ts b/libs/application-generic/src/services/log-level/index.ts new file mode 100644 index 00000000000..60379ad1ef4 --- /dev/null +++ b/libs/application-generic/src/services/log-level/index.ts @@ -0,0 +1 @@ +export * from './log-level.service'; diff --git a/libs/application-generic/src/services/log-level/log-level.service.ts b/libs/application-generic/src/services/log-level/log-level.service.ts new file mode 100644 index 00000000000..52c1e14e139 --- /dev/null +++ b/libs/application-generic/src/services/log-level/log-level.service.ts @@ -0,0 +1,97 @@ +import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; +import { FeatureFlagsKeysEnum } from '@novu/shared'; +import { PinoLogger } from 'nestjs-pino'; +import { getLogLevel, loggingLevelArr } from '../../logging'; +import { FeatureFlagsService } from '../feature-flags'; + +const LOG_CONTEXT = 'LogLevelService'; +const DEFAULT_POLLING_INTERVAL_MS = 60_000; // one minute + +@Injectable() +export class LogLevelService implements OnModuleInit, OnModuleDestroy { + private pollingInterval: NodeJS.Timeout | null = null; + private currentLogLevel: string; + private readonly pollingIntervalMs: number; + + constructor(private featureFlagsService: FeatureFlagsService) { + this.pollingIntervalMs = Number(process.env.LOG_LEVEL_POLLING_INTERVAL_MS) || DEFAULT_POLLING_INTERVAL_MS; + this.currentLogLevel = getLogLevel(); + } + + async onModuleInit(): Promise { + await this.updateLogLevel(); + + this.pollingInterval = setInterval(async () => { + try { + await this.updateLogLevel(); + } catch (error) { + Logger.error(`Failed to update log level: ${(error as Error).message}`, (error as Error).stack, LOG_CONTEXT); + } + }, this.pollingIntervalMs); + + Logger.log(`Log level polling started with interval of ${this.pollingIntervalMs}ms`, LOG_CONTEXT); + } + + onModuleDestroy(): void { + if (this.pollingInterval) { + clearInterval(this.pollingInterval); + this.pollingInterval = null; + Logger.log('Log level polling stopped', LOG_CONTEXT); + } + } + + private async updateLogLevel(): Promise { + const logLevelFromFlag = await this.getLogLevelFromFeatureFlag(); + + const newLogLevel = logLevelFromFlag || this.getFallbackLogLevel(); + + if (!this.isValidLogLevel(newLogLevel)) { + Logger.warn( + `Invalid log level "${newLogLevel}". Valid levels: ${loggingLevelArr.join(', ')}. Keeping current level: ${this.currentLogLevel}`, + LOG_CONTEXT + ); + + return; + } + + if (newLogLevel !== this.currentLogLevel) { + this.setLogLevel(newLogLevel); + Logger.log(`Log level changed from "${this.currentLogLevel}" to "${newLogLevel}"`, LOG_CONTEXT); + this.currentLogLevel = newLogLevel; + } + } + + private async getLogLevelFromFeatureFlag(): Promise { + try { + const flagValue = await this.featureFlagsService.getFlag({ + key: FeatureFlagsKeysEnum.LOG_LEVEL_STR, + defaultValue: undefined, + user: { _id: 'system' }, + }); + + if (flagValue && flagValue !== 'undefined') { + return flagValue; + } + + return undefined; + } catch (error) { + Logger.warn(`Failed to get log level from feature flag: ${(error as Error).message}`, LOG_CONTEXT); + + return undefined; + } + } + + private getFallbackLogLevel(): string { + return process.env.LOG_LEVEL || process.env.LOGGING_LEVEL || 'info'; + } + + private isValidLogLevel(level: string): boolean { + return loggingLevelArr.includes(level); + } + + private setLogLevel(level: string): void { + if (PinoLogger.root) { + PinoLogger.root.level = level; + } + } +} diff --git a/libs/application-generic/src/services/queues/standard-queue.service.ts b/libs/application-generic/src/services/queues/standard-queue.service.ts index 203439914e6..38d4f038fdf 100644 --- a/libs/application-generic/src/services/queues/standard-queue.service.ts +++ b/libs/application-generic/src/services/queues/standard-queue.service.ts @@ -46,7 +46,7 @@ export class StandardQueueService extends QueueBaseService { } const schedulerMode = await this.featureFlagsService.getFlag({ - key: FeatureFlagsKeysEnum.CF_SCHEDULER_MODE, + key: FeatureFlagsKeysEnum.CF_SCHEDULER_MODE_STR, defaultValue: CloudflareSchedulerMode.OFF, organization: { _id: data.data._organizationId, apiServiceLevel: organization.apiServiceLevel }, environment: { _id: data.data._environmentId }, diff --git a/packages/shared/src/types/feature-flags.ts b/packages/shared/src/types/feature-flags.ts index c4d76be029a..0988882540a 100644 --- a/packages/shared/src/types/feature-flags.ts +++ b/packages/shared/src/types/feature-flags.ts @@ -86,9 +86,12 @@ export enum FeatureFlagsKeysEnum { IS_BILLING_USAGE_CLICKHOUSE_ENABLED = 'IS_BILLING_USAGE_CLICKHOUSE_ENABLED', IS_BILLING_USAGE_CLICKHOUSE_SHADOW_ENABLED = 'IS_BILLING_USAGE_CLICKHOUSE_SHADOW_ENABLED', IS_BILLING_USAGE_DETAILED_DIAGNOSTICS_ENABLED = 'IS_BILLING_USAGE_DETAILED_DIAGNOSTICS_ENABLED', + IS_ANALYTICS_PAGE_ENABLED = 'IS_ANALYTICS_PAGE_ENABLED', + IS_LEGACY_SELECTOR_BUTTON_VISIBLE = 'IS_LEGACY_SELECTOR_BUTTON_VISIBLE', // String flags - CF_SCHEDULER_MODE = 'CF_SCHEDULER_MODE', // Values: "off" | "shadow" | "live" | "complete" + CF_SCHEDULER_MODE_STR = 'CF_SCHEDULER_MODE_STR', // Values: "off" | "shadow" | "live" | "complete" + LOG_LEVEL_STR = 'LOG_LEVEL_STR', // Values: "trace" | "debug" | "info" | "warn" | "error" | "fatal" | "none" // Numeric flags MAX_WORKFLOW_LIMIT_NUMBER = 'MAX_WORKFLOW_LIMIT_NUMBER', @@ -99,8 +102,6 @@ export enum FeatureFlagsKeysEnum { LOG_EXPIRATION_DAYS_NUMBER = 'LOG_EXPIRATION_DAYS_NUMBER', MAX_DATE_ANALYTICS_ENABLED_NUMBER = 'MAX_DATE_ANALYTICS_ENABLED_NUMBER', MAX_ENVIRONMENT_COUNT = 'MAX_ENVIRONMENT_COUNT', - IS_ANALYTICS_PAGE_ENABLED = 'IS_ANALYTICS_PAGE_ENABLED', - IS_LEGACY_SELECTOR_BUTTON_VISIBLE = 'IS_LEGACY_SELECTOR_BUTTON_VISIBLE', } export enum CloudflareSchedulerMode {