From 5b319fb57de16d5b7afae91f921da5b39399df01 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 4 Aug 2026 19:27:40 -0500 Subject: [PATCH 1/9] Reduce noisy change events --- .../OrganizationMaintenanceWorkItemHandler.cs | 2 +- .../ProjectMaintenanceWorkItemHandler.cs | 2 +- .../Migrations/UpdateEventUsage.cs | 4 +- .../Services/UsageService.cs | 6 +- .../organization-notifications.svelte | 17 ++-- .../organization-notifications.svelte.test.ts | 77 +++++++++++++++++++ .../Services/UsageServiceTests.cs | 41 ++++++++++ 7 files changed, 135 insertions(+), 14 deletions(-) create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte.test.ts diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/OrganizationMaintenanceWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/OrganizationMaintenanceWorkItemHandler.cs index 73e886505c..8765f9b19a 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/OrganizationMaintenanceWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/OrganizationMaintenanceWorkItemHandler.cs @@ -56,7 +56,7 @@ public override async Task HandleItemAsync(WorkItemContext context) } if (wi.UpgradePlans || wi.RemoveOldUsageStats) - await _organizationRepository.SaveAsync(results.Documents); + await _organizationRepository.SaveAsync(results.Documents, o => o.Notifications(wi.UpgradePlans)); // Sleep so we are not hammering the backend. await Task.Delay(TimeSpan.FromSeconds(2.5), _timeProvider); diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/ProjectMaintenanceWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/ProjectMaintenanceWorkItemHandler.cs index d6c16ca2fb..dfbb91caf0 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/ProjectMaintenanceWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/ProjectMaintenanceWorkItemHandler.cs @@ -58,7 +58,7 @@ public override async Task HandleItemAsync(WorkItemContext context) } if (workItem.UpdateDefaultBotList || workItem.IncrementConfigurationVersion || workItem.RemoveOldUsageStats) - await _projectRepository.SaveAsync(results.Documents); + await _projectRepository.SaveAsync(results.Documents, o => o.Notifications(workItem.UpdateDefaultBotList || workItem.IncrementConfigurationVersion)); // Sleep so we are not hammering the backend. await Task.Delay(TimeSpan.FromSeconds(2.5), _timeProvider); diff --git a/src/Exceptionless.Core/Migrations/UpdateEventUsage.cs b/src/Exceptionless.Core/Migrations/UpdateEventUsage.cs index 3860bd5848..675bd4c0ed 100644 --- a/src/Exceptionless.Core/Migrations/UpdateEventUsage.cs +++ b/src/Exceptionless.Core/Migrations/UpdateEventUsage.cs @@ -86,7 +86,7 @@ private async Task UpdateOrganizationsUsageAsync(MigrationContext context) } } - await _organizationRepository.SaveAsync(organization); + await _organizationRepository.SaveAsync(organization, o => o.Notifications(false)); await UpdateProjectsUsageAsync(context, organization); processed++; await context.Lock.RenewAsync(); @@ -142,7 +142,7 @@ private async Task UpdateProjectsUsageAsync(MigrationContext context, Organizati usage.Limit = organization.GetMaxEventsPerMonthWithBonus(_timeProvider); } - await _projectRepository.SaveAsync(project); + await _projectRepository.SaveAsync(project, o => o.Notifications(false)); } catch (Exception ex) { diff --git a/src/Exceptionless.Core/Services/UsageService.cs b/src/Exceptionless.Core/Services/UsageService.cs index 4a12491675..744acffa5b 100644 --- a/src/Exceptionless.Core/Services/UsageService.cs +++ b/src/Exceptionless.Core/Services/UsageService.cs @@ -117,7 +117,8 @@ await _cache.RemoveAllAsync(new[] { }); await _cache.SetAsync(GetTotalCacheKey(utcNow, organizationId), usage.Total, TimeSpan.FromHours(8)); - await _organizationRepository.SaveAsync(organization); + // Usage counters and last-event timestamps are operational updates, not user-facing entity changes. + await _organizationRepository.SaveAsync(organization, o => o.Notifications(false)); } } @@ -204,7 +205,8 @@ await _cache.RemoveAllAsync(new[] { await _cache.SetAsync(GetTotalCacheKey(utcNow, project.OrganizationId, projectId), usage.Total, TimeSpan.FromHours(8)); - await _projectRepository.SaveAsync(project); + // Project configuration changes have their own save path and should remain the source of ProjectChanged messages. + await _projectRepository.SaveAsync(project, o => o.Notifications(false)); } } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte index 8b3f7da879..358a393dac 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte @@ -9,7 +9,6 @@ import { ChangeType, type WebSocketMessageValue } from '$features/websockets/models'; import { useEventListener } from 'runed'; import { SvelteSet } from 'svelte/reactivity'; - import { debounce } from 'throttle-debounce'; import FreePlanNotification from './notifications/free-plan-notification.svelte'; import HourlyOverageNotification from './notifications/hourly-overage-notification.svelte'; @@ -95,19 +94,21 @@ ); const requiresPremiumUpgrade = $derived(requiresPremium && !organization?.has_premium_features && !needsProjectConfiguration); - const refetchConfigurationState = debounce(1500, async () => { - await Promise.all([organizationQuery.refetch(), projectsQuery.refetch()]); - }); - useEventListener(document, 'PersistentEventChanged', (event) => { const message = (event as CustomEvent>).detail; - - if (message.change_type === ChangeType.Removed || message.organization_id !== currentOrganizationId.current || !message.project_id) { + const projectNeedsConfiguration = projects.some((project) => project.id === message.project_id && project.is_configured === false); + + if ( + message.change_type === ChangeType.Removed || + message.organization_id !== currentOrganizationId.current || + !message.project_id || + !projectNeedsConfiguration || + configuredProjectIds.has(message.project_id) + ) { return; } configuredProjectIds.add(message.project_id); - void refetchConfigurationState(); }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte.test.ts new file mode 100644 index 0000000000..d2ec6d6aa4 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte.test.ts @@ -0,0 +1,77 @@ +import { ChangeType } from '$features/websockets/models'; +import { render } from '@testing-library/svelte'; +import { tick } from 'svelte'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import OrganizationNotifications from './organization-notifications.svelte'; + +const organizationRefetch = vi.hoisted(() => vi.fn()); +const projectsRefetch = vi.hoisted(() => vi.fn()); +const projects = vi.hoisted(() => [{ id: 'project-id', is_configured: false, organization_id: 'organization-id' }]); + +vi.mock('$features/organizations/api.svelte', () => ({ + getOrganizationQuery: () => ({ data: undefined, refetch: organizationRefetch }), + getOrganizationsQuery: () => ({ data: { data: [] } }) +})); + +vi.mock('$features/organizations/context.svelte', () => ({ + organization: { current: 'organization-id' } +})); + +vi.mock('$features/projects/api.svelte', () => ({ + getOrganizationProjectsQuery: () => ({ data: { data: projects }, isSuccess: true, refetch: projectsRefetch }) +})); + +vi.mock('$features/users/api.svelte', () => ({ + getMeQuery: () => ({ data: { organization_ids: ['organization-id'], roles: [] } }) +})); + +describe('OrganizationNotifications', () => { + beforeEach(() => { + organizationRefetch.mockReset(); + projectsRefetch.mockReset(); + projects[0]!.is_configured = false; + }); + + it('does not refetch organization or project state for persistent event changes', async () => { + render(OrganizationNotifications, { + isChatEnabled: false, + openChat: vi.fn() + }); + + const message = { + change_type: ChangeType.Added, + organization_id: 'organization-id', + project_id: 'project-id' + }; + + document.dispatchEvent(new CustomEvent('PersistentEventChanged', { detail: message })); + document.dispatchEvent(new CustomEvent('PersistentEventChanged', { detail: message })); + await tick(); + + expect(organizationRefetch).not.toHaveBeenCalled(); + expect(projectsRefetch).not.toHaveBeenCalled(); + }); + + it('does not refresh configuration state for an already configured project', async () => { + projects[0]!.is_configured = true; + render(OrganizationNotifications, { + isChatEnabled: false, + openChat: vi.fn() + }); + + document.dispatchEvent( + new CustomEvent('PersistentEventChanged', { + detail: { + change_type: ChangeType.Added, + organization_id: 'organization-id', + project_id: 'project-id' + } + }) + ); + await tick(); + + expect(organizationRefetch).not.toHaveBeenCalled(); + expect(projectsRefetch).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/Exceptionless.Tests/Services/UsageServiceTests.cs b/tests/Exceptionless.Tests/Services/UsageServiceTests.cs index 8a54c6cbd7..33f64e600a 100644 --- a/tests/Exceptionless.Tests/Services/UsageServiceTests.cs +++ b/tests/Exceptionless.Tests/Services/UsageServiceTests.cs @@ -9,6 +9,8 @@ using Foundatio.AsyncEx; using Foundatio.Messaging; using Foundatio.Repositories; +using Foundatio.Repositories.Elasticsearch; +using Foundatio.Repositories.Models; using Xunit; using LogLevel = Microsoft.Extensions.Logging.LogLevel; @@ -273,6 +275,45 @@ await messageBus.SubscribeAsync(po => Assert.Equal(0, usage.Deleted); } + [Fact] + public async Task SavePendingUsageAsync_UsageOnlyChanges_DoesNotPublishEntityChangedMessages() + { + // Arrange + var organization = await _organizationRepository.AddAsync(new Organization { Name = "Test", MaxEventsPerMonth = 750, PlanId = _plans.SmallPlan.Id }, o => o.ImmediateConsistency().Cache()); + var project = await _projectRepository.AddAsync(new Project { Name = "Test", OrganizationId = organization.Id, NextSummaryEndOfDayTicks = TimeProvider.GetUtcNow().UtcDateTime.Ticks }, o => o.ImmediateConsistency().Cache()); + var organizationRepository = Assert.IsAssignableFrom>(_organizationRepository); + var projectRepository = Assert.IsAssignableFrom>(_projectRepository); + int notificationCount = 0; + Func, Task> organizationHandler = (_, _) => + { + Interlocked.Increment(ref notificationCount); + return Task.CompletedTask; + }; + Func, Task> projectHandler = (_, _) => + { + Interlocked.Increment(ref notificationCount); + return Task.CompletedTask; + }; + organizationRepository.BeforePublishEntityChanged.AddHandler(organizationHandler); + projectRepository.BeforePublishEntityChanged.AddHandler(projectHandler); + + try + { + // Act + await _usageService.IncrementTotalAsync(organization.Id, project.Id); + TimeProvider.Advance(TimeSpan.FromMinutes(10)); + await _usageService.SavePendingUsageAsync(); + + // Assert + Assert.Equal(0, notificationCount); + } + finally + { + organizationRepository.BeforePublishEntityChanged.RemoveHandler(organizationHandler); + projectRepository.BeforePublishEntityChanged.RemoveHandler(projectHandler); + } + } + [Fact] public async Task CanGetEventsLeft() { From f8449f0ae412937a7da3e21f45e82b1205869a9b Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 4 Aug 2026 20:27:13 -0500 Subject: [PATCH 2/9] Handle plan overage refreshes --- .../Services/UsageService.cs | 4 +-- .../Api/Handlers/OrganizationHandler.cs | 1 + .../lib/features/organizations/api.svelte.ts | 6 ++++ .../lib/features/organizations/api.test.ts | 24 +++++++++++++++ .../src/lib/features/websockets/models.ts | 4 +++ .../ClientApp/src/routes/(app)/+layout.svelte | 13 +++++++-- .../Endpoints/OrganizationEndpointTests.cs | 29 +++++++++++++++++++ .../Services/UsageServiceTests.cs | 27 +++++++++++++++++ 8 files changed, 103 insertions(+), 5 deletions(-) create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.test.ts diff --git a/src/Exceptionless.Core/Services/UsageService.cs b/src/Exceptionless.Core/Services/UsageService.cs index 744acffa5b..8df6621b33 100644 --- a/src/Exceptionless.Core/Services/UsageService.cs +++ b/src/Exceptionless.Core/Services/UsageService.cs @@ -259,8 +259,8 @@ public async Task HandleOrganizationChangeAsync(Organization modified, Organizat if (bucketTotal.Value >= bucketLimit) { - await _messagePublisher.PublishAsync(new PlanOverage { OrganizationId = modified.Id, IsHourly = true }); await _cache.SetAsync(GetThrottledKey(utcNow, modified.Id), true, TimeSpan.FromMinutes(5)); + await _messagePublisher.PublishAsync(new PlanOverage { OrganizationId = modified.Id, IsHourly = true }); } } @@ -451,8 +451,8 @@ public async Task IncrementTotalAsync(string organizationId, string projectId, i if (bucketTotal >= bucketLimit && bucketTotal - bucketLimit < eventCount) { // org will be throttled during the current bucket of time - await _messagePublisher.PublishAsync(new PlanOverage { OrganizationId = organizationId, IsHourly = true }); await _cache.SetAsync(GetThrottledKey(utcNow, organizationId), true, TimeSpan.FromMinutes(5)); + await _messagePublisher.PublishAsync(new PlanOverage { OrganizationId = organizationId, IsHourly = true }); } } diff --git a/src/Exceptionless.Web/Api/Handlers/OrganizationHandler.cs b/src/Exceptionless.Web/Api/Handlers/OrganizationHandler.cs index c82f4d2901..fe49fd8710 100644 --- a/src/Exceptionless.Web/Api/Handlers/OrganizationHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/OrganizationHandler.cs @@ -929,6 +929,7 @@ private async Task AfterResultMapAsync(ICollection m currentUsage.Discarded = realTimeUsage.CurrentUsage.Discarded; currentUsage.TooBig = realTimeUsage.CurrentUsage.TooBig; currentUsage.Deleted = realTimeUsage.CurrentUsage.Deleted; + viewOrganization.IsOverMonthlyLimit = currentUsage.Limit >= 0 && currentUsage.Total >= currentUsage.Limit; var currentHourUsage = viewOrganization.GetCurrentHourlyUsage(timeProvider); currentHourUsage.Total = realTimeUsage.CurrentHourUsage.Total; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.svelte.ts index b5ad5c220c..fa3d61c803 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.svelte.ts @@ -23,6 +23,12 @@ export async function invalidateOrganizationQueries(queryClient: QueryClient, me } } +export async function invalidatePlanOverageQueries(queryClient: QueryClient, message: WebSocketMessageValue<'PlanOverage'>) { + await queryClient.invalidateQueries({ queryKey: queryKeys.id(message.organization_id, undefined) }); + await queryClient.invalidateQueries({ queryKey: queryKeys.id(message.organization_id, 'stats') }); + await queryClient.invalidateQueries({ queryKey: queryKeys.list(undefined) }); +} + export const queryKeys = { adminSearch: (params: GetAdminSearchOrganizationsParams) => [...queryKeys.list(params.mode), 'admin', { ...params }] as const, changePlan: (id: string | undefined) => [...queryKeys.type, id, 'change-plan'] as const, diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.test.ts new file mode 100644 index 0000000000..513fcff3d0 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.test.ts @@ -0,0 +1,24 @@ +import { QueryClient } from '@tanstack/svelte-query'; +import { describe, expect, it, vi } from 'vitest'; + +import { invalidatePlanOverageQueries, queryKeys } from './api.svelte'; + +describe('invalidatePlanOverageQueries', () => { + it('invalidates only the affected organization state', async () => { + // Arrange + const queryClient = new QueryClient(); + const invalidateSpy = vi.spyOn(queryClient, 'invalidateQueries').mockImplementation(async () => {}); + + // Act + await invalidatePlanOverageQueries(queryClient, { + is_hourly: false, + organization_id: 'organization-id' + }); + + // Assert + expect(invalidateSpy).toHaveBeenCalledTimes(3); + expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: queryKeys.id('organization-id', undefined) }); + expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: queryKeys.id('organization-id', 'stats') }); + expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: queryKeys.list(undefined) }); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/models.ts index 6883e6e193..461805e466 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/models.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/models.ts @@ -67,6 +67,10 @@ export function isEntityChangedType(message: { message: unknown; type: WebSocket return message.type !== 'PlanChanged' && message.type !== 'UserMembershipChanged' && message.type.endsWith('Changed'); } +export function isPlanOverageType(message: { message: unknown; type: WebSocketMessageType }): message is WebSocketMessage<'PlanOverage'> { + return message.type === 'PlanOverage'; +} + export function isWebSocketMessageType(type: string): type is WebSocketMessageType { return ( (['PlanChanged', 'PlanOverage', 'UserMembershipChanged', 'ReleaseNotification', 'SystemNotification'] as const).includes( diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte index e8a908b69d..327487cc0b 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte @@ -15,7 +15,12 @@ import { buildIntercomBootOptions, IntercomShell } from '$features/intercom'; import { shouldLoadIntercomOrganization } from '$features/intercom/config'; import Notifications from '$features/notifications/components/notifications.svelte'; - import { getOrganizationQuery, getOrganizationsQuery, invalidateOrganizationQueries } from '$features/organizations/api.svelte'; + import { + getOrganizationQuery, + getOrganizationsQuery, + invalidateOrganizationQueries, + invalidatePlanOverageQueries + } from '$features/organizations/api.svelte'; import OrganizationNotifications from '$features/organizations/components/organization-notifications.svelte'; import { organization, showOrganizationNotifications } from '$features/organizations/context.svelte'; import { premiumPage } from '$features/organizations/premium-page.svelte'; @@ -28,7 +33,7 @@ import { getMeQuery, invalidateUserQueries } from '$features/users/api.svelte'; import { getGravatarFromCurrentUser } from '$features/users/gravatar.svelte'; import { invalidateWebhookQueries } from '$features/webhooks/api.svelte'; - import { isEntityChangedType, type WebSocketMessageType } from '$features/websockets/models'; + import { isEntityChangedType, isPlanOverageType, type WebSocketMessageType } from '$features/websockets/models'; import { WebSocketClient } from '$features/websockets/web-socket-client.svelte'; import { Telemetry } from '$lib/telemetry'; import { useMiddleware } from '@foundatiofx/fetchclient'; @@ -129,7 +134,9 @@ }) ); - if (isEntityChangedType(data)) { + if (isPlanOverageType(data)) { + await invalidatePlanOverageQueries(queryClient, data.message); + } else if (isEntityChangedType(data)) { switch (data.type) { case 'OrganizationChanged': await invalidateOrganizationQueries(queryClient, data.message); diff --git a/tests/Exceptionless.Tests/Api/Endpoints/OrganizationEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/OrganizationEndpointTests.cs index 0a11eedbc2..0e4815b6e1 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/OrganizationEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/OrganizationEndpointTests.cs @@ -5,6 +5,7 @@ using Exceptionless.Core.Models; using Exceptionless.Core.Models.Billing; using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services; using Exceptionless.Core.Utility; using Exceptionless.Tests.Extensions; using Exceptionless.Tests.Utility; @@ -575,6 +576,34 @@ public async Task GetAsync_ViewOrganization_IncludesIsOverMonthlyLimit() Assert.False(viewOrg.IsOverMonthlyLimit); } + [Fact] + public async Task GetAsync_WithPendingUsageAtMonthlyLimit_ReturnsRealTimeOverageState() + { + // Arrange + var organization = await _organizationRepository.GetByIdAsync(SampleDataService.TEST_ORG_ID); + Assert.NotNull(organization); + + organization.MaxEventsPerMonth = 1; + organization.Usage.Clear(); + organization.UsageHours.Clear(); + await _organizationRepository.SaveAsync(organization, o => o.ImmediateConsistency().Cache().Originals()); + + var usageService = GetService(); + await usageService.IncrementTotalAsync(organization.Id, SampleDataService.TEST_PROJECT_ID); + + // Act + var viewOrganization = await SendRequestAsAsync(r => r + .AsTestOrganizationUser() + .AppendPaths("organizations", organization.Id) + .StatusCodeShouldBeOk() + ); + + // Assert + Assert.NotNull(viewOrganization); + Assert.Equal(1, viewOrganization.GetCurrentUsage(TimeProvider).Total); + Assert.True(viewOrganization.IsOverMonthlyLimit); + } + [Fact] public async Task PostAsync_NewOrganization_SetsCreatedAndUpdatedDates() { diff --git a/tests/Exceptionless.Tests/Services/UsageServiceTests.cs b/tests/Exceptionless.Tests/Services/UsageServiceTests.cs index 33f64e600a..553e01367a 100644 --- a/tests/Exceptionless.Tests/Services/UsageServiceTests.cs +++ b/tests/Exceptionless.Tests/Services/UsageServiceTests.cs @@ -336,6 +336,33 @@ public async Task CanGetEventsLeft() Assert.Equal(0, eventsLeft); } + [Fact] + public async Task IncrementTotalAsync_WhenHourlyLimitCrosses_PublishesAfterThrottleStateIsSet() + { + // Arrange + var organization = await _organizationRepository.AddAsync(new Organization { Name = "Test", MaxEventsPerMonth = 750, PlanId = _plans.SmallPlan.Id }, o => o.ImmediateConsistency().Cache()); + var project = await _projectRepository.AddAsync(new Project { Name = "Test", OrganizationId = organization.Id, NextSummaryEndOfDayTicks = TimeProvider.GetUtcNow().UtcDateTime.Ticks }, o => o.ImmediateConsistency().Cache()); + int eventsLeftInBucket = await _usageService.GetEventsLeftAsync(organization.Id); + var throttleState = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var messageBus = GetService(); + await messageBus.SubscribeAsync(async overage => + { + if (overage.OrganizationId != organization.Id || !overage.IsHourly) + { + return; + } + + var usage = await _usageService.GetUsageAsync(organization.Id); + throttleState.TrySetResult(usage.IsThrottled); + }, TestCancellationToken); + + // Act + await _usageService.IncrementTotalAsync(organization.Id, project.Id, eventsLeftInBucket); + + // Assert + Assert.True(await throttleState.Task.WaitAsync(TimeSpan.FromSeconds(5), TestCancellationToken)); + } + [Fact] public async Task CanIncrementOverageUsageAsync() { From 15e27df966aac3c2aa2b69b800897738d145852b Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 4 Aug 2026 20:43:20 -0500 Subject: [PATCH 3/9] Notify when hourly throttling clears --- .../Services/UsageService.cs | 16 ++++++-- .../Services/UsageServiceTests.cs | 37 ++++++++++++++++++- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/src/Exceptionless.Core/Services/UsageService.cs b/src/Exceptionless.Core/Services/UsageService.cs index 8df6621b33..73c3d758c8 100644 --- a/src/Exceptionless.Core/Services/UsageService.cs +++ b/src/Exceptionless.Core/Services/UsageService.cs @@ -85,6 +85,7 @@ private async Task SavePendingOrganizationUsageAsync(DateTime utcNow) var bucketDiscarded = await _cache.GetAsync(GetBucketDiscardedCacheKey(bucketUtc, organizationId)); var bucketTooBig = await _cache.GetAsync(GetBucketTooBigCacheKey(bucketUtc, organizationId)); var bucketDeleted = await _cache.GetAsync(GetBucketDeletedCacheKey(bucketUtc, organizationId)); + var hourlyThrottleTransition = await _cache.GetAsync(GetHourlyThrottleTransitionKey(bucketUtc, organizationId)); bool hasIngestion = (bucketTotal?.Value ?? 0) > 0 || (bucketBlocked?.Value ?? 0) > 0 || (bucketDiscarded?.Value ?? 0) > 0 || (bucketTooBig?.Value ?? 0) > 0; if (hasIngestion) @@ -113,12 +114,13 @@ await _cache.RemoveAllAsync(new[] { GetBucketDiscardedCacheKey(bucketUtc, organizationId), GetBucketTooBigCacheKey(bucketUtc, organizationId), GetBucketDeletedCacheKey(bucketUtc, organizationId), - GetThrottledKey(bucketUtc, organizationId) + GetThrottledKey(bucketUtc, organizationId), + GetHourlyThrottleTransitionKey(bucketUtc, organizationId) }); await _cache.SetAsync(GetTotalCacheKey(utcNow, organizationId), usage.Total, TimeSpan.FromHours(8)); - // Usage counters and last-event timestamps are operational updates, not user-facing entity changes. - await _organizationRepository.SaveAsync(organization, o => o.Notifications(false)); + // Routine usage updates stay silent, but clients need one refresh when an hourly throttle clears. + await _organizationRepository.SaveAsync(organization, o => o.Notifications(hourlyThrottleTransition?.Value ?? false)); } } @@ -260,6 +262,7 @@ public async Task HandleOrganizationChangeAsync(Organization modified, Organizat if (bucketTotal.Value >= bucketLimit) { await _cache.SetAsync(GetThrottledKey(utcNow, modified.Id), true, TimeSpan.FromMinutes(5)); + await _cache.SetAsync(GetHourlyThrottleTransitionKey(utcNow, modified.Id), true, TimeSpan.FromHours(8)); await _messagePublisher.PublishAsync(new PlanOverage { OrganizationId = modified.Id, IsHourly = true }); } } @@ -452,6 +455,7 @@ public async Task IncrementTotalAsync(string organizationId, string projectId, i { // org will be throttled during the current bucket of time await _cache.SetAsync(GetThrottledKey(utcNow, organizationId), true, TimeSpan.FromMinutes(5)); + await _cache.SetAsync(GetHourlyThrottleTransitionKey(utcNow, organizationId), true, TimeSpan.FromHours(8)); await _messagePublisher.PublishAsync(new PlanOverage { OrganizationId = organizationId, IsHourly = true }); } } @@ -616,6 +620,12 @@ private string GetThrottledKey(DateTime utcTime, string organizationId) return $"usage:{bucket}:{organizationId}:throttled"; } + private string GetHourlyThrottleTransitionKey(DateTime utcTime, string organizationId) + { + int bucket = GetCurrentBucket(utcTime); + return $"usage:{bucket}:{organizationId}:throttled-transition"; + } + private string GetProjectSetKey(DateTime utcTime) { int bucket = GetCurrentBucket(utcTime); diff --git a/tests/Exceptionless.Tests/Services/UsageServiceTests.cs b/tests/Exceptionless.Tests/Services/UsageServiceTests.cs index 553e01367a..f14e08f26b 100644 --- a/tests/Exceptionless.Tests/Services/UsageServiceTests.cs +++ b/tests/Exceptionless.Tests/Services/UsageServiceTests.cs @@ -279,7 +279,7 @@ await messageBus.SubscribeAsync(po => public async Task SavePendingUsageAsync_UsageOnlyChanges_DoesNotPublishEntityChangedMessages() { // Arrange - var organization = await _organizationRepository.AddAsync(new Organization { Name = "Test", MaxEventsPerMonth = 750, PlanId = _plans.SmallPlan.Id }, o => o.ImmediateConsistency().Cache()); + var organization = await _organizationRepository.AddAsync(new Organization { Name = "Test", MaxEventsPerMonth = 1_000_000, PlanId = _plans.SmallPlan.Id }, o => o.ImmediateConsistency().Cache()); var project = await _projectRepository.AddAsync(new Project { Name = "Test", OrganizationId = organization.Id, NextSummaryEndOfDayTicks = TimeProvider.GetUtcNow().UtcDateTime.Ticks }, o => o.ImmediateConsistency().Cache()); var organizationRepository = Assert.IsAssignableFrom>(_organizationRepository); var projectRepository = Assert.IsAssignableFrom>(_projectRepository); @@ -314,6 +314,41 @@ public async Task SavePendingUsageAsync_UsageOnlyChanges_DoesNotPublishEntityCha } } + [Fact] + public async Task SavePendingUsageAsync_HourlyThrottleClears_PublishesOrganizationChangedMessage() + { + // Arrange + var organization = await _organizationRepository.AddAsync(new Organization { Name = "Test", MaxEventsPerMonth = 750, PlanId = _plans.SmallPlan.Id }, o => o.ImmediateConsistency().Cache()); + var project = await _projectRepository.AddAsync(new Project { Name = "Test", OrganizationId = organization.Id, NextSummaryEndOfDayTicks = TimeProvider.GetUtcNow().UtcDateTime.Ticks }, o => o.ImmediateConsistency().Cache()); + int eventsLeftInBucket = await _usageService.GetEventsLeftAsync(organization.Id); + var organizationRepository = Assert.IsAssignableFrom>(_organizationRepository); + int notificationCount = 0; + Func, Task> organizationHandler = (_, _) => + { + Interlocked.Increment(ref notificationCount); + return Task.CompletedTask; + }; + organizationRepository.BeforePublishEntityChanged.AddHandler(organizationHandler); + + try + { + await _usageService.IncrementTotalAsync(organization.Id, project.Id, eventsLeftInBucket); + Assert.True((await _usageService.GetUsageAsync(organization.Id)).IsThrottled); + + // Act + TimeProvider.Advance(TimeSpan.FromMinutes(10)); + await _usageService.SavePendingUsageAsync(); + + // Assert + Assert.False((await _usageService.GetUsageAsync(organization.Id)).IsThrottled); + Assert.Equal(1, notificationCount); + } + finally + { + organizationRepository.BeforePublishEntityChanged.RemoveHandler(organizationHandler); + } + } + [Fact] public async Task CanGetEventsLeft() { From e3e1f6d5a288e18dfda7b37afe5bfed728aaecdd Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Tue, 4 Aug 2026 20:55:47 -0500 Subject: [PATCH 4/9] Refresh legacy organization cache on plan overage --- .../components/organization/organization-service.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js b/src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js index e19f1c10bf..390d242fe2 100644 --- a/src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js +++ b/src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js @@ -10,6 +10,7 @@ $rootScope.$on("auth:logout", _cache.removeAll); $rootScope.$on("OrganizationChanged", _cache.removeAll); $rootScope.$on("ProjectChanged", _cache.removeAll); + $rootScope.$on("PlanOverage", _cache.removeAll); $rootScope.$on("StackChanged", function ($event, data) { if (data.added) { From 88954057acf49d672538f00f7f25b9b5426a0f2d Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Wed, 5 Aug 2026 23:14:01 -0500 Subject: [PATCH 5/9] Preserve targeted client refreshes --- .../Services/UsageService.cs | 20 ++++++- .../organization-notifications.svelte | 37 +++++++----- .../organization-notifications.svelte.test.ts | 21 ++++++- .../Services/UsageServiceTests.cs | 58 ++++++++++++++++++- 4 files changed, 116 insertions(+), 20 deletions(-) diff --git a/src/Exceptionless.Core/Services/UsageService.cs b/src/Exceptionless.Core/Services/UsageService.cs index 73c3d758c8..009601a794 100644 --- a/src/Exceptionless.Core/Services/UsageService.cs +++ b/src/Exceptionless.Core/Services/UsageService.cs @@ -91,6 +91,16 @@ private async Task SavePendingOrganizationUsageAsync(DateTime utcNow) if (hasIngestion) organization.LastEventDateUtc = _timeProvider.GetUtcNow().UtcDateTime; + var currentMonthUtc = bucketUtc.StartOfMonth(); + var previousMonthUtc = currentMonthUtc.AddMonths(-1); + bool hasCurrentMonthUsage = organization.Usage.Any(u => u.Date.Year == currentMonthUtc.Year && u.Date.Month == currentMonthUtc.Month); + var previousMonthUsage = organization.Usage.FirstOrDefault(u => u.Date.Year == previousMonthUtc.Year && u.Date.Month == previousMonthUtc.Month); + bool monthlyOverageCleared = !hasCurrentMonthUsage && previousMonthUsage is { Limit: > 0 } && previousMonthUsage.Total >= previousMonthUsage.Limit; + int bucketLimit = GetBucketEventLimit(organization.GetMaxEventsPerMonthWithBonus(_timeProvider), bucketUtc); + bool hourlyThrottleCleared = hourlyThrottleTransition is { HasValue: true } transition + ? transition.Value + : bucketLimit >= 0 && bucketTotal is { HasValue: true } total && total.Value >= bucketLimit; + var usage = organization.GetUsage(bucketUtc, _timeProvider); usage.Limit = organization.GetMaxEventsPerMonthWithBonus(_timeProvider); usage.Total += bucketTotal?.Value ?? 0; @@ -119,8 +129,8 @@ await _cache.RemoveAllAsync(new[] { }); await _cache.SetAsync(GetTotalCacheKey(utcNow, organizationId), usage.Total, TimeSpan.FromHours(8)); - // Routine usage updates stay silent, but clients need one refresh when an hourly throttle clears. - await _organizationRepository.SaveAsync(organization, o => o.Notifications(hourlyThrottleTransition?.Value ?? false)); + // Routine usage updates stay silent, but clients need one refresh when an overage clears. + await _organizationRepository.SaveAsync(organization, o => o.Notifications(hourlyThrottleCleared || monthlyOverageCleared)); } } @@ -533,11 +543,15 @@ public async Task IncrementDeletedAsync(string organizationId, string? projectId } private int GetBucketEventLimit(int maxEventsPerMonth) + { + return GetBucketEventLimit(maxEventsPerMonth, _timeProvider.GetUtcNow().UtcDateTime); + } + + private int GetBucketEventLimit(int maxEventsPerMonth, DateTime utcNow) { if (maxEventsPerMonth < 5000) return maxEventsPerMonth; - var utcNow = _timeProvider.GetUtcNow().UtcDateTime; var timeLeftInMonth = utcNow.EndOfMonth() - utcNow; if (timeLeftInMonth < TimeSpan.FromDays(1)) return maxEventsPerMonth; diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte index 358a393dac..b90bcfe2a8 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte @@ -1,3 +1,25 @@ + + diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte.test.ts index d2ec6d6aa4..ddfc3a7396 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte.test.ts @@ -3,7 +3,7 @@ import { render } from '@testing-library/svelte'; import { tick } from 'svelte'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -import OrganizationNotifications from './organization-notifications.svelte'; +import OrganizationNotifications, { recordConfiguredProjectId } from './organization-notifications.svelte'; const organizationRefetch = vi.hoisted(() => vi.fn()); const projectsRefetch = vi.hoisted(() => vi.fn()); @@ -33,6 +33,25 @@ describe('OrganizationNotifications', () => { projects[0]!.is_configured = false; }); + it('records configuration events before project data loads', () => { + const configuredProjectIds = new Set(); + + const recorded = recordConfiguredProjectId( + { + change_type: ChangeType.Added, + data: {}, + organization_id: 'organization-id', + project_id: 'project-id', + type: 'PersistentEvent' + }, + 'organization-id', + configuredProjectIds + ); + + expect(recorded).toBe(true); + expect(configuredProjectIds).toContain('project-id'); + }); + it('does not refetch organization or project state for persistent event changes', async () => { render(OrganizationNotifications, { isChatEnabled: false, diff --git a/tests/Exceptionless.Tests/Services/UsageServiceTests.cs b/tests/Exceptionless.Tests/Services/UsageServiceTests.cs index f14e08f26b..552f13718f 100644 --- a/tests/Exceptionless.Tests/Services/UsageServiceTests.cs +++ b/tests/Exceptionless.Tests/Services/UsageServiceTests.cs @@ -5,8 +5,10 @@ using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; using Exceptionless.Core.Services; +using Exceptionless.DateTimeExtensions; using Exceptionless.Tests.Extensions; using Foundatio.AsyncEx; +using Foundatio.Caching; using Foundatio.Messaging; using Foundatio.Repositories; using Foundatio.Repositories.Elasticsearch; @@ -314,8 +316,10 @@ public async Task SavePendingUsageAsync_UsageOnlyChanges_DoesNotPublishEntityCha } } - [Fact] - public async Task SavePendingUsageAsync_HourlyThrottleClears_PublishesOrganizationChangedMessage() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SavePendingUsageAsync_HourlyThrottleClears_PublishesOrganizationChangedMessage(bool removeTransitionMarker) { // Arrange var organization = await _organizationRepository.AddAsync(new Organization { Name = "Test", MaxEventsPerMonth = 750, PlanId = _plans.SmallPlan.Id }, o => o.ImmediateConsistency().Cache()); @@ -334,6 +338,11 @@ public async Task SavePendingUsageAsync_HourlyThrottleClears_PublishesOrganizati { await _usageService.IncrementTotalAsync(organization.Id, project.Id, eventsLeftInBucket); Assert.True((await _usageService.GetUsageAsync(organization.Id)).IsThrottled); + if (removeTransitionMarker) + { + int bucket = TimeProvider.GetUtcNow().UtcDateTime.Floor(TimeSpan.FromMinutes(5)).ToEpoch(); + await GetService().RemoveAsync($"usage:{bucket}:{organization.Id}:throttled-transition"); + } // Act TimeProvider.Advance(TimeSpan.FromMinutes(10)); @@ -349,6 +358,51 @@ public async Task SavePendingUsageAsync_HourlyThrottleClears_PublishesOrganizati } } + [Fact] + public async Task SavePendingUsageAsync_WhenMonthlyOverageClearsAfterPlanIncrease_PublishesOrganizationChangedMessage() + { + // Arrange + TimeProvider.SetUtcNow(new DateTime(2015, 2, 28, 23, 55, 0, DateTimeKind.Utc)); + var organization = new Organization { Name = "Test", MaxEventsPerMonth = 750, PlanId = _plans.SmallPlan.Id }; + organization.GetCurrentUsage(TimeProvider).Total = organization.MaxEventsPerMonth; + organization.MaxEventsPerMonth = 1_500; + organization = await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency().Cache()); + var project = await _projectRepository.AddAsync(new Project { Name = "Test", OrganizationId = organization.Id, NextSummaryEndOfDayTicks = TimeProvider.GetUtcNow().UtcDateTime.Ticks }, o => o.ImmediateConsistency().Cache()); + var organizationRepository = Assert.IsAssignableFrom>(_organizationRepository); + int notificationCount = 0; + Func, Task> organizationHandler = (_, _) => + { + Interlocked.Increment(ref notificationCount); + return Task.CompletedTask; + }; + organizationRepository.BeforePublishEntityChanged.AddHandler(organizationHandler); + + try + { + TimeProvider.SetUtcNow(new DateTime(2015, 3, 1, 0, 0, 0, DateTimeKind.Utc)); + await _usageService.IncrementTotalAsync(organization.Id, project.Id); + + // Act + TimeProvider.Advance(TimeSpan.FromMinutes(10)); + await _usageService.SavePendingUsageAsync(); + + // Assert + organization = await _organizationRepository.GetByIdAsync(organization.Id); + Assert.NotNull(organization); + Assert.False(organization.IsOverMonthlyLimit(TimeProvider)); + Assert.Equal(1, notificationCount); + + await _usageService.IncrementTotalAsync(organization.Id, project.Id); + TimeProvider.Advance(TimeSpan.FromMinutes(10)); + await _usageService.SavePendingUsageAsync(); + Assert.Equal(1, notificationCount); + } + finally + { + organizationRepository.BeforePublishEntityChanged.RemoveHandler(organizationHandler); + } + } + [Fact] public async Task CanGetEventsLeft() { From 2e96f54f3669d3100351f8281938b38da980b6ff Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 6 Aug 2026 00:18:34 -0500 Subject: [PATCH 6/9] Fix usage state refresh boundaries --- .../Migrations/UpdateEventUsage.cs | 9 +++- .../Services/UsageService.cs | 9 +--- .../Api/Handlers/OrganizationHandler.cs | 2 +- .../organization/organization-service.js | 18 ++++++- .../lib/features/organizations/api.svelte.ts | 18 +++++-- .../lib/features/organizations/api.test.ts | 21 ++++++-- .../lib/features/organizations/utils.test.ts | 10 ++++ .../src/lib/features/organizations/utils.ts | 7 +++ .../src/lib/features/projects/api.svelte.ts | 4 +- .../ClientApp/src/routes/(app)/+layout.svelte | 17 +++++++ .../[organizationId]/usage/+page.svelte | 3 +- .../project/[projectId]/usage/+page.svelte | 4 +- .../Endpoints/OrganizationEndpointTests.cs | 31 ++++++++++++ .../UpdateEventUsageMigrationTests.cs | 48 +++++++++++++++++++ .../Services/UsageServiceTests.cs | 6 +-- 15 files changed, 184 insertions(+), 23 deletions(-) create mode 100644 src/Exceptionless.Web/ClientApp/src/lib/features/organizations/utils.test.ts diff --git a/src/Exceptionless.Core/Migrations/UpdateEventUsage.cs b/src/Exceptionless.Core/Migrations/UpdateEventUsage.cs index 675bd4c0ed..32ddb62143 100644 --- a/src/Exceptionless.Core/Migrations/UpdateEventUsage.cs +++ b/src/Exceptionless.Core/Migrations/UpdateEventUsage.cs @@ -70,6 +70,9 @@ private async Task UpdateOrganizationsUsageAsync(MigrationContext context) using var _ = _logger.BeginScope(new ExceptionlessState().Organization(organization.Id)); try { + int effectiveMonthlyLimit = organization.GetMaxEventsPerMonthWithBonus(_timeProvider); + var currentMonthUtc = _timeProvider.GetUtcNow().UtcDateTime.StartOfMonth(); + int currentMonthTotalBeforeRepair = organization.Usage.FirstOrDefault(u => u.Date == currentMonthUtc)?.Total ?? 0; var result = await _eventRepository.CountAsync(q => q.Organization(organization.Id).AggregationsExpression("date:date~1M")); var dateAggs = result.Aggregations.DateHistogram("date_date"); if (dateAggs?.Buckets is null) @@ -86,7 +89,11 @@ private async Task UpdateOrganizationsUsageAsync(MigrationContext context) } } - await _organizationRepository.SaveAsync(organization, o => o.Notifications(false)); + int currentMonthTotalAfterRepair = organization.Usage.FirstOrDefault(u => u.Date == currentMonthUtc)?.Total ?? 0; + bool monthlyOverageStarted = effectiveMonthlyLimit >= 0 + && currentMonthTotalBeforeRepair < effectiveMonthlyLimit + && currentMonthTotalAfterRepair >= effectiveMonthlyLimit; + await _organizationRepository.SaveAsync(organization, o => o.Notifications(monthlyOverageStarted)); await UpdateProjectsUsageAsync(context, organization); processed++; await context.Lock.RenewAsync(); diff --git a/src/Exceptionless.Core/Services/UsageService.cs b/src/Exceptionless.Core/Services/UsageService.cs index 009601a794..3310268c34 100644 --- a/src/Exceptionless.Core/Services/UsageService.cs +++ b/src/Exceptionless.Core/Services/UsageService.cs @@ -91,11 +91,6 @@ private async Task SavePendingOrganizationUsageAsync(DateTime utcNow) if (hasIngestion) organization.LastEventDateUtc = _timeProvider.GetUtcNow().UtcDateTime; - var currentMonthUtc = bucketUtc.StartOfMonth(); - var previousMonthUtc = currentMonthUtc.AddMonths(-1); - bool hasCurrentMonthUsage = organization.Usage.Any(u => u.Date.Year == currentMonthUtc.Year && u.Date.Month == currentMonthUtc.Month); - var previousMonthUsage = organization.Usage.FirstOrDefault(u => u.Date.Year == previousMonthUtc.Year && u.Date.Month == previousMonthUtc.Month); - bool monthlyOverageCleared = !hasCurrentMonthUsage && previousMonthUsage is { Limit: > 0 } && previousMonthUsage.Total >= previousMonthUsage.Limit; int bucketLimit = GetBucketEventLimit(organization.GetMaxEventsPerMonthWithBonus(_timeProvider), bucketUtc); bool hourlyThrottleCleared = hourlyThrottleTransition is { HasValue: true } transition ? transition.Value @@ -129,8 +124,8 @@ await _cache.RemoveAllAsync(new[] { }); await _cache.SetAsync(GetTotalCacheKey(utcNow, organizationId), usage.Total, TimeSpan.FromHours(8)); - // Routine usage updates stay silent, but clients need one refresh when an overage clears. - await _organizationRepository.SaveAsync(organization, o => o.Notifications(hourlyThrottleCleared || monthlyOverageCleared)); + // Routine usage updates stay silent, but clients need one refresh when hourly throttling clears. + await _organizationRepository.SaveAsync(organization, o => o.Notifications(hourlyThrottleCleared)); } } diff --git a/src/Exceptionless.Web/Api/Handlers/OrganizationHandler.cs b/src/Exceptionless.Web/Api/Handlers/OrganizationHandler.cs index fe49fd8710..a64badd407 100644 --- a/src/Exceptionless.Web/Api/Handlers/OrganizationHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/OrganizationHandler.cs @@ -923,7 +923,7 @@ private async Task AfterResultMapAsync(ICollection m viewOrganization.TrimUsage(timeProvider); var currentUsage = viewOrganization.GetCurrentUsage(timeProvider); - currentUsage.Limit = realTimeUsage.CurrentUsage.Limit; + currentUsage.Limit = viewOrganization.GetMaxEventsPerMonthWithBonus(timeProvider); currentUsage.Total = realTimeUsage.CurrentUsage.Total; currentUsage.Blocked = realTimeUsage.CurrentUsage.Blocked; currentUsage.Discarded = realTimeUsage.CurrentUsage.Discarded; diff --git a/src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js b/src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js index 390d242fe2..ec78a0730d 100644 --- a/src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js +++ b/src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js @@ -3,8 +3,9 @@ angular .module("exceptionless.organization", ["restangular"]) - .factory("organizationService", function ($cacheFactory, $rootScope, objectIDService, Restangular) { + .factory("organizationService", function ($cacheFactory, $interval, $rootScope, objectIDService, Restangular) { var _cache = $cacheFactory("http:organization"); + var _usageMonth = getUsageMonth(); $rootScope.$on("cache:clear", _cache.removeAll); $rootScope.$on("cache:clear-organization", _cache.removeAll); $rootScope.$on("auth:logout", _cache.removeAll); @@ -12,6 +13,21 @@ $rootScope.$on("ProjectChanged", _cache.removeAll); $rootScope.$on("PlanOverage", _cache.removeAll); + $interval(function () { + var usageMonth = getUsageMonth(); + if (usageMonth === _usageMonth) { + return; + } + + _usageMonth = usageMonth; + $rootScope.$broadcast("OrganizationChanged"); + }, 60000); + + function getUsageMonth() { + var now = new Date(); + return now.getUTCFullYear() * 12 + now.getUTCMonth(); + } + $rootScope.$on("StackChanged", function ($event, data) { if (data.added) { _cache.removeAll(); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.svelte.ts index fa3d61c803..3c64d862d4 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.svelte.ts @@ -23,10 +23,18 @@ export async function invalidateOrganizationQueries(queryClient: QueryClient, me } } +export async function invalidateOrganizationUsageQueries(queryClient: QueryClient, organizationId?: string) { + const invalidations = [queryClient.invalidateQueries({ queryKey: queryKeys.list(undefined) })]; + if (organizationId) { + invalidations.push(queryClient.invalidateQueries({ exact: true, queryKey: queryKeys.id(organizationId, undefined) })); + invalidations.push(queryClient.invalidateQueries({ exact: true, queryKey: queryKeys.id(organizationId, 'stats') })); + } + + await Promise.all(invalidations); +} + export async function invalidatePlanOverageQueries(queryClient: QueryClient, message: WebSocketMessageValue<'PlanOverage'>) { - await queryClient.invalidateQueries({ queryKey: queryKeys.id(message.organization_id, undefined) }); - await queryClient.invalidateQueries({ queryKey: queryKeys.id(message.organization_id, 'stats') }); - await queryClient.invalidateQueries({ queryKey: queryKeys.list(undefined) }); + await invalidateOrganizationUsageQueries(queryClient, message.organization_id); } export const queryKeys = { @@ -114,6 +122,7 @@ export interface GetOrganizationRequest { params?: { mode: 'stats' | undefined; }; + refetchInterval?: false | number; route: { id: string | undefined; }; @@ -366,7 +375,8 @@ export function getOrganizationQuery(request: GetOrganizationRequest) { return response.data!; }, - queryKey: queryKeys.id(request.route.id, request.params?.mode) + queryKey: queryKeys.id(request.route.id, request.params?.mode), + refetchInterval: request.refetchInterval })); } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.test.ts index 513fcff3d0..ee43ecbd4f 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.test.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.test.ts @@ -1,7 +1,7 @@ import { QueryClient } from '@tanstack/svelte-query'; import { describe, expect, it, vi } from 'vitest'; -import { invalidatePlanOverageQueries, queryKeys } from './api.svelte'; +import { invalidateOrganizationUsageQueries, invalidatePlanOverageQueries, queryKeys } from './api.svelte'; describe('invalidatePlanOverageQueries', () => { it('invalidates only the affected organization state', async () => { @@ -17,8 +17,23 @@ describe('invalidatePlanOverageQueries', () => { // Assert expect(invalidateSpy).toHaveBeenCalledTimes(3); - expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: queryKeys.id('organization-id', undefined) }); - expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: queryKeys.id('organization-id', 'stats') }); + expect(invalidateSpy).toHaveBeenCalledWith({ exact: true, queryKey: queryKeys.id('organization-id', undefined) }); + expect(invalidateSpy).toHaveBeenCalledWith({ exact: true, queryKey: queryKeys.id('organization-id', 'stats') }); + expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: queryKeys.list(undefined) }); + }); +}); + +describe('invalidateOrganizationUsageQueries', () => { + it('invalidates organization lists when there is no active organization', async () => { + // Arrange + const queryClient = new QueryClient(); + const invalidateSpy = vi.spyOn(queryClient, 'invalidateQueries').mockImplementation(async () => {}); + + // Act + await invalidateOrganizationUsageQueries(queryClient); + + // Assert + expect(invalidateSpy).toHaveBeenCalledOnce(); expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: queryKeys.list(undefined) }); }); }); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/utils.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/utils.test.ts new file mode 100644 index 0000000000..92ccda8eec --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/utils.test.ts @@ -0,0 +1,10 @@ +import { describe, expect, it } from 'vitest'; + +import { getUtcMonthKey } from './utils'; + +describe('getUtcMonthKey', () => { + it('changes only when the UTC month changes', () => { + expect(getUtcMonthKey(new Date('2026-08-01T00:00:00.000Z'))).toBe(getUtcMonthKey(new Date('2026-08-31T23:59:59.999Z'))); + expect(getUtcMonthKey(new Date('2026-09-01T00:00:00.000Z'))).not.toBe(getUtcMonthKey(new Date('2026-08-31T23:59:59.999Z'))); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/utils.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/utils.ts index 11cdb6569a..223b06d788 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/utils.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/utils.ts @@ -2,6 +2,9 @@ import { isSameUtcMonth } from '$features/shared/dates'; import type { ViewOrganization } from './models'; +export const ORGANIZATION_USAGE_REFETCH_INTERVAL_MS = 5 * 60 * 1000; +export const ORGANIZATION_USAGE_ROLLOVER_CHECK_INTERVAL_MS = 60 * 1000; + export function getNextBillingDateUtc(organization?: ViewOrganization): Date { if (organization?.subscribe_date) { console.log('Organization subscribe date for next billing date:', organization.subscribe_date); @@ -30,3 +33,7 @@ export function getRemainingEventLimit(organization?: ViewOrganization): number return organization.max_events_per_month + bonusEvents; } + +export function getUtcMonthKey(date = new Date()): number { + return date.getUTCFullYear() * 12 + date.getUTCMonth(); +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/projects/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/projects/api.svelte.ts index 65560321ba..f4342e2a14 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/projects/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/projects/api.svelte.ts @@ -117,6 +117,7 @@ export interface GetProjectIntegrationNotificationSettingsRequest { } export interface GetProjectRequest { + refetchInterval?: false | number; route: { id: string | undefined; }; @@ -396,7 +397,8 @@ export function getProjectQuery(request: GetProjectRequest) { return response.data!; }, - queryKey: queryKeys.id(request.route.id) + queryKey: queryKeys.id(request.route.id), + refetchInterval: request.refetchInterval })); } diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte index 327487cc0b..ef2d6d4f18 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte @@ -19,11 +19,13 @@ getOrganizationQuery, getOrganizationsQuery, invalidateOrganizationQueries, + invalidateOrganizationUsageQueries, invalidatePlanOverageQueries } from '$features/organizations/api.svelte'; import OrganizationNotifications from '$features/organizations/components/organization-notifications.svelte'; import { organization, showOrganizationNotifications } from '$features/organizations/context.svelte'; import { premiumPage } from '$features/organizations/premium-page.svelte'; + import { getUtcMonthKey, ORGANIZATION_USAGE_ROLLOVER_CHECK_INTERVAL_MS } from '$features/organizations/utils'; import { invalidateProjectQueries } from '$features/projects/api.svelte'; import { getSavedViewsQuery, invalidateSavedViewQueries, isSavedViewDeleted } from '$features/saved-views/api.svelte'; import { savedViewHref } from '$features/saved-views/slugs'; @@ -38,6 +40,7 @@ import { Telemetry } from '$lib/telemetry'; import { useMiddleware } from '@foundatiofx/fetchclient'; import { useQueryClient } from '@tanstack/svelte-query'; + import { useInterval } from 'runed'; import { tick } from 'svelte'; import { fade } from 'svelte/transition'; @@ -120,6 +123,20 @@ }); const queryClient = useQueryClient(); + let organizationUsageMonth = getUtcMonthKey(); + useInterval(() => ORGANIZATION_USAGE_ROLLOVER_CHECK_INTERVAL_MS, { + callback: () => { + const currentMonth = getUtcMonthKey(); + if (currentMonth === organizationUsageMonth) { + return; + } + + organizationUsageMonth = currentMonth; + void invalidateOrganizationUsageQueries(queryClient, organization.current); + }, + immediate: false + }); + async function onMessage(message: MessageEvent) { const data: { message: unknown; type: WebSocketMessageType } = message.data ? JSON.parse(message.data) : null; diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/organization/[organizationId]/usage/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/organization/[organizationId]/usage/+page.svelte index 0c42001cd2..c8ac8059bb 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/organization/[organizationId]/usage/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/organization/[organizationId]/usage/+page.svelte @@ -10,13 +10,14 @@ import { env } from '$env/dynamic/public'; import { ChangePlanDialog } from '$features/billing'; import { getOrganizationQuery } from '$features/organizations/api.svelte'; - import { getNextBillingDateUtc, getRemainingEventLimit } from '$features/organizations/utils'; + import { getNextBillingDateUtc, getRemainingEventLimit, ORGANIZATION_USAGE_REFETCH_INTERVAL_MS } from '$features/organizations/utils'; import { formatDateLabel, formatLongDate } from '$shared/dates'; import { scaleUtc } from 'd3-scale'; import { curveNatural } from 'd3-shape'; import { AreaChart } from 'layerchart'; const organizationQuery = getOrganizationQuery({ + refetchInterval: ORGANIZATION_USAGE_REFETCH_INTERVAL_MS, route: { get id() { return page.params.organizationId || ''; diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/usage/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/usage/+page.svelte index 27f5590cde..e63eef973b 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/usage/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/usage/+page.svelte @@ -11,7 +11,7 @@ import { ChangePlanDialog } from '$features/billing'; import { getOrganizationQuery } from '$features/organizations/api.svelte'; import { organization } from '$features/organizations/context.svelte'; - import { getNextBillingDateUtc, getRemainingEventLimit } from '$features/organizations/utils'; + import { getNextBillingDateUtc, getRemainingEventLimit, ORGANIZATION_USAGE_REFETCH_INTERVAL_MS } from '$features/organizations/utils'; import { getProjectQuery } from '$features/projects/api.svelte'; import { formatDateLabel, formatLongDate } from '$shared/dates'; import { scaleUtc } from 'd3-scale'; @@ -22,6 +22,7 @@ params: { mode: 'stats' }, + refetchInterval: ORGANIZATION_USAGE_REFETCH_INTERVAL_MS, route: { get id() { return organization.current; @@ -35,6 +36,7 @@ const nextBillingDate = $derived(getNextBillingDateUtc(organizationQuery.data)); const projectQuery = getProjectQuery({ + refetchInterval: ORGANIZATION_USAGE_REFETCH_INTERVAL_MS, route: { get id() { return page.params.projectId || ''; diff --git a/tests/Exceptionless.Tests/Api/Endpoints/OrganizationEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/OrganizationEndpointTests.cs index 0e4815b6e1..e7227cd0ca 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/OrganizationEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/OrganizationEndpointTests.cs @@ -604,6 +604,37 @@ public async Task GetAsync_WithPendingUsageAtMonthlyLimit_ReturnsRealTimeOverage Assert.True(viewOrganization.IsOverMonthlyLimit); } + [Fact] + public async Task GetAsync_WithExpiredBonusAndStaleUsageLimit_ReturnsLiveOverageState() + { + // Arrange + var organization = await _organizationRepository.GetByIdAsync(SampleDataService.TEST_ORG_ID); + Assert.NotNull(organization); + + organization.MaxEventsPerMonth = 1; + organization.BonusEventsPerMonth = 10; + organization.BonusExpiration = TimeProvider.GetUtcNow().UtcDateTime.Subtract(TimeSpan.FromMinutes(1)); + organization.Usage.Clear(); + organization.UsageHours.Clear(); + var currentUsage = organization.GetCurrentUsage(TimeProvider); + currentUsage.Limit = organization.MaxEventsPerMonth + organization.BonusEventsPerMonth; + currentUsage.Total = 2; + await _organizationRepository.SaveAsync(organization, o => o.ImmediateConsistency().Cache().Originals()); + + // Act + var viewOrganization = await SendRequestAsAsync(r => r + .AsTestOrganizationUser() + .AppendPaths("organizations", organization.Id) + .StatusCodeShouldBeOk() + ); + + // Assert + Assert.NotNull(viewOrganization); + Assert.Equal(organization.MaxEventsPerMonth, viewOrganization.GetCurrentUsage(TimeProvider).Limit); + Assert.Equal(2, viewOrganization.GetCurrentUsage(TimeProvider).Total); + Assert.True(viewOrganization.IsOverMonthlyLimit); + } + [Fact] public async Task PostAsync_NewOrganization_SetsCreatedAndUpdatedDates() { diff --git a/tests/Exceptionless.Tests/Migrations/UpdateEventUsageMigrationTests.cs b/tests/Exceptionless.Tests/Migrations/UpdateEventUsageMigrationTests.cs index 24a9c1aea9..d2b2ed31ec 100644 --- a/tests/Exceptionless.Tests/Migrations/UpdateEventUsageMigrationTests.cs +++ b/tests/Exceptionless.Tests/Migrations/UpdateEventUsageMigrationTests.cs @@ -1,12 +1,15 @@ using Exceptionless.Core.Billing; using Exceptionless.Core.Extensions; using Exceptionless.Core.Migrations; +using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; using Exceptionless.DateTimeExtensions; using Exceptionless.Tests.Utility; using Foundatio.Lock; using Foundatio.Repositories; +using Foundatio.Repositories.Elasticsearch; using Foundatio.Repositories.Migrations; +using Foundatio.Repositories.Models; using Foundatio.Utility; using Xunit; @@ -85,4 +88,49 @@ public async Task ShouldPopulateUsageStats() Assert.Equal(10, currentMonthsUsage.Total); Assert.Equal(limit, currentMonthsUsage.Limit); } + + [Fact] + public async Task RunAsync_WhenCurrentMonthRepairCrossesLimit_PublishesOneOrganizationChangedMessage() + { + // Arrange + var billingPlans = GetService(); + var organization = _organizationData.GenerateSampleOrganizationWithPlan(GetService(), billingPlans, billingPlans.SmallPlan); + organization.MaxEventsPerMonth = 10; + organization.GetCurrentUsage(TimeProvider).Total = 5; + organization = await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + var currentMonthUsageDate = DateTime.UtcNow.StartOfMonth(); + await _eventRepository.AddAsync(_eventData.GenerateEvents(count: 10, stackId: stack.Id, startDate: currentMonthUsageDate, endDate: DateTime.UtcNow), o => o.ImmediateConsistency()); + + var organizationRepository = Assert.IsAssignableFrom>(GetService()); + int notificationCount = 0; + Func, Task> organizationHandler = (_, _) => + { + Interlocked.Increment(ref notificationCount); + return Task.CompletedTask; + }; + organizationRepository.BeforePublishEntityChanged.AddHandler(organizationHandler); + + try + { + var migration = GetService(); + var context = new MigrationContext(GetService(), _logger, TestCancellationToken); + + // Act + await migration.RunAsync(context); + await migration.RunAsync(context); + + // Assert + Assert.Equal(1, notificationCount); + var updatedOrganization = await _organizationRepository.GetByIdAsync(organization.Id); + Assert.NotNull(updatedOrganization); + Assert.Equal(10, updatedOrganization.GetCurrentUsage(TimeProvider).Total); + } + finally + { + organizationRepository.BeforePublishEntityChanged.RemoveHandler(organizationHandler); + } + } } diff --git a/tests/Exceptionless.Tests/Services/UsageServiceTests.cs b/tests/Exceptionless.Tests/Services/UsageServiceTests.cs index 552f13718f..9c986b4d1e 100644 --- a/tests/Exceptionless.Tests/Services/UsageServiceTests.cs +++ b/tests/Exceptionless.Tests/Services/UsageServiceTests.cs @@ -359,7 +359,7 @@ public async Task SavePendingUsageAsync_HourlyThrottleClears_PublishesOrganizati } [Fact] - public async Task SavePendingUsageAsync_WhenMonthlyOverageClearsAfterPlanIncrease_PublishesOrganizationChangedMessage() + public async Task SavePendingUsageAsync_WhenMonthRollsOver_DoesNotPublishOrganizationChangedMessage() { // Arrange TimeProvider.SetUtcNow(new DateTime(2015, 2, 28, 23, 55, 0, DateTimeKind.Utc)); @@ -390,12 +390,12 @@ public async Task SavePendingUsageAsync_WhenMonthlyOverageClearsAfterPlanIncreas organization = await _organizationRepository.GetByIdAsync(organization.Id); Assert.NotNull(organization); Assert.False(organization.IsOverMonthlyLimit(TimeProvider)); - Assert.Equal(1, notificationCount); + Assert.Equal(0, notificationCount); await _usageService.IncrementTotalAsync(organization.Id, project.Id); TimeProvider.Advance(TimeSpan.FromMinutes(10)); await _usageService.SavePendingUsageAsync(); - Assert.Equal(1, notificationCount); + Assert.Equal(0, notificationCount); } finally { From d9508d20f66864d696b8341502760fa4bd7278d8 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Thu, 6 Aug 2026 11:50:53 -0500 Subject: [PATCH 7/9] Fix legacy rollover notification payload --- .../components/organization/organization-service.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js b/src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js index ec78a0730d..b6eb9b6c6e 100644 --- a/src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js +++ b/src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js @@ -20,7 +20,7 @@ } _usageMonth = usageMonth; - $rootScope.$broadcast("OrganizationChanged"); + $rootScope.$broadcast("OrganizationChanged", {}); }, 60000); function getUsageMonth() { From e7246645de3274f02e5fb4da80cb0a72d0474827 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Fri, 7 Aug 2026 11:57:45 -0500 Subject: [PATCH 8/9] Retry hourly overage notifications --- .../Services/UsageService.cs | 26 ++++++++- .../Services/UsageServiceTests.cs | 57 +++++++++++++++++++ 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/src/Exceptionless.Core/Services/UsageService.cs b/src/Exceptionless.Core/Services/UsageService.cs index 3310268c34..8fedeaef6a 100644 --- a/src/Exceptionless.Core/Services/UsageService.cs +++ b/src/Exceptionless.Core/Services/UsageService.cs @@ -69,6 +69,9 @@ private async Task SavePendingOrganizationUsageAsync(DateTime utcNow) { if (organizationIdsValue.HasValue) { + foreach (string? organizationId in organizationIdsValue.Value) + await PublishPendingHourlyOverageAsync(bucketUtc, organizationId); + // Should we wait to remove this in case there is a failure? We should just remove the organization id once processed. await _cache.RemoveAsync(GetOrganizationSetKey(bucketUtc)); @@ -268,7 +271,8 @@ public async Task HandleOrganizationChangeAsync(Organization modified, Organizat { await _cache.SetAsync(GetThrottledKey(utcNow, modified.Id), true, TimeSpan.FromMinutes(5)); await _cache.SetAsync(GetHourlyThrottleTransitionKey(utcNow, modified.Id), true, TimeSpan.FromHours(8)); - await _messagePublisher.PublishAsync(new PlanOverage { OrganizationId = modified.Id, IsHourly = true }); + await _cache.SetAsync(GetHourlyOveragePendingKey(utcNow, modified.Id), true, TimeSpan.FromHours(8)); + await PublishPendingHourlyOverageAsync(utcNow, modified.Id); } } @@ -461,10 +465,22 @@ public async Task IncrementTotalAsync(string organizationId, string projectId, i // org will be throttled during the current bucket of time await _cache.SetAsync(GetThrottledKey(utcNow, organizationId), true, TimeSpan.FromMinutes(5)); await _cache.SetAsync(GetHourlyThrottleTransitionKey(utcNow, organizationId), true, TimeSpan.FromHours(8)); - await _messagePublisher.PublishAsync(new PlanOverage { OrganizationId = organizationId, IsHourly = true }); + await _cache.SetAsync(GetHourlyOveragePendingKey(utcNow, organizationId), true, TimeSpan.FromHours(8)); + await PublishPendingHourlyOverageAsync(utcNow, organizationId); } } + private async Task PublishPendingHourlyOverageAsync(DateTime utcTime, string organizationId) + { + string pendingKey = GetHourlyOveragePendingKey(utcTime, organizationId); + var pending = await _cache.GetAsync(pendingKey); + if (!pending.HasValue || !pending.Value) + return; + + await _messagePublisher.PublishAsync(new PlanOverage { OrganizationId = organizationId, IsHourly = true }); + await _cache.RemoveAsync(pendingKey); + } + public async Task IncrementBlockedAsync(string organizationId, string? projectId, int eventCount = 1) { if (eventCount <= 0) @@ -635,6 +651,12 @@ private string GetHourlyThrottleTransitionKey(DateTime utcTime, string organizat return $"usage:{bucket}:{organizationId}:throttled-transition"; } + private string GetHourlyOveragePendingKey(DateTime utcTime, string organizationId) + { + int bucket = GetCurrentBucket(utcTime); + return $"usage:{bucket}:{organizationId}:hourly-overage-pending"; + } + private string GetProjectSetKey(DateTime utcTime) { int bucket = GetCurrentBucket(utcTime); diff --git a/tests/Exceptionless.Tests/Services/UsageServiceTests.cs b/tests/Exceptionless.Tests/Services/UsageServiceTests.cs index 9c986b4d1e..b55c01446b 100644 --- a/tests/Exceptionless.Tests/Services/UsageServiceTests.cs +++ b/tests/Exceptionless.Tests/Services/UsageServiceTests.cs @@ -452,6 +452,48 @@ await messageBus.SubscribeAsync(async overage => Assert.True(await throttleState.Task.WaitAsync(TimeSpan.FromSeconds(5), TestCancellationToken)); } + [Fact] + public async Task IncrementTotalAsync_WhenHourlyOveragePublicationFails_RetriesDuringPendingUsageSave() + { + // Arrange + var organization = await _organizationRepository.AddAsync(new Organization { Name = "Test", MaxEventsPerMonth = 750, PlanId = _plans.SmallPlan.Id }, o => o.ImmediateConsistency().Cache()); + var project = await _projectRepository.AddAsync(new Project { Name = "Test", OrganizationId = organization.Id, NextSummaryEndOfDayTicks = TimeProvider.GetUtcNow().UtcDateTime.Ticks }, o => o.ImmediateConsistency().Cache()); + int eventsLeftInBucket = await _usageService.GetEventsLeftAsync(organization.Id); + var cache = GetService(); + var messageBus = GetService(); + var publisher = new FailOnceHourlyPlanOveragePublisher(messageBus); + var usageService = new UsageService( + _organizationRepository, + _projectRepository, + cache, + publisher, + _notificationService, + TimeProvider, + GetService()); + var published = new AsyncCountdownEvent(1); + await messageBus.SubscribeAsync(overage => + { + if (overage.OrganizationId == organization.Id && overage.IsHourly) + published.Signal(); + }, TestCancellationToken); + + int bucket = TimeProvider.GetUtcNow().UtcDateTime.Floor(TimeSpan.FromMinutes(5)).ToEpoch(); + string pendingPublicationKey = $"usage:{bucket}:{organization.Id}:hourly-overage-pending"; + + // Act + await Assert.ThrowsAsync(() => usageService.IncrementTotalAsync(organization.Id, project.Id, eventsLeftInBucket)); + + // Assert + Assert.True((await cache.GetAsync(pendingPublicationKey)).Value); + + TimeProvider.Advance(TimeSpan.FromMinutes(10)); + await usageService.SavePendingUsageAsync(); + await published.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(2, publisher.HourlyAttempts); + Assert.False((await cache.GetAsync(pendingPublicationKey)).HasValue); + } + [Fact] public async Task CanIncrementOverageUsageAsync() { @@ -812,4 +854,19 @@ public async Task RunBenchmarkAsync() sw.Stop(); _logger.LogInformation("Time: {Duration:g}, Avg: ({AverageTickDuration:g}ticks | {AverageDuration}ms)", sw.Elapsed, sw.ElapsedTicks / iterations, sw.ElapsedMilliseconds / iterations); } + + private sealed class FailOnceHourlyPlanOveragePublisher(IMessagePublisher inner) : IMessagePublisher + { + private int _hourlyAttempts; + + public int HourlyAttempts => _hourlyAttempts; + + public Task PublishAsync(Type messageType, object message, MessageOptions? options = null, CancellationToken cancellationToken = default) + { + if (message is PlanOverage { IsHourly: true } && Interlocked.Increment(ref _hourlyAttempts) == 1) + return Task.FromException(new InvalidOperationException("Simulated hourly overage publication failure.")); + + return inner.PublishAsync(messageType, message, options, cancellationToken); + } + } } From 5728e5599f8a27d1a85be1224a743fdafb8087d6 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Fri, 7 Aug 2026 12:15:55 -0500 Subject: [PATCH 9/9] Refresh legacy usage views periodically --- .../app/organization/manage/manage.tpl.html | 2 +- .../ClientApp.angular/app/project/manage/manage.tpl.html | 2 +- .../components/organization/organization-service.js | 7 ++++++- .../components/project/project-service.js | 1 + 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Exceptionless.Web/ClientApp.angular/app/organization/manage/manage.tpl.html b/src/Exceptionless.Web/ClientApp.angular/app/organization/manage/manage.tpl.html index 21660d7c5d..6f73ad4ff4 100644 --- a/src/Exceptionless.Web/ClientApp.angular/app/organization/manage/manage.tpl.html +++ b/src/Exceptionless.Web/ClientApp.angular/app/organization/manage/manage.tpl.html @@ -2,7 +2,7 @@
diff --git a/src/Exceptionless.Web/ClientApp.angular/app/project/manage/manage.tpl.html b/src/Exceptionless.Web/ClientApp.angular/app/project/manage/manage.tpl.html index 38d47a13ae..e6308587de 100644 --- a/src/Exceptionless.Web/ClientApp.angular/app/project/manage/manage.tpl.html +++ b/src/Exceptionless.Web/ClientApp.angular/app/project/manage/manage.tpl.html @@ -2,7 +2,7 @@
diff --git a/src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js b/src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js index b6eb9b6c6e..dab8cf8ec8 100644 --- a/src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js +++ b/src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js @@ -12,6 +12,7 @@ $rootScope.$on("OrganizationChanged", _cache.removeAll); $rootScope.$on("ProjectChanged", _cache.removeAll); $rootScope.$on("PlanOverage", _cache.removeAll); + $rootScope.$on("UsageChanged", _cache.removeAll); $interval(function () { var usageMonth = getUsageMonth(); @@ -23,6 +24,10 @@ $rootScope.$broadcast("OrganizationChanged", {}); }, 60000); + $interval(function () { + $rootScope.$broadcast("UsageChanged"); + }, 300000); + function getUsageMonth() { var now = new Date(); return now.getUTCFullYear() * 12 + now.getUTCMonth(); @@ -51,7 +56,7 @@ plan_id: options.planId, stripe_token: options.stripeToken, last4: options.last4, - coupon_id: options.couponId + coupon_id: options.couponId, }; return Restangular.one("organizations", id).customPOST(body, "change-plan"); } diff --git a/src/Exceptionless.Web/ClientApp.angular/components/project/project-service.js b/src/Exceptionless.Web/ClientApp.angular/components/project/project-service.js index f7391b7b0c..338bd0bbcb 100644 --- a/src/Exceptionless.Web/ClientApp.angular/components/project/project-service.js +++ b/src/Exceptionless.Web/ClientApp.angular/components/project/project-service.js @@ -10,6 +10,7 @@ $rootScope.$on("auth:logout", _cache.removeAll); $rootScope.$on("OrganizationChanged", _cache.removeAll); $rootScope.$on("ProjectChanged", _cache.removeAll); + $rootScope.$on("UsageChanged", _cache.removeAll); $rootScope.$on("StackChanged", function ($event, data) { if (data.added) {