diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/OrganizationMaintenanceWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/OrganizationMaintenanceWorkItemHandler.cs index 73e886505c..8765f9b19a 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/OrganizationMaintenanceWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/OrganizationMaintenanceWorkItemHandler.cs @@ -56,7 +56,7 @@ public override async Task HandleItemAsync(WorkItemContext context) } if (wi.UpgradePlans || wi.RemoveOldUsageStats) - await _organizationRepository.SaveAsync(results.Documents); + await _organizationRepository.SaveAsync(results.Documents, o => o.Notifications(wi.UpgradePlans)); // Sleep so we are not hammering the backend. await Task.Delay(TimeSpan.FromSeconds(2.5), _timeProvider); diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/ProjectMaintenanceWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/ProjectMaintenanceWorkItemHandler.cs index d6c16ca2fb..dfbb91caf0 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/ProjectMaintenanceWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/ProjectMaintenanceWorkItemHandler.cs @@ -58,7 +58,7 @@ public override async Task HandleItemAsync(WorkItemContext context) } if (workItem.UpdateDefaultBotList || workItem.IncrementConfigurationVersion || workItem.RemoveOldUsageStats) - await _projectRepository.SaveAsync(results.Documents); + await _projectRepository.SaveAsync(results.Documents, o => o.Notifications(workItem.UpdateDefaultBotList || workItem.IncrementConfigurationVersion)); // Sleep so we are not hammering the backend. await Task.Delay(TimeSpan.FromSeconds(2.5), _timeProvider); diff --git a/src/Exceptionless.Core/Migrations/UpdateEventUsage.cs b/src/Exceptionless.Core/Migrations/UpdateEventUsage.cs index 3860bd5848..32ddb62143 100644 --- a/src/Exceptionless.Core/Migrations/UpdateEventUsage.cs +++ b/src/Exceptionless.Core/Migrations/UpdateEventUsage.cs @@ -70,6 +70,9 @@ private async Task UpdateOrganizationsUsageAsync(MigrationContext context) using var _ = _logger.BeginScope(new ExceptionlessState().Organization(organization.Id)); try { + int effectiveMonthlyLimit = organization.GetMaxEventsPerMonthWithBonus(_timeProvider); + var currentMonthUtc = _timeProvider.GetUtcNow().UtcDateTime.StartOfMonth(); + int currentMonthTotalBeforeRepair = organization.Usage.FirstOrDefault(u => u.Date == currentMonthUtc)?.Total ?? 0; var result = await _eventRepository.CountAsync(q => q.Organization(organization.Id).AggregationsExpression("date:date~1M")); var dateAggs = result.Aggregations.DateHistogram("date_date"); if (dateAggs?.Buckets is null) @@ -86,7 +89,11 @@ private async Task UpdateOrganizationsUsageAsync(MigrationContext context) } } - await _organizationRepository.SaveAsync(organization); + int currentMonthTotalAfterRepair = organization.Usage.FirstOrDefault(u => u.Date == currentMonthUtc)?.Total ?? 0; + bool monthlyOverageStarted = effectiveMonthlyLimit >= 0 + && currentMonthTotalBeforeRepair < effectiveMonthlyLimit + && currentMonthTotalAfterRepair >= effectiveMonthlyLimit; + await _organizationRepository.SaveAsync(organization, o => o.Notifications(monthlyOverageStarted)); await UpdateProjectsUsageAsync(context, organization); processed++; await context.Lock.RenewAsync(); @@ -142,7 +149,7 @@ private async Task UpdateProjectsUsageAsync(MigrationContext context, Organizati usage.Limit = organization.GetMaxEventsPerMonthWithBonus(_timeProvider); } - await _projectRepository.SaveAsync(project); + await _projectRepository.SaveAsync(project, o => o.Notifications(false)); } catch (Exception ex) { diff --git a/src/Exceptionless.Core/Services/UsageService.cs b/src/Exceptionless.Core/Services/UsageService.cs index 4a12491675..3310268c34 100644 --- a/src/Exceptionless.Core/Services/UsageService.cs +++ b/src/Exceptionless.Core/Services/UsageService.cs @@ -85,11 +85,17 @@ private async Task SavePendingOrganizationUsageAsync(DateTime utcNow) var bucketDiscarded = await _cache.GetAsync(GetBucketDiscardedCacheKey(bucketUtc, organizationId)); var bucketTooBig = await _cache.GetAsync(GetBucketTooBigCacheKey(bucketUtc, organizationId)); var bucketDeleted = await _cache.GetAsync(GetBucketDeletedCacheKey(bucketUtc, organizationId)); + var hourlyThrottleTransition = await _cache.GetAsync(GetHourlyThrottleTransitionKey(bucketUtc, organizationId)); bool hasIngestion = (bucketTotal?.Value ?? 0) > 0 || (bucketBlocked?.Value ?? 0) > 0 || (bucketDiscarded?.Value ?? 0) > 0 || (bucketTooBig?.Value ?? 0) > 0; if (hasIngestion) organization.LastEventDateUtc = _timeProvider.GetUtcNow().UtcDateTime; + int bucketLimit = GetBucketEventLimit(organization.GetMaxEventsPerMonthWithBonus(_timeProvider), bucketUtc); + bool hourlyThrottleCleared = hourlyThrottleTransition is { HasValue: true } transition + ? transition.Value + : bucketLimit >= 0 && bucketTotal is { HasValue: true } total && total.Value >= bucketLimit; + var usage = organization.GetUsage(bucketUtc, _timeProvider); usage.Limit = organization.GetMaxEventsPerMonthWithBonus(_timeProvider); usage.Total += bucketTotal?.Value ?? 0; @@ -113,11 +119,13 @@ await _cache.RemoveAllAsync(new[] { GetBucketDiscardedCacheKey(bucketUtc, organizationId), GetBucketTooBigCacheKey(bucketUtc, organizationId), GetBucketDeletedCacheKey(bucketUtc, organizationId), - GetThrottledKey(bucketUtc, organizationId) + GetThrottledKey(bucketUtc, organizationId), + GetHourlyThrottleTransitionKey(bucketUtc, organizationId) }); await _cache.SetAsync(GetTotalCacheKey(utcNow, organizationId), usage.Total, TimeSpan.FromHours(8)); - await _organizationRepository.SaveAsync(organization); + // Routine usage updates stay silent, but clients need one refresh when hourly throttling clears. + await _organizationRepository.SaveAsync(organization, o => o.Notifications(hourlyThrottleCleared)); } } @@ -204,7 +212,8 @@ await _cache.RemoveAllAsync(new[] { await _cache.SetAsync(GetTotalCacheKey(utcNow, project.OrganizationId, projectId), usage.Total, TimeSpan.FromHours(8)); - await _projectRepository.SaveAsync(project); + // Project configuration changes have their own save path and should remain the source of ProjectChanged messages. + await _projectRepository.SaveAsync(project, o => o.Notifications(false)); } } @@ -257,8 +266,9 @@ public async Task HandleOrganizationChangeAsync(Organization modified, Organizat if (bucketTotal.Value >= bucketLimit) { - await _messagePublisher.PublishAsync(new PlanOverage { OrganizationId = modified.Id, IsHourly = true }); await _cache.SetAsync(GetThrottledKey(utcNow, modified.Id), true, TimeSpan.FromMinutes(5)); + await _cache.SetAsync(GetHourlyThrottleTransitionKey(utcNow, modified.Id), true, TimeSpan.FromHours(8)); + await _messagePublisher.PublishAsync(new PlanOverage { OrganizationId = modified.Id, IsHourly = true }); } } @@ -449,8 +459,9 @@ public async Task IncrementTotalAsync(string organizationId, string projectId, i if (bucketTotal >= bucketLimit && bucketTotal - bucketLimit < eventCount) { // org will be throttled during the current bucket of time - await _messagePublisher.PublishAsync(new PlanOverage { OrganizationId = organizationId, IsHourly = true }); await _cache.SetAsync(GetThrottledKey(utcNow, organizationId), true, TimeSpan.FromMinutes(5)); + await _cache.SetAsync(GetHourlyThrottleTransitionKey(utcNow, organizationId), true, TimeSpan.FromHours(8)); + await _messagePublisher.PublishAsync(new PlanOverage { OrganizationId = organizationId, IsHourly = true }); } } @@ -527,11 +538,15 @@ public async Task IncrementDeletedAsync(string organizationId, string? projectId } private int GetBucketEventLimit(int maxEventsPerMonth) + { + return GetBucketEventLimit(maxEventsPerMonth, _timeProvider.GetUtcNow().UtcDateTime); + } + + private int GetBucketEventLimit(int maxEventsPerMonth, DateTime utcNow) { if (maxEventsPerMonth < 5000) return maxEventsPerMonth; - var utcNow = _timeProvider.GetUtcNow().UtcDateTime; var timeLeftInMonth = utcNow.EndOfMonth() - utcNow; if (timeLeftInMonth < TimeSpan.FromDays(1)) return maxEventsPerMonth; @@ -614,6 +629,12 @@ private string GetThrottledKey(DateTime utcTime, string organizationId) return $"usage:{bucket}:{organizationId}:throttled"; } + private string GetHourlyThrottleTransitionKey(DateTime utcTime, string organizationId) + { + int bucket = GetCurrentBucket(utcTime); + return $"usage:{bucket}:{organizationId}:throttled-transition"; + } + private string GetProjectSetKey(DateTime utcTime) { int bucket = GetCurrentBucket(utcTime); diff --git a/src/Exceptionless.Web/Api/Handlers/OrganizationHandler.cs b/src/Exceptionless.Web/Api/Handlers/OrganizationHandler.cs index c82f4d2901..a64badd407 100644 --- a/src/Exceptionless.Web/Api/Handlers/OrganizationHandler.cs +++ b/src/Exceptionless.Web/Api/Handlers/OrganizationHandler.cs @@ -923,12 +923,13 @@ private async Task AfterResultMapAsync(ICollection m viewOrganization.TrimUsage(timeProvider); var currentUsage = viewOrganization.GetCurrentUsage(timeProvider); - currentUsage.Limit = realTimeUsage.CurrentUsage.Limit; + currentUsage.Limit = viewOrganization.GetMaxEventsPerMonthWithBonus(timeProvider); currentUsage.Total = realTimeUsage.CurrentUsage.Total; currentUsage.Blocked = realTimeUsage.CurrentUsage.Blocked; currentUsage.Discarded = realTimeUsage.CurrentUsage.Discarded; currentUsage.TooBig = realTimeUsage.CurrentUsage.TooBig; currentUsage.Deleted = realTimeUsage.CurrentUsage.Deleted; + viewOrganization.IsOverMonthlyLimit = currentUsage.Limit >= 0 && currentUsage.Total >= currentUsage.Limit; var currentHourUsage = viewOrganization.GetCurrentHourlyUsage(timeProvider); currentHourUsage.Total = realTimeUsage.CurrentHourUsage.Total; diff --git a/src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js b/src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js index e19f1c10bf..b6eb9b6c6e 100644 --- a/src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js +++ b/src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js @@ -3,13 +3,30 @@ angular .module("exceptionless.organization", ["restangular"]) - .factory("organizationService", function ($cacheFactory, $rootScope, objectIDService, Restangular) { + .factory("organizationService", function ($cacheFactory, $interval, $rootScope, objectIDService, Restangular) { var _cache = $cacheFactory("http:organization"); + var _usageMonth = getUsageMonth(); $rootScope.$on("cache:clear", _cache.removeAll); $rootScope.$on("cache:clear-organization", _cache.removeAll); $rootScope.$on("auth:logout", _cache.removeAll); $rootScope.$on("OrganizationChanged", _cache.removeAll); $rootScope.$on("ProjectChanged", _cache.removeAll); + $rootScope.$on("PlanOverage", _cache.removeAll); + + $interval(function () { + var usageMonth = getUsageMonth(); + if (usageMonth === _usageMonth) { + return; + } + + _usageMonth = usageMonth; + $rootScope.$broadcast("OrganizationChanged", {}); + }, 60000); + + function getUsageMonth() { + var now = new Date(); + return now.getUTCFullYear() * 12 + now.getUTCMonth(); + } $rootScope.$on("StackChanged", function ($event, data) { if (data.added) { diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.svelte.ts index b5ad5c220c..3c64d862d4 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.svelte.ts @@ -23,6 +23,20 @@ export async function invalidateOrganizationQueries(queryClient: QueryClient, me } } +export async function invalidateOrganizationUsageQueries(queryClient: QueryClient, organizationId?: string) { + const invalidations = [queryClient.invalidateQueries({ queryKey: queryKeys.list(undefined) })]; + if (organizationId) { + invalidations.push(queryClient.invalidateQueries({ exact: true, queryKey: queryKeys.id(organizationId, undefined) })); + invalidations.push(queryClient.invalidateQueries({ exact: true, queryKey: queryKeys.id(organizationId, 'stats') })); + } + + await Promise.all(invalidations); +} + +export async function invalidatePlanOverageQueries(queryClient: QueryClient, message: WebSocketMessageValue<'PlanOverage'>) { + await invalidateOrganizationUsageQueries(queryClient, message.organization_id); +} + export const queryKeys = { adminSearch: (params: GetAdminSearchOrganizationsParams) => [...queryKeys.list(params.mode), 'admin', { ...params }] as const, changePlan: (id: string | undefined) => [...queryKeys.type, id, 'change-plan'] as const, @@ -108,6 +122,7 @@ export interface GetOrganizationRequest { params?: { mode: 'stats' | undefined; }; + refetchInterval?: false | number; route: { id: string | undefined; }; @@ -360,7 +375,8 @@ export function getOrganizationQuery(request: GetOrganizationRequest) { return response.data!; }, - queryKey: queryKeys.id(request.route.id, request.params?.mode) + queryKey: queryKeys.id(request.route.id, request.params?.mode), + refetchInterval: request.refetchInterval })); } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.test.ts new file mode 100644 index 0000000000..ee43ecbd4f --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.test.ts @@ -0,0 +1,39 @@ +import { QueryClient } from '@tanstack/svelte-query'; +import { describe, expect, it, vi } from 'vitest'; + +import { invalidateOrganizationUsageQueries, invalidatePlanOverageQueries, queryKeys } from './api.svelte'; + +describe('invalidatePlanOverageQueries', () => { + it('invalidates only the affected organization state', async () => { + // Arrange + const queryClient = new QueryClient(); + const invalidateSpy = vi.spyOn(queryClient, 'invalidateQueries').mockImplementation(async () => {}); + + // Act + await invalidatePlanOverageQueries(queryClient, { + is_hourly: false, + organization_id: 'organization-id' + }); + + // Assert + expect(invalidateSpy).toHaveBeenCalledTimes(3); + expect(invalidateSpy).toHaveBeenCalledWith({ exact: true, queryKey: queryKeys.id('organization-id', undefined) }); + expect(invalidateSpy).toHaveBeenCalledWith({ exact: true, queryKey: queryKeys.id('organization-id', 'stats') }); + expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: queryKeys.list(undefined) }); + }); +}); + +describe('invalidateOrganizationUsageQueries', () => { + it('invalidates organization lists when there is no active organization', async () => { + // Arrange + const queryClient = new QueryClient(); + const invalidateSpy = vi.spyOn(queryClient, 'invalidateQueries').mockImplementation(async () => {}); + + // Act + await invalidateOrganizationUsageQueries(queryClient); + + // Assert + expect(invalidateSpy).toHaveBeenCalledOnce(); + expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: queryKeys.list(undefined) }); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte index 8b3f7da879..b90bcfe2a8 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte @@ -1,3 +1,25 @@ + + diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte.test.ts new file mode 100644 index 0000000000..ddfc3a7396 --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/components/organization-notifications.svelte.test.ts @@ -0,0 +1,96 @@ +import { ChangeType } from '$features/websockets/models'; +import { render } from '@testing-library/svelte'; +import { tick } from 'svelte'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import OrganizationNotifications, { recordConfiguredProjectId } from './organization-notifications.svelte'; + +const organizationRefetch = vi.hoisted(() => vi.fn()); +const projectsRefetch = vi.hoisted(() => vi.fn()); +const projects = vi.hoisted(() => [{ id: 'project-id', is_configured: false, organization_id: 'organization-id' }]); + +vi.mock('$features/organizations/api.svelte', () => ({ + getOrganizationQuery: () => ({ data: undefined, refetch: organizationRefetch }), + getOrganizationsQuery: () => ({ data: { data: [] } }) +})); + +vi.mock('$features/organizations/context.svelte', () => ({ + organization: { current: 'organization-id' } +})); + +vi.mock('$features/projects/api.svelte', () => ({ + getOrganizationProjectsQuery: () => ({ data: { data: projects }, isSuccess: true, refetch: projectsRefetch }) +})); + +vi.mock('$features/users/api.svelte', () => ({ + getMeQuery: () => ({ data: { organization_ids: ['organization-id'], roles: [] } }) +})); + +describe('OrganizationNotifications', () => { + beforeEach(() => { + organizationRefetch.mockReset(); + projectsRefetch.mockReset(); + projects[0]!.is_configured = false; + }); + + it('records configuration events before project data loads', () => { + const configuredProjectIds = new Set(); + + const recorded = recordConfiguredProjectId( + { + change_type: ChangeType.Added, + data: {}, + organization_id: 'organization-id', + project_id: 'project-id', + type: 'PersistentEvent' + }, + 'organization-id', + configuredProjectIds + ); + + expect(recorded).toBe(true); + expect(configuredProjectIds).toContain('project-id'); + }); + + it('does not refetch organization or project state for persistent event changes', async () => { + render(OrganizationNotifications, { + isChatEnabled: false, + openChat: vi.fn() + }); + + const message = { + change_type: ChangeType.Added, + organization_id: 'organization-id', + project_id: 'project-id' + }; + + document.dispatchEvent(new CustomEvent('PersistentEventChanged', { detail: message })); + document.dispatchEvent(new CustomEvent('PersistentEventChanged', { detail: message })); + await tick(); + + expect(organizationRefetch).not.toHaveBeenCalled(); + expect(projectsRefetch).not.toHaveBeenCalled(); + }); + + it('does not refresh configuration state for an already configured project', async () => { + projects[0]!.is_configured = true; + render(OrganizationNotifications, { + isChatEnabled: false, + openChat: vi.fn() + }); + + document.dispatchEvent( + new CustomEvent('PersistentEventChanged', { + detail: { + change_type: ChangeType.Added, + organization_id: 'organization-id', + project_id: 'project-id' + } + }) + ); + await tick(); + + expect(organizationRefetch).not.toHaveBeenCalled(); + expect(projectsRefetch).not.toHaveBeenCalled(); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/utils.test.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/utils.test.ts new file mode 100644 index 0000000000..92ccda8eec --- /dev/null +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/utils.test.ts @@ -0,0 +1,10 @@ +import { describe, expect, it } from 'vitest'; + +import { getUtcMonthKey } from './utils'; + +describe('getUtcMonthKey', () => { + it('changes only when the UTC month changes', () => { + expect(getUtcMonthKey(new Date('2026-08-01T00:00:00.000Z'))).toBe(getUtcMonthKey(new Date('2026-08-31T23:59:59.999Z'))); + expect(getUtcMonthKey(new Date('2026-09-01T00:00:00.000Z'))).not.toBe(getUtcMonthKey(new Date('2026-08-31T23:59:59.999Z'))); + }); +}); diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/utils.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/utils.ts index 11cdb6569a..223b06d788 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/utils.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/organizations/utils.ts @@ -2,6 +2,9 @@ import { isSameUtcMonth } from '$features/shared/dates'; import type { ViewOrganization } from './models'; +export const ORGANIZATION_USAGE_REFETCH_INTERVAL_MS = 5 * 60 * 1000; +export const ORGANIZATION_USAGE_ROLLOVER_CHECK_INTERVAL_MS = 60 * 1000; + export function getNextBillingDateUtc(organization?: ViewOrganization): Date { if (organization?.subscribe_date) { console.log('Organization subscribe date for next billing date:', organization.subscribe_date); @@ -30,3 +33,7 @@ export function getRemainingEventLimit(organization?: ViewOrganization): number return organization.max_events_per_month + bonusEvents; } + +export function getUtcMonthKey(date = new Date()): number { + return date.getUTCFullYear() * 12 + date.getUTCMonth(); +} diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/projects/api.svelte.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/projects/api.svelte.ts index 65560321ba..f4342e2a14 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/projects/api.svelte.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/projects/api.svelte.ts @@ -117,6 +117,7 @@ export interface GetProjectIntegrationNotificationSettingsRequest { } export interface GetProjectRequest { + refetchInterval?: false | number; route: { id: string | undefined; }; @@ -396,7 +397,8 @@ export function getProjectQuery(request: GetProjectRequest) { return response.data!; }, - queryKey: queryKeys.id(request.route.id) + queryKey: queryKeys.id(request.route.id), + refetchInterval: request.refetchInterval })); } diff --git a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/models.ts b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/models.ts index 6883e6e193..461805e466 100644 --- a/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/models.ts +++ b/src/Exceptionless.Web/ClientApp/src/lib/features/websockets/models.ts @@ -67,6 +67,10 @@ export function isEntityChangedType(message: { message: unknown; type: WebSocket return message.type !== 'PlanChanged' && message.type !== 'UserMembershipChanged' && message.type.endsWith('Changed'); } +export function isPlanOverageType(message: { message: unknown; type: WebSocketMessageType }): message is WebSocketMessage<'PlanOverage'> { + return message.type === 'PlanOverage'; +} + export function isWebSocketMessageType(type: string): type is WebSocketMessageType { return ( (['PlanChanged', 'PlanOverage', 'UserMembershipChanged', 'ReleaseNotification', 'SystemNotification'] as const).includes( diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte index e8a908b69d..ef2d6d4f18 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/+layout.svelte @@ -15,10 +15,17 @@ import { buildIntercomBootOptions, IntercomShell } from '$features/intercom'; import { shouldLoadIntercomOrganization } from '$features/intercom/config'; import Notifications from '$features/notifications/components/notifications.svelte'; - import { getOrganizationQuery, getOrganizationsQuery, invalidateOrganizationQueries } from '$features/organizations/api.svelte'; + import { + getOrganizationQuery, + getOrganizationsQuery, + invalidateOrganizationQueries, + invalidateOrganizationUsageQueries, + invalidatePlanOverageQueries + } from '$features/organizations/api.svelte'; import OrganizationNotifications from '$features/organizations/components/organization-notifications.svelte'; import { organization, showOrganizationNotifications } from '$features/organizations/context.svelte'; import { premiumPage } from '$features/organizations/premium-page.svelte'; + import { getUtcMonthKey, ORGANIZATION_USAGE_ROLLOVER_CHECK_INTERVAL_MS } from '$features/organizations/utils'; import { invalidateProjectQueries } from '$features/projects/api.svelte'; import { getSavedViewsQuery, invalidateSavedViewQueries, isSavedViewDeleted } from '$features/saved-views/api.svelte'; import { savedViewHref } from '$features/saved-views/slugs'; @@ -28,11 +35,12 @@ import { getMeQuery, invalidateUserQueries } from '$features/users/api.svelte'; import { getGravatarFromCurrentUser } from '$features/users/gravatar.svelte'; import { invalidateWebhookQueries } from '$features/webhooks/api.svelte'; - import { isEntityChangedType, type WebSocketMessageType } from '$features/websockets/models'; + import { isEntityChangedType, isPlanOverageType, type WebSocketMessageType } from '$features/websockets/models'; import { WebSocketClient } from '$features/websockets/web-socket-client.svelte'; import { Telemetry } from '$lib/telemetry'; import { useMiddleware } from '@foundatiofx/fetchclient'; import { useQueryClient } from '@tanstack/svelte-query'; + import { useInterval } from 'runed'; import { tick } from 'svelte'; import { fade } from 'svelte/transition'; @@ -115,6 +123,20 @@ }); const queryClient = useQueryClient(); + let organizationUsageMonth = getUtcMonthKey(); + useInterval(() => ORGANIZATION_USAGE_ROLLOVER_CHECK_INTERVAL_MS, { + callback: () => { + const currentMonth = getUtcMonthKey(); + if (currentMonth === organizationUsageMonth) { + return; + } + + organizationUsageMonth = currentMonth; + void invalidateOrganizationUsageQueries(queryClient, organization.current); + }, + immediate: false + }); + async function onMessage(message: MessageEvent) { const data: { message: unknown; type: WebSocketMessageType } = message.data ? JSON.parse(message.data) : null; @@ -129,7 +151,9 @@ }) ); - if (isEntityChangedType(data)) { + if (isPlanOverageType(data)) { + await invalidatePlanOverageQueries(queryClient, data.message); + } else if (isEntityChangedType(data)) { switch (data.type) { case 'OrganizationChanged': await invalidateOrganizationQueries(queryClient, data.message); diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/organization/[organizationId]/usage/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/organization/[organizationId]/usage/+page.svelte index 0c42001cd2..c8ac8059bb 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/organization/[organizationId]/usage/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/organization/[organizationId]/usage/+page.svelte @@ -10,13 +10,14 @@ import { env } from '$env/dynamic/public'; import { ChangePlanDialog } from '$features/billing'; import { getOrganizationQuery } from '$features/organizations/api.svelte'; - import { getNextBillingDateUtc, getRemainingEventLimit } from '$features/organizations/utils'; + import { getNextBillingDateUtc, getRemainingEventLimit, ORGANIZATION_USAGE_REFETCH_INTERVAL_MS } from '$features/organizations/utils'; import { formatDateLabel, formatLongDate } from '$shared/dates'; import { scaleUtc } from 'd3-scale'; import { curveNatural } from 'd3-shape'; import { AreaChart } from 'layerchart'; const organizationQuery = getOrganizationQuery({ + refetchInterval: ORGANIZATION_USAGE_REFETCH_INTERVAL_MS, route: { get id() { return page.params.organizationId || ''; diff --git a/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/usage/+page.svelte b/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/usage/+page.svelte index 27f5590cde..e63eef973b 100644 --- a/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/usage/+page.svelte +++ b/src/Exceptionless.Web/ClientApp/src/routes/(app)/project/[projectId]/usage/+page.svelte @@ -11,7 +11,7 @@ import { ChangePlanDialog } from '$features/billing'; import { getOrganizationQuery } from '$features/organizations/api.svelte'; import { organization } from '$features/organizations/context.svelte'; - import { getNextBillingDateUtc, getRemainingEventLimit } from '$features/organizations/utils'; + import { getNextBillingDateUtc, getRemainingEventLimit, ORGANIZATION_USAGE_REFETCH_INTERVAL_MS } from '$features/organizations/utils'; import { getProjectQuery } from '$features/projects/api.svelte'; import { formatDateLabel, formatLongDate } from '$shared/dates'; import { scaleUtc } from 'd3-scale'; @@ -22,6 +22,7 @@ params: { mode: 'stats' }, + refetchInterval: ORGANIZATION_USAGE_REFETCH_INTERVAL_MS, route: { get id() { return organization.current; @@ -35,6 +36,7 @@ const nextBillingDate = $derived(getNextBillingDateUtc(organizationQuery.data)); const projectQuery = getProjectQuery({ + refetchInterval: ORGANIZATION_USAGE_REFETCH_INTERVAL_MS, route: { get id() { return page.params.projectId || ''; diff --git a/tests/Exceptionless.Tests/Api/Endpoints/OrganizationEndpointTests.cs b/tests/Exceptionless.Tests/Api/Endpoints/OrganizationEndpointTests.cs index 0a11eedbc2..e7227cd0ca 100644 --- a/tests/Exceptionless.Tests/Api/Endpoints/OrganizationEndpointTests.cs +++ b/tests/Exceptionless.Tests/Api/Endpoints/OrganizationEndpointTests.cs @@ -5,6 +5,7 @@ using Exceptionless.Core.Models; using Exceptionless.Core.Models.Billing; using Exceptionless.Core.Repositories; +using Exceptionless.Core.Services; using Exceptionless.Core.Utility; using Exceptionless.Tests.Extensions; using Exceptionless.Tests.Utility; @@ -575,6 +576,65 @@ public async Task GetAsync_ViewOrganization_IncludesIsOverMonthlyLimit() Assert.False(viewOrg.IsOverMonthlyLimit); } + [Fact] + public async Task GetAsync_WithPendingUsageAtMonthlyLimit_ReturnsRealTimeOverageState() + { + // Arrange + var organization = await _organizationRepository.GetByIdAsync(SampleDataService.TEST_ORG_ID); + Assert.NotNull(organization); + + organization.MaxEventsPerMonth = 1; + organization.Usage.Clear(); + organization.UsageHours.Clear(); + await _organizationRepository.SaveAsync(organization, o => o.ImmediateConsistency().Cache().Originals()); + + var usageService = GetService(); + await usageService.IncrementTotalAsync(organization.Id, SampleDataService.TEST_PROJECT_ID); + + // Act + var viewOrganization = await SendRequestAsAsync(r => r + .AsTestOrganizationUser() + .AppendPaths("organizations", organization.Id) + .StatusCodeShouldBeOk() + ); + + // Assert + Assert.NotNull(viewOrganization); + Assert.Equal(1, viewOrganization.GetCurrentUsage(TimeProvider).Total); + Assert.True(viewOrganization.IsOverMonthlyLimit); + } + + [Fact] + public async Task GetAsync_WithExpiredBonusAndStaleUsageLimit_ReturnsLiveOverageState() + { + // Arrange + var organization = await _organizationRepository.GetByIdAsync(SampleDataService.TEST_ORG_ID); + Assert.NotNull(organization); + + organization.MaxEventsPerMonth = 1; + organization.BonusEventsPerMonth = 10; + organization.BonusExpiration = TimeProvider.GetUtcNow().UtcDateTime.Subtract(TimeSpan.FromMinutes(1)); + organization.Usage.Clear(); + organization.UsageHours.Clear(); + var currentUsage = organization.GetCurrentUsage(TimeProvider); + currentUsage.Limit = organization.MaxEventsPerMonth + organization.BonusEventsPerMonth; + currentUsage.Total = 2; + await _organizationRepository.SaveAsync(organization, o => o.ImmediateConsistency().Cache().Originals()); + + // Act + var viewOrganization = await SendRequestAsAsync(r => r + .AsTestOrganizationUser() + .AppendPaths("organizations", organization.Id) + .StatusCodeShouldBeOk() + ); + + // Assert + Assert.NotNull(viewOrganization); + Assert.Equal(organization.MaxEventsPerMonth, viewOrganization.GetCurrentUsage(TimeProvider).Limit); + Assert.Equal(2, viewOrganization.GetCurrentUsage(TimeProvider).Total); + Assert.True(viewOrganization.IsOverMonthlyLimit); + } + [Fact] public async Task PostAsync_NewOrganization_SetsCreatedAndUpdatedDates() { diff --git a/tests/Exceptionless.Tests/Migrations/UpdateEventUsageMigrationTests.cs b/tests/Exceptionless.Tests/Migrations/UpdateEventUsageMigrationTests.cs index 24a9c1aea9..d2b2ed31ec 100644 --- a/tests/Exceptionless.Tests/Migrations/UpdateEventUsageMigrationTests.cs +++ b/tests/Exceptionless.Tests/Migrations/UpdateEventUsageMigrationTests.cs @@ -1,12 +1,15 @@ using Exceptionless.Core.Billing; using Exceptionless.Core.Extensions; using Exceptionless.Core.Migrations; +using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; using Exceptionless.DateTimeExtensions; using Exceptionless.Tests.Utility; using Foundatio.Lock; using Foundatio.Repositories; +using Foundatio.Repositories.Elasticsearch; using Foundatio.Repositories.Migrations; +using Foundatio.Repositories.Models; using Foundatio.Utility; using Xunit; @@ -85,4 +88,49 @@ public async Task ShouldPopulateUsageStats() Assert.Equal(10, currentMonthsUsage.Total); Assert.Equal(limit, currentMonthsUsage.Limit); } + + [Fact] + public async Task RunAsync_WhenCurrentMonthRepairCrossesLimit_PublishesOneOrganizationChangedMessage() + { + // Arrange + var billingPlans = GetService(); + var organization = _organizationData.GenerateSampleOrganizationWithPlan(GetService(), billingPlans, billingPlans.SmallPlan); + organization.MaxEventsPerMonth = 10; + organization.GetCurrentUsage(TimeProvider).Total = 5; + organization = await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency()); + + var project = await _projectRepository.AddAsync(_projectData.GenerateSampleProject(), o => o.ImmediateConsistency()); + var stack = await _stackRepository.AddAsync(_stackData.GenerateSampleStack(), o => o.ImmediateConsistency()); + var currentMonthUsageDate = DateTime.UtcNow.StartOfMonth(); + await _eventRepository.AddAsync(_eventData.GenerateEvents(count: 10, stackId: stack.Id, startDate: currentMonthUsageDate, endDate: DateTime.UtcNow), o => o.ImmediateConsistency()); + + var organizationRepository = Assert.IsAssignableFrom>(GetService()); + int notificationCount = 0; + Func, Task> organizationHandler = (_, _) => + { + Interlocked.Increment(ref notificationCount); + return Task.CompletedTask; + }; + organizationRepository.BeforePublishEntityChanged.AddHandler(organizationHandler); + + try + { + var migration = GetService(); + var context = new MigrationContext(GetService(), _logger, TestCancellationToken); + + // Act + await migration.RunAsync(context); + await migration.RunAsync(context); + + // Assert + Assert.Equal(1, notificationCount); + var updatedOrganization = await _organizationRepository.GetByIdAsync(organization.Id); + Assert.NotNull(updatedOrganization); + Assert.Equal(10, updatedOrganization.GetCurrentUsage(TimeProvider).Total); + } + finally + { + organizationRepository.BeforePublishEntityChanged.RemoveHandler(organizationHandler); + } + } } diff --git a/tests/Exceptionless.Tests/Services/UsageServiceTests.cs b/tests/Exceptionless.Tests/Services/UsageServiceTests.cs index 8a54c6cbd7..9c986b4d1e 100644 --- a/tests/Exceptionless.Tests/Services/UsageServiceTests.cs +++ b/tests/Exceptionless.Tests/Services/UsageServiceTests.cs @@ -5,10 +5,14 @@ using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; using Exceptionless.Core.Services; +using Exceptionless.DateTimeExtensions; using Exceptionless.Tests.Extensions; using Foundatio.AsyncEx; +using Foundatio.Caching; using Foundatio.Messaging; using Foundatio.Repositories; +using Foundatio.Repositories.Elasticsearch; +using Foundatio.Repositories.Models; using Xunit; using LogLevel = Microsoft.Extensions.Logging.LogLevel; @@ -273,6 +277,132 @@ await messageBus.SubscribeAsync(po => Assert.Equal(0, usage.Deleted); } + [Fact] + public async Task SavePendingUsageAsync_UsageOnlyChanges_DoesNotPublishEntityChangedMessages() + { + // Arrange + var organization = await _organizationRepository.AddAsync(new Organization { Name = "Test", MaxEventsPerMonth = 1_000_000, PlanId = _plans.SmallPlan.Id }, o => o.ImmediateConsistency().Cache()); + var project = await _projectRepository.AddAsync(new Project { Name = "Test", OrganizationId = organization.Id, NextSummaryEndOfDayTicks = TimeProvider.GetUtcNow().UtcDateTime.Ticks }, o => o.ImmediateConsistency().Cache()); + var organizationRepository = Assert.IsAssignableFrom>(_organizationRepository); + var projectRepository = Assert.IsAssignableFrom>(_projectRepository); + int notificationCount = 0; + Func, Task> organizationHandler = (_, _) => + { + Interlocked.Increment(ref notificationCount); + return Task.CompletedTask; + }; + Func, Task> projectHandler = (_, _) => + { + Interlocked.Increment(ref notificationCount); + return Task.CompletedTask; + }; + organizationRepository.BeforePublishEntityChanged.AddHandler(organizationHandler); + projectRepository.BeforePublishEntityChanged.AddHandler(projectHandler); + + try + { + // Act + await _usageService.IncrementTotalAsync(organization.Id, project.Id); + TimeProvider.Advance(TimeSpan.FromMinutes(10)); + await _usageService.SavePendingUsageAsync(); + + // Assert + Assert.Equal(0, notificationCount); + } + finally + { + organizationRepository.BeforePublishEntityChanged.RemoveHandler(organizationHandler); + projectRepository.BeforePublishEntityChanged.RemoveHandler(projectHandler); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SavePendingUsageAsync_HourlyThrottleClears_PublishesOrganizationChangedMessage(bool removeTransitionMarker) + { + // Arrange + var organization = await _organizationRepository.AddAsync(new Organization { Name = "Test", MaxEventsPerMonth = 750, PlanId = _plans.SmallPlan.Id }, o => o.ImmediateConsistency().Cache()); + var project = await _projectRepository.AddAsync(new Project { Name = "Test", OrganizationId = organization.Id, NextSummaryEndOfDayTicks = TimeProvider.GetUtcNow().UtcDateTime.Ticks }, o => o.ImmediateConsistency().Cache()); + int eventsLeftInBucket = await _usageService.GetEventsLeftAsync(organization.Id); + var organizationRepository = Assert.IsAssignableFrom>(_organizationRepository); + int notificationCount = 0; + Func, Task> organizationHandler = (_, _) => + { + Interlocked.Increment(ref notificationCount); + return Task.CompletedTask; + }; + organizationRepository.BeforePublishEntityChanged.AddHandler(organizationHandler); + + try + { + await _usageService.IncrementTotalAsync(organization.Id, project.Id, eventsLeftInBucket); + Assert.True((await _usageService.GetUsageAsync(organization.Id)).IsThrottled); + if (removeTransitionMarker) + { + int bucket = TimeProvider.GetUtcNow().UtcDateTime.Floor(TimeSpan.FromMinutes(5)).ToEpoch(); + await GetService().RemoveAsync($"usage:{bucket}:{organization.Id}:throttled-transition"); + } + + // Act + TimeProvider.Advance(TimeSpan.FromMinutes(10)); + await _usageService.SavePendingUsageAsync(); + + // Assert + Assert.False((await _usageService.GetUsageAsync(organization.Id)).IsThrottled); + Assert.Equal(1, notificationCount); + } + finally + { + organizationRepository.BeforePublishEntityChanged.RemoveHandler(organizationHandler); + } + } + + [Fact] + public async Task SavePendingUsageAsync_WhenMonthRollsOver_DoesNotPublishOrganizationChangedMessage() + { + // Arrange + TimeProvider.SetUtcNow(new DateTime(2015, 2, 28, 23, 55, 0, DateTimeKind.Utc)); + var organization = new Organization { Name = "Test", MaxEventsPerMonth = 750, PlanId = _plans.SmallPlan.Id }; + organization.GetCurrentUsage(TimeProvider).Total = organization.MaxEventsPerMonth; + organization.MaxEventsPerMonth = 1_500; + organization = await _organizationRepository.AddAsync(organization, o => o.ImmediateConsistency().Cache()); + var project = await _projectRepository.AddAsync(new Project { Name = "Test", OrganizationId = organization.Id, NextSummaryEndOfDayTicks = TimeProvider.GetUtcNow().UtcDateTime.Ticks }, o => o.ImmediateConsistency().Cache()); + var organizationRepository = Assert.IsAssignableFrom>(_organizationRepository); + int notificationCount = 0; + Func, Task> organizationHandler = (_, _) => + { + Interlocked.Increment(ref notificationCount); + return Task.CompletedTask; + }; + organizationRepository.BeforePublishEntityChanged.AddHandler(organizationHandler); + + try + { + TimeProvider.SetUtcNow(new DateTime(2015, 3, 1, 0, 0, 0, DateTimeKind.Utc)); + await _usageService.IncrementTotalAsync(organization.Id, project.Id); + + // Act + TimeProvider.Advance(TimeSpan.FromMinutes(10)); + await _usageService.SavePendingUsageAsync(); + + // Assert + organization = await _organizationRepository.GetByIdAsync(organization.Id); + Assert.NotNull(organization); + Assert.False(organization.IsOverMonthlyLimit(TimeProvider)); + Assert.Equal(0, notificationCount); + + await _usageService.IncrementTotalAsync(organization.Id, project.Id); + TimeProvider.Advance(TimeSpan.FromMinutes(10)); + await _usageService.SavePendingUsageAsync(); + Assert.Equal(0, notificationCount); + } + finally + { + organizationRepository.BeforePublishEntityChanged.RemoveHandler(organizationHandler); + } + } + [Fact] public async Task CanGetEventsLeft() { @@ -295,6 +425,33 @@ public async Task CanGetEventsLeft() Assert.Equal(0, eventsLeft); } + [Fact] + public async Task IncrementTotalAsync_WhenHourlyLimitCrosses_PublishesAfterThrottleStateIsSet() + { + // Arrange + var organization = await _organizationRepository.AddAsync(new Organization { Name = "Test", MaxEventsPerMonth = 750, PlanId = _plans.SmallPlan.Id }, o => o.ImmediateConsistency().Cache()); + var project = await _projectRepository.AddAsync(new Project { Name = "Test", OrganizationId = organization.Id, NextSummaryEndOfDayTicks = TimeProvider.GetUtcNow().UtcDateTime.Ticks }, o => o.ImmediateConsistency().Cache()); + int eventsLeftInBucket = await _usageService.GetEventsLeftAsync(organization.Id); + var throttleState = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var messageBus = GetService(); + await messageBus.SubscribeAsync(async overage => + { + if (overage.OrganizationId != organization.Id || !overage.IsHourly) + { + return; + } + + var usage = await _usageService.GetUsageAsync(organization.Id); + throttleState.TrySetResult(usage.IsThrottled); + }, TestCancellationToken); + + // Act + await _usageService.IncrementTotalAsync(organization.Id, project.Id, eventsLeftInBucket); + + // Assert + Assert.True(await throttleState.Task.WaitAsync(TimeSpan.FromSeconds(5), TestCancellationToken)); + } + [Fact] public async Task CanIncrementOverageUsageAsync() {