Reduce noisy change events

src/Exceptionless.Core/Jobs/WorkItemHandlers/OrganizationMaintenanceWorkItemHandler.cs

@@ -56,7 +56,7 @@ public override async Task HandleItemAsync(WorkItemContext context)
             }
 
             if (wi.UpgradePlans || wi.RemoveOldUsageStats)
-                await _organizationRepository.SaveAsync(results.Documents);
+                await _organizationRepository.SaveAsync(results.Documents, o => o.Notifications(wi.UpgradePlans));
 
             // Sleep so we are not hammering the backend.
             await Task.Delay(TimeSpan.FromSeconds(2.5), _timeProvider);

src/Exceptionless.Core/Jobs/WorkItemHandlers/ProjectMaintenanceWorkItemHandler.cs

@@ -58,7 +58,7 @@ public override async Task HandleItemAsync(WorkItemContext context)
             }
 
             if (workItem.UpdateDefaultBotList || workItem.IncrementConfigurationVersion || workItem.RemoveOldUsageStats)
-                await _projectRepository.SaveAsync(results.Documents);
+                await _projectRepository.SaveAsync(results.Documents, o => o.Notifications(workItem.UpdateDefaultBotList || workItem.IncrementConfigurationVersion));
 
             // Sleep so we are not hammering the backend.
             await Task.Delay(TimeSpan.FromSeconds(2.5), _timeProvider);

src/Exceptionless.Core/Migrations/UpdateEventUsage.cs

@@ -70,6 +70,9 @@ private async Task UpdateOrganizationsUsageAsync(MigrationContext context)
                 using var _ = _logger.BeginScope(new ExceptionlessState().Organization(organization.Id));
                 try
                 {
+                    int effectiveMonthlyLimit = organization.GetMaxEventsPerMonthWithBonus(_timeProvider);
+                    var currentMonthUtc = _timeProvider.GetUtcNow().UtcDateTime.StartOfMonth();
+                    int currentMonthTotalBeforeRepair = organization.Usage.FirstOrDefault(u => u.Date == currentMonthUtc)?.Total ?? 0;
                     var result = await _eventRepository.CountAsync(q => q.Organization(organization.Id).AggregationsExpression("date:date~1M"));
                     var dateAggs = result.Aggregations.DateHistogram("date_date");
                     if (dateAggs?.Buckets is null)
@@ -86,7 +89,11 @@ private async Task UpdateOrganizationsUsageAsync(MigrationContext context)
                         }
                     }
 
-                    await _organizationRepository.SaveAsync(organization);
+                    int currentMonthTotalAfterRepair = organization.Usage.FirstOrDefault(u => u.Date == currentMonthUtc)?.Total ?? 0;
+                    bool monthlyOverageStarted = effectiveMonthlyLimit >= 0
+                        && currentMonthTotalBeforeRepair < effectiveMonthlyLimit
+                        && currentMonthTotalAfterRepair >= effectiveMonthlyLimit;
+                    await _organizationRepository.SaveAsync(organization, o => o.Notifications(monthlyOverageStarted));
                     await UpdateProjectsUsageAsync(context, organization);
                     processed++;
                     await context.Lock.RenewAsync();
@@ -142,7 +149,7 @@ private async Task UpdateProjectsUsageAsync(MigrationContext context, Organizati
                             usage.Limit = organization.GetMaxEventsPerMonthWithBonus(_timeProvider);
                     }
 
-                    await _projectRepository.SaveAsync(project);
+                    await _projectRepository.SaveAsync(project, o => o.Notifications(false));
                 }
                 catch (Exception ex)
                 {

src/Exceptionless.Core/Services/UsageService.cs

@@ -69,6 +69,9 @@ private async Task SavePendingOrganizationUsageAsync(DateTime utcNow)
         {
             if (organizationIdsValue.HasValue)
             {
+                foreach (string? organizationId in organizationIdsValue.Value)
+                    await PublishPendingHourlyOverageAsync(bucketUtc, organizationId);
+
                 // Should we wait to remove this in case there is a failure? We should just remove the organization id once processed.
                 await _cache.RemoveAsync(GetOrganizationSetKey(bucketUtc));
 
@@ -85,11 +88,17 @@ private async Task SavePendingOrganizationUsageAsync(DateTime utcNow)
                     var bucketDiscarded = await _cache.GetAsync<int>(GetBucketDiscardedCacheKey(bucketUtc, organizationId));
                     var bucketTooBig = await _cache.GetAsync<int>(GetBucketTooBigCacheKey(bucketUtc, organizationId));
                     var bucketDeleted = await _cache.GetAsync<int>(GetBucketDeletedCacheKey(bucketUtc, organizationId));
+                    var hourlyThrottleTransition = await _cache.GetAsync<bool>(GetHourlyThrottleTransitionKey(bucketUtc, organizationId));
 
                     bool hasIngestion = (bucketTotal?.Value ?? 0) > 0 || (bucketBlocked?.Value ?? 0) > 0 || (bucketDiscarded?.Value ?? 0) > 0 || (bucketTooBig?.Value ?? 0) > 0;
                     if (hasIngestion)
                         organization.LastEventDateUtc = _timeProvider.GetUtcNow().UtcDateTime;
 
+                    int bucketLimit = GetBucketEventLimit(organization.GetMaxEventsPerMonthWithBonus(_timeProvider), bucketUtc);
+                    bool hourlyThrottleCleared = hourlyThrottleTransition is { HasValue: true } transition
+                        ? transition.Value
+                        : bucketLimit >= 0 && bucketTotal is { HasValue: true } total && total.Value >= bucketLimit;
+
                     var usage = organization.GetUsage(bucketUtc, _timeProvider);
                     usage.Limit = organization.GetMaxEventsPerMonthWithBonus(_timeProvider);
                     usage.Total += bucketTotal?.Value ?? 0;
@@ -113,11 +122,13 @@ await _cache.RemoveAllAsync(new[] {
                         GetBucketDiscardedCacheKey(bucketUtc, organizationId),
                         GetBucketTooBigCacheKey(bucketUtc, organizationId),
                         GetBucketDeletedCacheKey(bucketUtc, organizationId),
-                        GetThrottledKey(bucketUtc, organizationId)
+                        GetThrottledKey(bucketUtc, organizationId),
+                        GetHourlyThrottleTransitionKey(bucketUtc, organizationId)
                     });
 
                     await _cache.SetAsync(GetTotalCacheKey(utcNow, organizationId), usage.Total, TimeSpan.FromHours(8));
-                    await _organizationRepository.SaveAsync(organization);
+                    // Routine usage updates stay silent, but clients need one refresh when hourly throttling clears.
+                    await _organizationRepository.SaveAsync(organization, o => o.Notifications(hourlyThrottleCleared));
                 }
             }
 
@@ -204,7 +215,8 @@ await _cache.RemoveAllAsync(new[] {
 
                     await _cache.SetAsync(GetTotalCacheKey(utcNow, project.OrganizationId, projectId), usage.Total, TimeSpan.FromHours(8));
 
-                    await _projectRepository.SaveAsync(project);
+                    // Project configuration changes have their own save path and should remain the source of ProjectChanged messages.
+                    await _projectRepository.SaveAsync(project, o => o.Notifications(false));
                 }
             }
 
@@ -257,8 +269,10 @@ public async Task HandleOrganizationChangeAsync(Organization modified, Organizat
 
         if (bucketTotal.Value >= bucketLimit)
         {
-            await _messagePublisher.PublishAsync(new PlanOverage { OrganizationId = modified.Id, IsHourly = true });
             await _cache.SetAsync(GetThrottledKey(utcNow, modified.Id), true, TimeSpan.FromMinutes(5));
+            await _cache.SetAsync(GetHourlyThrottleTransitionKey(utcNow, modified.Id), true, TimeSpan.FromHours(8));
+            await _cache.SetAsync(GetHourlyOveragePendingKey(utcNow, modified.Id), true, TimeSpan.FromHours(8));
+            await PublishPendingHourlyOverageAsync(utcNow, modified.Id);
         }
     }
 
@@ -449,11 +463,24 @@ public async Task IncrementTotalAsync(string organizationId, string projectId, i
         if (bucketTotal >= bucketLimit && bucketTotal - bucketLimit < eventCount)
         {
             // org will be throttled during the current bucket of time
-            await _messagePublisher.PublishAsync(new PlanOverage { OrganizationId = organizationId, IsHourly = true });
             await _cache.SetAsync(GetThrottledKey(utcNow, organizationId), true, TimeSpan.FromMinutes(5));
+            await _cache.SetAsync(GetHourlyThrottleTransitionKey(utcNow, organizationId), true, TimeSpan.FromHours(8));
+            await _cache.SetAsync(GetHourlyOveragePendingKey(utcNow, organizationId), true, TimeSpan.FromHours(8));
+            await PublishPendingHourlyOverageAsync(utcNow, organizationId);
         }
     }
 
+    private async Task PublishPendingHourlyOverageAsync(DateTime utcTime, string organizationId)
+    {
+        string pendingKey = GetHourlyOveragePendingKey(utcTime, organizationId);
+        var pending = await _cache.GetAsync<bool>(pendingKey);
+        if (!pending.HasValue || !pending.Value)
+            return;
+
+        await _messagePublisher.PublishAsync(new PlanOverage { OrganizationId = organizationId, IsHourly = true });
+        await _cache.RemoveAsync(pendingKey);
+    }
+
     public async Task IncrementBlockedAsync(string organizationId, string? projectId, int eventCount = 1)
     {
         if (eventCount <= 0)
@@ -527,11 +554,15 @@ public async Task IncrementDeletedAsync(string organizationId, string? projectId
     }
 
     private int GetBucketEventLimit(int maxEventsPerMonth)
+    {
+        return GetBucketEventLimit(maxEventsPerMonth, _timeProvider.GetUtcNow().UtcDateTime);
+    }
+
+    private int GetBucketEventLimit(int maxEventsPerMonth, DateTime utcNow)
     {
         if (maxEventsPerMonth < 5000)
             return maxEventsPerMonth;
 
-        var utcNow = _timeProvider.GetUtcNow().UtcDateTime;
         var timeLeftInMonth = utcNow.EndOfMonth() - utcNow;
         if (timeLeftInMonth < TimeSpan.FromDays(1))
             return maxEventsPerMonth;
@@ -614,6 +645,18 @@ private string GetThrottledKey(DateTime utcTime, string organizationId)
         return $"usage:{bucket}:{organizationId}:throttled";
     }
 
+    private string GetHourlyThrottleTransitionKey(DateTime utcTime, string organizationId)
+    {
+        int bucket = GetCurrentBucket(utcTime);
+        return $"usage:{bucket}:{organizationId}:throttled-transition";
+    }
+
+    private string GetHourlyOveragePendingKey(DateTime utcTime, string organizationId)
+    {
+        int bucket = GetCurrentBucket(utcTime);
+        return $"usage:{bucket}:{organizationId}:hourly-overage-pending";
+    }
+
     private string GetProjectSetKey(DateTime utcTime)
     {
         int bucket = GetCurrentBucket(utcTime);

src/Exceptionless.Web/Api/Handlers/OrganizationHandler.cs

@@ -923,12 +923,13 @@ private async Task AfterResultMapAsync<TDestination>(ICollection<TDestination> m
             viewOrganization.TrimUsage(timeProvider);
 
             var currentUsage = viewOrganization.GetCurrentUsage(timeProvider);
-            currentUsage.Limit = realTimeUsage.CurrentUsage.Limit;
+            currentUsage.Limit = viewOrganization.GetMaxEventsPerMonthWithBonus(timeProvider);
             currentUsage.Total = realTimeUsage.CurrentUsage.Total;
             currentUsage.Blocked = realTimeUsage.CurrentUsage.Blocked;
             currentUsage.Discarded = realTimeUsage.CurrentUsage.Discarded;
             currentUsage.TooBig = realTimeUsage.CurrentUsage.TooBig;
             currentUsage.Deleted = realTimeUsage.CurrentUsage.Deleted;
+            viewOrganization.IsOverMonthlyLimit = currentUsage.Limit >= 0 && currentUsage.Total >= currentUsage.Limit;
 
             var currentHourUsage = viewOrganization.GetCurrentHourlyUsage(timeProvider);
             currentHourUsage.Total = realTimeUsage.CurrentHourUsage.Total;

src/Exceptionless.Web/ClientApp.angular/app/organization/manage/manage.tpl.html

@@ -2,7 +2,7 @@
 
 <div
     class="hbox hbox-auto-xs hbox-auto-sm"
-    refresh-on="OrganizationChanged"
+    refresh-on="OrganizationChanged UsageChanged"
     refresh-action="vm.get(data)"
     refresh-debounce="5000"
 >

src/Exceptionless.Web/ClientApp.angular/app/project/manage/manage.tpl.html

@@ -2,7 +2,7 @@
 
 <div
     class="hbox hbox-auto-xs hbox-auto-sm"
-    refresh-on="ProjectChanged"
+    refresh-on="ProjectChanged UsageChanged"
     refresh-action="vm.get(data)"
     refresh-throttle="10000"
 >

src/Exceptionless.Web/ClientApp.angular/components/organization/organization-service.js

@@ -3,13 +3,35 @@
 
     angular
         .module("exceptionless.organization", ["restangular"])
-        .factory("organizationService", function ($cacheFactory, $rootScope, objectIDService, Restangular) {
+        .factory("organizationService", function ($cacheFactory, $interval, $rootScope, objectIDService, Restangular) {
             var _cache = $cacheFactory("http:organization");
+            var _usageMonth = getUsageMonth();
             $rootScope.$on("cache:clear", _cache.removeAll);
             $rootScope.$on("cache:clear-organization", _cache.removeAll);
             $rootScope.$on("auth:logout", _cache.removeAll);
             $rootScope.$on("OrganizationChanged", _cache.removeAll);
             $rootScope.$on("ProjectChanged", _cache.removeAll);
+            $rootScope.$on("PlanOverage", _cache.removeAll);
+            $rootScope.$on("UsageChanged", _cache.removeAll);
+
+            $interval(function () {
+                var usageMonth = getUsageMonth();
+                if (usageMonth === _usageMonth) {
+                    return;
+                }
+
+                _usageMonth = usageMonth;
+                $rootScope.$broadcast("OrganizationChanged", {});
+            }, 60000);
+
+            $interval(function () {
+                $rootScope.$broadcast("UsageChanged");
+            }, 300000);
+
+            function getUsageMonth() {
+                var now = new Date();
+                return now.getUTCFullYear() * 12 + now.getUTCMonth();
+            }
 
             $rootScope.$on("StackChanged", function ($event, data) {
                 if (data.added) {
@@ -34,7 +56,7 @@
                     plan_id: options.planId,
                     stripe_token: options.stripeToken,
                     last4: options.last4,
-                    coupon_id: options.couponId
+                    coupon_id: options.couponId,
                 };
                 return Restangular.one("organizations", id).customPOST(body, "change-plan");
             }

src/Exceptionless.Web/ClientApp.angular/components/project/project-service.js

@@ -10,6 +10,7 @@
             $rootScope.$on("auth:logout", _cache.removeAll);
             $rootScope.$on("OrganizationChanged", _cache.removeAll);
             $rootScope.$on("ProjectChanged", _cache.removeAll);
+            $rootScope.$on("UsageChanged", _cache.removeAll);
 
             $rootScope.$on("StackChanged", function ($event, data) {
                 if (data.added) {

src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.svelte.ts

@@ -23,6 +23,20 @@ export async function invalidateOrganizationQueries(queryClient: QueryClient, me
     }
 }
 
+export async function invalidateOrganizationUsageQueries(queryClient: QueryClient, organizationId?: string) {
+    const invalidations = [queryClient.invalidateQueries({ queryKey: queryKeys.list(undefined) })];
+    if (organizationId) {
+        invalidations.push(queryClient.invalidateQueries({ exact: true, queryKey: queryKeys.id(organizationId, undefined) }));
+        invalidations.push(queryClient.invalidateQueries({ exact: true, queryKey: queryKeys.id(organizationId, 'stats') }));
+    }
+
+    await Promise.all(invalidations);
+}
+
+export async function invalidatePlanOverageQueries(queryClient: QueryClient, message: WebSocketMessageValue<'PlanOverage'>) {
+    await invalidateOrganizationUsageQueries(queryClient, message.organization_id);
+}
+
 export const queryKeys = {
     adminSearch: (params: GetAdminSearchOrganizationsParams) => [...queryKeys.list(params.mode), 'admin', { ...params }] as const,
     changePlan: (id: string | undefined) => [...queryKeys.type, id, 'change-plan'] as const,
@@ -108,6 +122,7 @@ export interface GetOrganizationRequest {
     params?: {
         mode: 'stats' | undefined;
     };
+    refetchInterval?: false | number;
     route: {
         id: string | undefined;
     };
@@ -360,7 +375,8 @@ export function getOrganizationQuery(request: GetOrganizationRequest) {
 
             return response.data!;
         },
-        queryKey: queryKeys.id(request.route.id, request.params?.mode)
+        queryKey: queryKeys.id(request.route.id, request.params?.mode),
+        refetchInterval: request.refetchInterval
     }));
 }
 

src/Exceptionless.Web/ClientApp/src/lib/features/organizations/api.test.ts

@@ -0,0 +1,39 @@
+import { QueryClient } from '@tanstack/svelte-query';
+import { describe, expect, it, vi } from 'vitest';
+
+import { invalidateOrganizationUsageQueries, invalidatePlanOverageQueries, queryKeys } from './api.svelte';
+
+describe('invalidatePlanOverageQueries', () => {
+    it('invalidates only the affected organization state', async () => {
+        // Arrange
+        const queryClient = new QueryClient();
+        const invalidateSpy = vi.spyOn(queryClient, 'invalidateQueries').mockImplementation(async () => {});
+
+        // Act
+        await invalidatePlanOverageQueries(queryClient, {
+            is_hourly: false,
+            organization_id: 'organization-id'
+        });
+
+        // Assert
+        expect(invalidateSpy).toHaveBeenCalledTimes(3);
+        expect(invalidateSpy).toHaveBeenCalledWith({ exact: true, queryKey: queryKeys.id('organization-id', undefined) });
+        expect(invalidateSpy).toHaveBeenCalledWith({ exact: true, queryKey: queryKeys.id('organization-id', 'stats') });
+        expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: queryKeys.list(undefined) });
+    });
+});
+
+describe('invalidateOrganizationUsageQueries', () => {
+    it('invalidates organization lists when there is no active organization', async () => {
+        // Arrange
+        const queryClient = new QueryClient();
+        const invalidateSpy = vi.spyOn(queryClient, 'invalidateQueries').mockImplementation(async () => {});
+
+        // Act
+        await invalidateOrganizationUsageQueries(queryClient);
+
+        // Assert
+        expect(invalidateSpy).toHaveBeenCalledOnce();
+        expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: queryKeys.list(undefined) });
+    });
+});