Fix empty page filtering stops

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 2.1.8
+
+### Fixed
+
+- Fixed an issue where paging would stop after three consecutive empty pages. The limit is now 1000, meaning up to 1000 consecutive empty pages are followed, and results after empty pages with a nextLink (Intune `$filter`) are returned. An empty page repeating a nextLink ends paging with a warning. Applies to `-All` and fan-out.
+
 ## 2.1.7
 
 ### Added

Directory.Build.props

@@ -1,7 +1,7 @@
 <Project>
 
   <PropertyGroup>
-    <Version>2.1.7</Version>
+    <Version>2.1.8</Version>
     <Authors>Thomas Maillo Grome, Fabien Tschanz</Authors>
     <Company>Mgx</Company>
     <Copyright>(c) 2026 Thomas Maillo Grome, (c) 2026 Fabien Tschanz. All rights reserved.</Copyright>

Modules/M365DSC.mgx/M365DSC.mgx.psd1

@@ -1,6 +1,6 @@
 @{
     RootModule        = 'M365DSC.mgx.psm1'
-    ModuleVersion     = '2.1.7'
+    ModuleVersion     = '2.1.8'
     GUID              = 'f978315f-75c0-48f5-b929-ca7a7757d1d2'
     Author            = 'Thomas Maillo Grome, Fabien Tschanz'
     CompanyName       = 'Mgx'
@@ -49,6 +49,9 @@
             LicenseUri   = 'https://github.com/Microsoft365DSC/mgx/blob/main/LICENSE'
             ProjectUri   = 'https://github.com/Microsoft365DSC/mgx'
             ReleaseNotes = @'
+v2.1.8
+- Fixed an issue where paging would stop after three consecutive empty pages. The limit is now 1000.
+
 v2.1.7
 - Added Invoke-MgxRequest -Envelope, emitting the payload as the service sent it, collection envelope and all, so a collection holding one item can be told from a single entity.
 - Fixed the batch retry loop waiting out its exponential fallback without regard for MaxRetryAfterSeconds, which capped only the delays a server asked for.

src/Mgx.Engine/Pagination/ConcurrentFanOut.cs

@@ -77,6 +77,7 @@ public async Task<FanOutResult> FetchAllAsync(
                 var allItems = new List<JsonElement>();
                 string? nextLink = url;
                 int consecutiveEmptyPages = 0;
+                var seenNextLinks = new HashSet<string>(StringComparer.Ordinal) { url };
 
                 // Extract expected host for nextLink validation (SSRF prevention)
                 Uri? expectedHost = Uri.TryCreate(url, UriKind.Absolute, out var parsed) ? parsed : null;
@@ -89,7 +90,7 @@ public async Task<FanOutResult> FetchAllAsync(
                     if (page.Value.Length == 0)
                     {
                         consecutiveEmptyPages++;
-                        if (consecutiveEmptyPages >= 3)
+                        if (consecutiveEmptyPages >= PageIterator.MaxConsecutiveEmptyPages)
                             break;
                     }
                     else
@@ -108,6 +109,12 @@ public async Task<FanOutResult> FetchAllAsync(
                     // Validate nextLink host matches initial URL (prevents SSRF via
                     // crafted Graph responses redirecting authenticated requests)
                     nextLink = NextLinkValidator.ValidateOrThrow(page.NextLink, expectedHost);
+                    if (nextLink != null && !seenNextLinks.Add(nextLink) && page.Value.Length == 0)
+                    {
+                        _client.EnqueueWarning(
+                            "Paging stopped: an empty page repeated an @odata.nextLink.");
+                        break;
+                    }
                 }
 
                 results[url] = allItems.ToArray();

src/Mgx.Engine/Pagination/PageIterator.cs

@@ -22,8 +22,10 @@ public sealed class PageIterator
 {
     private readonly ResilientGraphClient _client;
 
-    private const int MaxConsecutiveEmptyPages = 3;
-    private const int MaxConsecutiveEmptyPagesDelta = 1000;
+    /// <summary>
+    /// Intune filters per page, so matches can follow many empty pages. Bounds endless paging only.
+    /// </summary>
+    internal const int MaxConsecutiveEmptyPages = 1000;
 
     public PageIterator(ResilientGraphClient client)
     {
@@ -57,7 +59,7 @@ public async IAsyncEnumerable<JsonElement> StreamAllWithCountAsync(
         bool isFirstPage = true;
         bool countCaptured = false;
         int consecutiveEmptyPages = 0;
-        var emptyPageLimit = onDeltaLink != null ? MaxConsecutiveEmptyPagesDelta : MaxConsecutiveEmptyPages;
+        var seenNextLinks = new HashSet<string>(StringComparer.Ordinal) { nextLink };
 
         while (nextLink != null)
         {
@@ -92,7 +94,7 @@ public async IAsyncEnumerable<JsonElement> StreamAllWithCountAsync(
             if (page.Value.Length == 0)
             {
                 consecutiveEmptyPages++;
-                if (consecutiveEmptyPages >= emptyPageLimit)
+                if (consecutiveEmptyPages >= MaxConsecutiveEmptyPages)
                     break;
             }
             else
@@ -117,6 +119,12 @@ public async IAsyncEnumerable<JsonElement> StreamAllWithCountAsync(
             }
 
             nextLink = NextLinkValidator.ValidateOrThrow(page.NextLink, expectedHost);
+            if (nextLink != null && !seenNextLinks.Add(nextLink) && page.Value.Length == 0)
+            {
+                _client.EnqueueWarning(
+                    "Paging stopped: an empty page repeated an @odata.nextLink.");
+                break;
+            }
             isFirstPage = false;
 
             onPageComplete?.Invoke(new PageCompletedInfo(nextLink));

tests/Mgx.IntegrationTests/Cmdlets/DeltaQueryTests.cs

@@ -414,9 +414,9 @@ public async Task PageIterator_CapturesDeltaLink_AfterMultipleEmptyPages()
     }
 
     [Fact]
-    public async Task PageIterator_RegularEndpoint_StillBreaksAfter3EmptyPages()
+    public async Task PageIterator_RegularEndpoint_FollowsEmptyPagesToTheItemsBehindThem()
     {
-        // Without onDeltaLink, the regular limit of 3 applies
+        // Intune filters per page: the match can follow empty pages.
         ResiliencePipelineFactory.Reset();
         var handler = new MockHttpHandler();
 
@@ -426,6 +426,7 @@ public async Task PageIterator_RegularEndpoint_StillBreaksAfter3EmptyPages()
             handler.QueueResponse(HttpStatusCode.OK,
                 $"{{\"value\": [], \"@odata.nextLink\": \"https://graph.microsoft.com/v1.0/users?$skiptoken=empty{i}\"}}");
         }
+        handler.QueueResponse(HttpStatusCode.OK, "{\"value\": [{\"id\": \"behind-empty-pages\"}]}");
 
         using var httpClient = new HttpClient(handler);
         using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true });
@@ -434,13 +435,43 @@ public async Task PageIterator_RegularEndpoint_StillBreaksAfter3EmptyPages()
         var items = new List<JsonElement>();
 
         await foreach (var item in iterator.StreamAllWithCountAsync(
-            "https://graph.microsoft.com/v1.0/users", 0, null)) // No onDeltaLink = regular limit
+            "https://graph.microsoft.com/v1.0/users", 0, null))
         {
             items.Add(item);
         }
 
-        Assert.Equal(2, items.Count); // Items from page 1 only
-        Assert.Equal(4, handler.RequestCount); // Page 1 + 3 empty pages (stopped at limit)
+        Assert.Equal(3, items.Count);
+        Assert.Equal("behind-empty-pages", items[2].GetProperty("id").GetString());
+        Assert.Equal(7, handler.RequestCount);
+    }
+
+    [Fact]
+    public async Task PageIterator_RegularEndpoint_StopsWhenTheServiceRepeatsANextLink()
+    {
+        ResiliencePipelineFactory.Reset();
+        var handler = new MockHttpHandler();
+
+        handler.QueueResponse(HttpStatusCode.OK, TestData.UsersPage1);
+        for (int i = 0; i < 5; i++)
+        {
+            handler.QueueResponse(HttpStatusCode.OK,
+                "{\"value\": [], \"@odata.nextLink\": \"https://graph.microsoft.com/v1.0/users?$skiptoken=loop\"}");
+        }
+
+        using var httpClient = new HttpClient(handler);
+        using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true });
+
+        var iterator = new PageIterator(client);
+        var items = new List<JsonElement>();
+
+        await foreach (var item in iterator.StreamAllWithCountAsync(
+            "https://graph.microsoft.com/v1.0/users", 0, null))
+        {
+            items.Add(item);
+        }
+
+        Assert.Equal(2, items.Count);
+        Assert.Equal(3, handler.RequestCount);
     }
 
     // --- StreamAllWithCountAsync also captures deltaLink ---

tests/Mgx.IntegrationTests/Engine/Errors/FaultInjectionCatalog.cs

@@ -249,9 +249,8 @@ private static IReadOnlyDictionary<InjectionPoint, FaultCell> Cells(
             Class = null,
             Production = "a page whose @odata.nextLink is its own URL",
             Cells = Cells(
-                Pinned("page 3 points back at page 3. Nothing stops a self-loop of non-empty "
-                    + "pages today - only -Top / maxItems ends it; an empty self-loop ends at "
-                    + "the consecutive-empty-page limit"),
+                Pinned("page 3 points back at page 3. Only -Top / maxItems ends a non-empty "
+                    + "self-loop; an empty one ends at the repeated link"),
                 Pinned("the third url's page points back at itself, ended the same two ways"),
                 NotApplicable(NoLinksInABatch),
                 NotApplicable(NoLinksInADownload)),

tests/Mgx.IntegrationTests/Engine/PageIteratorTests.cs

@@ -112,7 +112,7 @@ await IdsOf(iterator.StreamAllWithCountAsync(
     }
 
     [Fact]
-    public async Task Three_consecutive_empty_pages_end_a_regular_stream()
+    public async Task A_repeated_nextLink_ends_a_regular_stream()
     {
         var handler = new StubHttpMessageHandler()
             .EnqueueRepeated(10, _ => new HttpResponseMessage(System.Net.HttpStatusCode.OK)
@@ -127,7 +127,7 @@ public async Task Three_consecutive_empty_pages_end_a_regular_stream()
         var ids = await IdsOf(iterator.StreamAllWithCountAsync(Start, 0, null, cancellationToken: Ct));
 
         Assert.Empty(ids);
-        Assert.Equal(3, handler.RequestCount);
+        Assert.Equal(2, handler.RequestCount);
     }
 
     [Fact]

tests/Mgx.IntegrationTests/Engine/Pagination/FaultInjectionFanOutTests.cs

@@ -246,7 +246,7 @@ private static async Task PinASelfReferencingNextLink()
         Assert.Equal(5, looping.RequestCount);
         Assert.Equal(3, looping.CapturedRequests.Count(r => r.Uri.Contains("/g3/")));
 
-        // Empty pages: the consecutive-empty-page limit is what does stop a self-loop today.
+        // Empty pages: the repeated link stops it.
         var empty = new MockHttpHandler();
         empty.When(r => r.Uri.Contains("/g1/")).Respond(HttpStatusCode.OK, PageBody(["m1", "m2"], null));
         empty.When(r => r.Uri.Contains("/g2/")).Respond(HttpStatusCode.OK, PageBody(["m3", "m4"], null));
@@ -256,7 +256,7 @@ private static async Task PinASelfReferencingNextLink()
 
         Assert.False(stopped.HasErrors);
         Assert.Empty(stopped.Results[Url3]);
-        Assert.Equal(5, empty.RequestCount);
+        Assert.Equal(3, empty.RequestCount);
     }
 
     private static async Task PinAnExpiredTokenAtTheThirdItem(FaultEntry entry)

tests/Mgx.IntegrationTests/Engine/Pagination/FaultInjectionPaginationTests.cs

@@ -235,7 +235,7 @@ private static async Task PinAStalledPageBody(FaultEntry entry)
     private static async Task PinASelfReferencingNextLink()
     {
         // Non-empty pages: the link points back at the page that produced it, every time, and
-        // nothing in the iterator notices. Only the item cap ends the walk - which is why this
+        // nothing in the iterator stops it. Only the item cap ends the walk - which is why this
         // case has to pass one.
         var looping = new MockHttpHandler();
         looping.When(r => r.Uri.Contains("skiptoken=page2"))
@@ -251,7 +251,7 @@ private static async Task PinASelfReferencingNextLink()
         Assert.Equal(5, looping.RequestCount);
         Assert.Equal(3, looping.CapturedRequests.Count(r => r.Uri.Contains("skiptoken=page3")));
 
-        // Empty pages: the consecutive-empty-page limit is what does stop a self-loop today.
+        // Empty pages: the repeated link stops it.
         var empty = new MockHttpHandler();
         empty.When(r => r.Uri.Contains("skiptoken=page2"))
              .Respond(HttpStatusCode.OK, PageBody(["u3", "u4"], Page3Url));
@@ -263,7 +263,7 @@ private static async Task PinASelfReferencingNextLink()
 
         Assert.Null(emptyThrown);
         Assert.Equal(4, emptyItems.Count);
-        Assert.Equal(5, empty.RequestCount);
+        Assert.Equal(3, empty.RequestCount);
     }
 
     private static async Task PinAnExpiredTokenAtPageThree(FaultEntry entry)