From ec71694647b6cc5908330b8dd261d7a7b3169ce4 Mon Sep 17 00:00:00 2001 From: gromedev <61883549+gromedev@users.noreply.github.com> Date: Fri, 21 Aug 2026 16:39:43 +0200 Subject: [PATCH 01/10] mgx 2.1.0 --- .github/workflows/build-and-test.yml | 78 +- .gitignore | 5 +- CHANGELOG.md | 139 ++- Directory.Build.props | 2 +- README.md | 179 +++- build.ps1 | 39 +- examples/01-get-all-users.ps1 | 12 - examples/03-get-managers-fan-out.ps1 | 18 - examples/05-delta-sync.ps1 | 18 - examples/07-enrich-users-with-manager.ps1 | 14 - examples/11-beta-endpoint.ps1 | 17 - examples/13-disabled-accounts-report.ps1 | 18 - examples/18-tune-rate-limits.ps1 | 26 - examples/20-conditional-access-export.ps1 | 20 - examples/22-mixed-endpoint-batch.ps1 | 37 - examples/README.md | 90 +- .../bulk-delete-whatif.ps1} | 15 +- .../bulk-update.ps1} | 6 + .../dead-letter-retry.ps1} | 25 + .../mixed-endpoint-batch.ps1 | 61 ++ examples/delta-and-sync/delta-sync.ps1 | 34 + .../delta-and-sync/drive-delta-resumable.ps1 | 49 ++ .../delta-and-sync/tenant-change-feed.ps1 | 82 ++ .../drive-content/drive-content-triage.ps1 | 155 ++++ .../drive-content/partial-content-hash.ps1 | 48 + .../chained-relation-expansion.ps1} | 14 + .../enrich-users-with-manager.ps1 | 32 + .../get-managers-fan-out.ps1 | 36 + .../group-members-multipage.ps1} | 10 + examples/getting-started/beta-endpoint.ps1 | 34 + .../export-users-to-jsonl.ps1} | 10 + examples/getting-started/get-all-users.ps1 | 29 + .../app-secrets-expiry.ps1} | 15 +- .../conditional-access-export.ps1 | 45 + .../disabled-accounts-report.ps1 | 33 + .../export-sign-in-logs.ps1} | 10 + .../guest-users-report.ps1} | 12 + .../stale-devices-report.ps1} | 14 + .../pacing-observability.ps1 | 178 ++++ .../resilience-for-existing-scripts.ps1} | 6 + .../resilience-status.ps1} | 18 +- .../resource-unit-budgeting.ps1 | 134 +++ .../telemetry.ps1} | 25 + .../tune-rate-limits.ps1 | 63 ++ module/en-US/Mgx.Cmdlets.dll-Help.xml | 678 ++++++++++++++- module/en-US/about_Mgx.help.txt | 30 +- module/en-US/about_Mgx_Tuning.help.txt | 37 + module/help/Enable-MgxResilience.md | 2 +- module/help/Get-MgxContent.md | 248 ++++++ module/help/Get-MgxOption.md | 2 +- module/help/Get-MgxTelemetry.md | 13 +- module/help/Invoke-MgxBatchRequest.md | 4 +- module/help/Set-MgxOption.md | 34 +- module/help/Sync-MgxDelta.md | 98 ++- module/mgx.Format.ps1xml | 6 + module/mgx.psd1 | 196 ++--- module/mgx.psm1 | 6 +- src/Mgx.Cmdlets/Base/ArgumentCompleters.cs | 25 + src/Mgx.Cmdlets/Base/MgxCmdletBase.cs | 287 +++++- src/Mgx.Cmdlets/Base/MgxCmdletCore.cs | 25 +- .../Cmdlets/Batch/InvokeMgxBatchRequest.cs | 95 +- .../Configuration/EnableMgxResilience.cs | 5 + .../Cmdlets/Configuration/GetMgxOption.cs | 1 + .../Cmdlets/Configuration/GetMgxTelemetry.cs | 5 + .../Cmdlets/Configuration/SetMgxOption.cs | 41 +- .../Cmdlets/Content/GetMgxContent.cs | 352 ++++++++ src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs | 661 +++++++++++++- .../Cmdlets/Expand/ExpandMgxRelation.cs | 61 +- .../Cmdlets/Export/ExportMgxCollection.cs | 202 +++-- src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs | 18 +- src/Mgx.Cmdlets/Mgx.Cmdlets.csproj | 18 +- src/Mgx.Cmdlets/Models/OutputTypes.cs | 14 + src/Mgx.Engine/Http/AdaptivePacing.cs | 153 ++++ src/Mgx.Engine/Http/AdaptiveRequestPacer.cs | 449 ++++++++++ src/Mgx.Engine/Http/DownloadUrlValidator.cs | 62 ++ src/Mgx.Engine/Http/GraphBatchClient.cs | 97 ++- src/Mgx.Engine/Http/GraphContentClient.cs | 441 ++++++++++ src/Mgx.Engine/Http/GraphRequestTracer.cs | 45 +- src/Mgx.Engine/Http/MgxTelemetryCollector.cs | 32 +- .../Http/ResiliencePipelineFactory.cs | 23 + .../Http/ResilientDelegatingHandler.cs | 61 +- src/Mgx.Engine/Http/ResilientGraphClient.cs | 70 +- .../Http/ResilientGraphClientOptions.cs | 7 + src/Mgx.Engine/Models/GraphResponse.cs | 10 + .../Models/GraphServiceException.cs | 44 +- src/Mgx.Engine/Pagination/ConcurrentFanOut.cs | 2 +- src/Mgx.Engine/Pagination/DeltaState.cs | 42 +- .../Pagination/NextLinkValidator.cs | 28 + src/Mgx.Engine/Pagination/PageIterator.cs | 13 +- .../Pagination/PaginationCheckpoint.cs | 40 +- tests/Live/Mgx.Live.Tests.ps1 | 167 ++++ tests/Live/README.md | 50 ++ .../{ => Base}/AuthFingerprintTests.cs | 0 .../Cmdlets/Base/BetaHintTests.cs | 132 +++ .../{ => Base}/JsonToHashtableTests.cs | 0 .../{ => Base}/PipelineContractTests.cs | 0 .../Cmdlets/DeltaCheckpointIntegrityTests.cs | 536 ++++++++++++ .../{ => Cmdlets}/DeltaQueryTests.cs | 623 +++++++++++++ .../Cmdlets/DeltaRecoveryTests.cs | 537 ++++++++++++ .../Cmdlets/ExportCheckpointIntegrityTests.cs | 517 +++++++++++ .../Cmdlets/ResilienceWrapTests.cs | 113 +++ .../Cmdlets/SetOptionGuardTests.cs | 85 ++ .../Cmdlets/TopWithAllTests.cs | 166 ++++ .../Engine/AdaptivePacingTests.cs | 110 --- .../{ => Engine/Batch}/BatchEdgeCaseTests.cs | 0 .../{ => Engine/Batch}/BatchWriteTests.cs | 0 .../{ => Engine/Batch}/BulkWriteTests.cs | 0 .../Engine/Batch/PartialBatchResultTests.cs | 143 +++ .../{ => Engine/Batch}/SpeedWinsTests.cs | 0 .../Engine/ErrorBodyShapeTests.cs | 73 ++ .../Engine/Http/AdaptivePacingTests.cs | 169 ++++ .../Engine/Http/AdaptiveRequestPacerTests.cs | 576 ++++++++++++ .../Engine/Http/BatchPacingStateTests.cs | 50 ++ .../Engine/Http/BodyRedactionTests.cs | 80 ++ .../{ => Engine/Http}/CancellationTests.cs | 0 .../{ => Engine/Http}/CircuitBreakerTests.cs | 0 .../{ => Engine/Http}/ConcurrencyTests.cs | 0 .../Http}/ConfigurableOptionsTests.cs | 0 .../{ => Engine/Http}/ContentTypeTests.cs | 0 .../Engine/Http/DownloadUrlValidatorTests.cs | 49 ++ .../Engine/Http/GraphContentClientTests.cs | 818 ++++++++++++++++++ .../{ => Engine/Http}/GuardTests.cs | 0 .../{ => Engine/Http}/ObservabilityTests.cs | 0 .../Http}/RateLimiterBatchTests.cs | 0 .../{ => Engine/Http}/RateLimiterTests.cs | 0 .../Engine/{ => Http}/RequestTraceTests.cs | 30 + .../Engine/Http/RetryAfterDateFormTests.cs | 63 ++ .../{ => Engine/Http}/RetryTests.cs | 0 .../{ => Engine/Http}/TransportConfigTests.cs | 0 .../Pagination/AtomicSaveHygieneTests.cs | 77 ++ .../Pagination/CheckpointRetentionTests.cs | 74 ++ .../Pagination}/CheckpointTests.cs | 0 .../{ => Engine/Pagination}/FanOutTests.cs | 0 .../Engine/Pagination/OrphanAdoptionTests.cs | 101 +++ .../Pagination}/PaginationTests.cs | 34 + .../Pagination}/SsrfValidationTests.cs | 111 ++- .../{ => Fakes}/MockHttpHandler.cs | 4 +- .../TestSetup/PacerTestDefaults.cs | 18 + tests/RELEASING.md | 59 ++ tests/TestHarness.psm1 | 12 +- .../Mgx.Behaviour.Tests.ps1} | 376 ++++++-- tests/Unit/Mgx.HelpFreshness.Tests.ps1 | 188 ++++ tests/Unit/Mgx.LiveCoverage.Tests.ps1 | 59 ++ tests/Unit/Mgx.Surface.Tests.ps1 | 83 ++ tests/benchmarks/01-list-users.ps1 | 10 +- tests/benchmarks/02-fanout-lookup.ps1 | 8 +- tests/benchmarks/05-memory-export.ps1 | 8 +- tests/benchmarks/06-fault-gauntlet.ps1 | 6 +- .../10-pacing-under-real-throttling.ps1 | 82 ++ tests/benchmarks/11-throttle-accuracy.ps1 | 219 +++++ tests/benchmarks/12-delta-replay.ps1 | 91 ++ tests/benchmarks/13-resource-unit-rate.ps1 | 156 ++++ tests/benchmarks/14-pacing-cold-cost.ps1 | 57 ++ tests/benchmarks/README.md | 19 + tests/benchmarks/common.ps1 | 183 +++- tests/benchmarks/run.ps1 | 5 +- .../tenant-resilience-comparison.ps1 | 0 157 files changed, 12839 insertions(+), 976 deletions(-) delete mode 100644 examples/01-get-all-users.ps1 delete mode 100644 examples/03-get-managers-fan-out.ps1 delete mode 100644 examples/05-delta-sync.ps1 delete mode 100644 examples/07-enrich-users-with-manager.ps1 delete mode 100644 examples/11-beta-endpoint.ps1 delete mode 100644 examples/13-disabled-accounts-report.ps1 delete mode 100644 examples/18-tune-rate-limits.ps1 delete mode 100644 examples/20-conditional-access-export.ps1 delete mode 100644 examples/22-mixed-endpoint-batch.ps1 rename examples/{17-bulk-delete-whatif.ps1 => bulk-writes-and-batching/bulk-delete-whatif.ps1} (69%) rename examples/{06-bulk-update.ps1 => bulk-writes-and-batching/bulk-update.ps1} (95%) rename examples/{09-dead-letter-retry.ps1 => bulk-writes-and-batching/dead-letter-retry.ps1} (62%) create mode 100644 examples/bulk-writes-and-batching/mixed-endpoint-batch.ps1 create mode 100644 examples/delta-and-sync/delta-sync.ps1 create mode 100644 examples/delta-and-sync/drive-delta-resumable.ps1 create mode 100644 examples/delta-and-sync/tenant-change-feed.ps1 create mode 100644 examples/drive-content/drive-content-triage.ps1 create mode 100644 examples/drive-content/partial-content-hash.ps1 rename examples/{19-chained-relation-expansion.ps1 => fan-out-and-relations/chained-relation-expansion.ps1} (63%) create mode 100644 examples/fan-out-and-relations/enrich-users-with-manager.ps1 create mode 100644 examples/fan-out-and-relations/get-managers-fan-out.ps1 rename examples/{12-group-members-multipage.ps1 => fan-out-and-relations/group-members-multipage.ps1} (83%) create mode 100644 examples/getting-started/beta-endpoint.ps1 rename examples/{02-export-users-to-jsonl.ps1 => getting-started/export-users-to-jsonl.ps1} (72%) create mode 100644 examples/getting-started/get-all-users.ps1 rename examples/{16-app-secrets-expiry.ps1 => reporting-and-audit/app-secrets-expiry.ps1} (65%) create mode 100644 examples/reporting-and-audit/conditional-access-export.ps1 create mode 100644 examples/reporting-and-audit/disabled-accounts-report.ps1 rename examples/{08-export-sign-in-logs.ps1 => reporting-and-audit/export-sign-in-logs.ps1} (73%) rename examples/{14-guest-users-report.ps1 => reporting-and-audit/guest-users-report.ps1} (57%) rename examples/{15-stale-devices-report.ps1 => reporting-and-audit/stale-devices-report.ps1} (57%) create mode 100644 examples/resilience-and-telemetry/pacing-observability.ps1 rename examples/{04-resilience-for-existing-scripts.ps1 => resilience-and-telemetry/resilience-for-existing-scripts.ps1} (91%) rename examples/{21-resilience-status.ps1 => resilience-and-telemetry/resilience-status.ps1} (70%) create mode 100644 examples/resilience-and-telemetry/resource-unit-budgeting.ps1 rename examples/{10-telemetry.ps1 => resilience-and-telemetry/telemetry.ps1} (50%) create mode 100644 examples/resilience-and-telemetry/tune-rate-limits.ps1 create mode 100644 module/help/Get-MgxContent.md create mode 100644 src/Mgx.Cmdlets/Cmdlets/Content/GetMgxContent.cs create mode 100644 src/Mgx.Engine/Http/AdaptivePacing.cs create mode 100644 src/Mgx.Engine/Http/AdaptiveRequestPacer.cs create mode 100644 src/Mgx.Engine/Http/DownloadUrlValidator.cs create mode 100644 src/Mgx.Engine/Http/GraphContentClient.cs create mode 100644 tests/Live/Mgx.Live.Tests.ps1 create mode 100644 tests/Live/README.md rename tests/Mgx.IntegrationTests/Cmdlets/{ => Base}/AuthFingerprintTests.cs (100%) create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/Base/BetaHintTests.cs rename tests/Mgx.IntegrationTests/Cmdlets/{ => Base}/JsonToHashtableTests.cs (100%) rename tests/Mgx.IntegrationTests/Cmdlets/{ => Base}/PipelineContractTests.cs (100%) create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/DeltaCheckpointIntegrityTests.cs rename tests/Mgx.IntegrationTests/{ => Cmdlets}/DeltaQueryTests.cs (62%) create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/DeltaRecoveryTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/ExportCheckpointIntegrityTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/ResilienceWrapTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/SetOptionGuardTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/TopWithAllTests.cs delete mode 100644 tests/Mgx.IntegrationTests/Engine/AdaptivePacingTests.cs rename tests/Mgx.IntegrationTests/{ => Engine/Batch}/BatchEdgeCaseTests.cs (100%) rename tests/Mgx.IntegrationTests/{ => Engine/Batch}/BatchWriteTests.cs (100%) rename tests/Mgx.IntegrationTests/{ => Engine/Batch}/BulkWriteTests.cs (100%) create mode 100644 tests/Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs rename tests/Mgx.IntegrationTests/{ => Engine/Batch}/SpeedWinsTests.cs (100%) create mode 100644 tests/Mgx.IntegrationTests/Engine/ErrorBodyShapeTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Http/AdaptivePacingTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Http/AdaptiveRequestPacerTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Http/BatchPacingStateTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Http/BodyRedactionTests.cs rename tests/Mgx.IntegrationTests/{ => Engine/Http}/CancellationTests.cs (100%) rename tests/Mgx.IntegrationTests/{ => Engine/Http}/CircuitBreakerTests.cs (100%) rename tests/Mgx.IntegrationTests/{ => Engine/Http}/ConcurrencyTests.cs (100%) rename tests/Mgx.IntegrationTests/{ => Engine/Http}/ConfigurableOptionsTests.cs (100%) rename tests/Mgx.IntegrationTests/{ => Engine/Http}/ContentTypeTests.cs (100%) create mode 100644 tests/Mgx.IntegrationTests/Engine/Http/DownloadUrlValidatorTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Http/GraphContentClientTests.cs rename tests/Mgx.IntegrationTests/{ => Engine/Http}/GuardTests.cs (100%) rename tests/Mgx.IntegrationTests/{ => Engine/Http}/ObservabilityTests.cs (100%) rename tests/Mgx.IntegrationTests/{ => Engine/Http}/RateLimiterBatchTests.cs (100%) rename tests/Mgx.IntegrationTests/{ => Engine/Http}/RateLimiterTests.cs (100%) rename tests/Mgx.IntegrationTests/Engine/{ => Http}/RequestTraceTests.cs (77%) create mode 100644 tests/Mgx.IntegrationTests/Engine/Http/RetryAfterDateFormTests.cs rename tests/Mgx.IntegrationTests/{ => Engine/Http}/RetryTests.cs (100%) rename tests/Mgx.IntegrationTests/{ => Engine/Http}/TransportConfigTests.cs (100%) create mode 100644 tests/Mgx.IntegrationTests/Engine/Pagination/AtomicSaveHygieneTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Pagination/CheckpointRetentionTests.cs rename tests/Mgx.IntegrationTests/{ => Engine/Pagination}/CheckpointTests.cs (100%) rename tests/Mgx.IntegrationTests/{ => Engine/Pagination}/FanOutTests.cs (100%) create mode 100644 tests/Mgx.IntegrationTests/Engine/Pagination/OrphanAdoptionTests.cs rename tests/Mgx.IntegrationTests/{ => Engine/Pagination}/PaginationTests.cs (77%) rename tests/Mgx.IntegrationTests/{ => Engine/Pagination}/SsrfValidationTests.cs (67%) rename tests/Mgx.IntegrationTests/{ => Fakes}/MockHttpHandler.cs (93%) create mode 100644 tests/Mgx.IntegrationTests/TestSetup/PacerTestDefaults.cs create mode 100644 tests/RELEASING.md rename tests/{Mgx.Tests.ps1 => Unit/Mgx.Behaviour.Tests.ps1} (81%) create mode 100644 tests/Unit/Mgx.HelpFreshness.Tests.ps1 create mode 100644 tests/Unit/Mgx.LiveCoverage.Tests.ps1 create mode 100644 tests/benchmarks/10-pacing-under-real-throttling.ps1 create mode 100644 tests/benchmarks/11-throttle-accuracy.ps1 create mode 100644 tests/benchmarks/12-delta-replay.ps1 create mode 100644 tests/benchmarks/13-resource-unit-rate.ps1 create mode 100644 tests/benchmarks/14-pacing-cold-cost.ps1 rename examples/23-benchmark-resilience.ps1 => tests/benchmarks/tenant-resilience-comparison.ps1 (100%) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index f405a54..59d0886 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -30,6 +30,14 @@ jobs: - name: Restore run: dotnet restore Mgx.slnx + - name: Install platyPS + shell: pwsh + run: | + # build.ps1 regenerates the compiled help from module/help and refuses to build without + # this. Help that is not regenerated ships stale, which is how Get-Help came to describe + # output shapes two releases out of date. + Install-PSResource -Name platyPS -TrustRepository -Scope AllUsers + - name: Build module shell: pwsh run: ./build.ps1 -Configuration Release @@ -41,6 +49,21 @@ jobs: --logger "trx;LogFileName=xunit-results.trx" --results-directory TestResults + # A run that executes nothing exits 0 and looks identical to a run that executes + # everything, so assert on the count as well as the exit code. Raise the floor as the + # suite grows. + - name: Assert xUnit actually ran + shell: pwsh + run: | + $floor = 530 + $trx = Get-ChildItem TestResults -Filter *.trx | Select-Object -First 1 + if (-not $trx) { throw "No .trx produced - the test run did not report results." } + $c = ([xml](Get-Content $trx.FullName -Raw)).TestRun.ResultSummary.Counters + "xUnit executed $($c.total) test(s): $($c.passed) passed, $($c.failed) failed" | Write-Host + if ([int]$c.total -lt $floor) { + throw "Only $($c.total) xUnit tests executed, expected at least $floor. The suite is not running." + } + - name: Install test prerequisites shell: pwsh run: | @@ -55,9 +78,16 @@ jobs: run: | Import-Module ./tests/TestHarness.psm1 $result = Invoke-TestHarness -IgnoreCodeCoverage + "Pester executed $($result.TotalCount): $($result.PassedCount) passed, $($result.FailedCount) failed, $($result.SkippedCount) skipped" | Write-Host if ($result.FailedCount -gt 0) { throw "$($result.FailedCount) Pester test(s) failed." } + # Same reasoning as the xUnit floor above: silence is not success. + $floor = 215 + $ran = [int]$result.PassedCount + [int]$result.SkippedCount + if ($ran -lt $floor) { + throw "Only $ran Pester test(s) ran, expected at least $floor. The suite is not running." + } - name: Validate module manifest shell: pwsh @@ -78,9 +108,11 @@ jobs: tests/TestResults.xml if-no-files-found: ignore - # The manifest claims PowerShell 7.4+. That claim is only honest if every supported - # host actually imports the module and passes the surface tests, so each one gets a leg. - # 7.4 is the LTS floor; without this job the floor is an assertion rather than a result. + # The manifest claims PowerShell 7.4+. That claim is only honest if the supported range + # actually imports the module and passes the surface tests, so its ends each get a leg: + # 7.4 is the LTS floor, and the newest release is the "+". Without this job the floor is + # an assertion rather than a result. The middle of the range is not covered - a 7.5-only + # regression reaches a user before it reaches CI, which is the accepted cost of the trim. compat: name: PowerShell ${{ matrix.psversion }} runs-on: windows-latest @@ -89,7 +121,7 @@ jobs: strategy: fail-fast: false matrix: - psversion: ['7.4', '7.5', '7.6'] + psversion: ['7.4', '7.6'] steps: - name: Checkout uses: actions/checkout@v6 @@ -107,28 +139,48 @@ jobs: dotnet tool install --global PowerShell --version '${{ matrix.psversion }}.*' "$env:USERPROFILE\.dotnet\tools" | Out-File -Append -Encoding utf8 $env:GITHUB_PATH + - name: Install platyPS + shell: pwsh + run: | + # build.ps1 regenerates the compiled help from module/help and refuses to build without + # this. Help that is not regenerated ships stale, which is how Get-Help came to describe + # output shapes two releases out of date. + Install-PSResource -Name platyPS -TrustRepository -Scope AllUsers + - name: Build module shell: pwsh run: ./build.ps1 -Configuration Release - # Written to a file and invoked with -File rather than passed to -Command. The previous - # form nested a double-quoted PowerShell command inside a YAML run block and escaped its - # dollars as \$, which is bash escaping - PowerShell uses a backtick. This step is also the - # only one in the job that omitted `shell: pwsh`, so the backslashes reached the inner - # pwsh literally and it died with "The term '\$r.FailedCount' is not recognized" before a - # single test ran. Every leg of this matrix had been failing that way since 2.0.0, so the - # 7.4 floor was never actually verified despite the job appearing to test it. + # Written to a file and invoked with -File rather than passed to -Command: nesting a + # double-quoted PowerShell command inside a YAML run block invites bash-style \$ escaping, + # where PowerShell needs a backtick. Keep `shell: pwsh` on this step so the here-string + # reaches the inner host intact. - name: Import and run surface tests on ${{ matrix.psversion }} shell: pwsh run: | $probe = @' "running on PowerShell $($PSVersionTable.PSVersion.ToString())" | Write-Host + # Assert the pin took effect. Printing the version proves nothing: if the install or + # PATH precedence silently gave us the wrong host, both matrix legs would test the same + # runtime and stay green while claiming to cover two. + $want = $env:MGX_EXPECT_PSVERSION + $got = "$($PSVersionTable.PSVersion.Major).$($PSVersionTable.PSVersion.Minor)" + if ($want -and $got -ne $want) { throw "expected PowerShell $want, got $got - the version pin did not take effect" } Install-PSResource -Name Pester -TrustRepository -Scope CurrentUser Import-Module ./module/mgx.psd1 -ErrorAction Stop - $r = Invoke-Pester -Path ./tests/Unit -PassThru - "$($r.PassedCount) passed, $($r.FailedCount) failed" | Write-Host + # Through the harness, not Invoke-Pester directly: tests/Unit contains Live-tagged + # blocks that need a real Graph session, and only the harness filters them out. + Import-Module ./tests/TestHarness.psm1 -Force + $r = Invoke-TestHarness -IgnoreCodeCoverage + "$($r.PassedCount) passed, $($r.FailedCount) failed, $($r.SkippedCount) skipped" | Write-Host if ($r.FailedCount -gt 0) { throw "$($r.FailedCount) test(s) failed" } + $floor = 215 + $ran = [int]$r.PassedCount + [int]$r.SkippedCount + if ($ran -lt $floor) { + throw "Only $ran test(s) ran on this host, expected at least $floor." + } '@ Set-Content -Path compat-surface-tests.ps1 -Value $probe -Encoding utf8 + $env:MGX_EXPECT_PSVERSION = '${{ matrix.psversion }}' pwsh -NoProfile -File ./compat-surface-tests.ps1 if ($LASTEXITCODE -ne 0) { throw "surface tests failed on PowerShell ${{ matrix.psversion }}" } diff --git a/.gitignore b/.gitignore index fbfe4dc..7609840 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,7 @@ module/.build-hash # Test outputs tests/TestResults.xml -tests/Benchmarks/results/* \ No newline at end of file +tests/benchmarks/results/* + +# Generated by the CI compat job when run locally +compat-surface-tests.ps1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cbf2ac..c4ba26c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,89 +1,144 @@ # Changelog +## 2.1.0 + +Adds proactive throttle avoidance, ranged content downloads, and resumable enumeration. + +### Added + +- Adaptive request pacing, on by default. Requests are spaced before they are sent, per workload; opt out with Set-MgxOption -NoAdaptivePacing. +- Get-MgxContent, to download file and media content whole or by byte range (-First). +- Sync-MgxDelta -CheckpointPath, to resume interrupted enumerations. +- Sync-MgxDelta -Latest, to baseline state without enumerating. +- Sync-MgxDelta -Prefer, to send drive delta Prefer tokens. +- Resource units consumed, in telemetry output, benchmarks and examples. + +### Fixed + +- -Latest was honored after a state invalidation, dropping every change since the last sync. +- Delta resume dropped the interrupted run's items when an earlier run had already completed, and advanced the delta token past them. +- Export resume dropped the interrupted run's items when an earlier export had already written the output file. +- Export resume wrote a page's items twice when a run was interrupted twice inside the same page. +- Delta resume kept the previous sync's rows in front of the interrupted sync's output. +- Export resume started the export over when the interrupted run had died on a transient error. +- A resume checkpoint naming a file that was never a temp consumed it as data. +- A refused delta token was reported as an endpoint not supporting delta queries. +- Sync-MgxDelta as a session's first cmdlet sent its request to the public cloud endpoint. +- A denied or read-only output file ended a delta sync with an unhandled error on Windows. +- A missing drive item suggested retrying in beta instead of reporting the item absent. +- -Top was ignored when combined with -All, returning the whole collection. +- Get-Help described output shapes and parameters from before 2.0.0. +- Enumeration returned part of a collection without error when a nextLink was refused. +- A delta sync stopped with an error when a page contained a non-object item. +- Get-MgxContent left an unwritable -OutFile as an unhandled error naming a temp path. +- A non-Graph JSON error body threw from inside the exception it was being used to build, surfacing a CDN failure as a security refusal. +- A $batch 429 set a pacing cap that nothing enforced and nothing cleared, so telemetry reported it forever. +- Get-MgxContent ignored Retry-After when a download host sent it as an HTTP date. +- Get-Help was missing the examples for -CheckpointPath, -Latest, -Prefer, adaptive pacing, and two Get-MgxContent endpoints, and did not document Set-MgxOption -BatchChunkConcurrency. +- Seven examples rendered blank tables, and one reported password secrets as certificates. +- Repeat 429s could raise the adaptive cap above -RateLimitPerSecond. +- Delta state did not record its API version, so runs omitting -ApiVersion silently synced the other one. +- -Debug wrote pre-authenticated download URLs verbatim, from redirect headers and from response bodies. +- Get-MgxContent -OutFile with piped input downloaded every item but kept only the last. +- -Offset was ignored when a server returned a whole-body 200. +- An offset past end of file overwrote an existing -OutFile with an empty one. +- Two-hop content downloads were reported as failed requests. +- Adaptive pacing fired synchronized bursts when sleep times were clamped. +- Enable-MgxResilience recorded no telemetry: request counts, HTTP time and rate-limiter wait all stayed 0. +- Invoke-MgGraphRequest with a relative URI failed while resilience was enabled. +- $batch envelopes were bucketed as Other rather than their target workload. +- Slow start opened above ceilings set below 4 rps. +- ResiliencePipelineFactory.Reset() reverted configuration and kept batch pacer state across credential changes. +- Ctrl-C during file cleanup raised disposed-object errors. + +### Documentation + +- Graph delta responses repeat objects across pages; deduplicate on id, or baseline with -Latest. +- Adaptive pacing applies to every request except batch outer POSTs. +- Measured throttling behavior: resource unit budgets are scoped per application and tenant, and x-ms-throttle-limit-percentage is not emitted. + ## 2.0.1 Patch release. Three fixes; no feature or API changes. ### Fixed -- The rate limiter was disposed on a timer while live clients still held it, so any Set-MgxOption call left Enable-MgxResilience sessions throwing ObjectDisposedException minutes later. It is now retired by dropping the reference rather than disposed. -- Enable-MgxResilience and Disable-MgxResilience disposed the injected HTTP client synchronously, cancelling SDK requests still in flight. Restoring the SDK's own client already stops new traffic. -- Sync-MgxDelta -Uri with a query string or trailing slash failed on the second run with a DeltaLinkPathMismatch error. The resource-path check now compares paths to paths. +- The rate limiter was disposed on a timer while live clients held it, so any Set-MgxOption call broke Enable-MgxResilience sessions minutes later. +- Enable-MgxResilience and Disable-MgxResilience cancelled SDK requests still in flight. +- Sync-MgxDelta -Uri with a query string or trailing slash failed on the second run. ## 2.0.0 ### Breaking -- Output shape changed for Invoke-MgxRequest, Invoke-MgxBatchRequest, Expand-MgxRelation, and Sync-MgxDelta to emit case-insensitive Hashtables instead of PSObjects, matching Invoke-MgGraphRequest output. Raw JSON output is still accessible via -Raw | ConvertFrom-Json. -- Preserved @odata.type verbatim under its own key for read-modify-write round-trips while stripping all other @odata.- transport metadata (including @odata.etag to prevent phantom drift in state comparisons). -- Removed custom Graph entity format views (Mgx.User, Mgx.Group, Mgx.Application, Mgx.ServicePrincipal, Mgx.DirectoryRole, Mgx.BatchResult) since dictionaries render using PowerShell's built-in Name/Value view. -- Lowered the platform floor to target .NET 8, adding support for PowerShell 7.4 (LTS) and later (down from 7.5). Replaced the .NET 9 API in the -Debug tracer with a WaitAsync equivalent. -- Removed Microsoft.Graph.Authentication from RequiredModules to decouple the module from the full Graph SDK. Cmdlets needing authentication now report a GraphAuthModuleNotLoaded error with installation instructions if the module is missing. +- Invoke-MgxRequest, Invoke-MgxBatchRequest, Expand-MgxRelation and Sync-MgxDelta now emit case-insensitive Hashtables instead of PSObjects, matching Invoke-MgGraphRequest. Use -Raw | ConvertFrom-Json for the old shape. +- @odata.type is preserved under its own key; all other @odata.* transport metadata is stripped, including @odata.etag. +- Removed the custom format views (Mgx.User, Mgx.Group, and the rest); dictionaries render with PowerShell's built-in Name/Value view. +- Lowered the floor to PowerShell 7.4 (LTS) and .NET 8, down from 7.5. +- Removed Microsoft.Graph.Authentication from RequiredModules. Cmdlets needing auth now report GraphAuthModuleNotLoaded with install instructions. ### Fixed -- Action endpoints returning collection envelopes (e.g., /directoryObjects/getByIds) now correctly unwrap and emit individual items across all response paths, including fan-out bulk writes, with truncation warnings if @odata.nextLink is present. -- Fan-out requests piped to endpoints like '/users/{id}' now extract id members from both Hashtables and PSCustomObjects, throwing a MissingPipelineId error if neither is found instead of embedding literal type names in the URL. -- Invoke-MgxBatchRequest now parses input items from both Hashtables and PSCustomObjects, enabling failed items to be piped directly back in for retries. +- Action endpoints returning collection envelopes (e.g. /directoryObjects/getByIds) now unwrap and emit individual items on every response path. +- Fan-out to endpoints like /users/{id} now extracts id from both Hashtables and PSCustomObjects, raising MissingPipelineId if absent. +- Invoke-MgxBatchRequest accepts both Hashtables and PSCustomObjects, so failed items can be piped straight back in. ### Changed -- Updated dependencies to Polly.Core 8.7.0 and System.Threading.RateLimiting 10.0.10. +- Updated to Polly.Core 8.7.0 and System.Threading.RateLimiting 10.0.10. ## 1.0.5 ### Fixed -- Export-MgxCollection no longer loses progress on interrupted first runs. Cancelled runs promote temp files to the output path with matching checkpoints, and killed or crashed runs recover on the next invocation by trimming orphaned temp files to the last checkpointed item count. The "Resume with:" hint now accurately reflects first-run states. -- Get-MgxTelemetry now includes per-item Retry-After wait times from $batch processing into RetryDelayMs instead of incorrectly reporting zero retry delay for throttled batch sessions. -- Count-discrepancy warnings now also trigger when an enumeration returns more items than @odata.count reported (using a 0.5% threshold with a 50-item floor) and recommend deduplicating on id downstream. +- Export-MgxCollection no longer loses progress on an interrupted first run. +- Get-MgxTelemetry now counts per-item Retry-After waits from $batch in RetryDelayMs. +- Count-discrepancy warnings now also fire when an enumeration returns more items than @odata.count reported. ### Documentation -- Added write-cost pacing documentation to about_Mgx_Tuning, explaining that Graph throttles write operations rather than batch items (e.g., a group create with 20 member bindings costs ~21 writes) and how to budget BatchItemsPerSecond accordingly. -- Updated Export-MgxCollection help with details on mid-page checkpointing, first-run recovery, and downstream id deduplication hygiene. -- Adjusted performance claims for Invoke-MgxRequest, updating the batch-vs-fan-out speedup from 3–4x down to the measured ~1.5x for PATCH operations at 1k scale. -- Rebuilt the README based on measured results from the new benchmark suite. - +- about_Mgx_Tuning now covers write-cost pacing: Graph throttles writes, not batch items. +- Export-MgxCollection help covers mid-page checkpointing, first-run recovery and id deduplication. +- Corrected the batch-vs-fan-out speedup for PATCH at 1k scale, from 3-4x to the measured ~1.5x. +- Rebuilt the README from the new benchmark suite. ## 1.0.4 ### Fixed -- Fixed Mgx cmdlets caching credentials from only the first Connect-MgGraph call. HTTP clients are now keyed on a fingerprint of the full authentication context so reconnecting with a different application, certificate, account, scope set, or secret properly updates the identity. -- Fixed JSON string input to -Body being silently converted to an empty object ({}) when passed as a PSObject. IDictionary, PSCustomObject, array, and nested bodies now serialize correctly. -- Fixed Enable-MgxResilience remaining bound to pre-reconnect SDK clients by automatically re-injecting resilience settings when the Graph identity changes. -- Fixed Set-MgxOption -TotalTimeoutSeconds failing to update the HTTP client timeout for existing sessions by rebuilding the client on value changes. -- Fixed temporary 429 throttling permanently slowing Invoke-MgxBatchRequest rates for the remainder of the session; write pacing now recovers after clean chunks and fully resets after five unthrottled minutes. -- Fixed the internal type cache failing to invalidate when Microsoft.Graph.Authentication was re-imported at a different version or in a new load context. -- Fixed JSON integers larger than 2^53 losing precision due to double-precision floating-point conversion. +- Cmdlets cached credentials from only the first Connect-MgGraph call. Clients are now keyed on the full authentication context. +- JSON string input to -Body was silently converted to an empty object when passed as a PSObject. +- Enable-MgxResilience stayed bound to pre-reconnect SDK clients. +- Set-MgxOption -TotalTimeoutSeconds did not update the HTTP client timeout for existing sessions. +- Temporary 429s permanently slowed Invoke-MgxBatchRequest for the rest of the session. +- The internal type cache did not invalidate when Microsoft.Graph.Authentication was re-imported. +- JSON integers larger than 2^53 lost precision. ### Changed -- Batch items with invalid JSON bodies now fail individually instead of aborting the entire batch. -- Passing -Body on a GET request now emits a warning instead of being silently ignored. -- SdkVersion header strings are now derived automatically from the assembly version set in Directory.Build.props. +- Batch items with invalid JSON bodies now fail individually instead of aborting the batch. +- Passing -Body on a GET now warns instead of being silently ignored. +- SdkVersion header strings are derived from the assembly version. ### Added -- Added -Debug request and response tracing across all cmdlets (single requests, pagination, fan-out, $batch) with credential redaction and a 4 KB body truncation limit. -- Added xUnit and Pester test suites alongside a build-and-test CI workflow to the repository. - +- -Debug request and response tracing across all cmdlets, with credential redaction and a 4 KB body limit. +- xUnit and Pester suites, and a build-and-test CI workflow. ## 1.0.3 -- Fixed `Remove-Module Mgx` failing and leaving the module loaded. Static-state cleanup moved into `AlcInitializer.OnRemove`, ahead of the ALC resolver detaching; it previously ran from the module `OnRemove` scriptblock, by which point `Polly.Core` was unresolvable. Only triggered when no Graph request had run in the session -- Fixed the `SdkVersion` request header reporting `mgx/0.3.0` regardless of the installed version. It now matches the module version, and `build.ps1` fails the build if the constant and the manifest ever disagree -- Extracted cmdlet lifecycle and JSON conversion to `MgxCmdletCore`. Internal refactor; no change to the cmdlet surface -- Treat `CA1416` as an error in both projects, so a Windows-only API reaching the cross-platform code paths fails the build instead of throwing `PlatformNotSupportedException` on Linux or macOS +- Remove-Module Mgx failed and left the module loaded, when no Graph request had run in the session. +- The SdkVersion header reported mgx/0.3.0 regardless of the installed version. +- Extracted cmdlet lifecycle and JSON conversion to MgxCmdletCore. Internal refactor; no surface change. +- CA1416 is now an error, so a Windows-only API in cross-platform code fails the build rather than throwing at runtime. ## 1.0.2 -- Fixed Linux install: renamed `Mgx.psd1`, `Mgx.psm1`, and `Mgx.Format.ps1xml` to lowercase so `Install-Module Mgx` works on case-sensitive filesystems (PSGallery lowercases the module folder name) -- Updated `about_Mgx_Tuning` version reference to v1.0.1 +- Fixed Linux install: lowercased the manifest, module and format filenames for case-sensitive filesystems. +- Updated the about_Mgx_Tuning version reference to v1.0.1. ## 1.0.1 -- Added tab completion for the Uri parameter on all cmdlets that accept Graph API paths (Invoke-MgxRequest, Invoke-MgxBatchRequest, Export-MgxCollection, Expand-MgxRelation, Sync-MgxDelta) -- Extracted `CircuitBreakerMessage` protected property on `MgxCmdletBase` to eliminate repeated inline circuit breaker message strings across six cmdlet files -- Removed redundant XML doc comments on self-documenting members in `MgxCmdletBase` and `ResilientGraphClient` \ No newline at end of file +- Added tab completion for the Uri parameter on every cmdlet that accepts a Graph path. +- Extracted CircuitBreakerMessage on MgxCmdletBase, replacing six inline copies. +- Removed redundant XML doc comments on self-documenting members. diff --git a/Directory.Build.props b/Directory.Build.props index d511574..611c34a 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,7 +1,7 @@ - 2.0.1 + 2.1.0 Thomas Maillo Grome Mgx (c) 2026 Thomas Maillo Grome. All rights reserved. diff --git a/README.md b/README.md index 7412441..18295e5 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,18 @@ # Mgx +Microsoft Graph becomes difficult when a PowerShell process turns into a long-running, concurrent, stateful workload operating against a throttled and eventually consistent API. Sequential lookups are slow at scale, long-running requests can hang on dead connections, and scripts that do not explicitly handle throttling or transient failures can lose data or require manual recovery. -Microsoft Graph PowerShell works well for ordinary scripting, but high-volume operations can become difficult to handle efficiently. Sequential lookups are slow at scale, long-running requests can hang on dead connections, and scripts that do not explicitly handle throttling or transient failures can lose data or require manual recovery. +Given a large Graph workload, Mgx can execute it efficiently, preserve correctness, adapt to Graph's limits, survive transient failure and state changes, remain observable, and recover without forcing the caller to implement the machinery itself. -Mgx adds concurrency, streaming pagination, batching, and resilience features to Microsoft Graph PowerShell while using the existing `Connect-MgGraph` authentication context. +Mgx does this by adding concurrency, streaming pagination, batching, and resilience features to Microsoft Graph PowerShell while (optionally) using the existing `Connect-MgGraph` authentication context. -In benchmarks, operations that benefit from concurrency and batching were **5–12× faster** than the Microsoft Graph SDK. Mgx also provides resilience for throttling, transient failures, dead connections, and interrupted long-running operations. In a 24-hour fault-injection test against a 100,000-user tenant, Mgx completed every operation without losing a single object. +In benchmarks, operations that benefit from concurrency and batching were **4–5× faster** than the Microsoft Graph SDK. On plain enumeration, where there is nothing to parallelize, it comes out about level with a tuned SDK - the difference being that it needs no tuning to get there. Mgx also provides resilience for throttling, transient failures, dead connections, and interrupted long-running operations: enumerating a 100,000-user tenant while its request budget was held at the ceiling, it returned all 100,000, where a plain `Invoke-RestMethod` loop returned 1,700.¹ ¹ Tested under sustained throttle waves, injected 429/503 faults, dead sockets, and `kill -9` during export. See [`tests/benchmarks/`](tests/benchmarks/). ## Quick start ```powershell +Install-Module Microsoft.Graph.Authentication # Connect-MgGraph lives here Install-Module Mgx Connect-MgGraph -Scopes "User.Read.All" Invoke-MgxRequest /users -All -Property displayName,mail @@ -19,40 +21,112 @@ Invoke-MgxRequest /users -All -Property displayName,mail ## Key Capabilities * **Performance:** Streaming pagination, concurrent fan-out, and batched writes of up to 20 sub-requests per HTTP call. -* **Resilience:** Proactive rate limiting, exponential backoff with jitter, circuit breakers, adaptive write pacing, body-read timeouts, and connection recycling. +* **Resilience:** Proactive rate limiting, exponential backoff with jitter, circuit breakers, adaptive per-workload request pacing, body-read timeouts, and connection recycling. * **Operations:** Streamed JSONL exports with checkpoint/resume, delta sync token management, and resilience injection for existing Graph SDK scripts through `Enable-MgxResilience`. +* **Content:** File and media downloads with `Get-MgxContent`, whole or by byte range, over a redirect path validated against a host allowlist and followed without the bearer token. * **Observability:** Execution metrics via `Get-MgxTelemetry`, including HTTP timing, throttle waits, retries, and resource consumption. --- ## Benchmarks -> **Environment:** Entra ID test tenant with 100k users, 15.4k groups, and 705 service principals. PowerShell 7.4 with Microsoft.Graph SDK 2.34. +> **Environment:** Entra ID test tenant with 100k users and 15.8k groups. PowerShell 7.6.4 (.NET 10) with the Microsoft.Graph SDK, app-only via certificate. The module's supported floor is 7.4 (.NET 8), which is verified by the test suite but is not where these figures were measured. Every row below comes from one run of the suite against one tenant on one build; rows that could not be re-measured have been removed rather than carried forward. ### Performance & Throughput -| Operation | Mgx | SDK (`Get-MgUser`) | Raw REST (`Invoke-RestMethod`) | Speedup vs SDK | -| ---------------------------------------- | ---------: | ---------------------: | -----------------------------: | ----------------: | -| **List 100,000 users** | **78.2s** | 505.0s¹ | 153.3s | **6.5×** | -| **Look up 5,000 users by ID** | **422.0s** | 2,183.0s | 5,500s² | **5.2×** | -| **User report** *(1k users + groups)* | **87.8s** | 450.0s | 1,132.0s | **5.1×** | -| **Create 10,000 users** | **510.0s** | 6,158.0s | Failed *(dead socket hang)* | **12.1×** | -| **Recurring delta sync** *(200 changes)* | **0.9s** | 84.5s *(full re-pull)* | Manual | **94× less work** | - -¹ SDK measured at its default 100-item page size. Passing `-PageSize 999` reduces the SDK result to 84.5s; Mgx remains slightly faster without additional configuration. - -² The baseline completed 3,472 of 5,000 lookups before its token expired. Three attempts failed in different ways. - -### Resilience Under Faults & Throttling - -**Fault injection:** 15% `429` throttling and 3% `503` errors across 1,000 lookups. - -* **Mgx:** Completed **1,000 / 1,000** in **43.2s** with 0 failures. -* **Bare SDK:** Completed **1,000 / 1,000** in **211.7s**. -* **Naive REST:** Completed in **0.9s**, but silently dropped **180 items**. -* **SDK + `Enable-MgxResilience`:** 100% completion with the same wall time as the bare SDK when no faults occur. -* **Adaptive write pacing:** Dynamically adjusts throughput during throttling. In a 77,000-object seeding campaign, Mgx executed 3,710 requests with **0 lost objects** and **0 circuit-breaker trips**. -* **Connection recovery:** Per-request body-read timeouts and periodic connection recycling prevent long-running operations from remaining stuck on dead sockets. +| Operation | Mgx | SDK (`Get-MgUser`) | Raw REST (`Invoke-RestMethod`) | Speedup vs SDK | +| ------------------------------------- | ---------: | -----------------: | -----------------------------: | -------------: | +| **List 100,000 users** | **47.1s** | 53.2s¹ | 58.5s | 1.1× | +| **Look up 5,000 users by ID** | **98.8s** | 521.0s | 985.3s | **5.3×** | +| **User report** *(1k users + groups)* | **23.9s** | 107.9s | 205.8s | **4.5×** | +| **Full delta enumeration** *(130,233 items)* | **145.6s** | - | - | - | + +¹ Both figures are the SDK at `-PageSize 999`, which is what the benchmark runs. At the SDK's *default* page size the same enumeration takes **about 2.5-3x** as long - measured 3.0x (168.2s against 57.0s), 2.66x and 2.52x on separate runs of the same tenant, which is the spread this figure has — the practical difference is that mgx needs no tuning to be fast, not that it out-runs a tuned SDK on plain enumeration. + +Graph charges resource units per request, by the shape of the query rather than by which client +sent it - so every column above spends the same: about 102 units for the enumeration, 5,002 for +the lookups, 2,003 for the report. That equality holds only because all three issue equivalent +requests - the page size noted above changes the request count, and the cost with it. +What mgx adds is not a cheaper request but a visible one: it accumulates `x-ms-resource-unit` +across the session and paces against the budget they are charged to. See +[Resource units: what queries actually cost](#resource-units-what-queries-actually-cost). + +Measured from `x-ms-resource-unit`. `/users/{id}` costs 1 RU; `transitiveMembers` with `$select` and `$top` costs 3, matching the documented cost table exactly. At 5,002 RU the heaviest row above spends about 1.3% of the 8,000 RU / 10s Identity & Access budget for one application + tenant pair. The delta endpoint does not emit the header, so no figure is given. + +### Resilience under throttling + +Enumerating all 100,000 users of a test tenant while a second client held the application's +resource-unit budget at its ceiling. Ground truth is `/users/$count`, which the directory +serves from its own index rather than by walking pages, so it cannot inherit a pagination +fault from the thing it is being used to check. + +| Contender | Retrieved | Missing | Duplicates | Wall time | +| ---------------------------------- | --------------------: | ------: | ---------: | --------: | +| `Invoke-MgxRequest -All` | **100,000** / 100,000 | 0 | 0 | 715.2s | +| `Get-MgUser -All` | **100,000** / 100,000 | 0 | 0 | 729.9s | +| `Invoke-RestMethod` + fixed retry | 8,000 / 100,000 | 92,000 | 0 | 120.0s | +| `Invoke-RestMethod`, no retry | 1,700 / 100,000 | 98,300 | 0 | 5.5s | + +The bottom row finishes in five and a half seconds holding 1.7% of the tenant. Both the SDK +and Mgx return everything - the SDK's retry handler is correct here, and fifteen seconds +between them across twelve minutes is not a difference worth reading into. What separates the +table is whether a contender honors `Retry-After` at all, not which library it comes from. + +Run again with the budget untouched, all four return exactly 100,000 with no duplicates. +The gap is produced by throttling, not by enumeration. + +Adding `Enable-MgxResilience` to the SDK neither helps nor hurts here, because the SDK's own +retry handler already covers throttling. What it adds is not covered by this table: per-request +body-read timeouts and periodic connection recycling, so a dead socket surfaces as a retryable +error rather than an indefinite hang. The benchmark suite runs its SDK baselines inside +watchdogged child processes for exactly that reason - bare SDK cmdlets have no default +body-read timeout and have hung indefinitely during these runs. + +### Delta enumerations repeat themselves + +A delta run does not paginate a snapshot, so one object can arrive on several pages of the +same enumeration. Against a tenant of 15,779 groups, over 1,135 pages: + +| Contender | Emitted | Distinct | +| ----------------------------------------- | ------: | -------: | +| `Invoke-RestMethod` over `/groups/delta` | 172,740 | 17,467 | +| `Sync-MgxDelta` over the same resource | 172,740 | 17,467 | + +Ten objects handed back for every one the directory holds. Mgx does not deduplicate this: it +is 126 seconds quicker over the same pages and exactly as repetitive. Deduplicating correctly +means choosing which occurrence of an id wins when a single run reports the same object both +live and `@removed`, and which occurrence is last is not knowable while streaming without +retaining every id seen. Until that is settled, dedupe downstream on `id`. + +Distinct exceeds the group count because a full delta also returns tombstones for deleted +groups - 16,324 of the objects above carry `@removed`. + +### What adaptive pacing costs when nothing is throttling + +It depends entirely on how long the run is, and the honest answer is "nothing on a long one, +a lot on a short one". + +The pacer opens cold at 4 rps and doubles each clean second until it reaches the ceiling, then +deactivates. A run long enough to finish ramping pays for the ramp once and never notices. A run +that finishes *during* it pays for the whole thing. Both measured on the 100k-user tenant: + +| workload | pacing on *(default)* | `-NoAdaptivePacing` | measured by | +| --- | ---: | ---: | --- | +| 8,000 concurrent reads | 39.4s | 40.3s | [benchmark 07](tests/benchmarks/07-adaptive-pacing.ps1) | +| 50 lookups, concurrency 8, cold session | 4.4s | 1.0s | [benchmark 14](tests/benchmarks/14-pacing-cold-cost.ps1) | + +The large run is free - 0 pacing activations, 0 throttle retries, and the paced run is if +anything the faster of the two. The small one costs **4-6x** depending on the day (4.6x above; +separate runs have given 4.4x and about 6x), and the reason is visible in telemetry: roughly 30 +seconds of accumulated `AdaptivePacingWaitMs` spread across a fan-out that would otherwise take +one. If your work is short, interactive, and against a tenant you are not hammering, +`Set-MgxOption -NoAdaptivePacing` is the switch. + +No 429s appeared at any concurrency the module permits. It caps at 128, and this tenant took +700 RU/s for 90s without refusing anything - a raw client only crosses the ceiling at around 200 +requests in flight, which is past what Mgx will issue. Graph absorbed the load by inflating latency +from 113ms to ~1s instead. That is why pacing reacts to `Retry-After` and latency drift rather than +to a header that never arrives. ### Memory Efficiency & Recovery @@ -60,11 +134,11 @@ Export 100,000 users to JSONL: | | Mgx `Export-MgxCollection` | Mgx `Invoke-MgxRequest \| file` | `Invoke-RestMethod` buffer+write | | ------------------ | -------------------------: | ------------------------------: | -------------------------------: | -| Wall time | 74.0s | 76.0s | 157.2s | -| Peak working set | 341MB | **225MB** | **504MB** | -| Managed heap delta | +19.8MB | **+8.0MB** | **+428.9MB** | +| Wall time | 53.0s | 47.0s | 64.0s | +| Peak working set | 305MB | **265MB** | **555MB** | +| Managed heap delta | +19.8MB | **+5.0MB** | **+415.7MB** | -* **Low memory footprint:** JSONL export peaks as low as **225MB RAM**, compared with 504MB for buffered REST. +* **Low memory footprint:** streaming to a file adds **5MB** to the managed heap over the whole export, against 416MB for the buffer-then-write approach. Peak working set is 265MB against 555MB. * **Kill-safe resume:** `Export-MgxCollection` checkpoints progress continuously. After interruption, rerunning the export resumes from the checkpoint without duplicating objects. --- @@ -114,6 +188,14 @@ Invoke-MgxRequest /users -Top 50 | # Delta sync with automatic token handling Sync-MgxDelta /users/delta -DeltaPath ./delta.json -Property displayName,mail + +# Enumerate a whole drive, resumable mid-run; later runs return only changes +Sync-MgxDelta /me/drive/root/delta -DeltaPath ./drive.json -CheckpointPath ./drive.cp -OutputFile drive.jsonl + +# Ranged reads: first 256 KB of each file instead of the whole thing +Invoke-MgxRequest "/me/drive/items/$folderId/children" -All | + Where-Object { $_.file } | + Get-MgxContent -First 262144 ``` See [`examples/`](examples/) for additional examples. @@ -127,9 +209,9 @@ See [`examples/`](examples/) for additional examples. | **Bulk Lookups** | `$ids \| ForEach-Object { Get-MgUser -UserId $_ }` | `$ids \| Invoke-MgxRequest '/users/{id}'` | | **Bulk Updates** | `$ids \| ForEach-Object { Update-MgUser ... }` | `$urls \| Invoke-MgxBatchRequest -Method PATCH -Body @{...}` | | **Exporting Data** | `$all = Get-MgUser -All; $all \| Export-Csv ...` | `Export-MgxCollection /users -OutputFile users.jsonl` | -| **Fault Protection** | Manual script handling required | Built-in or via `Enable-MgxResilience` | -| **Observability** | No retry/throttle metrics | `Get-MgxTelemetry` | -| **Dead Connection Handling** | Indefinite hangs possible | Body-read timeouts + connection recycling | +| **Fault Protection** | Written per script | Built-in, or added to existing scripts with `Enable-MgxResilience` | +| **Observability** | Timed by the caller | `Get-MgxTelemetry` | +| **Dead Connection Handling** | Left to the default HTTP timeouts | Body-read timeouts + connection recycling | | **Beta Endpoints** | Requires `Microsoft.Graph.Beta` | `-ApiVersion beta` | --- @@ -171,7 +253,8 @@ Invoke-MgxRequest /users -All -Raw | ConvertFrom-Json | `Invoke-MgxBatchRequest` | Batches up to 20 requests into a single HTTP POST call | | `Export-MgxCollection` | Streams paginated API results directly to JSONL with checkpointing | | `Expand-MgxRelation` | Performs concurrent fan-out lookups to expand related object attributes | -| `Sync-MgxDelta` | Manages stateful delta queries and state token storage | +| `Sync-MgxDelta` | Manages stateful delta queries and state token storage, with mid-run crash resume | +| `Get-MgxContent` | Downloads content bytes (whole or ranged) with a token-free, host-validated download path | | `Set-MgxOption` / `Get-MgxOption` | Configures global limits, retry counts, timeouts, and circuit-breaker thresholds | | `Enable-MgxResilience` / `Disable-MgxResilience` | Injects or removes Mgx resilience policies from native Microsoft.Graph SDK cmdlets | | `Get-MgxTelemetry` | Outputs execution statistics including HTTP duration, rate-limit delays, retry counts, and resource units | @@ -187,7 +270,31 @@ All HTTP operations pass through four layered [Polly 8.x](https://github.com/App 3. **Circuit breaker:** Opens when error rates exceed 10%, with a half-open probe after 15 seconds. 4. **Timeout:** 30-second per-request limit and 300-second cumulative ceiling across retries. -Batch writes additionally use an adaptive AIMD pacing engine to adjust throughput during throttling. +### Resource units: what queries actually cost + +Graph throttles directory workloads on a **resource-unit budget**, not on request count or bandwidth. Mgx reads `x-ms-resource-unit` on every response and accumulates it, so `Get-MgxTelemetry` reports what a session actually spent - see [examples/resilience-and-telemetry/resource-unit-budgeting.ps1](examples/resilience-and-telemetry/resource-unit-budgeting.ps1). + +Measured against a 15,779-group test tenant: + +| Query shape | RU | +|---|---| +| `transitiveMembers?$top=5` | 4 | +| `transitiveMembers?$top=5&$select=id` | **3** | +| `groups/{id}` (single read) | 1 | + +Both figures match the [documented cost table](https://learn.microsoft.com/en-us/graph/throttling-limits) exactly: `transitiveMembers` is published at 5 RU, `$select` takes one off, and `$top` under 20 takes another. Cost is a property of the query *shape*, not the number of objects returned. On a per-group fan-out across that tenant, `$select` alone is the difference between 47,337 and 63,116 resource units. + +Three findings from pushing a single client until the tenant pushed back: + +- **The budget behaves like a token bucket, and burst hides its rate.** The published limit for a tenant of this size is 8,000 RU per 10 s per *application + tenant pair* (800 RU/s). Held at a fixed rate for 90 s from a single client: 300 and 700 RU/s were never refused - both spending well past the bucket - while 1,200 RU/s began refusing after 19 s, and once saturated the tenant served about **730 RU/s** however hard it was pushed. A short run at any of those rates shows nothing - the bucket starts full, so roughly the first 20,000 RU are free regardless of how fast they are spent, which is why a burst measurement reads as a much higher sustained ceiling than the tenant actually has. The ceiling itself moves: the day before, the same probe was refused at 631 RU/s and saturated near 440. Re-measure with [`tests/benchmarks/13-resource-unit-rate.ps1`](tests/benchmarks/13-resource-unit-rate.ps1) rather than trusting these constants. +- **`x-ms-throttle-limit-percentage` was never emitted** - not at 1.5x the documented budget, and not while the tenant was actively returning 429s. Mgx therefore treats it as opportunistic and paces on 429 + `Retry-After` and latency drift, which are reliable. +- **Batching does not buy cheaper units.** Unbatched requests sustained a *higher* RU/s before throttling than the same requests inside `$batch`. Batching saves round-trips, not budget. + +Limits are scoped per **application + tenant pair**, and separately per service - Intune, Excel, Education and the rest each carry their own quota. That is why pacing state is partitioned by workload rather than pooled: a throttled directory fan-out says nothing about the budget remaining for Teams. + +A failed request costs 0 RU - so a fan-out that fails uniformly consumes no budget, triggers no throttling, and finishes fast while measuring nothing. Check status codes, not duration. + +Ahead of those four, an **adaptive pacing gate** spaces requests before they are sent. It is on by default and spaces requests on both the `Invoke-Mgx*` path and the `Enable-MgxResilience` SDK-bridge path. It learns a per-workload rate from throttling signals (additive increase, multiplicative decrease), keeps Drive, Directory and other workloads in separate buckets so one throttled workload does not slow the rest, and starts conservatively on a cold session. The learning happens only on the `Invoke-Mgx*` path: the SDK's own retry handler sits inside the wrapped chain and answers a 429 before the pipeline sees it, so a bridged session keeps spacing but never adapts. Batch outer POSTs are the one exemption - `GraphBatchClient` runs its own item-level AIMD, and stacking two controllers on one workload would compound their backoff. Disable with `Set-MgxOption -NoAdaptivePacing`; inspect the learned state in `Get-MgxTelemetry`. --- diff --git a/build.ps1 b/build.ps1 index e6b341a..fdcf96a 100644 --- a/build.ps1 +++ b/build.ps1 @@ -19,7 +19,11 @@ Write-Host "Building Mgx ($Configuration)..." -ForegroundColor Cyan # checked before the compile, failing in a second instead of a minute. $manifestVersion = (Import-PowerShellDataFile (Join-Path $ModuleRoot 'mgx.psd1')).ModuleVersion $propsFile = Join-Path $PSScriptRoot 'Directory.Build.props' -$propsVersion = ([xml](Get-Content $propsFile -Raw)).Project.PropertyGroup.Version +# Select-Object -First on a filtered list: .Project.PropertyGroup.Version returns an ARRAY the +# moment a second top-level exists, and comparing an array to a string does not +# fail loudly - it just stops gating. +$propsVersion = @(([xml](Get-Content $propsFile -Raw)).Project.PropertyGroup.Version | + Where-Object { $_ }) | Select-Object -First 1 if (-not $propsVersion) { throw "Version gate failed: no found in $propsFile" } @@ -44,7 +48,7 @@ if ($LASTEXITCODE -ne 0) { throw "Build failed with exit code $LASTEXITCODE" } # Copy Cmdlets + Engine DLLs (load in default ALC) # Detect TFM from csproj instead of hardcoding (survives TFM upgrades) $csproj = [xml](Get-Content "$PSScriptRoot/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj") -$tfm = $csproj.Project.PropertyGroup.TargetFramework +$tfm = @($csproj.Project.PropertyGroup.TargetFramework | Where-Object { $_ }) | Select-Object -First 1 $CmdletsOutput = Join-Path $PSScriptRoot "src/Mgx.Cmdlets/bin/$Configuration/$tfm" Copy-Item "$CmdletsOutput/Mgx.Cmdlets.dll" $ModuleRoot -Force Copy-Item "$CmdletsOutput/Mgx.Cmdlets.pdb" $ModuleRoot -Force -ErrorAction SilentlyContinue @@ -100,18 +104,27 @@ if ($orphans) { throw "Module integrity check failed: Mgx assemblies found in Dependencies/ (should only be in root): $($orphans.Name -join ', ')" } -# Write build hash for staleness detection -$hashInputFiles = @( - (Join-Path $ModuleRoot 'Mgx.Cmdlets.dll'), - (Join-Path $ModuleRoot 'Mgx.Engine.dll') -) | Where-Object { Test-Path $_ } -$combinedHash = ($hashInputFiles | ForEach-Object { (Get-FileHash $_ -Algorithm SHA256).Hash }) -join '|' -$buildStamp = @{ - Hash = (Get-FileHash -InputStream ([System.IO.MemoryStream]::new([System.Text.Encoding]::UTF8.GetBytes($combinedHash))) -Algorithm SHA256).Hash - Timestamp = (Get-Date -Format 'o') - Configuration = $Configuration +# Compiled help is generated from module/help/*.md, and nothing regenerated it: the shipped +# MAML drifted two releases behind its own source, so Get-Help documented output shapes removed +# in 2.0.0, denied a parameter added in 2.1.0, and gave a range the cmdlet rejects. Regenerating +# on every build is what stops the two diverging again. +$helpSource = Join-Path $PSScriptRoot 'module/help' +$helpOutput = Join-Path $ModuleRoot 'en-US' +if (Test-Path $helpSource) { + # Already-imported counts: -ListAvailable only searches PSModulePath, so a platyPS loaded + # from an explicit path (Save-Module into a folder, then Import-Module by path - which is how + # a CI box without a profile gets one) looked absent and failed the build. + if ((Get-Module platyPS) -or (Get-Module -ListAvailable platyPS)) { + if (-not (Get-Module platyPS)) { Import-Module platyPS -ErrorAction Stop } + New-ExternalHelp -Path $helpSource -OutputPath $helpOutput -Force | Out-Null + Write-Host "Regenerated compiled help from module/help" -ForegroundColor DarkGray + } + else { + throw ("platyPS is not installed, so module/en-US/Mgx.Cmdlets.dll-Help.xml cannot be " + + "regenerated and would ship stale - which is how Get-Help came to describe output " + + "shapes removed two releases earlier. Install-Module platyPS -Scope CurrentUser.") + } } -$buildStamp | ConvertTo-Json | Set-Content (Join-Path $ModuleRoot '.build-hash') -Force Write-Host "`nBuild complete!" -ForegroundColor Green Write-Host "Module output: $ModuleRoot" -ForegroundColor Yellow diff --git a/examples/01-get-all-users.ps1 b/examples/01-get-all-users.ps1 deleted file mode 100644 index ced884c..0000000 --- a/examples/01-get-all-users.ps1 +++ /dev/null @@ -1,12 +0,0 @@ -# Get all users from your tenant and display them in a table. -# -# The Microsoft.Graph SDK equivalent (Get-MgUser -All) buffers every user -# in memory before returning. This streams results to the pipeline as each -# page arrives, keeping memory constant regardless of tenant size. -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All" - -Import-Module Mgx - -Invoke-MgxRequest /users -All -Property displayName,mail,department,jobTitle | - Format-Table displayName, mail, department, jobTitle -AutoSize diff --git a/examples/03-get-managers-fan-out.ps1 b/examples/03-get-managers-fan-out.ps1 deleted file mode 100644 index 9307f95..0000000 --- a/examples/03-get-managers-fan-out.ps1 +++ /dev/null @@ -1,18 +0,0 @@ -# Get the manager for every user in the tenant using concurrent fan-out. -# -# The naive approach is a foreach loop: one HTTP call per user. -# With {id} template substitution, Invoke-MgxRequest dispatches all -# requests concurrently (default concurrency: 10) and streams results -# back as they complete. -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All" - -Import-Module Mgx - -# Get all users first, then fan-out to fetch each manager concurrently. -$users = Invoke-MgxRequest /users -All -Property id,displayName - -$users | - Invoke-MgxRequest '/users/{id}/manager' -SkipNotFound | - Select-Object id, displayName | - Format-Table -AutoSize diff --git a/examples/05-delta-sync.ps1 b/examples/05-delta-sync.ps1 deleted file mode 100644 index 853dc8b..0000000 --- a/examples/05-delta-sync.ps1 +++ /dev/null @@ -1,18 +0,0 @@ -# Incremental sync: only fetch users that changed since the last run. -# -# The first run downloads all users and saves a delta token to disk. -# Every subsequent run fetches only changes (new, modified, deleted) -# since the previous sync. Token management is automatic. -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All" - -Import-Module Mgx - -$deltaFile = "./users-delta.json" - -$changes = Sync-MgxDelta /users/delta ` - -DeltaPath $deltaFile ` - -Property id,displayName,mail,accountEnabled - -Write-Host "Changes since last sync: $($changes.Count)" -$changes | Format-Table id, displayName, mail, accountEnabled -AutoSize diff --git a/examples/07-enrich-users-with-manager.ps1 b/examples/07-enrich-users-with-manager.ps1 deleted file mode 100644 index 453a34b..0000000 --- a/examples/07-enrich-users-with-manager.ps1 +++ /dev/null @@ -1,14 +0,0 @@ -# Enrich a list of users with their manager's display name. -# -# Expand-MgxRelation concurrently fetches the related data for each -# piped object and attaches it as a new property. This replaces a -# manual foreach loop with N serial HTTP calls. -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All" - -Import-Module Mgx - -Invoke-MgxRequest /users -Top 50 -Property id,displayName,mail | - Expand-MgxRelation '/users/{id}/manager' -As Manager -Flatten -SkipNotFound | - Select-Object displayName, mail, @{ n='Manager'; e={ $_.Manager.displayName } } | - Format-Table -AutoSize diff --git a/examples/11-beta-endpoint.ps1 b/examples/11-beta-endpoint.ps1 deleted file mode 100644 index 2af4bcc..0000000 --- a/examples/11-beta-endpoint.ps1 +++ /dev/null @@ -1,17 +0,0 @@ -# Access beta Graph endpoints without installing Microsoft.Graph.Beta.*. -# -# The -ApiVersion parameter switches between v1.0 (default) and beta -# on the same cmdlet, with the same resilience and pagination support. -# No extra module installs needed. -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All" - -Import-Module Mgx - -# Get users with beta-only properties (e.g. customSecurityAttributes) -$users = Invoke-MgxRequest /users ` - -ApiVersion beta ` - -Top 10 ` - -Property id,displayName,customSecurityAttributes - -$users | Format-Table id, displayName -AutoSize diff --git a/examples/13-disabled-accounts-report.ps1 b/examples/13-disabled-accounts-report.ps1 deleted file mode 100644 index c6f9192..0000000 --- a/examples/13-disabled-accounts-report.ps1 +++ /dev/null @@ -1,18 +0,0 @@ -# Report all disabled user accounts in the tenant. -# -# Uses server-side filtering so only matching records are transferred. -# -ConsistencyLevel eventual enables advanced query support required -# for certain filter expressions on large tenants. -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All" - -Import-Module Mgx - -$disabled = Invoke-MgxRequest /users ` - -All ` - -Filter "accountEnabled eq false" ` - -Property displayName,mail,userPrincipalName,createdDateTime ` - -ConsistencyLevel eventual - -Write-Host "Disabled accounts: $($disabled.Count)" -$disabled | Sort-Object createdDateTime | Format-Table displayName, mail, createdDateTime -AutoSize diff --git a/examples/18-tune-rate-limits.ps1 b/examples/18-tune-rate-limits.ps1 deleted file mode 100644 index 9e53e7d..0000000 --- a/examples/18-tune-rate-limits.ps1 +++ /dev/null @@ -1,26 +0,0 @@ -# Tune resilience options for a specific workload. -# -# Set-MgxOption lets you adjust the rate limiter, retry count, circuit -# breaker, and timeouts at runtime without restarting the session. -# Only the parameters you pass are changed; everything else stays. -# -# Use Get-MgxOption to inspect current settings. -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All" - -Import-Module Mgx - -# View current defaults -Get-MgxOption - -# Slow down for a tenant that throttles aggressively -Set-MgxOption -BatchItemsPerSecond 5 -MaxRetryAttempts 10 - -# Run your workload -$users = Invoke-MgxRequest /users -All -Property id,displayName -Write-Host "Users: $($users.Count)" - -# Restore all defaults when done -Set-MgxOption -Reset - -Get-MgxOption diff --git a/examples/20-conditional-access-export.ps1 b/examples/20-conditional-access-export.ps1 deleted file mode 100644 index 32c9830..0000000 --- a/examples/20-conditional-access-export.ps1 +++ /dev/null @@ -1,20 +0,0 @@ -# Export all Conditional Access policies to JSON. -# -# CA policies are only available on the beta endpoint. -ApiVersion beta -# accesses them without installing any extra modules. -# -# Requirements: Connect-MgGraph -Scopes "Policy.Read.All" - -Import-Module Mgx - -$policies = Invoke-MgxRequest /identity/conditionalAccess/policies ` - -All ` - -ApiVersion beta ` - -Property id,displayName,state,createdDateTime,modifiedDateTime - -Write-Host "Conditional Access policies: $($policies.Count)" -$policies | Sort-Object displayName | Format-Table displayName, state, modifiedDateTime -AutoSize - -# Export full policy details to JSON for backup/audit -$policies | ConvertTo-Json -Depth 10 | Out-File "./conditional-access-policies.json" -Write-Host "Full export saved to conditional-access-policies.json" diff --git a/examples/22-mixed-endpoint-batch.ps1 b/examples/22-mixed-endpoint-batch.ps1 deleted file mode 100644 index a5ad77e..0000000 --- a/examples/22-mixed-endpoint-batch.ps1 +++ /dev/null @@ -1,37 +0,0 @@ -# Query multiple Graph endpoints in a single HTTP round-trip. -# -# Invoke-MgxBatchRequest accepts mixed URLs, methods, and bodies -# in one batch. This avoids serial requests when you need data -# from unrelated endpoints (users, groups, apps, skus, etc.). -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" - -Import-Module Mgx - -# Mixed GET: pull a snapshot from four different endpoints at once -$results = @( - "/users?`$top=5&`$select=displayName,userPrincipalName" - "/groups?`$top=5&`$select=displayName,groupTypes" - "/applications?`$top=5&`$select=displayName,appId" - "/subscribedSkus" -) | Invoke-MgxBatchRequest - -Write-Host "Batch returned $($results.Count) responses" -$results | ForEach-Object { - $name = $_.displayName ?? $_.skuPartNumber ?? '(collection)' - Write-Host " $name" -} - -# Mixed methods: read some things from different endpoints, all in one call -$requests = @( - [PSCustomObject]@{ Url = "/me"; Method = "GET" } - [PSCustomObject]@{ Url = "/organization"; Method = "GET" } - [PSCustomObject]@{ Url = "/users?`$top=1"; Method = "GET" } -) -$mixed = $requests | Invoke-MgxBatchRequest - -Write-Host "`nMixed-method batch:" -$mixed | ForEach-Object { - $name = $_.displayName ?? '(response)' - Write-Host " $name" -} diff --git a/examples/README.md b/examples/README.md index 5f15e16..cfd3022 100644 --- a/examples/README.md +++ b/examples/README.md @@ -2,6 +2,8 @@ Runnable scripts covering common Microsoft Graph scenarios. Each script is self-contained and includes the required scopes in its header comment. +Scripts are grouped by purpose - pick the directory that matches what you are trying to do. + ## Prerequisites ```powershell @@ -12,63 +14,68 @@ Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All" # adjust scopes per ## Scripts -### Users +### Getting Started (`getting-started/`) | Script | Description | Scopes | |--------|-------------|--------| -| [01-get-all-users.ps1](01-get-all-users.ps1) | Stream all users to the console | `User.Read.All` | -| [02-export-users-to-jsonl.ps1](02-export-users-to-jsonl.ps1) | Export all users to a JSONL file with checkpoint/resume | `User.Read.All` | -| [03-get-managers-fan-out.ps1](03-get-managers-fan-out.ps1) | Fetch every user's manager concurrently | `User.Read.All` | -| [07-enrich-users-with-manager.ps1](07-enrich-users-with-manager.ps1) | Attach manager as a property on each user object | `User.Read.All` | -| [13-disabled-accounts-report.ps1](13-disabled-accounts-report.ps1) | Report all disabled accounts | `User.Read.All` | -| [14-guest-users-report.ps1](14-guest-users-report.ps1) | Report all guest (external) accounts | `User.Read.All` | -| [17-bulk-delete-whatif.ps1](17-bulk-delete-whatif.ps1) | Bulk delete stale guests with `-WhatIf` preview | `User.ReadWrite.All` | -| [19-chained-relation-expansion.ps1](19-chained-relation-expansion.ps1) | Enrich users with manager + licenses in one pass | `User.Read.All` | +| [get-all-users.ps1](getting-started/get-all-users.ps1) | Stream all users to the console | `User.Read.All` | +| [export-users-to-jsonl.ps1](getting-started/export-users-to-jsonl.ps1) | Export all users to a JSONL file with checkpoint/resume | `User.Read.All` | +| [beta-endpoint.ps1](getting-started/beta-endpoint.ps1) | Access beta endpoints without installing extra modules | `User.Read.All` | -### Groups +### Fan-Out & Relations (`fan-out-and-relations/`) | Script | Description | Scopes | |--------|-------------|--------| -| [12-group-members-multipage.ps1](12-group-members-multipage.ps1) | Stream all members from all groups concurrently | `Group.Read.All`, `User.Read.All` | +| [get-managers-fan-out.ps1](fan-out-and-relations/get-managers-fan-out.ps1) | Fetch every user's manager concurrently | `User.Read.All` | +| [enrich-users-with-manager.ps1](fan-out-and-relations/enrich-users-with-manager.ps1) | Attach manager as a property on each user object | `User.Read.All` | +| [chained-relation-expansion.ps1](fan-out-and-relations/chained-relation-expansion.ps1) | Enrich users with manager + licenses in one pass | `User.Read.All` | +| [group-members-multipage.ps1](fan-out-and-relations/group-members-multipage.ps1) | Stream all members from all groups concurrently | `Group.Read.All`, `User.Read.All` | -### Devices & Apps +### Reporting & Audit (`reporting-and-audit/`) | Script | Description | Scopes | |--------|-------------|--------| -| [15-stale-devices-report.ps1](15-stale-devices-report.ps1) | Report devices inactive for 90+ days | `Device.Read.All` | -| [16-app-secrets-expiry.ps1](16-app-secrets-expiry.ps1) | Find app secrets and certificates expiring within 30 days | `Application.Read.All` | -| [20-conditional-access-export.ps1](20-conditional-access-export.ps1) | Export all Conditional Access policies (beta endpoint) | `Policy.Read.All` | +| [disabled-accounts-report.ps1](reporting-and-audit/disabled-accounts-report.ps1) | Report all disabled accounts | `User.Read.All` | +| [guest-users-report.ps1](reporting-and-audit/guest-users-report.ps1) | Report all guest (external) accounts | `User.Read.All` | +| [stale-devices-report.ps1](reporting-and-audit/stale-devices-report.ps1) | Report devices inactive for 90+ days | `Device.Read.All` | +| [app-secrets-expiry.ps1](reporting-and-audit/app-secrets-expiry.ps1) | Find app secrets and certificates expiring within 30 days | `Application.Read.All` | +| [conditional-access-export.ps1](reporting-and-audit/conditional-access-export.ps1) | Export all Conditional Access policies (beta endpoint) | `Policy.Read.All` | +| [export-sign-in-logs.ps1](reporting-and-audit/export-sign-in-logs.ps1) | Export sign-in logs to JSONL with checkpoint/resume | `AuditLog.Read.All` | -### Audit Logs +### Bulk Writes & Batching (`bulk-writes-and-batching/`) | Script | Description | Scopes | |--------|-------------|--------| -| [08-export-sign-in-logs.ps1](08-export-sign-in-logs.ps1) | Export sign-in logs to JSONL with checkpoint/resume | `AuditLog.Read.All` | +| [bulk-update.ps1](bulk-writes-and-batching/bulk-update.ps1) | PATCH multiple users via `$batch` (20 per HTTP call) | `User.ReadWrite.All` | +| [dead-letter-retry.ps1](bulk-writes-and-batching/dead-letter-retry.ps1) | Bulk create users with dead-letter tracking for failures | `User.ReadWrite.All` | +| [bulk-delete-whatif.ps1](bulk-writes-and-batching/bulk-delete-whatif.ps1) | Bulk delete stale guests with `-WhatIf` preview | `User.ReadWrite.All` | +| [mixed-endpoint-batch.ps1](bulk-writes-and-batching/mixed-endpoint-batch.ps1) | Query users, groups, apps, and SKUs in one HTTP call | `User.Read.All`, `Group.Read.All`, `Application.Read.All` | -### Bulk Operations +### Delta & Sync (`delta-and-sync/`) | Script | Description | Scopes | |--------|-------------|--------| -| [06-bulk-update.ps1](06-bulk-update.ps1) | PATCH multiple users via `$batch` (20 per HTTP call) | `User.ReadWrite.All` | -| [09-dead-letter-retry.ps1](09-dead-letter-retry.ps1) | Bulk create users with dead-letter tracking for failures | `User.ReadWrite.All` | -| [22-mixed-endpoint-batch.ps1](22-mixed-endpoint-batch.ps1) | Query users, groups, apps, and SKUs in one HTTP call | `User.Read.All`, `Group.Read.All`, `Application.Read.All` | +| [delta-sync.ps1](delta-and-sync/delta-sync.ps1) | Full sync on first run, incremental changes thereafter | `User.Read.All` | +| [drive-delta-resumable.ps1](delta-and-sync/drive-delta-resumable.ps1) | Resumable OneDrive/SharePoint delta sync with checkpointing | `Files.Read.All` | +| [tenant-change-feed.ps1](delta-and-sync/tenant-change-feed.ps1) | A resumable tenant-wide change feed, baselined with `-Latest` | `Group.Read.All`, `Application.Read.All` | -### Delta Sync +### Drive Content (`drive-content/`) | Script | Description | Scopes | |--------|-------------|--------| -| [05-delta-sync.ps1](05-delta-sync.ps1) | Full sync on first run, incremental changes thereafter | `User.Read.All` | +| [partial-content-hash.ps1](drive-content/partial-content-hash.ps1) | Identify files by hashing a byte range instead of downloading them whole | `Files.Read.All` | +| [drive-content-triage.ps1](drive-content/drive-content-triage.ps1) | Sniff 4 KB of each file, then download only what survives triage | `Files.Read.All` | -### Resilience & Configuration +### Resilience & Telemetry (`resilience-and-telemetry/`) | Script | Description | Scopes | |--------|-------------|--------| -| [04-resilience-for-existing-scripts.ps1](04-resilience-for-existing-scripts.ps1) | Add retry/circuit breaker to existing SDK scripts (zero code changes) | `User.Read.All` | -| [21-resilience-status.ps1](21-resilience-status.ps1) | Check, enable, and disable SDK resilience injection | `User.Read.All` | -| [10-telemetry.ps1](10-telemetry.ps1) | View request counts, retries, and throttle events for the session | `User.Read.All` | -| [11-beta-endpoint.ps1](11-beta-endpoint.ps1) | Access beta endpoints without installing extra modules | `User.Read.All` | -| [18-tune-rate-limits.ps1](18-tune-rate-limits.ps1) | Tune rate limiter, retry count, and timeouts at runtime | `User.Read.All` | -| [23-benchmark-resilience.ps1](23-benchmark-resilience.ps1) | Benchmark: bare SDK vs MgxResilience vs Invoke-MgxRequest vs Export | `User.Read.All` | +| [resilience-for-existing-scripts.ps1](resilience-and-telemetry/resilience-for-existing-scripts.ps1) | Add retry/circuit breaker to existing SDK scripts (zero code changes) | `User.Read.All` | +| [resilience-status.ps1](resilience-and-telemetry/resilience-status.ps1) | Check, enable, and disable SDK resilience injection | `User.Read.All` | +| [telemetry.ps1](resilience-and-telemetry/telemetry.ps1) | View request counts, retries, and throttle events for the session | `User.Read.All` | +| [tune-rate-limits.ps1](resilience-and-telemetry/tune-rate-limits.ps1) | Tune rate limiter, retry count, and timeouts at runtime | `User.Read.All` | +| [resource-unit-budgeting.ps1](resilience-and-telemetry/resource-unit-budgeting.ps1) | Measure what query shapes cost in resource units, and size a fan-out from the measurement | `Group.Read.All`, `User.Read.All` | +| [pacing-observability.ps1](resilience-and-telemetry/pacing-observability.ps1) | Watch the adaptive pacer: slow start, per-workload buckets, and which throttle signals are real | `User.Read.All`, `Files.Read.All` | ## Cmdlet Coverage @@ -76,14 +83,15 @@ Every Mgx cmdlet is demonstrated in at least one script: | Cmdlet | Scripts | |--------|---------| -| `Invoke-MgxRequest` | 01, 03, 12, 13, 14, 15, 17, 19, 20 | -| `Invoke-MgxBatchRequest` | 06, 09, 22 | -| `Export-MgxCollection` | 02, 08, 23 | -| `Expand-MgxRelation` | 07, 19 | -| `Sync-MgxDelta` | 05 | -| `Enable-MgxResilience` | 04, 21, 23 | -| `Disable-MgxResilience` | 21 | -| `Get-MgxResilience` | 21 | -| `Set-MgxOption` | 18 | -| `Get-MgxOption` | 18 | -| `Get-MgxTelemetry` | 10, 23 | +| `Invoke-MgxRequest` | get-all-users, beta-endpoint, get-managers-fan-out, enrich-users-with-manager, chained-relation-expansion, group-members-multipage, disabled-accounts-report, guest-users-report, stale-devices-report, app-secrets-expiry, conditional-access-export, bulk-delete-whatif, partial-content-hash, drive-content-triage, telemetry, tune-rate-limits, tenant-resilience-comparison, resource-unit-budgeting, pacing-observability | +| `Invoke-MgxBatchRequest` | bulk-update, dead-letter-retry, mixed-endpoint-batch | +| `Export-MgxCollection` | export-users-to-jsonl, export-sign-in-logs, tenant-resilience-comparison | +| `Expand-MgxRelation` | enrich-users-with-manager, chained-relation-expansion | +| `Sync-MgxDelta` | delta-sync, drive-delta-resumable, tenant-change-feed | +| `Enable-MgxResilience` | resilience-for-existing-scripts, resilience-status, tenant-resilience-comparison | +| `Disable-MgxResilience` | resilience-status, tenant-resilience-comparison | +| `Get-MgxResilience` | resilience-status | +| `Set-MgxOption` | tune-rate-limits, pacing-observability | +| `Get-MgxOption` | tune-rate-limits | +| `Get-MgxTelemetry` | telemetry, tenant-resilience-comparison, resource-unit-budgeting, pacing-observability, partial-content-hash, drive-content-triage | +| `Get-MgxContent` | partial-content-hash, drive-content-triage | diff --git a/examples/17-bulk-delete-whatif.ps1 b/examples/bulk-writes-and-batching/bulk-delete-whatif.ps1 similarity index 69% rename from examples/17-bulk-delete-whatif.ps1 rename to examples/bulk-writes-and-batching/bulk-delete-whatif.ps1 index 2b74be2..1d19bd5 100644 --- a/examples/17-bulk-delete-whatif.ps1 +++ b/examples/bulk-writes-and-batching/bulk-delete-whatif.ps1 @@ -25,7 +25,20 @@ if ($targets.Count -eq 0) { return } -$targets | Format-Table displayName, mail, createdDateTime -AutoSize +$targets | Select-Object displayName, mail, createdDateTime | Format-Table -AutoSize # Preview - remove -WhatIf to execute $targets | Invoke-MgxRequest '/users/{id}' -Method DELETE -WhatIf + +<# +Expected output: + +Targets: 2 + +displayName mail createdDateTime +----------- ---- --------------- +Bianca Pisani bianca.pisani@fabrikam.com 7/3/2025 8:13:54 PM +Raul Razo raul.razo@woodgrovebank.com 12/30/2025 11:43:39 AM + +What if: Performing the operation "Bulk write" on target "DELETE 2 items via /users/{id}". +#> diff --git a/examples/06-bulk-update.ps1 b/examples/bulk-writes-and-batching/bulk-update.ps1 similarity index 95% rename from examples/06-bulk-update.ps1 rename to examples/bulk-writes-and-batching/bulk-update.ps1 index 5cdc768..8718dc5 100644 --- a/examples/06-bulk-update.ps1 +++ b/examples/bulk-writes-and-batching/bulk-update.ps1 @@ -24,3 +24,9 @@ $results = $userIds ` $ok = ($results | Where-Object { $_.Status -lt 400 }).Count $fail = ($results | Where-Object { $_.Status -ge 400 }).Count Write-Host "Updated: $ok Failed: $fail" + +<# +Expected output: + +Updated: 3 Failed: 0 +#> diff --git a/examples/09-dead-letter-retry.ps1 b/examples/bulk-writes-and-batching/dead-letter-retry.ps1 similarity index 62% rename from examples/09-dead-letter-retry.ps1 rename to examples/bulk-writes-and-batching/dead-letter-retry.ps1 index 76909df..cd5e14e 100644 --- a/examples/09-dead-letter-retry.ps1 +++ b/examples/bulk-writes-and-batching/dead-letter-retry.ps1 @@ -41,3 +41,28 @@ if (Test-Path $deadLetterFile) { $retry | Invoke-MgxBatchRequest Remove-Item $deadLetterFile } + +<# +Expected output: + +Invoke-MgxBatchRequest: /tmp/mgx-examples/dead-letter-retry.ps1:33 +Line | + 33 | ... = $requests | Invoke-MgxBatchRequest -DeadLetterPath $deadLetterFile + | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + | POST /users: Request_BadRequest: Another object with the same value for property + | userPrincipalName already exists. +... +WARNING: 10 of 50 batch items failed after all retry attempts. Check $Error for details on each failed item. +Created: 40 +Failed: 10 +Retrying failed items... + +Name Value +---- ----- +Status 400 +Url /users +Body {[error, System.Collections.Hashtable]} +Method POST +... +WARNING: 10 of 10 batch items failed after all retry attempts. Check $Error for details on each failed item. +#> diff --git a/examples/bulk-writes-and-batching/mixed-endpoint-batch.ps1 b/examples/bulk-writes-and-batching/mixed-endpoint-batch.ps1 new file mode 100644 index 0000000..4c0010d --- /dev/null +++ b/examples/bulk-writes-and-batching/mixed-endpoint-batch.ps1 @@ -0,0 +1,61 @@ +# Query multiple Graph endpoints in a single HTTP round-trip. +# +# Invoke-MgxBatchRequest accepts mixed URLs, methods, and bodies +# in one batch. This avoids serial requests when you need data +# from unrelated endpoints (users, groups, apps, skus, etc.). +# +# Requirements: Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" + +Import-Module Mgx + +# Mixed GET: pull a snapshot from four different endpoints at once +$results = @( + "/users?`$top=5&`$select=displayName,userPrincipalName" + "/groups?`$top=5&`$select=displayName,groupTypes" + "/applications?`$top=5&`$select=displayName,appId" + "/subscribedSkus" +) | Invoke-MgxBatchRequest + +Write-Host "Batch returned $($results.Count) responses" +# A batch result is a Hashtable of Url / Method / Status / Body - the entity itself lives under +# Body. Reading $_.displayName straight off the result finds nothing, and a Hashtable returns +# $null for a missing key rather than erroring, so the mistake prints a fallback string forever +# instead of failing loudly. +$results | ForEach-Object { + $count = if ($_.Body -and $_.Body.value) { @($_.Body.value).Count } else { 0 } + Write-Host (" {0,-3} {1,-46} {2} item(s)" -f $_.Status, $_.Url, $count) +} + +# Mixed methods: read some things from different endpoints, all in one call +$requests = @( + [PSCustomObject]@{ Url = "/me"; Method = "GET" } + [PSCustomObject]@{ Url = "/organization"; Method = "GET" } + [PSCustomObject]@{ Url = "/users?`$top=1"; Method = "GET" } +) +$mixed = $requests | Invoke-MgxBatchRequest + +Write-Host "`nMixed-method batch:" +$mixed | ForEach-Object { + # Single-entity responses put the object directly in Body; collections nest it under + # Body.value. Checking for value distinguishes the two without guessing per endpoint. + $summary = if ($_.Body -and $_.Body.value) { "$(@($_.Body.value).Count) item(s)" } + elseif ($_.Body -and $_.Body.displayName) { $_.Body.displayName } + elseif ($_.Body -and $_.Body.id) { $_.Body.id } + else { '(no body)' } + Write-Host (" {0,-6} {1,-3} {2,-24} {3}" -f $_.Method, $_.Status, $_.Url, $summary) +} + +<# +Expected output: + +Batch returned 4 responses + 200 /users?$top=5&$select=displayName,userPrincipalName 5 item(s) + 200 /groups?$top=5&$select=displayName,groupTypes 5 item(s) + 200 /applications?$top=5&$select=displayName,appId 5 item(s) + 200 /subscribedSkus 3 item(s) + +Mixed-method batch: + GET 200 /me MOD Administrator + GET 200 /organization 1 item(s) + GET 200 /users?$top=1 1 item(s) +#> diff --git a/examples/delta-and-sync/delta-sync.ps1 b/examples/delta-and-sync/delta-sync.ps1 new file mode 100644 index 0000000..5bc8b59 --- /dev/null +++ b/examples/delta-and-sync/delta-sync.ps1 @@ -0,0 +1,34 @@ +# Incremental sync: only fetch users that changed since the last run. +# +# The first run downloads all users and saves a delta token to disk. +# Every subsequent run fetches only changes (new, modified, deleted) +# since the previous sync. Token management is automatic. +# +# Requirements: Connect-MgGraph -Scopes "User.Read.All" + +Import-Module Mgx + +$deltaFile = "./users-delta.json" + +$changes = Sync-MgxDelta /users/delta ` + -DeltaPath $deltaFile ` + -Property id,displayName,mail,accountEnabled + +Write-Host "Changes since last sync: $($changes.Count)" +$changes | Select-Object id, displayName, mail, accountEnabled | Format-Table -AutoSize + +<# +Expected output: + +Changes since last sync: 28 + +id displayName mail accountEnabled +-- ----------- ---- -------------- +05c76d47-f457-4be9-bf0a-a75b57af73c1 Adele Vance adelev@contoso.com True +b92d487c-5d51-460b-963a-c395ddc747ba Alex Wilber alexw@contoso.com True +02109083-4f10-4e5e-ab8b-17ec81caa674 Allan Deyoung allande@contoso.com True +aea6b38c-19f7-4225-adb4-feeeea0b4f20 Brian Johnson brianj@contoso.com False +d1e24810-61ae-41fb-a250-88e2672cb4e1 Cameron White cameronw@contoso.com False +27640233-6c77-4337-b2f9-693608bfd60f Christie Cline christiec@contoso.com True +... +#> diff --git a/examples/delta-and-sync/drive-delta-resumable.ps1 b/examples/delta-and-sync/drive-delta-resumable.ps1 new file mode 100644 index 0000000..c0f999b --- /dev/null +++ b/examples/delta-and-sync/drive-delta-resumable.ps1 @@ -0,0 +1,49 @@ +# Enumerate an entire OneDrive/SharePoint drive to JSONL - resumable. +# +# Drive enumerations at scale run for hours; a crash, Ctrl+C, or network +# death should not restart them from zero. -CheckpointPath saves the +# position at every page boundary, so re-running the identical command +# continues where the last run stopped. On successful completion the +# checkpoint is deleted and the delta token is saved: the NEXT run +# returns only what changed - the incremental backbone for backup-shaped +# workloads. +# +# Requirements: Connect-MgGraph -Scopes "Files.Read.All" + +Import-Module Mgx + +$deltaFile = "./drive-delta.json" +$checkpointFile = "./drive-delta.checkpoint" +$outputFile = "./drive-items.jsonl" + +# First run: full enumeration (resumable). Later runs: changes only. +Sync-MgxDelta /me/drive/root/delta ` + -DeltaPath $deltaFile ` + -CheckpointPath $checkpointFile ` + -OutputFile $outputFile ` + -Prefer deltashowremovedasdeleted ` + -Verbose + +# The JSONL is one driveItem per line. Deleted items carry the "deleted" +# facet (because of the Prefer token above); resume is at-least-once, so +# deduplicate on id if this feeds a downstream system: +$items = Get-Content $outputFile | ConvertFrom-Json +$live = $items | Where-Object { -not $_.deleted } | Sort-Object id -Unique +Write-Host "$($live.Count) live items, $(@($items | Where-Object deleted).Count) deleted" + +# Tip - track changes WITHOUT the initial enumeration: baseline from now +# with -Latest, then every later run returns only changes: +# Sync-MgxDelta /me/drive/root/delta -DeltaPath $deltaFile -Latest + +<# +Expected output: + +VERBOSE: No existing delta state. Performing full initial sync. +VERBOSE: Delta state saved to '/tmp/tenant-sync/drive-delta.json'. +VERBOSE: Delta sync complete: 11 items in 0.2s. Output: /tmp/tenant-sync/drive-items.jsonl +10 live items, 1 deleted + +drive-items.jsonl (first 2 lines): +{"id":"01381BR3EHVNURTDV4B1GIK7R5GN7ON4SL","name":"Q4-Revenue-Report.pdf","size":2418912,"lastM... +{"id":"019NF2ODIGGGJDG696J7PRMO65G9CEUEJ2","name":"Meeting-Notes.txt","size":12480,"lastModifie... +#> diff --git a/examples/delta-and-sync/tenant-change-feed.ps1 b/examples/delta-and-sync/tenant-change-feed.ps1 new file mode 100644 index 0000000..9cb4756 --- /dev/null +++ b/examples/delta-and-sync/tenant-change-feed.ps1 @@ -0,0 +1,82 @@ +# A tenant change feed: everything that changed since the last run, resumable. +# +# Baseline once with -Latest (instant - no initial enumeration), then each +# scheduled run appends only the changes to a JSONL log. Delta state makes +# it incremental; -CheckpointPath makes a killed run continue instead of +# restarting. "What changed in the tenant since yesterday" as a cron job. +# +# Requirements: Connect-MgGraph -Scopes "Group.Read.All","Application.Read.All" + +Import-Module Mgx + +$stateDir = "./tenant-feed" +New-Item -ItemType Directory -Path $stateDir -Force | Out-Null + +# --- groups: membership and lifecycle changes --- +# +# Baseline with -Latest on the very first run. This matters more than it looks: without it +# the first call enumerates every group in the tenant, and Graph does not promise one +# response per object. Measured on a 15,779-group tenant, a full enumeration emitted +# 156,413 objects - the same groups repeated ~10x across pages, which is explicitly allowed +# ("can't ensure that entities are unified in a single response"). -Latest skips all of it +# and starts the feed from now, which is what a change feed actually wants. +$groupState = "$stateDir/groups.delta" +if (-not (Test-Path $groupState)) { + Write-Host "First run: baselining groups from now (no initial enumeration)." + $null = Sync-MgxDelta /groups/delta -DeltaPath $groupState -Latest +} + +$groupChanges = Sync-MgxDelta /groups/delta ` + -DeltaPath $groupState ` + -CheckpointPath "$stateDir/groups.checkpoint" + +# Dedup by id, last write wins. Incremental rounds are usually clean, but the replay above +# is a property of delta itself, not of the initial page - so anything that treats one +# emitted object as one change event needs this. Keep the LAST occurrence: for an object +# that changed twice in one window, that is the current state. +$latestById = [ordered]@{} +foreach ($g in $groupChanges) { $latestById[[string]$g.id] = $g } +foreach ($g in $latestById.Values) { + $g | ConvertTo-Json -Compress -Depth 10 | Add-Content "$stateDir/groups-feed.jsonl" +} +if ($groupChanges.Count -ne $latestById.Count) { + Write-Host (" collapsed {0} emitted objects into {1} distinct groups" -f ` + $groupChanges.Count, $latestById.Count) +} + +# --- service principals: the security variant - new app registrations, +# credential changes, and deletions stand out in this feed --- +$spChanges = Sync-MgxDelta /servicePrincipals/delta ` + -DeltaPath "$stateDir/sp.delta" ` + -CheckpointPath "$stateDir/sp.checkpoint" + +$new = $spChanges | Where-Object { -not $_.'@removed' -and $_.appId } +$removed = $spChanges | Where-Object { $_.'@removed' } +foreach ($sp in $spChanges) { + $sp | ConvertTo-Json -Compress -Depth 10 | Add-Content "$stateDir/sp-feed.jsonl" +} + +Write-Host ("Groups changed: {0} | Service principals: {1} changed, {2} removed" -f ` + $groupChanges.Count, $new.Count, $removed.Count) +if ($new.Count -gt 0) { + Write-Host "Review new/changed service principals:" + $new | Select-Object appDisplayName, appId, accountEnabled | Format-Table -AutoSize +} + +<# +Expected output: + +First run: baselining groups from now (no initial enumeration). +Groups changed: 3 | Service principals: 3 changed, 0 removed +Review new/changed service principals: + +appDisplayName appId accountEnabled +-------------- ----- -------------- +Contoso Expense Portal dbc7b719-a215-42e9-bf03-7f78957e21a9 True +Contoso HR Sync 01d9b6b1-564d-4c5d-843f-10c58eccdffd True +Contoso Reporting Service 14b89564-33a9-402c-ab8e-2bc84a7164ab False + +tenant-feed/groups-feed.jsonl (first 2 lines): +{"displayName":"Mark 8 Project Team","groupTypes":["Unified"],"id":"5cf66b99-8a06-436a-8107-644... +{"displayName":"Engineering All Hands","groupTypes":["Unified"],"id":"24ffeed6-38b4-4bee-ae29-f... +#> diff --git a/examples/drive-content/drive-content-triage.ps1 b/examples/drive-content/drive-content-triage.ps1 new file mode 100644 index 0000000..debb7d4 --- /dev/null +++ b/examples/drive-content/drive-content-triage.ps1 @@ -0,0 +1,155 @@ +# Triage a drive by reading 4 KB of each file, then download only what survives triage. +# +# +# Two properties of Get-MgxContent make this cheap and safe: +# +# * A ranged read is a Range request. -First 4096 transfers 4 KB, not 4 KB of a 2 GB download +# you abandon. Some endpoints ignore Range and answer 200 with the whole body (profile +# photos do); mgx detects that and truncates client-side, so you never receive more than you +# asked for either way. +# * Piping a DriveItem uses its own @microsoft.graph.downloadUrl - a pre-authenticated URL - +# so there is no Graph round trip at all for the bytes. No resource units, and no bearer +# token sent to the download host. That URL is itself a credential for the file, which is +# why mgx redacts it from -Debug output. +# +# Requirements: Connect-MgGraph -Scopes "Files.Read.All" +# Read-only against Graph. Writes only to the local -OutputDirectory. + +param( + [string]$DrivePath = '/me/drive', + [int]$SniffBytes = 4096, + [int]$MaxFiles = 40, + [string]$OutputDirectory = './triaged' +) + +Import-Module Mgx + +# Magic-byte signatures. Enough to separate "worth fetching" from "skip" without a full download. +$Signatures = @( + @{ Name = 'PNG'; Bytes = [byte[]](0x89,0x50,0x4E,0x47) } + @{ Name = 'JPEG'; Bytes = [byte[]](0xFF,0xD8,0xFF) } + @{ Name = 'PDF'; Bytes = [byte[]](0x25,0x50,0x44,0x46) } + @{ Name = 'ZIP'; Bytes = [byte[]](0x50,0x4B,0x03,0x04) } # also docx/xlsx/pptx + @{ Name = 'GZIP'; Bytes = [byte[]](0x1F,0x8B) } + @{ Name = 'SQLite';Bytes = [byte[]](0x53,0x51,0x4C,0x69,0x74,0x65) } +) + +function Get-FileKind { + param([byte[]]$Head) + foreach ($sig in $Signatures) { + if ($Head.Length -ge $sig.Bytes.Length) { + $match = $true + for ($i = 0; $i -lt $sig.Bytes.Length; $i++) { + if ($Head[$i] -ne $sig.Bytes[$i]) { $match = $false; break } + } + if ($match) { return $sig.Name } + } + } + # No signature: decide whether it is plausibly text by looking for NUL bytes, which almost + # never appear in text and almost always appear in binary. + if ($Head.Length -gt 0 -and ($Head | Where-Object { $_ -eq 0 }).Count -eq 0) { return 'TEXT' } + return 'UNKNOWN' +} + +Write-Host "Enumerating $DrivePath ..." -ForegroundColor Cyan +$items = @(Invoke-MgxRequest "$DrivePath/root/children" -All -PageSize 200 ` + -Property id,name,size,file | + Where-Object { $_.ContainsKey('file') } | + Select-Object -First $MaxFiles) + +if (-not $items) { Write-Warning "No files found under $DrivePath/root."; return } +" $($items.Count) file(s) to triage" + +$before = (Get-MgxTelemetry).ContentBytesDownloaded +$results = [System.Collections.Generic.List[object]]::new() + +foreach ($item in $items) { + # Ranged read of the head only. On a 2 GB file this transfers 4 KB. + try { + $head = Get-MgxContent "$DrivePath/items/$($item.id)/content" -First $SniffBytes -ErrorAction Stop + } + catch { + $results.Add([pscustomobject]@{ + Name = $item.name; Size = $item.size; Kind = 'ERROR'; Sniffed = 0 + Note = ($_.Exception.Message -replace '\s+', ' ') + }) + continue + } + + $kind = Get-FileKind -Head $head + $results.Add([pscustomobject]@{ + Name = $item.name; Size = $item.size; Kind = $kind; Sniffed = $head.Length + Id = $item.id; Note = '' + }) +} + +Write-Host "`n=== Triage ===" -ForegroundColor Cyan +$results | Sort-Object Kind, Name | + Format-Table Name, @{N='Size';E={'{0:N0}' -f $_.Size};A='Right'}, + Kind, @{N='Read';E={$_.Sniffed};A='Right'}, Note -AutoSize + +# --- fetch only what triage selected --- +# Change this predicate to whatever your triage actually is. The point is that the decision was +# made from 4 KB per file rather than from downloading everything first. +$wanted = $results | Where-Object { $_.Kind -in 'PDF','TEXT' -and $_.Size -gt 0 -and $_.Size -lt 8MB } + +$sniffBytesTotal = ($results | Measure-Object Sniffed -Sum).Sum +$allBytes = ($results | Measure-Object Size -Sum).Sum + +Write-Host "`n=== Fetching $($wanted.Count) selected file(s) whole ===" -ForegroundColor Cyan +if ($wanted) { + New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null + foreach ($w in $wanted) { + $dest = Join-Path $OutputDirectory ($w.Name -replace '[\\/:*?"<>|]', '_') + # -OutFile streams to a temp and moves atomically, so an interrupted download never + # leaves a half-written file at the destination. + Get-MgxContent "$DrivePath/items/$($w.Id)/content" -OutFile $dest + " {0,-44} {1,10:N0} bytes" -f $w.Name, (Get-Item $dest).Length + } +} +else { " nothing matched the triage predicate" } + +$downloaded = (Get-MgxTelemetry).ContentBytesDownloaded - $before + +Write-Host "`n=== What triage saved ===" -ForegroundColor Cyan +" files examined : {0:N0}" -f $results.Count +" total size on drive : {0:N0} bytes" -f $allBytes +" read for triage : {0:N0} bytes ({1:P2} of the total)" -f $sniffBytesTotal, ($(if($allBytes){$sniffBytesTotal/$allBytes}else{0})) +" transferred overall : {0:N0} bytes (triage + the {1} full downloads)" -f $downloaded, $wanted.Count +" resource units : {0}" -f (Get-MgxTelemetry).ResourceUnitsConsumed +'' +" Scale that up: reading 4 KB of 112,000 files is ~460 MB regardless of whether those files" +" total 60 GB or 600 GB. The saving is the ratio, and it grows with the library." + +<# +Expected output: + +Enumerating /me/drive ... + 6 file(s) to triage + +=== Triage === + +Name Size Kind Read Note +---- ---- ---- ---- ---- +Q4-Revenue-Report.pdf 2,418,912 PDF 4096 +Contoso-Logo.png 148,326 PNG 4096 +Meeting-Notes.txt 12,480 TEXT 4096 +placeholder.txt 0 UNKNOWN 0 +telemetry.bin 65,536 UNKNOWN 4096 +Campaign-Assets.zip 5,242,880 ZIP 4096 + + +=== Fetching 2 selected file(s) whole === + Q4-Revenue-Report.pdf 2,418,912 bytes + Meeting-Notes.txt 12,480 bytes + +=== What triage saved === + files examined : 6 + total size on drive : 7,888,134 bytes + read for triage : 20,480 bytes (0.26% of the total) + transferred overall : 2,451,872 bytes (triage + the 2 full downloads) + resource units : 9 + + Scale that up: reading 4 KB of 112,000 files is ~460 MB regardless of whether those files + total 60 GB or 600 GB. The saving is the ratio, and it grows with the library. +#> diff --git a/examples/drive-content/partial-content-hash.ps1 b/examples/drive-content/partial-content-hash.ps1 new file mode 100644 index 0000000..f96e9d4 --- /dev/null +++ b/examples/drive-content/partial-content-hash.ps1 @@ -0,0 +1,48 @@ +# Identify files by content WITHOUT downloading them: ranged reads. +# +# Get-MgxContent: a metadata-extraction job +# over 20k drive files needed only each file's first 256 KB (magic bytes, +# EXIF, partial hash) - ranged reads moved ~5 GB instead of 63 GB. +# +# Requirements: Connect-MgGraph -Scopes "Files.Read.All" + +Import-Module Mgx + +$folderId = "01FOLDERID" # a folder's driveItem id + +Invoke-MgxRequest "/me/drive/items/$folderId/children" -All | + Where-Object { $_.file } | + ForEach-Object { + $head = $_ | Get-MgxContent -First 262144 + + [PSCustomObject]@{ + Name = $_.name + Size = $_.size + Magic = [BitConverter]::ToString($head[0..([Math]::Min(7, $head.Length - 1))]) + PartialSha = [BitConverter]::ToString( + [System.Security.Cryptography.SHA256]::HashData($head) + ).Replace('-', '').ToLowerInvariant() + # Graph reports quickXorHash in BASE64; rclone and most tools print + # HEX. Convert before comparing across tools: + QuickXorHex = if ($_.file.hashes.quickXorHash) { + [System.Convert]::ToHexString( + [System.Convert]::FromBase64String($_.file.hashes.quickXorHash) + ).ToLowerInvariant() + } + } + } | Format-Table -AutoSize + +# Bytes moved vs. bytes identified: +(Get-MgxTelemetry).ContentBytesDownloaded + +<# +Expected output: + +Name Size Magic PartialSha QuickXorHex +---- ---- ----- ---------- ----------- +Employee-Handbook.pdf 1204224 25-50-44-46-2D-31-2E-37 fcdec4adc77794abd24402810879c26beaee6f95f515b42be79fc032e69b6f5c e3bc4fcdfa266aadc7e2148a472720e3877b5de6 +README.txt 5120 61-62-63-64-65-66-67-68 a31ba217553a0e1b431276ab8ee0d0df43b958686c77041ec6d2eb859943e02d f9e21c1a070d2467204274dcec5d93d20613645d +Team-Photo.png 742912 89-50-4E-47-0D-0A-1A-0A bc3891906da735372e276ea14e9edcf82ee78e402a19ad70872b2b410d7fb2bf 51c7556a48958d776b8ff3ac36d2b5a5e8071fec + +529408 +#> diff --git a/examples/19-chained-relation-expansion.ps1 b/examples/fan-out-and-relations/chained-relation-expansion.ps1 similarity index 63% rename from examples/19-chained-relation-expansion.ps1 rename to examples/fan-out-and-relations/chained-relation-expansion.ps1 index a072875..d9bdbf1 100644 --- a/examples/19-chained-relation-expansion.ps1 +++ b/examples/fan-out-and-relations/chained-relation-expansion.ps1 @@ -18,3 +18,17 @@ foreach ($user in $enriched) { if (-not $licenses) { $licenses = "(none)" } Write-Host "$($user.displayName) | Manager: $mgr | Licenses: $licenses" } + +<# +Expected output: + +Adele Vance | Manager: Nestor Wilke | Licenses: SPE_E5 +Alex Wilber | Manager: Megan Bowen | Licenses: SPE_E5 +Allan Deyoung | Manager: Nestor Wilke | Licenses: SPE_E5, EMSPREMIUM +Brian Johnson | Manager: Lee Gu | Licenses: SPE_E3 +Cameron White | Manager: Adele Vance | Licenses: SPE_E3 +Christie Cline | Manager: Nestor Wilke | Licenses: SPE_E5 +Debra Berger | Manager: Nestor Wilke | Licenses: SPE_E5 +Delia Dennis | Manager: Allan Deyoung | Licenses: SPE_E5, EMSPREMIUM +... +#> diff --git a/examples/fan-out-and-relations/enrich-users-with-manager.ps1 b/examples/fan-out-and-relations/enrich-users-with-manager.ps1 new file mode 100644 index 0000000..29a648a --- /dev/null +++ b/examples/fan-out-and-relations/enrich-users-with-manager.ps1 @@ -0,0 +1,32 @@ +# Enrich a list of users with their manager's display name. +# +# Expand-MgxRelation concurrently fetches the related data for each +# piped object and attaches it as a new property. This replaces a +# manual foreach loop with N serial HTTP calls. +# +# Requirements: Connect-MgGraph -Scopes "User.Read.All" + +Import-Module Mgx + +Invoke-MgxRequest /users -Top 50 -Property id,displayName,mail | + Expand-MgxRelation '/users/{id}/manager' -As Manager -Flatten -SkipNotFound | + Select-Object displayName, mail, @{ n='Manager'; e={ $_.Manager.displayName } } | + Format-Table -AutoSize + +<# +Expected output: + +WARNING: Skipped 4 entities due to 404 (Not Found) responses. + +displayName mail Manager +----------- ---- ------- +Adele Vance adelev@contoso.com Nestor Wilke +Alex Wilber alexw@contoso.com Megan Bowen +Allan Deyoung allande@contoso.com Nestor Wilke +Brian Johnson brianj@contoso.com Lee Gu +Cameron White cameronw@contoso.com Adele Vance +Christie Cline christiec@contoso.com Nestor Wilke +Debra Berger debrab@contoso.com Nestor Wilke +Delia Dennis deliad@contoso.com Allan Deyoung +... +#> diff --git a/examples/fan-out-and-relations/get-managers-fan-out.ps1 b/examples/fan-out-and-relations/get-managers-fan-out.ps1 new file mode 100644 index 0000000..7fdefd8 --- /dev/null +++ b/examples/fan-out-and-relations/get-managers-fan-out.ps1 @@ -0,0 +1,36 @@ +# Get the manager for every user in the tenant using concurrent fan-out. +# +# The naive approach is a foreach loop: one HTTP call per user. +# With {id} template substitution, Invoke-MgxRequest dispatches all +# requests concurrently (default concurrency: 10) and streams results +# back as they complete. +# +# Requirements: Connect-MgGraph -Scopes "User.Read.All" + +Import-Module Mgx + +# Get all users first, then fan-out to fetch each manager concurrently. +$users = Invoke-MgxRequest /users -All -Property id,displayName + +$users | + Invoke-MgxRequest '/users/{id}/manager' -SkipNotFound | + Select-Object id, displayName | + Format-Table -AutoSize + +<# +Expected output: + +WARNING: This endpoint may only be available in beta. Retry with -ApiVersion beta. +WARNING: Skipped 4 entities due to 404 (Not Found) responses. +id displayName +-- ----------- +6426ded0-36f8-4bf7-9fe4-1353bd002dbe Nestor Wilke +2d871c04-0885-491c-bcbb-f14ab6f4a375 Megan Bowen +6426ded0-36f8-4bf7-9fe4-1353bd002dbe Nestor Wilke +c5151c70-bc16-4934-acf2-1a776694b6af Lee Gu +05c76d47-f457-4be9-bf0a-a75b57af73c1 Adele Vance +6426ded0-36f8-4bf7-9fe4-1353bd002dbe Nestor Wilke +6426ded0-36f8-4bf7-9fe4-1353bd002dbe Nestor Wilke +02109083-4f10-4e5e-ab8b-17ec81caa674 Allan Deyoung +... +#> diff --git a/examples/12-group-members-multipage.ps1 b/examples/fan-out-and-relations/group-members-multipage.ps1 similarity index 83% rename from examples/12-group-members-multipage.ps1 rename to examples/fan-out-and-relations/group-members-multipage.ps1 index a5bbd41..732c6e2 100644 --- a/examples/12-group-members-multipage.ps1 +++ b/examples/fan-out-and-relations/group-members-multipage.ps1 @@ -14,3 +14,13 @@ $allMembers = Invoke-MgxRequest /groups -All -Property id,displayName | Write-Host "Total members across all groups: $($allMembers.Count)" $allMembers | Group-Object '@odata.type' | Format-Table Name, Count -AutoSize + +<# +Expected output: + +Total members across all groups: 30 + +Name Count +---- ----- +#microsoft.graph.user 30 +#> diff --git a/examples/getting-started/beta-endpoint.ps1 b/examples/getting-started/beta-endpoint.ps1 new file mode 100644 index 0000000..3ad801f --- /dev/null +++ b/examples/getting-started/beta-endpoint.ps1 @@ -0,0 +1,34 @@ +# Access beta Graph endpoints without installing Microsoft.Graph.Beta.*. +# +# The -ApiVersion parameter switches between v1.0 (default) and beta +# on the same cmdlet, with the same resilience and pagination support. +# No extra module installs needed. +# +# Requirements: Connect-MgGraph -Scopes "User.Read.All" + +Import-Module Mgx + +# Get users with beta-only properties (e.g. customSecurityAttributes) +$users = Invoke-MgxRequest /users ` + -ApiVersion beta ` + -Top 10 ` + -Property id,displayName,customSecurityAttributes + +$users | Select-Object id, displayName | Format-Table -AutoSize + +<# +Expected output: + +id displayName +-- ----------- +05c76d47-f457-4be9-bf0a-a75b57af73c1 Adele Vance +b92d487c-5d51-460b-963a-c395ddc747ba Alex Wilber +02109083-4f10-4e5e-ab8b-17ec81caa674 Allan Deyoung +aea6b38c-19f7-4225-adb4-feeeea0b4f20 Brian Johnson +d1e24810-61ae-41fb-a250-88e2672cb4e1 Cameron White +27640233-6c77-4337-b2f9-693608bfd60f Christie Cline +af9691ee-cc4e-4748-8646-4605b2921a9c Debra Berger +29d7068e-0922-41e9-aa7a-7e97f477f67b Delia Dennis +f736ca7a-71d9-4708-8aae-3cfa2111a106 Diego Siciliani +26f05c3d-0bcd-4160-9789-cbea7b676305 Emily Braun +#> diff --git a/examples/02-export-users-to-jsonl.ps1 b/examples/getting-started/export-users-to-jsonl.ps1 similarity index 72% rename from examples/02-export-users-to-jsonl.ps1 rename to examples/getting-started/export-users-to-jsonl.ps1 index 25028c2..812646b 100644 --- a/examples/02-export-users-to-jsonl.ps1 +++ b/examples/getting-started/export-users-to-jsonl.ps1 @@ -21,3 +21,13 @@ $result = Export-MgxCollection /users ` -Property id,displayName,mail,department,jobTitle,accountEnabled Write-Host "Exported $($result.ItemCount) users to $outputFile" + +<# +Expected output: + +Exported 28 users to ./users.jsonl + +users.jsonl (first 2 lines): +{"displayName":"Adele Vance","jobTitle":"Retail Manager","accountEnabled":true,"department":"Re... +{"displayName":"Alex Wilber","jobTitle":"Marketing Assistant","accountEnabled":true,"department... +#> diff --git a/examples/getting-started/get-all-users.ps1 b/examples/getting-started/get-all-users.ps1 new file mode 100644 index 0000000..6874180 --- /dev/null +++ b/examples/getting-started/get-all-users.ps1 @@ -0,0 +1,29 @@ +# Get all users from your tenant and display them in a table. +# +# The Microsoft.Graph SDK equivalent (Get-MgUser -All) buffers every user +# in memory before returning. This streams results to the pipeline as each +# page arrives, keeping memory constant regardless of tenant size. +# +# Requirements: Connect-MgGraph -Scopes "User.Read.All" + +Import-Module Mgx + +Invoke-MgxRequest /users -All -Property displayName,mail,department,jobTitle | + Select-Object displayName, mail, department, jobTitle | + Format-Table -AutoSize + +<# +Expected output: + +displayName mail department jobTitle +----------- ---- ---------- -------- +Adele Vance adelev@contoso.com Retail Retail Manager +Alex Wilber alexw@contoso.com Marketing Marketing Assistant +Allan Deyoung allande@contoso.com IT Corporate Security Officer +Brian Johnson brianj@contoso.com Engineering Senior Engineer +Cameron White cameronw@contoso.com Retail Store Associate +Christie Cline christiec@contoso.com Human Resources Benefits Specialist +Debra Berger debrab@contoso.com Operations Administrative Assistant +Delia Dennis deliad@contoso.com IT Support Engineer +... +#> diff --git a/examples/16-app-secrets-expiry.ps1 b/examples/reporting-and-audit/app-secrets-expiry.ps1 similarity index 65% rename from examples/16-app-secrets-expiry.ps1 rename to examples/reporting-and-audit/app-secrets-expiry.ps1 index b5be1e5..8579c64 100644 --- a/examples/16-app-secrets-expiry.ps1 +++ b/examples/reporting-and-audit/app-secrets-expiry.ps1 @@ -20,9 +20,9 @@ $expiring = foreach ($app in $apps) { if ($expiry -le $warningDate) { [PSCustomObject]@{ App = $app.displayName - Type = if ($cred.PSObject.Properties.Name -contains 'secretText') { 'Secret' } else { 'Certificate' } + Type = if ($cred.ContainsKey('secretText')) { 'Secret' } else { 'Certificate' } ExpiresOn = $expiry.ToString('yyyy-MM-dd') - DaysLeft = [math]::Floor(($expiry - (Get-Date)).TotalDays) + DaysLeft = [int][math]::Floor(($expiry - (Get-Date)).TotalDays) } } } @@ -33,3 +33,14 @@ if ($expiring) { } else { Write-Host "No credentials expiring within 30 days." } + +<# +Expected output: + +App Type ExpiresOn DaysLeft +--- ---- --------- -------- +Contoso Partner Portal Certificate 2026-08-23 2 +Contoso Expense Portal Secret 2026-08-26 5 +Contoso HR Sync Certificate 2026-09-10 20 +Contoso Field Service App Secret 2026-09-17 27 +#> diff --git a/examples/reporting-and-audit/conditional-access-export.ps1 b/examples/reporting-and-audit/conditional-access-export.ps1 new file mode 100644 index 0000000..b6ce2e6 --- /dev/null +++ b/examples/reporting-and-audit/conditional-access-export.ps1 @@ -0,0 +1,45 @@ +# Export all Conditional Access policies to JSON. +# +# CA policies are only available on the beta endpoint. -ApiVersion beta +# accesses them without installing any extra modules. +# +# Requirements: Connect-MgGraph -Scopes "Policy.Read.All" + +Import-Module Mgx + +$policies = Invoke-MgxRequest /identity/conditionalAccess/policies ` + -All ` + -ApiVersion beta ` + -Property id,displayName,state,createdDateTime,modifiedDateTime + +Write-Host "Conditional Access policies: $($policies.Count)" +$policies | Sort-Object displayName | Select-Object displayName, state, modifiedDateTime | Format-Table -AutoSize + +# Export full policy details to JSON for backup/audit +$policies | ConvertTo-Json -Depth 10 | Out-File "./conditional-access-policies.json" +Write-Host "Full export saved to conditional-access-policies.json" + +<# +Expected output: + +Conditional Access policies: 4 + +displayName state modifiedDateTime +----------- ----- ---------------- +Block legacy authentication enabled 7/20/2026 12:48:56 PM +Block sign-in from unsupported countries disabled 5/15/2026 12:17:15 AM +Require compliant device for Exchange enabledForReportingButNotEnforced 8/16/2026 2:13:54 AM +Require MFA for all users enabled 8/11/2026 9:47:30 AM + +Full export saved to conditional-access-policies.json + +conditional-access-policies.json (first lines): +[ + { + "createdDateTime": "2025-07-04T09:20:08Z", + "state": "enabled", + "displayName": "Block legacy authentication", + "id": "d7d8695f-36ef-4a02-9045-cc9efadb0eb1", + "modifiedDateTime": "2026-07-20T12:48:56Z" +... +#> diff --git a/examples/reporting-and-audit/disabled-accounts-report.ps1 b/examples/reporting-and-audit/disabled-accounts-report.ps1 new file mode 100644 index 0000000..9f010fa --- /dev/null +++ b/examples/reporting-and-audit/disabled-accounts-report.ps1 @@ -0,0 +1,33 @@ +# Report all disabled user accounts in the tenant. +# +# Uses server-side filtering so only matching records are transferred. +# -ConsistencyLevel eventual enables advanced query support required +# for certain filter expressions on large tenants. +# +# Requirements: Connect-MgGraph -Scopes "User.Read.All" + +Import-Module Mgx + +$disabled = Invoke-MgxRequest /users ` + -All ` + -Filter "accountEnabled eq false" ` + -Property displayName,mail,userPrincipalName,createdDateTime ` + -ConsistencyLevel eventual + +Write-Host "Disabled accounts: $($disabled.Count)" +$disabled | Sort-Object createdDateTime | Select-Object displayName, mail, createdDateTime | Format-Table -AutoSize + +<# +Expected output: + +Disabled accounts: 6 + +displayName mail createdDateTime +----------- ---- --------------- +Brian Johnson brianj@contoso.com 5/21/2025 8:22:32 PM +Isaiah Langer isaiahl@contoso.com 6/24/2025 8:18:13 PM +Bianca Pisani bianca.pisani@fabrikam.com 7/3/2025 8:13:54 PM +Pradeep Gupta pradeepg@contoso.com 8/13/2025 12:23:01 AM +Cameron White cameronw@contoso.com 11/11/2025 5:13:25 PM +Raul Razo raul.razo@woodgrovebank.com 12/30/2025 11:43:39 AM +#> diff --git a/examples/08-export-sign-in-logs.ps1 b/examples/reporting-and-audit/export-sign-in-logs.ps1 similarity index 73% rename from examples/08-export-sign-in-logs.ps1 rename to examples/reporting-and-audit/export-sign-in-logs.ps1 index e2fc498..5ce9042 100644 --- a/examples/08-export-sign-in-logs.ps1 +++ b/examples/reporting-and-audit/export-sign-in-logs.ps1 @@ -23,3 +23,13 @@ $result = Export-MgxCollection /auditLogs/signIns ` -ApiVersion beta Write-Host "Exported $($result.ItemCount) sign-in records to $outputFile" + +<# +Expected output: + +Exported 246 sign-in records to ./signins.jsonl + +signins.jsonl (first 2 lines): +{"id":"f8984da0-286a-464f-9631-fb15a7f1075f","createdDateTime":"2026-08-20T12:14:58Z","userDisp... +{"id":"cd13719f-f4de-483e-8ab6-b81a8c6b2ed3","createdDateTime":"2026-08-20T12:10:49Z","userDisp... +#> diff --git a/examples/14-guest-users-report.ps1 b/examples/reporting-and-audit/guest-users-report.ps1 similarity index 57% rename from examples/14-guest-users-report.ps1 rename to examples/reporting-and-audit/guest-users-report.ps1 index 02df58b..e690b1d 100644 --- a/examples/14-guest-users-report.ps1 +++ b/examples/reporting-and-audit/guest-users-report.ps1 @@ -19,3 +19,15 @@ $guests | Sort-Object createdDateTime -Descending | Select-Object displayName, mail, externalUserState, createdDateTime | Format-Table -AutoSize + +<# +Expected output: + +Guest accounts: 3 + +displayName mail externalUserState createdDateTime +----------- ---- ----------------- --------------- +Gerhart Moller gerhart.moller@northwindtraders.com PendingAcceptance 6/23/2026 4:04:46 PM +Raul Razo raul.razo@woodgrovebank.com Accepted 12/30/2025 11:43:39 AM +Bianca Pisani bianca.pisani@fabrikam.com Accepted 7/3/2025 8:13:54 PM +#> diff --git a/examples/15-stale-devices-report.ps1 b/examples/reporting-and-audit/stale-devices-report.ps1 similarity index 57% rename from examples/15-stale-devices-report.ps1 rename to examples/reporting-and-audit/stale-devices-report.ps1 index cc305cf..5af8922 100644 --- a/examples/15-stale-devices-report.ps1 +++ b/examples/reporting-and-audit/stale-devices-report.ps1 @@ -21,3 +21,17 @@ $stale | Sort-Object approximateLastSignInDateTime | Select-Object displayName, operatingSystem, approximateLastSignInDateTime, accountEnabled | Format-Table -AutoSize + +<# +Expected output: + +Stale devices (90+ days): 5 + +displayName operatingSystem approximateLastSignInDateTime accountEnabled +----------- --------------- ----------------------------- -------------- +ISAIAHL-LAPTOP Windows 9/13/2025 1:55:10 AM False +ADELEV-SURFACE Windows 1/17/2026 10:17:15 PM True +CONTOSO-KIOSK-02 Windows 2/14/2026 3:15:20 AM True +CONTOSO-MAC-04 macOS 4/10/2026 1:10:32 PM True +MEGANB-IPAD iOS 5/16/2026 5:23:58 AM True +#> diff --git a/examples/resilience-and-telemetry/pacing-observability.ps1 b/examples/resilience-and-telemetry/pacing-observability.ps1 new file mode 100644 index 0000000..97816fe --- /dev/null +++ b/examples/resilience-and-telemetry/pacing-observability.ps1 @@ -0,0 +1,178 @@ +# What the adaptive pacer is doing, and why it is not doing what the docs suggest. +# +# The 2.1 pacer spaces requests BEFORE sending, learning a rate per workload. This script makes +# that state visible: what a cold session looks like, what slow start does to a fan-out, how +# workloads stay independent, and which signals the pacer can actually rely on. +# +# The last point is the one worth reading. Microsoft documents an x-ms-throttle-limit-percentage +# header that warns when you pass 0.8 of the resource-unit budget. Measured against a live +# tenant it was never emitted - not at 1.5x the documented budget, and not while the tenant was +# actively returning 429s. So the pacer treats it as opportunistic and relies on Retry-After and +# latency drift, which are real. This script shows you that for your own tenant. +# +# Requirements: Connect-MgGraph -Scopes "User.Read.All","Files.Read.All" +# Read-only. + +Import-Module Mgx + +function Show-PacingState { + param([string]$Label) + $t = Get-MgxTelemetry + Write-Host "`n$Label" -ForegroundColor Cyan + " requests : $($t.Requests) ($($t.Succeeded) ok, $($t.Failed) failed)" + " pacing waits : $($t.AdaptivePacingWaitMs) ms over $($t.AdaptivePacingActivations) activations" + " throttle retries : $($t.ThrottleRetries) rate-limiter waits: $($t.RateLimiterWaitMs) ms" + " resource units : $($t.ResourceUnitsConsumed)" + # PacingState is one line per workload bucket that has seen traffic. A bucket with nothing + # to say is absent rather than zeroed, so the line grows as you touch more workloads. + " pacing state : $(if ($t.PacingState) { $t.PacingState } else { '(no bucket has seen traffic yet)' })" + " last throttle % : $($t.LastThrottlePercentage)" +} + +Write-Host "=== 1. A cold session has no pacing state ===" -ForegroundColor Green +Show-PacingState "before any request" + +Write-Host "`n=== 2. Slow start on a cold directory workload ===" -ForegroundColor Green +# A cold bucket opens at 4 rps and doubles each clean second. Request #1 is never delayed - the +# cap only bites once a fan-out is in flight, which is exactly the case that gets throttled +# before it has returned a single item. +$sw = [Diagnostics.Stopwatch]::StartNew() +$users = Invoke-MgxRequest /users -All -PageSize 5 -Top 25 -Property id,displayName +$sw.Stop() +" fetched $(@($users).Count) users in $($sw.Elapsed.TotalSeconds.ToString('F1'))s" +Show-PacingState "after a directory read" + +Write-Host "`n=== 3. A second workload gets its own independent bucket ===" -ForegroundColor Green +# Drive and directory are separate buckets. A throttled Teams or Drive workload must not slow an +# Entra fan-out, because the budgets really are separate: Microsoft documents limits per service +# and per application+tenant pair, and a two-identity A/B run confirmed one identity absorbing +# 8,203 throttles while another in the same tenant was never refused. +try { + $null = Invoke-MgxRequest '/me/drive?$select=id,driveType' -ErrorAction Stop + " drive reachable - watch a second bucket appear in the state line below" +} +catch { + " no drive on this account ($($_.Exception.Message -replace '\s+', ' '))" + " the directory bucket below should be unaffected either way - that is the point" +} +Show-PacingState "after touching a second workload" + +Write-Host "`n=== 4. Did Graph ever send the proximity header? ===" -ForegroundColor Green +$t = Get-MgxTelemetry +if ($t.LastThrottlePercentage -lt 0) { + @" + LastThrottlePercentage = $($t.LastThrottlePercentage) + + Negative means Graph has not sent x-ms-throttle-limit-percentage on any response in this + session. That is the expected result. In testing it was never emitted, including during runs + that were actively being throttled 8,203 times - so a control loop built on it would be dead + code that looks like a feature. + + What the pacer uses instead, in order of reliability: + 1. 429 + Retry-After - unambiguous, always honored + 2. latency drift - throughput fell ~19x under load before recovering + 3. this percentage - welcome when present, assumed absent +"@ +} +else { + " LastThrottlePercentage = $($t.LastThrottlePercentage) - Graph DID send it." + " Worth recording: that contradicts what was measured elsewhere. Above 0.8 means you are" + " near the budget and the pacer is damping; at 1.2 it spaces to ~2s per request." +} + +Write-Host "`n=== 5. A/B: the same fan-out with pacing off ===" -ForegroundColor Green +# The honest comparison. Pacing costs a little wall-clock on a cold sequential run and roughly +# nothing on a warm one; what it buys is not being throttled at scale. Do not read a single +# small run as evidence either way. +$ids = @($users | Select-Object -First 8 | ForEach-Object { $_.id }) +if ($ids.Count -lt 2) { + " need at least 2 users to compare; skipping" + return +} + +Set-MgxOption -NoAdaptivePacing +$swOff = [Diagnostics.Stopwatch]::StartNew() +$null = $ids | Invoke-MgxRequest -Uri '/users/{id}?$select=id' -Concurrency 4 +$swOff.Stop() + +Set-MgxOption -Reset +$swOn = [Diagnostics.Stopwatch]::StartNew() +$null = $ids | Invoke-MgxRequest -Uri '/users/{id}?$select=id' -Concurrency 4 +$swOn.Stop() + +" {0} lookups, pacing OFF : {1:F2}s" -f $ids.Count, $swOff.Elapsed.TotalSeconds +" {0} lookups, pacing ON : {1:F2}s" -f $ids.Count, $swOn.Elapsed.TotalSeconds +" difference : {0:F2}s" -f ($swOn.Elapsed.TotalSeconds - $swOff.Elapsed.TotalSeconds) +"" +" A cold bucket pays slow start; a warm one usually pays nothing. The cost that matters is the" +" one you avoid - a throttled fan-out that collapses from 322 rps to 17 rps and takes 15 minutes" +" to drain." + +Show-PacingState "final" +Set-MgxOption -Reset + +<# +Expected output: + +=== 1. A cold session has no pacing state === + +before any request + requests : 0 (0 ok, 0 failed) + pacing waits : 0 ms over 0 activations + throttle retries : 0 rate-limiter waits: 0 ms + resource units : 0 + pacing state : (no bucket has seen traffic yet) + last throttle % : -1 + +=== 2. Slow start on a cold directory workload === + fetched 28 users in 0.4s + +after a directory read + requests : 1 (1 ok, 0 failed) + pacing waits : 0 ms over 0 activations + throttle retries : 0 rate-limiter waits: 0 ms + resource units : 1 + pacing state : directory: slow-start 4 rps, latency 241ms (1.0x of 241ms baseline) + last throttle % : -1 + +=== 3. A second workload gets its own independent bucket === + drive reachable - watch a second bucket appear in the state line below + +after touching a second workload + requests : 2 (2 ok, 0 failed) + pacing waits : 0 ms over 0 activations + throttle retries : 0 rate-limiter waits: 0 ms + resource units : 2 + pacing state : drive: slow-start 4 rps; directory: slow-start 4 rps, latency 241ms (1.0x of 241ms baseline) + last throttle % : -1 + +=== 4. Did Graph ever send the proximity header? === + LastThrottlePercentage = -1 + + Negative means Graph has not sent x-ms-throttle-limit-percentage on any response in this + session. That is the expected result. In testing it was never emitted, including during runs + that were actively being throttled 8,203 times - so a control loop built on it would be dead + code that looks like a feature. + + What the pacer uses instead, in order of reliability: + 1. 429 + Retry-After - unambiguous, always honored + 2. latency drift - throughput fell ~19x under load before recovering + 3. this percentage - welcome when present, assumed absent + +=== 5. A/B: the same fan-out with pacing off === + 8 lookups, pacing OFF : 0.71s + 8 lookups, pacing ON : 1.93s + difference : 1.22s + + A cold bucket pays slow start; a warm one usually pays nothing. The cost that matters is the + one you avoid - a throttled fan-out that collapses from 322 rps to 17 rps and takes 15 minutes + to drain. + +final + requests : 18 (18 ok, 0 failed) + pacing waits : 6193 ms over 8 activations + throttle retries : 0 rate-limiter waits: 0 ms + resource units : 18 + pacing state : drive: slow-start 4 rps; directory: slow-start 8 rps, latency 198ms (0.9x of 224ms baseline) + last throttle % : -1 +#> diff --git a/examples/04-resilience-for-existing-scripts.ps1 b/examples/resilience-and-telemetry/resilience-for-existing-scripts.ps1 similarity index 91% rename from examples/04-resilience-for-existing-scripts.ps1 rename to examples/resilience-and-telemetry/resilience-for-existing-scripts.ps1 index 03f37ad..60d467f 100644 --- a/examples/04-resilience-for-existing-scripts.ps1 +++ b/examples/resilience-and-telemetry/resilience-for-existing-scripts.ps1 @@ -16,3 +16,9 @@ Enable-MgxResilience # retry and circuit breaker protection. $users = Get-MgUser -All -Property displayName, mail Write-Host "Got $($users.Count) users via SDK (with Mgx resilience)" + +<# +Expected output: + +Got 28 users via SDK (with Mgx resilience) +#> diff --git a/examples/21-resilience-status.ps1 b/examples/resilience-and-telemetry/resilience-status.ps1 similarity index 70% rename from examples/21-resilience-status.ps1 rename to examples/resilience-and-telemetry/resilience-status.ps1 index 2522246..b432f36 100644 --- a/examples/21-resilience-status.ps1 +++ b/examples/resilience-and-telemetry/resilience-status.ps1 @@ -14,7 +14,7 @@ Get-MgxResilience # Inject Enable-MgxResilience -Get-MgxResilience # IsInjected: True +Get-MgxResilience # IsEnabled: True, IsActive: True # SDK cmdlets now have retry + circuit breaker $users = Get-MgUser -Top 5 -Property displayName @@ -22,4 +22,18 @@ Write-Host "Got $($users.Count) users via SDK (with resilience)" # Remove injection - restore original SDK behavior Disable-MgxResilience -Get-MgxResilience # IsInjected: False +Get-MgxResilience # IsEnabled: False, IsActive: False + +<# +Expected output: + +IsEnabled : False +IsActive : False + +IsEnabled : True +IsActive : True + +Got 5 users via SDK (with resilience) +IsEnabled : False +IsActive : False +#> diff --git a/examples/resilience-and-telemetry/resource-unit-budgeting.ps1 b/examples/resilience-and-telemetry/resource-unit-budgeting.ps1 new file mode 100644 index 0000000..5a21ca2 --- /dev/null +++ b/examples/resilience-and-telemetry/resource-unit-budgeting.ps1 @@ -0,0 +1,134 @@ +# Resource units: what a query actually costs, and how to size a fan-out before running it. +# +# Graph throttles directory workloads on a resource-unit budget, not on request count or +# bandwidth. Two queries that look alike can differ 5x in cost, and the difference is invisible +# unless you read x-ms-resource-unit. Mgx accumulates it for you: Get-MgxTelemetry reports +# ResourceUnitsConsumed for the session. +# +# This script measures the cost of the query shapes you are choosing between, then uses the +# measured cost - not a guess - to size a tenant-scale fan-out. +# +# Requirements: Connect-MgGraph -Scopes "Group.Read.All","User.Read.All" + +Import-Module Mgx + +function Measure-QueryCost { + <# + Cost of one call, measured rather than assumed. The documented RU table is directionally + right but not exact: transitiveMembers is published at 5 RU and bills 4 here, and the + documented "-1 RU for $select" discount does apply on top of that. + #> + param([string]$Label, [string]$Uri) + + $before = (Get-MgxTelemetry).ResourceUnitsConsumed + try { $null = Invoke-MgxRequest $Uri -ErrorAction Stop } + catch { Write-Warning "$Label : $($_.Exception.Message)"; return } + $cost = (Get-MgxTelemetry).ResourceUnitsConsumed - $before + + [pscustomobject]@{ Query = $Label; ResourceUnits = $cost; Uri = $Uri } +} + +$sample = (Invoke-MgxRequest '/groups?$top=1&$select=id').id +if (-not $sample) { throw 'No groups in this tenant to measure against.' } + +Write-Host "`n=== What each query shape costs ===" -ForegroundColor Cyan +$costs = @( + Measure-QueryCost 'group members (bare)' "/groups/$sample/members?`$top=5" + Measure-QueryCost 'group members ($select)' "/groups/$sample/members?`$top=5&`$select=id" + Measure-QueryCost 'transitiveMembers (bare)' "/groups/$sample/transitiveMembers?`$top=5" + Measure-QueryCost 'transitiveMembers ($select)' "/groups/$sample/transitiveMembers?`$top=5&`$select=id" + Measure-QueryCost 'single group read' "/groups/$sample" + Measure-QueryCost 'users list ($top=25)' "/users?`$top=25" + Measure-QueryCost 'users list ($select)' "/users?`$top=25&`$select=id,displayName" +) +$costs | Where-Object { $_ } | Format-Table Query, ResourceUnits -AutoSize + +# --- size the fan-out from the measured cost --- +# +# Published budget: 8,000 RU per 10 seconds per app per tenant, i.e. 800 RU/s. Measured onset of +# throttling in one test tenant was appreciably higher (clean at 882 RU/s, 429s from ~1,200 +# RU/s), so treat 800 as a conservative floor rather than a hard ceiling - and never as a target. +$BudgetRuPerSecond = 800 +$chosen = $costs | Where-Object { $_ -and $_.Query -eq 'transitiveMembers ($select)' } | Select-Object -First 1 +if (-not $chosen -or $chosen.ResourceUnits -le 0) { + Write-Warning 'No RU header observed - this tenant or endpoint may not be RU-metered. Skipping the projection.' + return +} + +$groupCount = 0 +$null = Invoke-MgxRequest '/groups' -Top 1 -CountVariable groupCount +$totalRu = $groupCount * $chosen.ResourceUnits +$seconds = $totalRu / $BudgetRuPerSecond + +Write-Host "`n=== Sizing a per-group fan-out over the whole tenant ===" -ForegroundColor Cyan +" groups in tenant : {0:N0}" -f $groupCount +" cost per group : {0} RU ({1})" -f $chosen.ResourceUnits, $chosen.Query +" total cost : {0:N0} RU" -f $totalRu +" floor at {0} RU/s : {1:N0}s ({2:N1} min) of pure budget consumption" -f $BudgetRuPerSecond, $seconds, ($seconds / 60) +'' +" Dropping `$select would cost {0:N0} RU more ({1:P0} increase)." -f ` + ($groupCount * 1), (1 / $chosen.ResourceUnits) + +# --- what the session actually spent --- +Write-Host "`n=== Session telemetry ===" -ForegroundColor Cyan +$t = Get-MgxTelemetry +" requests : $($t.Requests) ($($t.Succeeded) ok, $($t.Failed) failed)" +" resource units : $($t.ResourceUnitsConsumed)" +" throttle retries : $($t.ThrottleRetries)" +" pacing waits : $($t.AdaptivePacingWaitMs) ms over $($t.AdaptivePacingActivations) activations" +if ($t.Requests -gt 0) { + " average cost : {0:N2} RU per request" -f ($t.ResourceUnitsConsumed / $t.Requests) +} + +Write-Host @" + +Why this matters + * Cost is a property of the query SHAPE, not the object count. Adding `$select reduces it; + `$expand increases it. Measure the shape you are about to run 15,000 times. + * A 403 or 400 costs 0 RU. A fan-out that fails uniformly consumes no budget and triggers no + throttling - it looks fast and healthy while measuring nothing. Check status, not duration. + * x-ms-throttle-limit-percentage is documented as a proximity warning above 0.8 of budget. It + was never observed in testing, even while the tenant was actively returning 429s. Do not + build a control loop that depends on it; rely on 429 + Retry-After, which is reliable. +"@ -ForegroundColor DarkGray + +<# +Expected output: + +=== What each query shape costs === + +Query ResourceUnits +----- ------------- +group members (bare) 3 +group members ($select) 2 +transitiveMembers (bare) 4 +transitiveMembers ($select) 3 +single group read 1 +users list ($top=25) 2 +users list ($select) 1 + + +=== Sizing a per-group fan-out over the whole tenant === + groups in tenant : 15,779 + cost per group : 3 RU (transitiveMembers ($select)) + total cost : 47,337 RU + floor at 800 RU/s : 59s (1.0 min) of pure budget consumption + + Dropping $select would cost 15,779 RU more (33% increase). + +=== Session telemetry === + requests : 9 (9 ok, 0 failed) + resource units : 18 + throttle retries : 0 + pacing waits : 1496 ms over 8 activations + average cost : 2.00 RU per request + +Why this matters + * Cost is a property of the query SHAPE, not the object count. Adding $select reduces it; + $expand increases it. Measure the shape you are about to run 15,000 times. + * A 403 or 400 costs 0 RU. A fan-out that fails uniformly consumes no budget and triggers no + throttling - it looks fast and healthy while measuring nothing. Check status, not duration. + * x-ms-throttle-limit-percentage is documented as a proximity warning above 0.8 of budget. It + was never observed in testing, even while the tenant was actively returning 429s. Do not + build a control loop that depends on it; rely on 429 + Retry-After, which is reliable. +#> diff --git a/examples/10-telemetry.ps1 b/examples/resilience-and-telemetry/telemetry.ps1 similarity index 50% rename from examples/10-telemetry.ps1 rename to examples/resilience-and-telemetry/telemetry.ps1 index a9cee95..096b334 100644 --- a/examples/10-telemetry.ps1 +++ b/examples/resilience-and-telemetry/telemetry.ps1 @@ -17,3 +17,28 @@ Write-Host "Groups: $($groups.Count)" # See what happened Get-MgxTelemetry + +<# +Expected output: + +Users: 28 +Groups: 6 + +Requests : 2 +Succeeded : 2 +Failed : 0 +Throttle Retries (429) : 0 +Other Retries (5xx) : 0 +Circuit Breaker Trips : 0 +Rate Limiter Wait (ms) : 0 +Retry Delay (ms) : 0 +HTTP Time (ms) : 486 +Total Elapsed (ms) : 704 +Resource Units : 2 +Batch Item Throttles : 0 +Pacing Wait (ms) : 201 +Pacing Activations : 1 +Last Throttle % : never seen +Pacing State : directory: slow-start 4 rps, latency 237ms (1.0x of 241ms baseline) +Content Bytes : 0 +#> diff --git a/examples/resilience-and-telemetry/tune-rate-limits.ps1 b/examples/resilience-and-telemetry/tune-rate-limits.ps1 new file mode 100644 index 0000000..1a3ba43 --- /dev/null +++ b/examples/resilience-and-telemetry/tune-rate-limits.ps1 @@ -0,0 +1,63 @@ +# Tune resilience options for a specific workload. +# +# Set-MgxOption lets you adjust the rate limiter, retry count, circuit +# breaker, and timeouts at runtime without restarting the session. +# Only the parameters you pass are changed; everything else stays. +# +# Use Get-MgxOption to inspect current settings. +# +# Requirements: Connect-MgGraph -Scopes "User.Read.All" + +Import-Module Mgx + +# View current defaults +Get-MgxOption + +# Slow down for a tenant that throttles aggressively +Set-MgxOption -BatchItemsPerSecond 5 -MaxRetryAttempts 10 + +# Run your workload +$users = Invoke-MgxRequest /users -All -Property id,displayName +Write-Host "Users: $($users.Count)" + +# Restore all defaults when done +Set-MgxOption -Reset + +Get-MgxOption + +<# +Expected output: + +Rate Limit Burst : 200 +Rate Limit/sec : 50 +No Rate Limit : False +No Adaptive Pacing : False +Queue Limit : 500 +Max Retry Attempts : 7 +Max Retry-After (s) : 120 +Total Timeout (s) : 300 +Attempt Timeout (s) : 30 +CB Duration (s) : 15 +CB Failure Ratio : 0.1 +CB Min Throughput : 40 +CB Sampling (s) : 30 +Batch Chunk Concurrency : 1 +Batch Items/sec : 20 + +Users: 28 +Rate Limit Burst : 200 +Rate Limit/sec : 50 +No Rate Limit : False +No Adaptive Pacing : False +Queue Limit : 500 +Max Retry Attempts : 7 +Max Retry-After (s) : 120 +Total Timeout (s) : 300 +Attempt Timeout (s) : 30 +CB Duration (s) : 15 +CB Failure Ratio : 0.1 +CB Min Throughput : 40 +CB Sampling (s) : 30 +Batch Chunk Concurrency : 1 +Batch Items/sec : 20 +#> diff --git a/module/en-US/Mgx.Cmdlets.dll-Help.xml b/module/en-US/Mgx.Cmdlets.dll-Help.xml index cc14a1c..c652c4b 100644 --- a/module/en-US/Mgx.Cmdlets.dll-Help.xml +++ b/module/en-US/Mgx.Cmdlets.dll-Help.xml @@ -252,7 +252,7 @@ - The SDK's built-in RetryHandler still runs inside the wrapped chain. Retries can compound, bounded by the total timeout (default 300s) and circuit breaker. + The SDK's built-in RetryHandler still runs inside the wrapped chain, and it answers 429 and 503 itself. Two consequences, both measured: retries compound (four intended attempts can reach the wire eight times, because the SDK sleeps Retry-After inside the call and Mgx's own attempt timeout then fires), and a throttle never reaches Mgx's pipeline - so the adaptive pacer does not learn from it and Get-MgxTelemetry reports ThrottleRetries as 0 through a live throttle, booking the waiting as OtherRetries instead. The total timeout (default 300s) bounds the compounding; the circuit breaker does not, because the exhaustion path surfaces as an AggregateException that its predicate does not match. Use the Invoke-Mgx* cmdlets where throttle-aware pacing matters. @@ -340,11 +340,11 @@ Enable-MgxResilience # Re-inject for the new connection InputObject - The object to enrich with related data. Accepts any PSObject from the pipeline. Must have a property matching -IdProperty (default: id) whose value is substituted into the -Uri template. + The object to enrich with related data. Accepts a hashtable (what the Mgx cmdlets emit) or a PSCustomObject from the pipeline. Must have a member matching -IdProperty (default: id) whose value is substituted into the -Uri template. The relation is attached in the same shape the object arrived in. - PSObject + Object - PSObject + Object None @@ -352,7 +352,7 @@ Enable-MgxResilience # Re-inject for the new connection Flatten - Unwrap single-value relations. When the endpoint returns exactly one item (or a singleton object like /manager), the result is attached as a single PSObject instead of a one-element array. If multiple items are returned, a warning is emitted and the array is returned as-is. + Unwrap single-value relations. When the endpoint returns exactly one item (or a singleton object like /manager), the result is attached as a single object instead of a one-element array. If multiple items are returned, a warning is emitted and the array is returned as-is. SwitchParameter @@ -460,11 +460,11 @@ Enable-MgxResilience # Re-inject for the new connection InputObject - The object to enrich with related data. Accepts any PSObject from the pipeline. Must have a property matching -IdProperty (default: id) whose value is substituted into the -Uri template. + The object to enrich with related data. Accepts a hashtable (what the Mgx cmdlets emit) or a PSCustomObject from the pipeline. Must have a member matching -IdProperty (default: id) whose value is substituted into the -Uri template. The relation is attached in the same shape the object arrived in. - PSObject + Object - PSObject + Object None @@ -496,7 +496,7 @@ Enable-MgxResilience # Re-inject for the new connection Flatten - Unwrap single-value relations. When the endpoint returns exactly one item (or a singleton object like /manager), the result is attached as a single PSObject instead of a one-element array. If multiple items are returned, a warning is emitted and the array is returned as-is. + Unwrap single-value relations. When the endpoint returns exactly one item (or a singleton object like /manager), the result is attached as a single object instead of a one-element array. If multiple items are returned, a warning is emitted and the array is returned as-is. SwitchParameter @@ -605,20 +605,20 @@ Enable-MgxResilience # Re-inject for the new connection - System.Management.Automation.PSObject + System.Object - Any PSObject with a property matching -IdProperty (default: id). + A hashtable or PSCustomObject with a member matching -IdProperty (default: id). - System.Management.Automation.PSObject + System.Collections.Hashtable - The input object with an additional property (named by -As) containing the relation data. Collection endpoints produce an array of PSObjects. Singleton endpoints produce a single PSObject when -Flatten is used, or a one-element array otherwise. + The input object with an additional member (named by -As) containing the relation data, in the same shape the input arrived in. Collection endpoints produce an array of hashtables. Singleton endpoints produce a single hashtable when -Flatten is used, or a one-element array otherwise. @@ -644,7 +644,7 @@ Enable-MgxResilience # Re-inject for the new connection Expand-MgxRelation -Uri '/users/{id}/manager' -As Manager -Flatten -SkipNotFound | Select-Object displayName, @{N='ManagerName'; E={$_.Manager.displayName}} - The /manager endpoint returns a single object, not a collection. -Flatten unwraps it from the array so Manager is a PSObject (not a one-element array). Users without a manager get $null. + The /manager endpoint returns a single object, not a collection. -Flatten unwraps it from the array so Manager is a single object (not a one-element array). Users without a manager get $null. @@ -709,8 +709,9 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP - Export-MgxCollection streams paginated Microsoft Graph API results directly to a JSONL (JSON Lines) file. One JSON object per line, no PSObject conversion, minimal memory pressure regardless of collection size. - Supports checkpoint/resume: specify -CheckpointPath to save progress at page boundaries. On interruption and re-run, export resumes from the last checkpoint. The checkpoint file is deleted on successful completion. + Export-MgxCollection streams paginated Microsoft Graph API results directly to a JSONL (JSON Lines) file. One JSON object per line, no object conversion, minimal memory pressure regardless of collection size. + Supports checkpoint/resume: specify -CheckpointPath to save progress at page boundaries and mid-page (every 500 items). On interruption - Ctrl+C, a crash, or a killed process - re-running the same command resumes from the last checkpoint; an interrupted first run's partial temp file is recovered and trimmed to exactly the checkpointed item count, so the output never contains duplicates or torn lines. The checkpoint file is deleted on successful completion. + Downstream note: Graph pagination can transiently serve an overlapping page (observed in the wild as one extra page of items versus @odata.count). Mgx warns when the returned count overshoots the reported count; if the JSONL feeds another system, deduplicate on the id field as a matter of hygiene. Returns a summary PSObject with ItemCount and OutputFile properties. @@ -1239,6 +1240,430 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP + + + Get-MgxContent + Get + MgxContent + + Fetch content bytes from Graph $value / /content endpoints, whole or as a byte range. + + + + Get-MgxContent downloads file content from Microsoft Graph content endpoints - drive items (`/drives/{id}/items/{id}/content`), mail attachments (`.../attachments/{id}/$value`), profile photos (`/me/photo/$value`), and anything else that serves bytes. + Ranged reads are the point: `-First 262144` pulls the first 256 KB of a file (enough for format sniffing, EXIF, or partial hashing) instead of the whole thing. Across 20k files that is gigabytes moved instead of tens of gigabytes. + Under the hood, drive content is a two-hop flow: the authenticated Graph request (full resilience pipeline, adaptive pacing, rate limiting) answers with a redirect to a short-lived pre-authenticated download URL, which mgx fetches with a separate token-free HTTP client. The bearer token never reaches the download host, and the host is validated against a Microsoft-hosts allowlist (SharePoint/OneDrive download domains) before any bytes move - including on every mid-download redirect. + Output is a single `byte[]` to the pipeline (guarded at 100 MB - use -OutFile beyond that), or a file written via temp-plus-atomic-move so a failed download never truncates an existing file. + Piped DriveItems (from Invoke-MgxRequest or Sync-MgxDelta) use their `@microsoft.graph.downloadUrl` directly when present - no extra Graph round trip - falling back to `/drives/{driveId}/items/{id}/content` from the item's `id` and `parentReference.driveId`. + Some endpoints ignore the Range header and answer 200 with the full body (profile photos do this). Get-MgxContent detects that, keeps only the requested bytes, and aborts the rest of the transfer. + + + + Get-MgxContent + + Uri + + Relative Graph path to a content endpoint (e.g., /me/drive/items/{id}/content). Warns when the path does not look like a content endpoint (/content or /$value). + + String + + String + + + None + + + First + + First N bytes (Range: bytes=0..N-1). Mutually exclusive with -Offset/-Length. + + Int64 + + Int64 + + + None + + + Offset + + Range start in bytes; requires -Length. + + Int64 + + Int64 + + + 0 + + + Length + + Range length in bytes, starting at -Offset (default 0). + + Int64 + + Int64 + + + None + + + OutFile + + Write the content to this file (temp + atomic move) instead of emitting byte[] to the pipeline. Required for content larger than 100 MB. + + String + + String + + + None + + + ApiVersion + + Graph API version. Default: v1.0. + + + v1.0 + beta + + String + + String + + + v1.0 + + + Headers + + Custom request headers for the Graph request (hop 1 only; never sent to the download host). + + Hashtable + + Hashtable + + + None + + + ProgressAction + + Determines how the cmdlet responds to progress updates. + + ActionPreference + + ActionPreference + + + None + + + + Get-MgxContent + + InputObject + + A DriveItem-shaped object from the pipeline. Uses `@microsoft.graph.downloadUrl` when present (validated against the download-host allowlist), else `/drives/{parentReference.driveId}/items/{id}/content`. + + Object + + Object + + + None + + + First + + First N bytes (Range: bytes=0..N-1). Mutually exclusive with -Offset/-Length. + + Int64 + + Int64 + + + None + + + Offset + + Range start in bytes; requires -Length. + + Int64 + + Int64 + + + 0 + + + Length + + Range length in bytes, starting at -Offset (default 0). + + Int64 + + Int64 + + + None + + + OutFile + + Write the content to this file (temp + atomic move) instead of emitting byte[] to the pipeline. Required for content larger than 100 MB. + + String + + String + + + None + + + ApiVersion + + Graph API version. Default: v1.0. + + + v1.0 + beta + + String + + String + + + v1.0 + + + Headers + + Custom request headers for the Graph request (hop 1 only; never sent to the download host). + + Hashtable + + Hashtable + + + None + + + ProgressAction + + Determines how the cmdlet responds to progress updates. + + ActionPreference + + ActionPreference + + + None + + + + + + Uri + + Relative Graph path to a content endpoint (e.g., /me/drive/items/{id}/content). Warns when the path does not look like a content endpoint (/content or /$value). + + String + + String + + + None + + + InputObject + + A DriveItem-shaped object from the pipeline. Uses `@microsoft.graph.downloadUrl` when present (validated against the download-host allowlist), else `/drives/{parentReference.driveId}/items/{id}/content`. + + Object + + Object + + + None + + + First + + First N bytes (Range: bytes=0..N-1). Mutually exclusive with -Offset/-Length. + + Int64 + + Int64 + + + None + + + Offset + + Range start in bytes; requires -Length. + + Int64 + + Int64 + + + 0 + + + Length + + Range length in bytes, starting at -Offset (default 0). + + Int64 + + Int64 + + + None + + + OutFile + + Write the content to this file (temp + atomic move) instead of emitting byte[] to the pipeline. Required for content larger than 100 MB. + + String + + String + + + None + + + ApiVersion + + Graph API version. Default: v1.0. + + String + + String + + + v1.0 + + + Headers + + Custom request headers for the Graph request (hop 1 only; never sent to the download host). + + Hashtable + + Hashtable + + + None + + + ProgressAction + + Determines how the cmdlet responds to progress updates. + + ActionPreference + + ActionPreference + + + None + + + + + + System.Object + + + DriveItem-shaped Hashtables or PSCustomObjects (from Invoke-MgxRequest or Sync-MgxDelta). + + + + + + + System.Byte[] + + + The content bytes as a single array (not enumerated). Nothing is emitted when -OutFile is used. + + + + + + Content downloads require the mgx-owned HTTP transport. If mgx had to fall back to the Graph SDK's client (rare - it means the clean client could not be built), Get-MgxContent refuses with ContentRequiresOwnedTransport rather than let a transport that auto-follows redirects fetch from an unvalidated host. + Graph reports drive-item hashes as quickXorHash in base64; rclone and many tools print hex. Convert before comparing: `[System.Convert]::ToHexString([System.Convert]::FromBase64String($item.file.hashes.quickXorHash))` + The pre-authenticated download URL on piped items is short-lived (~1 hour). On expiry mgx re-requests it automatically for the /content path; for stale piped `@microsoft.graph.downloadUrl` values, re-fetch the item. + Intune report export jobs are not a /content flow: the completed job's `url` property is a pre-authenticated Azure Blob URL - download it directly with Invoke-WebRequest; it needs nothing from mgx. + + + + + ------------------ Example 1: Download a file ------------------ + Get-MgxContent "/me/drive/items/01ABC/content" -OutFile ./report.xlsx + + Downloads the drive item to a local file. + + + + Example 2: First 256 KB of every file in a folder (format sniffing / partial hash) + Invoke-MgxRequest "/me/drive/items/01FOLDER/children" -All | + Where-Object { $_.file } | + ForEach-Object { + $bytes = $_ | Get-MgxContent -First 262144 + [PSCustomObject]@{ name = $_.name; header = [BitConverter]::ToString($bytes[0..7]) } + } + + Moves kilobytes per file instead of the full content - the metadata-extraction economy case. + + + + --------------- Example 3: A specific byte range --------------- + Get-MgxContent "/me/drive/items/01ABC/content" -Offset 1048576 -Length 65536 + + Returns bytes 1,048,576 through 1,114,111 (Range: bytes=1048576-1114111). + + + + ----------------- Example 4: A mail attachment ----------------- + Get-MgxContent "/me/messages/AAMk.../attachments/AAMk.../`$value" -OutFile ./attachment.pdf + + Attachment bytes are served directly by Graph (no download-host hop). + + + + ------------------- Example 5: Profile photo ------------------- + Get-MgxContent "/me/photo/`$value" -OutFile ./me.jpg + + Note the backtick: `$value` would otherwise be expanded by PowerShell. + + + + + + Online Version: + https://github.com/gromedev/mgx/blob/main/module/help/Get-MgxContent.md + + + Invoke-MgxRequest + + + + Sync-MgxDelta + + + + Get-MgxTelemetry + + + + Get-MgxOption @@ -1270,7 +1695,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP - System.Management.Automation.PSObject + Mgx.Cmdlets.Models.MgxOptionOutput A PSObject with properties for each configuration setting. @@ -1396,6 +1821,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP Get-MgxTelemetry returns accumulated telemetry from the current session's resilience pipeline. Reports request counts, retry/throttle breakdown, and timing per category so you can determine whether the bottleneck is throttling, retry backoff, rate-limiter queuing, or network latency. + Adaptive pacing surfaces here too: AdaptivePacingWaitMs / AdaptivePacingActivations count proactive waits, LastThrottlePercentage is the most recent x-ms-throttle-limit-percentage Graph reported (-1 = never seen), and PacingState is a per-workload line showing active rate caps, slow start, throttle proximity, and current latency against the session baseline. -Reset clears the counters but deliberately not the pacer's learned rate caps - those describe the tenant's current throttle regime, not accumulated statistics. @@ -1440,7 +1866,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP - System.Management.Automation.PSObject + Mgx.Cmdlets.Models.MgxTelemetryOutput Telemetry summary with request counts, retry/throttle breakdown, and timing. @@ -1470,11 +1896,18 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP ---------------- Example 3: Check throttle rate ---------------- $t = Get-MgxTelemetry -"Throttle rate: $([math]::Round($t.ThrottleCount / $t.TotalRequests * 100, 1))%" +"Throttle rate: $(if ($t.Requests) { [math]::Round($t.ThrottleRetries / $t.Requests * 100, 1) } else { 0 })%" Calculates the percentage of requests that were throttled. + + ------ Example 4: Watch adaptive pacing during a fan-out ------ + Get-MgxTelemetry | Select-Object AdaptivePacingWaitMs, AdaptivePacingActivations, LastThrottlePercentage, PacingState + + Shows how much proactive pacing has happened and the current per-workload pacing state. + + @@ -1503,7 +1936,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP Invoke-MgxBatchRequest bundles multiple Microsoft Graph API requests into /$batch calls, sending up to 20 requests per HTTP round-trip (the Graph API maximum). This is 3-4x faster than individual requests for bulk operations. Supports GET, POST, PATCH, PUT, and DELETE methods with optional request bodies. Auto-chunks input into 20-request batches. - Source: Combine multiple HTTP requests using JSON batching (https://learn.microsoft.com/en-us/graph/json-batching)Pipeline input can be string URLs (for GET, or combined with -Method/-Body for the same operation on all) or PSObjects with Url, Method, and Body properties for per-item control. + Source: Combine multiple HTTP requests using JSON batching (https://learn.microsoft.com/en-us/graph/json-batching)Pipeline input can be string URLs (for GET, or combined with -Method/-Body for the same operation on all) or hashtables or PSCustomObjects with Url, Method, and Body members for per-item control. Failed items are surfaced as PowerShell ErrorRecords. Use -ErrorAction Stop to halt on the first failure, or inspect $Error after completion. @@ -1512,7 +1945,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP Uri - Graph API URLs to batch. Accepts absolute URLs (https://graph.microsoft.com/v1.0/users/id) or relative URLs (/users/id). Also accepts PSObjects with Url, Method, and Body properties for per-item control. + Graph API URLs to batch. Accepts absolute URLs (https://graph.microsoft.com/v1.0/users/id) or relative URLs (/users/id). Also accepts hashtables or PSCustomObjects with Url, Method, and Body members for per-item control. Object[] @@ -1540,7 +1973,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP Body - Request body for all requests when piping string URLs. Ignored when pipeline input contains PSObjects with their own Body property. + Request body for all requests when piping string URLs. Ignored when pipeline input carries its own Body member. Object @@ -1552,7 +1985,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP DeadLetterPath - Path to a JSONL file where failed batch items (status >= 400) are appended. Each line contains Url, Method, Body (with sensitive fields redacted), Status, Error, and Timestamp. The file can be re-piped to Invoke-MgxBatchRequest for retry: `Get-Content dead.jsonl | ConvertFrom-Json | Invoke-MgxBatchRequest`. + Path to a JSONL file where failed batch items (status >= 400) are appended, along with any that were never sent because an earlier chunk failed. Each line contains Url, Method, Body (with sensitive fields redacted), Status, Error, and Timestamp. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Read it to decide what to resubmit, and supply the sensitive fields again yourself. String @@ -1605,7 +2038,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP Method - HTTP method for all requests when piping string URLs. Default: GET. Ignored when pipeline input contains PSObjects with their own Method property. + HTTP method for all requests when piping string URLs. Default: GET. Ignored when pipeline input carries its own Method member. GET @@ -1662,7 +2095,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP Body - Request body for all requests when piping string URLs. Ignored when pipeline input contains PSObjects with their own Body property. + Request body for all requests when piping string URLs. Ignored when pipeline input carries its own Body member. Object @@ -1674,7 +2107,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP DeadLetterPath - Path to a JSONL file where failed batch items (status >= 400) are appended. Each line contains Url, Method, Body (with sensitive fields redacted), Status, Error, and Timestamp. The file can be re-piped to Invoke-MgxBatchRequest for retry: `Get-Content dead.jsonl | ConvertFrom-Json | Invoke-MgxBatchRequest`. + Path to a JSONL file where failed batch items (status >= 400) are appended, along with any that were never sent because an earlier chunk failed. Each line contains Url, Method, Body (with sensitive fields redacted), Status, Error, and Timestamp. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Read it to decide what to resubmit, and supply the sensitive fields again yourself. String @@ -1735,7 +2168,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP Method - HTTP method for all requests when piping string URLs. Default: GET. Ignored when pipeline input contains PSObjects with their own Method property. + HTTP method for all requests when piping string URLs. Default: GET. Ignored when pipeline input carries its own Method member. String @@ -1747,7 +2180,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP Uri - Graph API URLs to batch. Accepts absolute URLs (https://graph.microsoft.com/v1.0/users/id) or relative URLs (/users/id). Also accepts PSObjects with Url, Method, and Body properties for per-item control. + Graph API URLs to batch. Accepts absolute URLs (https://graph.microsoft.com/v1.0/users/id) or relative URLs (/users/id). Also accepts hashtables or PSCustomObjects with Url, Method, and Body members for per-item control. Object[] @@ -1787,17 +2220,17 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP System.Object[] - String URLs or PSObjects with Url, Method, and Body properties. + String URLs, or hashtables or PSCustomObjects with Url, Method, and Body members. - System.Management.Automation.PSObject + System.Collections.Hashtable - Per-request results with Url, Status, and Body properties. + Per-request results with Url, Method, Status, and Body keys, plus NotSent on any operation that was never sent because an earlier chunk failed. Body here is the RESPONSE body - for a failure that is the error envelope, not the request - so piping results straight back resubmits the wrong thing. Use Url and Method to rebuild the requests you want to retry. @@ -1931,7 +2364,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Invoke-MgxRequest is a general-purpose resilient client for any Microsoft Graph endpoint. It supports streaming pagination, fan-out concurrency, write operations (POST, PATCH, PUT, DELETE), and checkpoint/resume. - Results are returned as PSObjects with properties matching the Graph API JSON response. DateTime strings are automatically parsed to DateTimeOffset, and @odata.type is preserved as an ODataType property. + Results are returned as case-insensitive hashtables with keys matching the Graph API JSON response, the same shape Invoke-MgGraphRequest returns. DateTime strings are automatically parsed, @odata.type is preserved verbatim under its own key, and all other @odata.* metadata is stripped (including @odata.etag, which changes on every write; use -Raw when you need the If-Match tag). Key order is undefined: use Select-Object or Format-Table to pin column order. Use -Raw | ConvertFrom-Json to get PSCustomObjects instead. For bulk writes involving more than 10 items, consider using Invoke-MgxBatchRequest instead, which is 3-4x faster due to fewer HTTP round-trips. @@ -1979,7 +2412,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Body - Request body for write operations (POST, PATCH, PUT). Accepts a hashtable or PSObject, which is serialized to JSON. + Request body for write operations (POST, PATCH, PUT). Accepts a hashtable, PSObject or array, which is serialized to JSON, or a string already holding JSON, which is sent verbatim. Ignored (with a warning) on GET. Object @@ -2152,7 +2585,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Raw - Return raw JSON strings instead of PSObjects. Useful for piping to ConvertFrom-Json or writing to files. + Return raw JSON strings instead of hashtables. Useful for piping to ConvertFrom-Json or writing to files. SwitchParameter @@ -2283,7 +2716,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Body - Request body for write operations (POST, PATCH, PUT). Accepts a hashtable or PSObject, which is serialized to JSON. + Request body for write operations (POST, PATCH, PUT). Accepts a hashtable, PSObject or array, which is serialized to JSON, or a string already holding JSON, which is sent verbatim. Ignored (with a warning) on GET. Object @@ -2451,7 +2884,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Raw - Return raw JSON strings instead of PSObjects. Useful for piping to ConvertFrom-Json or writing to files. + Return raw JSON strings instead of hashtables. Useful for piping to ConvertFrom-Json or writing to files. SwitchParameter @@ -2575,17 +3008,17 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } System.String - Pipeline input for fan-out operations. Each string value replaces {id} in the URI template. + Pipeline input for fan-out operations. Accepts an id string, or an object carrying one: the id member of a hashtable or PSCustomObject is extracted and replaces {id} in the URI template. - System.Management.Automation.PSObject + System.Collections.Hashtable - Graph API response objects with properties matching the JSON fields. + Graph API response objects as case-insensitive hashtables with keys matching the JSON fields. @@ -2748,6 +3181,18 @@ Import-Csv ./users.csv | Set-MgxOption + + BatchChunkConcurrency + + Number of $batch chunks executed concurrently. The default of 1 runs chunks sequentially, which is what keeps inter-chunk pacing meaningful; raising it trades that pacing for throughput and makes server-side write throttling more likely. Range: 1-10. Default: 1. + + Int32 + + Int32 + + + 1 + BatchItemsPerSecond @@ -2847,7 +3292,18 @@ Import-Csv ./users.csv | NoRateLimit - Disable the client-side rate limiter entirely. + Disable the client-side rate limiter entirely. This also disables batch item pacing, so any -BatchItemsPerSecond you set alongside it has no effect; Set-MgxOption warns when both are given. + + + SwitchParameter + + + False + + + NoAdaptivePacing + + Disable adaptive request pacing (AIMD back-off after 429s, slow start on cold workloads, and throttle-proximity damping). Pacing is ON by default and independent of -NoRateLimit: the token bucket is the hard backstop, the pacer is the proactive layer in front of it. See about_Mgx_Tuning "ADAPTIVE PACING". SwitchParameter @@ -2917,6 +3373,18 @@ Import-Csv ./users.csv | + + BatchChunkConcurrency + + Number of $batch chunks executed concurrently. The default of 1 runs chunks sequentially, which is what keeps inter-chunk pacing meaningful; raising it trades that pacing for throughput and makes server-side write throttling more likely. Range: 1-10. Default: 1. + + Int32 + + Int32 + + + 1 + BatchItemsPerSecond @@ -3016,7 +3484,19 @@ Import-Csv ./users.csv | NoRateLimit - Disable the client-side rate limiter entirely. + Disable the client-side rate limiter entirely. This also disables batch item pacing, so any -BatchItemsPerSecond you set alongside it has no effect; Set-MgxOption warns when both are given. + + SwitchParameter + + SwitchParameter + + + False + + + NoAdaptivePacing + + Disable adaptive request pacing (AIMD back-off after 429s, slow start on cold workloads, and throttle-proximity damping). Pacing is ON by default and independent of -NoRateLimit: the token bucket is the hard backstop, the pacer is the proactive layer in front of it. See about_Mgx_Tuning "ADAPTIVE PACING". SwitchParameter @@ -3174,9 +3654,11 @@ Import-Csv ./users.csv | Sync-MgxDelta retrieves incremental changes from Microsoft Graph delta endpoints. On the first run, it performs a full sync and saves a delta token. Subsequent runs retrieve only items that changed since the last sync. - Delta tokens are saved to the file specified by -DeltaPath and persist across successful completions. This is different from -CheckpointPath (used by Export-MgxCollection) which is ephemeral and deleted on success. - Items that no longer match the query appear with an `@removed` property containing `{"reason": "changed"}` (item moved out of scope or soft-deleted) or `{"reason": "deleted"}` (permanently deleted). Filter these with `Where-Object { -not $_.'@removed' }`. - Delta tokens expire after approximately 7 days for directory objects (users, groups, applications). When a token expires, Graph returns HTTP 410 Gone. Sync-MgxDelta handles this automatically by deleting the stale token and performing a full re-sync with a warning. + Delta tokens are saved to the file specified by -DeltaPath and persist across successful completions. -CheckpointPath is different: it is an ephemeral mid-run resume position, saved at page boundaries while an enumeration is in flight and deleted the moment it completes. Use both on long enumerations - a killed 3-hour drive sync resumes where it stopped instead of starting over. + Drive delta is fully supported: `/me/drive/root/delta`, `/drives/{id}/root/delta`, `/users/{id}/drive/root/delta`, and `/groups/{id}/drive/root/delta` all work, including drive-specific `-Prefer` behaviors and the drive form of `-Latest`. + Items that no longer match the query appear with an `@removed` property containing `{"reason": "changed"}` (item moved out of scope or soft-deleted) or `{"reason": "deleted"}` (permanently deleted). Filter these with `Where-Object { -not $_.'@removed' }`. The completion message counts them separately. + Delta tokens expire after approximately 7 days for directory objects (users, groups, applications). When a token expires, Graph returns HTTP 410 Gone. Sync-MgxDelta handles this automatically by deleting the stale token (and any resume checkpoint) and performing a full re-sync with a warning. + Resume semantics are at-least-once: in pipeline mode the page in flight at a crash is re-emitted in full on resume; in JSONL mode mid-page checkpoints keep the window to at most 500 items. Deduplicate on `id` downstream if exact-once matters. Source: Use delta query to track changes in Microsoft Graph data (https://learn.microsoft.com/en-us/graph/delta-query-overview) @@ -3210,6 +3692,18 @@ Import-Csv ./users.csv | v1.0 + + CheckpointPath + + Path for the ephemeral mid-run resume checkpoint. Saved at page boundaries (and every 500 items in JSONL mode) while an enumeration is in flight; deleted on successful completion. Any event that invalidates the enumeration - HTTP 410 Gone, -FullSync, or a -Property/-Filter/-Prefer change - deletes it too. Must differ from -DeltaPath and -OutputFile. + + String + + String + + + None + DeltaPath @@ -3245,6 +3739,29 @@ Import-Csv ./users.csv | False + + Latest + + Baseline without enumerating: request only the latest delta token ("sync from now"). Returns no data; the next run returns everything that changed since. Drive resources take `?token=latest`, directory and other resources `$deltatoken=latest` - the form is chosen automatically from the URI shape. Ignored with a warning when usable delta state already exists (delete it or use -FullSync to re-baseline). + + + SwitchParameter + + + False + + + Prefer + + Prefer-header tokens joined into a single Prefer header on every page request. Tab completion offers the documented drive delta tokens: `deltashowremovedasdeleted`, `deltatraversepermissiongaps`, `deltashowsharingchanges` (requires the other two and Sites.FullControl.All), and `hierarchicalsharing`. A change against the stored state forces a full re-sync, like -Property and -Filter. `deltaExcludeParent` is a standalone request header, not a Prefer token - pass it via -Headers. + + String[] + + String[] + + + None + Headers @@ -3320,6 +3837,18 @@ Import-Csv ./users.csv | v1.0 + + CheckpointPath + + Path for the ephemeral mid-run resume checkpoint. Saved at page boundaries (and every 500 items in JSONL mode) while an enumeration is in flight; deleted on successful completion. Any event that invalidates the enumeration - HTTP 410 Gone, -FullSync, or a -Property/-Filter/-Prefer change - deletes it too. Must differ from -DeltaPath and -OutputFile. + + String + + String + + + None + DeltaPath @@ -3356,6 +3885,30 @@ Import-Csv ./users.csv | False + + Latest + + Baseline without enumerating: request only the latest delta token ("sync from now"). Returns no data; the next run returns everything that changed since. Drive resources take `?token=latest`, directory and other resources `$deltatoken=latest` - the form is chosen automatically from the URI shape. Ignored with a warning when usable delta state already exists (delete it or use -FullSync to re-baseline). + + SwitchParameter + + SwitchParameter + + + False + + + Prefer + + Prefer-header tokens joined into a single Prefer header on every page request. Tab completion offers the documented drive delta tokens: `deltashowremovedasdeleted`, `deltatraversepermissiongaps`, `deltashowsharingchanges` (requires the other two and Sites.FullControl.All), and `hierarchicalsharing`. A change against the stored state forces a full re-sync, like -Property and -Filter. `deltaExcludeParent` is a standalone request header, not a Prefer token - pass it via -Headers. + + String[] + + String[] + + + None + Headers @@ -3442,10 +3995,10 @@ Import-Csv ./users.csv | - System.Management.Automation.PSObject + System.Collections.Hashtable - Graph API objects with properties matching the JSON fields. Deleted items include an `@removed` property. + Graph API objects as case-insensitive hashtables with keys matching the JSON fields. Deleted items include an `@removed` key. @@ -3453,7 +4006,15 @@ Import-Csv ./users.csv | Delta queries follow all @odata.nextLink pages automatically (equivalent to -All on other cmdlets). The delta token is saved only after all pages are successfully retrieved. Query parameters ($select, $filter) are encoded into the delta token on the first request. Do not re-specify them on subsequent runs; they are automatically applied from the saved token. If you change -Property between runs, Sync-MgxDelta detects the mismatch and performs a full re-sync. - Supported delta endpoints include: /users/delta, /groups/delta, /applications/delta, /servicePrincipals/delta, /devices/delta, /directoryRoles/delta, and many others. Source: Use delta query to track changes in Microsoft Graph data (https://learn.microsoft.com/en-us/graph/delta-query-overview) + Supported delta endpoints include: /users/delta, /groups/delta, /applications/delta, /servicePrincipals/delta, /devices/delta, /directoryRoles/delta, and many others. Source: Use delta query to track changes in Microsoft Graph data (https://learn.microsoft.com/en-us/graph/delta-query-overview)DUPLICATE OBJECTS ACROSS PAGES + Graph does not guarantee that an object appears only once in a delta response: it "can't ensure that entities are unified in a single response." Deduplicate by `id` before treating emitted objects as change events. + This is most visible on an initial full enumeration. Measured against a tenant with 15,779 groups, `/groups/delta` emitted 156,413 objects across all pages - a replay factor of roughly 10x - with every object accounted for by a repeat rather than a distinct group. Incremental rounds off an established token are typically clean, but nothing in the contract promises that, so the dedup belongs in the consumer either way. + Two consequences worth designing around: + - For a change feed, baseline with `-Latest` so the initial enumeration never happens. It records a sync-from-now token and returns nothing, and every later run reports only real changes. + - When counting, count distinct ids. An object that changed twice inside one window is one object; keeping the last occurrence gives its current state. + + `-CheckpointPath` writes item counts as emitted, not deduplicated, so a resumed run's progress figures reflect the same replay. + Source: Use delta query to track changes (https://learn.microsoft.com/en-us/graph/delta-query-overview#other-considerations) @@ -3502,6 +4063,29 @@ Write-Host "$($changed.Count) changed, $($removed.Count) removed" Suitable for scheduled tasks. If the delta token expires (>7 days since last run), Sync-MgxDelta automatically performs a full re-sync and warns. + + -------- Example 7: Enumerate a whole drive, resumable -------- + Sync-MgxDelta /me/drive/root/delta -DeltaPath drive.delta -CheckpointPath drive.checkpoint -OutputFile drive.jsonl + + First run enumerates the entire drive to JSONL. If the run is killed at any point, re-running the identical command resumes from the last checkpointed page instead of starting over. Subsequent runs after completion return only changed items. + + + + ---- Example 8: Baseline from now, then track changes only ---- + Sync-MgxDelta /groups/delta -DeltaPath groups.delta -Latest # instant baseline, no data +# ...later... +Sync-MgxDelta /groups/delta -DeltaPath groups.delta # only what changed since + + -Latest skips the initial full enumeration and stores a "from now" token. The form is chosen automatically: drive resources use `?token=latest`, directory and everything else `$deltatoken=latest`. + + + + --------- Example 9: Drive delta with Prefer behaviors --------- + Sync-MgxDelta /me/drive/root/delta -DeltaPath drive.delta -Prefer deltashowremovedasdeleted + + Removed drive items carry the `deleted` facet instead of appearing as bare deletions. Note: `deltaExcludeParent` is a standalone request header, not a Prefer token - pass it as `-Headers @{ deltaExcludeParent = "true" }`. + + diff --git a/module/en-US/about_Mgx.help.txt b/module/en-US/about_Mgx.help.txt index fd814bb..e951a42 100644 --- a/module/en-US/about_Mgx.help.txt +++ b/module/en-US/about_Mgx.help.txt @@ -54,6 +54,13 @@ LONG DESCRIPTION Expand-MgxRelation enriches Graph objects with related data via concurrent fan-out, handling both collection and singleton endpoints. + Content downloads + Get-MgxContent downloads file and media content whole or by byte + range. Graph answers a content request with a redirect to a + pre-authenticated download host; that second hop carries no bearer + token, its host is validated against an allowlist before anything is + sent, and the capability URL is kept out of debug output. + CMDLETS Invoke-MgxRequest General-purpose resilient client for any Microsoft Graph endpoint. @@ -70,6 +77,10 @@ CMDLETS Sync-MgxDelta Incremental sync via Microsoft Graph delta queries. + Get-MgxContent + Download file and media content, whole or by byte range, over the + two-hop pre-authenticated download path. + Enable-MgxResilience Inject Polly resilience into the Microsoft.Graph SDK HTTP transport. @@ -90,7 +101,23 @@ CMDLETS RESILIENCE All Mgx cmdlets share a single Polly resilience pipeline composed of - four layers: + four layers, preceded by an adaptive pacing gate: + + Adaptive pacing gate (on by default) + Spaces requests BEFORE they are sent, rather than reacting after a + 429. It learns a rate per workload from throttling signals - + additive increase while clean, multiplicative decrease on a + throttle - and keeps Drive, Directory and other workloads in + separate buckets, so one throttled workload does not slow the + rest. A cold session starts conservatively and ramps up. + + It applies to every request, including the Enable-MgxResilience + SDK-bridge path. Batch outer POSTs are the one exemption: batching + runs its own item-level AIMD, and two controllers on one workload + would compound their backoff. + + Disable with Set-MgxOption -NoAdaptivePacing. Inspect the learned + state with Get-MgxTelemetry. Rate limiter (token bucket) Prevents outbound request bursts from triggering Graph API @@ -196,6 +223,7 @@ SEE ALSO Export-MgxCollection Expand-MgxRelation Sync-MgxDelta + Get-MgxContent Enable-MgxResilience Disable-MgxResilience Get-MgxResilience diff --git a/module/en-US/about_Mgx_Tuning.help.txt b/module/en-US/about_Mgx_Tuning.help.txt index 750b813..99e57f3 100644 --- a/module/en-US/about_Mgx_Tuning.help.txt +++ b/module/en-US/about_Mgx_Tuning.help.txt @@ -23,6 +23,43 @@ CLIENT-SIDE RATE LIMITING (ON BY DEFAULT) Disable explicitly with Set-MgxOption -NoRateLimit. +ADAPTIVE PACING (ON BY DEFAULT) + + A proactive gate ahead of the token bucket. The bucket enforces a hard + request-rate ceiling; the pacer slows down BEFORE Graph throttles, based + on what the tenant is signaling. Independent of -NoRateLimit; disable + explicitly with Set-MgxOption -NoAdaptivePacing. + + State is kept per workload (drive/SharePoint, directory, other), so a + throttle on one service never slows an unrelated workload in the same + session. Three mechanisms, all inactive (zero delay) in steady state: + + - AIMD back-off. A 429 caps the workload's request rate (4 req/sec on + the first throttle, halving on repeat, floor 2). The cap climbs back + a tenth of the ceiling per clean second and expires outright after + five quiet minutes. A server Retry-After additionally holds ALL + requests for that workload, not just the throttled one. + + - Slow start. A workload that is cold (first use, or idle five minutes) + opens with a 4 req/sec rate cap that doubles each clean second until + it reaches the ceiling. The first request is never delayed; the cap + only bites when demand exceeds it, i.e. fan-outs -- the pattern that + otherwise throttles before completing a single item. + + - Throttle-proximity damping. When Graph reports + x-ms-throttle-limit-percentage (documented to appear above 80% of + budget), requests are spaced with a delay that ramps from 0ms at 0.8 + to 2000ms at 1.2, while the report stays fresh (30s). + + Batch outer /$batch POSTs are exempt from the gate -- batch throughput + is owned by BatchItemsPerSecond and its item-level pacing above -- but + their responses still feed the pacer's signals. + + Watch it work: Get-MgxTelemetry shows total pacing wait, activation + count, the last reported throttle percentage, and a per-workload state + line (active caps, slow start, latency vs. baseline). Run any cmdlet + with -Verbose to see individual pacing waits. + RETRY BUDGET CAPS Setting Default Hard Max Purpose diff --git a/module/help/Enable-MgxResilience.md b/module/help/Enable-MgxResilience.md index c8edef5..28bbfbb 100644 --- a/module/help/Enable-MgxResilience.md +++ b/module/help/Enable-MgxResilience.md @@ -102,7 +102,7 @@ This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable ### None ## NOTES -The SDK's built-in RetryHandler still runs inside the wrapped chain. Retries can compound, bounded by the total timeout (default 300s) and circuit breaker. +The SDK's built-in RetryHandler still runs inside the wrapped chain, and it answers 429 and 503 itself. Two consequences, both measured: retries compound (four intended attempts can reach the wire eight times, because the SDK sleeps Retry-After inside the call and Mgx's own attempt timeout then fires), and a throttle never reaches Mgx's pipeline - so the adaptive pacer does not learn from it and Get-MgxTelemetry reports ThrottleRetries as 0 through a live throttle, booking the waiting as OtherRetries instead. The total timeout (default 300s) bounds the compounding; the circuit breaker does not, because the exhaustion path surfaces as an AggregateException that its predicate does not match. Use the Invoke-Mgx* cmdlets where throttle-aware pacing matters. ## RELATED LINKS [Disable-MgxResilience](Disable-MgxResilience.md) diff --git a/module/help/Get-MgxContent.md b/module/help/Get-MgxContent.md new file mode 100644 index 0000000..4de0601 --- /dev/null +++ b/module/help/Get-MgxContent.md @@ -0,0 +1,248 @@ +--- +external help file: Mgx.Cmdlets.dll-Help.xml +Module Name: Mgx +online version: https://github.com/gromedev/mgx/blob/main/module/help/Get-MgxContent.md +schema: 2.0.0 +--- + +# Get-MgxContent + +## SYNOPSIS +Fetch content bytes from Graph $value / /content endpoints, whole or as a byte range. + +## SYNTAX + +### Uri (Default) +``` +Get-MgxContent [-Uri] [-First ] [-Offset ] [-Length ] + [-OutFile ] [-ApiVersion ] [-Headers ] + [-ProgressAction ] [] +``` + +### InputObject +``` +Get-MgxContent -InputObject [-First ] [-Offset ] [-Length ] + [-OutFile ] [-ApiVersion ] [-Headers ] + [-ProgressAction ] [] +``` + +## DESCRIPTION +Get-MgxContent downloads file content from Microsoft Graph content endpoints - drive items (`/drives/{id}/items/{id}/content`), mail attachments (`.../attachments/{id}/$value`), profile photos (`/me/photo/$value`), and anything else that serves bytes. + +Ranged reads are the point: `-First 262144` pulls the first 256 KB of a file (enough for format sniffing, EXIF, or partial hashing) instead of the whole thing. Across 20k files that is gigabytes moved instead of tens of gigabytes. + +Under the hood, drive content is a two-hop flow: the authenticated Graph request (full resilience pipeline, adaptive pacing, rate limiting) answers with a redirect to a short-lived pre-authenticated download URL, which mgx fetches with a separate token-free HTTP client. The bearer token never reaches the download host, and the host is validated against a Microsoft-hosts allowlist (SharePoint/OneDrive download domains) before any bytes move - including on every mid-download redirect. + +Output is a single `byte[]` to the pipeline (guarded at 100 MB - use -OutFile beyond that), or a file written via temp-plus-atomic-move so a failed download never truncates an existing file. + +Piped DriveItems (from Invoke-MgxRequest or Sync-MgxDelta) use their `@microsoft.graph.downloadUrl` directly when present - no extra Graph round trip - falling back to `/drives/{driveId}/items/{id}/content` from the item's `id` and `parentReference.driveId`. + +Some endpoints ignore the Range header and answer 200 with the full body (profile photos do this). Get-MgxContent detects that, keeps only the requested bytes, and aborts the rest of the transfer. + +## EXAMPLES + +### Example 1: Download a file +```powershell +Get-MgxContent "/me/drive/items/01ABC/content" -OutFile ./report.xlsx +``` + +Downloads the drive item to a local file. + +### Example 2: First 256 KB of every file in a folder (format sniffing / partial hash) +```powershell +Invoke-MgxRequest "/me/drive/items/01FOLDER/children" -All | + Where-Object { $_.file } | + ForEach-Object { + $bytes = $_ | Get-MgxContent -First 262144 + [PSCustomObject]@{ name = $_.name; header = [BitConverter]::ToString($bytes[0..7]) } + } +``` + +Moves kilobytes per file instead of the full content - the metadata-extraction economy case. + +### Example 3: A specific byte range +```powershell +Get-MgxContent "/me/drive/items/01ABC/content" -Offset 1048576 -Length 65536 +``` + +Returns bytes 1,048,576 through 1,114,111 (Range: bytes=1048576-1114111). + +### Example 4: A mail attachment +```powershell +Get-MgxContent "/me/messages/AAMk.../attachments/AAMk.../`$value" -OutFile ./attachment.pdf +``` + +Attachment bytes are served directly by Graph (no download-host hop). + +### Example 5: Profile photo +```powershell +Get-MgxContent "/me/photo/`$value" -OutFile ./me.jpg +``` + +Note the backtick: `$value` would otherwise be expanded by PowerShell. + +## PARAMETERS + +### -Uri +Relative Graph path to a content endpoint (e.g., /me/drive/items/{id}/content). Warns when the path does not look like a content endpoint (/content or /$value). + +```yaml +Type: String +Parameter Sets: Uri +Aliases: Resource + +Required: True +Position: 0 +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -InputObject +A DriveItem-shaped object from the pipeline. Uses `@microsoft.graph.downloadUrl` when present (validated against the download-host allowlist), else `/drives/{parentReference.driveId}/items/{id}/content`. + +```yaml +Type: Object +Parameter Sets: InputObject +Aliases: + +Required: True +Position: Named +Default value: None +Accept pipeline input: True (ByValue) +Accept wildcard characters: False +``` + +### -First +First N bytes (Range: bytes=0..N-1). Mutually exclusive with -Offset/-Length. + +```yaml +Type: Int64 +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Offset +Range start in bytes; requires -Length. + +```yaml +Type: Int64 +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: 0 +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Length +Range length in bytes, starting at -Offset (default 0). + +```yaml +Type: Int64 +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -OutFile +Write the content to this file (temp + atomic move) instead of emitting byte[] to the pipeline. Required for content larger than 100 MB. + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ApiVersion +Graph API version. Default: v1.0. + +```yaml +Type: String +Parameter Sets: (All) +Aliases: +Accepted values: v1.0, beta + +Required: False +Position: Named +Default value: v1.0 +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Headers +Custom request headers for the Graph request (hop 1 only; never sent to the download host). + +```yaml +Type: Hashtable +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -ProgressAction +Determines how the cmdlet responds to progress updates. + +```yaml +Type: ActionPreference +Parameter Sets: (All) +Aliases: proga + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### CommonParameters +This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see [about_CommonParameters](http://go.microsoft.com/fwlink/?LinkID=113216). + +## INPUTS + +### System.Object +DriveItem-shaped Hashtables or PSCustomObjects (from Invoke-MgxRequest or Sync-MgxDelta). + +## OUTPUTS + +### System.Byte[] +The content bytes as a single array (not enumerated). Nothing is emitted when -OutFile is used. + +## NOTES +Content downloads require the mgx-owned HTTP transport. If mgx had to fall back to the Graph SDK's client (rare - it means the clean client could not be built), Get-MgxContent refuses with ContentRequiresOwnedTransport rather than let a transport that auto-follows redirects fetch from an unvalidated host. + +Graph reports drive-item hashes as quickXorHash in base64; rclone and many tools print hex. Convert before comparing: +`[System.Convert]::ToHexString([System.Convert]::FromBase64String($item.file.hashes.quickXorHash))` + +The pre-authenticated download URL on piped items is short-lived (~1 hour). On expiry mgx re-requests it automatically for the /content path; for stale piped `@microsoft.graph.downloadUrl` values, re-fetch the item. + +Intune report export jobs are not a /content flow: the completed job's `url` property is a pre-authenticated Azure Blob URL - download it directly with Invoke-WebRequest; it needs nothing from mgx. + +## RELATED LINKS +[Invoke-MgxRequest](Invoke-MgxRequest.md) +[Sync-MgxDelta](Sync-MgxDelta.md) +[Get-MgxTelemetry](Get-MgxTelemetry.md) diff --git a/module/help/Get-MgxOption.md b/module/help/Get-MgxOption.md index bff4bb9..93f89c3 100644 --- a/module/help/Get-MgxOption.md +++ b/module/help/Get-MgxOption.md @@ -46,7 +46,7 @@ This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable ## OUTPUTS -### System.Management.Automation.PSObject +### Mgx.Cmdlets.Models.MgxOptionOutput A PSObject with properties for each configuration setting. ## NOTES diff --git a/module/help/Get-MgxTelemetry.md b/module/help/Get-MgxTelemetry.md index f73c966..09d6ff1 100644 --- a/module/help/Get-MgxTelemetry.md +++ b/module/help/Get-MgxTelemetry.md @@ -19,6 +19,8 @@ Get-MgxTelemetry [-Reset] [] ## DESCRIPTION Get-MgxTelemetry returns accumulated telemetry from the current session's resilience pipeline. Reports request counts, retry/throttle breakdown, and timing per category so you can determine whether the bottleneck is throttling, retry backoff, rate-limiter queuing, or network latency. +Adaptive pacing surfaces here too: AdaptivePacingWaitMs / AdaptivePacingActivations count proactive waits, LastThrottlePercentage is the most recent x-ms-throttle-limit-percentage Graph reported (-1 = never seen), and PacingState is a per-workload line showing active rate caps, slow start, throttle proximity, and current latency against the session baseline. -Reset clears the counters but deliberately not the pacer's learned rate caps - those describe the tenant's current throttle regime, not accumulated statistics. + ## EXAMPLES ### Example 1: View session telemetry @@ -38,11 +40,18 @@ Returns current telemetry and resets all counters to zero. ### Example 3: Check throttle rate ```powershell $t = Get-MgxTelemetry -"Throttle rate: $([math]::Round($t.ThrottleCount / $t.TotalRequests * 100, 1))%" +"Throttle rate: $(if ($t.Requests) { [math]::Round($t.ThrottleRetries / $t.Requests * 100, 1) } else { 0 })%" ``` Calculates the percentage of requests that were throttled. +### Example 4: Watch adaptive pacing during a fan-out +```powershell +Get-MgxTelemetry | Select-Object AdaptivePacingWaitMs, AdaptivePacingActivations, LastThrottlePercentage, PacingState +``` + +Shows how much proactive pacing has happened and the current per-workload pacing state. + ## PARAMETERS ### -Reset @@ -69,7 +78,7 @@ This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable ## OUTPUTS -### System.Management.Automation.PSObject +### Mgx.Cmdlets.Models.MgxTelemetryOutput Telemetry summary with request counts, retry/throttle breakdown, and timing. ## NOTES diff --git a/module/help/Invoke-MgxBatchRequest.md b/module/help/Invoke-MgxBatchRequest.md index d2e66fb..6dc3209 100644 --- a/module/help/Invoke-MgxBatchRequest.md +++ b/module/help/Invoke-MgxBatchRequest.md @@ -160,7 +160,7 @@ Accept wildcard characters: False ``` ### -DeadLetterPath -Path to a JSONL file where failed batch items (status >= 400) are appended. Each line contains Url, Method, Body (with sensitive fields redacted), Status, Error, and Timestamp. The file can be re-piped to Invoke-MgxBatchRequest for retry: `Get-Content dead.jsonl | ConvertFrom-Json | Invoke-MgxBatchRequest`. +Path to a JSONL file where failed batch items (status >= 400) are appended, along with any that were never sent because an earlier chunk failed. Each line contains Url, Method, Body (with sensitive fields redacted), Status, Error, and Timestamp. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Read it to decide what to resubmit, and supply the sensitive fields again yourself. ```yaml Type: String @@ -309,7 +309,7 @@ String URLs, or hashtables or PSCustomObjects with Url, Method, and Body members ## OUTPUTS ### System.Collections.Hashtable -Per-request results with Url, Method, Status, and Body keys. Results can be piped back into this cmdlet to retry failed items. +Per-request results with Url, Method, Status, and Body keys, plus NotSent on any operation that was never sent because an earlier chunk failed. Body here is the RESPONSE body - for a failure that is the error envelope, not the request - so piping results straight back resubmits the wrong thing. Use Url and Method to rebuild the requests you want to retry. ## NOTES Each batch item is retried individually on 429 (throttled) or 5xx errors (for idempotent methods). POST requests only retry on 429 because POST is non-idempotent - retrying a failed POST on 5xx could create duplicates if the server processed the request before the error. This matches the Kiota SDK retry behavior. Source: [Microsoft Graph error responses and resource types](https://learn.microsoft.com/en-us/graph/errors) diff --git a/module/help/Set-MgxOption.md b/module/help/Set-MgxOption.md index 0885172..6341a2d 100644 --- a/module/help/Set-MgxOption.md +++ b/module/help/Set-MgxOption.md @@ -13,7 +13,7 @@ Configure resilience options for all Mgx cmdlets. ## SYNTAX ``` -Set-MgxOption [-RateLimitBurst ] [-RateLimitPerSecond ] [-NoRateLimit] +Set-MgxOption [-RateLimitBurst ] [-RateLimitPerSecond ] [-NoRateLimit] [-NoAdaptivePacing] [-RateLimitQueueLimit ] [-MaxRetryAfterSeconds ] [-MaxRetryAttempts ] [-TotalTimeoutSeconds ] [-AttemptTimeoutSeconds ] [-CircuitBreakerDurationSeconds ] [-CircuitBreakerFailureRatio ] [-CircuitBreakerMinThroughput ] @@ -57,6 +57,21 @@ Restores all options to their default values. ## PARAMETERS +### -BatchChunkConcurrency +Number of $batch chunks executed concurrently. The default of 1 runs chunks sequentially, which is what keeps inter-chunk pacing meaningful; raising it trades that pacing for throughput and makes server-side write throttling more likely. Range: 1-10. Default: 1. + +```yaml +Type: Int32 +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: 1 +Accept pipeline input: False +Accept wildcard characters: False +``` + ### -BatchItemsPerSecond Target throughput for batch item pacing in items/sec. Controls inter-chunk delay in sequential batch execution to avoid burst-and-stall against Graph's server-side write throttle. Set to 0 to disable pacing. Range: 0-1000. Default: 20. @@ -178,7 +193,22 @@ Accept wildcard characters: False ``` ### -NoRateLimit -Disable the client-side rate limiter entirely. +Disable the client-side rate limiter entirely. This also disables batch item pacing, so any -BatchItemsPerSecond you set alongside it has no effect; Set-MgxOption warns when both are given. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -NoAdaptivePacing +Disable adaptive request pacing (AIMD back-off after 429s, slow start on cold workloads, and throttle-proximity damping). Pacing is ON by default and independent of -NoRateLimit: the token bucket is the hard backstop, the pacer is the proactive layer in front of it. See about_Mgx_Tuning "ADAPTIVE PACING". ```yaml Type: SwitchParameter diff --git a/module/help/Sync-MgxDelta.md b/module/help/Sync-MgxDelta.md index 027646c..4465403 100644 --- a/module/help/Sync-MgxDelta.md +++ b/module/help/Sync-MgxDelta.md @@ -13,19 +13,24 @@ Incremental sync via Microsoft Graph delta queries. ## SYNTAX ``` -Sync-MgxDelta [-Uri] -DeltaPath [-Property ] [-Filter ] [-Top ] - [-OutputFile ] [-FullSync] [-ApiVersion ] [-Headers ] +Sync-MgxDelta [-Uri] -DeltaPath [-Property ] [-Filter ] + [-Prefer ] [-Top ] [-OutputFile ] [-FullSync] [-Latest] + [-CheckpointPath ] [-ApiVersion ] [-Headers ] [-ProgressAction ] [] ``` ## DESCRIPTION Sync-MgxDelta retrieves incremental changes from Microsoft Graph delta endpoints. On the first run, it performs a full sync and saves a delta token. Subsequent runs retrieve only items that changed since the last sync. -Delta tokens are saved to the file specified by -DeltaPath and persist across successful completions. This is different from -CheckpointPath (used by Export-MgxCollection) which is ephemeral and deleted on success. +Delta tokens are saved to the file specified by -DeltaPath and persist across successful completions. -CheckpointPath is different: it is an ephemeral mid-run resume position, saved at page boundaries while an enumeration is in flight and deleted the moment it completes. Use both on long enumerations - a killed 3-hour drive sync resumes where it stopped instead of starting over. -Items that no longer match the query appear with an `@removed` property containing `{"reason": "changed"}` (item moved out of scope or soft-deleted) or `{"reason": "deleted"}` (permanently deleted). Filter these with `Where-Object { -not $_.'@removed' }`. +Drive delta is fully supported: `/me/drive/root/delta`, `/drives/{id}/root/delta`, `/users/{id}/drive/root/delta`, and `/groups/{id}/drive/root/delta` all work, including drive-specific `-Prefer` behaviors and the drive form of `-Latest`. -Delta tokens expire after approximately 7 days for directory objects (users, groups, applications). When a token expires, Graph returns HTTP 410 Gone. Sync-MgxDelta handles this automatically by deleting the stale token and performing a full re-sync with a warning. +Items that no longer match the query appear with an `@removed` property containing `{"reason": "changed"}` (item moved out of scope or soft-deleted) or `{"reason": "deleted"}` (permanently deleted). Filter these with `Where-Object { -not $_.'@removed' }`. The completion message counts them separately. + +Delta tokens expire after approximately 7 days for directory objects (users, groups, applications). When a token expires, Graph returns HTTP 410 Gone. Sync-MgxDelta handles this automatically by deleting the stale token (and any resume checkpoint) and performing a full re-sync with a warning. + +Resume semantics are at-least-once: in pipeline mode the page in flight at a crash is re-emitted in full on resume; in JSONL mode mid-page checkpoints keep the window to at most 500 items. Deduplicate on `id` downstream if exact-once matters. Source: [Use delta query to track changes in Microsoft Graph data](https://learn.microsoft.com/en-us/graph/delta-query-overview) @@ -76,6 +81,29 @@ Write-Host "$($changed.Count) changed, $($removed.Count) removed" Suitable for scheduled tasks. If the delta token expires (>7 days since last run), Sync-MgxDelta automatically performs a full re-sync and warns. +### Example 7: Enumerate a whole drive, resumable +```powershell +Sync-MgxDelta /me/drive/root/delta -DeltaPath drive.delta -CheckpointPath drive.checkpoint -OutputFile drive.jsonl +``` + +First run enumerates the entire drive to JSONL. If the run is killed at any point, re-running the identical command resumes from the last checkpointed page instead of starting over. Subsequent runs after completion return only changed items. + +### Example 8: Baseline from now, then track changes only +```powershell +Sync-MgxDelta /groups/delta -DeltaPath groups.delta -Latest # instant baseline, no data +# ...later... +Sync-MgxDelta /groups/delta -DeltaPath groups.delta # only what changed since +``` + +-Latest skips the initial full enumeration and stores a "from now" token. The form is chosen automatically: drive resources use `?token=latest`, directory and everything else `$deltatoken=latest`. + +### Example 9: Drive delta with Prefer behaviors +```powershell +Sync-MgxDelta /me/drive/root/delta -DeltaPath drive.delta -Prefer deltashowremovedasdeleted +``` + +Removed drive items carry the `deleted` facet instead of appearing as bare deletions. Note: `deltaExcludeParent` is a standalone request header, not a Prefer token - pass it as `-Headers @{ deltaExcludeParent = "true" }`. + ## PARAMETERS ### -ApiVersion @@ -94,6 +122,21 @@ Accept pipeline input: False Accept wildcard characters: False ``` +### -CheckpointPath +Path for the ephemeral mid-run resume checkpoint. Saved at page boundaries (and every 500 items in JSONL mode) while an enumeration is in flight; deleted on successful completion. Any event that invalidates the enumeration - HTTP 410 Gone, -FullSync, or a -Property/-Filter/-Prefer change - deletes it too. Must differ from -DeltaPath and -OutputFile. + +```yaml +Type: String +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + ### -DeltaPath Path to the delta state file. This file persists across successful completions and tracks the sync position. JSON format, human-readable. @@ -139,6 +182,36 @@ Accept pipeline input: False Accept wildcard characters: False ``` +### -Latest +Baseline without enumerating: request only the latest delta token ("sync from now"). Returns no data; the next run returns everything that changed since. Drive resources take `?token=latest`, directory and other resources `$deltatoken=latest` - the form is chosen automatically from the URI shape. Ignored with a warning when usable delta state already exists (delete it or use -FullSync to re-baseline). + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: False +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Prefer +Prefer-header tokens joined into a single Prefer header on every page request. Tab completion offers the documented drive delta tokens: `deltashowremovedasdeleted`, `deltatraversepermissiongaps`, `deltashowsharingchanges` (requires the other two and Sites.FullControl.All), and `hierarchicalsharing`. A change against the stored state forces a full re-sync, like -Property and -Filter. `deltaExcludeParent` is a standalone request header, not a Prefer token - pass it via -Headers. + +```yaml +Type: String[] +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + ### -Headers Custom request headers applied to each page request. @@ -248,6 +321,21 @@ Query parameters ($select, $filter) are encoded into the delta token on the firs Supported delta endpoints include: /users/delta, /groups/delta, /applications/delta, /servicePrincipals/delta, /devices/delta, /directoryRoles/delta, and many others. Source: [Use delta query to track changes in Microsoft Graph data](https://learn.microsoft.com/en-us/graph/delta-query-overview) +DUPLICATE OBJECTS ACROSS PAGES + +Graph does not guarantee that an object appears only once in a delta response: it "can't ensure that entities are unified in a single response." Deduplicate by `id` before treating emitted objects as change events. + +This is most visible on an initial full enumeration. Measured against a tenant with 15,779 groups, `/groups/delta` emitted 156,413 objects across all pages - a replay factor of roughly 10x - with every object accounted for by a repeat rather than a distinct group. Incremental rounds off an established token are typically clean, but nothing in the contract promises that, so the dedup belongs in the consumer either way. + +Two consequences worth designing around: + +- For a change feed, baseline with `-Latest` so the initial enumeration never happens. It records a sync-from-now token and returns nothing, and every later run reports only real changes. +- When counting, count distinct ids. An object that changed twice inside one window is one object; keeping the last occurrence gives its current state. + +`-CheckpointPath` writes item counts as emitted, not deduplicated, so a resumed run's progress figures reflect the same replay. + +Source: [Use delta query to track changes](https://learn.microsoft.com/en-us/graph/delta-query-overview#other-considerations) + ## RELATED LINKS [Export-MgxCollection](Export-MgxCollection.md) [Invoke-MgxRequest](Invoke-MgxRequest.md) diff --git a/module/mgx.Format.ps1xml b/module/mgx.Format.ps1xml index 5951df3..853b5e5 100644 --- a/module/mgx.Format.ps1xml +++ b/module/mgx.Format.ps1xml @@ -75,6 +75,7 @@ RateLimitBurst RateLimitPerSecond NoRateLimit + NoAdaptivePacing RateLimitQueueLimit MaxRetryAttempts MaxRetryAfterSeconds @@ -114,6 +115,11 @@ TotalElapsedMs ResourceUnitsConsumed BatchItemThrottles + AdaptivePacingWaitMs + AdaptivePacingActivations + if ($_.LastThrottlePercentage -lt 0) { 'never seen' } else { '{0:P0}' -f $_.LastThrottlePercentage } + if ($null -eq $_.PacingState) { 'idle' } else { $_.PacingState } + ContentBytesDownloaded diff --git a/module/mgx.psd1 b/module/mgx.psd1 index fc9ce7c..ccb713f 100644 --- a/module/mgx.psd1 +++ b/module/mgx.psd1 @@ -1,6 +1,6 @@ @{ RootModule = 'mgx.psm1' - ModuleVersion = '2.0.1' + ModuleVersion = '2.1.0' GUID = 'a3f7e8d2-5b4c-4a1e-9f6d-2c8b0e3a7d5f' Author = 'Thomas Maillo Grome' CompanyName = 'Mgx' @@ -39,6 +39,7 @@ 'Get-MgxResilience' 'Get-MgxTelemetry' 'Sync-MgxDelta' + 'Get-MgxContent' ) FunctionsToExport = @() @@ -51,149 +52,66 @@ LicenseUri = 'https://github.com/gromedev/mgx/blob/main/LICENSE' ProjectUri = 'https://github.com/gromedev/mgx' ReleaseNotes = @' -v2.0.1 -Patch release. Three fixes for defects that break live sessions; no feature or API changes. - -- The rate limiter was disposed on a timer while live clients still held it. Any Set-MgxOption call left every Enable-MgxResilience session throwing ObjectDisposedException minutes later, recoverable only by disabling and re-enabling resilience. The delay came from TotalTimeoutSeconds, a per-request timeout used as a resource lifetime. -- Enable-MgxResilience and Disable-MgxResilience disposed the injected HTTP client synchronously, cancelling SDK requests that were still in flight - including a paged read running when the user disabled resilience. Re-injection is not always user-initiated: an identity change triggers it, and the check compares the auth context by instance, so a same-tenant Connect-MgGraph re-run is enough. -- Sync-MgxDelta -Uri with a query string broke incremental sync. The resource-path check compared a value that keeps the query against one that never has it, so the first run succeeded and saved state while the second terminated with a SecurityError accusing that state file of tampering. "/users/delta?$select=id" is the shape Microsoft's delta documentation shows. - -v2.0.0 -Breaking release. Output shape changes to Hashtable; the floor moves DOWN to PowerShell 7.4. -Incorporates the breaking work contributed in the Microsoft365DSC fork by Fabien Tschanz. -- BREAKING: Invoke-MgxRequest, Invoke-MgxBatchRequest, Expand-MgxRelation and Sync-MgxDelta - emit case-insensitive Hashtables instead of PSObjects, matching Invoke-MgGraphRequest, so - results drop into scripts written for the Graph SDK. Migration for code that needs the old - shape: -Raw | ConvertFrom-Json -- BREAKING: @odata.type is returned verbatim under its own key instead of as ODataType. All - other @odata.* metadata is stripped, including @odata.etag: it changes on every write, so - keeping it makes two reads of an unchanged entity compare unequal. Use -Raw for the etag -- BREAKING: the six Graph entity format views are removed - PowerShell renders any dictionary - with its built-in Name/Value view, so a custom view can never apply. Key order is undefined: - use Select-Object or Format-Table to fix column order -- The floor moves DOWN: targets .NET 8 and supports PowerShell 7.4 (LTS) and later, where 1.x - required 7.5. Nothing ever needed the newer runtime - the net9.0 target dated to 1.0.1 and was - never code-driven. The single .NET 9 API (LoadIntoBufferAsync(CancellationToken), in the -Debug - tracer) is replaced with a WaitAsync equivalent that keeps the same timeout behaviour -- BREAKING: Microsoft.Graph.Authentication is no longer in RequiredModules. Auth was already - resolved reflectively at call time, so declaring it only forced the SDK onto every consumer - and installed a second copy beside one already loaded. mgx now imports without it; cmdlets - needing a token report GraphAuthModuleNotLoaded with install instructions -- Collection envelopes are unwrapped on every response path, so an action endpoint that - answers a write with {"value":[...]} (for example /directoryObjects/getByIds) now emits one - object per element instead of the envelope -- Pipeline input accepts the new shape end to end: fan-out reads 'id' from Hashtables and - PSCustomObjects instead of stringifying them into the URL, and Invoke-MgxBatchRequest - accepts its own output, so failed items can be piped straight back in for retry -- Polly.Core 8.7.0, System.Threading.RateLimiting 10.0.10 -v1.0.5 -Fixes and hardening driven by a rebuilt benchmark suite run against a seeded 100k-user tenant. -- Fixed Export-MgxCollection losing all progress when a first run was interrupted: a cancelled - run now promotes its temp file with a checkpoint matching its exact content, and a killed or - crashed run is recovered on the next invocation from the orphaned temp file, trimmed to the - checkpointed item count. The "Resume with:" hint is now truthful for first runs -- Fixed Get-MgxTelemetry reporting zero retry delay for batch-heavy sessions: per-item - Retry-After waits inside $batch processing now flow into RetryDelayMs -- The count-discrepancy warning now also fires when an enumeration returns more items than - @odata.count reported (detects transient duplicated pages; dedup on id downstream) -- Documented write-cost pacing in about_Mgx_Tuning: Graph throttles writes, not items - - budget BatchItemsPerSecond by writes per item (a 20-member group create costs ~21 writes) -- New reproducible benchmark suite under tests/benchmarks, including a fault-injection - gauntlet that runs against a local mock Graph and needs no tenant; README rebuilt on the - suite's measured results -v1.0.4 -Fixes ported from the Microsoft365DSC fork, contributed by Fabien Tschanz. -- Fixed Mgx cmdlets keeping the credentials of the first Connect-MgGraph call: the cached - HTTP client was keyed on tenant id alone, so reconnecting to the same tenant with a - different application, certificate, account, or scope set silently reused the previous - identity. The client is now keyed on a fingerprint of the full auth context -- Fixed a JSON string passed to -Body being silently dropped (sent as {}), so a piped or - ConvertTo-Json body now serializes correctly -- Fixed Enable-MgxResilience staying bound to the pre-reconnect SDK client; resilience is - re-injected when the Graph identity changes -- Fixed Set-MgxOption -TotalTimeoutSeconds not reaching the HTTP client -- Fixed a single 429 slowing Invoke-MgxBatchRequest for the rest of the session; the write - rate now recovers after clean chunks and resets after five quiet minutes -- Fixed the internal type cache never invalidating when Microsoft.Graph.Authentication was - re-imported at a different version -- Fixed JSON integers above 2^53 losing precision -- Added -Debug request/response tracing on all cmdlets, with credential redaction -- A batch item with an invalid JSON body now fails on its own instead of aborting the batch; - -Body on GET now warns instead of being silently ignored -- The SdkVersion header is now derived from the assembly version instead of a hand-maintained - constant +v2.1.0 +Adds proactive throttle avoidance, ranged content downloads, and resumable enumeration. -v1.0.3 -- Fixed Remove-Module Mgx failing and leaving the module loaded (cleanup now runs before the - ALC resolver detaches). Only triggered when no Graph request had run in the session -- Fixed the SdkVersion request header reporting mgx/0.3.0 regardless of installed version -- Internal: cmdlet lifecycle and JSON conversion extracted to MgxCmdletCore. No surface change +Added +- Adaptive request pacing, on by default. Requests are spaced before they are sent, per workload; opt out with Set-MgxOption -NoAdaptivePacing. +- Get-MgxContent, to download file and media content whole or by byte range (-First). +- Sync-MgxDelta -CheckpointPath, to resume interrupted enumerations. +- Sync-MgxDelta -Latest, to baseline state without enumerating. +- Sync-MgxDelta -Prefer, to send drive delta Prefer tokens. +- Resource units consumed, in telemetry output, benchmarks and examples. -v1.0.2 -- Fixed Linux install: renamed module files to lowercase so Install-Module works on case-sensitive filesystems -- Updated about_Mgx_Tuning version reference +Fixed +- -Latest was honored after a state invalidation, dropping every change since the last sync. +- Delta resume dropped the interrupted run's items when an earlier run had already completed, and advanced the delta token past them. +- Export resume dropped the interrupted run's items when an earlier export had already written the output file. +- Export resume wrote a page's items twice when a run was interrupted twice inside the same page. +- Delta resume kept the previous sync's rows in front of the interrupted sync's output. +- Export resume started the export over when the interrupted run had died on a transient error. +- A resume checkpoint naming a file that was never a temp consumed it as data. +- A refused delta token was reported as an endpoint not supporting delta queries. +- Sync-MgxDelta as a session's first cmdlet sent its request to the public cloud endpoint. +- A denied or read-only output file ended a delta sync with an unhandled error on Windows. +- A missing drive item suggested retrying in beta instead of reporting the item absent. +- -Top was ignored when combined with -All, returning the whole collection. +- Get-Help described output shapes and parameters from before 2.0.0. +- Enumeration returned part of a collection without error when a nextLink was refused. +- A delta sync stopped with an error when a page contained a non-object item. +- Get-MgxContent left an unwritable -OutFile as an unhandled error naming a temp path. +- A non-Graph JSON error body threw from inside the exception it was being used to build, surfacing a CDN failure as a security refusal. +- A $batch 429 set a pacing cap that nothing enforced and nothing cleared, so telemetry reported it forever. +- Get-MgxContent ignored Retry-After when a download host sent it as an HTTP date. +- Get-Help was missing the examples for -CheckpointPath, -Latest, -Prefer, adaptive pacing, and two Get-MgxContent endpoints, and did not document Set-MgxOption -BatchChunkConcurrency. +- Seven examples rendered blank tables, and one reported password secrets as certificates. +- Repeat 429s could raise the adaptive cap above -RateLimitPerSecond. +- Delta state did not record its API version, so runs omitting -ApiVersion silently synced the other one. +- -Debug wrote pre-authenticated download URLs verbatim, from redirect headers and from response bodies. +- Get-MgxContent -OutFile with piped input downloaded every item but kept only the last. +- -Offset was ignored when a server returned a whole-body 200. +- An offset past end of file overwrote an existing -OutFile with an empty one. +- Two-hop content downloads were reported as failed requests. +- Adaptive pacing fired synchronized bursts when sleep times were clamped. +- Enable-MgxResilience recorded no telemetry: request counts, HTTP time and rate-limiter wait all stayed 0. +- Invoke-MgGraphRequest with a relative URI failed while resilience was enabled. +- $batch envelopes were bucketed as Other rather than their target workload. +- Slow start opened above ceilings set below 4 rps. +- ResiliencePipelineFactory.Reset() reverted configuration and kept batch pacer state across credential changes. +- Ctrl-C during file cleanup raised disposed-object errors. -v1.0.1 -- Added tab completion for Uri parameter on all cmdlets -- Extracted CircuitBreakerMessage protected property on MgxCmdletBase -- Removed redundant XML doc comments on self-documenting members +Documentation +- Graph delta responses repeat objects across pages; deduplicate on id, or baseline with -Latest. +- Adaptive pacing applies to every request except batch outer POSTs. +- Measured throttling behavior: resource unit budgets are scoped per application and tenant, and x-ms-throttle-limit-percentage is not emitted. -v1.0.0 - Initial public release - -v0.3.0 - Plug-and-play resilience -- NEW: Expand-MgxRelation: pipeline-composable relation enrichment - - Enrich Graph objects with related data via concurrent fan-out - - Auto-detects collection vs singleton endpoints (no silent data loss) - - -Top caps per-relation items (server-side $top + client-side truncation) - - -Flatten unwraps single-value relations, warns on multi-item - - -SkipNotFound/-SkipForbidden for partial failure tolerance - - Chaining: pipe through multiple Expand-MgxRelation stages -- NEW: Enable-MgxResilience / Disable-MgxResilience / Get-MgxResilience: zero-change resilience injection - - Wraps SDK HttpClient with Polly retry, circuit breaker, and rate limiting - - All Microsoft.Graph SDK cmdlets (Get-MgUser, etc.) gain resilience automatically - - Preserves full SDK handler chain (OData, NationalCloud, Redirect, Auth) - - Re-call after Connect-MgGraph to re-inject - - Thread-safe for concurrent runspaces - - Sovereign cloud support (auto-detects Graph endpoint) -- NEW: Get-MgxOption / Set-MgxOption: runtime resilience pipeline tuning - - 11 configurable parameters: rate limiting, retry, circuit breaker, timeouts - - CircuitBreakerSamplingDurationSeconds: control the failure measurement window (5-300s) - - Partial updates: only explicitly passed values are changed - - Set-MgxOption -Reset restores all defaults -- FIXED: Circuit breaker and rate limiter now shared across cmdlet invocations - - Previously created fresh per invocation, making both non-functional - - New ResiliencePipelineFactory manages shared static Polly pipeline -- FIXED: Rate limiter GC root leak on options change - - TokenBucketRateLimiter holds an internal Timer (GC root via AutoReplenishment) - - Old limiters now disposed after TotalTimeoutSeconds delay to avoid ObjectDisposedException -- FIXED: nextLink SSRF validation strengthened - - Rejects non-HTTPS scheme (prevents token leak over plaintext) - - Compares full authority (host + port), not just host - - ConcurrentFanOut rejects all nextLinks if initial URL parse fails -- FIXED: FindType() cache no longer caches null results - - Previously, importing Mgx before Microsoft.Graph permanently broke lookups -- IMPROVED: Request cloning copies Options, Version, and all content headers on retry +v2.0.1 +Patch release. Three fixes; no feature or API changes. +- The rate limiter was disposed on a timer while live clients held it, so any Set-MgxOption call broke Enable-MgxResilience sessions minutes later. +- Enable-MgxResilience and Disable-MgxResilience cancelled SDK requests still in flight. +- Sync-MgxDelta -Uri with a query string or trailing slash failed on the second run. -v0.2.0 - Restructure as companion module -- NEW: Invoke-MgxRequest: general-purpose resilient client for any Graph endpoint - - Streaming pagination with -All, -Top, -PageSize - - Fan-out concurrency with {id} template substitution - - Write operations (POST, PATCH, PUT, DELETE) - - -ApiVersion (v1.0/beta), -ConsistencyLevel, -NoPageSize - - -SkipNotFound/-SkipForbidden for fan-out error handling - - -Raw for JSON string output, -CheckpointPath for resume - - Progress reporting, pipeline stop support - - ArgumentCompleters for tab completion -- IMPROVED: @odata.type preserved as ODataType property (polymorphic queries) -- IMPROVED: DateTime strings parsed to DateTimeOffset -- NEW: Export-MgxCollection: JSONL streaming export with checkpoint/resume - - Raw JSON to disk, no PSObject overhead, constant memory - - -All, -Top, -CheckpointPath for resumable exports - - SupportsShouldProcess for -WhatIf/-Confirm -- IMPROVED: Set-MgxOption only overrides explicitly passed values -- FIXED: Set-MgxOption referenced deleted base class -- REMOVED: Get-MgxUser, Get-MgxGroup, Get-MgxGroupMember, Get-MgxApplication, Get-MgxServicePrincipal, Get-MgxDirectoryRole - - Use Invoke-MgxRequest '/users', '/groups', etc. instead +Earlier releases: https://github.com/gromedev/Mgx/blob/main/CHANGELOG.md '@ } } diff --git a/module/mgx.psm1 b/module/mgx.psm1 index f0a276a..91565ce 100644 --- a/module/mgx.psm1 +++ b/module/mgx.psm1 @@ -36,6 +36,10 @@ $script:UriCompletions = @( @{ Text = 'drives'; Tip = 'OneDrive drives' } @{ Text = 'auditLogs/signIns'; Tip = 'Sign-in logs' } @{ Text = 'auditLogs/directoryAudits'; Tip = 'Directory audit logs' } + @{ Text = 'me/drive/root/delta'; Tip = 'Drive delta: your OneDrive' } + @{ Text = 'users/delta'; Tip = 'Delta: user changes' } + @{ Text = 'groups/delta'; Tip = 'Delta: group changes' } + @{ Text = 'servicePrincipals/delta'; Tip = 'Delta: service principal changes' } ) $script:UriCompleter = { @@ -45,6 +49,6 @@ $script:UriCompleter = { } } -foreach ($cmd in 'Invoke-MgxRequest', 'Invoke-MgxBatchRequest', 'Export-MgxCollection', 'Expand-MgxRelation', 'Sync-MgxDelta') { +foreach ($cmd in 'Invoke-MgxRequest', 'Invoke-MgxBatchRequest', 'Export-MgxCollection', 'Expand-MgxRelation', 'Sync-MgxDelta', 'Get-MgxContent') { Register-ArgumentCompleter -CommandName $cmd -ParameterName Uri -ScriptBlock $script:UriCompleter } diff --git a/src/Mgx.Cmdlets/Base/ArgumentCompleters.cs b/src/Mgx.Cmdlets/Base/ArgumentCompleters.cs index 6036d79..d9326c9 100644 --- a/src/Mgx.Cmdlets/Base/ArgumentCompleters.cs +++ b/src/Mgx.Cmdlets/Base/ArgumentCompleters.cs @@ -36,6 +36,31 @@ public IEnumerable CompleteArgument( } } +/// +/// Prefer-header tokens for drive delta queries. deltaExcludeParent is deliberately absent: +/// it is a standalone request header, not a Prefer token - pass it via +/// -Headers @{ deltaExcludeParent = "true" }. +/// +internal sealed class DeltaPreferCompleter : IArgumentCompleter +{ + public IEnumerable CompleteArgument( + string commandName, string parameterName, string wordToComplete, + CommandAst commandAst, + System.Collections.IDictionary fakeBoundParameters) + { + (string Value, string Tooltip)[] tokens = + [ + ("deltashowremovedasdeleted", "Removed items carry the deleted facet"), + ("deltatraversepermissiongaps", "Traverse permission gaps in the hierarchy"), + ("deltashowsharingchanges", "Annotate permission-driven changes (requires the other two and Sites.FullControl.All)"), + ("hierarchicalsharing", "Sharing information only for hierarchy roots and explicit changes"), + ]; + return tokens + .Where(t => t.Value.StartsWith(wordToComplete, StringComparison.OrdinalIgnoreCase)) + .Select(t => new CompletionResult(t.Value, t.Value, CompletionResultType.ParameterValue, t.Tooltip)); + } +} + internal sealed class ApiVersionCompleter : IArgumentCompleter { public IEnumerable CompleteArgument( diff --git a/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs b/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs index 63838d0..1625b7b 100644 --- a/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs +++ b/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs @@ -690,6 +690,14 @@ private static void ScheduleDelayedHttpClientDispose(HttpClient? client, bool ow }, TaskScheduler.Default); } + /// + /// Whether the active transport is the mgx-owned clean client (AllowAutoRedirect off) + /// rather than the borrowed SDK client. The content path requires ownership: the SDK + /// client ships a RedirectHandler that auto-follows a content 302 to a host mgx never + /// validated, so Get-MgxContent fails closed when this is false. + /// + protected static bool TransportIsOwned => s_ownsHttpClient; + /// /// Drain buffered verbose messages from the resilience pipeline. /// Must be called on the pipeline thread (after .GetAwaiter().GetResult() returns). @@ -725,6 +733,246 @@ internal static void SetClientOptions(ResilientGraphClientOptions options) s_clientOptions = options ?? ResilientGraphClientOptions.Default; } + /// + /// True when nothing else holds the file. FileShare.None is honoured between .NET processes + /// on both Windows and Unix, so a writer that has it open makes this fail rather than let a + /// sweep take a file out from under it. + /// + private static bool CanTakeExclusively(string path) + { + try + { + using var _ = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.None); + return true; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return false; + } + } + + /// + /// Remove leftover "{outputPath}.{guid}.tmp" files. Called only when no resume is pending, + /// where every such file is an orphan by definition - except one a live run still holds, + /// which CanTakeExclusively keeps out of reach. + /// + protected void DeleteStaleTemps(string outputPath) + { + try + { + var dir = Path.GetDirectoryName(outputPath); + if (string.IsNullOrEmpty(dir) || !Directory.Exists(dir)) return; + foreach (var stale in Directory.EnumerateFiles(dir, Path.GetFileName(outputPath) + ".*.tmp").ToList()) + { + // "Orphan" is an assumption about a file this run did not create, and a second + // export running against the same output right now owns a file matching the same + // glob. Windows refuses to delete a file someone holds open, so it declined by + // accident; Unix does not, and the other run went on writing into an unlinked + // inode and lost everything it had fetched. Ask for the file exclusively first - + // if that fails, someone is using it and it is not an orphan. + if (!CanTakeExclusively(stale)) + { + WriteVerbose($"Left '{Path.GetFileName(stale)}' alone: another run is writing to it."); + continue; + } + try + { + File.Delete(stale); + WriteVerbose($"Deleted an orphaned temp file from an earlier interrupted run: {Path.GetFileName(stale)}"); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + WriteWarning($"Could not delete orphaned temp file '{stale}': {ex.Message}. Delete it manually."); + } + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // Best effort; a sweep failure must never stop the run. + } + } + + /// + /// The temp a checkpoint names, or null when it cannot be used. A checkpoint is untrusted + /// input once it is on disk, so the recorded name must be one a run could actually have + /// written - "{output}.{32-hex}.tmp" - and must not be the output itself. Anything else + /// (the checkpoint file, the delta state, a crafted path) would be copied into the output + /// as data and then deleted as the spent temp. The file must also be at least as long as + /// the checkpoint promised, since a shorter one means the items it counted are not all + /// there. + /// + private static string? ResolveNamedTemp(string outputPath, string tempFileName, long dataLength) + { + if (dataLength <= 0) return null; + var dir = Path.GetDirectoryName(outputPath); + if (string.IsNullOrEmpty(dir) || !Directory.Exists(dir)) return null; + if (!string.Equals(tempFileName, Path.GetFileName(tempFileName), StringComparison.Ordinal)) + return null; + if (!IsRunTempName(Path.GetFileName(outputPath), tempFileName)) return null; + var tempPath = Path.Combine(dir, tempFileName); + if (string.Equals(Path.GetFullPath(tempPath), Path.GetFullPath(outputPath), + StringComparison.OrdinalIgnoreCase)) + return null; + if (!File.Exists(tempPath)) return null; + if (new FileInfo(tempPath).Length < dataLength) return null; + return tempPath; + } + + /// + /// True when is a name a fresh run gives its temp: + /// the output's own name, a dot, 32 lowercase hex digits (Guid "N"), and ".tmp". + /// + private static bool IsRunTempName(string outputFileName, string candidate) + { + var prefix = outputFileName + "."; + const string suffix = ".tmp"; + if (candidate.Length != prefix.Length + 32 + suffix.Length) return false; + if (!candidate.StartsWith(prefix, StringComparison.Ordinal)) return false; + if (!candidate.EndsWith(suffix, StringComparison.Ordinal)) return false; + for (var i = prefix.Length; i < prefix.Length + 32; i++) + { + if (candidate[i] is (>= '0' and <= '9') or (>= 'a' and <= 'f')) continue; + return false; + } + return true; + } + + /// + /// Replace the output with the first bytes of a named temp, + /// then remove the temp. A fresh run that finishes moves its temp over whatever the output + /// held, so recovering an unfinished one has to reach that same file; appending instead + /// would leave the previous run's rows - already consumed - in front of this one's. Unlike + /// the glob-and-newest form, this takes exactly the file the checkpoint recorded, so a + /// leftover from an unrelated run cannot be merged in. Bytes rather than lines: the length + /// was taken from the writer's own position, so it cannot disagree with itself about line + /// endings or a torn final line. Returns false when the temp is absent or shorter than the + /// checkpoint promised, which means the caller must not resume past the items it counted. + /// + protected static bool TryPromoteNamedTemp(string outputPath, string tempFileName, long dataLength) + { + try + { + var tempPath = ResolveNamedTemp(outputPath, tempFileName, dataLength); + if (tempPath == null) return false; + + // Staged like the other forms, so the destination is replaced in one Move rather + // than truncated and refilled in place. + var adoptPath = outputPath + ".adopt"; + using (var writer = new FileStream(adoptPath, FileMode.Create, FileAccess.Write)) + { + using var temp = new FileStream(tempPath, FileMode.Open, FileAccess.Read); + var buffer = new byte[81920]; + long remaining = dataLength; + while (remaining > 0) + { + var read = temp.Read(buffer, 0, (int)Math.Min(buffer.Length, remaining)); + if (read <= 0) break; + writer.Write(buffer, 0, read); + remaining -= read; + } + if (remaining > 0) + { + writer.Dispose(); + File.Delete(adoptPath); + return false; + } + } + File.Move(adoptPath, outputPath, overwrite: true); + try { File.Delete(tempPath); } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { } + return true; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return false; + } + } + + /// + /// Cut the output back to the length a checkpoint recorded for it, dropping anything the + /// interrupted run wrote after its last save. Those items are re-fetched, so dropping them + /// is what keeps a resume from duplicating them. Returns false when the output is shorter + /// than recorded, which means it is no longer the file the checkpoint describes. + /// + protected static bool TryTrimOutputToCheckpoint(string outputPath, long dataLength) + { + try + { + // A checkpoint is untrusted input once it is on disk, and SetLength rejects a + // negative length with an ArgumentOutOfRangeException the catch below does not + // cover - so a hand-edited length escaped as a terminating error naming neither the + // checkpoint nor the file, and left itself on disk to fail the same way next run. + // ResolveNamedTemp already guards its own length; this is the same guard. + if (dataLength < 0) return false; + if (!File.Exists(outputPath)) return false; + var actual = new FileInfo(outputPath).Length; + if (actual < dataLength) return false; + if (actual == dataLength) return true; + using var fs = new FileStream(outputPath, FileMode.Open, FileAccess.Write); + fs.SetLength(dataLength); + return true; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return false; + } + } + + /// + /// Recovers a fresh-run JSONL job that was interrupted before its temp file was promoted, + /// from a checkpoint that predates the recorded temp name and length. With only a line + /// count and the newest matching temp to go on, this is safe solely when no output exists: + /// everything the run wrote is then in its temp, and creating the output from it is what a + /// finishing run's own promotion would have done. Against an existing output there is no + /// way to tell whose items the temp holds, so the caller must re-enumerate instead. + /// Copies exactly lines - content beyond the last flush may + /// be absent or torn - then removes the temp. Returns false when nothing usable exists, + /// leaving the caller to the stale-checkpoint path. + /// + protected static bool TryAdoptOrphanedTemp(string outputPath, long itemCount) + { + try + { + if (itemCount <= 0) return false; + if (File.Exists(outputPath)) return false; + var dir = Path.GetDirectoryName(outputPath); + if (string.IsNullOrEmpty(dir) || !Directory.Exists(dir)) return false; + var temp = Directory.EnumerateFiles(dir, Path.GetFileName(outputPath) + ".*.tmp") + .Select(p => new FileInfo(p)) + .OrderByDescending(f => f.LastWriteTimeUtc) + .FirstOrDefault(); + if (temp == null) return false; + + // Staged so the output appears in one Move. A merge that dies halfway leaves only + // the staging file behind, and the caller keeps its checkpoint - the safe direction. + long copied = 0; + var adoptPath = outputPath + ".adopt"; + using (var writer = new StreamWriter(adoptPath, append: false)) + { + using var reader = new StreamReader(temp.FullName); + string? line; + while (copied < itemCount && (line = reader.ReadLine()) != null) + { + writer.WriteLine(line); + copied++; + } + } + if (copied < itemCount) + { + // Temp holds less than the checkpoint promises - unusable. + File.Delete(adoptPath); + return false; + } + File.Move(adoptPath, outputPath, overwrite: true); + temp.Delete(); + return true; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return false; + } + } + #region Shared URL and header builders protected static string NormalizePath(string path) @@ -823,13 +1071,38 @@ protected static string BuildListUrl(string versionedBaseUrl, string relativeUri $"Wait {s_clientOptions.CircuitBreakerDurationSeconds}s or run Get-MgxTelemetry for details. " + $"Tune with Set-MgxOption -CircuitBreakerFailureRatio / -CircuitBreakerMinThroughput."; - protected void WriteBetaHintIfApplicable(HttpStatusCode statusCode, string apiVersion) + /// + /// Codes measured to mean the PATH was fine and the OBJECT was not there, so a beta hint + /// over them sends the caller to re-run a request that fails there too. Only codes with + /// demonstrated semantics belong here. Request_ResourceNotFound does NOT qualify: Graph + /// returns it both for a missing directory object and for a beta-only segment on v1.0 + /// (measured against /users/{id} and /users/{id}/profile), so it cannot be told apart and + /// the hedged hint stays. itemNotFound is the drive service reporting an absent item - an + /// unknown drive segment is a 400, not a 404, so the ambiguity does not arise there. + /// + private static readonly HashSet ObjectMissingCodes = new(StringComparer.OrdinalIgnoreCase) { - if (statusCode == HttpStatusCode.NotFound && - string.Equals(apiVersion, "v1.0", StringComparison.OrdinalIgnoreCase)) - { - WriteWarning("This endpoint may only be available in beta. Retry with -ApiVersion beta."); - } + "itemNotFound", + }; + + /// True when the exception is a Graph 404 that names a missing object. + protected static bool IsObjectMissing(Exception ex) => + ex is GraphServiceException { StatusCode: HttpStatusCode.NotFound } g + && g.ErrorCode != null + && ObjectMissingCodes.Contains(g.ErrorCode); + + protected void WriteBetaHintIfApplicable(HttpStatusCode statusCode, string apiVersion, + string? errorCode = null) + { + if (statusCode != HttpStatusCode.NotFound || + !string.Equals(apiVersion, "v1.0", StringComparison.OrdinalIgnoreCase)) + return; + + // A missing user, group or drive item is not an absent endpoint. + if (errorCode != null && ObjectMissingCodes.Contains(errorCode)) + return; + + WriteWarning("This endpoint may only be available in beta. Retry with -ApiVersion beta."); } protected static ErrorCategory MapStatusToCategory(HttpStatusCode statusCode) => statusCode switch @@ -858,7 +1131,7 @@ protected bool WriteGraphError(Exception ex, object? target, string? apiVersion { case GraphServiceException gex: if (apiVersion != null) - WriteBetaHintIfApplicable(gex.StatusCode, apiVersion); + WriteBetaHintIfApplicable(gex.StatusCode, apiVersion, gex.ErrorCode); WriteError(new ErrorRecord(gex, gex.ErrorCode ?? "GraphError", MapStatusToCategory(gex.StatusCode), target)); return true; diff --git a/src/Mgx.Cmdlets/Base/MgxCmdletCore.cs b/src/Mgx.Cmdlets/Base/MgxCmdletCore.cs index 00477b0..42d6f0f 100644 --- a/src/Mgx.Cmdlets/Base/MgxCmdletCore.cs +++ b/src/Mgx.Cmdlets/Base/MgxCmdletCore.cs @@ -19,6 +19,7 @@ namespace Mgx.Cmdlets.Base; public abstract class MgxCmdletCore : PSCmdlet, IDisposable { private CancellationTokenSource _cts = new(); + private readonly CancellationToken _token; private int _disposed; // 0 = not disposed, 1 = disposed (Interlocked for thread safety) // Regex gate for DateTime parsing: requires YYYY-MM-DDT prefix. @@ -26,10 +27,32 @@ public abstract class MgxCmdletCore : PSCmdlet, IDisposable private static readonly Regex Iso8601Pattern = new( @"^\d{4}-\d{2}-\d{2}[T ]", RegexOptions.Compiled); - protected CancellationToken CancellationToken => _cts.Token; + /// + /// The cancellation token for this invocation. + /// + /// + /// Deliberately a cached COPY rather than _cts.Token. StopProcessing cancels and then + /// disposes the source, and reading Token on a disposed source is documented to throw + /// ObjectDisposedException on every framework moniker including net8.0. Several catch blocks + /// read this property while unwinding a Ctrl-C - SyncMgxDelta and ExportMgxCollection consult + /// it before deciding whether to promote or delete a temp file - so a throw there would + /// replace the OperationCanceledException and skip the cleanup, orphaning a partial file. + /// + /// .NET 10 happens not to throw in practice, but the documented contract specifies + /// ObjectDisposedException for every moniker including net8.0, which is what the module + /// targets. A token struct copied before disposal stays fully usable, so caching costs + /// nothing and removes the dependency on undocumented leniency. + /// + protected CancellationToken CancellationToken => _token; #region Lifecycle + protected MgxCmdletCore() + { + // Copy the token once, while the source is guaranteed alive. + _token = _cts.Token; + } + protected override void StopProcessing() { _cts.Cancel(); diff --git a/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs b/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs index 4244308..c2913d8 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs @@ -133,29 +133,33 @@ protected override void EndProcessing() return; } - // ShouldProcess gate: only for batches containing write operations + // -WhatIf is documented as "the cmdlet is not run", without qualification, so the + // gate covers reads as well. A read-only batch changes nothing on the server, but it + // spends resource units, can be throttled, and emits objects into the pipeline - + // none of which is "not run", and none of which the caller asked for. var writeOps = _collected.Where(c => !string.Equals(c.Method, "GET", StringComparison.OrdinalIgnoreCase)).ToList(); - if (writeOps.Count > 0) + string target; + if (writeOps.Count == 0) { - string target; - if (writeOps.All(o => string.Equals(o.Method, writeOps[0].Method, StringComparison.OrdinalIgnoreCase))) - { - target = $"{writeOps[0].Method} {_collected.Count} requests via $batch"; - } - else - { - var breakdown = writeOps - .GroupBy(o => o.Method.ToUpperInvariant()) - .OrderByDescending(g => g.Count()) - .Select(g => $"{g.Count()} {g.Key}"); - target = $"{_collected.Count} requests ({string.Join(", ", breakdown)}) via $batch"; - } - - if (!ShouldProcess(target, "Send batch")) - return; + target = $"GET {_collected.Count} requests via $batch"; } + else if (writeOps.All(o => string.Equals(o.Method, writeOps[0].Method, StringComparison.OrdinalIgnoreCase))) + { + target = $"{writeOps[0].Method} {_collected.Count} requests via $batch"; + } + else + { + var breakdown = writeOps + .GroupBy(o => o.Method.ToUpperInvariant()) + .OrderByDescending(g => g.Count()) + .Select(g => $"{g.Count()} {g.Key}"); + target = $"{_collected.Count} requests ({string.Join(", ", breakdown)}) via $batch"; + } + + if (!ShouldProcess(target, "Send batch")) + return; var client = GetClient(); var mergedHeaders = Headers != null ? new System.Collections.Hashtable(Headers) : null; @@ -227,25 +231,24 @@ protected override void EndProcessing() : null }; + // Status 0 means the operation was never sent, because a chunk before it failed. + // It is not a success and must not read as one - the caller has to be able to + // tell a write that may have landed from one that certainly did not. + if (item.Status == GraphBatchClient.NotSentStatus) + result["NotSent"] = true; + // Single-argument WriteObject does not enumerate, so the Hashtable is emitted whole WriteObject(result); } - // Emit errors for failed items (enables -ErrorAction Stop, populates $Error) - for (int i = 0; i < results.Count; i++) + // A chunk's POST failed after earlier chunks were applied. Their results are above; + // this says why the rest never went, and NotSent names them. + if (batchResult.ChunkFailure != null) { - var (_, item) = results[i]; - if (item.Status >= 400) - { - var input = submitted[i]; - var graphMessage = TryExtractBatchErrorMessage(item); - var errorMessage = graphMessage != null - ? $"{input.Method} {input.Url}: {graphMessage}" - : $"HTTP {item.Status} for {input.Method} {input.Url}"; - var ex = new InvalidOperationException(errorMessage); - WriteError(new ErrorRecord(ex, "BatchItemError", - MapStatusToCategory((HttpStatusCode)item.Status), input.Url)); - } + var notSent = batchResult.NotSent.Count; + WriteError(new ErrorRecord(batchResult.ChunkFailure, "BatchChunkFailed", + ErrorCategory.NotSpecified, + $"{notSent} of {results.Count} operations were not sent")); } // Write failed items to dead-letter file (append mode) @@ -258,7 +261,7 @@ protected override void EndProcessing() for (int i = 0; i < results.Count; i++) { var (_, item) = results[i]; - if (item.Status < 400) continue; + if (item.Status < 400 && item.Status != GraphBatchClient.NotSentStatus) continue; var input = submitted[i]; var deadLetter = new JsonObject @@ -288,6 +291,32 @@ protected override void EndProcessing() catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { WriteWarning($"Failed to write dead-letter file '{resolvedDeadLetterPath}': {ex.Message}"); + + // Emit errors for failed items (enables -ErrorAction Stop, populates $Error) + for (int i = 0; i < results.Count; i++) + { + var (_, item) = results[i]; + if (item.Status == GraphBatchClient.NotSentStatus) + { + var skipped = submitted[i]; + WriteError(new ErrorRecord( + new InvalidOperationException( + $"{skipped.Method} {skipped.Url} was not sent: an earlier chunk failed."), + "BatchItemNotSent", ErrorCategory.NotSpecified, skipped.Url)); + } + else if (item.Status >= 400) + { + var input = submitted[i]; + var graphMessage = TryExtractBatchErrorMessage(item); + var errorMessage = graphMessage != null + ? $"{input.Method} {input.Url}: {graphMessage}" + : $"HTTP {item.Status} for {input.Method} {input.Url}"; + var itemError = new InvalidOperationException(errorMessage); + WriteError(new ErrorRecord(itemError, "BatchItemError", + MapStatusToCategory((HttpStatusCode)item.Status), input.Url)); + } + } + } if (failedCount > 0) diff --git a/src/Mgx.Cmdlets/Cmdlets/Configuration/EnableMgxResilience.cs b/src/Mgx.Cmdlets/Cmdlets/Configuration/EnableMgxResilience.cs index d9ea7d9..8af04e9 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Configuration/EnableMgxResilience.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Configuration/EnableMgxResilience.cs @@ -258,8 +258,13 @@ internal static void RefreshInjectedClient(Action warn, Action v }; ActiveHandler = resilientHandler; + // BaseAddress must come along: Invoke-MgGraphRequest resolves a relative -Uri + // against the active client's BaseAddress before any handler runs. Default + // request headers are NOT copied - the bridge delegates to sdkClient.SendAsync, + // which applies the original client's defaults to each request anyway. return new HttpClient(resilientHandler) { + BaseAddress = sdkClient.BaseAddress, Timeout = sdkClient.Timeout }; } diff --git a/src/Mgx.Cmdlets/Cmdlets/Configuration/GetMgxOption.cs b/src/Mgx.Cmdlets/Cmdlets/Configuration/GetMgxOption.cs index ee38ccb..18a91b3 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Configuration/GetMgxOption.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Configuration/GetMgxOption.cs @@ -21,6 +21,7 @@ protected override void ProcessRecord() RateLimitPerSecond = opts.RateLimitPerSecond, NoRateLimit = opts.NoRateLimit, RateLimitQueueLimit = opts.RateLimitQueueLimit, + NoAdaptivePacing = opts.NoAdaptivePacing, MaxRetryAttempts = opts.MaxRetryAttempts, MaxRetryAfterSeconds = opts.MaxRetryAfterSeconds, TotalTimeoutSeconds = opts.TotalTimeoutSeconds, diff --git a/src/Mgx.Cmdlets/Cmdlets/Configuration/GetMgxTelemetry.cs b/src/Mgx.Cmdlets/Cmdlets/Configuration/GetMgxTelemetry.cs index 611f48d..7e55228 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Configuration/GetMgxTelemetry.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Configuration/GetMgxTelemetry.cs @@ -38,6 +38,11 @@ protected override void ProcessRecord() TotalElapsedMs = summary.ElapsedMs, ResourceUnitsConsumed = summary.ResourceUnitsConsumed, BatchItemThrottles = summary.BatchItemThrottles, + AdaptivePacingWaitMs = summary.AdaptivePacingWaitMs, + AdaptivePacingActivations = summary.AdaptivePacingActivations, + LastThrottlePercentage = summary.LastThrottlePercentage, + PacingState = summary.PacingState, + ContentBytesDownloaded = summary.ContentBytesDownloaded, }); } } diff --git a/src/Mgx.Cmdlets/Cmdlets/Configuration/SetMgxOption.cs b/src/Mgx.Cmdlets/Cmdlets/Configuration/SetMgxOption.cs index 9fdfffe..caa9ed9 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Configuration/SetMgxOption.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Configuration/SetMgxOption.cs @@ -28,6 +28,9 @@ public class SetMgxOption : PSCmdlet [ValidateRange(0, 100_000)] public int RateLimitQueueLimit { get; set; } + [Parameter] + public SwitchParameter NoAdaptivePacing { get; set; } + [Parameter] [ValidateRange(1, 600)] public int MaxRetryAfterSeconds { get; set; } @@ -71,6 +74,17 @@ public class SetMgxOption : PSCmdlet [Parameter] public SwitchParameter Reset { get; set; } + /// + /// Parameters PowerShell adds to every cmdlet. Present in BoundParameters, but none of them + /// is a change to an option. + /// + private static readonly HashSet CommonParameterNames = new(StringComparer.OrdinalIgnoreCase) + { + "Verbose", "Debug", "ErrorAction", "WarningAction", "InformationAction", "ProgressAction", + "ErrorVariable", "WarningVariable", "InformationVariable", "OutVariable", "OutBuffer", + "PipelineVariable", "WhatIf", "Confirm", + }; + protected override void ProcessRecord() { var bound = MyInvocation.BoundParameters; @@ -91,8 +105,12 @@ protected override void ProcessRecord() } // No parameters passed: nothing to do (avoids unnecessary pipeline rebuild - // which would destroy circuit breaker failure history) - if (bound.Count == 0) + // which would destroy circuit breaker failure history). + // + // Counted without the common parameters, which BoundParameters includes: -Verbose alone + // made this look like a real change, so `Set-MgxOption -Verbose` rebuilt the pipeline and + // discarded exactly the history this guard was written to keep. + if (!bound.Keys.Cast().Any(k => !CommonParameterNames.Contains(k))) { WriteVerbose("No parameters specified. Options unchanged."); return; @@ -114,6 +132,9 @@ protected override void ProcessRecord() RateLimitBurst = bound.ContainsKey(nameof(RateLimitBurst)) ? RateLimitBurst : current.RateLimitBurst, RateLimitPerSecond = bound.ContainsKey(nameof(RateLimitPerSecond)) ? RateLimitPerSecond : current.RateLimitPerSecond, NoRateLimit = noRateLimit, + NoAdaptivePacing = bound.ContainsKey(nameof(NoAdaptivePacing)) + ? NoAdaptivePacing.IsPresent + : current.NoAdaptivePacing, RateLimitQueueLimit = bound.ContainsKey(nameof(RateLimitQueueLimit)) ? RateLimitQueueLimit : current.RateLimitQueueLimit, MaxRetryAfterSeconds = bound.ContainsKey(nameof(MaxRetryAfterSeconds)) ? MaxRetryAfterSeconds : current.MaxRetryAfterSeconds, MaxRetryAttempts = bound.ContainsKey(nameof(MaxRetryAttempts)) ? MaxRetryAttempts : current.MaxRetryAttempts, @@ -134,9 +155,23 @@ protected override void ProcessRecord() "Retries are effectively disabled because the total timeout will fire on the first attempt."); } + // -NoRateLimit switches off batch item pacing as well, which is a second mechanism + // against Graph's server-side WRITE throttle rather than the client-side limiter. Both + // are documented separately and -BatchItemsPerSecond 0 is the documented way to turn + // pacing off, so a caller who set both deserves to hear that one of them lost. + if (options.NoRateLimit && bound.ContainsKey(nameof(BatchItemsPerSecond)) + && options.BatchItemsPerSecond > 0) + { + WriteWarning( + $"-NoRateLimit also disables batch item pacing, so -BatchItemsPerSecond " + + $"{options.BatchItemsPerSecond} has no effect. Drop -NoRateLimit to keep pacing, " + + "or set -BatchItemsPerSecond 0 to disable it explicitly."); + } + MgxCmdletBase.SetClientOptions(options); WriteVerbose($"Mgx options updated: Burst={options.RateLimitBurst}, Rate={options.RateLimitPerSecond}/s, " + - $"NoRateLimit={options.NoRateLimit}, QueueLimit={options.RateLimitQueueLimit}, " + + $"NoRateLimit={options.NoRateLimit}, NoAdaptivePacing={options.NoAdaptivePacing}, " + + $"QueueLimit={options.RateLimitQueueLimit}, " + $"MaxRetry={options.MaxRetryAttempts}, TotalTimeout={options.TotalTimeoutSeconds}s, " + $"AttemptTimeout={options.AttemptTimeoutSeconds}s, CBDuration={options.CircuitBreakerDurationSeconds}s, " + $"CBFailureRatio={options.CircuitBreakerFailureRatio}, CBMinThroughput={options.CircuitBreakerMinThroughput}, " + diff --git a/src/Mgx.Cmdlets/Cmdlets/Content/GetMgxContent.cs b/src/Mgx.Cmdlets/Cmdlets/Content/GetMgxContent.cs new file mode 100644 index 0000000..7c496e1 --- /dev/null +++ b/src/Mgx.Cmdlets/Cmdlets/Content/GetMgxContent.cs @@ -0,0 +1,352 @@ +using System.Management.Automation; +using System.Net; +using System.Net.Http.Headers; +using Mgx.Cmdlets.Base; +using Mgx.Engine.Http; +using Mgx.Engine.Models; +using Polly.CircuitBreaker; + +namespace Mgx.Cmdlets.Cmdlets.Content; + +/// +/// Get-MgxContent: Fetch content bytes ($value / /content endpoints), whole or as a byte +/// range. Ranged reads are the point: pulling 256 KB headers off 20k files moves ~5 GB +/// instead of 63 GB for a metadata job. +/// +/// Two hops under the hood: the authenticated Graph request (full resilience pipeline, +/// pacing, rate limiting), then - when Graph 302s to a pre-authenticated download host - a +/// token-free fetch whose target is validated against a Microsoft-hosts allowlist. The +/// bearer token never reaches the download host. See GraphContentClient for the mechanism +/// and the transport preconditions. +/// +/// Output is a single byte[] to the pipeline, or a file via -OutFile (temp + atomic move). +/// Piped DriveItems use their @microsoft.graph.downloadUrl directly (validated first) or +/// fall back to /drives/{driveId}/items/{id}/content. +/// +[Cmdlet(VerbsCommon.Get, "MgxContent", DefaultParameterSetName = "Uri")] +[OutputType(typeof(byte[]))] +public class GetMgxContent : MgxCmdletBase +{ + /// Pipeline output guard: a byte[] larger than this must go to -OutFile. + internal const long MaxPipelineBytes = 100L * 1024 * 1024; + + [Parameter(Mandatory = true, Position = 0, ParameterSetName = "Uri")] + [Alias("Resource")] + public string Uri { get; set; } = string.Empty; + + [Parameter(Mandatory = true, ValueFromPipeline = true, ParameterSetName = "InputObject")] + public object? InputObject { get; set; } + + /// First N bytes (Range: bytes=0..N-1). Mutually exclusive with -Offset/-Length. + [Parameter] + [ValidateRange(1, long.MaxValue)] + public long First { get; set; } + + /// Range start, used with -Length (Range: bytes=Offset..Offset+Length-1). + [Parameter] + [ValidateRange(0, long.MaxValue)] + public long Offset { get; set; } + + /// Range length, from -Offset (default 0). + [Parameter] + [ValidateRange(1, long.MaxValue)] + public long Length { get; set; } + + [Parameter] + public string? OutFile { get; set; } + + [Parameter] + [ValidateSet("v1.0", "beta")] + [ArgumentCompleter(typeof(ApiVersionCompleter))] + public string ApiVersion { get; set; } = "v1.0"; + + [Parameter] + public System.Collections.Hashtable? Headers { get; set; } + + private string VersionedBaseUrl => $"{s_graphEndpoint}/{ApiVersion}"; + private string? _resolvedOutFile; + + // -OutFile carries no ParameterSetName, so it binds in the InputObject set too and the help + // documents that combination. With a single destination and File.Move(overwrite: true) per + // record, piping N items downloaded all N and left one file - the last writer winning, with + // no warning and full transfer cost for the discarded ones. Export-MgxCollection and + // Sync-MgxDelta guard the analogous case; this did not. + private bool _wroteOutFile; + private bool _transportChecked; + + protected override void BeginProcessing() + { + if (ParameterSetName == "Uri") + { + if (Uri.StartsWith("https://", StringComparison.OrdinalIgnoreCase) || + Uri.StartsWith("http://", StringComparison.OrdinalIgnoreCase)) + { + ThrowTerminatingError(new ErrorRecord( + new ArgumentException( + $"-Uri must be a relative Graph path (e.g., /me/drive/items/{{id}}/content), not an absolute URL. Got: '{Uri}'"), + "AbsoluteUriNotAllowed", ErrorCategory.InvalidArgument, null)); + return; + } + + if (!Uri.Contains("/content", StringComparison.OrdinalIgnoreCase) + && !Uri.Contains("$value", StringComparison.OrdinalIgnoreCase)) + { + WriteWarning( + $"URI '{Uri}' does not look like a content endpoint (/content or /$value). " + + "The response may be JSON metadata rather than file bytes."); + } + } + + // -First XOR -Offset/-Length: two ways to say the same thing must not disagree. + var boundParams = MyInvocation.BoundParameters; + var hasFirst = boundParams.ContainsKey(nameof(First)); + var hasOffset = boundParams.ContainsKey(nameof(Offset)); + var hasLength = boundParams.ContainsKey(nameof(Length)); + if (hasFirst && (hasOffset || hasLength)) + { + ThrowTerminatingError(new ErrorRecord( + new ArgumentException("-First cannot be combined with -Offset/-Length. Use one range form."), + "RangeParameterConflict", ErrorCategory.InvalidArgument, null)); + return; + } + if (hasOffset && !hasLength) + { + ThrowTerminatingError(new ErrorRecord( + new ArgumentException("-Offset requires -Length (Range: bytes=Offset..Offset+Length-1)."), + "OffsetRequiresLength", ErrorCategory.InvalidArgument, null)); + return; + } + + if (OutFile != null) + _resolvedOutFile = GetUnresolvedProviderPathFromPSPath(OutFile); + } + + protected override void ProcessRecord() + { + if (ParameterSetName == "Uri") + { + FetchContent(downloadUrl: null, + relativeUri: Uri, + errorTarget: Uri); + return; + } + + if (InputObject == null) + { + WriteVerbose("Skipping null pipeline input."); + return; + } + + var value = UnwrapPSObject(InputObject); + + // Prefer the item's own pre-authenticated download URL: no hop-1 round trip, no + // request-budget charge. Validated against the allowlist before any fetch. + if (TryGetMember(value, "@microsoft.graph.downloadUrl")?.ToString() is { Length: > 0 } itemDownloadUrl) + { + FetchContent(downloadUrl: itemDownloadUrl, relativeUri: null, errorTarget: InputObject); + return; + } + + // Fall back to /drives/{driveId}/items/{id}/content from the item's identifiers. + var id = Cmdlets.InvokeMgxRequest.ResolvePipelineId(value); + var driveId = TryGetMember(TryGetMember(value, "parentReference"), "driveId")?.ToString(); + if (string.IsNullOrEmpty(id) || string.IsNullOrEmpty(driveId)) + { + WriteError(new ErrorRecord( + new ArgumentException( + "Pipeline input needs either @microsoft.graph.downloadUrl, or both 'id' and " + + "'parentReference.driveId', to locate the content. Pipe DriveItems from " + + "Invoke-MgxRequest or Sync-MgxDelta."), + "MissingDriveItemInfo", ErrorCategory.InvalidArgument, InputObject)); + return; + } + + FetchContent(downloadUrl: null, + relativeUri: $"/drives/{driveId}/items/{id}/content", + errorTarget: InputObject); + } + + private RangeHeaderValue? BuildRange() + { + if (First > 0) return new RangeHeaderValue(0, First - 1); + if (Length > 0) return new RangeHeaderValue(Offset, Offset + Length - 1); + return null; + } + + /// Bytes the caller actually asked for; null = the whole file. + private long? RequestedBytes => First > 0 ? First : Length > 0 ? Length : null; + + private void FetchContent(string? downloadUrl, string? relativeUri, object? errorTarget) + { + var client = GetClient(); + + // Fail closed on non-owned transports: the 302-interception design requires + // AllowAutoRedirect off, which only the mgx-owned clean client guarantees. The SDK + // fallback client ships a RedirectHandler that would auto-follow to an unvalidated + // host. Checked once per invocation, after GetClient() decided the transport. + if (!_transportChecked) + { + _transportChecked = true; + if (!TransportIsOwned) + { + ThrowTerminatingError(new ErrorRecord( + new InvalidOperationException( + "Content downloads require the mgx-owned HTTP transport, but this session " + + "is using the Graph SDK's client (mgx could not build its own). The SDK " + + "transport auto-follows redirects, which defeats the download-host " + + "validation. Reconnect with Connect-MgGraph and retry."), + "ContentRequiresOwnedTransport", ErrorCategory.SecurityError, null)); + return; + } + } + + var range = BuildRange(); + + try + { + using var result = downloadUrl != null + ? GraphContentClient.GetFromDownloadUrlAsync( + downloadUrl, range, client.BodyReadTimeout, CancellationToken) + .GetAwaiter().GetResult() + : client.GetContentAsync( + $"{VersionedBaseUrl}{NormalizePath(relativeUri!)}", range, + BuildRequestHeaders(null, Headers), CancellationToken) + .GetAwaiter().GetResult(); + + DrainClientMessages(); + + // Truncation path: a range was requested but the server ignored it and answered + // 200 with the full body (profile photos do this). Copy only the requested bytes; + // disposing the result aborts the rest of the transfer. + long? maxBytes = null; + long skipBytes = 0; + if (range != null && result.StatusCode == HttpStatusCode.OK) + { + maxBytes = RequestedBytes; + // The server ignored the offset too, not just the length. Discarding the head + // locally is the only way -Offset can mean what it says on this path; without + // it the caller silently receives bytes 0..Length-1 and is told it worked. + skipBytes = Offset; + WriteVerbose(skipBytes > 0 + ? $"Server ignored the Range header (HTTP 200). Discarding the first {skipBytes:N0} bytes and taking {maxBytes:N0}." + : $"Server ignored the Range header (HTTP 200). Truncating to the requested {maxBytes} bytes."); + } + + if (_resolvedOutFile != null) + { + if (_wroteOutFile) + { + // Refuse rather than overwrite. Checked here, not in BeginProcessing, so the + // legitimate single-piped-item case still works. + ThrowTerminatingError(new ErrorRecord( + new InvalidOperationException( + "-OutFile writes a single file, but more than one item was piped in. " + + "Each item would overwrite the last. Pipe one item, or omit -OutFile " + + "and redirect the byte[] output per item."), + "OutFileWithMultipleInputs", ErrorCategory.InvalidOperation, OutFile)); + return; + } + WriteToFile(result, maxBytes, skipBytes); + _wroteOutFile = true; + } + else + { + WriteToPipeline(result, maxBytes, skipBytes, errorTarget); + } + } + catch (OperationCanceledException) when (CancellationToken.IsCancellationRequested) + { + DrainClientMessages(); + WriteWarning("Content download cancelled."); + } + catch (InvalidOperationException ex) + { + // Validator refusal or transport-redirect detection: a security boundary, not a + // transient fault - surface as such, per item. + DrainClientMessages(); + WriteError(new ErrorRecord(ex, "ContentDownloadRefused", ErrorCategory.SecurityError, errorTarget)); + } + catch (Exception ex) when (ex is GraphServiceException or BrokenCircuitException or HttpRequestException) + { + WriteGraphError(ex, errorTarget, ApiVersion); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // UnauthorizedAccessException is NOT an IOException. FileStream throws it for an + // unwritable parent directory or a destination that is itself a directory, and + // File.Move(overwrite: true) throws it for a read-only destination on Windows - so + // an unwritable -OutFile escaped every filter here, skipped DrainClientMessages, and + // surfaced as an unhandled error naming a temp path the caller never supplied. + // Eleven other sites in this codebase already pair the two; this was the only + // file-writing cmdlet that did not. + DrainClientMessages(); + WriteError(new ErrorRecord(ex, "IOError", ErrorCategory.WriteError, OutFile)); + } + } + + private void WriteToFile(GraphContentResult result, long? maxBytes, long skipBytes) + { + // Temp + atomic move, like every other file-writing path in mgx: a failed download + // never truncates an existing file. + var tempPath = $"{_resolvedOutFile}.{Guid.NewGuid():N}.tmp"; + long copied; + try + { + using (var file = new FileStream(tempPath, FileMode.Create, FileAccess.Write)) + { + copied = GraphContentClient.CopyWithIdleTimeoutAsync( + result.Content, file, maxBytes, GetClient().BodyReadTimeout, + CancellationToken, skipBytes) + .GetAwaiter().GetResult(); + } + File.Move(tempPath, _resolvedOutFile!, overwrite: true); + } + catch + { + try { if (File.Exists(tempPath)) File.Delete(tempPath); } catch { } + throw; + } + + WriteVerbose($"{copied:N0} bytes ({(int)result.StatusCode} {result.StatusCode}" + + (result.ContentRange != null ? $", {result.ContentRange}" : "") + + $") -> {_resolvedOutFile}"); + } + + private void WriteToPipeline(GraphContentResult result, long? maxBytes, long skipBytes, object? errorTarget) + { + // Known-oversized before a single byte moves: refuse early. + var expected = maxBytes ?? result.ContentLength; + if (expected > MaxPipelineBytes) + { + ThrowTerminatingError(new ErrorRecord( + new InvalidOperationException( + $"Content is {expected:N0} bytes; the pipeline guard is {MaxPipelineBytes:N0} (100 MB). " + + "Use -OutFile for large downloads, or a byte range (-First / -Offset -Length)."), + "ContentTooLargeForPipeline", ErrorCategory.LimitsExceeded, errorTarget)); + return; + } + + // Unknown length (chunked): enforce the guard during the copy - cap one byte over + // the limit so hitting the cap is distinguishable from an exact-limit body. + var copyLimit = maxBytes ?? MaxPipelineBytes + 1; + using var buffer = new MemoryStream(expected is > 0 and <= int.MaxValue ? (int)expected : 0); + var copied = GraphContentClient.CopyWithIdleTimeoutAsync( + result.Content, buffer, copyLimit, GetClient().BodyReadTimeout, + CancellationToken, skipBytes) + .GetAwaiter().GetResult(); + + if (copied > MaxPipelineBytes) + { + ThrowTerminatingError(new ErrorRecord( + new InvalidOperationException( + $"Content exceeded the {MaxPipelineBytes:N0}-byte (100 MB) pipeline guard mid-download. " + + "Use -OutFile for large downloads, or a byte range (-First / -Offset -Length)."), + "ContentTooLargeForPipeline", ErrorCategory.LimitsExceeded, errorTarget)); + return; + } + + WriteVerbose($"{copied:N0} bytes ({(int)result.StatusCode} {result.StatusCode}" + + (result.ContentRange != null ? $", {result.ContentRange}" : "") + ")"); + WriteObject(buffer.ToArray(), enumerateCollection: false); + } +} diff --git a/src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs b/src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs index e7e2b94..77d97a8 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs @@ -16,6 +16,10 @@ namespace Mgx.Cmdlets.Cmdlets.Delta; /// First run performs a full sync and saves the delta token. /// Subsequent runs retrieve only items changed since the last sync. /// Delta state persists across successful completions (unlike CheckpointPath which is ephemeral). +/// -CheckpointPath adds mid-run crash resume: the enumeration position is saved at page +/// boundaries (and mid-page in JSONL mode), so a killed sync continues where it stopped +/// instead of re-enumerating from scratch. Resume is at-least-once: in pipeline mode the +/// page in flight at the crash is re-emitted in full. /// [Cmdlet(VerbsData.Sync, "MgxDelta")] [OutputType(typeof(Hashtable))] @@ -34,6 +38,16 @@ public class SyncMgxDelta : MgxCmdletBase [Parameter] public string? Filter { get; set; } + /// + /// Prefer-header tokens joined into a single Prefer header (drive delta behaviors such as + /// deltashowremovedasdeleted). A change against the stored state forces a full re-sync, + /// like -Property and -Filter. Note: deltaExcludeParent is a standalone request header, + /// not a Prefer token - pass it via -Headers. + /// + [Parameter] + [ArgumentCompleter(typeof(DeltaPreferCompleter))] + public string[]? Prefer { get; set; } + [Parameter] [ValidateRange(1, 999)] public int Top { get; set; } @@ -44,6 +58,23 @@ public class SyncMgxDelta : MgxCmdletBase [Parameter] public SwitchParameter FullSync { get; set; } + /// + /// Baseline without enumerating: request only the latest delta token ("sync from now"). + /// Drive resources take ?token=latest; directory and other resources take + /// $deltatoken=latest - the form is chosen automatically from the URI shape. + /// Ignored (with a warning) when usable delta state already exists. + /// + [Parameter] + public SwitchParameter Latest { get; set; } + + /// + /// Path for the ephemeral mid-run resume checkpoint. Deleted on successful completion; + /// any event that invalidates the enumeration (410 Gone, -FullSync, a -Property/-Filter/ + /// -Prefer change) deletes it too, so a stale position can never be resumed. + /// + [Parameter] + public string? CheckpointPath { get; set; } + [Parameter] [ValidateSet("v1.0", "beta")] [ArgumentCompleter(typeof(ApiVersionCompleter))] @@ -57,6 +88,7 @@ public class SyncMgxDelta : MgxCmdletBase /// /// Normalize $select for stable comparison: sort, deduplicate, trim, case-insensitive. /// Saved to DeltaState.Select so future comparisons are order-independent. + /// Also used for Prefer tokens - the same normalization semantics apply. /// private static string NormalizeSelect(string? s) => string.IsNullOrEmpty(s) ? "" : string.Join(",", @@ -83,9 +115,10 @@ protected override void BeginProcessing() DeltaState.ValidateWriteAccess(resolvedDeltaPath); // Validate -OutputFile writability before HTTP calls + string? resolvedOutputPath = null; if (OutputFile != null) { - var resolvedOutputPath = GetUnresolvedProviderPathFromPSPath(OutputFile); + resolvedOutputPath = GetUnresolvedProviderPathFromPSPath(OutputFile); if (string.Equals(resolvedDeltaPath, resolvedOutputPath, StringComparison.OrdinalIgnoreCase)) { ThrowTerminatingError(new ErrorRecord( @@ -96,6 +129,28 @@ protected override void BeginProcessing() DeltaState.ValidateWriteAccess(resolvedOutputPath); } + // The checkpoint must not collide with either state file: sharing a path would + // corrupt both the position and the data it describes. + if (CheckpointPath != null) + { + var resolvedCheckpointPath = GetUnresolvedProviderPathFromPSPath(CheckpointPath); + if (string.Equals(resolvedCheckpointPath, resolvedDeltaPath, StringComparison.OrdinalIgnoreCase)) + { + ThrowTerminatingError(new ErrorRecord( + new ArgumentException("-CheckpointPath and -DeltaPath must be different files."), + "CheckpointDeltaPathCollision", ErrorCategory.InvalidArgument, CheckpointPath)); + return; + } + if (resolvedOutputPath != null + && string.Equals(resolvedCheckpointPath, resolvedOutputPath, StringComparison.OrdinalIgnoreCase)) + { + ThrowTerminatingError(new ErrorRecord( + new ArgumentException("-CheckpointPath and -OutputFile must be different files."), + "CheckpointOutputCollision", ErrorCategory.InvalidArgument, CheckpointPath)); + return; + } + } + // Warn if URI doesn't look like a delta endpoint if (!Uri.Contains("/delta", StringComparison.OrdinalIgnoreCase)) { @@ -112,46 +167,84 @@ protected override void ProcessRecord() var resolvedOutputPath = OutputFile != null ? GetUnresolvedProviderPathFromPSPath(OutputFile) : null; + var resolvedCheckpointPath = CheckpointPath != null + ? GetUnresolvedProviderPathFromPSPath(CheckpointPath) + : null; - // Handle -FullSync: delete existing delta state - if (FullSync.IsPresent && File.Exists(resolvedDeltaPath)) + // Handle -FullSync: delete existing delta state and any resume checkpoint - the + // position it describes belongs to the enumeration being discarded. + if (FullSync.IsPresent) { - if (DeltaState.Delete(resolvedDeltaPath)) - { - WriteVerbose("Full sync requested. Deleted existing delta state."); - } - else + if (File.Exists(resolvedDeltaPath)) { - WriteWarning($"Full sync requested but could not delete '{DeltaPath}' (file may be locked). " + - "The existing delta state will be ignored and a full sync will proceed."); + if (DeltaState.Delete(resolvedDeltaPath)) + { + WriteVerbose("Full sync requested. Deleted existing delta state."); + } + else + { + WriteWarning($"Full sync requested but could not delete '{DeltaPath}' (file may be locked). " + + "The existing delta state will be ignored and a full sync will proceed."); + } } + DeleteCheckpoint(resolvedCheckpointPath, "full sync requested"); } - // Normalize $select for order-independent comparison + // Normalize $select and Prefer for order-independent comparison var normalizedSelect = NormalizeSelect(Property != null ? string.Join(",", Property) : null); + var normalizedPrefer = NormalizeSelect(Prefer != null ? string.Join(",", Prefer) : null); var currentFilter = Filter; string requestUrl; // LoadWithResult distinguishes "not found" from "corrupt". - // Validate delta state BEFORE GetClient() so validation errors - // are surfaced without requiring a Graph connection. + // The endpoint-independent state checks run BEFORE GetClient() so their + // errors surface without requiring a Graph connection; the checks that + // compare against the session's endpoint run after it. var (existingState, loadResult) = DeltaState.LoadWithResult(resolvedDeltaPath); + // -Latest means "baseline from now, return nothing". That is right for a first run and + // catastrophic after a state invalidation: the user is told a full re-sync is starting, + // gets zero items, and a fresh baseline token is persisted - so every change since the + // last successful sync is dropped permanently. The guard that warns "-Latest ignored" + // lives in the resume branch, which an invalidated state never reaches. Track it here + // and clear it wherever state is discarded. + var honourLatest = Latest.IsPresent; + + // A live resume checkpoint is not a fresh run either. Without delta state the guards + // below never fire, so -Latest was honored on top of an interrupted enumeration: the + // checkpoint is dropped a moment later as "a different enumeration" (the token=latest + // suffix changes requestUrl), the items the crashed run collected stay stranded in its + // temp, and an empty page still saves a from-now token - so everything before this + // moment is permanently unreachable. -FullSync deletes the checkpoint above, so + // "-FullSync -Latest" still re-baselines from now, which is what the warning below + // tells people to use. + var hasResumableCheckpoint = resolvedCheckpointPath != null && File.Exists(resolvedCheckpointPath); + if (honourLatest && hasResumableCheckpoint) + honourLatest = false; + if (loadResult == DeltaLoadResult.Corrupt) { WriteWarning($"Delta state file '{DeltaPath}' is corrupt. Starting full sync."); + // A corrupt state means the previous position is unknown, which is exactly when + // baselining from now would hide the most: everything since the last good sync. + honourLatest = false; } if (existingState != null) { - // Validate graph endpoint matches current session - if (!string.Equals(existingState.GraphEndpoint, s_graphEndpoint, StringComparison.OrdinalIgnoreCase)) + // The deltaLink is absolute and carries its own version, so a run that omits + // -ApiVersion silently keeps syncing whichever version built the state - the + // caller believes they are on the default and are not. Empty means a pre-2.0.1 + // state file: unknown, not mismatched, so upgrades are not broken by this check. + if (!string.IsNullOrEmpty(existingState.ApiVersion) + && !string.Equals(existingState.ApiVersion, ApiVersion, StringComparison.OrdinalIgnoreCase)) { ThrowTerminatingError(new ErrorRecord( new InvalidOperationException( - $"Delta state was created against '{existingState.GraphEndpoint}' " - + $"but current session is connected to '{s_graphEndpoint}'. " - + "Use -FullSync to start fresh, or reconnect to the original endpoint."), - "DeltaEndpointMismatch", ErrorCategory.InvalidOperation, null)); + $"Delta state was created against Graph {existingState.ApiVersion} " + + $"but this run requests {ApiVersion}. Re-run with " + + $"-ApiVersion {existingState.ApiVersion}, or use -FullSync to rebuild " + + $"against {ApiVersion}."), + "DeltaApiVersionMismatch", ErrorCategory.InvalidOperation, null)); return; } @@ -176,7 +269,28 @@ protected override void ProcessRecord() + " Starting full re-sync to capture all selected properties."); if (!DeltaState.Delete(resolvedDeltaPath)) WriteVerbose($"Could not delete old delta state at '{DeltaPath}' (file may be locked). It will be overwritten."); + DeleteCheckpoint(resolvedCheckpointPath, "property selection changed"); existingState = null; + honourLatest = false; // a discarded state is not a fresh run + } + + // Detect Prefer change between runs: the tokens shape what the enumeration + // returns (removed facets, sharing annotations), so mixing states is unsound. + if (existingState != null) + { + var storedPrefer = NormalizeSelect(existingState.Prefer); + if (!string.Equals(storedPrefer, normalizedPrefer, StringComparison.OrdinalIgnoreCase)) + { + WriteWarning( + "Prefer headers changed since last sync " + + $"(was: '{(string.IsNullOrEmpty(storedPrefer) ? "(none)" : storedPrefer)}', now: '{(string.IsNullOrEmpty(normalizedPrefer) ? "(none)" : normalizedPrefer)}')." + + " Starting full re-sync."); + if (!DeltaState.Delete(resolvedDeltaPath)) + WriteVerbose($"Could not delete old delta state at '{DeltaPath}' (file may be locked). It will be overwritten."); + DeleteCheckpoint(resolvedCheckpointPath, "Prefer headers changed"); + existingState = null; + honourLatest = false; // a discarded state is not a fresh run + } } // Detect filter change between runs @@ -189,12 +303,34 @@ protected override void ProcessRecord() + " Starting full re-sync."); if (!DeltaState.Delete(resolvedDeltaPath)) WriteVerbose($"Could not delete old delta state at '{DeltaPath}' (file may be locked). It will be overwritten."); + DeleteCheckpoint(resolvedCheckpointPath, "filter changed"); existingState = null; + honourLatest = false; // a discarded state is not a fresh run } } + // GetClient() sits between the two validation halves on purpose. It runs after the + // state-file checks above so their errors surface without a Graph connection, and + // before everything below because it is the only thing that refreshes s_graphEndpoint + // from the session: on the first call of a session the endpoint comparison and the + // request URL would otherwise be built against the default endpoint instead of the + // connected one. Invoke-MgxRequest sequences GetClient() first for the same reason. + var client = GetClient(); + if (existingState != null) { + // Validate graph endpoint matches current session + if (!string.Equals(existingState.GraphEndpoint, s_graphEndpoint, StringComparison.OrdinalIgnoreCase)) + { + ThrowTerminatingError(new ErrorRecord( + new InvalidOperationException( + $"Delta state was created against '{existingState.GraphEndpoint}' " + + $"but current session is connected to '{s_graphEndpoint}'. " + + "Use -FullSync to start fresh, or reconnect to the original endpoint."), + "DeltaEndpointMismatch", ErrorCategory.InvalidOperation, null)); + return; + } + // SSRF validation: deltaLink is untrusted (from a file on disk) var deltaUri = new System.Uri(s_graphEndpoint); var validated = NextLinkValidator.Validate(existingState.DeltaLink, deltaUri); @@ -228,6 +364,13 @@ protected override void ProcessRecord() return; } + if (Latest.IsPresent) + { + WriteWarning( + $"-Latest ignored: usable delta state already exists at '{DeltaPath}'. " + + "Delete it or use -FullSync to re-baseline from now."); + } + requestUrl = validated; WriteVerbose($"Resuming delta sync from {existingState.LastSync:u} ({existingState.ItemCount} items in previous sync)."); } @@ -235,13 +378,144 @@ protected override void ProcessRecord() { requestUrl = BuildListUrl(VersionedBaseUrl, Uri, new ODataListParams(false, Top, Top > 0 ? Top : 999, Filter, Property, null, null, 0, null)); - WriteVerbose("No existing delta state. Performing full initial sync."); + + if (Latest.IsPresent && !honourLatest) + { + WriteWarning(hasResumableCheckpoint + ? "-Latest ignored: a resume checkpoint exists, so an interrupted enumeration " + + "is still in progress. Baselining from now would abandon what it collected " + + "and drop every change before now. Delete the checkpoint, or use -FullSync " + + "to re-baseline." + : "-Latest ignored: the previous delta state was discarded, so this run must " + + "enumerate to rebuild it. Baselining from now would silently drop every " + + "change since the last successful sync."); + } + else if (honourLatest) + { + // "Sync from now": returns an empty page plus a deltaLink; the existing + // empty-page-still-saves-token path persists the baseline. The token form + // differs by service: OneDrive/SharePoint take token=latest, directory and + // everything else $deltatoken=latest. + var tokenParam = AdaptivePacing.Classify(Uri) == WorkloadBucket.Drive + ? "token=latest" + : "$deltatoken=latest"; + requestUrl += (requestUrl.Contains('?') ? "&" : "?") + tokenParam; + WriteVerbose($"No existing delta state. Requesting latest delta token only ({tokenParam})."); + } + else + { + WriteVerbose("No existing delta state. Performing full initial sync."); + } } - // GetClient() after validation so delta state errors surface without Graph connection - var client = GetClient(); ExecuteDeltaSync(client, requestUrl, resolvedDeltaPath, resolvedOutputPath, - normalizedSelect, currentFilter, sw); + resolvedCheckpointPath, normalizedSelect, normalizedPrefer, currentFilter, sw); + } + + /// + /// The Graph API version a deltaLink was issued by, read from the link itself, or null when + /// it cannot be read. + /// + private static string? ApiVersionOfLink(string? deltaLink) + { + if (string.IsNullOrEmpty(deltaLink)) return null; + if (!System.Uri.TryCreate(deltaLink, UriKind.Absolute, out var u)) return null; + var first = u.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(); + return string.Equals(first, "v1.0", StringComparison.OrdinalIgnoreCase) + || string.Equals(first, "beta", StringComparison.OrdinalIgnoreCase) + ? first + : null; + } + + private void DeleteCheckpoint(string? checkpointPath, string reason) + { + if (checkpointPath == null || !File.Exists(checkpointPath)) return; + if (PaginationCheckpoint.Delete(checkpointPath)) + WriteVerbose($"Deleted resume checkpoint ({reason})."); + else + WriteWarning($"Could not delete resume checkpoint at '{checkpointPath}' ({reason}). " + + "Delete it manually before the next run."); + } + + + /// + /// Put the files into the state the checkpoint claims, or delete the checkpoint. A + /// checkpoint records which file its items were written to and how many bytes of that file + /// they occupy, which makes three cases decidable instead of guessed. + /// + /// A temp is named, so the interrupted run was fresh and its items are in that temp while + /// the output still holds the PREVIOUS sync's rows. Those rows were the previous run's + /// result, already replaced by every path that completes - a fresh run that finishes moves + /// its temp over the output, and a cancellation promotes the same way - so recovery + /// promotes too. Appending instead would put rows the caller has already consumed back in + /// front of this sync's. + /// + /// None is named, so the run was appending to the output and its items are already there, + /// past the recorded length only if it wrote more after its last save. Cutting back to that + /// length is what stops those from being written twice. + /// + /// Neither is recorded, so the checkpoint cannot say which file its items are in, and an + /// appending run's checkpoint cannot be told from a fresh one's. When no output exists the + /// ambiguity is harmless - everything the run wrote is in its temp - but against an + /// existing output the only recovery that cannot lose or repeat items is re-enumerating. + /// + /// When the counted items turn out to be in no file, the delta link has not moved, so + /// re-enumerating costs time and loses nothing while resuming past them loses them for good. + /// + private void ReconcileCheckpointWithFiles(string checkpointPath, string outputPath, PaginationCheckpoint checkpoint) + { + if (checkpoint.DataLength is not { } dataLength) + { + if (!File.Exists(outputPath)) + { + if (TryAdoptOrphanedTemp(outputPath, checkpoint.ItemsCollected)) + WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted sync's temp file. Resuming from checkpoint."); + else + { + WriteWarning("Checkpoint found but output file is missing. Deleting stale checkpoint and starting fresh."); + PaginationCheckpoint.Delete(checkpointPath); + } + return; + } + + WriteWarning( + "The resume checkpoint does not record which file the interrupted sync's items are in. " + + "Re-enumerating from the last saved delta token; no changes are lost."); + PaginationCheckpoint.Delete(checkpointPath); + return; + } + + if (checkpoint.TempFile != null) + { + if (TryPromoteNamedTemp(outputPath, checkpoint.TempFile, dataLength)) + { + WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted sync's temp file. Resuming from checkpoint."); + // Those items are the output now. Repoint the checkpoint at it immediately, + // so a second interruption cannot promote the same temp a second time. + checkpoint.TempFile = null; + checkpoint.DataLength = new FileInfo(outputPath).Length; + try { checkpoint.Save(checkpointPath); } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + WriteWarning($"Checkpoint save failed after recovery: {ex.Message}"); + } + return; + } + + WriteWarning( + $"The interrupted sync's temp file is missing or incomplete, so the {checkpoint.ItemsCollected} items it " + + "recorded are not on disk. Re-enumerating from the last saved delta token; no changes are lost."); + PaginationCheckpoint.Delete(checkpointPath); + return; + } + + if (!TryTrimOutputToCheckpoint(outputPath, dataLength)) + { + WriteWarning( + $"'{outputPath}' no longer holds the {checkpoint.ItemsCollected} items the resume checkpoint records. " + + "Re-enumerating from the last saved delta token; no changes are lost."); + PaginationCheckpoint.Delete(checkpointPath); + } } private void ExecuteDeltaSync( @@ -249,7 +523,9 @@ private void ExecuteDeltaSync( string requestUrl, string deltaPath, string? outputPath, + string? checkpointPath, string? select, + string? prefer, string? filter, Stopwatch sw) { @@ -260,37 +536,225 @@ private void ExecuteDeltaSync( try { var headers = BuildRequestHeaders(null, Headers); + if (Prefer is { Length: > 0 }) + { + // Dedicated parameter wins over a Prefer key in -Headers (matches the + // ConsistencyLevel convention in BuildRequestHeaders). + headers ??= new Dictionary(); + headers["Prefer"] = string.Join(",", Prefer); + } + + // --- resume from checkpoint, when one exists for THIS enumeration --- + ResumeState? resume = null; + long resumedItemCount = 0; + var currentFetchUrl = requestUrl; + var appendOutput = false; + + if (checkpointPath != null && File.Exists(checkpointPath)) + { + // JSONL crash: the checkpoint survives but the output was never promoted from + // its temp file. Promote the temp (trimmed to the checkpointed length) so + // resume appends to real data instead of declaring staleness. Without this + // the resume restarts at checkpoint.NextLink and the crashed run's items, + // sitting only in the temp, are never emitted - while the delta token + // advances past them on success. + if (outputPath != null) + { + var orphanCp = PaginationCheckpoint.Load(checkpointPath); + + // The resource is checked before anything is merged. The temp glob is + // shaped from the output path, so a checkpoint belonging to a different + // enumeration must never be allowed to pull a file into this one. The + // mismatch itself is handled a few lines below; here we only decline. + var resourceMatches = orphanCp != null + && string.Equals(orphanCp.Resource, requestUrl, StringComparison.Ordinal); + + if (resourceMatches && orphanCp!.NextLink != null) + { + ReconcileCheckpointWithFiles(checkpointPath, outputPath, orphanCp); + } + else if (!File.Exists(outputPath)) + { + WriteWarning("Checkpoint found but output file is missing. Deleting stale checkpoint and starting fresh."); + PaginationCheckpoint.Delete(checkpointPath); + } + } + + if (File.Exists(checkpointPath)) + { + var checkpoint = PaginationCheckpoint.Load(checkpointPath); + if (checkpoint?.NextLink == null) + { + WriteVerbose("Checkpoint is stale (corrupt or completed). Deleting."); + PaginationCheckpoint.Delete(checkpointPath); + } + else if (!string.Equals(checkpoint.Resource, requestUrl, StringComparison.Ordinal)) + { + // A checkpoint from a different enumeration (different deltaLink, + // different parameters, a completed sync in between). + WriteWarning("Checkpoint belongs to a different enumeration. Deleting checkpoint and starting fresh."); + PaginationCheckpoint.Delete(checkpointPath); + } + else + { + // SSRF validation: the checkpoint nextLink is untrusted (a file on disk) + var expectedHost = new System.Uri(requestUrl); + var validatedLink = NextLinkValidator.Validate(checkpoint.NextLink, expectedHost); + if (validatedLink != null + && checkpoint.ItemsCollected >= 0 + && checkpoint.PageItemsAlreadyWritten >= 0) + { + resume = new ResumeState( + validatedLink, + checkpoint.PageItemsAlreadyWritten, + checkpoint.ItemsCollected); + currentFetchUrl = validatedLink; + resumedItemCount = checkpoint.ItemsCollected; + appendOutput = outputPath != null && File.Exists(outputPath); + WriteVerbose($"Resuming delta enumeration from checkpoint: {resumedItemCount} items already processed" + + (checkpoint.PageItemsAlreadyWritten > 0 + ? $", skipping {checkpoint.PageItemsAlreadyWritten} items on first page." + : ".")); + } + else + { + WriteWarning("Checkpoint nextLink failed validation. Deleting checkpoint and starting fresh."); + PaginationCheckpoint.Delete(checkpointPath); + } + } + } + } + var iterator = new PageIterator(client); string? capturedDeltaLink = null; long itemCount = 0; + long removedCount = 0; + long totalProcessed = resumedItemCount; + // Seeded from the resume skip, not 0. PageIterator drops the skipped items before + // the consumer ever sees them (PageIterator.cs: "if (isFirstPage && skippedOnPage + // < skipOnFirstPage) continue;"), so a counter starting at 0 records only the + // NEWLY written items of the first resumed page. A mid-page checkpoint there then + // claimed fewer items of that page than the output actually held, and the next + // resume skipped too few and re-emitted the difference - up to a page's worth of + // duplicate lines, which is exactly what the comment below says cannot happen. + int pageItemsWritten = resume?.SkipOnFirstPage ?? 0; + + // What the next two checkpoint sites should say about WHERE the counted items + // are. Set once the writer exists; null on the pipeline path, which has no file. + string? checkpointTempFile = null; + long? checkpointDataLength = null; + + void OnPageComplete(PageCompletedInfo info) + { + if (info.NextPageUrl != null) + currentFetchUrl = info.NextPageUrl; + pageItemsWritten = 0; + } - var enumerable = iterator.StreamAllWithCountAsync( - requestUrl, - maxItems: 0, - onCount: null, - headers: headers, - onDeltaLink: dl => capturedDeltaLink = dl, - cancellationToken: CancellationToken); + void SaveBoundaryCheckpoint(PageCompletedInfo info) + { + if (checkpointPath == null || info.NextPageUrl == null) return; + try + { + new PaginationCheckpoint + { + Resource = requestUrl, + NextLink = info.NextPageUrl, + ItemsCollected = totalProcessed, + PageItemsAlreadyWritten = 0, + TempFile = checkpointTempFile, + DataLength = checkpointDataLength + }.Save(checkpointPath); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + WriteWarning($"Checkpoint save failed (page boundary): {ex.Message}"); + } + } if (outputPath != null) { - // JSONL output mode - var writePath = $"{outputPath}.{Guid.NewGuid():N}.tmp"; + // JSONL output mode. Fresh runs write to a temp file and promote on + // success; checkpointed resumes append to the already-promoted output. + if (!appendOutput) + { + // Nothing is being resumed, so no temp on disk describes recoverable + // work. Anything left over was orphaned - by -FullSync, by a -Property + // /-Filter/-Prefer change, by a checkpoint from a different enumeration, + // or by an adoption that declined a torn temp - and orphans are not + // inert: TryAdoptOrphanedTemp picks the NEWEST file matching + // outputPath + ".*.tmp" with nothing but a line count to go on, so a + // survivor from an unrelated enumeration is adoptable by some later + // crash's checkpoint, and one success makes those rows permanent. + // Sweeping here is what keeps "a temp exists only while a checkpoint + // describing it exists" true across runs. + DeleteStaleTemps(outputPath); + } + var writePath = appendOutput ? outputPath : $"{outputPath}.{Guid.NewGuid():N}.tmp"; + // A resumed run appends to the output itself, so there is no temp to name. + checkpointTempFile = appendOutput ? null : Path.GetFileName(writePath); try { - using (var writer = new StreamWriter(writePath, append: false)) + using (var writer = new StreamWriter(writePath, appendOutput)) { + var enumerable = iterator.StreamAllWithCountAsync( + requestUrl, + maxItems: 0, + onCount: null, + headers: headers, + resume: resume, + onPageComplete: info => + { + writer.Flush(); + checkpointDataLength = writer.BaseStream.Position; + SaveBoundaryCheckpoint(info); + OnPageComplete(info); + }, + onDeltaLink: dl => capturedDeltaLink = dl, + cancellationToken: CancellationToken); + var enumerator = enumerable.GetAsyncEnumerator(CancellationToken); try { while (enumerator.MoveNextAsync().AsTask().GetAwaiter().GetResult()) { writer.WriteLine(enumerator.Current.GetRawText()); + // TryGetProperty throws on anything that is not an object, + // and the item is whatever the service put in "value". + if (enumerator.Current.ValueKind == JsonValueKind.Object + && enumerator.Current.TryGetProperty("@removed", out _)) + removedCount++; itemCount++; + pageItemsWritten++; + totalProcessed++; DrainClientMessages(); - if (itemCount % 500 == 0) + if (totalProcessed % 500 == 0) + { writer.Flush(); + checkpointDataLength = writer.BaseStream.Position; + // Mid-page checkpoint: tracks items written from the + // current page so crash resume skips them (no dupes). + if (checkpointPath != null) + { + try + { + new PaginationCheckpoint + { + Resource = requestUrl, + NextLink = currentFetchUrl, + ItemsCollected = totalProcessed, + PageItemsAlreadyWritten = pageItemsWritten, + TempFile = checkpointTempFile, + DataLength = checkpointDataLength + }.Save(checkpointPath); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + WriteWarning($"Mid-page checkpoint save failed: {ex.Message}"); + } + } + } } } finally @@ -298,25 +762,102 @@ private void ExecuteDeltaSync( enumerator.DisposeAsync().AsTask().GetAwaiter().GetResult(); } } - File.Move(writePath, outputPath, overwrite: true); + if (!appendOutput) + File.Move(writePath, outputPath, overwrite: true); } - catch + catch (Exception attemptEx) { - try { if (File.Exists(writePath)) File.Delete(writePath); } catch { } + if (!appendOutput) + { + // User cancellation of a checkpointed fresh run: promote the temp + // file (the using block already flushed it on unwind) and save a + // checkpoint matching its exact content, so resume works on first + // runs too. Otherwise clean the temp up as before. + var cancelled = attemptEx is OperationCanceledException + && CancellationToken.IsCancellationRequested; + var promoted = false; + if (cancelled && checkpointPath != null && itemCount > 0) + { + try + { + // Promote first: once the move lands the items are in the + // output, so that is what the checkpoint must point at. If + // the save then fails, the previous checkpoint still names + // a temp that no longer exists, which reads as unusable and + // costs a re-enumeration rather than a wrong resume. + var promotedLength = new FileInfo(writePath).Length; + File.Move(writePath, outputPath, overwrite: true); + promoted = true; + new PaginationCheckpoint + { + Resource = requestUrl, + NextLink = currentFetchUrl, + ItemsCollected = totalProcessed, + PageItemsAlreadyWritten = pageItemsWritten, + TempFile = null, + DataLength = promotedLength + }.Save(checkpointPath); + } + catch (Exception promoteEx) when (promoteEx is IOException or UnauthorizedAccessException) + { + // Promotion is best-effort; fall back to the old cleanup. + } + } + if (!promoted) + { + // A surviving checkpoint describes items that exist ONLY in this + // temp: SaveBoundaryCheckpoint flushes the writer before recording + // the position, so the temp always holds at least ItemsCollected. + // Deleting it leaves the checkpoint pointing past data that is + // nowhere - and the next run then finds a checkpoint, an output + // and no temp, which is the routine "nothing to promote" state, so + // it resumes in APPEND mode against an output that never received + // these pages and the delta token advances past them. + // Keep the temp for the next run to promote. It is deleted by + // promotion, by a later fresh run's own failure once the checkpoint + // is gone, or on the missing-output path below. + var resumable = checkpointPath != null && File.Exists(checkpointPath); + if (!resumable) + { + try { if (File.Exists(writePath)) File.Delete(writePath); } catch { } + } + } + } throw; } } else { - // Pipeline output mode + // Pipeline output mode. Checkpoints save at page boundaries only: + // emitted objects cannot be un-emitted, so resume re-emits the page in + // flight at the crash (at-least-once, documented). + var enumerable = iterator.StreamAllWithCountAsync( + requestUrl, + maxItems: 0, + onCount: null, + headers: headers, + resume: resume, + onPageComplete: info => + { + SaveBoundaryCheckpoint(info); + OnPageComplete(info); + }, + onDeltaLink: dl => capturedDeltaLink = dl, + cancellationToken: CancellationToken); + var enumerator = enumerable.GetAsyncEnumerator(CancellationToken); try { while (enumerator.MoveNextAsync().AsTask().GetAwaiter().GetResult()) { + if (enumerator.Current.ValueKind == JsonValueKind.Object + && enumerator.Current.TryGetProperty("@removed", out _)) + removedCount++; var ht = JsonToHashtable(enumerator.Current); WriteObject(ht); itemCount++; + pageItemsWritten++; + totalProcessed++; DrainClientMessages(); } } @@ -326,6 +867,11 @@ private void ExecuteDeltaSync( } } + // Success: the checkpoint's job is done - delete it BEFORE saving delta + // state, so a crash between the two leaves a fresh incremental (correct) + // rather than a resumable position into a completed enumeration (wrong). + DeleteCheckpoint(checkpointPath, "sync completed"); + // Save delta state ONLY after successful completion (Architect P0). // Zero-item responses still save the token (Adversarial P0). if (capturedDeltaLink != null) @@ -335,9 +881,17 @@ private void ExecuteDeltaSync( DeltaLink = capturedDeltaLink, Select = select, // Normalized value for stable future comparisons Filter = filter, + Prefer = prefer, // Normalized, like Select Resource = Uri, - ItemCount = itemCount, - GraphEndpoint = s_graphEndpoint + ItemCount = totalProcessed, + GraphEndpoint = s_graphEndpoint, + // The version the LINK carries, not the one that was asked for. A state + // file written before this field existed has none, so the mismatch check + // is skipped and the run proceeds - against whatever version the stored + // deltaLink names, which may not be the requested one. Stamping the + // request there recorded a version the token was never issued by, and + // every later run then refused with advice pointing the wrong way. + ApiVersion = ApiVersionOfLink(capturedDeltaLink) ?? ApiVersion }.Save(deltaPath); WriteVerbose($"Delta state saved to '{deltaPath}'."); } @@ -350,7 +904,10 @@ private void ExecuteDeltaSync( sw.Stop(); WriteVerbose( - $"Delta sync complete: {itemCount} items in {sw.Elapsed.TotalSeconds:F1}s" + $"Delta sync complete: {itemCount} items" + + (removedCount > 0 ? $" ({removedCount} removed)" : "") + + $" in {sw.Elapsed.TotalSeconds:F1}s" + + (resumedItemCount > 0 ? $" (resumed after {resumedItemCount})" : "") + (isFullResync ? " (full re-sync after 410 Gone)" : "") + (outputPath != null ? $". Output: {outputPath}" : ".")); @@ -361,11 +918,13 @@ private void ExecuteDeltaSync( && ex.StatusCode == HttpStatusCode.Gone) { // 410 Gone: delta token expired (>7 days for directory objects). - // Delete delta state and restart with full sync. + // Delete delta state AND any checkpoint - both describe the dead + // enumeration - and restart with full sync. // Second attempt builds fresh URL (no delta token), so 410 won't recur. DrainClientMessages(); if (!DeltaState.Delete(deltaPath)) WriteVerbose("Could not delete expired delta state (file may be locked). It will be overwritten."); + DeleteCheckpoint(checkpointPath, "delta token expired (410 Gone)"); isFullResync = true; requestUrl = BuildListUrl(VersionedBaseUrl, Uri, new ODataListParams(false, Top, Top > 0 ? Top : 999, Filter, Property, null, null, 0, null)); @@ -377,7 +936,11 @@ private void ExecuteDeltaSync( catch (OperationCanceledException) when (CancellationToken.IsCancellationRequested) { DrainClientMessages(); - WriteWarning("Delta sync cancelled."); + var resumeHint = checkpointPath != null + ? $" Resume with: Sync-MgxDelta '{Uri}' -DeltaPath '{DeltaPath}' -CheckpointPath '{CheckpointPath}'" + + (OutputFile != null ? $" -OutputFile '{OutputFile}'" : "") + : " Use -CheckpointPath to enable mid-run resume."; + WriteWarning($"Delta sync cancelled.{resumeHint}"); return; } catch (Exception ex) when (ex is GraphServiceException or BrokenCircuitException or HttpRequestException) @@ -392,6 +955,18 @@ private void ExecuteDeltaSync( ErrorCategory.WriteError, OutputFile)); return; } + catch (UnauthorizedAccessException ex) + { + // Not an IOException, so the catch above never saw it. It is what Windows raises + // for a denying ACL, a read-only file, or an -OutputFile naming a directory, and + // leaving it out made every one of those an unhandled error there while the same + // failure was a clean error record on Unix. Export-MgxCollection already reports + // it this way. + DrainClientMessages(); + WriteError(new ErrorRecord(ex, "AccessDenied", + ErrorCategory.PermissionDenied, OutputFile)); + return; + } catch (Exception) { // Drain buffered messages for unexpected exception types diff --git a/src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs b/src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs index ab007df..534ee15 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs @@ -221,7 +221,7 @@ private void ExecuteFanOut() ? nl.GetString() : null; Uri? expectedHost = System.Uri.TryCreate(url, UriKind.Absolute, out var parsed) ? parsed : null; - nextLink = NextLinkValidator.Validate(nextLink, expectedHost); + nextLink = NextLinkValidator.ValidateOrThrow(nextLink, expectedHost); int consecutiveEmptyPages = 0; while (nextLink != null) @@ -248,7 +248,7 @@ private void ExecuteFanOut() break; } - nextLink = NextLinkValidator.Validate(page.NextLink, expectedHost); + nextLink = NextLinkValidator.ValidateOrThrow(page.NextLink, expectedHost); } results[url] = items.ToArray(); @@ -284,7 +284,6 @@ private void OutputBufferedObjects(Dictionary resultsById { // Cache converted results per ID to avoid redundant JsonToHashtable calls // when multiple buffer objects share the same ID - var convertedCache = new Dictionary(); HashSet? flattenWarned = Flatten.IsPresent ? [] : null; foreach (var obj in _buffer) @@ -295,30 +294,29 @@ private void OutputBufferedObjects(Dictionary resultsById if (id != null && resultsById.ContainsKey(id)) { - if (!convertedCache.TryGetValue(id, out relationValue)) + // Converted per input object rather than once per id. Two inputs carrying the + // same id used to receive the SAME hashtables, so writing to one output's + // relation silently rewrote the others'. The conversion only repeats when an id + // actually appears twice, which is the same case that made the sharing visible. + var items = resultsById[id]; + var converted = items.Select(JsonToHashtable).ToArray(); + + if (Flatten.IsPresent) { - var items = resultsById[id]; - var converted = items.Select(JsonToHashtable).ToArray(); - - if (Flatten.IsPresent) + if (converted.Length <= 1) { - if (converted.Length <= 1) - { - relationValue = converted.Length == 1 ? converted[0] : null; - } - else - { - if (flattenWarned!.Add(id)) - WriteWarning($"-Flatten: entity '{id}' returned {converted.Length} items instead of 1. Returning array."); - relationValue = converted; - } + relationValue = converted.Length == 1 ? converted[0] : null; } else { + if (flattenWarned!.Add(id)) + WriteWarning($"-Flatten: entity '{id}' returned {converted.Length} items instead of 1. Returning array."); relationValue = converted; } - - convertedCache[id] = relationValue; + } + else + { + relationValue = converted; } } @@ -359,10 +357,12 @@ private string BuildUrl(string id) var resolved = IdPlaceholder.Replace(Uri, System.Uri.EscapeDataString(id)); var url = $"{VersionedBaseUrl}{NormalizePath(resolved)}"; - // Pass $top to Graph so it returns only the items we need, - // avoiding full-page downloads when only a few items are wanted. - // Client-side truncation in the lambda remains as a safety net. - if (Top > 0) + // Pass $top to Graph so it returns only the items we need, avoiding full-page downloads + // when only a few items are wanted. Skipped when the caller already wrote one into + // -Uri: Graph rejects a URL carrying $top twice, which turned -Top into a 400 for + // every input object. + // The client-side truncation in the lambda enforces -Top either way. + if (Top > 0 && !HasTopQueryOption(url)) { var separator = url.Contains('?') ? "&" : "?"; url = $"{url}{separator}$top={Top}"; @@ -371,6 +371,19 @@ private string BuildUrl(string id) return url; } + /// True when the URL already carries a $top query option. + private static bool HasTopQueryOption(string url) + { + var q = url.IndexOf('?'); + if (q < 0) return false; + foreach (var part in url[(q + 1)..].Split('&')) + { + var name = part.Split('=', 2)[0]; + if (string.Equals(name, "$top", StringComparison.OrdinalIgnoreCase)) return true; + } + return false; + } + private void HandleFanOutErrors( Dictionary errors, Dictionary urlToId) diff --git a/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs b/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs index 548764a..61bc07a 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs @@ -121,13 +121,16 @@ protected override void ProcessRecord() return; } - // Determine max items: -All = unlimited (overrides -Top), -Top N = N, neither = single page + // -All says how far to page, -Top says how much to return. They answer different + // questions, so -All no longer discards the cap: -Top is documented as the total + // maximum, and Invoke-MgxRequest already honours it either way. Neither, and a single + // page is the limit. int maxItems; bool defaultedToPageSize = false; - if (All.IsPresent) - maxItems = 0; // unlimited; -All always overrides -Top - else if (Top > 0) + if (Top > 0) maxItems = Top; + else if (All.IsPresent) + maxItems = 0; // unlimited else { maxItems = PageSize; // single page worth @@ -140,14 +143,14 @@ protected override void ProcessRecord() // adopt it: trim to exactly the item count the checkpoint recorded (content // past the last flush may be missing or torn) and promote it to the output // path. Only when no usable temp exists is the checkpoint truly stale. - if (cpPath != null && File.Exists(cpPath) && !File.Exists(outputPath)) + if (cpPath != null && File.Exists(cpPath)) { var orphanCp = PaginationCheckpoint.Load(cpPath); - if (orphanCp?.NextLink != null && TryAdoptOrphanedTemp(outputPath, orphanCp.ItemsCollected)) + if (orphanCp?.NextLink != null) { - WriteWarning($"Recovered {orphanCp.ItemsCollected} items from an interrupted export's temp file. Resuming from checkpoint."); + ReconcileCheckpointWithFiles(cpPath, outputPath, orphanCp); } - else + else if (!File.Exists(outputPath)) { WriteWarning("Checkpoint found but output file is missing. Deleting stale checkpoint and starting fresh."); PaginationCheckpoint.Delete(cpPath); @@ -242,9 +245,26 @@ protected override void ProcessRecord() // This protects any pre-existing output file from truncation if // the Graph request fails on the first page. // Use GUID to prevent collision when multiple exports target the same file. + if (!append) + { + // No resume is pending, so every "{output}.{guid}.tmp" on disk is an orphan. + // Leaving them is not inert: the pre-length adoption path picks the NEWEST + // match with only a line count to go on, so a survivor of an unrelated run is + // adoptable by some later crash's checkpoint. + DeleteStaleTemps(outputPath); + } var writePath = append ? outputPath : $"{outputPath}.{Guid.NewGuid():N}.tmp"; + // What the checkpoint sites below should say about WHERE the counted items are. + // A resumed run appends to the output itself, so it has no temp to name. + string? checkpointTempFile = append ? null : Path.GetFileName(writePath); + long? checkpointDataLength = null; long itemCount = 0; - int pageItemsWritten = 0; + // Seeded from the resume skip, not 0. PageIterator drops the skipped items before + // this loop ever sees them, so a counter starting at 0 records only the NEWLY + // written items of a resumed first page. A mid-page checkpoint saved there then + // claims fewer items of that page than the output holds, and the next resume skips + // too few and writes the difference twice. + int pageItemsWritten = resume?.SkipOnFirstPage ?? 0; long totalWritten = resumedItemCount; long? reportedODataCount = null; @@ -268,12 +288,15 @@ protected override void ProcessRecord() try { writer.Flush(); + checkpointDataLength = writer.BaseStream.Position; new PaginationCheckpoint { Resource = url, NextLink = info.NextPageUrl, ItemsCollected = totalWritten, - PageItemsAlreadyWritten = 0 + PageItemsAlreadyWritten = 0, + TempFile = checkpointTempFile, + DataLength = checkpointDataLength }.Save(cpPath); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) @@ -308,12 +331,15 @@ protected override void ProcessRecord() { try { + checkpointDataLength = writer.BaseStream.Position; new PaginationCheckpoint { Resource = url, NextLink = currentFetchUrl, ItemsCollected = totalWritten, - PageItemsAlreadyWritten = pageItemsWritten + PageItemsAlreadyWritten = pageItemsWritten, + TempFile = checkpointTempFile, + DataLength = checkpointDataLength }.Save(cpPath); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) @@ -356,15 +382,23 @@ protected override void ProcessRecord() { try { + // Move first, then record. A checkpoint that named the output + // before the move existed would describe a file that is not + // there yet, and a move that then failed would leave it saying + // so. The length is the temp's, taken before the move, because + // it is the same bytes under a different name afterwards. + var promotedLength = new FileInfo(writePath).Length; + File.Move(writePath, outputPath, overwrite: true); + promoted = true; new PaginationCheckpoint { Resource = url, NextLink = currentFetchUrl, ItemsCollected = totalWritten, - PageItemsAlreadyWritten = pageItemsWritten + PageItemsAlreadyWritten = pageItemsWritten, + TempFile = null, + DataLength = promotedLength }.Save(cpPath); - File.Move(writePath, outputPath, overwrite: true); - promoted = true; } catch (Exception promoteEx) when (promoteEx is IOException or UnauthorizedAccessException) { @@ -373,7 +407,20 @@ protected override void ProcessRecord() } if (!promoted) { - try { if (File.Exists(writePath)) File.Delete(writePath); } catch { } + // A surviving checkpoint counts items that exist only in this temp: + // every checkpoint site flushes the writer before recording the + // position, so the temp always holds at least what it promises. + // Deleting it made the next run's recovery find the checkpoint + // naming a missing file and start the export over - resume worked + // after a kill or a Ctrl-C but never after a handled error, which + // is the common way a long export dies. Keep the temp for the next + // run to promote; it is deleted by promotion or by the stale-temp + // sweep once the checkpoint is gone. + var resumable = cpPath != null && File.Exists(cpPath); + if (!resumable) + { + try { if (File.Exists(writePath)) File.Delete(writePath); } catch { } + } } } throw; // re-throw to retry catch or outer catch blocks @@ -466,62 +513,99 @@ protected override void ProcessRecord() } } + private string BuildUrl(bool includeCount, bool suppressTop = false) => BuildListUrl( + VersionedBaseUrl, Uri, + new ODataListParams(NoPageSize.IsPresent || suppressTop, Top, PageSize, Filter, + Property, Sort, Search, Skip, ExpandProperty, + IncludeCount: includeCount)); + + private Dictionary? BuildHeaders() => + BuildRequestHeaders(ConsistencyLevel, Headers); + /// - /// Recovers a fresh-run export that was interrupted before its temp file was - /// promoted (kill/crash: the temp survives; the checkpoint reflects the last - /// flush). Copies exactly lines from the newest - /// matching temp file to the output path - content beyond the last flush may - /// be absent or torn - then removes the temp. Returns false when nothing - /// usable exists, leaving the caller to the stale-checkpoint path. + /// Put the files into the state the checkpoint claims, or delete the checkpoint. A + /// checkpoint records which file its items were written to and how many bytes of that file + /// they occupy, which makes three cases decidable instead of guessed. + /// + /// A temp is named, so the interrupted run was fresh and its items are in that temp while + /// the output still holds a PREVIOUS export. An export is a snapshot and a fresh run that + /// finishes moves its temp over the output, so recovery promotes the temp the same way - + /// appending would leave the previous export's rows in front of this one's. + /// + /// None is named, so the run was appending to the output and its items are already there, + /// past the recorded length only if it wrote more after its last save. Cutting back to that + /// length is what stops those from being written twice. + /// + /// Neither is recorded, so the checkpoint predates this and is handled as it was before. + /// + /// When the counted items turn out to be in no file, nothing has been promoted and no token + /// has moved, so starting over costs a pass and loses nothing, while resuming past them + /// would drop them from the output for good. /// - private static bool TryAdoptOrphanedTemp(string outputPath, long itemCount) + private void ReconcileCheckpointWithFiles(string checkpointPath, string outputPath, PaginationCheckpoint checkpoint) { - try + if (checkpoint.DataLength is not { } dataLength) { - if (itemCount <= 0) return false; - var dir = Path.GetDirectoryName(outputPath); - if (string.IsNullOrEmpty(dir) || !Directory.Exists(dir)) return false; - var temp = Directory.EnumerateFiles(dir, Path.GetFileName(outputPath) + ".*.tmp") - .Select(p => new FileInfo(p)) - .OrderByDescending(f => f.LastWriteTimeUtc) - .FirstOrDefault(); - if (temp == null) return false; - - long copied = 0; - var adoptPath = outputPath + ".adopt"; - using (var reader = new StreamReader(temp.FullName)) - using (var writer = new StreamWriter(adoptPath, append: false)) + // Written before any of this was recorded. Adoption then has only a line count and + // the newest matching temp to go on, which is safe to attempt only when there is no + // output it could be merged into - exactly the case this path used to be limited to. + if (!File.Exists(outputPath)) { - string? line; - while (copied < itemCount && (line = reader.ReadLine()) != null) + if (TryAdoptOrphanedTemp(outputPath, checkpoint.ItemsCollected)) + WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted export's temp file. Resuming from checkpoint."); + else { - writer.WriteLine(line); - copied++; + WriteWarning("Checkpoint found but output file is missing. Deleting stale checkpoint and starting fresh."); + PaginationCheckpoint.Delete(checkpointPath); } + return; } - if (copied < itemCount) + + // The output exists and the checkpoint cannot say whether its items are in it. Both + // shapes are possible from a release that recorded neither field: a run that was + // appending, whose items ARE there, and a fresh run killed mid-flight, whose items + // are in a temp while the output still holds a PREVIOUS export. Resuming assumed the + // first, so upgrading with the second on disk appended the remainder of the + // enumeration onto the earlier export - a 100,000-row file coming back with 163,037. + // Undecidable means start over: an export re-runs from the first page and replaces + // the output, which costs a pass and cannot leave a wrong file behind. + WriteWarning( + "The resume checkpoint does not record which file the interrupted export's items are in. " + + "Exporting again from the beginning; nothing is lost."); + PaginationCheckpoint.Delete(checkpointPath); + return; + } + + if (checkpoint.TempFile != null) + { + if (TryPromoteNamedTemp(outputPath, checkpoint.TempFile, dataLength)) { - // Temp holds less than the checkpoint promises - unusable. - File.Delete(adoptPath); - return false; + WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted export's temp file. Resuming from checkpoint."); + // Those items are the output now. Repoint the checkpoint at it before anything + // else can fail, so a second interruption cannot promote the same temp again. + checkpoint.TempFile = null; + checkpoint.DataLength = new FileInfo(outputPath).Length; + try { checkpoint.Save(checkpointPath); } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + WriteWarning($"Checkpoint save failed after recovery: {ex.Message}"); + } + return; } - File.Move(adoptPath, outputPath, overwrite: true); - temp.Delete(); - return true; + + WriteWarning( + $"The interrupted export's temp file is missing or incomplete, so the {checkpoint.ItemsCollected} items it " + + "recorded are not on disk. Exporting again from the beginning; nothing is lost."); + PaginationCheckpoint.Delete(checkpointPath); + return; } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + + if (!TryTrimOutputToCheckpoint(outputPath, dataLength)) { - return false; + WriteWarning( + $"'{outputPath}' no longer holds the {checkpoint.ItemsCollected} items the resume checkpoint records. " + + "Exporting again from the beginning; nothing is lost."); + PaginationCheckpoint.Delete(checkpointPath); } } - - private string BuildUrl(bool includeCount, bool suppressTop = false) => BuildListUrl( - VersionedBaseUrl, Uri, - new ODataListParams(NoPageSize.IsPresent || suppressTop, Top, PageSize, Filter, - Property, Sort, Search, Skip, ExpandProperty, - IncludeCount: includeCount)); - - private Dictionary? BuildHeaders() => - BuildRequestHeaders(ConsistencyLevel, Headers); - } diff --git a/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs b/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs index 2114f2d..9059a4c 100644 --- a/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs +++ b/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs @@ -294,7 +294,11 @@ private void ExecuteList(string relativeUri, string? sourceId) includeCount: !string.IsNullOrEmpty(CountVariable) || includeAutoCount, noPageSize: suppressTop); var iterator = new PageIterator(GetClient()); - var maxItems = (All.IsPresent || Top <= 0) ? 0 : Top; + // -All says how far to page, -Top says how much to return, and they are not the + // same question: -All used to zero the cap, so asking for a bounded slice of a + // large collection walked all of it. Worse, -Top also sets the page size, so the + // walk ran at the slice's page size - 150 rows at a time across the whole tenant. + var maxItems = Top > 0 ? Top : 0; var headers = BuildHeaders(); long itemCount = 0; @@ -598,8 +602,8 @@ private void ExecuteCollectionFanOut(ConcurrentFanOut fanOut, List uniqu for (int i = 0; i < uniqueIds.Count; i++) urlToSourceId[urls[i]] = uniqueIds[i]; - // Respect -Top limit per URL - var maxItems = (All.IsPresent || Top <= 0) ? 0 : Top; + // Respect -Top limit per URL, -All or not. + var maxItems = Top > 0 ? Top : 0; // Pass headers to FetchAllAsync var fanOutResult = fanOut.FetchAllAsync(urls, maxItems, headers, CancellationToken) @@ -765,6 +769,9 @@ private void HandleBulkWriteErrors(IReadOnlyList errors) { var statusCode = (HttpStatusCode)error.StatusCode; + // A batch result carries no error code, only a status and a message, so a missing + // object cannot be told from a missing endpoint here the way it can on the fan-out + // path below. This is the write path, where a 404 is the likelier of the two. if (statusCode == HttpStatusCode.NotFound) has404 = true; @@ -802,7 +809,10 @@ private void HandleFanOutErrors(Dictionary errors) { var statusCode = GetStatusCodeFromException(ex); - if (statusCode == HttpStatusCode.NotFound) + // Only a 404 that might mean "no such endpoint" is worth a beta hint. A 404 naming + // a missing object says the path was fine, and hinting over it sends the caller to + // re-run against beta for a request that fails there too. + if (statusCode == HttpStatusCode.NotFound && !IsObjectMissing(ex)) has404 = true; if (SkipNotFound.IsPresent && statusCode == HttpStatusCode.NotFound) diff --git a/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj b/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj index b1cc118..110d517 100644 --- a/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj +++ b/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj @@ -23,11 +23,21 @@ - - + + - $(MSBuildThisFileDirectory)..\..\module - $(ModuleRoot)\Dependencies + $(MSBuildThisFileDirectory)../../module + $(ModuleRoot)/Dependencies diff --git a/src/Mgx.Cmdlets/Models/OutputTypes.cs b/src/Mgx.Cmdlets/Models/OutputTypes.cs index c8fe49f..eaaf3da 100644 --- a/src/Mgx.Cmdlets/Models/OutputTypes.cs +++ b/src/Mgx.Cmdlets/Models/OutputTypes.cs @@ -19,6 +19,19 @@ public sealed class MgxTelemetryOutput public long TotalElapsedMs { get; set; } public long ResourceUnitsConsumed { get; set; } public long BatchItemThrottles { get; set; } + public long AdaptivePacingWaitMs { get; set; } + public long AdaptivePacingActivations { get; set; } + + /// Most recent x-ms-throttle-limit-percentage seen (raw ratio, e.g. 0.85); + /// -1 when Graph never sent one this session. + public double LastThrottlePercentage { get; set; } + + /// Human-readable per-workload pacing state (adapted caps, slow start, + /// proximity, latency vs baseline); null when nothing is active. + public string? PacingState { get; set; } + + /// Bytes downloaded through Get-MgxContent (both hops). + public long ContentBytesDownloaded { get; set; } } /// @@ -30,6 +43,7 @@ public sealed class MgxOptionOutput public int RateLimitPerSecond { get; set; } public bool NoRateLimit { get; set; } public int RateLimitQueueLimit { get; set; } + public bool NoAdaptivePacing { get; set; } public int MaxRetryAttempts { get; set; } public int MaxRetryAfterSeconds { get; set; } public int TotalTimeoutSeconds { get; set; } diff --git a/src/Mgx.Engine/Http/AdaptivePacing.cs b/src/Mgx.Engine/Http/AdaptivePacing.cs new file mode 100644 index 0000000..9de4892 --- /dev/null +++ b/src/Mgx.Engine/Http/AdaptivePacing.cs @@ -0,0 +1,153 @@ +using System.Diagnostics; + +namespace Mgx.Engine.Http; + +/// +/// Coarse workload class for a Graph request URI. Pacing state is partitioned by bucket so a +/// throttle on one service (e.g. Teams) does not slow an unrelated fan-out (e.g. Entra) in the +/// same process. Also decides the delta "sync from now" token form: OneDrive/SharePoint +/// resources take token=latest, everything else $deltatoken=latest. +/// +public enum WorkloadBucket +{ + /// OneDrive / SharePoint content plane (drives, sites, shares). + Drive = 0, + + /// Entra directory objects (users, groups, servicePrincipals, ...). + Directory = 1, + + /// Everything else (Exchange, Teams, Intune, reports, ...). + Other = 2, + + /// + /// $batch envelopes. Kept separate because GraphBatchClient governs batch throughput with + /// its own item-level AIMD: a batch 429 landing in Other capped unrelated Exchange, Teams + /// and Intune traffic on evidence that had nothing to do with them, while the workload the + /// batch actually addressed went untouched. Measured budget isolation is per tenant and app + /// identity, so mixing signals across workloads discards real information. + /// + Batch = 3 +} + +/// +/// Shared AIMD (additive-increase / multiplicative-decrease) pacing math and the workload +/// classifier. Extracted from GraphBatchClient so the request-level pacer +/// (AdaptiveRequestPacer) and batch item pacing use one set of rules. Public for the +/// classifier alone (the cmdlet layer picks the -Latest token form with it); the pacing +/// math stays internal. +/// +public static class AdaptivePacing +{ + internal const int WorkloadBucketCount = 4; + + /// Floor for any adapted rate. Halving without a floor would eventually reach + /// zero, which disables pacing entirely - the opposite of what a throttled tenant needs. + internal const int MinAdaptiveRate = 2; + + /// + /// How long an adapted (reduced) rate persists after the last throttle before it no longer + /// describes the tenant's current state. Also the quiet period after which the request + /// pacer considers a workload "cold" and re-enters slow start. + /// + internal static readonly TimeSpan AdaptiveRecoveryWindow = TimeSpan.FromMinutes(5); + + /// Rate to fall back to after a throttle was observed. + internal static int ReduceRate(int rate) => Math.Max(rate / 2, MinAdaptiveRate); + + /// Rate to climb to after a clean interval, capped at the configured rate. + internal static int RecoverRate(int rate, int configuredRate) => + Math.Min(configuredRate, rate + Math.Max(1, configuredRate / 10)); + + /// + /// True when the persisted adapted rate is older than the recovery window, so the + /// throttling that produced it no longer describes the tenant's current state. + /// + internal static bool AdaptedRateHasExpired(long lastThrottleTicks, long nowTicks) => + lastThrottleTicks > 0 + && nowTicks - lastThrottleTicks > (long)(AdaptiveRecoveryWindow.TotalSeconds * Stopwatch.Frequency); + + // Segment sets for Classify. Precedence: drive markers anywhere in the path win (a user's + // OneDrive lives under /users/{id}/drive), then non-directory service markers anywhere + // (a group's calendar lives under /groups/{id}/events but is Exchange-backed), then the + // first segment decides directory membership. Other is the safe default: it only + // determines pacing-state partitioning and the -Latest token form. + private static readonly HashSet DriveMarkers = new(StringComparer.OrdinalIgnoreCase) + { + "drive", "drives", "sites", "shares" + }; + + private static readonly HashSet NonDirectoryMarkers = new(StringComparer.OrdinalIgnoreCase) + { + "messages", "mailfolders", "events", "calendar", "calendars", "calendarview", + "contactfolders", "chats", "teams", "channels", "teamwork", "onenote", "planner", + "todo", "photo", "photos", "presence", "insights", "onlinemeetings", "joinedteams" + }; + + private static readonly HashSet DirectoryRoots = new(StringComparer.OrdinalIgnoreCase) + { + "users", "groups", "serviceprincipals", "applications", "directoryobjects", + "devices", "directoryroles", "directoryroletemplates", "administrativeunits", + "oauth2permissiongrants", "organization", "contacts", "directory", "me", + "grouplifecyclepolicies", "subscribedskus", "domains" + }; + + /// + /// Classify a request URI (absolute or relative, with or without query) into a workload + /// bucket. Never throws; unparseable input lands in . + /// + public static WorkloadBucket Classify(string? requestUri) + { + if (string.IsNullOrEmpty(requestUri)) return WorkloadBucket.Other; + + // Reduce to the path: strip scheme+authority if absolute, then the query/fragment. + // The scheme check matters: on Unix, Uri.TryCreate parses a leading-slash relative + // path as an absolute file:// URI and folds the query into AbsolutePath. + var path = requestUri; + if (Uri.TryCreate(requestUri, UriKind.Absolute, out var abs) + && (abs.Scheme == Uri.UriSchemeHttps || abs.Scheme == Uri.UriSchemeHttp)) + { + path = abs.AbsolutePath; + } + else + { + var cut = path.IndexOfAny(['?', '#']); + if (cut >= 0) path = path[..cut]; + } + + var segments = path.Split('/', StringSplitOptions.RemoveEmptyEntries); + var start = 0; + + // Skip the API version segment when present ("v1.0" or "beta"). + if (segments.Length > 0 && + (segments[0].Equals("v1.0", StringComparison.OrdinalIgnoreCase) + || segments[0].Equals("beta", StringComparison.OrdinalIgnoreCase))) + { + start = 1; + } + + if (segments.Length <= start) return WorkloadBucket.Other; + + // $batch before anything else: the envelope's own URL says nothing about the workloads + // inside it, and guessing from the first inner operation would be worse than admitting + // the envelope is its own thing. + for (var i = start; i < segments.Length; i++) + { + if (segments[i].Equals("$batch", StringComparison.OrdinalIgnoreCase)) + return WorkloadBucket.Batch; + } + + for (var i = start; i < segments.Length; i++) + { + if (DriveMarkers.Contains(segments[i])) return WorkloadBucket.Drive; + } + + for (var i = start; i < segments.Length; i++) + { + if (NonDirectoryMarkers.Contains(segments[i])) return WorkloadBucket.Other; + } + + return DirectoryRoots.Contains(segments[start]) + ? WorkloadBucket.Directory + : WorkloadBucket.Other; + } +} diff --git a/src/Mgx.Engine/Http/AdaptiveRequestPacer.cs b/src/Mgx.Engine/Http/AdaptiveRequestPacer.cs new file mode 100644 index 0000000..f1bc093 --- /dev/null +++ b/src/Mgx.Engine/Http/AdaptiveRequestPacer.cs @@ -0,0 +1,449 @@ +using System.Diagnostics; +using System.Globalization; + +namespace Mgx.Engine.Http; + +/// +/// Process-wide proactive request pacer. Spaces outgoing requests *before* the token-bucket +/// lease so the process backs off ahead of Graph's throttle instead of only reacting to 429s. +/// State is partitioned by so a throttle on one service never +/// slows an unrelated workload in the same process. +/// +/// Three mechanisms, all off (zero delay) in steady clean state: +/// - AIMD cap: a 429 caps the bucket's rate (entry 4 rps, halving on repeat, floor 2 rps); +/// additive recovery once per clean second; the cap expires after a quiet +/// AdaptiveRecoveryWindow, matching GraphBatchClient semantics. +/// - Slow start: a cold bucket (first use, or quiet for AdaptiveRecoveryWindow) opens with a +/// conservative rate *cap* that doubles each clean second until it reaches the ceiling. +/// Cap-based, never delay-based: request #1 is never delayed; the cap only bites when +/// demand exceeds it, i.e. fan-outs - the "throttled before a single item" case. +/// - Proximity damping: when Graph reports x-ms-throttle-limit-percentage >= 0.8, a +/// per-request delay ramps linearly to its maximum at 1.2, while the header stays fresh. +/// +/// A 429's Retry-After additionally pushes the whole bucket's next send slot forward, so +/// concurrent callers hold off together instead of each discovering the throttle separately. +/// +/// Polly's DelayGenerator stays reactive-per-request (delays retries of a failed request); +/// this pacer is proactive-inter-request. They compose without double-delaying. Batch outer +/// POSTs skip the gate (GraphBatchClient owns batch throughput) but still feed signal state. +/// +internal static class AdaptiveRequestPacer +{ + // --- tuning constants --- + + /// Cap applied on the first 429 in a previously uncapped bucket. + internal const int ThrottledEntryRate = 4; + + /// Opening rate cap for a cold bucket (slow start). + internal const int SlowStartInitialRate = 4; + + /// Clean interval after which caps ramp (recovery / slow-start doubling). + internal static readonly TimeSpan RampInterval = TimeSpan.FromSeconds(1); + + /// How long a reported throttle percentage stays authoritative. + internal static readonly TimeSpan PercentageFreshness = TimeSpan.FromSeconds(30); + + /// Damping starts at 0.8 (per mille: 800) - the documented emission floor. + internal const int DampingStartPerMille = 800; + + /// Damping reaches its maximum at 1.2 (20% of requests already throttled). + internal const int DampingFullPerMille = 1200; + + /// Per-request delay at or above DampingFullPerMille. + internal const int DampingMaxDelayMs = 2000; + + private const int Buckets = AdaptivePacing.WorkloadBucketCount; + + // --- configuration (set via Configure, from ResiliencePipelineFactory.GetOrCreate) --- + + /// + /// Test seam. The integration suite disables the gate process-wide via a + /// [ModuleInitializer] - hundreds of pre-pacing tests would otherwise inherit slow-start + /// spacing and multiply suite time - and pacer tests re-enable it locally (serially, in + /// the Pipeline collection). Never set from production code. + /// + internal static volatile bool DisabledForTests; + + private static volatile bool s_enabled = true; + private static volatile int s_ceilingRate = 50; // full-recovery target (rps) + private static volatile int s_maxDelayMs = 120_000; // MaxRetryAfterSeconds clamp + + // --- per-bucket state --- + + private static readonly object[] s_stateLocks = CreateLocks(); + private static readonly long[] s_nextSlotTicks = new long[Buckets]; + private static readonly long[] s_lastRequestTicks = new long[Buckets]; + private static readonly long[] s_lastThrottleTicks = new long[Buckets]; + private static readonly long[] s_lastRampTicks = new long[Buckets]; + private static readonly int[] s_adaptedRate = new int[Buckets]; // 0 = inactive + private static readonly int[] s_slowStartRate = new int[Buckets]; // 0 = inactive + private static readonly int[] s_lastPercentagePerMille = InitPercentages(); + private static readonly long[] s_lastPercentageTicks = new long[Buckets]; + private static readonly long[] s_latencyBaselineMs = new long[Buckets]; // EMA, 0 = no data + private static readonly long[] s_lastLatencyMs = new long[Buckets]; + + private static object[] CreateLocks() + { + var locks = new object[Buckets]; + for (var i = 0; i < Buckets; i++) locks[i] = new object(); + return locks; + } + + private static int[] InitPercentages() + { + var p = new int[Buckets]; + Array.Fill(p, -1); + return p; + } + + /// + /// Apply option-derived configuration. Called from ResiliencePipelineFactory.GetOrCreate, + /// the chokepoint every client build passes through, so Set-MgxOption changes take effect + /// on the next cmdlet invocation like every other option. + /// + internal static void Configure(ResilientGraphClientOptions options) + { + s_enabled = !options.NoAdaptivePacing; + // The HTTP token bucket is the hard backstop; the pacer only ever caps *below* it. + // With the bucket disabled there is no configured rate to recover toward, so keep + // the default ceiling rather than inventing one. + s_ceilingRate = options.NoRateLimit ? 50 : options.RateLimitPerSecond; + s_maxDelayMs = options.MaxRetryAfterSeconds * 1000; + } + + /// Clears all learned state. Called on credential change + /// (ResiliencePipelineFactory.Reset) and from test setup. + internal static void Reset() + { + for (var i = 0; i < Buckets; i++) + { + lock (s_stateLocks[i]) + { + s_nextSlotTicks[i] = 0; + s_lastRequestTicks[i] = 0; + s_lastThrottleTicks[i] = 0; + s_lastRampTicks[i] = 0; + s_adaptedRate[i] = 0; + s_slowStartRate[i] = 0; + s_lastPercentagePerMille[i] = -1; + s_lastPercentageTicks[i] = 0; + s_latencyBaselineMs[i] = 0; + s_lastLatencyMs[i] = 0; + } + } + // Configuration is deliberately NOT reverted here. Reset clears LEARNED state - adapted + // caps, slow start, gauges, baselines. Resetting s_enabled/s_ceilingRate/s_maxDelayMs to + // defaults re-enabled pacing under a client built with NoAdaptivePacing, which then + // recorded activations until the next cmdlet invocation happened to re-Configure. The + // exposed window was in-flight fan-outs and parallel runspaces, where no re-Configure + // intervenes. Callers changing configuration call Configure; it is not Reset's business. + } + + // --- pure math (the testable core) --- + + /// Effective rate cap from the two cap sources. 0 = uncapped. + internal static int EffectiveRateCap(int adaptedRate, int slowStartRate) + { + if (adaptedRate > 0 && slowStartRate > 0) return Math.Min(adaptedRate, slowStartRate); + return adaptedRate > 0 ? adaptedRate : slowStartRate; + } + + /// + /// Per-request delay from the last reported throttle percentage. Zero below the documented + /// 0.8 emission floor or when the report has gone stale; linear ramp to the maximum at 1.2. + /// + internal static long DampingDelayMs(int perMille, long ageTicks) + { + if (perMille < DampingStartPerMille) return 0; + if (ageTicks > (long)(PercentageFreshness.TotalSeconds * Stopwatch.Frequency)) return 0; + + var span = DampingFullPerMille - DampingStartPerMille; + var over = Math.Min(perMille - DampingStartPerMille, span); + return DampingMaxDelayMs * over / span; + } + + /// Inter-request spacing in Stopwatch ticks: the stricter of the rate cap's + /// interval and the damping delay. 0 = no spacing. + internal static long ComputeIntervalTicks(int rateCap, long dampingDelayMs) + { + var capTicks = rateCap > 0 ? Stopwatch.Frequency / rateCap : 0; + var dampTicks = dampingDelayMs > 0 ? dampingDelayMs * Stopwatch.Frequency / 1000 : 0; + return Math.Max(capTicks, dampTicks); + } + + // --- the gate --- + + /// + /// Waits until this request may be sent, per the bucket's current pacing state. + /// Returns the milliseconds actually waited (0 on the fast path). The caller records + /// telemetry and verbose output so messages land in the right cmdlet's stream. + /// + internal static async ValueTask WaitAsync(WorkloadBucket bucket, CancellationToken cancellationToken) + { + // Batch envelopes are exempt by construction - GraphBatchClient passes paceGate: false + // and runs its own item-level AIMD. Guarding here as well means a batch can never claim + // a slot even if a future caller forgets the flag, and keeps two AIMD controllers from + // compounding their backoff on one workload. + if (bucket == WorkloadBucket.Batch) return 0; + + if (!s_enabled || DisabledForTests) return 0; + + var b = (int)bucket; + long intervalTicks; + + lock (s_stateLocks[b]) + { + var now = Stopwatch.GetTimestamp(); + + // Expire an adapted cap after a quiet recovery window. + if (s_adaptedRate[b] > 0 && AdaptivePacing.AdaptedRateHasExpired(s_lastThrottleTicks[b], now)) + s_adaptedRate[b] = 0; + + // Cold bucket (first use or quiet period): enter slow start. An active adapted + // cap wins over slow start, so don't stack one on top of the other. + var quietTicks = now - s_lastRequestTicks[b]; + var cold = s_lastRequestTicks[b] == 0 + || quietTicks > (long)(AdaptivePacing.AdaptiveRecoveryWindow.TotalSeconds * Stopwatch.Frequency); + if (cold && s_adaptedRate[b] == 0) + { + // Clamp to the ceiling, as the throttle path already does. Without it, a caller + // configuring -RateLimitPerSecond 1..3 (values the tuning help recommends) got a + // slow-start cap of 4 sitting ABOVE their configured rate, and telemetry + // reporting "slow-start 4 rps" against a 2 rps ceiling. + s_slowStartRate[b] = Math.Min(SlowStartInitialRate, s_ceilingRate); + s_lastRampTicks[b] = now; + } + + // Ramp caps once per clean interval: additive recovery for the adapted cap, + // doubling for slow start. A cap that reaches the ceiling deactivates. + var rampTicks = (long)(RampInterval.TotalSeconds * Stopwatch.Frequency); + if (now - s_lastRampTicks[b] >= rampTicks) + { + if (s_adaptedRate[b] > 0 && s_lastThrottleTicks[b] < s_lastRampTicks[b]) + { + var next = AdaptivePacing.RecoverRate(s_adaptedRate[b], s_ceilingRate); + s_adaptedRate[b] = next >= s_ceilingRate ? 0 : next; + } + if (s_slowStartRate[b] > 0) + { + var next = s_slowStartRate[b] * 2; + s_slowStartRate[b] = next >= s_ceilingRate ? 0 : next; + } + s_lastRampTicks[b] = now; + } + + s_lastRequestTicks[b] = now; + + var cap = EffectiveRateCap(s_adaptedRate[b], s_slowStartRate[b]); + var damping = DampingDelayMs(s_lastPercentagePerMille[b], now - s_lastPercentageTicks[b]); + intervalTicks = ComputeIntervalTicks(cap, damping); + } + + // Fast path: nothing active, but a Retry-After push may still hold the bucket. + long targetTicks; + long claimNow; + while (true) + { + var next = Interlocked.Read(ref s_nextSlotTicks[b]); + claimNow = Stopwatch.GetTimestamp(); + targetTicks = Math.Max(claimNow, next); + if (intervalTicks == 0 && next <= claimNow) + return 0; // no spacing and no pending hold - don't advance the slot + var newNext = targetTicks + intervalTicks; + if (Interlocked.CompareExchange(ref s_nextSlotTicks[b], newNext, next) == next) + break; + } + + var delayTicks = targetTicks - claimNow; + if (delayTicks <= 0) return 0; + + // Honour the claimed slot in full. Clamping the sleep here while still advancing the + // slot by the whole interval was a synchronised-burst generator: every claimant whose + // turn fell past the clamp woke at exactly the clamp and sent together, and the slot ran + // away ahead of any wait that would ever be served. That fires a burst precisely when + // the bucket is most oversubscribed - the opposite of the mechanism's purpose. + // + // Unbounded waiting is not the risk it looks like. The slot advances by at most one + // interval per request, and the interval is bounded by MinAdaptiveRate (2 rps => 500 ms) + // plus DampingMaxDelayMs (2 s). A server Retry-After is clamped to s_maxDelayMs where it + // is applied, in RecordThrottle, so a hostile or absurd Retry-After still cannot push the + // slot beyond the horizon. The wait is cancellable, which is the real escape hatch. + // + // s_maxDelayMs derives from MaxRetryAfterSeconds - a cap on how long to honour ONE + // server response. Reusing it as a queue-depth cap conflated two different limits. + var delayMs = delayTicks * 1000 / Stopwatch.Frequency; + if (delayMs <= 0) return 0; + + await Task.Delay((int)delayMs, cancellationToken); + return delayMs; + } + + // --- signal recording --- + + /// + /// Record a 429 for the bucket: cap the rate (entry rate on first throttle, halving on + /// repeat) and, when the server sent Retry-After, push the whole bucket's next send slot + /// past it so concurrent callers hold off together. Called from the shared pipeline's + /// OnRetry (before the throttled response is disposed) and on a final 429. + /// + internal static void RecordThrottle(WorkloadBucket bucket, TimeSpan? retryAfter) + { + if (!s_enabled || DisabledForTests) return; + + var b = (int)bucket; + lock (s_stateLocks[b]) + { + var now = Stopwatch.GetTimestamp(); + s_lastThrottleTicks[b] = now; + + // Batch is never gated - WaitAsync returns before touching state, because + // GraphBatchClient runs its own item-level AIMD. Setting an adapted cap for it + // therefore created a control nothing enforces AND nothing clears: the expiry check + // lives inside WaitAsync, so the cap survived forever and Get-MgxTelemetry reported + // "batch: capped 2/50 rps (last 429 3600s ago)" indefinitely. The throttle timestamp + // above is still recorded - it is real, and the batch client reads it. + // + // Returning here also skips the Retry-After slot hold below, deliberately: that hold + // writes s_nextSlotTicks, and nothing on the batch path ever reads it for the same + // reason - WaitAsync is gone before it looks. Per-item Retry-After is honored by + // GraphBatchClient itself. + if (bucket == WorkloadBucket.Batch) + return; + + // Both branches clamp to the ceiling. Only the entry branch used to, so a REPEAT + // throttle could raise the cap: ReduceRate floors at MinAdaptiveRate (2), so with + // -RateLimitPerSecond 1 - a value the tuning help recommends verbatim - the second + // 429 moved the cap from 1 to 2, halved the spacing, and printed the + // self-contradictory "capped 2/1 rps". A throttle must never widen the gate. + var reduced = s_adaptedRate[b] > 0 + ? AdaptivePacing.ReduceRate(s_adaptedRate[b]) + : ThrottledEntryRate; + s_adaptedRate[b] = Math.Min(reduced, s_ceilingRate); + s_slowStartRate[b] = 0; // the adapted cap governs from here + s_lastRampTicks[b] = now; + } + + if (retryAfter is { } ra && ra > TimeSpan.Zero) + { + var holdTicks = (long)(Math.Min(ra.TotalMilliseconds, s_maxDelayMs) / 1000.0 * Stopwatch.Frequency); + var target = Stopwatch.GetTimestamp() + holdTicks; + while (true) + { + var next = Interlocked.Read(ref s_nextSlotTicks[(int)bucket]); + if (next >= target) break; + if (Interlocked.CompareExchange(ref s_nextSlotTicks[(int)bucket], target, next) == next) + break; + } + } + } + + /// + /// Record the final response for a request: throttle-proximity percentage (fed into + /// damping and surfaced as a telemetry gauge) and a final 429 that exhausted its retries + /// (OnRetry never fires for the last attempt). + /// + internal static void RecordResponse(WorkloadBucket bucket, HttpResponseMessage response) + { + if (!s_enabled || DisabledForTests) return; + + if (response.Headers.TryGetValues("x-ms-throttle-limit-percentage", out var pctValues) + && double.TryParse(pctValues.FirstOrDefault(), NumberStyles.Any, CultureInfo.InvariantCulture, out var pct) + && pct > 0) + { + var b = (int)bucket; + Volatile.Write(ref s_lastPercentagePerMille[b], (int)(pct * 1000)); + Volatile.Write(ref s_lastPercentageTicks[b], Stopwatch.GetTimestamp()); + } + + if ((int)response.StatusCode == 429) + { + TimeSpan? retryAfter = response.Headers.RetryAfter?.Delta + ?? (response.Headers.RetryAfter?.Date is { } d ? d - DateTimeOffset.UtcNow : null); + RecordThrottle(bucket, retryAfter); + } + } + + /// + /// Record transport latency for the bucket (per attempt, network time only). Maintains a + /// slow EMA baseline so the SPO soft-clamp - latency stretching 5x+ with no 429s and no + /// headers - is visible in telemetry. Telemetry-only in 2.1; not a pacing input. + /// + internal static void RecordLatency(WorkloadBucket bucket, long httpMs) + { + if (httpMs <= 0 || !s_enabled || DisabledForTests) return; + var b = (int)bucket; + lock (s_stateLocks[b]) + { + s_lastLatencyMs[b] = httpMs; + s_latencyBaselineMs[b] = s_latencyBaselineMs[b] == 0 + ? httpMs + : s_latencyBaselineMs[b] + (httpMs - s_latencyBaselineMs[b]) / 8; + } + } + + // --- telemetry surface --- + + /// Most recently reported throttle percentage across all buckets (raw ratio, + /// e.g. 0.85), or -1 when never seen. + internal static double LastThrottlePercentage + { + get + { + long bestTicks = 0; + var best = -1; + for (var i = 0; i < Buckets; i++) + { + var t = Volatile.Read(ref s_lastPercentageTicks[i]); + if (t > bestTicks) + { + bestTicks = t; + best = Volatile.Read(ref s_lastPercentagePerMille[i]); + } + } + return best < 0 ? -1 : best / 1000.0; + } + } + + /// + /// Human-readable summary of active pacing state per bucket, or null when nothing is + /// active and no latency data exists. Rendered by Get-MgxTelemetry. + /// + internal static string? DescribeState() + { + var parts = new List(); + var now = Stopwatch.GetTimestamp(); + for (var i = 0; i < Buckets; i++) + { + var name = ((WorkloadBucket)i).ToString().ToLowerInvariant(); + List? facts = null; + lock (s_stateLocks[i]) + { + if (s_adaptedRate[i] > 0) + { + var ago = (now - s_lastThrottleTicks[i]) / Stopwatch.Frequency; + (facts ??= []).Add($"capped {s_adaptedRate[i]}/{s_ceilingRate} rps (last 429 {ago}s ago)"); + } + if (s_slowStartRate[i] > 0) + (facts ??= []).Add($"slow-start {s_slowStartRate[i]} rps"); + if (s_lastPercentagePerMille[i] >= DampingStartPerMille + && now - s_lastPercentageTicks[i] <= (long)(PercentageFreshness.TotalSeconds * Stopwatch.Frequency)) + (facts ??= []).Add($"proximity {s_lastPercentagePerMille[i] / 10}%"); + if (s_latencyBaselineMs[i] > 0) + { + var ratio = (double)s_lastLatencyMs[i] / s_latencyBaselineMs[i]; + (facts ??= []).Add($"latency {s_lastLatencyMs[i]}ms ({ratio:0.0}x of {s_latencyBaselineMs[i]}ms baseline)"); + } + } + if (facts != null) + parts.Add($"{name}: {string.Join(", ", facts)}"); + } + return parts.Count == 0 ? null : string.Join("; ", parts); + } + + // --- test accessors --- + + internal static int GetAdaptedRate(WorkloadBucket bucket) => Volatile.Read(ref s_adaptedRate[(int)bucket]); + internal static int GetSlowStartRate(WorkloadBucket bucket) => Volatile.Read(ref s_slowStartRate[(int)bucket]); + internal static long GetNextSlotTicks(WorkloadBucket bucket) => Interlocked.Read(ref s_nextSlotTicks[(int)bucket]); + internal static bool Enabled => s_enabled; +} diff --git a/src/Mgx.Engine/Http/DownloadUrlValidator.cs b/src/Mgx.Engine/Http/DownloadUrlValidator.cs new file mode 100644 index 0000000..5659bab --- /dev/null +++ b/src/Mgx.Engine/Http/DownloadUrlValidator.cs @@ -0,0 +1,62 @@ +namespace Mgx.Engine.Http; + +/// +/// Validates pre-authenticated content download URLs (hop-2 targets: the 302 Location from a +/// Graph /content request, or @microsoft.graph.downloadUrl off a driveItem) before the +/// token-free HTTP client fetches them. +/// +/// Threat model, precisely: the two-hop split guarantees the bearer token never reaches the +/// download host, and this allowlist keeps the token-free client pointed at Microsoft-operated +/// infrastructure (no SSRF into arbitrary hosts). It does NOT guarantee content origin - any +/// tenant on earth controls a *.sharepoint.com subdomain, so an allowlisted host is a host +/// that cannot steal the token, not a host whose bytes are trustworthy. +/// +/// Deliberately separate from NextLinkValidator, which stays same-authority-only: pagination +/// must never leave the Graph host, downloads legitimately do. +/// +public static class DownloadUrlValidator +{ + // Dot-anchored suffixes: the leading dot means "some subdomain of", so + // evilsharepoint.com and sharepoint.com.evil.example never match. Static in 2.1 - + // no user knob; nothing in scope downloads from outside this set. + private static readonly string[] AllowedHostSuffixes = + [ + ".sharepoint.com", + ".sharepoint.us", + ".sharepoint.cn", + ".sharepointonline.com", + ".files.1drv.com", + ".svc.ms" + ]; + + /// + /// Returns the URL unchanged when it passes every check, or null when the download must + /// be refused: non-HTTPS (token-free or not, bytes travel plaintext), a non-default port + /// (suffix matching alone would admit tenant.sharepoint.com:8443 shapes), embedded + /// userinfo (classic parser-confusion trick), or a host outside the allowlist. + /// + public static string? Validate(string? downloadUrl) + { + if (string.IsNullOrEmpty(downloadUrl)) return null; + + if (!Uri.TryCreate(downloadUrl, UriKind.Absolute, out var uri)) return null; + + if (!string.Equals(uri.Scheme, "https", StringComparison.OrdinalIgnoreCase)) + return null; + + if (!uri.IsDefaultPort) return null; + + if (!string.IsNullOrEmpty(uri.UserInfo)) return null; + + // IdnHost is the punycode-normalized form, so lookalike Unicode hosts are compared + // in the same alphabet the resolver will actually use. + var host = uri.IdnHost; + foreach (var suffix in AllowedHostSuffixes) + { + if (host.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)) + return downloadUrl; + } + + return null; + } +} diff --git a/src/Mgx.Engine/Http/GraphBatchClient.cs b/src/Mgx.Engine/Http/GraphBatchClient.cs index 4ac91a9..c06e0ff 100644 --- a/src/Mgx.Engine/Http/GraphBatchClient.cs +++ b/src/Mgx.Engine/Http/GraphBatchClient.cs @@ -70,27 +70,9 @@ public sealed class GraphBatchClient // lifetime of the session, with no way to recover short of restarting PowerShell. private static long s_lastThrottleTicks; - // Adaptive pacing bounds. Classic AIMD: a 429 halves the rate, and a run of clean chunks - // adds a fraction of the configured rate back, so the rate creeps up to the throttling - // threshold instead of jumping straight back into it. - private const int MinAdaptiveItemsPerSecond = 2; + // Chunk-loop AIMD tuning. The shared math (halve on throttle, additive recovery, expiry + // window) lives in AdaptivePacing so the request-level pacer applies identical rules. private const int AdaptiveRecoveryCleanChunks = 2; - internal static readonly TimeSpan AdaptiveRecoveryWindow = TimeSpan.FromMinutes(5); - - /// Rate to fall back to after a chunk reported throttling. - internal static int ReduceRate(int rate) => Math.Max(rate / 2, MinAdaptiveItemsPerSecond); - - /// Rate to climb to after a run of clean chunks, capped at the configured rate. - internal static int RecoverRate(int rate, int configuredRate) => - Math.Min(configuredRate, rate + Math.Max(1, configuredRate / 10)); - - /// - /// True when the persisted adapted rate is older than the recovery window, so the - /// throttling that produced it no longer describes the tenant's current state. - /// - internal static bool AdaptedRateHasExpired(long lastThrottleTicks, long nowTicks) => - lastThrottleTicks > 0 - && nowTicks - lastThrottleTicks > (long)(AdaptiveRecoveryWindow.TotalSeconds * Stopwatch.Frequency); public GraphBatchClient(ResilientGraphClient client, string graphBaseUrl = "https://graph.microsoft.com/v1.0", int maxRetryAfterSeconds = 120, int batchChunkConcurrency = 1, int batchItemsPerSecond = 0) @@ -105,7 +87,10 @@ public GraphBatchClient(ResilientGraphClient client, string graphBaseUrl = "http private static bool HasWriteOperations(IReadOnlyList operations) => operations.Any(op => !string.Equals(op.Method, "GET", StringComparison.OrdinalIgnoreCase)); - /// Test-only: reset cross-call pacing state between tests. + /// + /// Clear cross-call batch pacing state. Called by ResiliencePipelineFactory.Reset when the + /// credential changes - this state is learned per tenant and must not outlive it. + /// internal static void ResetPacingState() { Interlocked.Exchange(ref s_lastBatchCompletedTicks, 0); @@ -140,6 +125,14 @@ public async Task> ExecuteBatchAsync( /// After all chunks complete, items that exhausted per-chunk retries but have /// retryable status codes are retried once more as a follow-up batch. /// + /// + /// Status recorded for an operation that was never sent, because a chunk before it failed. + /// Distinct from any HTTP status so a caller can tell "not attempted" from "attempted and + /// refused" - the difference between a write that may have landed and one that certainly + /// did not. + /// + public const int NotSentStatus = 0; + public async Task ExecuteBatchIndexedAsync( IReadOnlyList operations, CancellationToken cancellationToken = default) @@ -151,6 +144,12 @@ public async Task ExecuteBatchIndexedAsync( var telemetry = new BatchTelemetry { TotalRequests = operations.Count }; var batchSw = Stopwatch.StartNew(); + // A chunk's own POST can fail after the chunks before it were applied. Their results are + // the only record of what landed, so they are returned with the failure rather than lost + // to it. + Exception? chunkFailure = null; + var notSent = new List(); + // Cross-call pacing: if a previous write batch completed recently, delay to maintain // target throughput. Only applies to batches containing writes (POST/PATCH/DELETE) - // GET-only batches don't hit Graph's write throttle. Delay is capped to one chunk's @@ -190,12 +189,12 @@ public async Task ExecuteBatchIndexedAsync( // so that successive Invoke-MgxBatchRequest invocations in a loop don't reset // to the full rate and immediately get re-throttled. var adapted = Volatile.Read(ref s_adaptedItemsPerSecond); - if (adapted > 0 && AdaptedRateHasExpired( + if (adapted > 0 && AdaptivePacing.AdaptedRateHasExpired( Interlocked.Read(ref s_lastThrottleTicks), Stopwatch.GetTimestamp())) { Volatile.Write(ref s_adaptedItemsPerSecond, 0); adapted = 0; - _pendingVerbose.Enqueue($"Adaptive pacing: no throttling for {AdaptiveRecoveryWindow.TotalMinutes:0} min, restored configured rate ({_batchItemsPerSecond} items/sec)"); + _pendingVerbose.Enqueue($"Adaptive pacing: no throttling for {AdaptivePacing.AdaptiveRecoveryWindow.TotalMinutes:0} min, restored configured rate ({_batchItemsPerSecond} items/sec)"); } int effectiveItemsPerSecond = (adapted > 0 && adapted < _batchItemsPerSecond) ? adapted @@ -232,8 +231,39 @@ public async Task ExecuteBatchIndexedAsync( } var chunkSw = Stopwatch.StartNew(); - var (chunkResults, throttleDelay, chunkRetries, chunkThrottles, chunkRetryDelayMs) = - await SendBatchWithRetryAsync(chunk, cancellationToken); + IReadOnlyList<(BatchOperation Operation, GraphBatchResponseItem Response)> chunkResults; + int throttleDelay, chunkRetries, chunkThrottles; + long chunkRetryDelayMs; + try + { + (chunkResults, throttleDelay, chunkRetries, chunkThrottles, chunkRetryDelayMs) = + await SendBatchWithRetryAsync(chunk, cancellationToken); + } + // Only an HTTP-level failure of the chunk itself. A malformed envelope - wrong + // response count, unknown ids - is a protocol violation rather than a chunk that + // did not land, and it still throws: nothing about the run can be trusted after it. + catch (GraphServiceException ex) + { + // The chunks before this one were applied on the server. Letting this + // exception out discards their results, which is the only record of what + // landed - and for writes that is the thing the caller most needs. Stop + // here, keep them, and hand the failure back alongside. + // + // The unsent slots are filled rather than dropped: callers line results up + // against their own input list by position, so a short list would misattribute + // every error after the gap. + chunkFailure = ex; + for (int i = globalOffset; i < operations.Count; i++) + { + notSent.Add(operations[i]); + results[i] = (operations[i], new GraphBatchResponseItem + { + Id = i.ToString(), + Status = NotSentStatus + }); + } + break; + } prevChunkElapsedMs = chunkSw.ElapsedMilliseconds; crossChunkDelaySeconds = throttleDelay; telemetry.AddItemRetries(chunkRetries); @@ -247,10 +277,10 @@ public async Task ExecuteBatchIndexedAsync( { Interlocked.Exchange(ref s_lastThrottleTicks, Stopwatch.GetTimestamp()); cleanChunks = 0; - if (effectiveItemsPerSecond > MinAdaptiveItemsPerSecond) + if (effectiveItemsPerSecond > AdaptivePacing.MinAdaptiveRate) { var prev = effectiveItemsPerSecond; - effectiveItemsPerSecond = ReduceRate(effectiveItemsPerSecond); + effectiveItemsPerSecond = AdaptivePacing.ReduceRate(effectiveItemsPerSecond); Volatile.Write(ref s_adaptedItemsPerSecond, effectiveItemsPerSecond); _pendingVerbose.Enqueue($"Adaptive pacing: {chunkThrottles} throttle(s) in chunk {chunkIndex}, reducing rate {prev} -> {effectiveItemsPerSecond} items/sec (persisted)"); } @@ -260,7 +290,7 @@ public async Task ExecuteBatchIndexedAsync( { cleanChunks = 0; var prev = effectiveItemsPerSecond; - effectiveItemsPerSecond = RecoverRate(effectiveItemsPerSecond, _batchItemsPerSecond); + effectiveItemsPerSecond = AdaptivePacing.RecoverRate(effectiveItemsPerSecond, _batchItemsPerSecond); // Persist 0 once fully recovered, so the next call starts clean instead of // pinning itself to a rate that merely equals the configured one today. Volatile.Write(ref s_adaptedItemsPerSecond, @@ -390,7 +420,7 @@ public async Task ExecuteBatchIndexedAsync( // Validate all slots populated var nullIndex = Array.FindIndex(results, r => r == null); - if (nullIndex >= 0) + if (chunkFailure is null && nullIndex >= 0) { throw new InvalidOperationException( $"Batch result slot {nullIndex} was not populated after processing all chunks. " @@ -414,7 +444,9 @@ public async Task ExecuteBatchIndexedAsync( return new BatchExecutionResult { Results = finalResults, - Telemetry = telemetry + Telemetry = telemetry, + ChunkFailure = chunkFailure, + NotSent = notSent }; } @@ -476,7 +508,10 @@ public async Task ExecuteBatchIndexedAsync( var json = JsonSerializer.Serialize(batchRequest, JsonOptions); var content = new StringContent(json, Encoding.UTF8, "application/json"); var httpSw = Stopwatch.StartNew(); - using var response = await _client.PostAsync(_batchUrl, content, cancellationToken); + // paceGate: false - this class owns batch throughput (cross-call pacing, item-level + // AIMD). Running the request pacer's gate on top would stack two independent AIMD + // controllers on one workload. Outer responses still feed pacer signal state. + using var response = await _client.PostAsync(_batchUrl, content, cancellationToken, paceGate: false); var httpMs = httpSw.ElapsedMilliseconds; if (httpMs > 5000) _pendingVerbose.Enqueue($"Batch HTTP slow: {httpMs}ms for {pendingIndices.Count} items (attempt {attempt + 1})"); diff --git a/src/Mgx.Engine/Http/GraphContentClient.cs b/src/Mgx.Engine/Http/GraphContentClient.cs new file mode 100644 index 0000000..433010e --- /dev/null +++ b/src/Mgx.Engine/Http/GraphContentClient.cs @@ -0,0 +1,441 @@ +using System.Net; +using System.Net.Http.Headers; +using Polly; +using Polly.Retry; + +namespace Mgx.Engine.Http; + +/// +/// Result of a content fetch. Owns the response: dispose it to release the connection. +/// +public sealed class GraphContentResult : IDisposable +{ + public required Stream Content { get; init; } + public HttpStatusCode StatusCode { get; init; } + public long? ContentLength { get; init; } + public string? ContentRange { get; init; } + public string? ContentType { get; init; } + public string? ETag { get; init; } + + /// True when the bytes came from the pre-authenticated download host (hop 2) + /// rather than directly from Graph. + public bool FromDownloadHost { get; init; } + + internal HttpResponseMessage? OwnedResponse { get; init; } + + public void Dispose() + { + Content.Dispose(); + OwnedResponse?.Dispose(); + } +} + +/// +/// The two-hop content path behind Get-MgxContent. +/// +/// Hop 1 (Graph, authenticated) goes through ResilientGraphClient.SendAsync - full pipeline, +/// pacer, bucket lease. A 2xx with a body is content served directly by Graph (attachments, +/// photos). A redirect is the drive-item case: the Location is a pre-authenticated URL on a +/// download host, validated against DownloadUrlValidator before hop 2 touches it. +/// +/// Hop 2 (download host, token-free) uses a static singleton HttpClient with NO auth handler: +/// the bearer must never reach the download host. AllowAutoRedirect is off and redirects are +/// followed manually (max 3), re-validating every Location - an open redirect on an +/// allowlisted host must not silently bypass the validator. Its own small retry pipeline +/// handles 429/5xx with Retry-After; deliberately NO circuit breaker (a CDN outage must not +/// poison the Graph circuit) and NO bucket charge (the request budget is Graph-side). +/// On 401/403 (expired pre-auth URL) the caller re-runs hop 1 once for a fresh URL. +/// +/// Transport precondition (fail closed): the 302 is only observable because the owned clean +/// HttpClient sets AllowAutoRedirect=false. On a transport that auto-follows (the SDK +/// fallback client ships Kiota's RedirectHandler; injected transports are unknown), the +/// bytes would arrive as a 2xx from a host mgx never validated - so any 2xx whose request +/// URI left the Graph host is rejected here, and the cmdlet layer refuses non-owned +/// transports outright before calling in. +/// +public static class GraphContentClient +{ + private const int MaxManualRedirects = 3; + + /// Statuses hop 1 treats as "go fetch from the download host". + private static bool IsRedirect(HttpStatusCode status) => + status is HttpStatusCode.Moved or HttpStatusCode.Found + or HttpStatusCode.SeeOther or HttpStatusCode.TemporaryRedirect + or (HttpStatusCode)308; + + /// Test seam: replaces the token-free hop-2 client so download-host behavior + /// (redirects, 429/5xx, auth-expiry) can be mocked. Never set from production code. + internal static HttpClient? DownloadClientForTests; + + // Token-free singleton for hop 2. No auth handler by construction; decompression stays + // off so ranged reads and Content-Length are byte-exact. + private static readonly HttpClient s_downloadClient = new(new SocketsHttpHandler + { + AllowAutoRedirect = false, + PooledConnectionLifetime = TransportDefaults.PooledConnectionLifetime, + MaxConnectionsPerServer = TransportDefaults.MaxConnectionsPerServer, + ConnectTimeout = TransportDefaults.ConnectTimeout + }) + { + // Covers until response headers (ResponseHeadersRead); the body copy is bounded by + // the caller's idle timeout, not this. + Timeout = TimeSpan.FromSeconds(100) + }; + + // Small reactive pipeline for hop 2: retry 3 with exponential backoff + jitter on + // 429/5xx/transport errors, honoring Retry-After clamped to two minutes. + /// + /// Delay for one download retry, from Retry-After. The header has two legal forms: + /// delta-seconds and an HTTP-date. Honoring only Delta silently fell back to plain + /// exponential backoff whenever a download host chose the date form - and the main pipeline + /// already handles both, so this path was the odd one out. A date already in the past yields + /// no delay rather than a negative one, and both forms are capped at 120s. + /// + /// Internal rather than inlined into the DelayGenerator lambda because the pipeline is a + /// private static field with no seam: a test could only reach the lambda by driving a real + /// 429 through GetContentAsync. The test that covers this used to hold its own copy of the + /// logic instead, which passed with the whole feature deleted from the product. + /// Polly's MaxDelay does NOT clamp a value returned from a DelayGenerator, so the cap here + /// is the only bound on the sleep. + /// + internal static TimeSpan? ResolveRetryDelay(RetryConditionHeaderValue? retryAfter) + { + var cap = TimeSpan.FromSeconds(120); + if (retryAfter?.Delta is { } delta) + return delta > cap ? cap : delta; + if (retryAfter?.Date is { } date) + { + var delay = date - DateTimeOffset.UtcNow; + if (delay > TimeSpan.Zero) + return delay > cap ? cap : delay; + } + return null; + } + + private static readonly ResiliencePipeline s_downloadPipeline = + new ResiliencePipelineBuilder() + .AddRetry(new RetryStrategyOptions + { + MaxRetryAttempts = 3, + BackoffType = DelayBackoffType.Exponential, + UseJitter = true, + Delay = TimeSpan.FromSeconds(1), + MaxDelay = TimeSpan.FromSeconds(120), + ShouldHandle = args => + { + if (args.Outcome.Result?.StatusCode is (HttpStatusCode)429 + or HttpStatusCode.InternalServerError + or HttpStatusCode.BadGateway + or HttpStatusCode.ServiceUnavailable + or HttpStatusCode.GatewayTimeout) + return ValueTask.FromResult(true); + return ValueTask.FromResult(args.Outcome.Exception is HttpRequestException); + }, + DelayGenerator = args => + ValueTask.FromResult(ResolveRetryDelay(args.Outcome.Result?.Headers.RetryAfter)), + OnRetry = args => + { + args.Outcome.Result?.Dispose(); + return default; + } + }) + .Build(); + + /// + /// Fetch content for a Graph $value / /content URI, optionally a byte range. + /// must be absolute (the cmdlet layer builds it). + /// + public static async Task GetContentAsync( + ResilientGraphClient graphClient, + string requestUri, + RangeHeaderValue? range, + Dictionary? headers, + CancellationToken cancellationToken) + { + var graphHost = new Uri(requestUri).Host; + var requestHeaders = headers != null + ? new Dictionary(headers) + : new Dictionary(); + if (range != null) + requestHeaders["Range"] = range.ToString(); + + // Two passes at most: a hop-2 401/403 means the pre-authenticated URL expired + // (they are short-lived), so re-run hop 1 once for a fresh one. + for (var authAttempt = 0; ; authAttempt++) + { + // redirectIsSuccess: this is the one caller that expects a 302 and follows it, so + // a redirect here is the documented success path, not a failed request. + var hop1 = await graphClient.SendAsync( + HttpMethod.Get, requestUri, headers: requestHeaders, + cancellationToken: cancellationToken, traceResponseBody: false, + redirectIsSuccess: true); + + if (hop1.IsSuccessStatusCode) + { + // Fail closed: a 2xx that did not come from the Graph host means the + // transport auto-followed the redirect - the host was never validated and + // bearer handling depended on transport internals. Refuse the bytes. + var finalHost = hop1.RequestMessage?.RequestUri?.Host; + if (finalHost != null && !string.Equals(finalHost, graphHost, StringComparison.OrdinalIgnoreCase)) + { + hop1.Dispose(); + throw new InvalidOperationException( + $"The transport followed a redirect to '{finalHost}' before mgx could validate it. " + + "Content downloads require the mgx-owned HTTP client (AllowAutoRedirect off)."); + } + return await WrapAsync(hop1, fromDownloadHost: false, graphClient.BodyReadTimeout, cancellationToken); + } + + if (IsRedirect(hop1.StatusCode)) + { + var location = hop1.Headers.Location; + hop1.Dispose(); + if (location == null) + throw new HttpRequestException("Graph returned a redirect without a Location header."); + + var absolute = location.IsAbsoluteUri + ? location.ToString() + : new Uri(new Uri(requestUri), location).ToString(); + var validated = DownloadUrlValidator.Validate(absolute) + ?? throw new InvalidOperationException( + $"Download host '{TryGetHost(absolute)}' is not on the allowed list " + + "(SharePoint/OneDrive download hosts only). Refusing to fetch content from it."); + + var hop2 = await FetchFromDownloadHostAsync(validated, range, cancellationToken); + + if (hop2.StatusCode is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden + && authAttempt == 0) + { + // Expired pre-auth URL: one fresh hop-1, one more try. + hop2.Dispose(); + continue; + } + + if (hop2.IsSuccessStatusCode) + return await WrapAsync(hop2, fromDownloadHost: true, graphClient.BodyReadTimeout, cancellationToken); + + var hop2Body = await ReadErrorBodyAsync(hop2, graphClient.BodyReadTimeout, cancellationToken); + var hop2Status = hop2.StatusCode; + hop2.Dispose(); + throw new Models.GraphServiceException(hop2Status, hop2Body); + } + + // Plain Graph error (404, 401, 416, ...): surface it like every other cmdlet path. + var body = await ReadErrorBodyAsync(hop1, graphClient.BodyReadTimeout, cancellationToken); + var status = hop1.StatusCode; + hop1.Dispose(); + throw new Models.GraphServiceException(status, body); + } + } + + /// + /// Fetch directly from a pre-authenticated download URL (a piped driveItem's + /// @microsoft.graph.downloadUrl), skipping hop 1. The caller MUST have validated the URL + /// through DownloadUrlValidator first; this method validates again and throws otherwise. + /// No auth refresh is possible on this path - the URL is short-lived, and a 401/403 means + /// the item must be re-fetched for a fresh one. + /// + public static async Task GetFromDownloadUrlAsync( + string downloadUrl, + RangeHeaderValue? range, + TimeSpan bodyReadTimeout, + CancellationToken cancellationToken) + { + var validated = DownloadUrlValidator.Validate(downloadUrl) + ?? throw new InvalidOperationException( + $"Download host '{TryGetHost(downloadUrl)}' is not on the allowed list " + + "(SharePoint/OneDrive download hosts only). Refusing to fetch content from it."); + + var response = await FetchFromDownloadHostAsync(validated, range, cancellationToken); + if (response.IsSuccessStatusCode) + return await WrapAsync(response, fromDownloadHost: true, bodyReadTimeout, cancellationToken); + + var body = await ReadErrorBodyAsync(response, bodyReadTimeout, cancellationToken); + var status = response.StatusCode; + response.Dispose(); + throw new Models.GraphServiceException(status, body); + } + + /// + /// Token-free fetch with manual, re-validated redirects. Every hop must pass the + /// allowlist: an open redirect on an allowlisted host is not a pass. + /// + private static async Task FetchFromDownloadHostAsync( + string url, RangeHeaderValue? range, CancellationToken cancellationToken) + { + var current = url; + for (var redirects = 0; ; redirects++) + { + var context = ResilienceContextPool.Shared.Get(cancellationToken); + HttpResponseMessage response; + try + { + var target = current; + response = await s_downloadPipeline.ExecuteAsync( + async ctx => + { + var request = new HttpRequestMessage(HttpMethod.Get, target); + if (range != null) + request.Headers.Range = range; + return await (DownloadClientForTests ?? s_downloadClient).SendAsync( + request, HttpCompletionOption.ResponseHeadersRead, ctx.CancellationToken); + }, + context); + } + finally + { + ResilienceContextPool.Shared.Return(context); + } + + if (!IsRedirect(response.StatusCode)) + return response; + + var location = response.Headers.Location; + response.Dispose(); + if (location == null) + throw new HttpRequestException("Download host returned a redirect without a Location header."); + if (redirects >= MaxManualRedirects) + throw new HttpRequestException( + $"Download exceeded {MaxManualRedirects} redirects. Refusing to continue."); + + var absolute = location.IsAbsoluteUri + ? location.ToString() + : new Uri(new Uri(current), location).ToString(); + current = DownloadUrlValidator.Validate(absolute) + ?? throw new InvalidOperationException( + $"Download host '{TryGetHost(absolute)}' is not on the allowed list " + + "(redirected mid-download). Refusing to follow."); + } + } + + private static async Task WrapAsync( + HttpResponseMessage response, bool fromDownloadHost, TimeSpan bodyReadTimeout, + CancellationToken cancellationToken) + { + // ResponseHeadersRead: opening the stream is cheap; the caller copies with an idle + // timeout (CopyWithIdleTimeoutAsync) so a stalled body cannot hang forever. + using var bodyCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + bodyCts.CancelAfter(bodyReadTimeout); + Stream stream; + try + { + stream = await response.Content.ReadAsStreamAsync(bodyCts.Token); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + response.Dispose(); + throw new HttpRequestException("Content stream open timed out."); + } + + return new GraphContentResult + { + Content = stream, + StatusCode = response.StatusCode, + ContentLength = response.Content.Headers.ContentLength, + ContentRange = response.Content.Headers.ContentRange?.ToString(), + ContentType = response.Content.Headers.ContentType?.ToString(), + ETag = response.Headers.ETag?.ToString(), + FromDownloadHost = fromDownloadHost, + OwnedResponse = response + }; + } + + private static async Task ReadErrorBodyAsync( + HttpResponseMessage response, TimeSpan bodyReadTimeout, CancellationToken cancellationToken) + { + try + { + using var bodyCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + bodyCts.CancelAfter(bodyReadTimeout); + return await response.Content.ReadAsStringAsync(bodyCts.Token); + } + catch (Exception) + { + return string.Empty; + } + } + + private static string TryGetHost(string url) => + Uri.TryCreate(url, UriKind.Absolute, out var uri) ? uri.Host : url; + + /// + /// Copy up to (null = unbounded) with a PER-READ idle + /// timeout: the clock resets on progress, so a slow-but-moving download of any size + /// survives while a stalled stream aborts within . + /// Returns the byte count copied and records it in telemetry. + /// The maxBytes cut is the truncation path: when a server ignores Range and answers 200 + /// with the full body, the caller passes the requested length and disposes the result, + /// aborting the rest of the transfer. + /// + public static async Task CopyWithIdleTimeoutAsync( + Stream source, Stream destination, long? maxBytes, TimeSpan idleTimeout, + CancellationToken cancellationToken, long skipBytes = 0) + { + var buffer = new byte[81920]; + + // Discard the bytes before the requested offset. Only reachable when a ranged request + // was answered with 200 and the whole body: the server ignored the offset, so it has to + // be honoured here instead. Without this, -Offset 1MB -Length 64KB returned bytes + // 0..65535 and reported success - the wrong bytes, silently, which is worse than an + // error because nothing downstream can tell. + long skipped = 0; + while (skipped < skipBytes) + { + var toSkip = (int)Math.Min(buffer.Length, skipBytes - skipped); + var r = await ReadWithIdleTimeoutAsync(source, buffer, toSkip, idleTimeout, cancellationToken); + if (r == 0) + { + // The body ended before the offset was reached, so the requested slice does not + // exist. Returning 0 was indistinguishable from a legitimately empty resource, + // and the -OutFile path does not inspect the count: it moved the empty temp over + // the destination, destroying an existing file and reporting success. Throwing + // is the only signal the caller can act on - its catch deletes the temp and + // rethrows, so the destination is left untouched, which is what the surrounding + // code already promises. + throw new InvalidOperationException( + $"The content ended after {skipped:N0} bytes, before the requested offset of " + + $"{skipBytes:N0}. The requested range does not exist in this resource."); + } + skipped += r; + } + + long total = 0; + while (maxBytes == null || total < maxBytes.Value) + { + var toRead = maxBytes == null + ? buffer.Length + : (int)Math.Min(buffer.Length, maxBytes.Value - total); + + var read = await ReadWithIdleTimeoutAsync(source, buffer, toRead, idleTimeout, cancellationToken); + if (read == 0) break; + await destination.WriteAsync(buffer.AsMemory(0, read), cancellationToken); + total += read; + } + + if (total > 0) + MgxTelemetryCollector.Current.RecordContentBytes(total); + return total; + } + + /// + /// One read, bounded by the idle timeout. A stalled body must not hang forever, and the + /// timeout applies per read rather than to the transfer as a whole so a slow-but-progressing + /// download is not killed. + /// + private static async Task ReadWithIdleTimeoutAsync( + Stream source, byte[] buffer, int count, TimeSpan idleTimeout, + CancellationToken cancellationToken) + { + using var readCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + readCts.CancelAfter(idleTimeout); + try + { + return await source.ReadAsync(buffer.AsMemory(0, count), readCts.Token); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new HttpRequestException( + $"Content download stalled: no data for {idleTimeout.TotalSeconds:F0}s."); + } + } +} diff --git a/src/Mgx.Engine/Http/GraphRequestTracer.cs b/src/Mgx.Engine/Http/GraphRequestTracer.cs index a954f3d..386095d 100644 --- a/src/Mgx.Engine/Http/GraphRequestTracer.cs +++ b/src/Mgx.Engine/Http/GraphRequestTracer.cs @@ -28,6 +28,26 @@ internal static partial class GraphRequestTracer RegexOptions.IgnoreCase)] private static partial Regex SensitiveJsonValue(); + /// + /// A pre-authenticated URL is a credential whose property name says nothing about it: + /// "@microsoft.graph.downloadUrl" contains none of password/secret/credential/token/key, so + /// SensitiveJsonValue leaves it whole and the tempauth JWT reaches -Debug output. Match on + /// the VALUE instead - a URL carrying one of the capability parameters Graph, SharePoint and + /// the download hosts actually use. Deliberately not "every URL": @odata.nextLink is a URL + /// too, and losing it from a trace would remove the thing paging bugs are diagnosed with. + /// + [GeneratedRegex("\"(https?://[^\"]*?(?:tempauth|guestaccesstoken|authkey|X-Amz-Signature|(? + /// Property names that carry a pre-authenticated URL even when the value does not expose a + /// recognisable parameter - the short-form download hosts put the capability in the path. + /// + [GeneratedRegex("\"([^\"]*downloadurl[^\"]*)\"\\s*:\\s*\"[^\"]*\"", + RegexOptions.IgnoreCase)] + private static partial Regex DownloadUrlProperty(); + /// /// Trace line for an outgoing request. is 1-based so retries are visible. /// @@ -76,7 +96,8 @@ internal static string FormatResponse(HttpResponseMessage response, long elapsed { sb.AppendLine().Append(" Headers:"); foreach (var header in headers) - sb.AppendLine().Append(" ").Append(header.Key).Append(": ").Append(Join(header.Value)); + sb.AppendLine().Append(" ").Append(header.Key).Append(": ") + .Append(RedactHeaderValue(header.Key, Join(header.Value))); } if (!string.IsNullOrEmpty(body)) @@ -95,10 +116,30 @@ private static bool IsTraced(string name) => private static string Join(IEnumerable values) => string.Join(", ", values); - /// Redact credential-looking JSON properties, then truncate. + /// + /// A content 302's Location is a pre-authenticated URL: it grants the file bytes without a + /// bearer token, so tracing it verbatim writes a live credential into -Debug output. The + /// capability is not always in the query - some download hosts carry it in the path - so + /// keep only scheme and host, which is the whole diagnostic value of the header anyway. + /// The JSON-body redaction cannot reach header values, hence this hook. + /// + private static string RedactHeaderValue(string name, string value) + { + if (!name.Equals("Location", StringComparison.OrdinalIgnoreCase)) return value; + return Uri.TryCreate(value, UriKind.Absolute, out var uri) + ? $"{uri.Scheme}://{uri.IdnHost}/" + : ""; + } + + /// Redact credential-looking JSON properties and pre-authenticated URLs, then truncate. private static string Sanitize(string body) { var redacted = SensitiveJsonValue().Replace(body, "\"$1\": \"\""); + // Property-name match first: covers a downloadUrl whose capability sits in the path. + redacted = DownloadUrlProperty().Replace(redacted, "\"$1\": \"\""); + // Then any remaining URL value carrying a capability parameter, keeping the parameter + // name visible so a trace still shows WHICH kind of URL was redacted. + redacted = CapabilityUrlValue().Replace(redacted, "\"$1\""); return redacted.Length <= MaxBodyChars ? redacted : redacted[..MaxBodyChars] + $"... [truncated, {redacted.Length - MaxBodyChars} more chars]"; diff --git a/src/Mgx.Engine/Http/MgxTelemetryCollector.cs b/src/Mgx.Engine/Http/MgxTelemetryCollector.cs index 86887b0..268c164 100644 --- a/src/Mgx.Engine/Http/MgxTelemetryCollector.cs +++ b/src/Mgx.Engine/Http/MgxTelemetryCollector.cs @@ -21,6 +21,9 @@ public sealed class MgxTelemetryCollector private long _elapsedMs; // Total wall-clock time in SendAsync (all phases) private long _resourceUnits; // x-ms-resource-unit sum across all responses private long _batchItemThrottles; // Per-item 429s inside $batch responses (distinct from Polly-level _throttleRetries) + private long _pacingWaitMs; // Time spent in the adaptive pacer's proactive gate + private long _pacingActivations; // Requests the pacer actually delayed (wait > 0) + private long _contentBytes; // Bytes downloaded via Get-MgxContent (both hops) public void RecordRequest(bool succeeded, long elapsedMs) { @@ -65,6 +68,16 @@ public void RecordBatchItemThrottles(int count) => public void RecordBatchRetryDelay(long ms) => Interlocked.Add(ref _retryDelayMs, ms); + /// A request the adaptive pacer held before sending. + public void RecordPacingWait(long ms) + { + Interlocked.Increment(ref _pacingActivations); + Interlocked.Add(ref _pacingWaitMs, ms); + } + + public void RecordContentBytes(long bytes) => + Interlocked.Add(ref _contentBytes, bytes); + public void Reset() { Interlocked.Exchange(ref _totalRequests, 0); @@ -79,6 +92,11 @@ public void Reset() Interlocked.Exchange(ref _elapsedMs, 0); Interlocked.Exchange(ref _resourceUnits, 0); Interlocked.Exchange(ref _batchItemThrottles, 0); + Interlocked.Exchange(ref _pacingWaitMs, 0); + Interlocked.Exchange(ref _pacingActivations, 0); + Interlocked.Exchange(ref _contentBytes, 0); + // Pacer control state (adapted rates, slow start) is deliberately NOT cleared here: + // it describes the tenant's current throttle regime, not accumulated statistics. } public MgxTelemetrySummary GetSummary() => new( @@ -93,7 +111,12 @@ public void Reset() HttpMs: Interlocked.Read(ref _httpMs), ElapsedMs: Interlocked.Read(ref _elapsedMs), ResourceUnitsConsumed: Interlocked.Read(ref _resourceUnits), - BatchItemThrottles: Interlocked.Read(ref _batchItemThrottles)); + BatchItemThrottles: Interlocked.Read(ref _batchItemThrottles), + AdaptivePacingWaitMs: Interlocked.Read(ref _pacingWaitMs), + AdaptivePacingActivations: Interlocked.Read(ref _pacingActivations), + LastThrottlePercentage: AdaptiveRequestPacer.LastThrottlePercentage, + PacingState: AdaptiveRequestPacer.DescribeState(), + ContentBytesDownloaded: Interlocked.Read(ref _contentBytes)); } /// @@ -111,4 +134,9 @@ public sealed record MgxTelemetrySummary( long HttpMs, long ElapsedMs, long ResourceUnitsConsumed, - long BatchItemThrottles); + long BatchItemThrottles, + long AdaptivePacingWaitMs, + long AdaptivePacingActivations, + double LastThrottlePercentage, + string? PacingState, + long ContentBytesDownloaded); diff --git a/src/Mgx.Engine/Http/ResiliencePipelineFactory.cs b/src/Mgx.Engine/Http/ResiliencePipelineFactory.cs index cd59828..8b56f67 100644 --- a/src/Mgx.Engine/Http/ResiliencePipelineFactory.cs +++ b/src/Mgx.Engine/Http/ResiliencePipelineFactory.cs @@ -82,6 +82,10 @@ public static (ResiliencePipeline Pipeline, TokenBucketRate s_rateLimiter = rateLimiter; s_cachedOptions = options; + // The chokepoint every client build passes through: apply pacing configuration + // here so Set-MgxOption changes reach the pacer on the next invocation. + AdaptiveRequestPacer.Configure(options); + return (s_pipeline, rateLimiter); } } @@ -100,6 +104,13 @@ public static void Reset() // Not disposed, for the reason documented in GetOrCreate. s_rateLimiter = null; s_cachedOptions = null; + // Learned pacing state describes the old tenant; clear it with the CB history. + AdaptiveRequestPacer.Reset(); + // GraphBatchClient keeps its own AIMD state in separate statics, so clearing the + // request pacer alone left the batch pacer describing the previous tenant: a fresh + // credential started at the throttled item rate learned elsewhere, and its first + // batch was additionally delayed by the old tenant's completion timestamp. + GraphBatchClient.ResetPacingState(); } } @@ -196,6 +207,18 @@ or HttpStatusCode.GatewayTimeout var attempt = args.AttemptNumber; var retryDelay = args.RetryDelay; + // Feed the pacer BEFORE the response is disposed below - the request URI + // and Retry-After are unreadable afterwards. Retried 429s never reach the + // SendAsync success path, so this is the only place they are observed. + if (status == (HttpStatusCode)429) + { + TimeSpan? serverDelay = retryAfter?.Delta + ?? (retryAfter?.Date is { } date ? date - DateTimeOffset.UtcNow : null); + AdaptiveRequestPacer.RecordThrottle( + AdaptivePacing.Classify(response?.RequestMessage?.RequestUri?.ToString()), + serverDelay); + } + // Dispose the previous response to drain the connection back to the pool. // With HttpCompletionOption.ResponseHeadersRead, the body stream stays open // until explicitly read or the response is disposed. Without this, retried diff --git a/src/Mgx.Engine/Http/ResilientDelegatingHandler.cs b/src/Mgx.Engine/Http/ResilientDelegatingHandler.cs index b51eeef..47bc7f7 100644 --- a/src/Mgx.Engine/Http/ResilientDelegatingHandler.cs +++ b/src/Mgx.Engine/Http/ResilientDelegatingHandler.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using System.Diagnostics; using System.Net.Http.Headers; using System.Threading.RateLimiting; using Polly; @@ -50,16 +51,36 @@ protected override async Task SendAsync( context.Properties.Set(ResiliencePipelineFactory.IsIdempotentKey, request.Method != HttpMethod.Post); context.Properties.Set(ResiliencePipelineFactory.VerboseWriterKey, (Action)(msg => _pendingVerbose.Enqueue(msg))); + var bucket = AdaptivePacing.Classify(request.RequestUri?.ToString()); + // Telemetry parity with ResilientGraphClient.SendAsync. Without it an + // Enable-MgxResilience session reported TotalRequests=0 alongside a non-zero retry + // count - self-contradictory, and a divide-by-zero for the throttle-rate calculation + // the Get-MgxTelemetry help tells users to compute. + var totalSw = System.Diagnostics.Stopwatch.StartNew(); + var succeeded = false; + var recorded = false; try { + // Same proactive gate as ResilientGraphClient.SendAsync. This is the + // Enable-MgxResilience path: SDK cmdlet traffic bypasses ResilientGraphClient + // entirely, so pacing hooked only there would skip SDK-wrapped workloads. + var pacedMs = await AdaptiveRequestPacer.WaitAsync(bucket, cancellationToken); + if (pacedMs > 0) + { + MgxTelemetryCollector.Current.RecordPacingWait(pacedMs); + _pendingVerbose.Enqueue($"Adaptive pacing: waited {pacedMs}ms before sending ({bucket} workload)"); + } + if (_rateLimiter != null) { + var limiterSw = Stopwatch.StartNew(); lease = await _rateLimiter.AcquireAsync(1, cancellationToken); + MgxTelemetryCollector.Current.RecordRateLimiterWait(limiterSw.ElapsedMilliseconds); if (!lease.IsAcquired) throw new InvalidOperationException("Rate limit exceeded. Too many concurrent requests. Reduce -Concurrency on fan-out cmdlets, increase the queue with Set-MgxOption -RateLimitQueueLimit, or disable with Set-MgxOption -NoRateLimit."); } - return await _pipeline.ExecuteAsync( + var result = await _pipeline.ExecuteAsync( async ctx => { // Clone on every attempt, including the first. On the SDK bridge path @@ -90,12 +111,48 @@ protected override async Task SendAsync( clone.Content = freshContent; } - return await base.SendAsync(clone, ctx.CancellationToken); + // Per-attempt network time, measured INSIDE the pipeline so retry delays are + // excluded - the same placement the owned client uses. Without this, + // Get-MgxTelemetry reported "HTTP Time (ms): 0" for every Enable-MgxResilience + // session, which is indistinguishable from no network time at all. + // NOTE: deliberately not AdaptiveRequestPacer.RecordLatency - see the comment + // after ExecuteAsync for why the pacer baseline stays off this path. + // Caveat: on this path the SDK's own retry handler sleeps INSIDE + // base.SendAsync, so its Retry-After waits land in HttpMs while + // RetryDelayMs stays 0 (that counts Polly's waits, and Polly is outside). + // Separating them would mean reaching into the SDK's handler chain. + // Bounded by AttemptTimeoutSeconds, and the alternative was reporting 0. + var httpSw = Stopwatch.StartNew(); + var attempt = await base.SendAsync(clone, ctx.CancellationToken); + MgxTelemetryCollector.Current.RecordHttpTime(httpSw.ElapsedMilliseconds); + return attempt; }, context); + + // Feed the pacer's signal state (proximity percentage, final 429). Latency is + // not recorded on this path: timing here would include retry delays inside + // ExecuteAsync, which would pollute the network-time baseline. + AdaptiveRequestPacer.RecordResponse(bucket, result); + + // A 3xx is an error on this path: the SDK bridge does not follow redirects itself + // and every caller surfaces them, so only 2xx counts as success here. + succeeded = result.IsSuccessStatusCode; + if (result.Headers.TryGetValues("x-ms-resource-unit", out var ruValues) + && long.TryParse(System.Linq.Enumerable.FirstOrDefault(ruValues), out var ru) + && ru > 0) + { + MgxTelemetryCollector.Current.RecordResourceUnit(ru); + } + return result; } finally { + if (!recorded) + { + recorded = true; + totalSw.Stop(); + MgxTelemetryCollector.Current.RecordRequest(succeeded, totalSw.ElapsedMilliseconds); + } // Drain buffered verbose messages on the calling thread if (VerboseWriter != null) { diff --git a/src/Mgx.Engine/Http/ResilientGraphClient.cs b/src/Mgx.Engine/Http/ResilientGraphClient.cs index 113a31e..d735e35 100644 --- a/src/Mgx.Engine/Http/ResilientGraphClient.cs +++ b/src/Mgx.Engine/Http/ResilientGraphClient.cs @@ -97,6 +97,10 @@ public void DrainVerboseMessages() VerboseWriter(msg); } + // For engine components that already report through this client's buffered channel + // (PageIterator runs on the enumeration thread, where a cmdlet cannot WriteWarning). + internal void EnqueueWarning(string message) => _pendingWarnings.Enqueue(message); + // Same threading contract as DrainVerboseMessages. public void DrainWarningMessages() { @@ -140,7 +144,10 @@ public async Task SendAsync( HttpContent? content = null, Dictionary? headers = null, CancellationToken cancellationToken = default, - int permitCount = 1) + int permitCount = 1, + bool paceGate = true, + bool traceResponseBody = true, + bool redirectIsSuccess = false) { // Buffer content bytes before pipeline so retries reconstruct fresh HttpContent. // Snapshot ALL content headers (not just ContentType) to preserve @@ -169,8 +176,22 @@ public async Task SendAsync( var totalSw = Stopwatch.StartNew(); bool succeeded = false; + var bucket = AdaptivePacing.Classify(requestUri); try { + // Proactive gate ahead of the bucket lease: the pacer spaces requests, the token + // bucket stays the hard backstop. Batch outer POSTs pass paceGate: false - + // GraphBatchClient owns batch throughput - but their responses still feed signals. + if (paceGate) + { + var pacedMs = await AdaptiveRequestPacer.WaitAsync(bucket, cancellationToken); + if (pacedMs > 0) + { + MgxTelemetryCollector.Current.RecordPacingWait(pacedMs); + _pendingVerbose.Enqueue($"Adaptive pacing: waited {pacedMs}ms before sending ({bucket} workload)"); + } + } + if (_rateLimiter != null) { var limiterSw = Stopwatch.StartNew(); @@ -209,19 +230,44 @@ public async Task SendAsync( // Stream response headers immediately instead of buffering entire body var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ctx.CancellationToken); MgxTelemetryCollector.Current.RecordHttpTime(httpSw.ElapsedMilliseconds); + // Per-attempt network time feeds the latency baseline (telemetry-only: + // makes the SPO soft-clamp visible; not a pacing input in 2.1). + AdaptiveRequestPacer.RecordLatency(bucket, httpSw.ElapsedMilliseconds); if (DebugEnabled) - await TraceResponseAsync(response, httpSw.ElapsedMilliseconds, ctx.CancellationToken); + { + if (traceResponseBody) + { + await TraceResponseAsync(response, httpSw.ElapsedMilliseconds, ctx.CancellationToken); + } + else + { + // Content path: buffering a multi-megabyte download to trace it + // would defeat the streaming read. Headers-only line instead. + _pendingDebug.Enqueue(GraphRequestTracer.FormatResponse(response, httpSw.ElapsedMilliseconds, null)); + } + } return response; }, context); - succeeded = result.IsSuccessStatusCode; + // A redirect counts as success only for the caller that expects one. Graph answers + // /content with a 302 to a pre-authenticated download host and GraphContentClient + // follows it, so booking that as a failure made every successful two-hop download + // register as failed. Every OTHER caller treats a 3xx as an error and throws - + // AllowAutoRedirect is off - so a blanket 3xx-is-success would book a request the + // user saw fail as succeeded. Hence the per-call flag rather than a status range. + succeeded = result.IsSuccessStatusCode + || (redirectIsSuccess && (int)result.StatusCode >= 300 && (int)result.StatusCode < 400); // Log throttle proximity and diagnostic headers to verbose. // These headers warn that requests are approaching throttle limits before 429s hit. LogThrottleHeaders(result); + // Feed the pacer's signal state: throttle-proximity percentage, and a final 429 + // that exhausted retries (OnRetry never fires for the last attempt). + AdaptiveRequestPacer.RecordResponse(bucket, result); + // Track x-ms-resource-unit for telemetry (Identity/Access uses RU-based throttling). // Safe to record here: _pipeline.ExecuteAsync returns only the final response; // retried responses are disposed in OnRetry and never reach this point. @@ -259,8 +305,22 @@ public Task PostAsync( HttpContent content, CancellationToken cancellationToken = default, Dictionary? headers = null, - int permitCount = 1) - => SendAsync(HttpMethod.Post, requestUri, content, headers, cancellationToken, permitCount); + int permitCount = 1, + bool paceGate = true) + => SendAsync(HttpMethod.Post, requestUri, content, headers, cancellationToken, permitCount, paceGate); + + /// + /// Fetch content bytes ($value / /content endpoints), optionally a byte range. + /// Two hops: the authenticated Graph request through the full pipeline, then - when Graph + /// 302s to a pre-authenticated download host - a token-free fetch through + /// GraphContentClient. See GraphContentClient for the transport preconditions. + /// + public Task GetContentAsync( + string requestUri, + System.Net.Http.Headers.RangeHeaderValue? range = null, + Dictionary? headers = null, + CancellationToken cancellationToken = default) + => GraphContentClient.GetContentAsync(this, requestUri, range, headers, cancellationToken); /// /// Fetch a collection page and deserialize. diff --git a/src/Mgx.Engine/Http/ResilientGraphClientOptions.cs b/src/Mgx.Engine/Http/ResilientGraphClientOptions.cs index 1957bd7..e3d42b0 100644 --- a/src/Mgx.Engine/Http/ResilientGraphClientOptions.cs +++ b/src/Mgx.Engine/Http/ResilientGraphClientOptions.cs @@ -65,6 +65,13 @@ public int RateLimitPerSecond /// Set to true to disable the rate limiter entirely. Default: false. public bool NoRateLimit { get; init; } + /// + /// Set to true to disable adaptive request pacing (AIMD back-off, slow start, and + /// throttle-proximity damping). Independent of NoRateLimit: the token bucket is the hard + /// backstop, the pacer is the proactive layer in front of it. Default: false (pacing ON). + /// + public bool NoAdaptivePacing { get; init; } + /// Maximum queue depth before rejecting requests. Range: 0-100,000. Default: 500. public int RateLimitQueueLimit { diff --git a/src/Mgx.Engine/Models/GraphResponse.cs b/src/Mgx.Engine/Models/GraphResponse.cs index 507c7d1..45b6662 100644 --- a/src/Mgx.Engine/Models/GraphResponse.cs +++ b/src/Mgx.Engine/Models/GraphResponse.cs @@ -105,6 +105,16 @@ public sealed class BatchExecutionResult { public required IReadOnlyList<(BatchOperation Operation, GraphBatchResponseItem Response)> Results { get; init; } public required BatchTelemetry Telemetry { get; init; } + + /// + /// Set when a chunk's own POST failed rather than its items. The chunks before it were + /// applied on the server, so their results are still in and the + /// caller needs both: what landed, and why the rest did not. Throwing loses the first half. + /// + public Exception? ChunkFailure { get; init; } + + /// Operations that were never sent, because a chunk before them failed. + public IReadOnlyList NotSent { get; init; } = []; } /// diff --git a/src/Mgx.Engine/Models/GraphServiceException.cs b/src/Mgx.Engine/Models/GraphServiceException.cs index c5105fb..2b388d4 100644 --- a/src/Mgx.Engine/Models/GraphServiceException.cs +++ b/src/Mgx.Engine/Models/GraphServiceException.cs @@ -32,10 +32,23 @@ private static string FormatAndExtract(HttpStatusCode statusCode, string respons try { using var doc = JsonDocument.Parse(responseBody); + + // Shape-check before every access. TryGetProperty throws InvalidOperationException on + // a non-object, and GetString() throws on a non-string - neither is a JsonException, + // so valid JSON of an unexpected shape used to escape this method as an exception + // thrown from inside an exception's own constructor. Graph always emits the OData + // envelope, but the content path's second hop talks to SharePoint, OneDrive and CDN + // hosts that are not Graph and answer with whatever they like. + if (doc.RootElement.ValueKind != JsonValueKind.Object) + return $"HTTP {(int)statusCode}: {statusCode}"; + if (doc.RootElement.TryGetProperty("error", out var errorObj)) { - var code = errorObj.TryGetProperty("code", out var c) ? c.GetString() : null; - var message = errorObj.TryGetProperty("message", out var m) ? m.GetString() : null; + if (errorObj.ValueKind != JsonValueKind.Object) + return $"HTTP {(int)statusCode}: {statusCode}"; + + var code = AsString(errorObj, "code"); + var message = AsString(errorObj, "message"); errorCode = code; // Build formatted message from whatever Graph provided @@ -51,10 +64,35 @@ private static string FormatAndExtract(HttpStatusCode statusCode, string respons return formatted; } } - catch (JsonException) { } + catch (Exception ex) when (ex is JsonException or InvalidOperationException) + { + // Backstop: the shape guards above cover the reachable cases, but an error body must + // never be able to throw out of here - callers are constructing an exception. + } return $"HTTP {(int)statusCode}: {statusCode}"; } + /// + /// A string property, or null when absent or not actually a string. Graph nests a non-string + /// "message" on some endpoints (an object with a "value"), which GetString() rejects outright. + /// + private static string? AsString(JsonElement obj, string name) + { + if (!obj.TryGetProperty(name, out var v)) return null; + if (v.ValueKind == JsonValueKind.String) return v.GetString(); + + // Some endpoints nest it as { "value": "..." } - the very shape this helper's own + // comment names. Returning null there produced "Code: " with an empty message, which + // is strictly worse than the text that was sitting one level down. + if (v.ValueKind == JsonValueKind.Object + && v.TryGetProperty("value", out var inner) + && inner.ValueKind == JsonValueKind.String) + { + return inner.GetString(); + } + return null; + } + /// /// Maps common Graph error codes to user-facing guidance strings. /// Returns null for unrecognized codes. diff --git a/src/Mgx.Engine/Pagination/ConcurrentFanOut.cs b/src/Mgx.Engine/Pagination/ConcurrentFanOut.cs index 050da11..25c8a7d 100644 --- a/src/Mgx.Engine/Pagination/ConcurrentFanOut.cs +++ b/src/Mgx.Engine/Pagination/ConcurrentFanOut.cs @@ -107,7 +107,7 @@ public async Task FetchAllAsync( // Validate nextLink host matches initial URL (prevents SSRF via // crafted Graph responses redirecting authenticated requests) - nextLink = NextLinkValidator.Validate(page.NextLink, expectedHost); + nextLink = NextLinkValidator.ValidateOrThrow(page.NextLink, expectedHost); } results[url] = allItems.ToArray(); diff --git a/src/Mgx.Engine/Pagination/DeltaState.cs b/src/Mgx.Engine/Pagination/DeltaState.cs index 8477067..69d1a41 100644 --- a/src/Mgx.Engine/Pagination/DeltaState.cs +++ b/src/Mgx.Engine/Pagination/DeltaState.cs @@ -23,6 +23,11 @@ public sealed class DeltaState [JsonPropertyName("filter")] public string? Filter { get; set; } + // Normalized Prefer tokens the enumeration was taken with (2.1 additive: state files + // written before this property deserialize to null - no migration needed). + [JsonPropertyName("prefer")] + public string? Prefer { get; set; } + [JsonPropertyName("resource")] public string Resource { get; set; } = string.Empty; @@ -35,6 +40,15 @@ public sealed class DeltaState [JsonPropertyName("graphEndpoint")] public string GraphEndpoint { get; set; } = string.Empty; + /// + /// Graph API version the deltaLink was issued by, read from the link rather than from the + /// request, so it names the version the token actually came from. Empty on state files + /// written before 2.1.0 - including every 2.0.1 one - which is treated as "unknown" rather + /// than a mismatch so upgrades keep working. + /// + [JsonPropertyName("apiVersion")] + public string ApiVersion { get; set; } = string.Empty; + private static readonly JsonSerializerOptions JsonOptions = new() { WriteIndented = true @@ -61,8 +75,11 @@ public static (DeltaState? State, DeltaLoadResult Result) LoadWithResult(string { return (null, DeltaLoadResult.Corrupt); } - catch (IOException) + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { + // Windows reports a denying ACL as UnauthorizedAccessException, which is not an + // IOException - catching only the latter made unreadable state throw there and read + // as absent everywhere else. return (null, DeltaLoadResult.NotFound); } } @@ -85,8 +102,18 @@ public void Save(string path) lock (lockObj) { var tmpPath = normalizedPath + ".tmp"; - File.WriteAllText(tmpPath, json); - File.Move(tmpPath, normalizedPath, overwrite: true); + try + { + File.WriteAllText(tmpPath, json); + File.Move(tmpPath, normalizedPath, overwrite: true); + } + catch + { + // The staging file is not the state; leaving it behind only invites a later run + // to wonder what it is. + try { if (File.Exists(tmpPath)) File.Delete(tmpPath); } catch { } + throw; + } } } @@ -110,7 +137,7 @@ public static bool Delete(string path) if (File.Exists(tmpPath)) File.Delete(tmpPath); return true; } - catch (IOException) + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { return false; } @@ -123,6 +150,13 @@ public static bool Delete(string path) /// public static void ValidateWriteAccess(string path) { + // NOTE: a directory passes the probe below, because the probe writes ".probe" + // NEXT to the target rather than to it, so the fail-fast check succeeds and the real + // write fails later. Rejecting a directory here would be better, but this method also + // validates -OutputFile, where the late failure is what + // DeltaQueryTests.Cmdlet_Checkpoint_MidPage_BoundsProgressLostWhenARunDiesInsideAPage + // uses to reach a mid-page abort - and that test guards more than this costs. Left as + // it is deliberately rather than by omission. var dir = Path.GetDirectoryName(Path.GetFullPath(path)); if (dir != null && !Directory.Exists(dir)) Directory.CreateDirectory(dir); diff --git a/src/Mgx.Engine/Pagination/NextLinkValidator.cs b/src/Mgx.Engine/Pagination/NextLinkValidator.cs index 16cb9dd..e611b98 100644 --- a/src/Mgx.Engine/Pagination/NextLinkValidator.cs +++ b/src/Mgx.Engine/Pagination/NextLinkValidator.cs @@ -36,4 +36,32 @@ public static class NextLinkValidator return nextLink; } + + /// + /// Validates a nextLink the service actually sent, throwing if it is refused. + /// Use this mid-pagination. A null from is ambiguous there: it means + /// both "there was no link" and "the link was refused", and treating the second like the + /// first ends the loop quietly and hands back a partial collection that looks complete. + /// Returns null only when there genuinely was no link, i.e. the end of the collection. + /// + public static string? ValidateOrThrow(string? nextLink, Uri? expectedHost, string? expectedPathPrefix = null) + { + if (nextLink == null) return null; + + var validated = Validate(nextLink, expectedHost, expectedPathPrefix); + if (validated == null) + throw new InvalidOperationException( + $"Pagination stopped: the service returned an @odata.nextLink that failed validation " + + $"({Describe(nextLink)}). Expected an https link on '{expectedHost?.Authority}'. " + + "Following it could send the access token to another host, and ignoring it would " + + "return part of the collection as though it were all of it."); + + return validated; + } + + /// Scheme, authority and path only - the query can carry anything on a crafted link. + private static string Describe(string link) => + Uri.TryCreate(link, UriKind.Absolute, out var u) + ? $"{u.Scheme}://{u.Authority}{u.AbsolutePath}" + : "unparseable"; } diff --git a/src/Mgx.Engine/Pagination/PageIterator.cs b/src/Mgx.Engine/Pagination/PageIterator.cs index 54cbb94..bb5bcab 100644 --- a/src/Mgx.Engine/Pagination/PageIterator.cs +++ b/src/Mgx.Engine/Pagination/PageIterator.cs @@ -84,12 +84,21 @@ public async IAsyncEnumerable StreamAllWithCountAsync( } // Capture deltaLink from the final page of a delta query response. - // Validated against expectedHost before surfacing. + // Validated against expectedHost before surfacing. A refused link is dropped, not + // thrown: every item has already been delivered by this point, the token simply + // does not advance, and the next sync repeats the enumeration - unlike a refused + // nextLink, where continuing silently would hand back a partial collection. Say + // so, or the caller reports "no delta token received" and misdiagnoses it. if (page.DeltaLink != null) { var validatedDelta = NextLinkValidator.Validate(page.DeltaLink, expectedHost); if (validatedDelta != null) onDeltaLink?.Invoke(validatedDelta); + else if (onDeltaLink != null) + _client.EnqueueWarning( + "The service returned an @odata.deltaLink that failed validation, so the delta " + + "token was not saved. Following it could send the access token to another " + + "host. The next sync will repeat this enumeration from the previous token."); } if (page.Value.Length == 0) @@ -119,7 +128,7 @@ public async IAsyncEnumerable StreamAllWithCountAsync( yield break; } - nextLink = NextLinkValidator.Validate(page.NextLink, expectedHost); + nextLink = NextLinkValidator.ValidateOrThrow(page.NextLink, expectedHost); isFirstPage = false; onPageComplete?.Invoke(new PageCompletedInfo(nextLink)); diff --git a/src/Mgx.Engine/Pagination/PaginationCheckpoint.cs b/src/Mgx.Engine/Pagination/PaginationCheckpoint.cs index 02aa8e5..ac00968 100644 --- a/src/Mgx.Engine/Pagination/PaginationCheckpoint.cs +++ b/src/Mgx.Engine/Pagination/PaginationCheckpoint.cs @@ -28,6 +28,23 @@ public sealed class PaginationCheckpoint [JsonPropertyName("pageItemsAlreadyWritten")] public int PageItemsAlreadyWritten { get; set; } + /// + /// File name (not path) of the temp the writing run was appending to, relative to the + /// output's directory - or null when that run wrote straight to the output. This is what + /// makes a checkpoint answerable: without it, "no temp on disk" cannot be told apart from + /// "a temp from some other run is on disk", and neither from "the run was appending". + /// + [JsonPropertyName("tempFile")] + public string? TempFile { get; set; } + + /// + /// Byte length of that file at the moment this checkpoint was saved, captured after the + /// writer was flushed. Null on checkpoints written before this was recorded, which are + /// treated as unverifiable rather than as zero-length. + /// + [JsonPropertyName("dataLength")] + public long? DataLength { get; set; } + [JsonPropertyName("timestamp")] public DateTimeOffset Timestamp { get; set; } = DateTimeOffset.UtcNow; @@ -56,9 +73,12 @@ public sealed class PaginationCheckpoint // Treat as no checkpoint; caller will start fresh. return null; } - catch (IOException) + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { - // File locked or inaccessible; treat as no checkpoint. + // Locked, or the account cannot read it; treat as no checkpoint either way. Windows + // reports a denying ACL as UnauthorizedAccessException, which is not an IOException, + // so catching only the latter made an unreadable checkpoint throw on Windows and + // resume cleanly everywhere else. return null; } } @@ -77,8 +97,18 @@ public void Save(string path) lock (lockObj) { var tmpPath = normalizedPath + ".tmp"; - File.WriteAllText(tmpPath, json); - File.Move(tmpPath, normalizedPath, overwrite: true); + try + { + File.WriteAllText(tmpPath, json); + File.Move(tmpPath, normalizedPath, overwrite: true); + } + catch + { + // Same reasoning as the delta state: a staging file that never became the + // checkpoint is litter, and adoption already has to reason about stray files. + try { if (File.Exists(tmpPath)) File.Delete(tmpPath); } catch { } + throw; + } } } @@ -99,7 +129,7 @@ public static bool Delete(string path) } return true; } - catch (IOException) + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { return false; } diff --git a/tests/Live/Mgx.Live.Tests.ps1 b/tests/Live/Mgx.Live.Tests.ps1 new file mode 100644 index 0000000..248e7ce --- /dev/null +++ b/tests/Live/Mgx.Live.Tests.ps1 @@ -0,0 +1,167 @@ +<# + These exist because the rest of the suite mocks HTTP. A mocked test proves the code does + what it was written to do; it cannot prove the request that leaves the process is one Graph + accepts. Enable-MgxResilience shipped a wrapper client with no BaseAddress and a green + suite: every test passed an absolute URI, and every relative-URI call would have thrown. + + So: relative URIs wherever a caller would use one, real responses, no mocks in this file. + + Read-only. Nothing here writes to a tenant. + + Run: Invoke-Pester -Path ./tests/Live + Needs AZURE_TENANT_ID / AZURE_CLIENT_ID / AZURE_CLIENT_CERTIFICATE_PATH for a directory + tenant, and optionally MGX_LIVE_CONTENT_* for the content tests (see tests/Live/README.md). +#> + +BeforeAll { + $repo = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent + if (-not $env:AZURE_TENANT_ID -or -not $env:AZURE_CLIENT_ID -or -not $env:AZURE_CLIENT_CERTIFICATE_PATH) { + throw "Live tests need AZURE_TENANT_ID, AZURE_CLIENT_ID and AZURE_CLIENT_CERTIFICATE_PATH. " + + "They are not skipped on purpose: a live suite that skips itself is a live suite nobody runs." + } + . "$repo/tests/benchmarks/common.ps1" + Import-MgxLocal + Connect-MgxBenchmark | Out-Null + + # Sized from the tenant rather than assumed. These tests have to hold on a 19-user tenant + # and a 100,000-user one: a live suite that only passes against one directory is one that + # gets switched off the first time it meets another. + $tok = Get-BenchAppToken + $script:UserCount = [int](Invoke-RestMethod -Uri 'https://graph.microsoft.com/v1.0/users/$count' ` + -Headers @{ Authorization = "Bearer $tok"; ConsistencyLevel = 'eventual' }) + # Enough to need more than one page where the tenant allows it, never more than it holds. + $script:Slice = [Math]::Min(150, $script:UserCount) + $script:SmallSlice = [Math]::Min(5, $script:UserCount) +} + +Describe 'Invoke-MgxRequest' { + It 'returns objects for a relative URI' { + (Invoke-MgxRequest '/users?$top=2' -WarningAction SilentlyContinue).Count | Should -Be 2 + } + It 'honors -Property' { + (Invoke-MgxRequest '/users?$top=1' -Property id -WarningAction SilentlyContinue).id | Should -Not -BeNullOrEmpty + } + It 'honors -Filter' { + Invoke-MgxRequest /users -Filter "startsWith(displayName,'b')" -Top 1 | Should -Not -BeNullOrEmpty + } + It 'caps the total at -Top' { + (Invoke-MgxRequest /users -Top $script:Slice -Property id | Measure-Object).Count | + Should -Be $script:Slice + } + It 'caps the total at -Top even with -All' { + (Invoke-MgxRequest /users -All -Top $script:Slice -Property id | Measure-Object).Count | + Should -Be $script:Slice + } +} + +Describe 'Invoke-MgxBatchRequest' { + It 'executes a batch of relative URIs' { + $ids = Invoke-MgxRequest /users -Top $script:SmallSlice -Property id | ForEach-Object { $_.id } + $res = $ids | ForEach-Object { "/users/$_" } | Invoke-MgxBatchRequest -Method GET + ($res | Measure-Object).Count | Should -Be $script:SmallSlice + } +} + +Describe 'Expand-MgxRelation' { + It 'fans out over a relation' { + $users = Invoke-MgxRequest /users -Top $script:SmallSlice -Property id,displayName + $r = $users | Expand-MgxRelation -Uri '/users/{id}/manager' -As manager -SkipNotFound -SkipForbidden + $r | Should -Not -BeNull + } +} + +Describe 'Export-MgxCollection' { + It 'writes one JSONL line per object' { + $f = Join-Path ([IO.Path]::GetTempPath()) "mgxlive-$([Guid]::NewGuid().ToString('N')).jsonl" + try { + Export-MgxCollection /users -OutputFile $f -Top $script:Slice -Property id | Out-Null + (Get-Content $f).Count | Should -Be $script:Slice + } finally { Remove-Item $f -ErrorAction SilentlyContinue } + } +} + +Describe 'Sync-MgxDelta' { + It 'baselines with -Latest and writes state' { + $d = Join-Path ([IO.Path]::GetTempPath()) "mgxlive-$([Guid]::NewGuid().ToString('N')).json" + try { + Sync-MgxDelta /users/delta -DeltaPath $d -Latest | Out-Null + Test-Path $d | Should -BeTrue + } finally { Remove-Item $d -ErrorAction SilentlyContinue } + } +} + +Describe 'Get-MgxOption' { + It 'returns the current settings' { Get-MgxOption | Should -Not -BeNull } +} + +Describe 'Set-MgxOption' { + It 'changes a setting and reports it back' { + $before = (Get-MgxOption).RateLimitPerSecond + try { + Set-MgxOption -RateLimitPerSecond 42 + (Get-MgxOption).RateLimitPerSecond | Should -Be 42 + } finally { if ($before) { Set-MgxOption -RateLimitPerSecond $before } } + } +} + +Describe 'Get-MgxTelemetry' { + It 'counts requests the session made' { + Invoke-MgxRequest '/users?$top=1' -WarningAction SilentlyContinue | Out-Null + (Get-MgxTelemetry).Requests | Should -BeGreaterThan 0 + } + It 'zeroes on -Reset' { + Get-MgxTelemetry -Reset | Out-Null + (Get-MgxTelemetry).Requests | Should -Be 0 + } +} + +Describe 'Enable-MgxResilience' { + # The regression this file was written for: the wrapper dropped BaseAddress, so a RELATIVE + # -Uri threw from PrepareUri before any handler ran. Absolute URIs were unaffected, which is + # why a full green suite said nothing. + AfterEach { Disable-MgxResilience -ErrorAction SilentlyContinue | Out-Null } + + It 'leaves a relative-URI SDK call working' { + Enable-MgxResilience | Out-Null + Invoke-MgGraphRequest -Method GET -Uri '/v1.0/users?$top=1' | Should -Not -BeNull + } + It 'leaves SDK cmdlets working' { + Enable-MgxResilience | Out-Null + (Get-MgUser -Top ([Math]::Min(2, $script:UserCount)) -Property id | Measure-Object).Count | + Should -Be ([Math]::Min(2, $script:UserCount)) + } +} + +Describe 'Disable-MgxResilience' { + It 'restores plain SDK calls' { + Enable-MgxResilience | Out-Null + Disable-MgxResilience | Out-Null + Invoke-MgGraphRequest -Method GET -Uri '/v1.0/users?$top=1' | Should -Not -BeNull + } +} + +Describe 'Get-MgxResilience' { + It 'reports state either side of a wrap' { + Get-MgxResilience | Should -Not -BeNull + Enable-MgxResilience | Out-Null + try { Get-MgxResilience | Should -Not -BeNull } finally { Disable-MgxResilience | Out-Null } + } +} + +Describe 'Get-MgxContent' { + # Content needs Files.Read.All / Sites.Read.All, which the directory tenant does not carry. + # Point MGX_LIVE_CONTENT_URI at a drive item on a tenant that does. + BeforeAll { $script:contentUri = $env:MGX_LIVE_CONTENT_URI } + + It 'downloads a byte range' -Skip:(-not $env:MGX_LIVE_CONTENT_URI) { + $bytes = Get-MgxContent $script:contentUri -First 256 + $bytes.Length | Should -Be 256 + } + It 'downloads a whole file to -OutFile' -Skip:(-not $env:MGX_LIVE_CONTENT_URI) { + $f = Join-Path ([IO.Path]::GetTempPath()) "mgxlive-$([Guid]::NewGuid().ToString('N')).bin" + try { + Get-MgxContent $script:contentUri -OutFile $f | Out-Null + (Get-Item $f).Length | Should -BeGreaterThan 0 + } finally { Remove-Item $f -ErrorAction SilentlyContinue } + } +} diff --git a/tests/Live/README.md b/tests/Live/README.md new file mode 100644 index 0000000..d764cb2 --- /dev/null +++ b/tests/Live/README.md @@ -0,0 +1,50 @@ +# Live tests + +The rest of the suite mocks HTTP. That proves the code does what it was written to do; it +cannot prove the request leaving the process is one Graph accepts. `Enable-MgxResilience` +shipped a wrapper client with no `BaseAddress` — every relative-URI SDK call threw before +reaching the wire — against a fully green suite, because every mocked test passed an absolute +URI. These tests exist for that class of defect. + +They are read-only. Nothing here writes to a tenant. + +## Running them + +```bash +export AZURE_TENANT_ID='...' +export AZURE_CLIENT_ID='...' +export AZURE_CLIENT_CERTIFICATE_PATH="$HOME/.certs/....pfx" +pwsh -c 'Invoke-Pester -Path ./tests/Live' +``` + +Missing credentials is a **failure**, not a skip. A live suite that skips itself when +unconfigured is one that silently stops running. + +The tests size themselves from the tenant, so a 19-user directory and a 100,000-user one both +work. **Prefer a tenant with a SharePoint licence**: `Get-MgxContent` is the only cmdlet that +needs one, and without it those two tests skip, so a tenant that has it is the only place the +suite covers everything. Point it at any drive item: + +```bash +export MGX_LIVE_CONTENT_URI='/drives//items//content' +``` + +A tenant with no SharePoint answers every sites/drives call with `BadRequest: Tenant does not +have a SPO license`, and no permission grant changes that — the content roles can be present and +still inert. + +## Keeping them honest + +`tests/Unit/Mgx.LiveCoverage.Tests.ps1` fails if an exported cmdlet has no `Describe` block +here. It parses this directory rather than running it, so it works in CI with no credentials. +Adding a cmdlet without a live test breaks the build. + +If a cmdlet genuinely cannot be exercised live, still add the `Describe` with a `-Skip` and a +reason. The point is that the decision is recorded rather than absent — which is how seven of +twelve cmdlets came to have no live coverage at all without anyone noticing. + +## What they will not catch + +They cover the happy path of each cmdlet. Failure modes, resume, throttling and accuracy are +covered by `tests/Mgx.IntegrationTests` (mocked, deterministic) and `tests/benchmarks` (live, +measured). A green run here means "the cmdlets work against Graph", not "the module is correct". diff --git a/tests/Mgx.IntegrationTests/Cmdlets/AuthFingerprintTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/Base/AuthFingerprintTests.cs similarity index 100% rename from tests/Mgx.IntegrationTests/Cmdlets/AuthFingerprintTests.cs rename to tests/Mgx.IntegrationTests/Cmdlets/Base/AuthFingerprintTests.cs diff --git a/tests/Mgx.IntegrationTests/Cmdlets/Base/BetaHintTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/Base/BetaHintTests.cs new file mode 100644 index 0000000..5b1c48b --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/Base/BetaHintTests.cs @@ -0,0 +1,132 @@ +using System.Management.Automation; +using System.Net; +using System.Reflection; +using System.Text; +using Mgx.Cmdlets.Base; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// The beta hint exists for one case: an endpoint that 404s on v1.0 because it only exists in +/// beta. Graph also 404s for a missing OBJECT on a perfectly good endpoint, and for a missing +/// drive item the hint pointed at beta over a request that fails there identically. The codes +/// overlap - Request_ResourceNotFound is returned both for a missing directory object and for a +/// beta-only segment, measured live - so only itemNotFound, whose meaning is unambiguous, is +/// suppressed. +/// +[Collection("Pipeline")] +public class BetaHintTests +{ + private sealed class SingleResponseHandler(HttpStatusCode status, string body) : HttpMessageHandler + { + public List Requests { get; } = []; + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + lock (Requests) Requests.Add(request.RequestUri!.ToString()); + return Task.FromResult(new HttpResponseMessage(status) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } + + private static void InjectMock(HttpMessageHandler handler) + { + ResiliencePipelineFactory.Reset(); + var t = typeof(MgxCmdletBase); + t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)! + .SetValue(null, new HttpClient(handler)); + t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)! + .SetValue(null, MgxCmdletBase.BuildAuthFingerprint( + new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); + // Get-MgxContent refuses to run over a transport mgx does not own (the SDK client + // auto-follows redirects, defeating download-host validation), so the mock must claim + // ownership or the cmdlet errors before any HTTP happens and the test is vacuous. + t.GetField("s_ownsHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, true); + t.GetField("s_graphEndpoint", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! + .SetValue(null, "https://graph.microsoft.com"); + var options = new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }; + t.GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! + .SetValue(null, options); + // Owning the client makes GetClient() check the cached timeout; a mismatch forces a + // rebuild that cannot succeed under a mock. Cache the value the options carry. + t.GetField("s_cachedTotalTimeoutSeconds", BindingFlags.NonPublic | BindingFlags.Static)! + .SetValue(null, options.TotalTimeoutSeconds); + ResiliencePipelineFactory.Reset(); + } + + private static void CleanupMock() + { + var t = typeof(MgxCmdletBase); + t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); + t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); + t.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); + ResiliencePipelineFactory.Reset(); + } + + private static (List Warnings, List Errors) Run( + string cmdlet, string uri, HttpStatusCode status, string body) + { + var handler = new SingleResponseHandler(status, body); + InjectMock(handler); + try + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand(cmdlet) + .AddParameter("Uri", uri) + .AddParameter("ErrorAction", ActionPreference.Continue); + string? invokeEx = null; + try { ps.Invoke(); } + catch (Exception ex) { invokeEx = $"{ex.GetType().Name}: {ex.Message}"; } + + var warnings = ps.Streams.Warning.Select(w => w.Message).ToList(); + var errors = ps.Streams.Error.Select(e => e.ToString()).ToList(); + if (invokeEx != null) errors.Add(invokeEx); + if (errors.Count == 0 && warnings.Count == 0) + errors.Add($"(silent; {handler.Requests.Count} HTTP requests: {string.Join(", ", handler.Requests)})"); + return (warnings, errors); + } + finally { CleanupMock(); } + } + + private const string ItemNotFound = + """{"error":{"code":"itemNotFound","message":"The resource could not be found."}}"""; + private const string ResourceNotFound = + """{"error":{"code":"Request_ResourceNotFound","message":"Resource 'profile' does not exist."}}"""; + + [Fact] + public void A_missing_drive_item_does_not_suggest_beta() + { + var (warnings, errors) = Run( + "Get-MgxContent", "/drives/b!x/items/01X/content", + HttpStatusCode.NotFound, ItemNotFound); + + Assert.DoesNotContain(warnings, w => w.Contains("beta")); + Assert.Contains(errors, e => e.Contains("itemNotFound")); + } + + [Fact] + public void An_ambiguous_404_still_suggests_beta() + { + // Request_ResourceNotFound cannot be told apart from a beta-only segment, so the + // hedged hint stays. This pins the suppression to codes with known semantics: widening + // it to this code would kill the hint's one legitimate case. + var (warnings, _) = Run( + "Invoke-MgxRequest", "/users/00000000-0000-0000-0000-000000000000/profile", + HttpStatusCode.NotFound, ResourceNotFound); + + Assert.Contains(warnings, w => w.Contains("beta")); + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/JsonToHashtableTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/Base/JsonToHashtableTests.cs similarity index 100% rename from tests/Mgx.IntegrationTests/Cmdlets/JsonToHashtableTests.cs rename to tests/Mgx.IntegrationTests/Cmdlets/Base/JsonToHashtableTests.cs diff --git a/tests/Mgx.IntegrationTests/Cmdlets/PipelineContractTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/Base/PipelineContractTests.cs similarity index 100% rename from tests/Mgx.IntegrationTests/Cmdlets/PipelineContractTests.cs rename to tests/Mgx.IntegrationTests/Cmdlets/Base/PipelineContractTests.cs diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DeltaCheckpointIntegrityTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DeltaCheckpointIntegrityTests.cs new file mode 100644 index 0000000..abe1b8a --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/DeltaCheckpointIntegrityTests.cs @@ -0,0 +1,536 @@ +using System.Management.Automation; +using System.Net; +using System.Reflection; +using Mgx.Cmdlets.Base; +using Mgx.Engine.Http; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// A checkpoint is a promise that ItemsCollected items are durably somewhere. Every test here +/// breaks that promise a different way and asserts the sync notices. Once the delta token +/// advances there is no re-fetch, so an item skipped here is gone permanently. +/// +[Collection("Pipeline")] +public class DeltaCheckpointIntegrityTests +{ + private const string DeltaLink1 = "https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D1"; + private const string Page2Url = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2"; + + private const string ChangesPage1 = """ + {"value":[{"id":"b1"},{"id":"b2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2"} + """; + private const string ChangesPage2 = """ + {"value":[{"id":"b3"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B3"} + """; + private const string ChangesPage3 = """ + {"value":[{"id":"b4"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D2"} + """; + private const string ChangesPage2Final = """ + {"value":[{"id":"b3"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D2"} + """; + private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + + private static void InjectMock(MockHttpHandler handler) + { + ResiliencePipelineFactory.Reset(); + var t = typeof(MgxCmdletBase); + t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)! + .SetValue(null, new HttpClient(handler)); + t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)! + .SetValue(null, MgxCmdletBase.BuildAuthFingerprint( + new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); + t.GetField("s_ownsHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, false); + t.GetField("s_graphEndpoint", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! + .SetValue(null, "https://graph.microsoft.com"); + t.GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! + .SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + ResiliencePipelineFactory.Reset(); + } + + private static void CleanupMock() + { + var t = typeof(MgxCmdletBase); + t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); + t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); + t.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); + ResiliencePipelineFactory.Reset(); + } + + private static PowerShell Shell() + { + var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + return ps; + } + + private sealed record Env(string Dir, string DeltaPath, string CheckpointPath, string OutputPath); + + private static Env NewEnv(string checkpointName = "run.checkpoint") + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-integrity-{Guid.NewGuid():N}")).FullName; + var env = new Env(dir, + Path.Combine(dir, "state.json"), + Path.Combine(dir, checkpointName), + Path.Combine(dir, "out.jsonl")); + // Delta state pins requestUrl to DeltaLink1 for every run below. + new DeltaState + { + DeltaLink = DeltaLink1, + Resource = "/users/delta", + GraphEndpoint = "https://graph.microsoft.com", + Select = "", + ApiVersion = "v1.0", + }.Save(env.DeltaPath); + return env; + } + + private static void Sync(Env env) + { + using var ps = Shell(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", env.DeltaPath) + .AddParameter("CheckpointPath", env.CheckpointPath) + .AddParameter("OutputFile", env.OutputPath); + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + } + + private static string[] Ids(string outputPath) => + File.Exists(outputPath) ? File.ReadAllLines(outputPath) : []; + + // ---------------------------------------------------------------- D5 + + /// + /// A temp shorter than the checkpoint promises. Adoption declines, and because the output + /// exists the checkpoint is kept and the run resumes at NextLink - past items that are in + /// neither file. The token then advances over them. + /// + [Fact] + public void A_torn_temp_does_not_let_the_token_advance_past_what_it_lost() + { + var env = NewEnv(); + try + { + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 1: b1,b2 + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 2 re-enumerates + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); + InjectMock(handler); + + Sync(env); + var temp = Directory.GetFiles(env.Dir, "out.jsonl.*.tmp").Single(); + Assert.Equal(2, PaginationCheckpoint.Load(env.CheckpointPath)!.ItemsCollected); + + // The writer flushed two items; the machine lost power and only one reached the + // platter. The checkpoint still promises two. + File.WriteAllLines(temp, ["{\"id\":\"b1\"}"]); + var before = handler.Requests.Count; + + Sync(env); + + // Resuming at NextLink would step over b2, which is now in no file at all. + Assert.Contains("$deltatoken=D1", handler.Requests[before].RequestUri!.ToString()); + Assert.Equal( + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + Ids(env.OutputPath)); + } + finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + } + + // ---------------------------------------------------------------- D6 + + /// + /// PaginationCheckpoint.Save writes "{checkpointPath}.tmp" and renames. With a checkpoint + /// named after the output - which the collision check permits, it only forbids equal paths - + /// that staging file matches the output's own "{output}.*.tmp" glob, so a crash mid-save + /// leaves a file that adoption will happily copy into the data. + /// + [Fact] + public void The_checkpoints_own_staging_file_is_never_adopted_as_data() + { + // -CheckpointPath named after -OutputFile is permitted: the collision check only forbids + // the two being the same file. PaginationCheckpoint.Save then stages through + // "out.jsonl.ckpt.tmp", which matches the output's own "out.jsonl.*.tmp" glob. + var env = NewEnv("out.jsonl.ckpt"); + try + { + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); + InjectMock(handler); + + Sync(env); + var temp = Directory.GetFiles(env.Dir, "out.jsonl.*.tmp") + .Single(f => !f.EndsWith(".ckpt.tmp", StringComparison.Ordinal)); + + // A save that died between writing the staging file and renaming it. It is newer + // than the real temp, so newest-wins selection prefers it. + var staged = $"{env.CheckpointPath}.tmp"; + File.WriteAllText(staged, File.ReadAllText(env.CheckpointPath)); + File.SetLastWriteTimeUtc(staged, File.GetLastWriteTimeUtc(temp).AddMinutes(5)); + + Sync(env); + + Assert.Equal( + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + Ids(env.OutputPath)); + } + finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + } + + // ---------------------------------------------------------------- D7 + + /// + /// File.Exists is presence, not content. An output truncated to nothing between runs still + /// reads as a valid baseline, so the run appends the tail to an empty file and the token + /// advances over everything the checkpoint said was already there. + /// + [Fact] + public void A_truncated_output_is_not_mistaken_for_the_baseline_it_describes() + { + var env = NewEnv(); + try + { + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 1: b1,b2 + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); // run 2: adopts, adds b3 + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 3 re-enumerates + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage3); + InjectMock(handler); + + Sync(env); + Sync(env); + // Run 2 resumed, so it wrote straight to the output and left no temp behind. + Assert.Empty(Directory.GetFiles(env.Dir, "out.jsonl.*.tmp")); + Assert.Equal(3, PaginationCheckpoint.Load(env.CheckpointPath)!.ItemsCollected); + + File.WriteAllText(env.OutputPath, ""); // emptied underneath the checkpoint + var before = handler.Requests.Count; + + Sync(env); + + // Appending the tail to an empty file would advance the token over b1..b3. + Assert.Contains("$deltatoken=D1", handler.Requests[before].RequestUri!.ToString()); + Assert.Equal( + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}", "{\"id\":\"b4\"}"], + Ids(env.OutputPath)); + } + finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + } + + // ---------------------------------------------------------------- D8 + + /// + /// An interrupted run leaves the output ahead of its checkpoint: everything it wrote after + /// the last save is on disk but uncounted. Those items are re-fetched on resume, so unless + /// the output is cut back to the length the checkpoint recorded they are written twice. + /// + [Fact] + public void Items_written_after_the_last_checkpoint_are_not_duplicated_on_resume() + { + var env = NewEnv(); + try + { + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 1: b1,b2 + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); // run 2: adopts, adds b3 + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage3); // run 3: b4 + InjectMock(handler); + + Sync(env); + Sync(env); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], Ids(env.OutputPath)); + + // Run 2 got partway into page 3 before it died: b4 reached the file, the checkpoint + // still ends at b3, and page 3 will be fetched again. + File.AppendAllLines(env.OutputPath, ["{\"id\":\"b4\"}"]); + + Sync(env); + + var ids = Ids(env.OutputPath); + Assert.Equal( + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}", "{\"id\":\"b4\"}"], + ids); + Assert.Equal(ids.Length, ids.Distinct().Count()); + } + finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + } + + // ---------------------------------------------------------------- D9 + + /// + /// Adoption ends in a move followed by a delete. A temp that outlives its own adoption - + /// the delete failed, or the process died in between - must not be merged a second time. + /// The run below fails before completing a page, so the only checkpoint on disk afterwards + /// is the one adoption itself left; if that still names the temp, the next run adopts it again. + /// + [Fact] + public void A_temp_that_survives_its_own_adoption_is_not_merged_twice() + { + var env = NewEnv(); + try + { + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 1: b1,b2 + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // run 2: adopts, then dies + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); // run 3: b3 + InjectMock(handler); + + Sync(env); + var temp = Directory.GetFiles(env.Dir, "out.jsonl.*.tmp").Single(); + var tempBytes = File.ReadAllBytes(temp); + + Sync(env); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], Ids(env.OutputPath)); + Assert.True(File.Exists(env.CheckpointPath), "the run failed, so its position survives"); + + // The delete never happened. + File.WriteAllBytes(temp, tempBytes); + + Sync(env); + + Assert.Equal( + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + Ids(env.OutputPath)); + } + finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + } + + /// + /// A checkpoint is read back from disk, so its contents are input, not a fact. One naming + /// the output as the temp to adopt would have the file appended to itself and then removed. + /// + [Fact] + public void A_checkpoint_naming_the_output_as_its_own_temp_does_not_consume_it() + { + var env = NewEnv(); + try + { + File.WriteAllLines(env.OutputPath, ["{\"id\":\"a1\"}", "{\"id\":\"a2\"}"]); + new PaginationCheckpoint + { + Resource = DeltaLink1, + NextLink = Page2Url, + ItemsCollected = 2, + TempFile = "out.jsonl", + DataLength = new FileInfo(env.OutputPath).Length, + }.Save(env.CheckpointPath); + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); + InjectMock(handler); + + Sync(env); + + Assert.True(File.Exists(env.OutputPath), "the output must survive"); + Assert.Equal( + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + Ids(env.OutputPath)); + } + finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + } + + /// + /// The recorded temp name accepts only what a run could have written: the output's name, + /// 32 hex digits, ".tmp". A checkpoint naming any other file - here, the checkpoint + /// itself - would have that file's bytes copied into the output as data and the file + /// deleted as the spent temp. + /// + [Fact] + public void A_checkpoint_naming_a_file_that_is_not_a_temp_does_not_consume_it() + { + var env = NewEnv(); + try + { + File.WriteAllLines(env.OutputPath, ["{\"id\":\"a1\"}", "{\"id\":\"a2\"}"]); + new PaginationCheckpoint + { + Resource = DeltaLink1, + NextLink = Page2Url, + ItemsCollected = 2, + TempFile = Path.GetFileName(env.CheckpointPath), + DataLength = 10, // shorter than the checkpoint file, so the length guard passes + }.Save(env.CheckpointPath); + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); + InjectMock(handler); + + Sync(env); + + // The unusable name reads as "items are in no file": re-enumerate, output replaced. + Assert.Equal( + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + Ids(env.OutputPath)); + } + finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + } + + // ---------------------------------------------------------------- D10 + + /// + /// Temp selection is newest-by-write-time with nothing tying a temp to the run that wrote it. + /// A newer file from an unrelated run wins over the one the checkpoint is actually about. + /// + [Fact] + public void Adoption_takes_the_temp_its_checkpoint_describes_not_the_newest_one() + { + var env = NewEnv(); + try + { + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); + InjectMock(handler); + + Sync(env); // leaves its own temp with b1,b2 + var own = Directory.GetFiles(env.Dir, "out.jsonl.*.tmp").Single(); + + // Something else drops a newer file matching the same glob. + var decoy = $"{env.OutputPath}.zzzzzzzz.tmp"; + File.WriteAllLines(decoy, ["{\"id\":\"decoy1\"}", "{\"id\":\"decoy2\"}", "{\"id\":\"decoy3\"}"]); + File.SetLastWriteTimeUtc(decoy, File.GetLastWriteTimeUtc(own).AddMinutes(5)); + + Sync(env); + + var ids = Ids(env.OutputPath); + Assert.DoesNotContain(ids, l => l.Contains("decoy")); + Assert.Contains("{\"id\":\"b1\"}", ids); + Assert.Contains("{\"id\":\"b2\"}", ids); + } + finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + } + + // ---------------------------------------------------------------- D11 + + /// + /// The item loop calls TryGetProperty("@removed") on every element. That throws + /// InvalidOperationException on anything that is not a JSON object, which escapes as a + /// terminating error naming neither the endpoint nor the item. + /// + [Fact] + public void A_non_object_item_in_a_page_does_not_terminate_the_run() + { + var env = NewEnv(); + try + { + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, + """{"value":[{"id":"b1"},"not-an-object",{"id":"b2"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D2"}"""); + InjectMock(handler); + + Exception? escaped = null; + using (var ps = Shell()) + { + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", env.DeltaPath) + .AddParameter("CheckpointPath", env.CheckpointPath) + .AddParameter("OutputFile", env.OutputPath); + try { ps.Invoke(); } + catch (Exception ex) { escaped = ex; } + } + + Assert.Null(escaped); + var ids = Ids(env.OutputPath); + Assert.Contains("{\"id\":\"b1\"}", ids); + Assert.Contains("{\"id\":\"b2\"}", ids); + } + finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + } + + // ---------------------------------------------------------------- D12 + + /// + /// A resumed run starts partway into a page, and PageIterator drops the skipped items before + /// the consumer ever sees them. A mid-page checkpoint saved while still on that first page + /// therefore has to count the skipped items as well as the new ones - it records a position + /// in the page, not how much this run wrote. Counting only the new ones understates the + /// output, and the next resume skips too few and writes the difference a second time. + /// + [Fact] + public void A_second_interruption_on_a_resumed_page_does_not_duplicate_its_items() + { + const string RefusedLink = "https://not-graph.example.com/v1.0/users/delta?$skiptoken=P2"; + const string Page2Link = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=P2"; + + static string Item(int i) => $"{{\"id\":\"p{i}\"}}"; + static string Page(string nextLink) => + $"{{\"value\":[{string.Join(",", Enumerable.Range(0, 1000).Select(Item))}]," + + $"\"@odata.nextLink\":\"{nextLink}\"}}"; + + var env = NewEnv(); + try + { + // Ctrl-C 500 items into the first page: the run promoted what it had written and + // recorded the output, 500 items in, all 500 of them from the page in flight. + File.WriteAllLines(env.OutputPath, Enumerable.Range(0, 500).Select(Item)); + new PaginationCheckpoint + { + Resource = DeltaLink1, + NextLink = DeltaLink1, + ItemsCollected = 500, + PageItemsAlreadyWritten = 500, + TempFile = null, + DataLength = new FileInfo(env.OutputPath).Length, + }.Save(env.CheckpointPath); + + var handler = new MockHttpHandler(); + // The resumed run writes the other 500 items of the page, saves the mid-page + // checkpoint that item 1000 triggers, and is then refused the page's nextLink - + // so it dies before the page boundary and that mid-page save is what survives. + handler.QueueResponse(HttpStatusCode.OK, Page(RefusedLink)); + // The run after it re-fetches the same page, this time paginating to the end. + handler.QueueResponse(HttpStatusCode.OK, Page(Page2Link)); + handler.QueueResponse(HttpStatusCode.OK, + """{"value":[{"id":"tail"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D2"}"""); + InjectMock(handler); + + Sync(env); + + Assert.Equal(1000, Ids(env.OutputPath).Length); + var checkpoint = PaginationCheckpoint.Load(env.CheckpointPath)!; + Assert.Equal(1000, checkpoint.ItemsCollected); + // 1000 items of this page are in the output, not the 500 this run put there. + Assert.Equal(1000, checkpoint.PageItemsAlreadyWritten); + + Sync(env); + + var ids = Ids(env.OutputPath); + Assert.Equal(ids.Length, ids.Distinct().Count()); + Assert.Equal(1001, ids.Length); + Assert.Equal("{\"id\":\"tail\"}", ids[^1]); + } + finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + } +} diff --git a/tests/Mgx.IntegrationTests/DeltaQueryTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DeltaQueryTests.cs similarity index 62% rename from tests/Mgx.IntegrationTests/DeltaQueryTests.cs rename to tests/Mgx.IntegrationTests/Cmdlets/DeltaQueryTests.cs index dde8a91..cc4a305 100644 --- a/tests/Mgx.IntegrationTests/DeltaQueryTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/DeltaQueryTests.cs @@ -232,6 +232,55 @@ public void DeltaState_SaveAndLoad_RoundTrips() } } + [Fact] + public void DeltaState_RoundTrips_TheApiVersion_ItWasBuiltAgainst() + { + var path = Path.Combine(Path.GetTempPath(), $"delta-test-{Guid.NewGuid()}.json"); + try + { + new DeltaState + { + DeltaLink = "https://graph.microsoft.com/beta/users/delta?$deltatoken=abc", + Resource = "/users/delta", + GraphEndpoint = "https://graph.microsoft.com", + ApiVersion = "beta" + }.Save(path); + + Assert.Equal("beta", DeltaState.Load(path)!.ApiVersion); + } + finally + { + DeltaState.Delete(path); + } + } + + [Fact] + public void DeltaState_WrittenBefore_2_0_1_LoadsWithAnUnknownApiVersion() + { + // Upgrade path: state files in the wild have no apiVersion key. They must load as + // "unknown" so the cmdlet's mismatch check skips them, not fail or default to a + // version that would spuriously conflict with the run's own -ApiVersion. + var path = Path.Combine(Path.GetTempPath(), $"delta-test-{Guid.NewGuid()}.json"); + try + { + File.WriteAllText(path, """ + { + "deltaLink": "https://graph.microsoft.com/v1.0/users/delta?$deltatoken=abc", + "resource": "/users/delta", + "graphEndpoint": "https://graph.microsoft.com" + } + """); + + var loaded = DeltaState.Load(path); + Assert.NotNull(loaded); + Assert.Equal(string.Empty, loaded!.ApiVersion); + } + finally + { + DeltaState.Delete(path); + } + } + // --- LoadWithResult tests (Fix 6) --- [Fact] @@ -1134,4 +1183,578 @@ public void DeltaState_ValidateWriteAccess_ThrowsOnUnwritablePath() Assert.ThrowsAny(() => DeltaState.ValidateWriteAccess(unwritablePath)); } + + // --- 2.1 Phase A: -Prefer / -Latest / -CheckpointPath --- + + [Fact] + public void DeltaState_RoundTrips_Prefer_And_OldSchemaLoadsNull() + { + var path = Path.Combine(Path.GetTempPath(), $"delta-prefer-{Guid.NewGuid()}.json"); + try + { + new DeltaState + { + DeltaLink = "https://graph.microsoft.com/v1.0/me/drive/root/delta?token=abc", + Resource = "/me/drive/root/delta", + GraphEndpoint = "https://graph.microsoft.com", + Prefer = "deltashowremovedasdeleted,hierarchicalsharing" + }.Save(path); + + var loaded = DeltaState.Load(path); + Assert.Equal("deltashowremovedasdeleted,hierarchicalsharing", loaded!.Prefer); + + // A 2.0-era state file has no "prefer" property: it must load with null, so an + // unchanged run (no -Prefer) does not trigger a phantom drift resync. + File.WriteAllText(path, """ + { + "deltaLink": "https://graph.microsoft.com/v1.0/users/delta?$deltatoken=old", + "resource": "/users/delta", + "lastSync": "2026-08-01T00:00:00+00:00", + "itemCount": 5, + "graphEndpoint": "https://graph.microsoft.com" + } + """); + var (oldState, result) = DeltaState.LoadWithResult(path); + Assert.Equal(DeltaLoadResult.Ok, result); + Assert.Null(oldState!.Prefer); + } + finally + { + DeltaState.Delete(path); + } + } + + [Fact] + public void Cmdlet_Prefer_SentOnEveryPage_AndPersistedNormalized() + { + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, DeltaPage1); + handler.QueueResponse(HttpStatusCode.OK, DeltaPage2WithToken); + + InjectMockHttpClient(handler); + var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-prefer-{Guid.NewGuid()}.json"); + + try + { + using var ps = CreateTestShell(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("Prefer", new[] { "hierarchicalsharing", "deltashowremovedasdeleted" }); + var results = ps.Invoke(); + + Assert.Equal(3, results.Count); + // The joined header goes out on every page request, not just the first. + Assert.Equal(2, handler.RequestCount); + foreach (var request in handler.Requests) + { + Assert.True(request.Headers.Contains("Prefer"), "every page request must carry Prefer"); + Assert.Equal("hierarchicalsharing,deltashowremovedasdeleted", + request.Headers.GetValues("Prefer").Single()); + } + // Stored normalized (sorted, case-insensitive) for order-independent drift checks. + var state = DeltaState.Load(deltaPath); + Assert.Equal("deltashowremovedasdeleted,hierarchicalsharing", state!.Prefer); + } + finally + { + DeltaState.Delete(deltaPath); + CleanupMockHttpClient(); + } + } + + [Fact] + public void Cmdlet_PreferDrift_ForcesFullResync_AndDeletesCheckpoint() + { + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, DeltaPage2WithToken); + + InjectMockHttpClient(handler); + var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-preferdrift-{Guid.NewGuid()}.json"); + var cpPath = Path.Combine(Path.GetTempPath(), $"cmdlet-preferdrift-{Guid.NewGuid()}.checkpoint"); + + try + { + new DeltaState + { + DeltaLink = "https://graph.microsoft.com/v1.0/users/delta?$deltatoken=stale", + Resource = "/users/delta", + GraphEndpoint = "https://graph.microsoft.com", + Select = "", + Prefer = "deltashowremovedasdeleted" + }.Save(deltaPath); + // A leftover checkpoint from the drifted enumeration must not survive the resync. + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users/delta?$deltatoken=stale", + NextLink = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=page9", + ItemsCollected = 42 + }.Save(cpPath); + + using var ps = CreateTestShell(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", cpPath); + ps.Invoke(); // no -Prefer: drift against the stored tokens + + Assert.Contains(ps.Streams.Warning, w => w.Message.Contains("Prefer headers changed")); + // Full resync, not the stale incremental link + Assert.DoesNotContain("deltatoken=stale", handler.Requests[0].RequestUri!.ToString()); + Assert.False(File.Exists(cpPath), "checkpoint from the drifted enumeration must be deleted"); + // New state carries the new (empty) Prefer + var state = DeltaState.Load(deltaPath); + Assert.True(string.IsNullOrEmpty(state!.Prefer)); + } + finally + { + DeltaState.Delete(deltaPath); + PaginationCheckpoint.Delete(cpPath); + CleanupMockHttpClient(); + } + } + + [Fact] + public void Cmdlet_Latest_UsesDeltatokenForm_ForDirectoryResources() + { + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, EmptyDeltaWithToken); + + InjectMockHttpClient(handler); + var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-latest-dir-{Guid.NewGuid()}.json"); + + try + { + using var ps = CreateTestShell(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("Latest", true); + var results = ps.Invoke(); + + Assert.Empty(results); // baseline only, no data + var url = handler.Requests[0].RequestUri!.ToString(); + Assert.Contains("$deltatoken=latest", url); + // The empty-page-still-saves-token path persists the baseline. + var state = DeltaState.Load(deltaPath); + Assert.Contains("$deltatoken=empty789", state!.DeltaLink); + } + finally + { + DeltaState.Delete(deltaPath); + CleanupMockHttpClient(); + } + } + + [Fact] + public void Cmdlet_Latest_UsesTokenForm_ForDrives() + { + // Drive resources take ?token=latest; $deltatoken=latest is directory-only. + // (Verified against driveitem-delta and delta-query-overview docs.) + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, """ + { + "value": [], + "@odata.deltaLink": "https://graph.microsoft.com/v1.0/me/drive/root/delta?token=drivebaseline" + } + """); + + InjectMockHttpClient(handler); + var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-latest-drive-{Guid.NewGuid()}.json"); + + try + { + using var ps = CreateTestShell(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/me/drive/root/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("Latest", true); + ps.Invoke(); + + var url = handler.Requests[0].RequestUri!.ToString(); + Assert.Contains("&token=latest", url); + Assert.DoesNotContain("$deltatoken", url); + var state = DeltaState.Load(deltaPath); + Assert.Contains("token=drivebaseline", state!.DeltaLink); + } + finally + { + DeltaState.Delete(deltaPath); + CleanupMockHttpClient(); + } + } + + [Fact] + public void Cmdlet_Latest_IgnoredWhenStateExists() + { + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, EmptyDeltaWithToken); + + InjectMockHttpClient(handler); + var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-latest-ignored-{Guid.NewGuid()}.json"); + + try + { + new DeltaState + { + DeltaLink = "https://graph.microsoft.com/v1.0/users/delta?$deltatoken=stale", + Resource = "/users/delta", + GraphEndpoint = "https://graph.microsoft.com", + Select = "" + }.Save(deltaPath); + + using var ps = CreateTestShell(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("Latest", true); + ps.Invoke(); + + Assert.Contains(ps.Streams.Warning, w => w.Message.Contains("-Latest ignored")); + // The stored incremental link is used, not a re-baseline. + Assert.Contains("deltatoken=stale", handler.Requests[0].RequestUri!.ToString()); + } + finally + { + DeltaState.Delete(deltaPath); + CleanupMockHttpClient(); + } + } + + [Fact] + public void Cmdlet_Checkpoint_SurvivesCrash_ThenResumesAndDeletesOnSuccess() + { + // Run 1 (pipeline mode): page 1 succeeds, page 2 dies on 500s. The page-boundary + // checkpoint must survive. Run 2: resumes from page 2's link (not from scratch), + // completes, deletes the checkpoint, and saves delta state. + var handler = new MockHttpHandler(); + var errorBody = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + handler.QueueResponse(HttpStatusCode.OK, DeltaPage1); + handler.QueueResponse(HttpStatusCode.InternalServerError, errorBody); + handler.QueueResponse(HttpStatusCode.InternalServerError, errorBody); + + InjectMockHttpClient(handler); + var optField = typeof(MgxCmdletBase).GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)!; + optField.SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + ResiliencePipelineFactory.Reset(); + + var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-cp-resume-{Guid.NewGuid()}.json"); + var cpPath = Path.Combine(Path.GetTempPath(), $"cmdlet-cp-resume-{Guid.NewGuid()}.checkpoint"); + + try + { + using (var ps = CreateTestShell()) + { + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", cpPath); + var run1 = ps.Invoke(); + + Assert.True(ps.HadErrors, "run 1 should fail on the 500s"); + Assert.Equal(2, run1.Count); // page 1 was emitted before the crash + } + + Assert.True(File.Exists(cpPath), "page-boundary checkpoint must survive the crash"); + Assert.False(File.Exists(deltaPath), "delta state must NOT be saved on failure"); + var checkpoint = PaginationCheckpoint.Load(cpPath); + Assert.Contains("skiptoken=page2", checkpoint!.NextLink); + Assert.Equal(2, checkpoint.ItemsCollected); + + // Heal the transport and resume. + handler.QueueResponse(HttpStatusCode.OK, DeltaPage2WithToken); + using (var ps = CreateTestShell()) + { + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", cpPath); + var run2 = ps.Invoke(); + + Assert.False(ps.HadErrors); + Assert.Single(run2); // only page 2 - no re-enumeration of page 1 + } + + // Run 2's single request went straight to the checkpointed position. + Assert.Contains("skiptoken=page2", handler.Requests[^1].RequestUri!.ToString()); + Assert.False(File.Exists(cpPath), "checkpoint must be deleted on success"); + var state = DeltaState.Load(deltaPath); + Assert.Contains("$deltatoken=abc123", state!.DeltaLink); + Assert.Equal(3, state.ItemCount); // 2 resumed + 1 from page 2 + } + finally + { + DeltaState.Delete(deltaPath); + PaginationCheckpoint.Delete(cpPath); + CleanupMockHttpClient(); + } + } + + [Fact] + public void Cmdlet_Checkpoint_JsonlOrphanedTemp_IsAdopted() + { + // A killed JSONL fresh run leaves the checkpoint plus a GUID-named temp and no + // output file. The next run must adopt the temp (trimmed to the checkpointed count) + // and resume appending - the same recovery Export-MgxCollection ships. + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, DeltaPage2WithToken); + + InjectMockHttpClient(handler); + var stem = $"cmdlet-cp-adopt-{Guid.NewGuid()}"; + var deltaPath = Path.Combine(Path.GetTempPath(), $"{stem}.json"); + var cpPath = Path.Combine(Path.GetTempPath(), $"{stem}.checkpoint"); + var outputPath = Path.Combine(Path.GetTempPath(), $"{stem}.jsonl"); + var tempPath = $"{outputPath}.deadbeefdeadbeefdeadbeefdeadbeef.tmp"; + + try + { + // Simulate the kill: checkpoint at page-1 boundary, temp holding page 1's items. + // Resource must equal the URL the cmdlet rebuilds for this parameter set. + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users/delta?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=page2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0 + }.Save(cpPath); + File.WriteAllLines(tempPath, + [ + """{"id": "user1", "displayName": "User One"}""", + """{"id": "user2", "displayName": "User Two"}""" + ]); + + using var ps = CreateTestShell(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", cpPath); + ps.Invoke(); + + Assert.False(ps.HadErrors); + Assert.Contains(ps.Streams.Warning, w => w.Message.Contains("Recovered 2 items")); + var lines = File.ReadAllLines(outputPath); + Assert.Equal(3, lines.Length); // 2 adopted + 1 from the resumed page + Assert.Contains("user3", lines[2]); + Assert.False(File.Exists(tempPath), "adopted temp must be removed"); + Assert.False(File.Exists(cpPath), "checkpoint must be deleted on success"); + } + finally + { + DeltaState.Delete(deltaPath); + PaginationCheckpoint.Delete(cpPath); + if (File.Exists(outputPath)) File.Delete(outputPath); + if (File.Exists(tempPath)) File.Delete(tempPath); + CleanupMockHttpClient(); + } + } + + [Fact] + public void Cmdlet_410Gone_DeletesCheckpointToo() + { + // 410 invalidates the whole enumeration: delta state AND checkpoint describe a dead + // position. A surviving checkpoint would resume into the expired enumeration. + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.Gone, + """{"error":{"code":"deltaTokenExpired","message":"Delta token has expired"}}"""); + handler.QueueResponse(HttpStatusCode.OK, DeltaPage2WithToken); + + InjectMockHttpClient(handler); + var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-410cp-{Guid.NewGuid()}.json"); + var cpPath = Path.Combine(Path.GetTempPath(), $"cmdlet-410cp-{Guid.NewGuid()}.checkpoint"); + + try + { + new DeltaState + { + DeltaLink = "https://graph.microsoft.com/v1.0/users/delta?$deltatoken=stale", + Resource = "/users/delta", + GraphEndpoint = "https://graph.microsoft.com", + Select = "" + }.Save(deltaPath); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users/delta?$deltatoken=stale", + NextLink = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=dead", + ItemsCollected = 7 + }.Save(cpPath); + + using var ps = CreateTestShell(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", cpPath); + var results = ps.Invoke(); + + Assert.True(results.Count > 0, "re-sync after 410 should return items"); + Assert.False(File.Exists(cpPath), "410 must delete the checkpoint with the delta state"); + var state = DeltaState.Load(deltaPath); + Assert.Contains("$deltatoken=abc123", state!.DeltaLink); + } + finally + { + DeltaState.Delete(deltaPath); + PaginationCheckpoint.Delete(cpPath); + CleanupMockHttpClient(); + } + } + + [Fact] + public void Cmdlet_RemovedItems_AreCountedAndPassedThrough() + { + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, DeltaPageWithRemoved); + + InjectMockHttpClient(handler); + var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-removed-{Guid.NewGuid()}.json"); + + try + { + using var ps = CreateTestShell(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("Verbose", true); + var results = ps.Invoke(); + + // Raw passthrough: the @removed item is emitted, not filtered. + Assert.Equal(2, results.Count); + var removed = results.Select(r => (System.Collections.Hashtable)r.BaseObject) + .Single(ht => ht.ContainsKey("@removed")); + Assert.Equal("user5", removed["id"]); + // Accounting reaches the completion message. + Assert.Contains(ps.Streams.Verbose, v => v.Message.Contains("(1 removed)")); + } + finally + { + DeltaState.Delete(deltaPath); + CleanupMockHttpClient(); + } + } + + /// + /// A single delta page carrying items, no nextLink, deltaLink present. + /// + private static string BuildSingleDeltaPage(int count) + { + var sb = new System.Text.StringBuilder(); + sb.Append("{\"value\":["); + for (int i = 0; i < count; i++) + { + if (i > 0) sb.Append(','); + sb.Append($"{{\"id\":\"user{i}\",\"displayName\":\"User {i}\"}}"); + } + sb.Append("],\"@odata.deltaLink\":\"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=bigpage\"}"); + return sb.ToString(); + } + + [Fact] + public void Cmdlet_Checkpoint_MidPage_BoundsProgressLostWhenARunDiesInsideAPage() + { + // A delta page can be long, and can be the only page of the run. If progress were saved + // at page boundaries alone, a run that died inside a page would resume from zero and + // re-enumerate everything it had already written. The contract is mid-page checkpointing: + // the position is saved often enough that dying inside a page costs at most + // MaxAcceptableLoss items of lost progress. + // + // The death staged here is the JSONL promotion: -OutputFile names an existing directory, + // so the temp file cannot be renamed onto it. That failure lands after the page has been + // consumed and before the success path deletes the checkpoint, so what remains on disk is + // exactly what a resume would find. + const int ItemsInPage = 1250; + const int MaxAcceptableLoss = 500; + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, BuildSingleDeltaPage(ItemsInPage)); + + InjectMockHttpClient(handler); + var stem = $"cmdlet-cp-midpage-{Guid.NewGuid()}"; + var deltaPath = Path.Combine(Path.GetTempPath(), $"{stem}.json"); + var cpPath = Path.Combine(Path.GetTempPath(), $"{stem}.checkpoint"); + var outputPath = Path.Combine(Path.GetTempPath(), $"{stem}.jsonl"); + Directory.CreateDirectory(outputPath); + + try + { + using var ps = CreateTestShell(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", cpPath); + ps.Invoke(); + + Assert.True(ps.HadErrors, "the run must fail: the output could not be promoted"); + Assert.False(File.Exists(deltaPath), "delta state must not be saved on failure"); + + var checkpoint = PaginationCheckpoint.Load(cpPath); + Assert.True(checkpoint != null, + $"the run consumed {ItemsInPage} items inside one page and left no checkpoint - " + + "every item would be re-enumerated on resume"); + Assert.True(checkpoint!.PageItemsAlreadyWritten > 0, + "the surviving checkpoint holds no in-page position, so a resume cannot skip " + + "what was already written"); + Assert.True(checkpoint.ItemsCollected >= ItemsInPage - MaxAcceptableLoss, + $"checkpoint stalled at {checkpoint.ItemsCollected} of {ItemsInPage} items: dying here " + + $"loses {ItemsInPage - checkpoint.ItemsCollected} items of progress, more than the " + + $"{MaxAcceptableLoss} the resume guarantee allows"); + // Single page: the in-page position is the whole position. + Assert.Equal(checkpoint.ItemsCollected, (long)checkpoint.PageItemsAlreadyWritten); + } + finally + { + DeltaState.Delete(deltaPath); + PaginationCheckpoint.Delete(cpPath); + if (Directory.Exists(outputPath)) Directory.Delete(outputPath, recursive: true); + foreach (var leftover in Directory.GetFiles(Path.GetTempPath(), $"{stem}*")) + { + try { File.Delete(leftover); } catch { } + } + CleanupMockHttpClient(); + } + } + [Fact] + public void Latest_is_refused_when_the_previous_state_was_discarded() + { + // -Latest baselines from now and returns nothing. After a state invalidation the user is + // told a full re-sync is starting; honouring -Latest there returns zero items and saves a + // fresh baseline, dropping every change since the last good sync. The "-Latest ignored" + // guard only covered the resume path, which an invalidated state never reaches - and + // -Latest's own help invites leaving it in a scheduled script, which is the setup that + // triggers it. + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, DeltaPage2WithToken); + InjectMockHttpClient(handler); + var deltaPath = Path.Combine(Path.GetTempPath(), $"latest-discard-{Guid.NewGuid()}.json"); + + try + { + // Stored state selected only id; this run asks for more, which invalidates it. + new DeltaState + { + DeltaLink = "https://graph.microsoft.com/v1.0/users/delta?$deltatoken=stale", + Resource = "/users/delta", + GraphEndpoint = "https://graph.microsoft.com", + Select = "id", + }.Save(deltaPath); + + using var ps = CreateTestShell(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("Property", new[] { "id", "mail" }) + .AddParameter("Latest"); + ps.Invoke(); + + var requested = handler.Requests.Select(r => r.RequestUri!.ToString()).ToList(); + Assert.NotEmpty(requested); + Assert.DoesNotContain(requested, + u => u.Contains("token=latest", StringComparison.OrdinalIgnoreCase)); + } + finally + { + DeltaState.Delete(deltaPath); + CleanupMockHttpClient(); + } + } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DeltaRecoveryTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DeltaRecoveryTests.cs new file mode 100644 index 0000000..2b5a144 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/DeltaRecoveryTests.cs @@ -0,0 +1,537 @@ +using System.Management.Automation; +using System.Net; +using System.Reflection; +using Mgx.Cmdlets.Base; +using Mgx.Engine.Http; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// End-to-end recovery for a JSONL delta sync, driven through the cmdlet against a mock +/// transport. OrphanAdoptionTests and CheckpointRetentionTests call TryAdoptOrphanedTemp +/// directly; both shipped bugs in this area lived in the CALLER, in the interaction between +/// adoption, the checkpoint-deletion branch and the temp-vs-direct write decision, so the +/// state machine needs coverage that actually runs it. +/// +/// The invariant under test: a checkpoint that survives a run must never describe items that +/// exist nowhere. Once the delta token advances there is no re-fetch, so an item skipped here +/// is gone permanently. +/// +[Collection("Pipeline")] +public class DeltaRecoveryTests +{ + private const string Baseline = """ + {"value":[{"id":"a1"},{"id":"a2"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D1"} + """; + private const string ChangesPage1 = """ + {"value":[{"id":"b1"},{"id":"b2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2"} + """; + private const string ChangesPage2 = """ + {"value":[{"id":"b3"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D2"} + """; + private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + + private static void InjectMock(MockHttpHandler handler) + { + ResiliencePipelineFactory.Reset(); + var t = typeof(MgxCmdletBase); + t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)! + .SetValue(null, new HttpClient(handler)); + t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)! + .SetValue(null, MgxCmdletBase.BuildAuthFingerprint( + new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); + t.GetField("s_ownsHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, false); + t.GetField("s_graphEndpoint", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! + .SetValue(null, "https://graph.microsoft.com"); + t.GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! + .SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + ResiliencePipelineFactory.Reset(); + } + + private static void CleanupMock() + { + var t = typeof(MgxCmdletBase); + t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); + t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); + t.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); + ResiliencePipelineFactory.Reset(); + } + + private static PowerShell Shell() + { + var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + return ps; + } + + private static void Sync(string deltaPath, string checkpointPath, string outputPath) + { + using var ps = Shell(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("OutputFile", outputPath); + // A run that dies is the subject here, so a terminating error is an expected outcome. + try { ps.Invoke(); } + catch (System.Management.Automation.CmdletInvocationException) { } + } + + /// + /// The steady-state failure: a completed run, then a run that dies on a transient 500 + /// partway through. The dead run's temp holds the pages the checkpoint counts. If that + /// temp is discarded, the next run sees checkpoint + output + no temp - which is the + /// ROUTINE "nothing to promote" state - resumes in append mode, and the token advances + /// past pages that were never written anywhere. And recovery must land on what a clean + /// run 2 would have produced: promotion replaces the completed run's rows, which the + /// caller has already consumed, rather than appending behind them. + /// + [Fact] + public void A_transient_error_midrun_does_not_strand_the_pages_the_checkpoint_counts() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-recovery-{Guid.NewGuid():N}")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var outputPath = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, Baseline); // run 1 + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 2, page 1 + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // run 2, page 2 + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // ... and its retry + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); // run 3 resumes + + InjectMock(handler); + try + { + Sync(deltaPath, checkpointPath, outputPath); + Assert.Equal(["{\"id\":\"a1\"}", "{\"id\":\"a2\"}"], File.ReadAllLines(outputPath)); + Assert.False(File.Exists(checkpointPath), "a completed run deletes its checkpoint"); + + Sync(deltaPath, checkpointPath, outputPath); + var checkpoint = PaginationCheckpoint.Load(checkpointPath); + Assert.NotNull(checkpoint); + Assert.Equal(2, checkpoint.ItemsCollected); + Assert.Contains("skiptoken=B2", checkpoint.NextLink); + // b1 and b2 are counted by the checkpoint but are not in the output, so they must + // still be somewhere: the temp is the only place they can be. + Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal( + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], + File.ReadAllLines(Directory.GetFiles(dir, "out.jsonl.*.tmp")[0])); + + Sync(deltaPath, checkpointPath, outputPath); + Assert.Equal( + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(outputPath)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + Assert.False(File.Exists(checkpointPath)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The semantics every recovery path has to agree with: a completed sync's output is that + /// sync's changes alone. A fresh run writes to a temp and moves it over the output, so the + /// previous sync's rows - already consumed by whatever reads the file - do not accumulate. + /// + [Fact] + public void A_completed_sync_replaces_the_previous_syncs_output() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-recovery-{Guid.NewGuid():N}")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var outputPath = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, Baseline); // run 1: a1,a2 + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 2: b1,b2 + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); // run 2: b3 + + InjectMock(handler); + try + { + Sync(deltaPath, checkpointPath, outputPath); + Assert.Equal(["{\"id\":\"a1\"}", "{\"id\":\"a2\"}"], File.ReadAllLines(outputPath)); + + Sync(deltaPath, checkpointPath, outputPath); + Assert.Equal( + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(outputPath)); + } + finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// A checkpoint that predates the recorded temp name and length cannot say which file its + /// items are in, so against an existing output it must not guess: an appending run's items + /// are in the output, a fresh run's are in a temp, and merging the wrong one either repeats + /// rows or hands a stale file to the resume. Re-enumerating from the token loses nothing. + /// + [Fact] + public void A_checkpoint_that_does_not_name_its_file_reenumerates_rather_than_guessing() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-recovery-{Guid.NewGuid():N}")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var outputPath = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, Baseline); // run 1 + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 2, page 1 + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // run 2 dies + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 3 re-enumerates + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + + InjectMock(handler); + try + { + Sync(deltaPath, checkpointPath, outputPath); + Sync(deltaPath, checkpointPath, outputPath); + + // The same position as recorded, minus the fields that locate the items. + var checkpoint = PaginationCheckpoint.Load(checkpointPath)!; + checkpoint.TempFile = null; + checkpoint.DataLength = null; + checkpoint.Save(checkpointPath); + var before = handler.Requests.Count; + + Sync(deltaPath, checkpointPath, outputPath); + + Assert.Contains("$deltatoken=D1", handler.Requests[before].RequestUri!.ToString()); + Assert.Equal( + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(outputPath)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The same failure with no output from an earlier run. Here the checkpoint IS stale in the + /// sense its warning means - the output is genuinely missing - but the temp still holds the + /// items, so adoption creates the output from it rather than the checkpoint being dropped. + /// + [Fact] + public void A_first_run_that_dies_midway_still_recovers_its_pages() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-recovery-{Guid.NewGuid():N}")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var outputPath = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + + InjectMock(handler); + try + { + Sync(deltaPath, checkpointPath, outputPath); + Assert.False(File.Exists(outputPath), "a fresh run promotes its temp only on success"); + Assert.True(File.Exists(checkpointPath)); + + Sync(deltaPath, checkpointPath, outputPath); + Assert.Equal( + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(outputPath)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// -Latest means "baseline from now". After an interrupted enumeration that is the same + /// catastrophe the code already guards against after a state discard: the collected items + /// are abandoned and the token jumps past every change that preceded it, with no way back. + /// Without delta state on disk none of the existing guards fire, so the checkpoint is the + /// only signal that this is not a fresh run. + /// + [Fact] + public void Latest_is_ignored_while_a_resume_checkpoint_exists() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-recovery-{Guid.NewGuid():N}")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var outputPath = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 1, page 1 + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // run 1, page 2 + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // ... and its retry + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); // run 2 resumes + + InjectMock(handler); + try + { + Sync(deltaPath, checkpointPath, outputPath); + Assert.True(File.Exists(checkpointPath)); + Assert.False(File.Exists(deltaPath), "run 1 never completed, so no token was saved"); + + using (var ps = Shell()) + { + ps.AddCommand("Sync-MgxDelta").AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath).AddParameter("CheckpointPath", checkpointPath) + .AddParameter("OutputFile", outputPath).AddParameter("Latest", true); + ps.Invoke(); + Assert.Contains(ps.Streams.Warning, w => w.Message.Contains("-Latest ignored")); + } + + // The interrupted enumeration finished instead of being baselined away. + Assert.Equal( + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(outputPath)); + var token = DeltaState.Load(deltaPath)!.DeltaLink; + Assert.Contains("$deltatoken=D2", token); + Assert.DoesNotContain("token=latest", token); + } + finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// A resume opens the output itself for append, and a denying ACL or read-only bit there + /// raises UnauthorizedAccessException - which does not derive from IOException, so a + /// handler catching only the latter turns an ordinary permission failure into an unhandled + /// error. It must surface the same way an unwritable path does everywhere else: as an + /// AccessDenied error record. + /// + [Fact] + public void A_denied_output_ends_the_resume_as_an_error_record_not_an_unhandled_failure() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-recovery-{Guid.NewGuid():N}")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var outputPath = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, Baseline); // run 1 + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 2, page 1 + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // run 2 dies + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); // run 3 resumes + + InjectMock(handler); + try + { + Sync(deltaPath, checkpointPath, outputPath); + Sync(deltaPath, checkpointPath, outputPath); + + // The shape a promoted cancellation leaves: the checkpoint points at the output + // itself, so the resume opens that file directly rather than merging a temp over + // it - a merge's rename would replace the file, permissions and all. + var checkpoint = PaginationCheckpoint.Load(checkpointPath)!; + checkpoint.TempFile = null; + checkpoint.DataLength = new FileInfo(outputPath).Length; + checkpoint.Save(checkpointPath); + + if (OperatingSystem.IsWindows()) + File.SetAttributes(outputPath, File.GetAttributes(outputPath) | FileAttributes.ReadOnly); + else + File.SetUnixFileMode(outputPath, UnixFileMode.UserRead); + + using var ps = Shell(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("OutputFile", outputPath); + var escaped = Record.Exception(() => ps.Invoke()); + + Assert.Null(escaped); + Assert.Contains(ps.Streams.Error, + e => e.FullyQualifiedErrorId.StartsWith("AccessDenied", StringComparison.Ordinal)); + } + finally + { + CleanupMock(); + try + { + if (OperatingSystem.IsWindows()) + File.SetAttributes(outputPath, File.GetAttributes(outputPath) & ~FileAttributes.ReadOnly); + else + File.SetUnixFileMode(outputPath, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } + catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A run that dies before any page boundary leaves no checkpoint, so there is nothing to + /// resume and nothing the temp is needed for. It must not be left behind: a stale temp is + /// adoptable by a later run whose checkpoint happens to match the output name. + /// + [Fact] + public void A_run_that_dies_with_no_checkpoint_leaves_no_temp() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-recovery-{Guid.NewGuid():N}")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var outputPath = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + + InjectMock(handler); + try + { + Sync(deltaPath, checkpointPath, outputPath); + Assert.False(File.Exists(checkpointPath), "no page completed, so no position to save"); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.False(File.Exists(outputPath)); + } + finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The escape hatch the ignored--Latest warning names has to work: -FullSync discards the + /// enumeration outright, checkpoint included, so there is nothing left to protect and + /// re-baselining from now is exactly what was asked for. + /// + [Fact] + public void FullSync_with_Latest_still_baselines_from_now_despite_a_checkpoint() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-recovery-{Guid.NewGuid():N}")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var outputPath = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.OK, + """{"value":[],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=FROMNOW"}"""); + + InjectMock(handler); + try + { + Sync(deltaPath, checkpointPath, outputPath); + Assert.True(File.Exists(checkpointPath)); + + using (var ps = Shell()) + { + ps.AddCommand("Sync-MgxDelta").AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath).AddParameter("CheckpointPath", checkpointPath) + .AddParameter("OutputFile", outputPath) + .AddParameter("FullSync", true).AddParameter("Latest", true); + ps.Invoke(); + Assert.DoesNotContain(ps.Streams.Warning, w => w.Message.Contains("-Latest ignored")); + } + + Assert.Contains("$deltatoken=FROMNOW", DeltaState.Load(deltaPath)!.DeltaLink); + Assert.Contains("deltatoken=latest", handler.Requests[^1].RequestUri!.ToString()); + } + finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// A temp file carries no identity: TryAdoptOrphanedTemp globs "{output}.*.tmp", takes the + /// newest, and checks only that it has enough lines. The Resource check added in 48ffe87 + /// validates the CHECKPOINT, so a CURRENT checkpoint plus a temp orphaned by an unrelated + /// run still merges that run's rows into a healthy output - and one success makes them + /// permanent. -FullSync and the -Property/-Filter/-Prefer discards all leave such orphans. + /// A fresh run has nothing to resume, so every leftover temp is an orphan and must go. + /// + [Fact] + public void A_fresh_run_clears_temps_orphaned_by_an_earlier_enumeration() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-recovery-{Guid.NewGuid():N}")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var outputPath = Path.Combine(dir, "out.jsonl"); + + // An orphan from an enumeration that no longer exists, long enough to satisfy any + // later checkpoint's line count. + var foreign = $"{outputPath}.deadbeef.tmp"; + File.WriteAllLines(foreign, Enumerable.Range(0, 50).Select(i => $"{{\"id\":\"foreign{i}\"}}")); + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, Baseline); + + InjectMock(handler); + try + { + Sync(deltaPath, checkpointPath, outputPath); + + Assert.False(File.Exists(foreign), "the orphan must not survive a fresh run"); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + var lines = File.ReadAllLines(outputPath); + Assert.Equal(["{\"id\":\"a1\"}", "{\"id\":\"a2\"}"], lines); + Assert.DoesNotContain(lines, l => l.Contains("foreign")); + } + finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// Resuming from a mid-page checkpoint must land exactly on the seam. PageItemsAlreadyWritten + /// counts every item of the in-flight page that is already in the output, including the ones + /// an earlier resume skipped rather than wrote, so the count and the file agree. + /// + [Fact] + public void Resuming_from_a_midpage_checkpoint_repeats_no_items() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-recovery-{Guid.NewGuid():N}")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var outputPath = Path.Combine(dir, "out.jsonl"); + var page2Url = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=P2"; + + // Stopped at totalProcessed = 1000: all 999 of page 1, plus item 0 of page 2. + var written = Enumerable.Range(0, 999).Select(i => $"{{\"id\":\"p{i}\"}}") + .Concat(["{\"id\":\"q0\"}"]).ToArray(); + File.WriteAllLines(outputPath, written); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users/delta?$top=999", + NextLink = page2Url, + ItemsCollected = 1000, + PageItemsAlreadyWritten = 1, + TempFile = null, + DataLength = new FileInfo(outputPath).Length, + }.Save(checkpointPath); + + var items = Enumerable.Range(0, 999).Select(i => $"{{\"id\":\"q{i}\"}}"); + var page2 = "{\"value\":[" + string.Join(",", items) + + "],\"@odata.deltaLink\":\"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D9\"}"; + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, page2); + + InjectMock(handler); + try + { + Sync(deltaPath, checkpointPath, outputPath); + + var lines = File.ReadAllLines(outputPath); + Assert.Equal(999 + 999, lines.Length); + Assert.Equal(lines.Length, lines.Distinct().Count()); + Assert.Equal("{\"id\":\"q0\"}", lines[999]); + Assert.Equal("{\"id\":\"q1\"}", lines[1000]); + } + finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ExportCheckpointIntegrityTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ExportCheckpointIntegrityTests.cs new file mode 100644 index 0000000..af80c78 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/ExportCheckpointIntegrityTests.cs @@ -0,0 +1,517 @@ +using System.Management.Automation; +using System.Net; +using System.Reflection; +using System.Text; +using Mgx.Cmdlets.Base; +using Mgx.Engine.Http; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// An export writes a fresh run to a temp and moves it over the output, so a checkpoint from +/// an interrupted export counts items that are in the temp, not in the output. Recovery has to +/// know which file holds them: an output left over from an EARLIER export satisfies "the output +/// exists" while holding none of them. +/// +[Collection("Pipeline")] +public class ExportCheckpointIntegrityTests +{ + private const string Page1 = """ + {"value":[{"id":"u1"},{"id":"u2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2"} + """; + private const string Page2 = """ + {"value":[{"id":"u3"}]} + """; + private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + + private sealed class ScriptedHandler : HttpMessageHandler + { + private readonly Queue<(HttpStatusCode Status, string Body)> _steps = new(); + private readonly object _lock = new(); + public List Requests { get; } = []; + + public void Queue(HttpStatusCode status, string body) + { + lock (_lock) _steps.Enqueue((status, body)); + } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + (HttpStatusCode Status, string Body) step; + lock (_lock) + { + Requests.Add(request.RequestUri!.ToString()); + step = _steps.Count > 0 ? _steps.Dequeue() : (HttpStatusCode.InternalServerError, ServerError); + } + return Task.FromResult(new HttpResponseMessage(step.Status) + { + RequestMessage = request, + Content = new StringContent(step.Body, Encoding.UTF8, "application/json") + }); + } + } + + private static void InjectMock(HttpMessageHandler handler) + { + ResiliencePipelineFactory.Reset(); + var t = typeof(MgxCmdletBase); + t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)! + .SetValue(null, new HttpClient(handler)); + t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)! + .SetValue(null, MgxCmdletBase.BuildAuthFingerprint( + new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); + t.GetField("s_ownsHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, false); + t.GetField("s_graphEndpoint", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! + .SetValue(null, "https://graph.microsoft.com"); + t.GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! + .SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + ResiliencePipelineFactory.Reset(); + } + + private static void CleanupMock() + { + var t = typeof(MgxCmdletBase); + t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); + t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); + t.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); + ResiliencePipelineFactory.Reset(); + } + + private static long? Export(string outputPath, string? checkpointPath, bool all, int top = 0) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + var cmd = ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", outputPath); + if (checkpointPath != null) cmd.AddParameter("CheckpointPath", checkpointPath); + if (all) cmd.AddParameter("All"); + if (top > 0) cmd.AddParameter("Top", top); + try + { + foreach (var r in ps.Invoke()) + if (r?.BaseObject is Mgx.Cmdlets.Models.MgxExportResult summary) + return summary.ItemCount; + } + catch (CmdletInvocationException) { } + return null; + } + + private static string[] Lines(string path) => File.Exists(path) ? File.ReadAllLines(path) : []; + + private static string NewDir() => + Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-export-ckpt-{Guid.NewGuid():N}")).FullName; + + /// + /// A narrow export, then a full one that dies, then the documented resume. The dead run's + /// first page went to a temp and its checkpoint counted those items; the output still holds + /// only the narrow export's. Resuming has to not treat that output as though it already + /// held them. + /// + [Fact] + public void A_resume_does_not_count_an_earlier_exports_output_as_this_ones_items() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + InjectMock(handler); + + // Run 1: a one-item probe export. Completes, no checkpoint. + handler.Queue(HttpStatusCode.OK, Page1); + Assert.Equal(1, Export(output, checkpointPath: null, all: false, top: 1)); + Assert.Equal(["{\"id\":\"u1\"}"], Lines(output)); + + // Run 2: the full export. Page 1 is written and checkpointed; page 2 fails. + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.InternalServerError, ServerError); + Export(output, checkpoint, all: true); + + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.Equal(2, cp!.ItemsCollected); // it counted u1 and u2... + Assert.Equal(["{\"id\":\"u1\"}"], Lines(output)); // ...which are not in the output + + // Run 3: the documented resume. Only the page the dead run never reached. + handler.Queue(HttpStatusCode.OK, Page2); + var reported = Export(output, checkpoint, all: true); + + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], Lines(output)); + Assert.Equal(3, reported); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + CleanupMock(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A fresh checkpointed export that dies on a transient error. The checkpoint survives and + /// counts items that exist only in the temp, so the temp has to survive with it - deleting + /// it leaves the checkpoint naming a missing file, and the next run starts the export over. + /// A kill or a Ctrl-C already kept the data; a handled error must not be the one way to + /// lose the position. + /// + [Fact] + public void A_transient_failure_keeps_the_temp_the_checkpoint_names() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + InjectMock(handler); + + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.InternalServerError, ServerError); + Export(output, checkpoint, all: true); + + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.NotNull(cp!.TempFile); + Assert.True(File.Exists(Path.Combine(dir, cp.TempFile!)), + "the checkpoint names a temp that is no longer on disk"); + } + finally + { + CleanupMock(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And the run after that failure resumes: it promotes the kept temp, fetches the page the + /// checkpoint recorded rather than the first one, and ends with every item exactly once. + /// + [Fact] + public void An_export_resumes_after_a_transient_failure_rather_than_starting_over() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + InjectMock(handler); + + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.InternalServerError, ServerError); + Export(output, checkpoint, all: true); + + handler.Queue(HttpStatusCode.OK, Page2); + var before = handler.Requests.Count; + var reported = Export(output, checkpoint, all: true); + + Assert.Contains("skiptoken=P2", handler.Requests[before]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], Lines(output)); + Assert.Equal(3, reported); + Assert.False(File.Exists(checkpoint)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + CleanupMock(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The ordinary re-baseline: the same export run again over its own previous output. A + /// resume must not leave the previous run's rows in front of this run's. + /// + [Fact] + public void A_resume_replaces_the_previous_export_rather_than_appending_to_it() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + InjectMock(handler); + + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + Assert.Equal(3, Export(output, checkpointPath: null, all: true)); + + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.InternalServerError, ServerError); + Export(output, checkpoint, all: true); + + handler.Queue(HttpStatusCode.OK, Page2); + var before = handler.Requests.Count; + var reported = Export(output, checkpoint, all: true); + + Assert.Contains("skiptoken=P2", handler.Requests[before]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], Lines(output)); + Assert.Equal(3, reported); + } + finally + { + CleanupMock(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A surviving temp is adopted rather than abandoned, even though an output from an earlier + /// export is sitting next to it. The temp is what a hard kill leaves - the process dies + /// before the cleanup that an ordinary failure runs - so it is staged directly here. + /// + [Fact] + public void A_surviving_temp_is_promoted_over_an_earlier_exports_output() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + InjectMock(handler); + + handler.Queue(HttpStatusCode.OK, Page1); + Assert.Equal(1, Export(output, checkpointPath: null, all: false, top: 1)); + + // What a kill -9 leaves partway through a fresh checkpointed run. + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + var bytes = "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"; + File.WriteAllText(temp, bytes); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = Path.GetFileName(temp), + DataLength = new FileInfo(temp).Length + }.Save(checkpoint); + + handler.Queue(HttpStatusCode.OK, Page2); + Export(output, checkpoint, all: true); + + // The temp's items replaced the earlier export's output; they are not behind it. + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], Lines(output)); + Assert.False(File.Exists(temp)); + } + finally + { + CleanupMock(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A checkpoint from a release that recorded neither the temp name nor a length cannot say + /// where its items are, and both shapes reach this code: a run that was appending, whose + /// items are in the output, and a fresh run killed mid-flight, whose items are in a temp + /// while the output still holds a PREVIOUS export. Assuming the first appended the rest of + /// the enumeration onto the earlier export. + /// + [Fact] + public void A_checkpoint_that_cannot_say_where_its_items_are_exports_again_rather_than_appending() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + InjectMock(handler); + + // A previous export completed and left its output behind. + File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n{\"id\":\"old3\"}\n"); + // A later fresh run was killed: its items are in a temp, not in that output. + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + // The shape a pre-2.1.0 release wrote: no temp name, no length. + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0 + }.Save(checkpoint); + + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + var reported = Export(output, checkpoint, all: true); + + // Replaced, not appended: the previous export's rows are gone and each id appears once. + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], Lines(output)); + Assert.Equal(3, reported); + Assert.DoesNotContain(Lines(output), l => l.Contains("old")); + } + finally + { + CleanupMock(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same shape as the delta case: an export resumed partway into a page has its first + /// items dropped by the iterator before it sees them, so a checkpoint saved while still on + /// that page has to count them too. Counting only what this run wrote makes the next resume + /// skip too few and write the difference twice. + /// + [Fact] + public void An_export_resumed_into_a_page_does_not_duplicate_it_on_a_second_interruption() + { + const string RefusedLink = "https://not-graph.example.com/v1.0/users?$skiptoken=P2"; + const string Page2Link = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2"; + + static string Item(int i) => $"{{\"id\":\"p{i}\"}}"; + static string BigPage(string nextLink) => + $"{{\"value\":[{string.Join(",", Enumerable.Range(0, 1000).Select(Item))}]," + + $"\"@odata.nextLink\":\"{nextLink}\"}}"; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + InjectMock(handler); + + // One real run, only so the checkpoint on disk carries the exact URL the cmdlet + // builds for these parameters. + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.InternalServerError, ServerError); + Export(output, checkpoint, all: true); + var resource = PaginationCheckpoint.Load(checkpoint)!.Resource; + foreach (var t in Directory.GetFiles(dir, "out.jsonl.*.tmp")) File.Delete(t); + + // Ctrl-C 500 items into the first page: that run promoted what it had written and + // recorded the output, 500 items in, all 500 from the page in flight. + File.WriteAllLines(output, Enumerable.Range(0, 500).Select(Item)); + new PaginationCheckpoint + { + Resource = resource, + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P1", + ItemsCollected = 500, + PageItemsAlreadyWritten = 500, + TempFile = null, + DataLength = new FileInfo(output).Length, + }.Save(checkpoint); + + // The resumed run writes the other 500 items, saves the mid-page checkpoint item + // 1000 triggers, and is then refused the page's nextLink - so it dies before the + // page boundary and that mid-page save is what survives. + handler.Queue(HttpStatusCode.OK, BigPage(RefusedLink)); + Export(output, checkpoint, all: true); + + Assert.Equal(1000, Lines(output).Length); + var mid = PaginationCheckpoint.Load(checkpoint)!; + Assert.Equal(1000, mid.ItemsCollected); + Assert.Equal(1000, mid.PageItemsAlreadyWritten); + + handler.Queue(HttpStatusCode.OK, BigPage(Page2Link)); + handler.Queue(HttpStatusCode.OK, """{"value":[{"id":"tail"}]}"""); + Export(output, checkpoint, all: true); + + var lines = Lines(output); + Assert.Equal(lines.Length, lines.Distinct().Count()); + Assert.Equal(1001, lines.Length); + } + finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// A checkpoint is untrusted input once it is on disk. A negative recorded length reached + /// SetLength, which rejects it with an exception the recovery path does not catch - so the + /// run died naming neither the checkpoint nor the file, and left the checkpoint behind to + /// fail the same way on every retry. + /// + [Fact] + public void A_checkpoint_recording_a_negative_length_does_not_wedge_the_export() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + InjectMock(handler); + + File.WriteAllText(output, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + DataLength = -1 + }.Save(checkpoint); + + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + var reported = Export(output, checkpoint, all: true); + + // The unusable checkpoint is discarded and the export runs from the beginning. + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], Lines(output)); + Assert.Equal(3, reported); + Assert.False(File.Exists(checkpoint)); + } + finally + { + CleanupMock(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A fresh run sweeps leftover temps, and "leftover" is an assumption: a second export + /// running against the same output owns a file matching the same glob right now. Windows + /// refuses to delete a file held open and so declined by accident; Unix deletes it, and the + /// other run keeps writing into an unlinked inode and loses everything it fetched. + /// + [Fact] + public void A_fresh_export_does_not_delete_a_temp_another_run_is_writing() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var live = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + try + { + var handler = new ScriptedHandler(); + InjectMock(handler); + + // Another export holds its temp open, exactly as StreamWriter does. + using (var held = new FileStream(live, FileMode.Create, FileAccess.Write, FileShare.Read)) + { + var bytes = System.Text.Encoding.UTF8.GetBytes("{\"id\":\"inflight\"}\n"); + held.Write(bytes, 0, bytes.Length); + held.Flush(); + + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + Export(output, checkpointPath: null, all: true); + + Assert.True(File.Exists(live), "the other run's temp was deleted out from under it"); + } + + Assert.Equal("{\"id\":\"inflight\"}", File.ReadAllLines(live).Single()); + } + finally + { + CleanupMock(); + try { Directory.Delete(dir, true); } catch { } + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ResilienceWrapTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ResilienceWrapTests.cs new file mode 100644 index 0000000..5e6695b --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/ResilienceWrapTests.cs @@ -0,0 +1,113 @@ +using System.Reflection; +using Mgx.Cmdlets.Cmdlets.Configuration; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// Enable-MgxResilience swaps the SDK's HttpClient for a wrapper that bridges to the +/// original. Invoke-MgGraphRequest resolves a relative -Uri against the active client's +/// BaseAddress before any handler runs, so the wrapper must carry the original client's +/// configuration or every relative-URI SDK call dies before reaching the wire. +/// +[Collection("Pipeline")] +public class ResilienceWrapTests +{ + private static HttpClient? BuildWrapper(HttpClient sdkClient, List warnings) + { + var build = typeof(EnableMgxResilience).GetMethod( + "BuildResilientSdkClient", BindingFlags.NonPublic | BindingFlags.Static)!; + try + { + return (HttpClient?)build.Invoke(null, [sdkClient, (Action)warnings.Add]); + } + finally + { + EnableMgxResilience.ActiveHandler = null; + ResiliencePipelineFactory.Reset(); + } + } + + [Fact] + public void The_wrapper_keeps_the_sdk_clients_base_address() + { + using var sdkClient = new HttpClient + { + BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") + }; + var warnings = new List(); + + var wrapper = BuildWrapper(sdkClient, warnings); + + Assert.NotNull(wrapper); + Assert.Empty(warnings); + Assert.Equal(sdkClient.BaseAddress, wrapper.BaseAddress); + } + + [Fact] + public void The_wrapper_keeps_the_sdk_clients_timeout() + { + using var sdkClient = new HttpClient { Timeout = TimeSpan.FromSeconds(123) }; + var warnings = new List(); + + var wrapper = BuildWrapper(sdkClient, warnings); + + Assert.NotNull(wrapper); + Assert.Equal(sdkClient.Timeout, wrapper.Timeout); + } + + /// Counts what the wire actually saw, and what it answered with. + private sealed class ThrottleThenOk : HttpMessageHandler + { + private readonly int _throttles; + public int Calls { get; private set; } + public ThrottleThenOk(int throttles) => _throttles = throttles; + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Calls++; + if (Calls <= _throttles) + { + var r = new HttpResponseMessage((System.Net.HttpStatusCode)429) { RequestMessage = request }; + r.Headers.TryAddWithoutValidation("Retry-After", "0"); + return Task.FromResult(r); + } + return Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent("{\"value\":[]}", System.Text.Encoding.UTF8, "application/json") + }); + } + } + + /// + /// A 429 answered through the bridge must be visible to mgx: the adaptive pacer learns from + /// throttles, and Get-MgxTelemetry reports them. + /// + /// Scope: this covers the BRIDGE, with the wire directly beneath it. A live session also has + /// the SDK's own RetryHandler inside the wrapped chain, which may answer a 429 before the + /// outer pipeline ever sees it - that is not exercised here, because Kiota's handlers are + /// only constructed by a real Graph client. What this pins is that the bridge itself does + /// not swallow a throttle on the way past. + /// + [Fact] + public void A_throttle_through_the_wrapper_is_visible_to_mgx() + { + var wire = new ThrottleThenOk(throttles: 1); + using var sdkClient = new HttpClient(wire) { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + var warnings = new List(); + + MgxTelemetryCollector.Current.Reset(); + var wrapper = BuildWrapper(sdkClient, warnings); + Assert.NotNull(wrapper); + + using var resp = wrapper!.GetAsync("users").GetAwaiter().GetResult(); + + var snapshot = MgxTelemetryCollector.Current.GetSummary(); + Assert.True(resp.IsSuccessStatusCode, $"final status was {(int)resp.StatusCode}"); + Assert.Equal(2, wire.Calls); // the 429 and the retry both hit the wire + Assert.True(snapshot.ThrottleRetries > 0, + $"mgx recorded {snapshot.ThrottleRetries} throttle retries for a 429 the wire served"); + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/SetOptionGuardTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/SetOptionGuardTests.cs new file mode 100644 index 0000000..0569e27 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/SetOptionGuardTests.cs @@ -0,0 +1,85 @@ +using System.Management.Automation; +using Mgx.Cmdlets.Base; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// Set-MgxOption rebuilds the resilience pipeline, which discards circuit-breaker failure +/// history. It guards against doing that for nothing - but the guard counted every bound +/// parameter, and PowerShell binds the common ones too, so -Verbose alone looked like a change. +/// +[Collection("Pipeline")] +public class SetOptionGuardTests +{ + private static PowerShell Shell() + { + var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Configuration.SetMgxOption).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + return ps; + } + + [Theory] + [InlineData("Verbose")] + [InlineData("ErrorAction")] + public void A_common_parameter_alone_is_not_a_change(string parameter) + { + using var ps = Shell(); + var cmd = ps.AddCommand("Set-MgxOption").AddParameter("Verbose", true); + if (parameter == "ErrorAction") cmd.AddParameter("ErrorAction", ActionPreference.Continue); + ps.Invoke(); + + var said = string.Join(" ", ps.Streams.Verbose.Select(v => v.Message)); + Assert.Contains("Options unchanged", said); + Assert.DoesNotContain("Mgx options updated", said); + } + + [Fact] + public void A_real_parameter_still_updates() + { + using var ps = Shell(); + ps.AddCommand("Set-MgxOption") + .AddParameter("RateLimitPerSecond", 42) + .AddParameter("Verbose", true); + ps.Invoke(); + + var said = string.Join(" ", ps.Streams.Verbose.Select(v => v.Message)); + Assert.Contains("Mgx options updated", said); + + // put it back + using var reset = Shell(); + reset.AddCommand("Set-MgxOption").AddParameter("Reset", true); + reset.Invoke(); + } + + /// + /// -NoRateLimit turns off batch item pacing too. That is a second mechanism, aimed at Graph's + /// server-side write throttle rather than the client-side limiter, and it has its own + /// documented off switch - so a caller setting both should be told which one won. + /// + [Fact] + public void Setting_NoRateLimit_beside_BatchItemsPerSecond_says_which_one_wins() + { + try + { + using var ps = Shell(); + ps.AddCommand("Set-MgxOption") + .AddParameter("NoRateLimit", true) + .AddParameter("BatchItemsPerSecond", 20); + ps.Invoke(); + + var warned = string.Join(" ", ps.Streams.Warning.Select(w => w.Message)); + Assert.Contains("has no effect", warned); + Assert.Contains("BatchItemsPerSecond", warned); + } + finally + { + using var reset = Shell(); + reset.AddCommand("Set-MgxOption").AddParameter("Reset", true); + reset.Invoke(); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/TopWithAllTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/TopWithAllTests.cs new file mode 100644 index 0000000..53111d1 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/TopWithAllTests.cs @@ -0,0 +1,166 @@ +using System.Management.Automation; +using System.Net; +using System.Reflection; +using System.Text; +using Mgx.Cmdlets.Base; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// -Top is documented as capping the total result set. -All follows every nextLink. Together +/// they meant -Top was discarded: the run walked the whole collection and handed back all of +/// it. Silent, and expensive twice over, because -Top also sets the page size - asking a +/// 100,000-object tenant for 150 items paged it 150 at a time to return all 100,000. +/// +[Collection("Pipeline")] +public class TopWithAllTests +{ + /// + /// Pages of items carrying a nextLink, up to a hard ceiling far + /// above what any test asks for. The ceiling exists only so an uncapped run ends: without + /// the fix -All ignores -Top and keeps asking, and a test that hangs reports nothing. + /// + private sealed class PagedHandler(int pageSize, int maxPages) : HttpMessageHandler + { + public int RequestCount { get; private set; } + private int _next; + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + RequestCount++; + var items = string.Join(",", + Enumerable.Range(_next, pageSize).Select(i => $"{{\"id\":\"u{i}\"}}")); + _next += pageSize; + var link = RequestCount >= maxPages + ? "" + : $",\"@odata.nextLink\":\"https://graph.microsoft.com/v1.0/users?$skiptoken={_next}\""; + var body = $"{{\"value\":[{items}]{link}}}"; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } + + private static void InjectMock(HttpMessageHandler handler) + { + ResiliencePipelineFactory.Reset(); + var t = typeof(MgxCmdletBase); + t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)! + .SetValue(null, new HttpClient(handler)); + t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)! + .SetValue(null, MgxCmdletBase.BuildAuthFingerprint( + new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); + t.GetField("s_ownsHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, false); + t.GetField("s_graphEndpoint", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! + .SetValue(null, "https://graph.microsoft.com"); + t.GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! + .SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + ResiliencePipelineFactory.Reset(); + } + + private static void CleanupMock() + { + var t = typeof(MgxCmdletBase); + t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); + t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); + t.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); + ResiliencePipelineFactory.Reset(); + } + + private static (int Count, int Requests) Enumerate(int top, bool all, int pageSize) + { + var handler = new PagedHandler(pageSize, maxPages: 40); + InjectMock(handler); + try + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + var cmd = ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("Top", top) + .AddParameter("PageSize", pageSize) + .AddParameter("WarningAction", ActionPreference.SilentlyContinue); + if (all) cmd.AddParameter("All"); + var output = ps.Invoke(); + return (output.Count, handler.RequestCount); + } + finally { CleanupMock(); } + } + + [Fact] + public void All_stops_at_Top_rather_than_walking_the_whole_collection() + { + // Forty pages of 50 are available; the caller asked for 150. Anything above 150 means + // the cap was discarded and the run kept walking. + var (count, _) = Enumerate(top: 150, all: true, pageSize: 50); + + Assert.Equal(150, count); + } + + [Fact] + public void All_with_Top_stops_requesting_once_it_has_enough() + { + // Not just "returns the right number": the cost is the point. Three pages of 50 covers + // 150, and a fourth request would be work done for output the caller never sees. + var (_, requests) = Enumerate(top: 150, all: true, pageSize: 50); + + Assert.Equal(3, requests); + } + + [Fact] + public void Top_alone_still_caps_the_total() + { + var (count, _) = Enumerate(top: 75, all: false, pageSize: 50); + + Assert.Equal(75, count); + } + + [Fact] + public void Export_stops_at_Top_rather_than_walking_the_whole_collection() + { + // The sibling of the Invoke-MgxRequest case. Both cmdlets document -Top as a total + // maximum, so they must not disagree about what -All -Top means. + var handler = new PagedHandler(pageSize: 50, maxPages: 40); + InjectMock(handler); + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-topall-{Guid.NewGuid():N}")).FullName; + var output = Path.Combine(dir, "out.jsonl"); + try + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", output) + .AddParameter("All") + .AddParameter("Top", 150) + .AddParameter("PageSize", 50) + .AddParameter("WarningAction", ActionPreference.SilentlyContinue); + ps.Invoke(); + + Assert.Equal(150, File.ReadAllLines(output).Length); + } + finally + { + CleanupMock(); + try { Directory.Delete(dir, true); } catch { } + } + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/AdaptivePacingTests.cs b/tests/Mgx.IntegrationTests/Engine/AdaptivePacingTests.cs deleted file mode 100644 index 7d3c27c..0000000 --- a/tests/Mgx.IntegrationTests/Engine/AdaptivePacingTests.cs +++ /dev/null @@ -1,110 +0,0 @@ -using System.Diagnostics; -using Mgx.Engine.Http; - -namespace Mgx.IntegrationTests.Engine; - -/// -/// Batch write pacing is halved whenever Graph returns 429, and the reduced rate is persisted -/// across Invoke-MgxBatchRequest calls. Without a way back up, one throttling episode slowed -/// every later batch for the lifetime of the process, so recovery is the point of these tests. -/// -public class AdaptivePacingTests -{ - [Theory] - [InlineData(20, 10)] - [InlineData(10, 5)] - [InlineData(5, 2)] - public void ReduceRate_halves_on_throttling(int rate, int expected) - { - Assert.Equal(expected, GraphBatchClient.ReduceRate(rate)); - } - - [Theory] - [InlineData(4)] - [InlineData(2)] - [InlineData(1)] - public void ReduceRate_never_falls_below_the_floor(int rate) - { - // Halving without a floor would eventually reach zero, which disables pacing - // entirely - the opposite of what a throttled tenant needs. - Assert.True(GraphBatchClient.ReduceRate(rate) >= 2); - } - - [Fact] - public void RecoverRate_climbs_additively_not_by_doubling() - { - // Additive increase against multiplicative decrease: the rate approaches the - // throttling threshold instead of leaping back onto it. - Assert.Equal(12, GraphBatchClient.RecoverRate(10, 20)); - } - - [Fact] - public void RecoverRate_always_makes_progress_at_small_configured_rates() - { - // configuredRate / 10 rounds to zero below 10 items/sec, which would stall recovery. - Assert.Equal(3, GraphBatchClient.RecoverRate(2, 5)); - } - - [Fact] - public void RecoverRate_stops_at_the_configured_rate() - { - Assert.Equal(20, GraphBatchClient.RecoverRate(19, 20)); - Assert.Equal(20, GraphBatchClient.RecoverRate(20, 20)); - } - - [Fact] - public void Reduced_rate_climbs_all_the_way_back_to_the_configured_rate() - { - // The regression this whole change is about: from halved, repeated clean chunks - // must terminate at the configured rate rather than converging below it. - const int configured = 20; - var rate = GraphBatchClient.ReduceRate(configured); - Assert.True(rate < configured); - - for (var i = 0; i < 100 && rate < configured; i++) - rate = GraphBatchClient.RecoverRate(rate, configured); - - Assert.Equal(configured, rate); - } - - // Both timestamps are synthesized rather than anchored to Stopwatch.GetTimestamp(). - private static long Ticks(double seconds) => (long)(seconds * Stopwatch.Frequency); - - private static long RecoveryWindowTicks => - Ticks(GraphBatchClient.AdaptiveRecoveryWindow.TotalSeconds); - - [Fact] - public void Adapted_rate_expires_after_a_long_quiet_period() - { - const long throttledAt = 1; - - Assert.True(GraphBatchClient.AdaptedRateHasExpired( - throttledAt, throttledAt + RecoveryWindowTicks + Ticks(60))); - } - - [Fact] - public void Adapted_rate_survives_a_recent_throttle() - { - const long throttledAt = 1; - - Assert.False(GraphBatchClient.AdaptedRateHasExpired(throttledAt, throttledAt + Ticks(5))); - } - - [Fact] - public void Adapted_rate_survives_right_up_to_the_window_boundary() - { - const long throttledAt = 1; - - Assert.False(GraphBatchClient.AdaptedRateHasExpired( - throttledAt, throttledAt + RecoveryWindowTicks)); - Assert.True(GraphBatchClient.AdaptedRateHasExpired( - throttledAt, throttledAt + RecoveryWindowTicks + 1)); - } - - [Fact] - public void Never_expires_when_no_throttle_was_ever_recorded() - { - // Zero means "no 429 seen yet", not "throttled at boot". - Assert.False(GraphBatchClient.AdaptedRateHasExpired(0, RecoveryWindowTicks * 10)); - } -} diff --git a/tests/Mgx.IntegrationTests/BatchEdgeCaseTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/BatchEdgeCaseTests.cs similarity index 100% rename from tests/Mgx.IntegrationTests/BatchEdgeCaseTests.cs rename to tests/Mgx.IntegrationTests/Engine/Batch/BatchEdgeCaseTests.cs diff --git a/tests/Mgx.IntegrationTests/BatchWriteTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/BatchWriteTests.cs similarity index 100% rename from tests/Mgx.IntegrationTests/BatchWriteTests.cs rename to tests/Mgx.IntegrationTests/Engine/Batch/BatchWriteTests.cs diff --git a/tests/Mgx.IntegrationTests/BulkWriteTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/BulkWriteTests.cs similarity index 100% rename from tests/Mgx.IntegrationTests/BulkWriteTests.cs rename to tests/Mgx.IntegrationTests/Engine/Batch/BulkWriteTests.cs diff --git a/tests/Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs new file mode 100644 index 0000000..5e580e8 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs @@ -0,0 +1,143 @@ +using System.Net; +using Mgx.Engine.Http; +using Mgx.Engine.Models; + +namespace Mgx.IntegrationTests; + +/// +/// A batch larger than Graph's 20-request limit is split into chunks and sent one after another. +/// If a later chunk's POST fails outright, the earlier chunks have already been applied - the +/// writes happened, on the server. Losing their results loses the only record of what landed, +/// which is exactly what a caller needs in order to know what to do next. +/// +[Collection("Pipeline")] +public class PartialBatchResultTests +{ + private static string BatchResponse(int firstId, int count, int status) => + "{ \"responses\": [" + string.Join(",", Enumerable.Range(firstId, count).Select(i => + $"{{ \"id\": \"{i}\", \"status\": {status}, \"body\": {{ \"id\": \"u{i}\" }} }}")) + "] }"; + + /// First POST answers a full chunk; every later POST is a hard failure. + private sealed class FirstChunkOnlyHandler : HttpMessageHandler + { + public int Posts { get; private set; } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Posts++; + if (Posts == 1) + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(BatchResponse(1, 20, 200), + System.Text.Encoding.UTF8, "application/json") + }); + } + // Not a per-item failure inside a 200 envelope - the POST itself fails. + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent( + "{\"error\":{\"code\":\"BadRequest\",\"message\":\"batch rejected\"}}", + System.Text.Encoding.UTF8, "application/json") + }); + } + } + + [Fact] + public async Task A_failed_chunk_does_not_discard_the_chunks_that_already_succeeded() + { + ResiliencePipelineFactory.Reset(); + var wire = new FirstChunkOnlyHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client); + + // 30 operations = two chunks. The first is applied; the second is refused. + var ops = Enumerable.Range(1, 30) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + // The 20 writes in chunk one happened. Their outcomes must survive the second chunk's + // failure, or the caller cannot tell an applied write from one that never ran. + var known = result.Results.Count(r => r.Response is not null); + Assert.True(known >= 20, + $"only {known} of 30 operations came back; the first chunk's 20 results were discarded"); + } + + /// Counts requests; answers an empty batch envelope. + private sealed class CountingHandler : HttpMessageHandler + { + public int Requests { get; private set; } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Requests++; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(BatchResponse(1, 2, 200), + System.Text.Encoding.UTF8, "application/json") + }); + } + } + + /// + /// The help says of -WhatIf: "Shows what would happen if the cmdlet runs. The cmdlet is not + /// run." A read-only batch changes nothing on the server, but it still spends resource units, + /// can be throttled, and emits objects into the pipeline - so running it is not "not run", + /// and it is not what the caller asked for. + /// + [Fact] + public void WhatIf_does_not_send_a_read_only_batch() + { + ResiliencePipelineFactory.Reset(); + var wire = new CountingHandler(); + var t = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); + t.GetField("s_graphHttpClient", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)! + .SetValue(null, new HttpClient(wire)); + t.GetField("s_cachedAuthFingerprint", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)! + .SetValue(null, Mgx.Cmdlets.Base.MgxCmdletBase.BuildAuthFingerprint( + new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); + t.GetField("s_ownsHttpClient", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)! + .SetValue(null, false); + t.GetField("s_graphEndpoint", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.Public)! + .SetValue(null, "https://graph.microsoft.com"); + t.GetField("s_clientOptions", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.Public)! + .SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + ResiliencePipelineFactory.Reset(); + + try + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new[] { "/users/u1", "/users/u2" }) + .AddParameter("Method", "GET") + .AddParameter("WhatIf", true); + var output = ps.Invoke(); + + Assert.Equal(0, wire.Requests); + Assert.Empty(output); + } + finally + { + t.GetField("s_graphHttpClient", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!.SetValue(null, null); + t.GetField("s_cachedAuthFingerprint", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!.SetValue(null, null); + ResiliencePipelineFactory.Reset(); + } + } +} diff --git a/tests/Mgx.IntegrationTests/SpeedWinsTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/SpeedWinsTests.cs similarity index 100% rename from tests/Mgx.IntegrationTests/SpeedWinsTests.cs rename to tests/Mgx.IntegrationTests/Engine/Batch/SpeedWinsTests.cs diff --git a/tests/Mgx.IntegrationTests/Engine/ErrorBodyShapeTests.cs b/tests/Mgx.IntegrationTests/Engine/ErrorBodyShapeTests.cs new file mode 100644 index 0000000..e9214a1 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/ErrorBodyShapeTests.cs @@ -0,0 +1,73 @@ +using System.Net; +using Mgx.Engine.Models; + +namespace Mgx.IntegrationTests.Engine; + +/// +/// Error-body parsing must never throw. The constructor runs while a caller is already building +/// an exception, and 2.1's content path follows a redirect to SharePoint, OneDrive and CDN hosts +/// that are not Graph and do not emit the OData error envelope. +/// +public class ErrorBodyShapeTests +{ + [Theory] + [InlineData("""{"error":{"code":"itemNotFound","message":"Item not found"}}""")] // the normal shape + [InlineData("""{"error":"just a string"}""")] // error is not an object + [InlineData("""{"error":{"code":404,"message":"numeric code"}}""")] // code is not a string + [InlineData("""{"error":{"message":{"value":"nested"}}}""")] // message is not a string + [InlineData("""["an","array","root"]""")] // root is an array + [InlineData("\"a bare json string\"")] // root is a string + [InlineData("12345")] // root is a number + [InlineData("null")] // root is null + [InlineData("503 from a CDN")] // not JSON at all + [InlineData("")] // empty + public void Never_throws_whatever_the_body_shape(string body) + { + var ex = new GraphServiceException(HttpStatusCode.ServiceUnavailable, body); + Assert.False(string.IsNullOrWhiteSpace(ex.Message)); + } + + [Fact] + public void Still_extracts_a_graph_shaped_error() + { + var ex = new GraphServiceException( + HttpStatusCode.NotFound, + """{"error":{"code":"itemNotFound","message":"Item not found"}}"""); + Assert.Contains("itemNotFound", ex.Message); + Assert.Contains("Item not found", ex.Message); + } + + [Fact] + public void Extracts_a_message_nested_under_value() + { + var ex = new GraphServiceException( + HttpStatusCode.BadRequest, + """{"error":{"code":"badRequest","message":{"value":"the real text"}}}"""); + Assert.Contains("the real text", ex.Message); + } + + /// + /// The theory above only asserts the message is non-empty, and the backstop catch at the + /// bottom of FormatAndExtract returns the status line for anything that throws - so all ten + /// cases stay green with every ValueKind guard deleted. The guards are two independently + /// sufficient controls with the catch, and the theory pins only their disjunction. + /// Pin the guards themselves: a body odd in ONE place must still yield the parts that are + /// well formed, instead of falling back wholesale. + /// + [Fact] + public void A_body_odd_in_one_place_still_yields_the_parts_that_are_well_formed() + { + var ex = new GraphServiceException( + HttpStatusCode.ServiceUnavailable, + """{"error":{"code":404,"message":"numeric code"}}"""); + Assert.Contains("numeric code", ex.Message); + Assert.Null(ex.ErrorCode); + } + + [Fact] + public void Falls_back_to_the_status_line_when_the_shape_is_wrong() + { + var ex = new GraphServiceException(HttpStatusCode.ServiceUnavailable, """{"error":404}"""); + Assert.Contains("503", ex.Message); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Http/AdaptivePacingTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/AdaptivePacingTests.cs new file mode 100644 index 0000000..f9233df --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/AdaptivePacingTests.cs @@ -0,0 +1,169 @@ +using System.Diagnostics; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests.Engine; + +/// +/// The shared AIMD math (AdaptivePacing): rates are halved whenever Graph returns 429, and +/// the reduced rate persists across calls. Without a way back up, one throttling episode +/// slowed everything for the lifetime of the process, so recovery is the point of these +/// tests. Used by both GraphBatchClient (per-item pacing) and AdaptiveRequestPacer. +/// +public class AdaptivePacingTests +{ + [Theory] + [InlineData(20, 10)] + [InlineData(10, 5)] + [InlineData(5, 2)] + public void ReduceRate_halves_on_throttling(int rate, int expected) + { + Assert.Equal(expected, AdaptivePacing.ReduceRate(rate)); + } + + [Theory] + [InlineData(4)] + [InlineData(2)] + [InlineData(1)] + public void ReduceRate_never_falls_below_the_floor(int rate) + { + // Halving without a floor would eventually reach zero, which disables pacing + // entirely - the opposite of what a throttled tenant needs. + Assert.True(AdaptivePacing.ReduceRate(rate) >= AdaptivePacing.MinAdaptiveRate); + } + + [Fact] + public void RecoverRate_climbs_additively_not_by_doubling() + { + // Additive increase against multiplicative decrease: the rate approaches the + // throttling threshold instead of leaping back onto it. + Assert.Equal(12, AdaptivePacing.RecoverRate(10, 20)); + } + + [Fact] + public void RecoverRate_always_makes_progress_at_small_configured_rates() + { + // configuredRate / 10 rounds to zero below 10 items/sec, which would stall recovery. + Assert.Equal(3, AdaptivePacing.RecoverRate(2, 5)); + } + + [Fact] + public void RecoverRate_stops_at_the_configured_rate() + { + Assert.Equal(20, AdaptivePacing.RecoverRate(19, 20)); + Assert.Equal(20, AdaptivePacing.RecoverRate(20, 20)); + } + + [Fact] + public void Reduced_rate_climbs_all_the_way_back_to_the_configured_rate() + { + // The regression this whole change is about: from halved, repeated clean chunks + // must terminate at the configured rate rather than converging below it. + const int configured = 20; + var rate = AdaptivePacing.ReduceRate(configured); + Assert.True(rate < configured); + + for (var i = 0; i < 100 && rate < configured; i++) + rate = AdaptivePacing.RecoverRate(rate, configured); + + Assert.Equal(configured, rate); + } + + // Both timestamps are synthesized rather than anchored to Stopwatch.GetTimestamp(). + private static long Ticks(double seconds) => (long)(seconds * Stopwatch.Frequency); + + private static long RecoveryWindowTicks => + Ticks(AdaptivePacing.AdaptiveRecoveryWindow.TotalSeconds); + + [Fact] + public void Adapted_rate_expires_after_a_long_quiet_period() + { + const long throttledAt = 1; + + Assert.True(AdaptivePacing.AdaptedRateHasExpired( + throttledAt, throttledAt + RecoveryWindowTicks + Ticks(60))); + } + + [Fact] + public void Adapted_rate_survives_a_recent_throttle() + { + const long throttledAt = 1; + + Assert.False(AdaptivePacing.AdaptedRateHasExpired(throttledAt, throttledAt + Ticks(5))); + } + + [Fact] + public void Adapted_rate_survives_right_up_to_the_window_boundary() + { + const long throttledAt = 1; + + Assert.False(AdaptivePacing.AdaptedRateHasExpired( + throttledAt, throttledAt + RecoveryWindowTicks)); + Assert.True(AdaptivePacing.AdaptedRateHasExpired( + throttledAt, throttledAt + RecoveryWindowTicks + 1)); + } + + [Fact] + public void Never_expires_when_no_throttle_was_ever_recorded() + { + // Zero means "no 429 seen yet", not "throttled at boot". + Assert.False(AdaptivePacing.AdaptedRateHasExpired(0, RecoveryWindowTicks * 10)); + } + + // --- workload classifier --- + // Drive markers anywhere in the path win (a user's OneDrive lives under /users/{id}/drive), + // then non-directory service markers (a group calendar is Exchange-backed), then the first + // segment decides directory membership. Other is the safe default. The same classifier + // picks the -Latest token form: Drive => token=latest, everything else $deltatoken=latest. + + [Theory] + [InlineData("/users")] + [InlineData("/v1.0/users?$top=5")] + [InlineData("https://graph.microsoft.com/v1.0/groups/abc/members")] + [InlineData("/beta/servicePrincipals")] + [InlineData("/groups/delta?$deltatoken=latest")] + [InlineData("/me")] + [InlineData("/me/memberOf")] + [InlineData("/organization")] + [InlineData("/directoryObjects/getByIds")] + public void Classify_directory_workloads(string uri) => + Assert.Equal(WorkloadBucket.Directory, AdaptivePacing.Classify(uri)); + + [Theory] + [InlineData("/me/drive/root/delta")] + [InlineData("https://graph.microsoft.com/v1.0/drives/b!x/items/01ABC/content")] + [InlineData("/users/user@contoso.com/drive/root/children")] + [InlineData("/sites/contoso.sharepoint.com,guid/lists/Documents/items/delta")] + [InlineData("/groups/abc/drive/root/delta?token=latest")] + [InlineData("/shares/u!abc123/driveItem")] + public void Classify_drive_workloads(string uri) => + Assert.Equal(WorkloadBucket.Drive, AdaptivePacing.Classify(uri)); + + [Theory] + [InlineData("/users/user@contoso.com/messages")] // Exchange, despite /users root + [InlineData("/groups/abc/events")] // group calendar is Exchange-backed + [InlineData("/me/photo/$value")] + [InlineData("/teams/abc/channels/xyz/messages/delta")] + [InlineData("/deviceManagement/managedDevices")] + [InlineData("/mailFolders('inbox')/messages/delta")] + [InlineData("")] + [InlineData("not a uri at all")] + public void Classify_everything_else_lands_in_other(string uri) => + Assert.Equal(WorkloadBucket.Other, AdaptivePacing.Classify(uri)); + + [Fact] + public void Classify_null_is_other() => + Assert.Equal(WorkloadBucket.Other, AdaptivePacing.Classify(null)); + + [Theory] + [InlineData("/$batch")] + [InlineData("https://graph.microsoft.com/v1.0/$batch")] + [InlineData("/beta/$batch")] + public void Classify_batch_envelopes_get_their_own_bucket(string uri) + { + // A $batch envelope says nothing about the workloads inside it. Filing it under Other + // meant a batch 429 capped unrelated Exchange, Teams and Intune traffic to the throttled + // entry rate on evidence that had nothing to do with them, while the workload the batch + // actually addressed was left untouched - inverting the isolation the buckets exist for. + Assert.Equal(WorkloadBucket.Batch, AdaptivePacing.Classify(uri)); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Http/AdaptiveRequestPacerTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/AdaptiveRequestPacerTests.cs new file mode 100644 index 0000000..50da15d --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/AdaptiveRequestPacerTests.cs @@ -0,0 +1,576 @@ +using System.Diagnostics; +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests.Engine; + +/// +/// The request-level pacer: pure math (damping, caps, intervals) plus the stateful behaviors +/// - slow start on cold buckets, AIMD capping on 429, Retry-After propagation, bucket +/// isolation - and the two integration guarantees: both SendAsync paths are gated, and batch +/// outer POSTs are not. +/// +/// Stateful tests run inside PacerScope (which re-enables the gate the suite-wide +/// ModuleInitializer turned off) and in the Pipeline collection so nothing runs concurrently +/// with them against the process-static state. +/// +[Collection("Pipeline")] +public class AdaptiveRequestPacerTests +{ + /// Re-enables the pacer for one test; restores the suite default on dispose. + private sealed class PacerScope : IDisposable + { + public PacerScope() + { + AdaptiveRequestPacer.Reset(); + // Reset clears learned state only - it deliberately no longer reverts configuration, + // because doing so re-enabled pacing under a client built with NoAdaptivePacing. So + // the scope must restore defaults explicitly, or a test that configured a low + // ceiling or disabled pacing leaks that setting into every later test in the run. + AdaptiveRequestPacer.Configure(ResilientGraphClientOptions.Default); + AdaptiveRequestPacer.DisabledForTests = false; + } + + public void Dispose() + { + AdaptiveRequestPacer.Reset(); + AdaptiveRequestPacer.Configure(ResilientGraphClientOptions.Default); + AdaptiveRequestPacer.DisabledForTests = true; + } + } + + // --- pure math --- + + [Theory] + [InlineData(0, 0, 0)] + [InlineData(10, 0, 10)] + [InlineData(0, 8, 8)] + [InlineData(10, 8, 8)] + [InlineData(4, 16, 4)] + public void EffectiveRateCap_takes_the_stricter_active_cap(int adapted, int slowStart, int expected) => + Assert.Equal(expected, AdaptiveRequestPacer.EffectiveRateCap(adapted, slowStart)); + + [Theory] + [InlineData(790, 0)] // below the documented 0.8 emission floor: no damping + [InlineData(800, 0)] // ramp starts AT 0.8, so 0.8 itself is zero delay + [InlineData(1000, 1000)] // 1.0 = throttling begins: half the maximum + [InlineData(1200, 2000)] // 1.2 = full damping + [InlineData(1800, 2000)] // clamped above 1.2 + public void DampingDelay_ramps_linearly_from_documented_emission_floor(int perMille, long expectedMs) => + Assert.Equal(expectedMs, AdaptiveRequestPacer.DampingDelayMs(perMille, ageTicks: 0)); + + [Fact] + public void DampingDelay_ignores_stale_reports() + { + var staleTicks = (long)((AdaptiveRequestPacer.PercentageFreshness.TotalSeconds + 1) * Stopwatch.Frequency); + Assert.Equal(0, AdaptiveRequestPacer.DampingDelayMs(1200, staleTicks)); + } + + [Fact] + public void ComputeInterval_zero_when_nothing_is_active() => + Assert.Equal(0, AdaptiveRequestPacer.ComputeIntervalTicks(rateCap: 0, dampingDelayMs: 0)); + + [Fact] + public void ComputeInterval_takes_the_stricter_of_cap_and_damping() + { + // 10 rps = 100ms interval; 500ms damping is stricter. + var ticks = AdaptiveRequestPacer.ComputeIntervalTicks(rateCap: 10, dampingDelayMs: 500); + Assert.Equal(500 * Stopwatch.Frequency / 1000, ticks); + + // 2 rps = 500ms interval; 100ms damping is weaker. + ticks = AdaptiveRequestPacer.ComputeIntervalTicks(rateCap: 2, dampingDelayMs: 100); + Assert.Equal(Stopwatch.Frequency / 2, ticks); + } + + // --- stateful behavior --- + + [Fact] + public async Task First_request_of_a_cold_bucket_is_never_delayed() + { + using var scope = new PacerScope(); + + var waited = await AdaptiveRequestPacer.WaitAsync(WorkloadBucket.Directory, CancellationToken.None); + + Assert.Equal(0, waited); + // ...but slow start is now armed for the requests that follow. + Assert.Equal(AdaptiveRequestPacer.SlowStartInitialRate, + AdaptiveRequestPacer.GetSlowStartRate(WorkloadBucket.Directory)); + } + + [Fact] + public async Task Slow_start_spaces_an_immediate_burst() + { + using var scope = new PacerScope(); + + await AdaptiveRequestPacer.WaitAsync(WorkloadBucket.Directory, CancellationToken.None); + var second = await AdaptiveRequestPacer.WaitAsync(WorkloadBucket.Directory, CancellationToken.None); + + // At the 4 rps opening cap the second request waits ~250ms. + Assert.True(second > 0, $"second request should have been paced, waited {second}ms"); + } + + [Fact] + public void Throttle_enters_the_cap_at_the_entry_rate_then_halves_to_the_floor() + { + using var scope = new PacerScope(); + + AdaptiveRequestPacer.RecordThrottle(WorkloadBucket.Directory, retryAfter: null); + Assert.Equal(AdaptiveRequestPacer.ThrottledEntryRate, + AdaptiveRequestPacer.GetAdaptedRate(WorkloadBucket.Directory)); + + AdaptiveRequestPacer.RecordThrottle(WorkloadBucket.Directory, retryAfter: null); + Assert.Equal(AdaptivePacing.ReduceRate(AdaptiveRequestPacer.ThrottledEntryRate), + AdaptiveRequestPacer.GetAdaptedRate(WorkloadBucket.Directory)); + + // Repeat throttles never fall below the floor. + AdaptiveRequestPacer.RecordThrottle(WorkloadBucket.Directory, retryAfter: null); + Assert.Equal(AdaptivePacing.MinAdaptiveRate, + AdaptiveRequestPacer.GetAdaptedRate(WorkloadBucket.Directory)); + } + + [Fact] + public async Task Throttle_cancels_slow_start_and_the_adapted_cap_governs() + { + using var scope = new PacerScope(); + + await AdaptiveRequestPacer.WaitAsync(WorkloadBucket.Drive, CancellationToken.None); + Assert.True(AdaptiveRequestPacer.GetSlowStartRate(WorkloadBucket.Drive) > 0); + + AdaptiveRequestPacer.RecordThrottle(WorkloadBucket.Drive, retryAfter: null); + + Assert.Equal(0, AdaptiveRequestPacer.GetSlowStartRate(WorkloadBucket.Drive)); + Assert.Equal(AdaptiveRequestPacer.ThrottledEntryRate, + AdaptiveRequestPacer.GetAdaptedRate(WorkloadBucket.Drive)); + } + + [Fact] + public void RetryAfter_pushes_the_whole_buckets_next_slot() + { + using var scope = new PacerScope(); + + var before = Stopwatch.GetTimestamp(); + AdaptiveRequestPacer.RecordThrottle(WorkloadBucket.Directory, TimeSpan.FromSeconds(3)); + + var slot = AdaptiveRequestPacer.GetNextSlotTicks(WorkloadBucket.Directory); + // The hold lands ~3s out; assert at least 2s to stay timing-tolerant. + Assert.True(slot > before + 2 * Stopwatch.Frequency, + "Retry-After should hold the whole bucket, not just the throttled request"); + } + + [Fact] + public void Buckets_are_isolated_a_throttle_on_one_never_caps_another() + { + using var scope = new PacerScope(); + + AdaptiveRequestPacer.RecordThrottle(WorkloadBucket.Other, retryAfter: null); // e.g. Teams + + Assert.True(AdaptiveRequestPacer.GetAdaptedRate(WorkloadBucket.Other) > 0); + Assert.Equal(0, AdaptiveRequestPacer.GetAdaptedRate(WorkloadBucket.Directory)); + Assert.Equal(0, AdaptiveRequestPacer.GetAdaptedRate(WorkloadBucket.Drive)); + } + + [Fact] + public void RecordResponse_captures_the_throttle_percentage_gauge() + { + using var scope = new PacerScope(); + + using var response = new HttpResponseMessage(HttpStatusCode.OK); + response.Headers.TryAddWithoutValidation("x-ms-throttle-limit-percentage", "0.95"); + + AdaptiveRequestPacer.RecordResponse(WorkloadBucket.Directory, response); + + Assert.Equal(0.95, AdaptiveRequestPacer.LastThrottlePercentage, precision: 3); + var state = AdaptiveRequestPacer.DescribeState(); + Assert.NotNull(state); + Assert.Contains("proximity 95%", state); + } + + [Fact] + public async Task Proximity_gauge_paces_requests_beyond_what_the_rate_cap_alone_would() + { + // The gauge is opportunistic - Graph emits x-ms-throttle-limit-percentage rarely, so it + // may never be seen live - but the wiring from RecordResponse into the gate must work + // when a header does arrive. Nothing else in the suite drives a reported percentage + // through WaitAsync, so severing damping from the gate is otherwise invisible. + using var scope = new PacerScope(); + + // Cold-bucket slow start is the only other spacing source here and it caps at 4 rps, + // i.e. a 250ms interval. A reported 1.0 ("throttling has begun") sits halfway up the + // damping ramp: 1000ms, four times anything the cap can produce. + await AdaptiveRequestPacer.WaitAsync(WorkloadBucket.Directory, CancellationToken.None); + + using var response = new HttpResponseMessage(HttpStatusCode.OK); + response.Headers.TryAddWithoutValidation("x-ms-throttle-limit-percentage", "1.0"); + AdaptiveRequestPacer.RecordResponse(WorkloadBucket.Directory, response); + + // The interval computed for a request governs the slot the *next* one claims, so the + // damped spacing is observable on the request after the first post-header send. + await AdaptiveRequestPacer.WaitAsync(WorkloadBucket.Directory, CancellationToken.None); + var damped = await AdaptiveRequestPacer.WaitAsync(WorkloadBucket.Directory, CancellationToken.None); + + Assert.True(damped > 600, + "a request sent while Graph reports it is at the throttle limit must be spaced by the " + + $"proximity damping (~1000ms), well past the 250ms the 4 rps cap alone gives; waited {damped}ms"); + } + + [Fact] + public void RecordResponse_treats_a_final_429_as_a_throttle() + { + using var scope = new PacerScope(); + + // OnRetry never fires for the last attempt, so a 429 that survives its retries is + // only observable on the final response. + using var response = new HttpResponseMessage((HttpStatusCode)429); + response.Headers.TryAddWithoutValidation("Retry-After", "2"); + + AdaptiveRequestPacer.RecordResponse(WorkloadBucket.Directory, response); + + Assert.Equal(AdaptiveRequestPacer.ThrottledEntryRate, + AdaptiveRequestPacer.GetAdaptedRate(WorkloadBucket.Directory)); + } + + [Fact] + public async Task NoAdaptivePacing_disables_the_gate_and_signal_recording() + { + using var scope = new PacerScope(); + AdaptiveRequestPacer.Configure(new ResilientGraphClientOptions { NoAdaptivePacing = true }); + + var first = await AdaptiveRequestPacer.WaitAsync(WorkloadBucket.Directory, CancellationToken.None); + var second = await AdaptiveRequestPacer.WaitAsync(WorkloadBucket.Directory, CancellationToken.None); + AdaptiveRequestPacer.RecordThrottle(WorkloadBucket.Directory, TimeSpan.FromSeconds(60)); + + Assert.Equal(0, first); + Assert.Equal(0, second); + Assert.Equal(0, AdaptiveRequestPacer.GetAdaptedRate(WorkloadBucket.Directory)); + } + + [Fact] + public void Reset_clears_learned_state() + { + using var scope = new PacerScope(); + + AdaptiveRequestPacer.RecordThrottle(WorkloadBucket.Directory, TimeSpan.FromSeconds(30)); + Assert.True(AdaptiveRequestPacer.GetAdaptedRate(WorkloadBucket.Directory) > 0); + + AdaptiveRequestPacer.Reset(); + + Assert.Equal(0, AdaptiveRequestPacer.GetAdaptedRate(WorkloadBucket.Directory)); + Assert.Equal(0, AdaptiveRequestPacer.GetNextSlotTicks(WorkloadBucket.Directory)); + Assert.Equal(-1, AdaptiveRequestPacer.LastThrottlePercentage); + } + + [Fact] + public async Task Slow_start_cap_doubles_after_a_clean_interval() + { + using var scope = new PacerScope(); + + await AdaptiveRequestPacer.WaitAsync(WorkloadBucket.Directory, CancellationToken.None); + Assert.Equal(AdaptiveRequestPacer.SlowStartInitialRate, + AdaptiveRequestPacer.GetSlowStartRate(WorkloadBucket.Directory)); + + // RampInterval is 1s; sleep past it with margin, then the next request ramps the cap. + await Task.Delay(1300); + await AdaptiveRequestPacer.WaitAsync(WorkloadBucket.Directory, CancellationToken.None); + + Assert.Equal(AdaptiveRequestPacer.SlowStartInitialRate * 2, + AdaptiveRequestPacer.GetSlowStartRate(WorkloadBucket.Directory)); + } + + [Fact] + public async Task Adapted_cap_climbs_back_out_of_a_throttle_after_clean_intervals() + { + // The "AI" of AIMD. A 429 halves the bucket's rate; if the additive recovery step never + // ran, one 429 would pace that workload forever and throughput would never return. + using var scope = new PacerScope(); + // A 5 rps ceiling makes one additive step (4 + max(1, 5/10)) reach the ceiling, so a + // recovered bucket becomes observable within two ramp intervals rather than the ten a + // 50 rps ceiling would need. + AdaptiveRequestPacer.Configure(new ResilientGraphClientOptions { RateLimitPerSecond = 5 }); + AdaptiveRequestPacer.DisabledForTests = false; + + AdaptiveRequestPacer.RecordThrottle(WorkloadBucket.Directory, retryAfter: null); + + // While the cap holds, back-to-back requests are spaced (4 rps => ~250ms). + await AdaptiveRequestPacer.WaitAsync(WorkloadBucket.Directory, CancellationToken.None); + var whileCapped = await AdaptiveRequestPacer.WaitAsync(WorkloadBucket.Directory, CancellationToken.None); + Assert.True(whileCapped > 0, "a bucket capped by a 429 must still be pacing before it recovers"); + + // Two clean ramp intervals. The first ramp after a throttle only re-arms the window - + // the throttle shares its timestamp - and the second performs the additive increase. + for (var i = 0; i < 2; i++) + { + await Task.Delay(1300); + await AdaptiveRequestPacer.WaitAsync(WorkloadBucket.Directory, CancellationToken.None); + } + + // Recovery is observable as restored throughput: a burst is no longer spaced at all. + var afterRecovery = new long[3]; + for (var i = 0; i < afterRecovery.Length; i++) + afterRecovery[i] = await AdaptiveRequestPacer.WaitAsync(WorkloadBucket.Directory, CancellationToken.None); + + Assert.True(afterRecovery.All(w => w == 0), + "a bucket that stayed clean must climb back out of its post-429 cap, but requests are still " + + $"being paced: [{string.Join(", ", afterRecovery)}] ms"); + } + + // --- integration: the two request paths and the batch exemption --- + + [Fact] + public async Task SendAsync_is_gated_and_a_burst_is_paced() + { + using var scope = new PacerScope(); + ResiliencePipelineFactory.Reset(); + AdaptiveRequestPacer.DisabledForTests = false; // factory Reset re-cleared pacer state only + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + + MgxTelemetryCollector.Current.Reset(); + await client.GetAsync("https://graph.microsoft.com/v1.0/users/user1"); + await client.GetAsync("https://graph.microsoft.com/v1.0/users/user2"); + await client.GetAsync("https://graph.microsoft.com/v1.0/users/user3"); + + var summary = MgxTelemetryCollector.Current.GetSummary(); + Assert.True(summary.AdaptivePacingActivations >= 1, + "an immediate 3-request burst on a cold bucket must trip the slow-start cap"); + Assert.True(summary.AdaptivePacingWaitMs > 0); + } + + [Fact] + public async Task A_429_through_SendAsync_feeds_the_pacer_with_the_right_bucket() + { + using var scope = new PacerScope(); + ResiliencePipelineFactory.Reset(); + AdaptiveRequestPacer.DisabledForTests = false; + + var handler = new MockHttpHandler(); + handler.QueueResponse((HttpStatusCode)429, null, new() { ["Retry-After"] = "0" }); + handler.QueueResponse(HttpStatusCode.OK, TestData.SingleUser); + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + + var response = await client.GetAsync("https://graph.microsoft.com/v1.0/users/user1"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(AdaptiveRequestPacer.ThrottledEntryRate, + AdaptiveRequestPacer.GetAdaptedRate(WorkloadBucket.Directory)); + Assert.Equal(0, AdaptiveRequestPacer.GetAdaptedRate(WorkloadBucket.Drive)); + } + + [Fact] + public async Task The_delegating_handler_path_is_gated_too() + { + // The Enable-MgxResilience bridge: SDK traffic bypasses ResilientGraphClient, so the + // pacer must sit in ResilientDelegatingHandler as well or M365DSC-style consumers + // get no pacing at all. + using var scope = new PacerScope(); + ResiliencePipelineFactory.Reset(); + AdaptiveRequestPacer.DisabledForTests = false; + + var options = new ResilientGraphClientOptions { NoRateLimit = true }; + var (pipeline, rateLimiter) = ResiliencePipelineFactory.GetOrCreate(options); + var mock = new MockHttpHandler(); + mock.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + using var resilient = new ResilientDelegatingHandler(pipeline, rateLimiter) { InnerHandler = mock }; + using var httpClient = new HttpClient(resilient); + + MgxTelemetryCollector.Current.Reset(); + await httpClient.GetAsync("https://graph.microsoft.com/v1.0/users/user1"); + await httpClient.GetAsync("https://graph.microsoft.com/v1.0/users/user2"); + await httpClient.GetAsync("https://graph.microsoft.com/v1.0/users/user3"); + + var summary = MgxTelemetryCollector.Current.GetSummary(); + Assert.True(summary.AdaptivePacingActivations >= 1, + "the SDK-bridge path must be paced exactly like ResilientGraphClient.SendAsync"); + + // Telemetry parity. Reporting pacing activations and retries while TotalRequests stayed + // 0 was self-contradictory, and made the throttle-rate formula in the Get-MgxTelemetry + // help divide by zero on exactly the path M365DSC-style consumers use. + Assert.Equal(3, summary.TotalRequests); + Assert.Equal(3, summary.Succeeded); + Assert.Equal(0, summary.Failed); + } + + /// + /// A handler that takes measurable time, so ElapsedMilliseconds does not truncate to 0. + /// + private sealed class SlowHandler : HttpMessageHandler + { + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + await Task.Delay(25, cancellationToken); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(TestData.SingleUser, System.Text.Encoding.UTF8, "application/json") + }; + } + } + + [Fact] + public async Task Sdk_bridge_path_records_http_time() + { + AdaptiveRequestPacer.DisabledForTests = true; // isolate network time from pacing delay + try + { + ResiliencePipelineFactory.Reset(); + var options = new ResilientGraphClientOptions { NoRateLimit = true }; + var (pipeline, rateLimiter) = ResiliencePipelineFactory.GetOrCreate(options); + using var slow = new SlowHandler(); + using var resilient = new ResilientDelegatingHandler(pipeline, rateLimiter) { InnerHandler = slow }; + using var httpClient = new HttpClient(resilient); + + MgxTelemetryCollector.Current.Reset(); + await httpClient.GetAsync("https://graph.microsoft.com/v1.0/users/a"); + + var summary = MgxTelemetryCollector.Current.GetSummary(); + // The 2.1 parity fix added RecordRequest and stopped there, so HttpMs stayed 0 for + // every Enable-MgxResilience session - indistinguishable from no network time. + Assert.True(summary.HttpMs > 0, + $"HttpMs must be recorded on the SDK-bridge path, got {summary.HttpMs}"); + } + finally + { + AdaptiveRequestPacer.DisabledForTests = false; + ResiliencePipelineFactory.Reset(); + } + } + + [Fact] + public async Task Sdk_bridge_path_records_rate_limiter_wait() + { + AdaptiveRequestPacer.DisabledForTests = true; // isolate the limiter from pacing delay + try + { + ResiliencePipelineFactory.Reset(); + // A burst of 1 at 1/sec guarantees the second and third acquisitions actually wait. + var options = new ResilientGraphClientOptions + { + RateLimitPerSecond = 1, + RateLimitBurst = 1, + RateLimitQueueLimit = 10 + }; + var (pipeline, rateLimiter) = ResiliencePipelineFactory.GetOrCreate(options); + var mock = new MockHttpHandler(); + mock.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + using var resilient = new ResilientDelegatingHandler(pipeline, rateLimiter) { InnerHandler = mock }; + using var httpClient = new HttpClient(resilient); + + MgxTelemetryCollector.Current.Reset(); + await httpClient.GetAsync("https://graph.microsoft.com/v1.0/users/a"); + await httpClient.GetAsync("https://graph.microsoft.com/v1.0/users/b"); + await httpClient.GetAsync("https://graph.microsoft.com/v1.0/users/c"); + + var summary = MgxTelemetryCollector.Current.GetSummary(); + Assert.True(summary.RateLimiterWaitMs > 0, + $"limiter wait must be recorded on the SDK-bridge path, got {summary.RateLimiterWaitMs}"); + } + finally + { + AdaptiveRequestPacer.DisabledForTests = false; + ResiliencePipelineFactory.Reset(); + } + } + + [Fact] + public async Task Concurrent_waiters_stay_spaced_when_the_queue_exceeds_the_delay_cap() + { + // The slot is claimed at full interval but the sleep used to be clamped to + // MaxRetryAfterSeconds, so every claimant queued past the clamp woke at the clamp and + // sent simultaneously - a synchronised burst produced exactly when the bucket was most + // oversubscribed, which is the opposite of what the gate is for. + using var scope = new PacerScope(); + // NoRateLimit pins the ceiling at 50, so cold-start slow start caps at 4 rps => a 250 ms + // interval. MaxRetryAfterSeconds = 1 sets the old 1000 ms sleep clamp, which 12 queued + // claims (~2.75 s of slots) overshoot decisively. + AdaptiveRequestPacer.Configure(new ResilientGraphClientOptions + { + NoRateLimit = true, + MaxRetryAfterSeconds = 1, + }); + AdaptiveRequestPacer.DisabledForTests = false; + + var sw = System.Diagnostics.Stopwatch.StartNew(); + var waits = await Task.WhenAll(Enumerable.Range(0, 12) + .Select(async _ => + { + await AdaptiveRequestPacer.WaitAsync(WorkloadBucket.Directory, CancellationToken.None); + return sw.ElapsedMilliseconds; + })); + + var ordered = waits.OrderBy(x => x).ToArray(); + // Under the old clamp every claim past ~1000 ms woke at the clamp: eight of these twelve + // would land inside one 200 ms window and send together. + var clustered = ordered.Count(t => t > 900 && t < 1100); + Assert.True(clustered <= 2, + $"waiters bunched at the delay cap instead of staying spaced: [{string.Join(", ", ordered)}]"); + Assert.True(ordered[^1] >= 2000, + $"the last waiter should honour its claimed slot (~2750ms), got {ordered[^1]}ms"); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + public void A_throttle_never_raises_the_cap_above_the_configured_ceiling(int ceiling) + { + // ReduceRate floors at MinAdaptiveRate (2), and only the first-throttle branch clamped to + // the ceiling. So a SECOND 429 under -RateLimitPerSecond 1 moved the cap from 1 to 2 - + // halving the spacing in response to being throttled, and reporting "capped 2/1 rps". + // Three throttles, because the bug needs a repeat to appear. + using var scope = new PacerScope(); + AdaptiveRequestPacer.Configure(new ResilientGraphClientOptions { RateLimitPerSecond = ceiling }); + + for (var i = 0; i < 3; i++) + { + AdaptiveRequestPacer.RecordThrottle(WorkloadBucket.Directory, null); + var cap = AdaptiveRequestPacer.GetAdaptedRate(WorkloadBucket.Directory); + Assert.True(cap <= ceiling, + $"after {i + 1} throttle(s) the cap was {cap}, above the configured ceiling {ceiling}"); + } + } + + [Fact] + public async Task Batch_bucket_never_claims_a_pacing_slot() + { + // GraphBatchClient owns batch throughput through its own item-level AIMD and passes + // paceGate: false. The gate refuses the bucket as well, so a caller that forgets the + // flag cannot accidentally stack two AIMD controllers on one workload. Before $batch + // had its own bucket it classified as Other, and an outer 429 capped unrelated + // Exchange, Teams and Intune traffic on evidence that had nothing to do with them. + using var scope = new PacerScope(); + AdaptiveRequestPacer.DisabledForTests = false; + + var waited = await AdaptiveRequestPacer.WaitAsync(WorkloadBucket.Batch, CancellationToken.None); + Assert.Equal(0, waited); + } + + [Fact] + public async Task Batch_outer_posts_skip_the_gate() + { + // GraphBatchClient owns batch throughput (cross-call pacing, item-level AIMD). + // Pacing the outer POSTs too would stack two AIMD controllers on one workload. + using var scope = new PacerScope(); + ResiliencePipelineFactory.Reset(); + AdaptiveRequestPacer.DisabledForTests = false; + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, + """{"responses":[{"id":"1","status":200,"body":{"id":"user1"}}]}"""); + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batch = new GraphBatchClient(client); + + MgxTelemetryCollector.Current.Reset(); + // Three back-to-back batch calls: gated traffic at this cadence would trip slow start. + await batch.ExecuteBatchAsync(["https://graph.microsoft.com/v1.0/users/user1"]); + await batch.ExecuteBatchAsync(["https://graph.microsoft.com/v1.0/users/user1"]); + await batch.ExecuteBatchAsync(["https://graph.microsoft.com/v1.0/users/user1"]); + + var summary = MgxTelemetryCollector.Current.GetSummary(); + Assert.Equal(0, summary.AdaptivePacingActivations); + } +} \ No newline at end of file diff --git a/tests/Mgx.IntegrationTests/Engine/Http/BatchPacingStateTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/BatchPacingStateTests.cs new file mode 100644 index 0000000..b3100de --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/BatchPacingStateTests.cs @@ -0,0 +1,50 @@ +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests.Engine.Http; + +/// +/// WaitAsync exempts the Batch bucket before touching state - GraphBatchClient runs its own +/// item-level AIMD - so an adapted cap recorded for Batch is enforced by nothing. Worse, the +/// expiry check lives inside WaitAsync, so such a cap is also cleared by nothing and +/// Get-MgxTelemetry reports "batch: capped 2/50 rps (last 429 3600s ago)" forever. +/// +/// Nothing pinned that: deleting the guard left the whole suite green. +/// +[Collection("Pipeline")] +public class BatchPacingStateTests +{ + [Fact] + public void A_batch_throttle_does_not_create_a_cap_that_nothing_can_clear() + { + // The suite disables the pacer globally (PacerTestDefaults), so it must be re-enabled + // here or RecordThrottle returns before touching any state and this passes vacuously. + AdaptiveRequestPacer.DisabledForTests = false; + AdaptiveRequestPacer.Reset(); + try + { + AdaptiveRequestPacer.RecordThrottle(WorkloadBucket.Batch, null); + + var state = AdaptiveRequestPacer.DescribeState(); + Assert.True( + state == null || !state.Contains("batch: capped", StringComparison.OrdinalIgnoreCase), + $"Batch must not carry an adapted cap - it is never gated and never expired. Got: {state}"); + } + finally { AdaptiveRequestPacer.Reset(); AdaptiveRequestPacer.DisabledForTests = true; } + } + + [Fact] + public void A_directory_throttle_still_creates_one() + { + AdaptiveRequestPacer.DisabledForTests = false; + AdaptiveRequestPacer.Reset(); + try + { + AdaptiveRequestPacer.RecordThrottle(WorkloadBucket.Directory, null); + + var state = AdaptiveRequestPacer.DescribeState(); + Assert.NotNull(state); + Assert.Contains("directory: capped", state, StringComparison.OrdinalIgnoreCase); + } + finally { AdaptiveRequestPacer.Reset(); AdaptiveRequestPacer.DisabledForTests = true; } + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Http/BodyRedactionTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/BodyRedactionTests.cs new file mode 100644 index 0000000..9dcea33 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/BodyRedactionTests.cs @@ -0,0 +1,80 @@ +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests.Engine.Http; + +/// +/// -Debug body redaction. A pre-authenticated download URL fetches the file bytes with no bearer +/// token, and -Debug output is what users paste into issue reports, so this is a credential +/// control - and drive-content-triage.ps1 tells users it exists. +/// +/// It previously had NO test at all: deleting both redaction passes left the entire suite green. +/// These pin both directions, because over-redaction is its own regression - @odata.nextLink is +/// how paging bugs get diagnosed. +/// +public class BodyRedactionTests +{ + private static string Trace(string body) => + GraphRequestTracer.FormatResponse(new HttpResponseMessage(HttpStatusCode.OK), 12, body); + + [Theory] + // SharePoint / OneDrive for Business: capability in the query string. + [InlineData("""{"@microsoft.graph.downloadUrl":"https://c-my.sharepoint.com/_layouts/15/download.aspx?UniqueId=1&tempauth=eyJ0.LEAKED.sig&ApiVersion=2.0"}""", "LEAKED")] + // Consumer OneDrive: capability in the PATH, which a query-only cut would miss entirely. + [InlineData("""{"@microsoft.graph.downloadUrl":"https://public.bl.files.1drv.com/y4mLEAKED/file.bin"}""", "LEAKED")] + // The older property name for the same thing. + [InlineData("""{"@content.downloadUrl":"https://c-my.sharepoint.com/x?tempauth=LEAKED"}""", "LEAKED")] + // Azure SAS. + [InlineData("""{"uploadUrl":"https://s.blob.core.windows.net/c/b?sv=2021&sig=LEAKED&se=2026"}""", "LEAKED")] + // Sharing link. + [InlineData("""{"link":"https://c.sharepoint.com/g?guestaccesstoken=LEAKED"}""", "LEAKED")] + // Presigned S3-style. + [InlineData("""{"url":"https://b.s3.amazonaws.com/k?X-Amz-Signature=LEAKED&X-Amz-Expires=60"}""", "LEAKED")] + // Nested inside a collection, which is how Graph actually returns driveItems. + [InlineData("""{"value":[{"id":"a"},{"id":"b","@microsoft.graph.downloadUrl":"https://c-my.sharepoint.com/d?tempauth=LEAKED"}]}""", "LEAKED")] + public void Redacts_pre_authenticated_urls(string body, string secret) + { + var trace = Trace(body); + Assert.DoesNotContain(secret, trace, StringComparison.Ordinal); + Assert.Contains("", trace); + } + + [Theory] + // Paging tokens are diagnostics, not credentials. Losing these would remove the thing + // pagination bugs are debugged with. + [InlineData("""{"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=RFNwdAoAAQAAA"}""", "RFNwdAoAAQAAA")] + [InlineData("""{"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=KEEPME"}""", "KEEPME")] + // Ordinary URLs on a driveItem. + [InlineData("""{"webUrl":"https://c.sharepoint.com/Shared%20Documents/KEEPME.xlsx"}""", "KEEPME")] + [InlineData("""{"siteUrl":"https://c.sharepoint.com/sites/KEEPME"}""", "KEEPME")] + // The (? + /// Three items, not two. Sanitize makes two passes that can each match a downloadUrl - one + /// by property name, one by value - so with only two items a first-match-only regression + /// was masked: pass one redacted FIRST, pass two redacted SECOND, and the test stayed green + /// while every third and subsequent capability URL leaked. + /// + [Fact] + public void Redacts_every_url_in_a_body_not_just_the_first() + { + var trace = Trace("""{"value":[{"@microsoft.graph.downloadUrl":"https://a.sharepoint.com/x?tempauth=FIRST"},{"@microsoft.graph.downloadUrl":"https://b.sharepoint.com/y?tempauth=SECOND"},{"@microsoft.graph.downloadUrl":"https://c.sharepoint.com/z?tempauth=THIRD"}]}"""); + Assert.DoesNotContain("FIRST", trace, StringComparison.Ordinal); + Assert.DoesNotContain("SECOND", trace, StringComparison.Ordinal); + Assert.DoesNotContain("THIRD", trace, StringComparison.Ordinal); + } +} diff --git a/tests/Mgx.IntegrationTests/CancellationTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/CancellationTests.cs similarity index 100% rename from tests/Mgx.IntegrationTests/CancellationTests.cs rename to tests/Mgx.IntegrationTests/Engine/Http/CancellationTests.cs diff --git a/tests/Mgx.IntegrationTests/CircuitBreakerTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/CircuitBreakerTests.cs similarity index 100% rename from tests/Mgx.IntegrationTests/CircuitBreakerTests.cs rename to tests/Mgx.IntegrationTests/Engine/Http/CircuitBreakerTests.cs diff --git a/tests/Mgx.IntegrationTests/ConcurrencyTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/ConcurrencyTests.cs similarity index 100% rename from tests/Mgx.IntegrationTests/ConcurrencyTests.cs rename to tests/Mgx.IntegrationTests/Engine/Http/ConcurrencyTests.cs diff --git a/tests/Mgx.IntegrationTests/ConfigurableOptionsTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/ConfigurableOptionsTests.cs similarity index 100% rename from tests/Mgx.IntegrationTests/ConfigurableOptionsTests.cs rename to tests/Mgx.IntegrationTests/Engine/Http/ConfigurableOptionsTests.cs diff --git a/tests/Mgx.IntegrationTests/ContentTypeTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/ContentTypeTests.cs similarity index 100% rename from tests/Mgx.IntegrationTests/ContentTypeTests.cs rename to tests/Mgx.IntegrationTests/Engine/Http/ContentTypeTests.cs diff --git a/tests/Mgx.IntegrationTests/Engine/Http/DownloadUrlValidatorTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/DownloadUrlValidatorTests.cs new file mode 100644 index 0000000..6971c37 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/DownloadUrlValidatorTests.cs @@ -0,0 +1,49 @@ +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// The hop-2 download allowlist. Companion to SsrfValidationTests (which covers +/// NextLinkValidator's same-authority rule); this validator is the one place a request is +/// allowed to leave the Graph host, so every bypass shape gets a deny test. +/// +public class DownloadUrlValidatorTests +{ + [Theory] + [InlineData("https://contoso.sharepoint.com/_layouts/15/download.aspx?UniqueId=abc&tempauth=xyz")] + [InlineData("https://contoso-my.sharepoint.com/personal/user/_layouts/15/download.aspx?share=abc")] + [InlineData("https://tenant.sharepoint.us/download?x=1")] + [InlineData("https://tenant.sharepoint.cn/download")] + [InlineData("https://tenant.sharepointonline.com/file")] + [InlineData("https://public.bn1303.files.1drv.com/y4mabc/file.jpg")] + [InlineData("https://southindia1-mediap.svc.ms/transform/thumbnail?provider=spo")] + public void Allows_documented_microsoft_download_hosts(string url) => + Assert.Equal(url, DownloadUrlValidator.Validate(url)); + + [Theory] + [InlineData("http://contoso.sharepoint.com/file")] // scheme downgrade + [InlineData("https://evilsharepoint.com/file")] // suffix not dot-anchored + [InlineData("https://sharepoint.com/file")] // apex, no subdomain + [InlineData("https://contoso.sharepoint.com.evil.example/file")] // suffix embedded mid-host + [InlineData("https://contoso.sharepoint.com:8443/file")] // non-default port + [InlineData("https://user@contoso.sharepoint.com/file")] // userinfo trick + [InlineData("https://user:pass@contoso.sharepoint.com/file")] // userinfo trick + [InlineData("https://mystore.blob.core.windows.net/container/file?sig=x")] // any Azure customer owns one + [InlineData("https://evil.example/redirect?to=contoso.sharepoint.com")] // suffix only in query + [InlineData("file:///etc/passwd")] + [InlineData("not a url")] + [InlineData("")] + public void Denies_everything_else(string url) => + Assert.Null(DownloadUrlValidator.Validate(url)); + + [Fact] + public void Denies_null() => + Assert.Null(DownloadUrlValidator.Validate(null)); + + [Fact] + public void Denies_unicode_lookalike_hosts() + { + // "ѕharepoint" with a Cyrillic dze: punycode-normalizes to xn--, never matches. + Assert.Null(DownloadUrlValidator.Validate("https://contoso.ѕharepoint.com/file")); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Http/GraphContentClientTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/GraphContentClientTests.cs new file mode 100644 index 0000000..55b1c4e --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/GraphContentClientTests.cs @@ -0,0 +1,818 @@ +using System.Collections.ObjectModel; +using System.Management.Automation; +using System.Net; +using System.Net.Http.Headers; +using System.Reflection; +using System.Text; +using Mgx.Cmdlets.Base; +using Mgx.Engine.Http; +using Mgx.Engine.Models; + +namespace Mgx.IntegrationTests; + +/// +/// The two-hop content path. The security-critical assertions live here: hop 2 carries no +/// Authorization header, every redirect is re-validated against the allowlist, an +/// auto-redirecting transport is refused, and the -Debug trace never leaks a pre-auth URL's +/// query string. Hop 1 uses a MockHttpHandler-backed ResilientGraphClient; hop 2 is mocked +/// through GraphContentClient.DownloadClientForTests. +/// +[Collection("Pipeline")] +public class GraphContentClientTests : IDisposable +{ + private const string GraphContentUrl = "https://graph.microsoft.com/v1.0/me/drive/items/01ABC/content"; + private const string CdnUrl = "https://contoso-my.sharepoint.com/_layouts/15/download.aspx?UniqueId=abc&tempauth=SECRETTOKEN"; + + private readonly MockHttpHandler _graphHandler = new(); + private readonly MockHttpHandler _cdnHandler = new(); + private readonly HttpClient _graphHttpClient; + private readonly ResilientGraphClient _client; + + public GraphContentClientTests() + { + ResiliencePipelineFactory.Reset(); + _graphHttpClient = new HttpClient(_graphHandler); + _client = new ResilientGraphClient(_graphHttpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + GraphContentClient.DownloadClientForTests = new HttpClient(_cdnHandler); + } + + public void Dispose() + { + GraphContentClient.DownloadClientForTests?.Dispose(); + GraphContentClient.DownloadClientForTests = null; + _client.Dispose(); + _graphHttpClient.Dispose(); + } + + private void QueueRedirectToCdn(string location = CdnUrl) => + _graphHandler.QueueResponse(HttpStatusCode.Found, null, new() { ["Location"] = location }); + + private static async Task ReadAllAsync(GraphContentResult result) + { + using var ms = new MemoryStream(); + await GraphContentClient.CopyWithIdleTimeoutAsync( + result.Content, ms, null, TimeSpan.FromSeconds(5), CancellationToken.None); + return Encoding.UTF8.GetString(ms.ToArray()); + } + + // --- the token-leak regression the whole design exists to prevent --- + + [Fact] + public async Task Hop2_request_carries_no_Authorization_header_and_the_range() + { + QueueRedirectToCdn(); + _cdnHandler.QueueResponse(HttpStatusCode.PartialContent, "12345"); + + using var result = await _client.GetContentAsync( + GraphContentUrl, new RangeHeaderValue(0, 4)); + + Assert.Equal("12345", await ReadAllAsync(result)); + Assert.True(result.FromDownloadHost); + + var cdnRequest = _cdnHandler.Requests.Single(); + Assert.Null(cdnRequest.Headers.Authorization); + Assert.False(cdnRequest.Headers.Contains("Authorization"), + "the bearer must never reach the download host"); + Assert.Equal("bytes=0-4", cdnRequest.Headers.Range!.ToString()); + } + + [Fact] + public void Debug_trace_never_contains_the_preauth_query_string() + { + using var response = new HttpResponseMessage(HttpStatusCode.Found); + response.Content = new StringContent(""); + response.Headers.TryAddWithoutValidation("Location", CdnUrl); + + var trace = GraphRequestTracer.FormatResponse(response, 12, null); + + Assert.DoesNotContain("SECRETTOKEN", trace); + Assert.DoesNotContain("tempauth", trace); + // Only scheme and host survive. The path goes too, deliberately: not every download + // host puts the capability in the query - some carry it in a path segment - and the + // tracer cannot tell those apart. The host answers the question -Debug is actually + // asked here ("where did this redirect go?"); the path adds little and can leak. + Assert.Contains("https://contoso-my.sharepoint.com/", trace); + Assert.DoesNotContain("_layouts", trace); + } + + // --- allowlist enforcement, both hops --- + + [Fact] + public async Task Hop1_redirect_to_a_disallowed_host_is_refused() + { + QueueRedirectToCdn("https://mystore.blob.core.windows.net/exfil/file?sig=x"); + + var ex = await Assert.ThrowsAsync( + () => _client.GetContentAsync(GraphContentUrl)); + + Assert.Contains("not on the allowed list", ex.Message); + Assert.Empty(_cdnHandler.Requests); // never fetched + } + + [Fact] + public async Task Hop2_redirect_to_a_disallowed_host_is_refused() + { + // An open redirect on an allowlisted host must not bypass the validator: every + // Location is re-validated, not just the first. + QueueRedirectToCdn(); + _cdnHandler.QueueResponse(HttpStatusCode.Found, null, + new() { ["Location"] = "https://evil.example/steal" }); + + var ex = await Assert.ThrowsAsync( + () => _client.GetContentAsync(GraphContentUrl)); + + Assert.Contains("not on the allowed list", ex.Message); + } + + [Fact] + public async Task Hop2_validated_redirect_is_followed() + { + QueueRedirectToCdn(); + _cdnHandler.QueueResponse(HttpStatusCode.Found, null, + new() { ["Location"] = "https://southindia1-mediap.svc.ms/transform?x=1" }); + _cdnHandler.QueueResponse(HttpStatusCode.OK, "bytes-from-cdn"); + + using var result = await _client.GetContentAsync(GraphContentUrl); + + Assert.Equal("bytes-from-cdn", await ReadAllAsync(result)); + Assert.Equal(2, _cdnHandler.Requests.Count); + } + + // --- transport precondition --- + + private sealed class AutoRedirectingHandler : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + // Simulates an SDK-style transport that already followed the 302: the response + // is a 200 whose RequestMessage points at the download host. + var followed = new HttpRequestMessage(HttpMethod.Get, CdnUrl); + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("cdn bytes via auto-redirect"), + RequestMessage = followed + }); + } + } + + [Fact] + public async Task A_transport_that_auto_followed_the_redirect_is_refused() + { + using var autoClient = new HttpClient(new AutoRedirectingHandler()); + using var client = new ResilientGraphClient(autoClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + + var ex = await Assert.ThrowsAsync( + () => client.GetContentAsync(GraphContentUrl)); + + Assert.Contains("followed a redirect", ex.Message); + } + + // --- the same precondition, one layer up: Get-MgxContent before it fetches anything --- + + private const string TestTenantId = "test-tenant-00000000-0000-0000-0000-000000000000"; + + /// + /// Points MgxCmdletBase's cached transport at this class's Graph mock and declares whether + /// mgx owns it (false = the Graph SDK's client, which ships a RedirectHandler). Reflection + /// because the statics are private; the fingerprint has to match what the test shell's + /// stubbed Get-MgContext produces, or GetClient() sees a credential change and rebuilds. + /// + private void InjectCmdletTransport(bool owned, HttpClient? client = null) + { + ResiliencePipelineFactory.Reset(); + var options = new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }; + SetCmdletStatic("s_graphHttpClient", client ?? _graphHttpClient); + SetCmdletStatic("s_cachedAuthFingerprint", + MgxCmdletBase.BuildAuthFingerprint(new { TenantId = TestTenantId }, null)); + SetCmdletStatic("s_cachedAuthContextRef", null); + SetCmdletStatic("s_ownsHttpClient", owned); + SetCmdletStatic("s_graphEndpoint", "https://graph.microsoft.com"); + SetCmdletStatic("s_clientOptions", options); + SetCmdletStatic("s_cachedTotalTimeoutSeconds", options.TotalTimeoutSeconds); + } + + private static void ResetCmdletTransport() + { + SetCmdletStatic("s_graphHttpClient", null); + SetCmdletStatic("s_cachedAuthFingerprint", null); + SetCmdletStatic("s_cachedAuthContextRef", null); + SetCmdletStatic("s_ownsHttpClient", false); + SetCmdletStatic("s_cachedTotalTimeoutSeconds", 0); + SetCmdletStatic("s_clientOptions", ResilientGraphClientOptions.Default); + ResiliencePipelineFactory.Reset(); + } + + private static void SetCmdletStatic(string field, object? value) => + typeof(MgxCmdletBase) + .GetField(field, BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Static)! + .SetValue(null, value); + + private static PowerShell CreateContentShell() + { + var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Content.GetMgxContent).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + // Stands in for a Graph connection: GetClient() only needs a context with a TenantId. + ps.AddScript($"function Get-MgContext {{ [PSCustomObject]@{{ TenantId = '{TestTenantId}' }} }}"); + ps.Invoke(); + ps.Commands.Clear(); + return ps; + } + + private static IEnumerable ErrorIds(PowerShell ps, Exception? thrown) + { + foreach (var record in ps.Streams.Error) + yield return record.FullyQualifiedErrorId; + if (thrown is IContainsErrorRecord containsRecord) + yield return containsRecord.ErrorRecord.FullyQualifiedErrorId; + } + + [Fact] + public void GetMgxContent_refuses_to_fetch_over_a_transport_mgx_does_not_own() + { + const string contentUri = "/me/drive/items/01ABC/content"; + // Bait, served to anyone who asks. Getting these bytes back means an authenticated + // request went out over a transport that can auto-follow a 302 to a host mgx never + // validated - the exact leak the two-hop design exists to prevent. + _graphHandler.SetDefaultResponse(HttpStatusCode.OK, "bytes-that-must-not-be-fetched"); + + try + { + // 1. Borrowed SDK transport: refuse, and refuse before the request is sent. + InjectCmdletTransport(owned: false); + using (var ps = CreateContentShell()) + { + ps.AddCommand("Get-MgxContent").AddParameter("Uri", contentUri); + var thrown = Record.Exception(() => ps.Invoke()); + + Assert.Contains(ErrorIds(ps, thrown), + id => id.Contains("ContentRequiresOwnedTransport", StringComparison.Ordinal)); + Assert.Equal(0, _graphHandler.RequestCount); + } + + // 2. The identical call on the mgx-owned transport has to go through, or step 1 + // proves nothing: a cmdlet that always refuses would pass it too. + InjectCmdletTransport(owned: true); + using (var ps = CreateContentShell()) + { + ps.AddCommand("Get-MgxContent").AddParameter("Uri", contentUri); + var results = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + var bytes = Assert.IsType(Assert.Single(results).BaseObject); + Assert.Equal("bytes-that-must-not-be-fetched", Encoding.UTF8.GetString(bytes)); + Assert.Equal(1, _graphHandler.RequestCount); + } + } + finally + { + ResetCmdletTransport(); + } + } + + // --- pipeline output guard --- + + /// + /// A body of bytes with an optionally declared + /// Content-Length. Declared and actual are independent on purpose: Graph declares the + /// length on a whole-file GET, so an oversized body can be refused before a byte moves, + /// while a chunked response declares nothing and the guard has to hold during the copy. + /// + private sealed class SizedBodyHandler(long? declaredLength, long actualBytes) : HttpMessageHandler + { + private GeneratedBodyStream? _body; + + /// Bytes the cmdlet actually pulled off the body stream. + public long BytesServed => _body?.BytesRead ?? 0; + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + _body = new GeneratedBodyStream(actualBytes); + var content = new StreamContent(_body); + // Non-seekable, so StreamContent computes no length: leaving this unset is the + // chunked case (ContentLength null), setting it is what Graph does. + if (declaredLength != null) + content.Headers.ContentLength = declaredLength; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = content, + RequestMessage = request + }); + } + } + + /// N zero bytes, generated on read - a large body without a large fixture. + private sealed class GeneratedBodyStream(long length) : Stream + { + public long BytesRead { get; private set; } + + public override int Read(byte[] buffer, int offset, int count) + => Read(buffer.AsSpan(offset, count)); + + public override int Read(Span buffer) + { + var remaining = length - BytesRead; + if (remaining <= 0) return 0; + var n = (int)Math.Min(buffer.Length, remaining); + buffer[..n].Clear(); + BytesRead += n; + return n; + } + + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken) + => ValueTask.FromResult(Read(buffer.Span)); + + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => Task.FromResult(Read(buffer.AsSpan(offset, count))); + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => BytesRead; set => throw new NotSupportedException(); } + public override void Flush() { } + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + } + + /// Runs Get-MgxContent -Uri over a transport serving one sized body. + private (Collection Output, IReadOnlyList ErrorIds, long BytesServed) InvokeContentOverSizedBody( + long? declaredLength, long actualBytes) + { + var handler = new SizedBodyHandler(declaredLength, actualBytes); + using var http = new HttpClient(handler); + try + { + InjectCmdletTransport(owned: true, client: http); + using var ps = CreateContentShell(); + ps.AddCommand("Get-MgxContent").AddParameter("Uri", "/me/drive/items/01ABC/content"); + Collection output = []; + var thrown = Record.Exception(() => output = ps.Invoke()); + return (output, [.. ErrorIds(ps, thrown)], handler.BytesServed); + } + finally + { + ResetCmdletTransport(); + } + } + + [Fact] + public void GetMgxContent_refuses_a_body_bigger_than_the_pipeline_guard() + { + const long MiB = 1024 * 1024; + + // 1. Graph declares 128 MB. Over the 100 MB guard, so the cmdlet must refuse without + // reading the body at all - the whole point is that these bytes never materialise + // as a byte[] in the host's memory. + var declared = InvokeContentOverSizedBody(declaredLength: 128 * MiB, actualBytes: 128 * MiB); + Assert.Contains(declared.ErrorIds, + id => id.Contains("ContentTooLargeForPipeline", StringComparison.Ordinal)); + Assert.Empty(declared.Output); + Assert.Equal(0, declared.BytesServed); + + // 2. Chunked: no Content-Length to check up front, so the guard has to trip during the + // copy. One byte over is the interesting case - it must not be admitted, and the + // cmdlet must stop reading rather than buffer the rest of a multi-gigabyte body. + var chunked = InvokeContentOverSizedBody(declaredLength: null, actualBytes: 100 * MiB + 1); + Assert.Contains(chunked.ErrorIds, + id => id.Contains("ContentTooLargeForPipeline", StringComparison.Ordinal)); + Assert.Empty(chunked.Output); + Assert.InRange(chunked.BytesServed, 100 * MiB + 1, 101 * MiB); + + // 3. Brackets the guard from below, so a cmdlet that simply refuses everything cannot + // pass: a body declared just under 100 MB still reaches the pipeline as bytes. + var admitted = InvokeContentOverSizedBody(declaredLength: 96 * MiB, actualBytes: 4096); + Assert.Empty(admitted.ErrorIds); + var bytes = Assert.IsType(Assert.Single(admitted.Output).BaseObject); + Assert.Equal(4096, bytes.Length); + } + + // --- byte ranges: what -First / -Offset -Length put on the wire --- + + /// + /// An origin that honours Range the way SharePoint and a well-behaved CDN do: it serves + /// exactly the bytes asked for and answers 206, so nothing downstream trims the body. That + /// makes the number of bytes the caller receives a direct readout of the range mgx asked + /// for - the local truncation path only runs on a 200, and would mask the difference. + /// + private sealed class RangeHonouringHandler(byte[] body) : HttpMessageHandler + { + /// The Range header as it went out, or null when none was sent. + public string? RangeHeader { get; private set; } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + RangeHeader = request.Headers.Range?.ToString(); + var span = request.Headers.Range?.Ranges.Single(); + + HttpResponseMessage response; + if (span?.From is not { } from) + { + response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(body) + }; + } + else + { + // Inclusive on both ends, per RFC 9110: bytes=0-4 is five bytes. + var to = (int)Math.Min(span.To ?? body.Length - 1, body.Length - 1); + response = new HttpResponseMessage(HttpStatusCode.PartialContent) + { + Content = new ByteArrayContent(body[(int)from..(to + 1)]) + }; + response.Content.Headers.TryAddWithoutValidation( + "Content-Range", $"bytes {from}-{to}/{body.Length}"); + } + + response.RequestMessage = request; + return Task.FromResult(response); + } + } + + /// Runs Get-MgxContent -Uri (plus the given range parameters) against that origin. + private (byte[] Bytes, string? RangeHeader) InvokeContentOverRangeHonouringOrigin( + byte[] body, Dictionary rangeParameters) + { + var handler = new RangeHonouringHandler(body); + using var http = new HttpClient(handler); + try + { + InjectCmdletTransport(owned: true, client: http); + using var ps = CreateContentShell(); + ps.AddCommand("Get-MgxContent").AddParameter("Uri", "/me/drive/items/01ABC/content"); + foreach (var (name, value) in rangeParameters) + ps.AddParameter(name, value); + + var output = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + return (Assert.IsType(Assert.Single(output).BaseObject), handler.RangeHeader); + } + finally + { + ResetCmdletTransport(); + } + } + + [Fact] + public void GetMgxContent_First_N_fetches_exactly_N_bytes() + { + var body = Encoding.UTF8.GetBytes("ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + + // -First 5 has to mean five bytes. Byte ranges are inclusive at both ends, so the last + // byte requested is 4: asking for 0-5 pulls a sixth byte off the wire that the caller + // never asked for and never sees flagged, because the server honored the range and no + // local truncation runs. On the 256 KB-header scans this parameter exists for, that is + // an extra byte per file and a header boundary landing one byte late. + var (first, firstRange) = InvokeContentOverRangeHonouringOrigin(body, new() { ["First"] = 5L }); + Assert.Equal("ABCDE", Encoding.UTF8.GetString(first)); + Assert.Equal("bytes=0-4", firstRange); + + // The -Offset/-Length spelling of the same idea: 4 bytes from offset 5, end inclusive. + var (window, windowRange) = InvokeContentOverRangeHonouringOrigin( + body, new() { ["Offset"] = 5L, ["Length"] = 4L }); + Assert.Equal("FGHI", Encoding.UTF8.GetString(window)); + Assert.Equal("bytes=5-8", windowRange); + + // Unranged, the very same origin hands over the whole file - so the byte counts above + // are the range mgx asked for, not the limit of the fixture. + var (whole, wholeRange) = InvokeContentOverRangeHonouringOrigin(body, new()); + Assert.Equal(body, whole); + Assert.Null(wholeRange); + } + + /// + /// An origin that ignores Range and answers 200 with the whole body - what the profile + /// photo endpoint and a few CDN edges actually do. The requested slice then has to be cut + /// out locally, or -Offset means nothing. + /// + private sealed class RangeIgnoringHandler(byte[] body) : HttpMessageHandler + { + /// The Range header the cmdlet sent, or null when none was sent. + public string? RangeHeader { get; private set; } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + RangeHeader = request.Headers.Range?.ToString(); + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(body), + RequestMessage = request + }); + } + } + + /// Runs Get-MgxContent -Uri (plus range parameters) against a range-ignoring origin. + private (byte[] Bytes, string? RangeHeader) InvokeContentOverRangeIgnoringOrigin( + byte[] body, Dictionary rangeParameters) + { + var handler = new RangeIgnoringHandler(body); + using var http = new HttpClient(handler); + try + { + InjectCmdletTransport(owned: true, client: http); + using var ps = CreateContentShell(); + ps.AddCommand("Get-MgxContent").AddParameter("Uri", "/me/drive/items/01ABC/content"); + foreach (var (name, value) in rangeParameters) + ps.AddParameter(name, value); + + var output = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + return (Assert.IsType(Assert.Single(output).BaseObject), handler.RangeHeader); + } + finally + { + ResetCmdletTransport(); + } + } + + [Fact] + public void GetMgxContent_honours_Offset_locally_when_the_server_ignores_the_range() + { + var body = Encoding.UTF8.GetBytes("ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + + // The server was asked for bytes 5-8 and sent all 26 anyway (HTTP 200). The caller + // still asked for a window starting at 5, so the head has to be discarded here: the + // failure mode this guards is silent, not loud - handing back "ABCD" and reporting + // success looks exactly like a correct read to everything downstream. + var (window, rangeHeader) = InvokeContentOverRangeIgnoringOrigin( + body, new() { ["Offset"] = 5L, ["Length"] = 4L }); + Assert.Equal("bytes=5-8", rangeHeader); + Assert.Equal("FGHI", Encoding.UTF8.GetString(window)); + + // -First 5 is the offset-zero spelling: nothing to discard, so the same path must take + // the bytes from the head. This is what stops "skip a fixed amount" from passing. + var (first, firstRange) = InvokeContentOverRangeIgnoringOrigin(body, new() { ["First"] = 5L }); + Assert.Equal("bytes=0-4", firstRange); + Assert.Equal("ABCDE", Encoding.UTF8.GetString(first)); + + // -OutFile takes the same truncated copy, and it is the path where wrong bytes get + // written to disk and outlive the session. + var outFile = Path.Combine(Path.GetTempPath(), $"mgx-offset-{Guid.NewGuid():N}.bin"); + var handler = new RangeIgnoringHandler(body); + using var http = new HttpClient(handler); + try + { + InjectCmdletTransport(owned: true, client: http); + using var ps = CreateContentShell(); + ps.AddCommand("Get-MgxContent") + .AddParameter("Uri", "/me/drive/items/01ABC/content") + .AddParameter("Offset", 20L) + .AddParameter("Length", 6L) + .AddParameter("OutFile", outFile); + ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + Assert.Equal("UVWXYZ", Encoding.UTF8.GetString(File.ReadAllBytes(outFile))); + } + finally + { + ResetCmdletTransport(); + if (File.Exists(outFile)) File.Delete(outFile); + } + } + + // --- content flows --- + + [Fact] + public async Task Direct_200_from_graph_is_returned_without_hop2() + { + // Attachments and photos serve bytes straight from Graph - no redirect involved. + _graphHandler.QueueResponse(HttpStatusCode.OK, "attachment-bytes"); + + using var result = await _client.GetContentAsync( + "https://graph.microsoft.com/v1.0/me/messages/1/attachments/2/$value"); + + Assert.Equal("attachment-bytes", await ReadAllAsync(result)); + Assert.False(result.FromDownloadHost); + Assert.Empty(_cdnHandler.Requests); + } + + [Fact] + public async Task Expired_preauth_url_gets_one_fresh_hop1_then_succeeds() + { + QueueRedirectToCdn(); + _cdnHandler.QueueResponse(HttpStatusCode.Forbidden); // expired pre-auth URL + QueueRedirectToCdn(); // fresh hop 1 + _cdnHandler.QueueResponse(HttpStatusCode.PartialContent, "x"); // fresh URL works + + using var result = await _client.GetContentAsync( + GraphContentUrl, new RangeHeaderValue(0, 0)); + + Assert.Equal("x", await ReadAllAsync(result)); + Assert.Equal(2, _graphHandler.RequestCount); + Assert.Equal(2, _cdnHandler.Requests.Count); + } + + [Fact] + public async Task Persistent_403_after_refresh_throws() + { + QueueRedirectToCdn(); + _cdnHandler.QueueResponse(HttpStatusCode.Forbidden); + QueueRedirectToCdn(); + _cdnHandler.QueueResponse(HttpStatusCode.Forbidden); + + var ex = await Assert.ThrowsAsync( + () => _client.GetContentAsync(GraphContentUrl)); + + Assert.Equal(HttpStatusCode.Forbidden, ex.StatusCode); + } + + [Fact] + public async Task Hop2_retries_transient_503() + { + QueueRedirectToCdn(); + _cdnHandler.QueueResponse(HttpStatusCode.ServiceUnavailable); + _cdnHandler.QueueResponse(HttpStatusCode.OK, "after-retry"); + + using var result = await _client.GetContentAsync(GraphContentUrl); + + Assert.Equal("after-retry", await ReadAllAsync(result)); + Assert.Equal(2, _cdnHandler.Requests.Count); + } + + [Fact] + public async Task Graph_error_surfaces_as_GraphServiceException() + { + _graphHandler.QueueResponse(HttpStatusCode.NotFound, + """{"error":{"code":"itemNotFound","message":"gone"}}"""); + + var ex = await Assert.ThrowsAsync( + () => _client.GetContentAsync(GraphContentUrl)); + + Assert.Equal(HttpStatusCode.NotFound, ex.StatusCode); + } + + // --- the copy helper: truncation and idle timeout --- + + [Fact] + public async Task Copy_truncates_at_maxBytes_when_the_server_ignored_range() + { + using var source = new MemoryStream(Encoding.UTF8.GetBytes("full body that the server sent anyway")); + using var destination = new MemoryStream(); + + var copied = await GraphContentClient.CopyWithIdleTimeoutAsync( + source, destination, maxBytes: 9, TimeSpan.FromSeconds(5), CancellationToken.None); + + Assert.Equal(9, copied); + Assert.Equal("full body", Encoding.UTF8.GetString(destination.ToArray())); + } + + [Fact] + public async Task Copy_honours_the_offset_when_the_server_ignored_range() + { + // The offset case of the test above. A server that answers a ranged request with 200 + // and the whole body ignored the START of the range as well as its length, so the head + // must be discarded locally. Before this was handled, -Offset 5 -Length 4 returned + // "full" instead of "body" and reported success - silently wrong bytes. + using var source = new MemoryStream(Encoding.UTF8.GetBytes("full body that the server sent anyway")); + using var destination = new MemoryStream(); + + var copied = await GraphContentClient.CopyWithIdleTimeoutAsync( + source, destination, maxBytes: 4, TimeSpan.FromSeconds(5), CancellationToken.None, + skipBytes: 5); + + Assert.Equal(4, copied); + Assert.Equal("body", Encoding.UTF8.GetString(destination.ToArray())); + } + + [Fact] + public async Task Copy_throws_when_the_body_ends_before_the_offset() + { + // Returning 0 here was wrong, and this test previously asserted it. A count of zero is + // indistinguishable from a legitimately empty resource, and WriteToFile does not inspect + // the count - it moved the empty temp over the destination with overwrite: true, + // destroying an existing file and exiting successfully. Throwing is the only signal the + // caller can act on. + using var source = new MemoryStream(Encoding.UTF8.GetBytes("short")); + using var destination = new MemoryStream(); + + var ex = await Assert.ThrowsAsync( + () => GraphContentClient.CopyWithIdleTimeoutAsync( + source, destination, maxBytes: 10, TimeSpan.FromSeconds(5), CancellationToken.None, + skipBytes: 500)); + + Assert.Contains("before the requested offset", ex.Message); + Assert.Empty(destination.ToArray()); + } + + [Fact] + public async Task An_empty_resource_still_writes_an_empty_result() + { + // The counterpart: a genuinely empty body with no offset requested must NOT throw. The + // fix above must distinguish "the slice does not exist" from "the resource is empty", + // which a bare copied == 0 check would not. + using var source = new MemoryStream([]); + using var destination = new MemoryStream(); + + var copied = await GraphContentClient.CopyWithIdleTimeoutAsync( + source, destination, maxBytes: null, TimeSpan.FromSeconds(5), CancellationToken.None); + + Assert.Equal(0, copied); + Assert.Empty(destination.ToArray()); + } + + [Fact] + public async Task A_redirect_counts_as_success_only_for_the_caller_that_expects_one() + { + // Content path: the 302 is the documented success path, so it must not book a failure. + QueueRedirectToCdn(); + _cdnHandler.QueueResponse(HttpStatusCode.OK, "payload!"); + MgxTelemetryCollector.Current.Reset(); + + using var result = await _client.GetContentAsync(GraphContentUrl); + await ReadAllAsync(result); + + Assert.Equal(0, MgxTelemetryCollector.Current.GetSummary().Failed); + + // Ordinary path: AllowAutoRedirect is off and every non-content caller throws on a 3xx, + // so booking it as succeeded would report success for a request the user saw fail. + _graphHandler.QueueResponse(HttpStatusCode.Found, null, + new() { ["Location"] = "https://contoso-my.sharepoint.com/download.aspx" }); + MgxTelemetryCollector.Current.Reset(); + + using var plain = await _client.SendAsync( + HttpMethod.Get, "https://graph.microsoft.com/v1.0/reports/x"); + + Assert.Equal(HttpStatusCode.Found, plain.StatusCode); + var summary = MgxTelemetryCollector.Current.GetSummary(); + Assert.Equal(1, summary.Failed); + Assert.Equal(0, summary.Succeeded); + } + + private sealed class StalledStream : Stream + { + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken) + { + await Task.Delay(Timeout.Infinite, cancellationToken); + return 0; + } + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => 0; set => throw new NotSupportedException(); } + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + } + + [Fact] + public async Task Copy_aborts_a_stalled_stream_within_the_idle_timeout() + { + using var destination = new MemoryStream(); + + var ex = await Assert.ThrowsAsync( + () => GraphContentClient.CopyWithIdleTimeoutAsync( + new StalledStream(), destination, null, + TimeSpan.FromMilliseconds(200), CancellationToken.None)); + + Assert.Contains("stalled", ex.Message); + } + + [Fact] + public async Task Content_bytes_reach_telemetry() + { + _graphHandler.QueueResponse(HttpStatusCode.OK, "12345678"); + MgxTelemetryCollector.Current.Reset(); + + using var result = await _client.GetContentAsync( + "https://graph.microsoft.com/v1.0/me/photo/$value"); + await ReadAllAsync(result); + + Assert.Equal(8, MgxTelemetryCollector.Current.GetSummary().ContentBytesDownloaded); + } + + [Fact] + public async Task Redirect_on_the_content_path_is_not_counted_as_a_failed_request() + { + // Graph answers /content with a 302 to a pre-authenticated download host. That is the + // documented success path, not an error - but IsSuccessStatusCode is 2xx-only, so + // classifying on it alone made every successful two-hop download register as a + // failure. Observed live: five successful downloads reported as Succeeded=1 Failed=5. + QueueRedirectToCdn(); + _cdnHandler.QueueResponse(HttpStatusCode.OK, "payload!"); + MgxTelemetryCollector.Current.Reset(); + + using var result = await _client.GetContentAsync(GraphContentUrl); + await ReadAllAsync(result); + + var summary = MgxTelemetryCollector.Current.GetSummary(); + Assert.Equal(0, summary.Failed); + Assert.True(summary.Succeeded >= 1, $"expected a success, got {summary.Succeeded}"); + } +} \ No newline at end of file diff --git a/tests/Mgx.IntegrationTests/GuardTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/GuardTests.cs similarity index 100% rename from tests/Mgx.IntegrationTests/GuardTests.cs rename to tests/Mgx.IntegrationTests/Engine/Http/GuardTests.cs diff --git a/tests/Mgx.IntegrationTests/ObservabilityTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/ObservabilityTests.cs similarity index 100% rename from tests/Mgx.IntegrationTests/ObservabilityTests.cs rename to tests/Mgx.IntegrationTests/Engine/Http/ObservabilityTests.cs diff --git a/tests/Mgx.IntegrationTests/RateLimiterBatchTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/RateLimiterBatchTests.cs similarity index 100% rename from tests/Mgx.IntegrationTests/RateLimiterBatchTests.cs rename to tests/Mgx.IntegrationTests/Engine/Http/RateLimiterBatchTests.cs diff --git a/tests/Mgx.IntegrationTests/RateLimiterTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/RateLimiterTests.cs similarity index 100% rename from tests/Mgx.IntegrationTests/RateLimiterTests.cs rename to tests/Mgx.IntegrationTests/Engine/Http/RateLimiterTests.cs diff --git a/tests/Mgx.IntegrationTests/Engine/RequestTraceTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/RequestTraceTests.cs similarity index 77% rename from tests/Mgx.IntegrationTests/Engine/RequestTraceTests.cs rename to tests/Mgx.IntegrationTests/Engine/Http/RequestTraceTests.cs index c155870..ea05233 100644 --- a/tests/Mgx.IntegrationTests/Engine/RequestTraceTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Http/RequestTraceTests.cs @@ -79,6 +79,36 @@ public async Task Traces_the_response_and_leaves_the_body_readable() Assert.Equal(payload, await response.Content.ReadAsStringAsync(Ct)); } + [Theory] + // Query-borne capability: the SharePoint/OneDrive-for-Business download shape. + [InlineData("https://contoso-my.sharepoint.com/personal/x/_layouts/15/download.aspx?UniqueId=1&tempauth=THE_SECRET&ApiVersion=2.0", "THE_SECRET")] + // Path-borne capability: some download hosts put it in a path segment instead, so cutting + // the trace at '?' is not enough - only host-level truncation covers both. + [InlineData("https://public.bl.files.1drv.com/y4mTHE_SECRET/file.bin", "THE_SECRET")] + public async Task Redacts_the_capability_in_a_content_redirect_Location(string location, string secret) + { + var handler = new StubHttpMessageHandler().Enqueue(_ => + { + var response = new HttpResponseMessage(HttpStatusCode.Found); + response.Headers.Location = new Uri(location); + return response; + }); + var (client, http, debug) = NewClient(handler, debugEnabled: true); + using var _ = http; + + using var response = await client.SendAsync(HttpMethod.Get, + "https://graph.microsoft.com/v1.0/drives/d/items/i/content", cancellationToken: Ct); + client.DrainDebugMessages(); + + var trace = Assert.Single(debug, m => m.StartsWith("[Mgx] Response", StringComparison.Ordinal)); + // A pre-authenticated URL grants the bytes without a bearer token. -Debug output is what + // users paste into issue reports, so the capability must never reach it. + Assert.DoesNotContain(secret, trace, StringComparison.Ordinal); + Assert.Contains("", trace); + // The host is the diagnostic value and is safe to keep. + Assert.Contains(new Uri(location).Host, trace, StringComparison.Ordinal); + } + [Fact] public async Task Redacts_credentials_from_the_traced_body() { diff --git a/tests/Mgx.IntegrationTests/Engine/Http/RetryAfterDateFormTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/RetryAfterDateFormTests.cs new file mode 100644 index 0000000..5d75582 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/RetryAfterDateFormTests.cs @@ -0,0 +1,63 @@ +using System.Net; +using System.Net.Http.Headers; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests.Engine.Http; + +/// +/// Retry-After has two legal forms - delta-seconds and an HTTP-date. The content path honoured +/// only Delta, so a download host choosing the date form fell back to plain exponential backoff +/// with nothing to say so. These pin the parsing both ways. +/// +public class RetryAfterDateFormTests +{ + // The production resolver itself. This used to be a local copy of it, annotated "mirrors + // GraphContentClient's DelayGenerator" - so every test below asserted that a copy of the + // code behaved like itself, and deleting the HTTP-date branch from the product left all + // 567 tests green. + private static TimeSpan? Resolve(RetryConditionHeaderValue? retryAfter) => + GraphContentClient.ResolveRetryDelay(retryAfter); + + [Fact] + public void Delta_form_is_honoured() + { + var d = Resolve(new RetryConditionHeaderValue(TimeSpan.FromSeconds(7))); + Assert.NotNull(d); + Assert.Equal(7, d!.Value.TotalSeconds, 0); + } + + [Fact] + public void Date_form_is_honoured() + { + var d = Resolve(new RetryConditionHeaderValue(DateTimeOffset.UtcNow.AddSeconds(30))); + Assert.NotNull(d); + Assert.InRange(d!.Value.TotalSeconds, 20, 31); + } + + [Fact] + public void A_past_date_yields_no_delay_rather_than_a_negative_one() + { + Assert.Null(Resolve(new RetryConditionHeaderValue(DateTimeOffset.UtcNow.AddSeconds(-60)))); + } + + [Fact] + public void A_far_future_date_is_capped_at_two_minutes() + { + var d = Resolve(new RetryConditionHeaderValue(DateTimeOffset.UtcNow.AddHours(6))); + Assert.NotNull(d); + Assert.Equal(120, d!.Value.TotalSeconds, 0); + } + + [Fact] + public void A_huge_delta_is_capped_too() + { + var d = Resolve(new RetryConditionHeaderValue(TimeSpan.FromHours(6))); + Assert.Equal(120, d!.Value.TotalSeconds, 0); + } + + [Fact] + public void No_header_yields_no_delay() + { + Assert.Null(Resolve(null)); + } +} diff --git a/tests/Mgx.IntegrationTests/RetryTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/RetryTests.cs similarity index 100% rename from tests/Mgx.IntegrationTests/RetryTests.cs rename to tests/Mgx.IntegrationTests/Engine/Http/RetryTests.cs diff --git a/tests/Mgx.IntegrationTests/TransportConfigTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/TransportConfigTests.cs similarity index 100% rename from tests/Mgx.IntegrationTests/TransportConfigTests.cs rename to tests/Mgx.IntegrationTests/Engine/Http/TransportConfigTests.cs diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/AtomicSaveHygieneTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/AtomicSaveHygieneTests.cs new file mode 100644 index 0000000..146f0a0 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/AtomicSaveHygieneTests.cs @@ -0,0 +1,77 @@ +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// Both state files are written to "<path>.tmp" and renamed, so a crash cannot leave a +/// half-written file. A save that FAILS should not leave the staging file either - adoption and +/// recovery already have to reason about stray files beside the output. +/// +[Collection("Pipeline")] +public class AtomicSaveHygieneTests +{ + private static string NewDir() => + Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-atomic-{Guid.NewGuid():N}")).FullName; + + [Fact] + public void A_failed_checkpoint_save_leaves_no_staging_file() + { + var dir = NewDir(); + var target = Path.Combine(dir, "run.checkpoint"); + try + { + // The rename cannot land on a directory, so the save fails after the staging file + // has been written - the window this is about. + Directory.CreateDirectory(target); + + var cp = new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=x", + ItemsCollected = 1 + }; + Assert.ThrowsAny(() => cp.Save(target)); + + Assert.False(File.Exists(target + ".tmp"), + "the staging file outlived the save that failed to promote it"); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } +} + +/// +/// A delta state file from before this field existed carries no apiVersion, so the mismatch +/// check is skipped and the run proceeds against whatever version the stored deltaLink names. +/// Recording the REQUESTED version at that point stamped one the token was never issued by, and +/// every later run refused with advice pointing at the wrong version. +/// +[Collection("Pipeline")] +public class DeltaApiVersionStampTests +{ + private static string? Stamp(string? link) + { + var m = typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).GetMethod( + "ApiVersionOfLink", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!; + return (string?)m.Invoke(null, [link]); + } + + [Theory] + [InlineData("https://graph.microsoft.com/v1.0/users/delta?$deltatoken=abc", "v1.0")] + [InlineData("https://graph.microsoft.com/beta/users/delta?$deltatoken=abc", "beta")] + public void The_version_comes_from_the_link(string link, string expected) + { + Assert.Equal(expected, Stamp(link)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData("not-a-url")] + [InlineData("https://graph.microsoft.com/v9.9/users/delta")] + public void An_unreadable_link_falls_back_to_the_request(string? link) + { + Assert.Null(Stamp(link)); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/CheckpointRetentionTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/CheckpointRetentionTests.cs new file mode 100644 index 0000000..999d963 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/CheckpointRetentionTests.cs @@ -0,0 +1,74 @@ +using System.Reflection; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests.Engine.Pagination; + +/// +/// TryAdoptOrphanedTemp's contract when there is nothing to adopt: report failure and leave the +/// existing output exactly as it was. +/// +/// Scope note: these are unit tests of the helper. They cannot guard the caller's decision about +/// what a failed adoption MEANS - the helper never opens the checkpoint, so no assertion here can +/// distinguish "checkpoint kept" from "checkpoint deleted". That decision lives in SyncMgxDelta +/// and is covered end to end by DeltaRecoveryTests, which drives the cmdlet against a mock +/// transport. +/// +public class CheckpointRetentionTests +{ + private static readonly MethodInfo Adopt = + typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( + "TryAdoptOrphanedTemp", BindingFlags.Static | BindingFlags.NonPublic)!; + + private static bool TryAdopt(string outputPath, long itemCount) => + (bool)Adopt.Invoke(null, [outputPath, itemCount])!; + + private static string NewDir() + { + var d = Path.Combine(Path.GetTempPath(), $"mgx-ckpt-{Guid.NewGuid():N}"); + Directory.CreateDirectory(d); + return d; + } + + /// + /// The exact second-interruption state: checkpoint + output + NO temp. Adoption must fail, + /// and the caller must NOT read that as a missing output. + /// + [Fact] + public void Adoption_fails_when_a_resumed_run_left_no_temp() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllLines(output, ["{\"id\":\"baseline-1\"}", "{\"id\":\"baseline-2\"}"]); + + Assert.False(TryAdopt(output, 2)); + + // The caller distinguishes these two facts. Conflating them is what destroyed data: + // adoption failing is routine; the output being absent is not. + Assert.True(File.Exists(output)); + Assert.Equal(2, File.ReadAllLines(output).Length); + } + finally { Directory.Delete(dir, true); } + } + + [Fact] + public void A_failed_adoption_leaves_the_existing_output_untouched() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllLines(output, ["{\"id\":\"baseline\"}"]); + // Temp holds fewer lines than the checkpoint promises: adoption must refuse. + File.WriteAllLines($"{output}.aaa.tmp", ["{\"id\":\"partial\"}"]); + + Assert.False(TryAdopt(output, 99)); + + Assert.Equal(["{\"id\":\"baseline\"}"], File.ReadAllLines(output)); + Assert.False(File.Exists($"{output}.adopt")); + } + finally { Directory.Delete(dir, true); } + } + +} diff --git a/tests/Mgx.IntegrationTests/CheckpointTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/CheckpointTests.cs similarity index 100% rename from tests/Mgx.IntegrationTests/CheckpointTests.cs rename to tests/Mgx.IntegrationTests/Engine/Pagination/CheckpointTests.cs diff --git a/tests/Mgx.IntegrationTests/FanOutTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/FanOutTests.cs similarity index 100% rename from tests/Mgx.IntegrationTests/FanOutTests.cs rename to tests/Mgx.IntegrationTests/Engine/Pagination/FanOutTests.cs diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/OrphanAdoptionTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/OrphanAdoptionTests.cs new file mode 100644 index 0000000..63a9ad2 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/OrphanAdoptionTests.cs @@ -0,0 +1,101 @@ +using System.Reflection; + +namespace Mgx.IntegrationTests.Engine.Pagination; + +/// +/// Orphan-temp adoption after a JSONL run is interrupted, for checkpoints that predate the +/// recorded temp name and length. With only a line count and a glob to go on, adoption is +/// safe solely when no output exists; against an existing output it must refuse and leave the +/// caller to re-enumerate, since nothing ties the newest temp to the run the checkpoint is +/// actually about. +/// +public class OrphanAdoptionTests +{ + private static readonly MethodInfo Adopt = + typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( + "TryAdoptOrphanedTemp", BindingFlags.Static | BindingFlags.NonPublic)!; + + private static bool Invoke(string outputPath, long itemCount) => + (bool)Adopt.Invoke(null, [outputPath, itemCount])!; + + private static string NewDir() + { + var d = Path.Combine(Path.GetTempPath(), $"mgx-adopt-{Guid.NewGuid():N}"); + Directory.CreateDirectory(d); + return d; + } + + [Fact] + public void Adopts_the_temp_when_no_output_exists_yet() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllLines($"{output}.abc.tmp", ["{\"id\":\"a\"}", "{\"id\":\"b\"}", "{\"id\":\"c\"}"]); + + Assert.True(Invoke(output, 2)); + + // Trimmed to the checkpointed count: the third line was written after the last flush. + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + Assert.Empty(Directory.GetFiles(dir, "*.tmp")); + } + finally { Directory.Delete(dir, true); } + } + + [Fact] + public void Refuses_when_an_output_already_exists() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + // Run 1 completed. + File.WriteAllLines(output, ["{\"id\":\"run1-1\"}", "{\"id\":\"run1-2\"}"]); + // Run 2 crashed: a temp is on disk, but nothing says it is run 2's rather than a + // survivor of some unrelated enumeration against the same output name. + File.WriteAllLines($"{output}.def.tmp", ["{\"id\":\"run2-1\"}", "{\"id\":\"run2-2\"}", "{\"id\":\"unflushed\"}"]); + + Assert.False(Invoke(output, 2)); + + // The output is untouched; recovery falls to re-enumeration, not a guessed merge. + Assert.Equal( + ["{\"id\":\"run1-1\"}", "{\"id\":\"run1-2\"}"], + File.ReadAllLines(output)); + } + finally { Directory.Delete(dir, true); } + } + + [Fact] + public void Refuses_when_the_temp_holds_less_than_the_checkpoint_promises() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllLines(output, ["{\"id\":\"kept\"}"]); + File.WriteAllLines($"{output}.ghi.tmp", ["{\"id\":\"only-one\"}"]); + + Assert.False(Invoke(output, 5)); + + // A refused adoption must not touch the existing output, nor leave staging behind. + Assert.Equal(["{\"id\":\"kept\"}"], File.ReadAllLines(output)); + Assert.False(File.Exists($"{output}.adopt")); + } + finally { Directory.Delete(dir, true); } + } + + [Fact] + public void Does_nothing_when_there_is_no_temp() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllLines(output, ["{\"id\":\"kept\"}"]); + Assert.False(Invoke(output, 3)); + Assert.Equal(["{\"id\":\"kept\"}"], File.ReadAllLines(output)); + } + finally { Directory.Delete(dir, true); } + } +} diff --git a/tests/Mgx.IntegrationTests/PaginationTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/PaginationTests.cs similarity index 77% rename from tests/Mgx.IntegrationTests/PaginationTests.cs rename to tests/Mgx.IntegrationTests/Engine/Pagination/PaginationTests.cs index 65c32b3..6ea5788 100644 --- a/tests/Mgx.IntegrationTests/PaginationTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/PaginationTests.cs @@ -151,4 +151,38 @@ public async Task PageIterator_PassesHeadersOnAllPages() Assert.Equal("eventual", req.Headers.GetValues("ConsistencyLevel").First()); } } + + [Fact] + public async Task PageIterator_RejectedNextLink_Throws_RatherThanTruncatingSilently() + { + // A nextLink that fails validation is indistinguishable, at the loop level, from "no + // more pages" - both leave the validator returning null. Ending the iteration quietly + // hands the caller a partial collection and exit code 0, which is the failure mode the + // validator exists to prevent: against a response that is actually hostile, silence + // lets the sender truncate the result set and never be noticed. + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, """ + { + "value": [ + {"id": "user1", "displayName": "User One"}, + {"id": "user2", "displayName": "User Two"} + ], + "@odata.nextLink": "https://evil.example.com/v1.0/users?$skiptoken=page2" + } + """); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var iterator = new PageIterator(client); + + var items = new List(); + var ex = await Assert.ThrowsAsync(async () => + { + await foreach (var item in iterator.StreamAllWithCountAsync("https://graph.microsoft.com/v1.0/users", 0, null)) + items.Add(item); + }); + + Assert.Contains("nextLink", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Equal(2, items.Count); // the valid page is still delivered before the failure + } } diff --git a/tests/Mgx.IntegrationTests/SsrfValidationTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/SsrfValidationTests.cs similarity index 67% rename from tests/Mgx.IntegrationTests/SsrfValidationTests.cs rename to tests/Mgx.IntegrationTests/Engine/Pagination/SsrfValidationTests.cs index 9daccb8..79426c4 100644 --- a/tests/Mgx.IntegrationTests/SsrfValidationTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/SsrfValidationTests.cs @@ -25,14 +25,58 @@ public async Task PageIterator_RejectsNextLink_DifferentHost() var iterator = new PageIterator(client); var items = new List(); - await foreach (var item in iterator.StreamAllWithCountAsync("https://graph.microsoft.com/v1.0/users", 0, null)) + await Assert.ThrowsAsync(async () => + { + await foreach (var item in iterator.StreamAllWithCountAsync("https://graph.microsoft.com/v1.0/users", 0, null)) + { + items.Add(item); + } + }); + + // Page 1 is still delivered and the malicious nextLink is never fetched - but the + // caller is told the collection is incomplete rather than left to assume it is whole. + Assert.Single(items); + Assert.Equal(1, handler.RequestCount); // Only initial request, no second page fetch + } + + /// + /// A refused deltaLink is dropped, not thrown: by the time it arrives every item has been + /// delivered, so the enumeration completes and only the token fails to advance. The drop + /// must not be silent - without a warning the caller reports "no delta token received" + /// and misdiagnoses a refused token as an endpoint that does not support delta. + /// + [Fact] + public async Task PageIterator_RejectedDeltaLink_CompletesWithAWarningInsteadOfSavingIt() + { + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, """ + { + "value": [{"id": "user1"}, {"id": "user2"}], + "@odata.deltaLink": "https://evil.com/steal-token?$deltatoken=abc" + } + """); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var iterator = new PageIterator(client); + + string? capturedDeltaLink = null; + var items = new List(); + await foreach (var item in iterator.StreamAllWithCountAsync( + "https://graph.microsoft.com/v1.0/users/delta", 0, null, + onDeltaLink: dl => capturedDeltaLink = dl)) { items.Add(item); } - // Should get page 1 items but NOT follow the malicious nextLink - Assert.Single(items); - Assert.Equal(1, handler.RequestCount); // Only initial request, no second page fetch + Assert.Equal(2, items.Count); // the enumeration completed + Assert.Null(capturedDeltaLink); // the poisoned token was never surfaced + Assert.Equal(1, handler.RequestCount); // and never fetched + + var warnings = new List(); + client.WarningWriter = warnings.Add; + client.DrainWarningMessages(); + Assert.Contains(warnings, w => w.Contains("deltaLink") && w.Contains("failed validation")); } [Fact] @@ -52,13 +96,16 @@ public async Task PageIterator_RejectsNextLink_HttpScheme() var iterator = new PageIterator(client); var items = new List(); - await foreach (var item in iterator.StreamAllWithCountAsync("https://graph.microsoft.com/v1.0/users", 0, null)) + await Assert.ThrowsAsync(async () => { - items.Add(item); - } - - Assert.Single(items); - Assert.Equal(1, handler.RequestCount); + await foreach (var item in iterator.StreamAllWithCountAsync("https://graph.microsoft.com/v1.0/users", 0, null)) + { + items.Add(item); + } + }); + + Assert.Single(items); // page 1 was still delivered + Assert.Equal(1, handler.RequestCount); // and the refused link was never fetched } [Fact] @@ -78,13 +125,16 @@ public async Task PageIterator_RejectsNextLink_DifferentPort() var iterator = new PageIterator(client); var items = new List(); - await foreach (var item in iterator.StreamAllWithCountAsync("https://graph.microsoft.com/v1.0/users", 0, null)) + await Assert.ThrowsAsync(async () => { - items.Add(item); - } - - Assert.Single(items); - Assert.Equal(1, handler.RequestCount); + await foreach (var item in iterator.StreamAllWithCountAsync("https://graph.microsoft.com/v1.0/users", 0, null)) + { + items.Add(item); + } + }); + + Assert.Single(items); // page 1 was still delivered + Assert.Equal(1, handler.RequestCount); // and the refused link was never fetched } [Fact] @@ -137,9 +187,11 @@ public async Task ConcurrentFanOut_RejectsNextLink_DifferentHost() var result = await fanOut.FetchAllAsync( ["https://graph.microsoft.com/v1.0/groups/g1/members"]); - // Should get page 1 items but NOT follow the malicious nextLink - Assert.True(result.Results.ContainsKey("https://graph.microsoft.com/v1.0/groups/g1/members")); - Assert.Single(result.Results["https://graph.microsoft.com/v1.0/groups/g1/members"]); + // The refused link is reported against its own URL instead of silently yielding a + // partial member list. Other URLs in the same fan-out are unaffected. + Assert.False(result.Results.ContainsKey("https://graph.microsoft.com/v1.0/groups/g1/members")); + Assert.True(result.Errors.ContainsKey("https://graph.microsoft.com/v1.0/groups/g1/members")); + Assert.IsType(result.Errors["https://graph.microsoft.com/v1.0/groups/g1/members"]); Assert.Equal(1, handler.RequestCount); } @@ -161,7 +213,8 @@ public async Task ConcurrentFanOut_RejectsNextLink_HttpScheme() var result = await fanOut.FetchAllAsync( ["https://graph.microsoft.com/v1.0/groups/g1/members"]); - Assert.Single(result.Results["https://graph.microsoft.com/v1.0/groups/g1/members"]); + Assert.False(result.Results.ContainsKey("https://graph.microsoft.com/v1.0/groups/g1/members")); + Assert.IsType(result.Errors["https://graph.microsoft.com/v1.0/groups/g1/members"]); Assert.Equal(1, handler.RequestCount); } @@ -183,7 +236,8 @@ public async Task ConcurrentFanOut_RejectsNextLink_DifferentPort() var result = await fanOut.FetchAllAsync( ["https://graph.microsoft.com/v1.0/groups/g1/members"]); - Assert.Single(result.Results["https://graph.microsoft.com/v1.0/groups/g1/members"]); + Assert.False(result.Results.ContainsKey("https://graph.microsoft.com/v1.0/groups/g1/members")); + Assert.IsType(result.Errors["https://graph.microsoft.com/v1.0/groups/g1/members"]); Assert.Equal(1, handler.RequestCount); } @@ -265,12 +319,15 @@ public async Task PageIterator_RejectsNextLink_MalformedUrl() var iterator = new PageIterator(client); var items = new List(); - await foreach (var item in iterator.StreamAllWithCountAsync("https://graph.microsoft.com/v1.0/users", 0, null)) + await Assert.ThrowsAsync(async () => { - items.Add(item); - } - - Assert.Single(items); - Assert.Equal(1, handler.RequestCount); + await foreach (var item in iterator.StreamAllWithCountAsync("https://graph.microsoft.com/v1.0/users", 0, null)) + { + items.Add(item); + } + }); + + Assert.Single(items); // page 1 was still delivered + Assert.Equal(1, handler.RequestCount); // and the refused link was never fetched } } diff --git a/tests/Mgx.IntegrationTests/MockHttpHandler.cs b/tests/Mgx.IntegrationTests/Fakes/MockHttpHandler.cs similarity index 93% rename from tests/Mgx.IntegrationTests/MockHttpHandler.cs rename to tests/Mgx.IntegrationTests/Fakes/MockHttpHandler.cs index 63c97a8..8b1e136 100644 --- a/tests/Mgx.IntegrationTests/MockHttpHandler.cs +++ b/tests/Mgx.IntegrationTests/Fakes/MockHttpHandler.cs @@ -73,7 +73,9 @@ protected override Task SendAsync(HttpRequestMessage reques if (mock.Exception != null) return Task.FromException(mock.Exception); - var response = new HttpResponseMessage(mock.StatusCode); + // Real transports (SocketsHttpHandler) set RequestMessage on the response; consumers + // like the pacer's OnRetry hook read the request URI off it. Mirror that here. + var response = new HttpResponseMessage(mock.StatusCode) { RequestMessage = request }; if (mock.Body != null) response.Content = new StringContent(mock.Body, Encoding.UTF8, "application/json"); diff --git a/tests/Mgx.IntegrationTests/TestSetup/PacerTestDefaults.cs b/tests/Mgx.IntegrationTests/TestSetup/PacerTestDefaults.cs new file mode 100644 index 0000000..a28a70a --- /dev/null +++ b/tests/Mgx.IntegrationTests/TestSetup/PacerTestDefaults.cs @@ -0,0 +1,18 @@ +using System.Runtime.CompilerServices; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// Disables the adaptive request pacer for the whole test assembly before any test runs. +/// Pacing is ON by default in production; without this, every pre-pacing test would inherit +/// slow-start spacing (a cold bucket caps at 4 rps) and pagination/fan-out tests would +/// multiply suite time. Pacer tests re-enable the gate locally via +/// AdaptiveRequestPacerTests.PacerScope, serialized through the Pipeline collection. +/// +internal static class PacerTestDefaults +{ + [ModuleInitializer] + internal static void DisablePacingForSuite() => + AdaptiveRequestPacer.DisabledForTests = true; +} diff --git a/tests/RELEASING.md b/tests/RELEASING.md new file mode 100644 index 0000000..c6acedb --- /dev/null +++ b/tests/RELEASING.md @@ -0,0 +1,59 @@ +# Releasing + +`main` carries one commit per release. There are no merge commits: a release is `dev` squashed +onto `main`, tagged, then published. + +Every command below runs from the repository root, not from this directory. + +## Before + +1. **Both suites, both platforms.** + ``` + dotnet test tests/Mgx.IntegrationTests/Mgx.IntegrationTests.csproj + pwsh -c 'Invoke-Pester -Path ./tests/Unit' + ``` + Run them on Windows too. A green run on one OS says nothing about file locking, path + comparison or the exceptions Windows raises where Unix raises different ones - each of those + has shipped a defect here. + +2. **The live suite, actually run.** + ``` + pwsh -c 'Invoke-Pester -Path ./tests/Live' + ``` + With credentials, against a real tenant. The mocked suites cannot see a request Graph + rejects: `Enable-MgxResilience` once shipped a client with no `BaseAddress`, breaking every + relative-URI call, with every test green. "Skipped" is not a pass - see `tests/Live/README.md`. + +3. **Install it the way a user does.** Build Release, then in a *fresh* shell with only the + Gallery dependencies present, import the staged `module/`, run `Test-ModuleManifest`, and + check `Get-Help` for every exported cmdlet. Stale generated help drops parameters silently. + +4. **Read the published numbers.** Every figure in `README.md` must be reproducible from + `tests/benchmarks` as it stands. Re-measure rather than reuse: the tenant's throttling + ceiling has moved by 40% between consecutive days. + +5. **Version and notes.** `ModuleVersion` in `module/mgx.psd1`, the manifest's `ReleaseNotes`, + and the `CHANGELOG.md` section all agree. + +## Squashing + +```bash +git checkout main && git pull origin main +git merge --squash dev +git commit -F + +git diff --stat dev # MUST be empty - if it prints anything, the squash lost something + +git tag -a vX.Y.Z -m "mgx X.Y.Z" +git push origin main && git push origin vX.Y.Z +``` + +The squash message is the public record of the release; the CHANGELOG carries the per-defect +detail. Describe what the release *is*, not the churn that produced it. + +## After + +`Publish-Module` from the tagged `main`. + +`dev` and `main` share no history once squashed, so `dev` reads as permanently ahead. Retire it +and branch a fresh one from `main` for the next version rather than carrying the divergence. diff --git a/tests/TestHarness.psm1 b/tests/TestHarness.psm1 index 61f20cc..e25ec60 100644 --- a/tests/TestHarness.psm1 +++ b/tests/TestHarness.psm1 @@ -71,7 +71,13 @@ function Invoke-TestHarness [Parameter()] [System.String] - $TestPath = (Join-Path $PSScriptRoot 'Unit') + $TestPath = (Join-Path $PSScriptRoot 'Unit'), + + # Live-tagged blocks need a real Graph session. They are excluded by default so CI + # and a cold clone both pass; pass this to run them against a connected tenant. + [Parameter()] + [Switch] + $IncludeLive ) $pesterModule = Get-Module -Name Pester -ListAvailable | @@ -98,6 +104,10 @@ function Invoke-TestHarness $configuration = New-PesterConfiguration $configuration.Run.Path = $TestPath + if (-not $IncludeLive.IsPresent) + { + $configuration.Filter.ExcludeTag = 'Live' + } $configuration.Run.PassThru = $true $configuration.Output.Verbosity = 'Detailed' $configuration.TestResult.Enabled = $true diff --git a/tests/Mgx.Tests.ps1 b/tests/Unit/Mgx.Behaviour.Tests.ps1 similarity index 81% rename from tests/Mgx.Tests.ps1 rename to tests/Unit/Mgx.Behaviour.Tests.ps1 index 5686541..89db075 100644 --- a/tests/Mgx.Tests.ps1 +++ b/tests/Unit/Mgx.Behaviour.Tests.ps1 @@ -13,7 +13,7 @@ #> BeforeAll { - $ModulePath = Join-Path $PSScriptRoot '../module/mgx.psd1' + $ModulePath = Join-Path $PSScriptRoot '../../module/mgx.psd1' Import-Module $ModulePath -Force } @@ -22,11 +22,11 @@ Describe 'Module Loading' { $module = Get-Module Mgx $module | Should -Not -BeNullOrEmpty # Read expected version from manifest to avoid hardcoded values breaking on version bumps - $manifest = Import-PowerShellDataFile (Join-Path $PSScriptRoot '../module/mgx.psd1') + $manifest = Import-PowerShellDataFile (Join-Path $PSScriptRoot '../../module/mgx.psd1') $module.Version | Should -Be $manifest.ModuleVersion } - It 'Should export exactly 11 cmdlets' { + It 'Should export exactly 12 cmdlets' { # Filtered to -CommandType Cmdlet on purpose: Get-Command -Module also counts # exported functions, so an unfiltered count would drift for reasons that have # nothing to do with the compiled surface. @@ -36,13 +36,14 @@ Describe 'Module Loading' { $commands | Should -Contain 'Export-MgxCollection' $commands | Should -Contain 'Expand-MgxRelation' $commands | Should -Contain 'Sync-MgxDelta' + $commands | Should -Contain 'Get-MgxContent' $commands | Should -Contain 'Set-MgxOption' $commands | Should -Contain 'Get-MgxOption' $commands | Should -Contain 'Enable-MgxResilience' $commands | Should -Contain 'Disable-MgxResilience' $commands | Should -Contain 'Get-MgxResilience' $commands | Should -Contain 'Get-MgxTelemetry' - $commands | Should -HaveCount 11 + $commands | Should -HaveCount 12 } It 'Should unload cleanly when no Graph request ever ran' { @@ -57,7 +58,7 @@ Describe 'Module Loading' { # # Runs in a child process because the precondition is "Polly.Core has never # been loaded", which the parent suite has already violated by this point. - $modulePath = Join-Path $PSScriptRoot '../module/mgx.psd1' + $modulePath = Join-Path $PSScriptRoot '../../module/mgx.psd1' $probe = @" Import-Module '$modulePath' -Force try { Remove-Module mgx -ErrorAction Stop } @@ -65,7 +66,30 @@ catch { Write-Output ('THREW: ' + `$_.Exception.Message.Split([char]10)[0]); exi if (Get-Module mgx) { Write-Output 'STILL LOADED'; exit 1 } Write-Output 'OK' "@ - $result = pwsh -NoProfile -Command $probe + # This assertion needs a FRESH session, so it must spawn a child host. Under a + # dotnet-tool PowerShell install on Unix there is no re-executable host: the + # process path is the dotnet muxer, and $PSHOME/pwsh is a managed wrapper that + # cannot be exec'd directly ("Exec format error"). Rather than fail there and + # look like an mgx defect, prove the host can re-exec first and skip if it cannot. + $hostExe = @( + (Get-Process -Id $PID).Path + (Get-Command pwsh -ErrorAction SilentlyContinue).Source + ) | Where-Object { $_ -and (Split-Path $_ -Leaf) -like 'pwsh*' } | Select-Object -First 1 + + if (-not $hostExe) { + Set-ItResult -Skipped -Because 'this host cannot re-exec itself (dotnet-tool PowerShell install)' + return + } + # A failed exec surfaces as a terminating NativeCommandFailed, not a return value, + # so the probe must be wrapped rather than compared. + $probeOk = $null + try { $probeOk = & $hostExe -NoProfile -Command '"alive"' 2>$null } catch { $probeOk = $null } + if ($probeOk -ne 'alive') { + Set-ItResult -Skipped -Because "host '$hostExe' cannot be launched as a child process" + return + } + + $result = & $hostExe -NoProfile -Command $probe $result | Should -Contain 'OK' $LASTEXITCODE | Should -Be 0 } @@ -946,22 +970,6 @@ Describe 'Sync-MgxDelta State Management' { "$warnings" | Should -BeLike '*does not contain*delta*' } - It 'Should detect endpoint mismatch in delta state' { - $dp = Join-Path $script:testDir 'endpoint-mismatch.json' - @{ - deltaLink = 'https://graph.microsoft.us/v1.0/users/delta?$deltatoken=abc' - select = '' - filter = $null - resource = '/users/delta' - lastSync = (Get-Date).ToString('o') - itemCount = 10 - graphEndpoint = 'https://graph.microsoft.us' - } | ConvertTo-Json | Set-Content $dp - - { Sync-MgxDelta -Uri '/users/delta' -DeltaPath $dp -ErrorAction Stop } | - Should -Throw '*graph.microsoft.us*graph.microsoft.com*' - } - It 'Should detect resource mismatch in delta state' { $dp = Join-Path $script:testDir 'resource-mismatch.json' @{ @@ -1018,38 +1026,6 @@ Describe 'Sync-MgxDelta State Management' { "$warnings" | Should -BeLike '*Filter changed*' } - It 'Should reject tampered deltaLink with wrong host (SSRF)' { - $dp = Join-Path $script:testDir 'ssrf.json' - @{ - deltaLink = 'https://evil.com/v1.0/users/delta?$deltatoken=stolen' - select = '' - filter = $null - resource = '/users/delta' - lastSync = (Get-Date).ToString('o') - itemCount = 10 - graphEndpoint = 'https://graph.microsoft.com' - } | ConvertTo-Json | Set-Content $dp - - { Sync-MgxDelta -Uri '/users/delta' -DeltaPath $dp -ErrorAction Stop } | - Should -Throw '*invalid or untrusted*' - } - - It 'Should reject tampered deltaLink with wrong resource path' { - $dp = Join-Path $script:testDir 'path-tamper.json' - @{ - deltaLink = 'https://graph.microsoft.com/v1.0/me/messages?$deltatoken=abc' - select = '' - filter = $null - resource = '/users/delta' - lastSync = (Get-Date).ToString('o') - itemCount = 10 - graphEndpoint = 'https://graph.microsoft.com' - } | ConvertTo-Json | Set-Content $dp - - { Sync-MgxDelta -Uri '/users/delta' -DeltaPath $dp -ErrorAction Stop } | - Should -Throw '*path*does not match*' - } - It 'Should warn on corrupt delta state file' { $dp = Join-Path $script:testDir 'corrupt.json' 'not valid json{{{' | Set-Content $dp @@ -1118,6 +1094,218 @@ Describe 'Sync-MgxDelta State Management' { } } +Describe 'Sync-MgxDelta Session Endpoint' { + # These scenarios need a session that is connected to a non-default endpoint. Each runs + # in a child pwsh: a custom Graph environment plus Connect-MgGraph -AccessToken gives a + # connected session without a tenant, and the child process keeps that connection (and + # the module's static endpoint state) out of this one. + BeforeAll { + $script:modulePath = Join-Path $PSScriptRoot '../../module/mgx.psd1' + $script:testDir = Join-Path ([System.IO.Path]::GetTempPath()) "mgx-endpoint-tests-$(New-Guid)" + New-Item -ItemType Directory -Path $script:testDir -Force | Out-Null + + $script:graphAuthAvailable = [bool](Get-Module -ListAvailable Microsoft.Graph.Authentication) + + # Same re-exec probe as 'Should unload cleanly when no Graph request ever ran': + # a dotnet-tool PowerShell install has no re-executable host. + $script:hostExe = @( + (Get-Process -Id $PID).Path + (Get-Command pwsh -ErrorAction SilentlyContinue).Source + ) | Where-Object { $_ -and (Split-Path $_ -Leaf) -like 'pwsh*' } | Select-Object -First 1 + if ($script:hostExe) { + $probeOk = $null + try { $probeOk = & $script:hostExe -NoProfile -Command '"alive"' 2>$null } catch { $probeOk = $null } + if ($probeOk -ne 'alive') { $script:hostExe = $null } + } + + # Custom environments persist across sessions, so every name is tracked and swept + # in AfterAll in case a child dies before its own cleanup runs. + $script:envNames = [System.Collections.Generic.List[string]]::new() + + function Get-SkipReason { + if (-not $script:hostExe) { return 'this host cannot launch a child pwsh' } + if (-not $script:graphAuthAvailable) { return 'Microsoft.Graph.Authentication is not installed' } + return $null + } + + function Invoke-ConnectedChild { + param([string]$GraphEndpoint, [string]$Body) + + $envName = "MgxTest-$([guid]::NewGuid().ToString('N').Substring(0, 8))" + $script:envNames.Add($envName) + $child = @" +Import-Module Microsoft.Graph.Authentication +Import-Module '$script:modulePath' +try { + Add-MgEnvironment -Name '$envName' -GraphEndpoint '$GraphEndpoint' -AzureADEndpoint 'https://login.microsoftonline.com' | Out-Null + Connect-MgGraph -Environment '$envName' -AccessToken (ConvertTo-SecureString 'not-a-real-token' -AsPlainText -Force) -NoWelcome + # -AccessToken leaves TenantId empty, and an empty TenantId reads as "not connected". + [Microsoft.Graph.PowerShell.Authentication.GraphSession]::Instance.AuthContext.TenantId = 'test-tenant' +$Body +} +finally { + Disconnect-MgGraph -ErrorAction SilentlyContinue | Out-Null + Remove-MgEnvironment -Name '$envName' -ErrorAction SilentlyContinue | Out-Null +} +"@ + & $script:hostExe -NoProfile -Command $child 2>&1 | ForEach-Object { "$_" } + } + + function New-DeltaStateFile { + param([string]$Name, [string]$DeltaLink, [string]$GraphEndpoint) + + $dp = Join-Path $script:testDir $Name + @{ + deltaLink = $DeltaLink + select = '' + filter = $null + resource = '/users/delta' + lastSync = (Get-Date).ToString('o') + itemCount = 10 + graphEndpoint = $GraphEndpoint + } | ConvertTo-Json | Set-Content $dp + return $dp + } + + function Get-FreePort { + $probe = [System.Net.Sockets.TcpListener]::new([System.Net.IPAddress]::Loopback, 0) + $probe.Start() + $port = ([System.Net.IPEndPoint]$probe.LocalEndpoint).Port + $probe.Stop() + return $port + } + } + + AfterAll { + if (Test-Path $script:testDir) { Remove-Item $script:testDir -Recurse -Force } + if ($script:hostExe -and $script:graphAuthAvailable -and $script:envNames.Count -gt 0) { + $names = "'" + ($script:envNames -join "','") + "'" + & $script:hostExe -NoProfile -Command "Import-Module Microsoft.Graph.Authentication; foreach (`$n in @($names)) { Remove-MgEnvironment -Name `$n -ErrorAction SilentlyContinue | Out-Null }" 2>$null | Out-Null + } + } + + It 'Should send the first request of a session to the connected endpoint' { + $reason = Get-SkipReason + if (-not $reason -and -not (Get-Command Start-ThreadJob -ErrorAction SilentlyContinue)) { + $reason = 'Start-ThreadJob is not available' + } + if ($reason) { Set-ItResult -Skipped -Because $reason; return } + + $port = Get-FreePort + $listener = [System.Net.HttpListener]::new() + $listener.Prefixes.Add("http://localhost:$port/") + $listener.Start() + $served = [System.Collections.Concurrent.ConcurrentQueue[string]]::new() + $server = Start-ThreadJob -ScriptBlock { + $listener = $using:listener + $served = $using:served + $body = [System.Text.Encoding]::UTF8.GetBytes( + '{"value":[{"id":"u1"},{"id":"u2"}],' + + '"@odata.deltaLink":"http://localhost:' + $using:port + '/v1.0/users/delta?$deltatoken=first"}') + while ($listener.IsListening) { + try { $context = $listener.GetContext() } catch { break } + $served.Enqueue($context.Request.Url.PathAndQuery) + $context.Response.ContentType = 'application/json' + $context.Response.OutputStream.Write($body, 0, $body.Length) + $context.Response.Close() + } + } + + $dp = Join-Path $script:testDir 'first-call.json' + try { + # Set-MgxOption does not initialize the transport, so Sync-MgxDelta is still the + # first cmdlet of the session to build the client; the tight limits keep a + # regression (which sends this request to the default endpoint instead of the + # connected one) from stalling the suite on retries. + $out = Invoke-ConnectedChild -GraphEndpoint "http://localhost:$port" -Body @" + Set-MgxOption -MaxRetryAttempts 1 -AttemptTimeoutSeconds 15 -TotalTimeoutSeconds 30 + try { + `$changes = Sync-MgxDelta /users/delta -DeltaPath '$dp' -ErrorAction Stop + Write-Output "SYNCED=`$(@(`$changes).Count)" + } catch { + Write-Output "ERROR-ID=`$(`$_.FullyQualifiedErrorId)" + } +"@ + ($out -join ' ') | Should -BeLike '*SYNCED=2*' + $requests = @($served) + $requests | Should -Not -BeNullOrEmpty -Because 'the first request of the session must reach the connected endpoint' + $requests[0] | Should -BeLike '/v1.0/users/delta*' + } + finally { + $listener.Stop() + $listener.Close() + Stop-Job $server -ErrorAction SilentlyContinue + Remove-Job $server -Force -ErrorAction SilentlyContinue + } + } + + It 'Should detect endpoint mismatch in delta state' { + $reason = Get-SkipReason + if ($reason) { Set-ItResult -Skipped -Because $reason; return } + + $port = Get-FreePort + $dp = New-DeltaStateFile -Name 'endpoint-mismatch.json' ` + -DeltaLink 'https://graph.microsoft.us/v1.0/users/delta?$deltatoken=abc' ` + -GraphEndpoint 'https://graph.microsoft.us' + + $out = Invoke-ConnectedChild -GraphEndpoint "http://localhost:$port" -Body @" + try { + Sync-MgxDelta /users/delta -DeltaPath '$dp' -ErrorAction Stop + Write-Output 'NO-ERROR' + } catch { + Write-Output "ERROR-ID=`$(`$_.FullyQualifiedErrorId)" + Write-Output "ERROR-MSG=`$(`$_.Exception.Message)" + } +"@ + $text = $out -join ' ' + $text | Should -BeLike '*DeltaEndpointMismatch*' + $text | Should -BeLike '*graph.microsoft.us*' + # The error must name the endpoint the session is connected to, not the default. + $text | Should -BeLike "*localhost:$port*" + } + + It 'Should reject tampered deltaLink with wrong host (SSRF)' { + $reason = Get-SkipReason + if ($reason) { Set-ItResult -Skipped -Because $reason; return } + + $port = Get-FreePort + $dp = New-DeltaStateFile -Name 'ssrf.json' ` + -DeltaLink 'https://evil.com/v1.0/users/delta?$deltatoken=stolen' ` + -GraphEndpoint "http://localhost:$port" + + $out = Invoke-ConnectedChild -GraphEndpoint "http://localhost:$port" -Body @" + try { + Sync-MgxDelta /users/delta -DeltaPath '$dp' -ErrorAction Stop + Write-Output 'NO-ERROR' + } catch { + Write-Output "ERROR-MSG=`$(`$_.Exception.Message)" + } +"@ + ($out -join ' ') | Should -BeLike '*invalid or untrusted*' + } + + It 'Should reject tampered deltaLink with wrong resource path' { + $reason = Get-SkipReason + if ($reason) { Set-ItResult -Skipped -Because $reason; return } + + # https so the link passes host validation and reaches the path check. + $port = Get-FreePort + $dp = New-DeltaStateFile -Name 'path-tamper.json' ` + -DeltaLink "https://localhost:$port/v1.0/me/messages?`$deltatoken=abc" ` + -GraphEndpoint "https://localhost:$port" + + $out = Invoke-ConnectedChild -GraphEndpoint "https://localhost:$port" -Body @" + try { + Sync-MgxDelta /users/delta -DeltaPath '$dp' -ErrorAction Stop + Write-Output 'NO-ERROR' + } catch { + Write-Output "ERROR-MSG=`$(`$_.Exception.Message)" + } +"@ + ($out -join ' ') | Should -BeLike '*path*does not match*' + } +} + Describe 'NormalizeSelect (via reflection)' { BeforeAll { $script:normalize = [Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta].GetMethod( @@ -1318,7 +1506,7 @@ Describe 'Expand-MgxRelation Buffer Warning' { It 'Source code contains 50k buffer warning guard' { # The actual live test (piping 50k items) requires Graph connection and takes ~15 min # due to fan-out HTTP calls. Verify the guard exists in source instead. - $source = Get-Content (Join-Path $PSScriptRoot '../src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs') -Raw + $source = Get-Content (Join-Path $PSScriptRoot '../../src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs') -Raw $source | Should -Match '50_000' $source | Should -Match 'WriteWarning' $source | Should -Match 'buffer|Buffer' @@ -1332,7 +1520,7 @@ Describe 'HttpClient Ownership Disposal Guard' { BeforeAll { # Ensure Mgx.Engine types are loaded (transitive dep of Mgx.Cmdlets) - $enginePath = Join-Path $PSScriptRoot '../module/Mgx.Engine.dll' + $enginePath = Join-Path $PSScriptRoot '../../module/Mgx.Engine.dll' [System.Reflection.Assembly]::LoadFrom($enginePath) | Out-Null $script:CmdletBaseType = [Mgx.Cmdlets.Base.MgxCmdletBase] @@ -1353,7 +1541,9 @@ Describe 'HttpClient Ownership Disposal Guard' { $script:CmdletBaseType.GetField('s_ownsHttpClient', $script:Flags).SetValue($null, $false) $script:CmdletBaseType.GetField('s_clientOptions', $script:Flags).SetValue( $null, [Mgx.Engine.Http.ResilientGraphClientOptions]::Default) - $script:CmdletBaseType.GetField('s_cachedTenantId', $script:Flags).SetValue($null, $null) + # Clients are keyed on a fingerprint of the whole auth context, not the tenant id. + # This teardown reflects on that field by name, so it must track any rename. + $script:CmdletBaseType.GetField('s_cachedAuthFingerprint', $script:Flags).SetValue($null, $null) } It 'Should NOT dispose HttpClient when s_ownsHttpClient is false' { @@ -1395,6 +1585,78 @@ Describe 'HttpClient Ownership Disposal Guard' { } } +Describe 'Enable-MgxResilience SDK Client Wrap' { + It 'Should keep the SDK BaseAddress on the wrapped client' { + # Invoke-MgGraphRequest resolves a relative -Uri against GraphHttpClient.BaseAddress + # before any handler runs, so the wrapper Enable-MgxResilience installs must carry + # the original client's BaseAddress or every relative-URI SDK call dies on the spot. + # + # Runs in a child process: the wrap mutates GraphSession.Instance and connects a + # mock session, neither of which may leak into the parent suite. No tenant and no + # listener are needed - the endpoint points at a dead port because Enable's init + # probe only has to run, not succeed, for the SDK to build its HttpClient. + if (-not (Get-Module -ListAvailable Microsoft.Graph.Authentication)) { + Set-ItResult -Skipped -Because 'Microsoft.Graph.Authentication is not installed' + return + } + + $modulePath = Join-Path $PSScriptRoot '../../module/mgx.psd1' + # Add-MgEnvironment PERSISTS custom environments, so the name is unique per run + # and removed again in the probe's finally block. + $envName = 'MgxWrap' + [guid]::NewGuid().ToString('N').Substring(0, 10) + $probe = @" +Import-Module Microsoft.Graph.Authentication +Import-Module '$modulePath' -Force +Add-MgEnvironment -Name $envName -GraphEndpoint 'http://localhost:59997' `` + -AzureADEndpoint 'https://login.microsoftonline.com' | Out-Null +try { + Connect-MgGraph -Environment $envName `` + -AccessToken (ConvertTo-SecureString 'mock-token-not-validated' -AsPlainText -Force) -NoWelcome + # -AccessToken auth leaves AuthContext.TenantId empty, and Mgx's auth fingerprint + # treats an empty TenantId as "not connected" (1.0.4 identity fix). Give it a value. + [Microsoft.Graph.PowerShell.Authentication.GraphSession]::Instance.AuthContext.TenantId = 'mock-tenant' + Enable-MgxResilience -WarningAction SilentlyContinue + `$wrapped = [Microsoft.Graph.PowerShell.Authentication.GraphSession]::Instance.GraphHttpClient + Write-Output ('WRAPPED [' + `$wrapped.BaseAddress + ']') + Disable-MgxResilience + `$restored = [Microsoft.Graph.PowerShell.Authentication.GraphSession]::Instance.GraphHttpClient + Write-Output ('RESTORED [' + `$restored.BaseAddress + ']') +} +finally { + Disconnect-MgGraph -ErrorAction SilentlyContinue | Out-Null + Remove-MgEnvironment -Name $envName -ErrorAction SilentlyContinue | Out-Null +} +"@ + # Same host re-exec dance as 'Should unload cleanly': prove the host can spawn a + # child before blaming mgx for a dotnet-tool install that cannot. + $hostExe = @( + (Get-Process -Id $PID).Path + (Get-Command pwsh -ErrorAction SilentlyContinue).Source + ) | Where-Object { $_ -and (Split-Path $_ -Leaf) -like 'pwsh*' } | Select-Object -First 1 + + if (-not $hostExe) { + Set-ItResult -Skipped -Because 'this host cannot re-exec itself (dotnet-tool PowerShell install)' + return + } + $probeOk = $null + try { $probeOk = & $hostExe -NoProfile -Command '"alive"' 2>$null } catch { $probeOk = $null } + if ($probeOk -ne 'alive') { + Set-ItResult -Skipped -Because "host '$hostExe' cannot be launched as a child process" + return + } + + $result = @(& $hostExe -NoProfile -Command $probe) + $restoredLine = $result | Where-Object { $_ -like 'RESTORED *' } + $wrappedLine = $result | Where-Object { $_ -like 'WRAPPED *' } + + # Disable restores the original SDK client, so RESTORED is the BaseAddress the + # wrapper was built from; the wrap must have carried it over verbatim. + $restoredLine | Should -Not -BeNullOrEmpty + $restoredLine | Should -Not -Be 'RESTORED []' -Because 'the original SDK client carries a BaseAddress' + $wrappedLine | Should -Be ('WRAPPED ' + ($restoredLine -replace '^RESTORED ')) + } +} + Describe 'Sync-MgxDelta ExecuteDeltaSync Tests' -Tag 'Live' { BeforeAll { $script:liveConnected = $null -ne (Get-MgContext -ErrorAction SilentlyContinue) diff --git a/tests/Unit/Mgx.HelpFreshness.Tests.ps1 b/tests/Unit/Mgx.HelpFreshness.Tests.ps1 new file mode 100644 index 0000000..c0eb0da --- /dev/null +++ b/tests/Unit/Mgx.HelpFreshness.Tests.ps1 @@ -0,0 +1,188 @@ +<# + The compiled MAML must say what its markdown source says. + + module/en-US/Mgx.Cmdlets.dll-Help.xml is generated from module/help/*.md, and for two + releases nothing regenerated it. Get-Help went on describing PSObjects and an ODataType + property that 2.0.0 replaced with hashtables, denied that Sync-MgxDelta has -CheckpointPath + while that was a headline 2.1.0 feature, and gave -BatchChunkConcurrency a range the cmdlet + rejects. The existing surface tests compare example COUNT and parameter PRESENCE, so none of + it registered. + + This compares the text. It is deliberately coarse - descriptions are reflowed and links are + rendered differently by the generator - so it checks that the MAML contains no parameter + description that the markdown has since changed, using a normalized comparison. +#> + +BeforeAll { + $repo = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent + $script:Maml = Join-Path $repo 'module/en-US/Mgx.Cmdlets.dll-Help.xml' + $script:HelpDir = Join-Path $repo 'module/help' + + function Normalize([string] $t) { + if (-not $t) { return '' } + # markdown link -> its text; code ticks and whitespace collapse + $t = [regex]::Replace($t, '\[([^\]]+)\]\([^)]+\)', '$1') + $t = $t -replace '`', '' + ($t -replace '\s+', ' ').Trim() + } + + $script:MamlXml = if (Test-Path $script:Maml) { [xml](Get-Content $script:Maml -Raw) } else { $null } +} + +Describe 'Compiled help matches its markdown source' { + It 'the MAML exists' { + Test-Path $script:Maml | Should -BeTrue + } + + It 'carries no description the markdown has replaced' { + # Phrases that were true of an older release and are now wrong. Each one shipped in + # Get-Help while the markdown already said otherwise. + $stale = @( + @{ Text = 'ODataType'; Why = '2.0.0 replaced PSObjects with hashtables; @odata.type is a key now' } + @{ Text = 'instead of PSObjects'; Why = '-Raw returns raw JSON instead of hashtables' } + @{ Text = 'ephemeral and deleted on success'; Why = 'Sync-MgxDelta gained -CheckpointPath in 2.1.0' } + ) + $raw = Get-Content $script:Maml -Raw + $hits = $stale | Where-Object { $raw -match [regex]::Escape($_.Text) } + ($hits | ForEach-Object { "$($_.Text) -- $($_.Why)" }) -join '; ' | + Should -BeNullOrEmpty -Because 'the MAML is generated from module/help; run ./build.ps1 to regenerate it' + } + + It 'carries the synopsis, description, outputs and examples its markdown gives' { + # The parameter check below was the whole guard, so a SYNOPSIS saying the wrong thing + # entirely, a replaced OUTPUTS section, or a renamed example all shipped green. Sections + # are compared by their opening sentence: the generator reflows prose, so a prefix is + # what can be compared honestly. + $mismatches = [System.Collections.Generic.List[string]]::new() + + foreach ($md in Get-ChildItem $script:HelpDir -Filter '*.md') { + $cmdlet = $md.BaseName + $lines = Get-Content $md.FullName + $node = $script:MamlXml.helpItems.command | + Where-Object { $_.details.name.Trim() -eq $cmdlet } | Select-Object -First 1 + if (-not $node) { $mismatches.Add("$cmdlet : absent from MAML"); continue } + + function SectionText([string[]] $l, [string] $heading) { + $out = [System.Collections.Generic.List[string]]::new() + $in = $false + foreach ($line in $l) { + if ($line -match "^## $heading\s*$") { $in = $true; continue } + if ($in -and $line -match '^#{1,3} ') { break } + if ($in -and $line.Trim()) { $out.Add($line) } + } + Normalize ($out -join ' ') + } + function Head([string] $t, [int] $n = 50) { + if ($t.Length -gt $n) { $t.Substring(0, $n) } else { $t } + } + + $wantSyn = SectionText $lines 'SYNOPSIS' + $gotSyn = Normalize $node.details.description.para + if ($wantSyn -and $gotSyn -notlike "*$(Head $wantSyn)*") { + $mismatches.Add("$cmdlet SYNOPSIS : markdown '$(Head $wantSyn)...' MAML '$(Head $gotSyn)...'") + } + + $wantOut = SectionText $lines 'OUTPUTS' + $gotOut = Normalize (($node.returnValues.returnValue.type.name | ForEach-Object { $_ }) -join ' ') + if ($wantOut -and $gotOut) { + # OUTPUTS opens with the type name under a ### heading; compare that token. + $wantType = ($lines | Where-Object { $_ -match '^### ' } | ForEach-Object { $_ } | + Select-Object -First 0) + $typeLine = $null + $inOut = $false + foreach ($line in $lines) { + if ($line -match '^## OUTPUTS\s*$') { $inOut = $true; continue } + if ($inOut -and $line -match '^## ') { break } + if ($inOut -and $line -match '^### (.+)$') { $typeLine = $Matches[1].Trim(); break } + } + if ($typeLine -and $gotOut -notlike "*$typeLine*") { + $mismatches.Add("$cmdlet OUTPUTS : markdown '$typeLine' MAML '$(Head $gotOut 60)'") + } + } + + $mdExamples = @($lines | Where-Object { $_ -match '^### Example' }) + $mamlExamples = @($node.examples.example) + for ($i = 0; $i -lt [Math]::Min($mdExamples.Count, $mamlExamples.Count); $i++) { + $want = Normalize ($mdExamples[$i] -replace '^### ', '') + $got = Normalize $mamlExamples[$i].title + if ($got -notlike "*$want*") { + $mismatches.Add("$cmdlet example $($i + 1) : markdown '$want' MAML '$got'") + } + } + } + + ($mismatches -join "`n") | Should -BeNullOrEmpty -Because 'run ./build.ps1 to regenerate the compiled help' + } + + It 'documents every parameter with the description its markdown gives' { + $mismatches = [System.Collections.Generic.List[string]]::new() + + foreach ($md in Get-ChildItem $script:HelpDir -Filter '*.md') { + $cmdlet = $md.BaseName + $lines = Get-Content $md.FullName + + # ### -Name followed by prose until the ```yaml block + for ($i = 0; $i -lt $lines.Count; $i++) { + if ($lines[$i] -notmatch '^### -(\w+)$') { continue } + $param = $Matches[1] + $desc = [System.Collections.Generic.List[string]]::new() + for ($j = $i + 1; $j -lt $lines.Count; $j++) { + if ($lines[$j] -match '^```' -or $lines[$j] -match '^### ' -or $lines[$j] -match '^## ') { break } + if ($lines[$j].Trim()) { $desc.Add($lines[$j]) } + } + $want = Normalize ($desc -join ' ') + if (-not $want) { continue } + + $node = $script:MamlXml.helpItems.command | + Where-Object { $_.details.name.Trim() -eq $cmdlet } | + ForEach-Object { $_.parameters.parameter } | + Where-Object { $_.name -eq $param } | + Select-Object -First 1 + if (-not $node) { $mismatches.Add("$cmdlet -$param : absent from MAML"); continue } + + $got = Normalize (($node.description.para | ForEach-Object { $_ }) -join ' ') + # Coarse: the MAML must open with what the markdown says. + $head = if ($want.Length -gt 60) { $want.Substring(0, 60) } else { $want } + if ($got -notlike "*$head*") { + $mismatches.Add("$cmdlet -$param : markdown says '$head...' MAML says '$($got.Substring(0,[Math]::Min(60,$got.Length)))...'") + } + } + } + + ($mismatches -join "`n") | Should -BeNullOrEmpty -Because 'run ./build.ps1 to regenerate the compiled help' + } +} + +Describe 'Documented output types match the cmdlets' { + # The guard above keeps the MAML matching its markdown. It cannot see markdown that is simply + # wrong about the code - both files said PSObject while the cmdlets declared and emitted + # their own types. This compares the documentation against [OutputType]. + It 'every cmdlet documents the type it declares' { + $repo = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent + Import-Module "$repo/module/mgx.psd1" -Force + $wrong = [System.Collections.Generic.List[string]]::new() + + foreach ($cmd in Get-Command -Module mgx -CommandType Cmdlet) { + $declared = @($cmd.OutputType | ForEach-Object { $_.Type.FullName } | Where-Object { $_ }) + if (-not $declared) { continue } + + $md = Join-Path $repo "module/help/$($cmd.Name).md" + if (-not (Test-Path $md)) { continue } + + $documented = $null + $inOutputs = $false + foreach ($line in Get-Content $md) { + if ($line -match '^## OUTPUTS\s*$') { $inOutputs = $true; continue } + if ($inOutputs -and $line -match '^## ') { break } + if ($inOutputs -and $line -match '^### (.+)$') { $documented = $Matches[1].Trim(); break } + } + if (-not $documented) { continue } + + if ($declared -notcontains $documented) { + $wrong.Add("$($cmd.Name): documents '$documented', declares '$($declared -join ", ")'") + } + } + + ($wrong -join "`n") | Should -BeNullOrEmpty -Because 'module/help OUTPUTS should name the type the cmdlet emits' + } +} diff --git a/tests/Unit/Mgx.LiveCoverage.Tests.ps1 b/tests/Unit/Mgx.LiveCoverage.Tests.ps1 new file mode 100644 index 0000000..a8b8b76 --- /dev/null +++ b/tests/Unit/Mgx.LiveCoverage.Tests.ps1 @@ -0,0 +1,59 @@ +<# + Asserts that every exported cmdlet is exercised against a real tenant by tests/Live. + + This runs in CI, where there are no credentials, because it reads the live tests rather than + running them. That is the point: the live suite proves the cmdlets work, and this proves the + live suite is complete. Without it, coverage decays silently - a cmdlet gets added, nobody + writes a live test, and the gap is invisible until a user finds it. + + It exists because that is exactly what happened: Enable-MgxResilience shipped a wrapper with + no BaseAddress, breaking every relative-URI SDK call, while seven of twelve cmdlets had never + been run against a tenant at all and nothing anywhere said so. +#> + +BeforeAll { + $repo = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent + Import-Module "$repo/module/mgx.psd1" -Force + $script:Exported = (Get-Command -Module mgx -CommandType Cmdlet,Function).Name | Sort-Object + + $script:LiveDir = "$repo/tests/Live" + $script:LiveText = if (Test-Path $script:LiveDir) { + (Get-ChildItem $script:LiveDir -Filter '*.Tests.ps1' | Get-Content -Raw) -join "`n" + } else { '' } + + # A Describe block naming the cmdlet is the contract. Parsed, not regexed loosely, so a + # mention inside a comment or a string cannot satisfy it. + $script:Described = @() + if ($script:LiveText) { + $ast = [System.Management.Automation.Language.Parser]::ParseInput($script:LiveText, [ref]$null, [ref]$null) + $script:Described = $ast.FindAll({ + param($n) + $n -is [System.Management.Automation.Language.CommandAst] -and + $n.GetCommandName() -eq 'Describe' + }, $true) | ForEach-Object { + $arg = $_.CommandElements | Select-Object -Skip 1 -First 1 + if ($arg -is [System.Management.Automation.Language.StringConstantExpressionAst]) { $arg.Value } + } + } +} + +Describe 'Live test coverage' { + It 'has a live test directory' { + Test-Path $script:LiveDir | Should -BeTrue -Because 'tests/Live is where cmdlets are proven against a real tenant' + } + + It 'covers every exported cmdlet' { + $missing = $script:Exported | Where-Object { $_ -notin $script:Described } + $missing -join ', ' | Should -BeNullOrEmpty -Because @' +every exported cmdlet needs a Describe block in tests/Live naming it. A mocked test proves the +code does what it was written to do; only a live one proves the request Graph receives is one it +accepts. If a cmdlet genuinely cannot be tested live, add the Describe with an explaining -Skip +so the decision is recorded rather than absent. +'@ + } + + It 'describes nothing that is not exported' { + $stale = $script:Described | Where-Object { $_ -notin $script:Exported } + $stale -join ', ' | Should -BeNullOrEmpty -Because 'a live test for a cmdlet that no longer exists is dead weight' + } +} diff --git a/tests/Unit/Mgx.Surface.Tests.ps1 b/tests/Unit/Mgx.Surface.Tests.ps1 index 831de8b..ba30087 100644 --- a/tests/Unit/Mgx.Surface.Tests.ps1 +++ b/tests/Unit/Mgx.Surface.Tests.ps1 @@ -19,6 +19,7 @@ $script:ExpectedCmdlets = @( 'Get-MgxResilience' 'Get-MgxTelemetry' 'Sync-MgxDelta' + 'Get-MgxContent' ) BeforeAll { @@ -41,6 +42,7 @@ BeforeAll { 'Get-MgxResilience' 'Get-MgxTelemetry' 'Sync-MgxDelta' + 'Get-MgxContent' ) } @@ -127,6 +129,25 @@ Describe 'Cmdlet safety attributes' { } } +Describe 'Adaptive pacing surface' { + It 'Set-MgxOption exposes -NoAdaptivePacing' { + (Get-Command -Name Set-MgxOption).Parameters.Keys | Should -Contain 'NoAdaptivePacing' + } + + It 'Get-MgxOption reports pacing enabled by default' { + (Get-MgxOption).NoAdaptivePacing | Should -BeFalse + } + + It 'Get-MgxTelemetry reports the pacing counters and state' { + $telemetry = Get-MgxTelemetry + $names = $telemetry.PSObject.Properties.Name + $names | Should -Contain 'AdaptivePacingWaitMs' + $names | Should -Contain 'AdaptivePacingActivations' + $names | Should -Contain 'LastThrottlePercentage' + $names | Should -Contain 'PacingState' + } +} + Describe 'Help' { It 'ships help content for <_>' -ForEach $script:ExpectedCmdlets { $help = Get-Help -Name $_ -ErrorAction SilentlyContinue @@ -152,3 +173,65 @@ Describe 'Module removal' { Should -Not -Throw } } + +Describe 'Help content is authored, not auto-generated' { + # The pre-existing check only asserted a non-empty Synopsis. PowerShell SYNTHESISES a Synopsis + # from the syntax block when no MAML exists, so that assertion passed with the help file + # deleted outright. These read the compiled MAML directly instead of going through Get-Help, + # which is both deterministic and immune to updatable-help lookups. + BeforeAll { + # The 'Module removal' Describe above unloads mgx, and Pester runs Describes in file + # order - so Get-Command here found nothing, every parameter list came back empty, and + # the comparison below passed having checked nothing. Re-import to be order-independent. + Import-Module -Name $script:Paths.ManifestPath -Force -ErrorAction Stop + + $script:MamlPath = Join-Path $script:Paths.ModuleRoot 'en-US/Mgx.Cmdlets.dll-Help.xml' + $script:HelpMd = Join-Path (Split-Path -Parent $script:Paths.ModuleRoot) 'module/help' + $script:Maml = if (Test-Path $script:MamlPath) { [xml](Get-Content $script:MamlPath -Raw) } else { $null } + } + + It 'ships a compiled MAML file' { + $script:MamlPath | Should -Exist + $script:Maml | Should -Not -BeNullOrEmpty + } + + It 'compiled help carries every markdown example for <_>' -ForEach $script:ExpectedCmdlets { + $md = Join-Path $script:HelpMd "$_.md" + if (-not (Test-Path $md)) { Set-ItResult -Skipped -Because "no markdown help for $_"; return } + + # Capture the name FIRST: inside Where-Object, $_ and $PSItem rebind to the pipeline + # item, so comparing against $PSItem there compares each command to itself. + $name = $PSItem + $inMarkdown = @(Select-String -Path $md -Pattern '^### Example' -AllMatches).Count + $cmd = @($script:Maml.helpItems.command | Where-Object { $_.details.name.Trim() -eq $name })[0] + $cmd | Should -Not -BeNullOrEmpty -Because "$name has no entry in the compiled MAML" + + # @($null).Count is 1 in PowerShell, so a cmdlet whose entire element is + # missing counted as 1 - and any cmdlet with exactly one markdown example then passed + # with its help gone. Count nodes, and require the element to exist at all. + $cmd.examples | Should -Not -BeNullOrEmpty -Because "$name has no element in the compiled MAML" + $inMaml = @($cmd.examples.example | Where-Object { $null -ne $_ }).Count + $inMaml | Should -Be $inMarkdown -Because "module/help/$name.md documents $inMarkdown example(s) but the MAML carries $inMaml - regenerate the compiled help" + } + + It 'compiled help documents every parameter of <_>' -ForEach $script:ExpectedCmdlets { + $common = [System.Management.Automation.PSCmdlet]::CommonParameters + + [System.Management.Automation.PSCmdlet]::OptionalCommonParameters + $name = $PSItem + # -ErrorAction Stop, and assert non-empty: without both, a Get-Command that cannot resolve + # the cmdlet leaves $real empty, $missing empty, and the assertion passes having checked + # nothing at all - the exact vacuous-green this whole block exists to prevent. + $cmdInfo = Get-Command -Name $name -ErrorAction Stop + $cmdInfo | Should -Not -BeNullOrEmpty -Because "$name must be resolvable for this comparison to mean anything" + # Several cmdlets (Get-MgxOption, the resilience trio) legitimately take no parameters + # beyond the common ones, so an empty list here is a real result, not a failure. + $real = @($cmdInfo.Parameters.Keys | Where-Object { $_ -notin $common }) + + $cmd = @($script:Maml.helpItems.command | Where-Object { $_.details.name.Trim() -eq $name })[0] + $cmd | Should -Not -BeNullOrEmpty -Because "$name has no entry in the compiled MAML" + $documented = @($cmd.parameters.parameter.name) + + $missing = @($real | Where-Object { $_ -notin $documented }) + $missing | Should -BeNullOrEmpty -Because "$name has parameter(s) absent from compiled help: $($missing -join ', ')" + } +} diff --git a/tests/benchmarks/01-list-users.ps1 b/tests/benchmarks/01-list-users.ps1 index 2badf87..8d8422f 100644 --- a/tests/benchmarks/01-list-users.ps1 +++ b/tests/benchmarks/01-list-users.ps1 @@ -30,6 +30,14 @@ $results.sdk = Measure-BenchMedian -Name 'SDK Get-MgUser -All' -Runs $Runs -Scri @{ count = $n } } +# The README footnote compares the SDK tuned against the SDK as it comes. Without this arm that +# ratio was a claim with nothing behind it - it said ten times, and it is three. +$results.sdkDefault = Measure-BenchMedian -Name 'SDK Get-MgUser -All (default page)' -Runs $Runs -Script { + $n = 0 + Get-MgUser -All -Property $props | ForEach-Object { $n++ } + @{ count = $n } +} + $results.rest = Measure-BenchMedian -Name 'raw Invoke-RestMethod' -Runs $Runs -Script { $tok = Get-BenchAppToken $headers = @{ Authorization = "Bearer $tok" } @@ -55,7 +63,7 @@ $results.ttfrSdk = Measure-BenchMedian -Name 'SDK time-to-first' -Runs $Runs -Sc Write-Host '' Write-Host '=== LIST USERS ===' -foreach ($k in 'mgx', 'sdk', 'rest') { +foreach ($k in 'mgx', 'sdk', 'sdkDefault', 'rest') { $m = $results[$k].Median Write-Host ("{0,-24} {1,8:F1}s count={2} peakWS={3}MB" -f $results[$k].Name, ($m.ElapsedMs / 1000), $m.Output.count, $m.PeakWorkingSetMB) } diff --git a/tests/benchmarks/02-fanout-lookup.ps1 b/tests/benchmarks/02-fanout-lookup.ps1 index c35b52e..4e2cbbf 100644 --- a/tests/benchmarks/02-fanout-lookup.ps1 +++ b/tests/benchmarks/02-fanout-lookup.ps1 @@ -74,11 +74,9 @@ if ($Contender) { } # ---------------- parent mode ---------------- -Write-Host "collecting $SampleSize bench user ids..." -$ids = [System.Collections.Generic.List[string]]::new() -Invoke-MgxRequest /users -All -Filter "startsWith(userPrincipalName,'bench.u')" -Property id | - Select-Object -First $SampleSize | ForEach-Object { $ids.Add($_.id) } -if ($ids.Count -lt $SampleSize) { throw "only found $($ids.Count) bench users; seed the tenant first" } +Write-Host "collecting $SampleSize user ids..." +# Read-only benchmark: seeded users preferred, ordinary users acceptable. See Get-BenchUserIds. +$ids = Get-BenchUserIds -Count $SampleSize Write-Host "got $($ids.Count) ids" $tmp = [System.IO.Path]::GetTempPath() diff --git a/tests/benchmarks/05-memory-export.ps1 b/tests/benchmarks/05-memory-export.ps1 index 4761a80..81339c6 100644 --- a/tests/benchmarks/05-memory-export.ps1 +++ b/tests/benchmarks/05-memory-export.ps1 @@ -1,8 +1,12 @@ # Benchmark 05: export every user to JSONL - the memory story. # Contenders: -# mgx-export - Export-MgxCollection: streams raw JSON to disk, no object conversion -# mgx-pipe - Invoke-MgxRequest -All | Out-File: streams but pays the conversion tax +# mgx-export - Export-MgxCollection: streams raw JSON to disk +# mgx-pipe - Invoke-MgxRequest -All -Raw | Out-File: streams raw JSON through the pipeline # rest - Invoke-RestMethod: buffers all pages in memory, then writes +# Both mgx contenders emit GetRawText() and neither converts to Hashtables - the pipe is run +# with -Raw - so the two mgx rows are close to like-for-like. What separates them is the export +# cmdlet's extra per-page work (temp promotion, checkpoint bookkeeping, count reconciliation), +# not object conversion. # Reports wall time, peak working set, and managed-heap delta for each. Single run per # contender by default (export is IO-heavy; variance is dominated by service latency). param( diff --git a/tests/benchmarks/06-fault-gauntlet.ps1 b/tests/benchmarks/06-fault-gauntlet.ps1 index a82541e..3e17847 100644 --- a/tests/benchmarks/06-fault-gauntlet.ps1 +++ b/tests/benchmarks/06-fault-gauntlet.ps1 @@ -17,7 +17,11 @@ param( Import-MgxLocal Import-Module Microsoft.Graph.Users -$ids = 1..$N +# Strings, not integers. Invoke-MgxRequest resolves a piped id as either a bare string or +# an 'id' member; an int is neither, so every record raised MissingPipelineId, -ErrorAction +# SilentlyContinue swallowed all of them, and the fan-out contender died at EndProcessing +# claiming no pipeline input. Real Graph ids are strings, so this matches actual usage. +$ids = @(1..$N | ForEach-Object { "$_" }) # Find a free port (a crashed prior run can leave a zombie listener behind) $Port = ($Port..($Port + 20)) | Where-Object { diff --git a/tests/benchmarks/10-pacing-under-real-throttling.ps1 b/tests/benchmarks/10-pacing-under-real-throttling.ps1 new file mode 100644 index 0000000..e323a8a --- /dev/null +++ b/tests/benchmarks/10-pacing-under-real-throttling.ps1 @@ -0,0 +1,82 @@ +# Benchmark 10: does adaptive pacing pay for itself against a service that throttles? +# +# Benchmark 06 answers "what does pacing cost" but structurally cannot answer "what does it +# buy": its mock injects 429s on a fixed schedule keyed on entity id, so the throttle rate is +# independent of send rate and backing off has no upside by construction. Real Graph throttles +# HARDER the faster you push, which is the whole premise of pacing. +# +# This drives the SAME read workload twice against a real tenant - once paced (2.1 default), +# once with -NoAdaptivePacing - hard enough to cross the resource-unit budget and provoke +# genuine 429s. Reads only: throttling is not write-specific, so this needs no write consent. +# +# Each round runs in a FRESH pwsh process because the adapted rate is static process state, +# with a cooldown between so the second round does not inherit a drained budget. +param( + [ValidateSet('both', 'paced', 'unpaced')] [string] $Mode = 'both', + [int] $Count = 20000, + [int] $Concurrency = 60, + [int] $CooldownSeconds = 300 +) + +. "$PSScriptRoot/common.ps1" + +if ($Mode -eq 'both') { + foreach ($m in 'paced', 'unpaced') { + Write-Host "=== round: $m ===" -ForegroundColor Cyan + $p = Start-Process pwsh -PassThru -Wait -NoNewWindow -ArgumentList ` + '-NoProfile', '-File', $PSCommandPath, '-Mode', $m, '-Count', $Count, '-Concurrency', $Concurrency + if ($p.ExitCode -ne 0) { throw "round '$m' failed (exit $($p.ExitCode))" } + if ($m -eq 'paced') { + Write-Host "cooldown ${CooldownSeconds}s (let the resource-unit budget refill)..." + Start-Sleep -Seconds $CooldownSeconds + } + } + Write-Host 'both rounds complete - see results/10-pacing-under-real-throttling.json' + return +} + +Import-MgxLocal +Connect-MgxBenchmark + +# A fixed, pre-materialised id list. Bounded work only: no loop that can run away. +Write-Host "collecting $Count ids..." +$ids = Get-BenchUserIds -Count $Count +Write-Host " got $($ids.Count)" + +# The token-bucket limiter would cap us well under the RU budget and we would never reach +# the throttling regime this benchmark exists to measure. Lift it; the pacer (or its absence) +# is the variable under test. +Set-MgxOption -NoRateLimit +if ($Mode -eq 'unpaced') { + Set-MgxOption -NoAdaptivePacing + Write-Host ' adaptive pacing DISABLED' +} else { + Write-Host ' adaptive pacing ON (2.1 default)' +} + +$result = Measure-BenchPass -Name "reads $Mode" -Script { + $items = @($ids | Invoke-MgxRequest '/users/{id}' -Property id,displayName ` + -Concurrency $Concurrency -ErrorVariable errs -ErrorAction SilentlyContinue) + @{ ok = $items.Count; failed = @($errs).Count } +} + +$t = Get-MgxTelemetry +Write-Host '' +Write-Host ("=== $Mode ===") +Write-Host (" wall {0,8:F1}s" -f ($result.ElapsedMs / 1000)) +Write-Host (" completed {0,8} failed {1}" -f $result.Output.ok, $result.Output.failed) +Write-Host (" throttle retries {0,8}" -f $t.ThrottleRetries) +Write-Host (" retry delay {0,8:F1}s (time spent honouring Retry-After)" -f ($t.RetryDelayMs / 1000)) +Write-Host (" pacing wait {0,8:F1}s over {1} activations" -f ($t.AdaptivePacingWaitMs / 1000), $t.AdaptivePacingActivations) +Write-Host (" resource units {0,8} ({1:F1} RU/s)" -f $t.ResourceUnitsConsumed, ($t.ResourceUnitsConsumed / ($result.ElapsedMs / 1000))) +Write-Host (" pacing state {0}" -f $t.PacingState) + +# Write-BenchResult attaches the telemetry snapshot itself, including resource units. +Write-BenchResult -Benchmark '10-pacing-under-real-throttling' -Result ([pscustomobject]@{ + Mode = $Mode + Count = $ids.Count + Concurrency = $Concurrency + WallMs = $result.ElapsedMs + Ok = $result.Output.ok + Failed = $result.Output.failed +}) diff --git a/tests/benchmarks/11-throttle-accuracy.ps1 b/tests/benchmarks/11-throttle-accuracy.ps1 new file mode 100644 index 0000000..4bee5fa --- /dev/null +++ b/tests/benchmarks/11-throttle-accuracy.ps1 @@ -0,0 +1,219 @@ +# Benchmark 11: does the data survive throttling? +# +# The other benchmarks ask how FAST a contender is. This one asks whether the rows it handed +# you are the rows the tenant actually holds. Under throttling an enumeration can come back +# short (a page that never succeeded) or long (a page counted twice), and both are silent - +# the script exits 0 and you get a CSV that looks fine. +# +# Ground truth is the server-side /users/$count, which is independent of pagination, of the +# SDK, and of Mgx. Each contender enumerates the same collection at the same page size and is +# scored on: +# Emitted objects handed to the pipeline +# Distinct unique ids among them +# Missing ground truth - Distinct (rows you will never know you lost) +# Duplicate Emitted - Distinct (rows you will count twice) +# +# Contenders run CONCURRENTLY, in separate processes, against one app registration - because +# the Graph budget is scoped per application+tenant, that is what makes them contend for the +# same tokens instead of each getting a quiet tenant to itself. -PressureConcurrency adds +# background load on top, which is the ordinary condition for a tenant running anything else. +param( + [int] $PageSize = 100, + # In-flight pressure requests. 400 measured 886 req/s against a 100k tenant and crossed the + # 800 RU/s ceiling in 21s, so this is the dial that decides whether the run throttles at all. + [int] $PressureConcurrency = 400, + [int] $TimeoutMinutes = 30, + [int] $CooldownSeconds = 90, + [switch] $NoPressure +) + +. "$PSScriptRoot/common.ps1" +Import-MgxLocal +Connect-MgxBenchmark + +if ($NoPressure) { $PressureConcurrency = 0 } + +# --- Ground truth ----------------------------------------------------------------------- +# $count is served from the directory's own index, not by walking pages, so it cannot inherit +# a pagination bug from the thing it is being used to check. +$token = Get-BenchAppToken +$truth = Invoke-RestMethod -Uri 'https://graph.microsoft.com/v1.0/users/$count' ` + -Headers @{ Authorization = "Bearer $token"; ConsistencyLevel = 'eventual' } +$truth = [int] $truth +Write-Host "ground truth: $truth users (from /users/`$count)" +Write-Host "page size $PageSize -> ~$([math]::Ceiling($truth / $PageSize)) requests per contender; pressure concurrency $PressureConcurrency" + +# --- Contenders ------------------------------------------------------------------------- +# Each returns the same shape. Every one of them pages the same collection at the same size, +# selecting only id, so the only variable is how the contender behaves when Graph says no. + +$restBody = { + param($PageSize, $Retry) + . "$using:PSScriptRoot/common.ps1" + $seen = [System.Collections.Generic.HashSet[string]]::new() + $emitted = 0; $throttles = 0; $requests = 0; $failure = $null + $token = Get-BenchAppToken + $sw = [Diagnostics.Stopwatch]::StartNew() + $url = 'https://graph.microsoft.com/v1.0/users?$top=' + $PageSize + '&$select=id' + while ($url) { + # Token lifetime is not the variable under test - a 401 an hour in would look like data + # loss without being any. Both REST contenders re-mint on the same schedule. + if ($requests -gt 0 -and $requests % 400 -eq 0) { $token = Get-BenchAppToken } + $page = $null + for ($attempt = 1; $attempt -le $Retry; $attempt++) { + try { + $requests++ + $page = Invoke-RestMethod -Uri $url -Headers @{ Authorization = "Bearer $token" } + break + } + catch { + $code = $_.Exception.Response.StatusCode.value__ + if ($code -eq 429) { $throttles++ } + if ($attempt -eq $Retry) { $failure = "HTTP $code after $attempt attempt(s)"; break } + # Deliberately a flat sleep: the naive retry loop people actually write does not + # read Retry-After, which is the whole reason it keeps arriving early. + Start-Sleep -Seconds 5 + } + } + if (-not $page) { break } + foreach ($u in $page.value) { $emitted++; [void]$seen.Add($u.id) } + $url = $page.'@odata.nextLink' + } + $sw.Stop() + @{ Emitted = $emitted; Distinct = $seen.Count; Throttles = $throttles + Requests = $requests; WallMs = $sw.ElapsedMilliseconds; Failure = $failure } +} + +$sdkBody = { + param($PageSize) + . "$using:PSScriptRoot/common.ps1" + Import-MgxLocal + Connect-MgxBenchmark + $seen = [System.Collections.Generic.HashSet[string]]::new() + $emitted = 0; $failure = $null + $sw = [Diagnostics.Stopwatch]::StartNew() + try { + Get-MgUser -All -PageSize $PageSize -Property id -ErrorAction Stop | + ForEach-Object { $emitted++; [void]$seen.Add($_.Id) } + } + catch { $failure = $_.Exception.Message } + $sw.Stop() + @{ Emitted = $emitted; Distinct = $seen.Count; Throttles = $null + Requests = $null; WallMs = $sw.ElapsedMilliseconds; Failure = $failure } +} + +$mgxBody = { + param($PageSize) + . "$using:PSScriptRoot/common.ps1" + Import-MgxLocal + Connect-MgxBenchmark + Get-MgxTelemetry -Reset | Out-Null + $seen = [System.Collections.Generic.HashSet[string]]::new() + $emitted = 0; $failure = $null + $sw = [Diagnostics.Stopwatch]::StartNew() + try { + Invoke-MgxRequest /users -All -PageSize $PageSize -Property id -ErrorAction Stop | + ForEach-Object { $emitted++; [void]$seen.Add($_.id) } + } + catch { $failure = $_.Exception.Message } + $sw.Stop() + $t = Get-MgxTelemetry + @{ Emitted = $emitted; Distinct = $seen.Count; Throttles = $t.ThrottleRetries + Requests = $t.Requests; WallMs = $sw.ElapsedMilliseconds; Failure = $failure } +} + +# Background load. Not a contender and not scored - it exists so the contenders are measured +# on a tenant that is busy, which is the only condition under which any of this matters. Graph +# scopes the budget per application+tenant pair, so this and the contenders draw on one bucket. +# Sequential paging cannot generate this: a contender walking nextLinks manages ~6 req/s, three +# orders short of the ceiling. Only high in-flight concurrency gets there. +$pressureBody = { + param($Minutes, $Concurrency) + . "$using:PSScriptRoot/common.ps1" + $token = Get-BenchAppToken + $handler = [System.Net.Http.SocketsHttpHandler]::new() + $handler.MaxConnectionsPerServer = $Concurrency + $client = [System.Net.Http.HttpClient]::new($handler) + $client.Timeout = [TimeSpan]::FromSeconds(120) + $client.DefaultRequestHeaders.Authorization = + [System.Net.Http.Headers.AuthenticationHeaderValue]::new('Bearer', $token) + $url = 'https://graph.microsoft.com/v1.0/users?$top=1&$select=id' + $deadline = [datetime]::UtcNow.AddMinutes($Minutes) + $sent = 0; $minted = [datetime]::UtcNow + while ([datetime]::UtcNow -lt $deadline) { + if (([datetime]::UtcNow - $minted).TotalMinutes -gt 40) { + $token = Get-BenchAppToken + $client.DefaultRequestHeaders.Authorization = + [System.Net.Http.Headers.AuthenticationHeaderValue]::new('Bearer', $token) + $minted = [datetime]::UtcNow + } + $tasks = foreach ($i in 1..$Concurrency) { $client.GetAsync($url) } + try { [System.Threading.Tasks.Task]::WaitAll($tasks) } catch { } + foreach ($t in $tasks) { if ($t.Status -eq 'RanToCompletion') { $t.Result.Dispose() } } + $sent += $Concurrency + } + $client.Dispose() + $sent +} + +# --- Run ---------------------------------------------------------------------------------- +$jobs = [ordered]@{} +if ($PressureConcurrency -gt 0) { + # A previous run may have left the bucket drained, which would throttle the contenders + # before the pressure job has even started and make the two indistinguishable. + Write-Host "cooldown ${CooldownSeconds}s (let the resource-unit budget refill)..." + Start-Sleep -Seconds $CooldownSeconds + $null = Start-Job -Name 'pressure' -ScriptBlock $pressureBody ` + -ArgumentList $TimeoutMinutes, $PressureConcurrency + Write-Host "draining the budget at concurrency $PressureConcurrency..." + Start-Sleep -Seconds 30 +} + +$jobs['rest-naive'] = Start-Job -Name 'rest-naive' -ScriptBlock $restBody -ArgumentList $PageSize, 1 +$jobs['rest-retry'] = Start-Job -Name 'rest-retry' -ScriptBlock $restBody -ArgumentList $PageSize, 3 +$jobs['sdk'] = Start-Job -Name 'sdk' -ScriptBlock $sdkBody -ArgumentList $PageSize +$jobs['mgx'] = Start-Job -Name 'mgx' -ScriptBlock $mgxBody -ArgumentList $PageSize + +Write-Host "all four running concurrently against one app registration..." +$null = Wait-Job -Job $jobs.Values -Timeout ($TimeoutMinutes * 60) + +$results = [ordered]@{} +foreach ($name in $jobs.Keys) { + $job = $jobs[$name] + if ($job.State -eq 'Running') { + Stop-Job $job + $results[$name] = @{ Emitted = 0; Distinct = 0; WallMs = $TimeoutMinutes * 60000 + Failure = "did not finish inside $TimeoutMinutes min" } + } + else { + $out = Receive-Job $job -ErrorAction SilentlyContinue | Where-Object { $_ -is [hashtable] } | Select-Object -Last 1 + $results[$name] = if ($out) { $out } else { @{ Emitted = 0; Distinct = 0; WallMs = 0; Failure = 'job produced no result' } } + } + Remove-Job $job -Force +} +Get-Job | Where-Object Name -like 'pressure*' | ForEach-Object { Stop-Job $_ -ErrorAction SilentlyContinue; Remove-Job $_ -Force } + +# --- Report ------------------------------------------------------------------------------- +$labels = [ordered]@{ + 'rest-naive' = 'Invoke-RestMethod' + 'rest-retry' = 'Invoke-RestMethod + retry' + 'sdk' = 'Get-MgUser -All' + 'mgx' = 'Invoke-MgxRequest -All' +} +Write-Host '' +Write-Host "=== THROTTLE ACCURACY (ground truth $truth users, page size $PageSize, $PressureConcurrency in flight) ===" +Write-Host ('{0,-28} {1,9} {2,9} {3,8} {4,10} {5,8} {6,6}' -f 'Contender', 'Emitted', 'Distinct', 'Missing', 'Duplicate', 'Wall', '429s') +foreach ($name in $labels.Keys) { + $r = $results[$name] + $r.Missing = $truth - $r.Distinct + $r.Duplicate = $r.Emitted - $r.Distinct + Write-Host ('{0,-28} {1,9} {2,9} {3,8} {4,10} {5,7:F1}s {6,6}' -f ` + $labels[$name], $r.Emitted, $r.Distinct, $r.Missing, $r.Duplicate, + ($r.WallMs / 1000), ($r.Throttles ?? '-')) + if ($r.Failure) { Write-Host (" ^ $($r.Failure)") } +} + +$results.groundTruth = $truth +$results.pageSize = $PageSize +$results.pressureConcurrency = $PressureConcurrency +Write-BenchResult -Benchmark '11-throttle-accuracy' -Result $results diff --git a/tests/benchmarks/12-delta-replay.ps1 b/tests/benchmarks/12-delta-replay.ps1 new file mode 100644 index 0000000..2dfedcc --- /dev/null +++ b/tests/benchmarks/12-delta-replay.ps1 @@ -0,0 +1,91 @@ +# Benchmark 12: how many times does a delta enumeration hand you the same object? +# +# Graph's delta endpoints do not paginate a snapshot. An object can appear on more than one +# page of a single enumeration, so the number of objects a delta run EMITS is not the number +# of objects that exist. A caller that counts rows, or appends them to a file, or feeds them +# into a report, gets a number larger than the directory holds and no indication why. +# +# Nothing here is a comparison between tools: Mgx does not deduplicate delta output either. +# The point is to measure the replay factor against known ground truth, so the size of the +# gap between "emitted" and "distinct" is a recorded number rather than folklore. +param( + [string] $Resource = '/groups/delta', + [string] $CountResource = '/groups/$count', + [int] $Top = 0 +) + +. "$PSScriptRoot/common.ps1" +Import-MgxLocal +Connect-MgxBenchmark + +# Ground truth from the directory's own index, not by walking pages. +$token = Get-BenchAppToken +$truth = [int] (Invoke-RestMethod -Uri "https://graph.microsoft.com/v1.0$CountResource" ` + -Headers @{ Authorization = "Bearer $token"; ConsistencyLevel = 'eventual' }) +Write-Host "ground truth: $truth objects (from $CountResource)" + +$results = [ordered]@{} + +# --- Raw REST walk of the delta pages ----------------------------------------------------- +Write-Host 'walking delta pages with Invoke-RestMethod...' +$seen = [System.Collections.Generic.HashSet[string]]::new() +$emitted = 0; $pages = 0; $removed = 0 +$url = "https://graph.microsoft.com/v1.0$Resource" +if ($Top -gt 0) { $url += '?$top=' + $Top } +$sw = [Diagnostics.Stopwatch]::StartNew() +while ($url) { + if ($pages -gt 0 -and $pages % 400 -eq 0) { $token = Get-BenchAppToken } + $page = Invoke-RestMethod -Uri $url -Headers @{ Authorization = "Bearer $token" } + $pages++ + foreach ($o in $page.value) { + $emitted++ + if ($o.PSObject.Properties.Name -contains '@removed') { $removed++ } + if ($o.id) { [void]$seen.Add($o.id) } + } + $url = $page.'@odata.nextLink' +} +$sw.Stop() +$results.rest = @{ + Emitted = $emitted; Distinct = $seen.Count; Pages = $pages; Removed = $removed + WallMs = $sw.ElapsedMilliseconds +} + +# --- Sync-MgxDelta over the same resource ------------------------------------------------- +Write-Host 'walking the same resource with Sync-MgxDelta...' +$deltaPath = Join-Path ([System.IO.Path]::GetTempPath()) 'bench12-delta.json' +Remove-Item $deltaPath -ErrorAction SilentlyContinue +$seenMgx = [System.Collections.Generic.HashSet[string]]::new() +$emittedMgx = 0 +Get-MgxTelemetry -Reset | Out-Null +$sw2 = [Diagnostics.Stopwatch]::StartNew() +$syncArgs = @{ Uri = $Resource; DeltaPath = $deltaPath } +if ($Top -gt 0) { $syncArgs.Top = $Top } +Sync-MgxDelta @syncArgs | ForEach-Object { + $emittedMgx++ + if ($_.id) { [void]$seenMgx.Add($_.id) } +} +$sw2.Stop() +$results.mgx = @{ + Emitted = $emittedMgx; Distinct = $seenMgx.Count; WallMs = $sw2.ElapsedMilliseconds +} +Remove-Item $deltaPath -ErrorAction SilentlyContinue + +# --- Report ------------------------------------------------------------------------------- +Write-Host '' +Write-Host "=== DELTA REPLAY ($Resource, ground truth $truth) ===" +Write-Host ('{0,-26} {1,10} {2,10} {3,8} {4,9}' -f 'Contender', 'Emitted', 'Distinct', 'Replay', 'Wall') +foreach ($k in 'rest', 'mgx') { + $r = $results[$k] + $label = if ($k -eq 'rest') { 'Invoke-RestMethod' } else { 'Sync-MgxDelta' } + $factor = if ($r.Distinct -gt 0) { $r.Emitted / $r.Distinct } else { 0 } + Write-Host ('{0,-26} {1,10} {2,10} {3,7:F1}x {4,8:F1}s' -f ` + $label, $r.Emitted, $r.Distinct, $factor, ($r.WallMs / 1000)) +} +Write-Host ("REST pages: $($results.rest.Pages), objects marked @removed: $($results.rest.Removed)") +Write-Host '' +Write-Host 'Distinct should equal ground truth. Emitted above Distinct is replay: the same object' +Write-Host 'handed to the caller more than once inside a single enumeration.' + +$results.groundTruth = $truth +$results.resource = $Resource +Write-BenchResult -Benchmark '12-delta-replay' -Result $results -WallMs $results.mgx.WallMs diff --git a/tests/benchmarks/13-resource-unit-rate.ps1 b/tests/benchmarks/13-resource-unit-rate.ps1 new file mode 100644 index 0000000..780b0c8 --- /dev/null +++ b/tests/benchmarks/13-resource-unit-rate.ps1 @@ -0,0 +1,156 @@ +# Benchmark 13: what resource-unit rate does the tenant actually sustain? +# +# The budget behaves like a token bucket, and the bucket starts full - so any measurement that +# does not outlast the initial burst allowance reads as a much higher sustained ceiling than +# the tenant has. This holds 1-RU requests (/users/{id}?$select=id) at a fixed send rate and +# records, per rate: when the first 429 arrived, how many units had been served by then, and +# what rate the tenant kept serving at once it was refusing. The burst allowance is the units +# served before the first refusal; the sustained ceiling is the served rate after it. +# +# A round that ends with no refusal proves sustainability ONLY if it spent well past the burst +# allowance; the report says so explicitly rather than leaving a short clean run to be read as +# a ceiling. That misreading is exactly how this suite once published 882 RU/s "sustained". +# +# Raw HttpClient, no retries, no pacing: the tenant is the subject, not the client. Each +# request counts once, and a 429 is a data point rather than something to recover from. +param( + [int[]] $Rates = @(300, 700, 1200), + [int] $Seconds = 90, + [int] $CooldownSeconds = 120, + [int] $MaxConnections = 512 +) + +. "$PSScriptRoot/common.ps1" +Import-MgxLocal +Connect-MgxBenchmark + +# Enough ids to avoid hammering one object; requests cycle through them. +$ids = Get-BenchUserIds -Count 5000 +Write-Host "using $($ids.Count) user ids; $($Rates.Count) round(s) of ${Seconds}s" + +$results = [ordered]@{} +$roundIndex = 0 +foreach ($rate in $Rates) { + $roundIndex++ + if ($roundIndex -gt 1) { + Write-Host "cooldown ${CooldownSeconds}s (let the resource-unit budget refill)..." + Start-Sleep -Seconds $CooldownSeconds + } + + $token = Get-BenchAppToken + $handler = [System.Net.Http.SocketsHttpHandler]::new() + $handler.MaxConnectionsPerServer = $MaxConnections + $client = [System.Net.Http.HttpClient]::new($handler) + $client.Timeout = [TimeSpan]::FromSeconds(120) + $client.DefaultRequestHeaders.Authorization = + [System.Net.Http.Headers.AuthenticationHeaderValue]::new('Bearer', $token) + + Write-Host "holding ~$rate RU/s for ${Seconds}s..." + $pending = [System.Collections.Generic.List[System.Threading.Tasks.Task[System.Net.Http.HttpResponseMessage]]]::new() + $sent = 0; $ok = 0; $throttled = 0; $failed = 0 + $okBeforeFirst429 = 0; $okAfterFirst429 = 0 + $first429Seconds = $null + $ruHeaderSample = $null + + $harvest = { + $still = [System.Collections.Generic.List[System.Threading.Tasks.Task[System.Net.Http.HttpResponseMessage]]]::new() + foreach ($t in $pending) { + if (-not $t.IsCompleted) { $still.Add($t); continue } + if ($t.Status -ne 'RanToCompletion') { $script:failed++; continue } + $r = $t.Result + $code = [int]$r.StatusCode + if ($code -eq 429) { + $script:throttled++ + if ($null -eq $script:first429Seconds) { + $script:first429Seconds = [math]::Round($sw.Elapsed.TotalSeconds, 1) + } + } + elseif ($code -ge 200 -and $code -lt 300) { + $script:ok++ + if ($null -eq $script:first429Seconds) { $script:okBeforeFirst429++ } + else { $script:okAfterFirst429++ } + if ($null -eq $script:ruHeaderSample) { + $v = $null + if ($r.Headers.TryGetValues('x-ms-resource-unit', [ref]$v)) { + $script:ruHeaderSample = @($v)[0] + } + } + } + else { $script:failed++ } + $r.Dispose() + } + $script:pending = $still + } + + $sw = [Diagnostics.Stopwatch]::StartNew() + while ($sw.Elapsed.TotalSeconds -lt $Seconds) { + # Cadence self-corrects: send whatever the elapsed time says should be out by now. + $due = [int][math]::Min($rate * $Seconds, [math]::Floor($rate * $sw.Elapsed.TotalSeconds)) + while ($sent -lt $due) { + $id = $ids[$sent % $ids.Count] + $pending.Add($client.GetAsync("https://graph.microsoft.com/v1.0/users/$id`?`$select=id")) + $sent++ + } + & $harvest + Start-Sleep -Milliseconds 50 + } + $sendWindow = $sw.Elapsed.TotalSeconds + + # Let stragglers land; they still carry statuses that belong to this round. + $deadline = [datetime]::UtcNow.AddSeconds(60) + while ($pending.Count -gt 0 -and [datetime]::UtcNow -lt $deadline) { + & $harvest + Start-Sleep -Milliseconds 200 + } + $failed += $pending.Count # anything still hanging after 60s + $sw.Stop() + $client.Dispose() + + $achieved = [math]::Round($sent / $sendWindow, 0) + $served = if ($null -ne $first429Seconds -and $sw.Elapsed.TotalSeconds -gt $first429Seconds) { + [math]::Round($okAfterFirst429 / ($sw.Elapsed.TotalSeconds - $first429Seconds), 0) + } else { $null } + + $results["rate$rate"] = @{ + TargetRate = $rate + AchievedRate = $achieved + Seconds = [math]::Round($sendWindow, 1) + Sent = $sent + Succeeded = $ok + Throttled = $throttled + Failed = $failed + First429Seconds = $first429Seconds + RuBeforeFirst429 = $okBeforeFirst429 # 1 RU each, so units == successes + ServedRateAfter429 = $served + RuHeaderSample = $ruHeaderSample + } + + if ($null -eq $first429Seconds) { + Write-Host (" no refusal in {0}s at ~{1} RU/s ({2} RU spent)." -f ` + [math]::Round($sendWindow), $achieved, $ok) + Write-Host (" NOTE: clean only proves sustainability if {0} RU is well past the burst allowance" -f $ok) + Write-Host (" a throttled round below reports that allowance as 'RU before first 429'.") + } + else { + Write-Host (" first 429 at {0}s; {1} RU served before it; ~{2} RU/s served while refusing." -f ` + $first429Seconds, $okBeforeFirst429, $served) + } +} + +# --- Report ------------------------------------------------------------------------------- +Write-Host '' +Write-Host "=== RESOURCE-UNIT RATE (1-RU requests, no retry) ===" +Write-Host ('{0,10} {1,10} {2,9} {3,11} {4,13} {5,15}' -f ` + 'Target', 'Achieved', 'Sent', 'First 429', 'RU before it', 'Served after it') +foreach ($k in $results.Keys) { + $r = $results[$k] + Write-Host ('{0,8}/s {1,8}/s {2,9} {3,11} {4,13} {5,13}/s' -f ` + $r.TargetRate, $r.AchievedRate, $r.Sent, + ($null -ne $r.First429Seconds ? "$($r.First429Seconds)s" : '-'), + ($null -ne $r.First429Seconds ? $r.RuBeforeFirst429 : '-'), + ($r.ServedRateAfter429 ?? '-')) +} + +$results.rates = $Rates +$results.seconds = $Seconds +Write-BenchResult -Benchmark '13-resource-unit-rate' -Result $results diff --git a/tests/benchmarks/14-pacing-cold-cost.ps1 b/tests/benchmarks/14-pacing-cold-cost.ps1 new file mode 100644 index 0000000..43e544a --- /dev/null +++ b/tests/benchmarks/14-pacing-cold-cost.ps1 @@ -0,0 +1,57 @@ +# Benchmark 14: what adaptive pacing costs a SHORT run. +# +# Benchmark 07 measures pacing across a long fan-out, where slow start is amortised away and the +# gate reads as free. That is true and it is only half the picture: the pacer opens cold at 4 rps +# and doubles each clean second, so a run that finishes DURING the ramp pays for all of it. The +# README table needs both numbers, and until this existed only the long one came from the suite. +# +# Each arm runs in a fresh process, because the adapted rate is per-process state and a warm +# pacer would measure nothing. +param( + [int] $Count = 50, + [int] $Concurrency = 8, + [ValidateSet('both', 'paced', 'unpaced')] [string] $Mode = 'both' +) + +. "$PSScriptRoot/common.ps1" + +if ($Mode -eq 'both') { + $results = [ordered]@{} + foreach ($m in 'paced', 'unpaced') { + $out = & pwsh -NoProfile -File $PSCommandPath -Mode $m -Count $Count -Concurrency $Concurrency + $line = $out | Where-Object { $_ -match '^RESULT\|' } | Select-Object -Last 1 + if (-not $line) { throw "arm '$m' produced no result" } + $null, $secs, $requests, $pacingMs = $line -split '\|' + $results[$m] = @{ Seconds = [double]$secs; Requests = [int]$requests; PacingWaitMs = [long]$pacingMs } + } + + Write-Host '' + Write-Host "=== ADAPTIVE PACING ON A COLD SHORT RUN ($Count lookups, concurrency $Concurrency) ===" + Write-Host ('{0,-28} {1,8} {2,10} {3,14}' -f 'Arm', 'Wall', 'Requests', 'PacingWaitMs') + foreach ($k in 'paced', 'unpaced') { + $r = $results[$k] + $label = if ($k -eq 'paced') { 'Adaptive pacing on (default)' } else { '-NoAdaptivePacing' } + Write-Host ('{0,-28} {1,7:F1}s {2,10} {3,14}' -f $label, $r.Seconds, $r.Requests, $r.PacingWaitMs) + } + if ($results.unpaced.Seconds -gt 0) { + Write-Host ('ratio {0,6:F1}x' -f ($results.paced.Seconds / $results.unpaced.Seconds)) + } + Write-Host 'Compare against benchmark 07, where the same gate costs nothing across a long run.' + Write-BenchResult -Benchmark '14-pacing-cold-cost' -Result $results -WallMs ([long]($results.paced.Seconds * 1000)) + return +} + +Import-MgxLocal +Connect-MgxBenchmark | Out-Null +if ($Mode -eq 'unpaced') { Set-MgxOption -NoAdaptivePacing } + +$ids = Invoke-MgxRequest /users -Top $Count -Property id -WarningAction SilentlyContinue | + ForEach-Object { $_.id } +Get-MgxTelemetry -Reset | Out-Null + +$sw = [Diagnostics.Stopwatch]::StartNew() +$null = $ids | Invoke-MgxRequest -Uri '/users/{id}?$select=id' -Concurrency $Concurrency +$sw.Stop() + +$t = Get-MgxTelemetry +"RESULT|{0}|{1}|{2}" -f $sw.Elapsed.TotalSeconds, $t.Requests, $t.AdaptivePacingWaitMs diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md index 391aa4b..0904c9d 100644 --- a/tests/benchmarks/README.md +++ b/tests/benchmarks/README.md @@ -2,6 +2,18 @@ Reproduces every number in the main README's Benchmarks section. Each script is self-contained; `run.ps1` runs the full suite and regenerates the README tables. +## Resource units + +Every result now carries a `Telemetry` block alongside the timing: resource units consumed, +request counts, throttle retries, and pacing waits. Wall time alone describes half the cost - +Graph throttles directory workloads on a resource-unit budget, so two runs with identical +durations can sit very differently against that budget. + +`RuPerRequest` is the figure to watch when comparing query shapes: adding `$select` measurably +lowers it. Arms that do not load Mgx (the bare-SDK comparisons) record `Telemetry: null`, since +there is no session telemetry to read - expected, not a failure. + + ## Prerequisites - PowerShell 7.5+, `Microsoft.Graph.Authentication`, `Microsoft.Graph.Users` @@ -26,6 +38,13 @@ Never point these scripts at a production tenant. Benchmarks 04 and 07 create an | 07 | `07-adaptive-pacing.ps1` | AIMD write pacing vs naive full-speed writes under real throttling | yes | | 08 | `08-delta-sync.ps1` | Delta sync: initial pull, then incremental cost | yes | | 09 | `09-kill-resume.ps1` | Checkpoint/resume correctness: exact count, zero duplicates | yes | +| 10 | `10-pacing-under-real-throttling.ps1` | Adaptive pacing pays for itself against genuine 429s | yes | +| 11 | `11-throttle-accuracy.ps1` | Under throttling, retrieval hinges on honoring `Retry-After` | yes | +| 12 | `12-delta-replay.ps1` | Delta enumerations repeat objects; replay factor vs ground truth | yes | +| 13 | `13-resource-unit-rate.ps1` | Sustained RU ceiling vs burst allowance, at held send rates | yes | + +11-13 are not in `run.ps1`: each deliberately drives the tenant to 429s, so they run standalone +(10 also throttles on purpose, which is why `run.ps1` puts it last). ## Methodology diff --git a/tests/benchmarks/common.ps1 b/tests/benchmarks/common.ps1 index bfc1cf2..cf43f7b 100644 --- a/tests/benchmarks/common.ps1 +++ b/tests/benchmarks/common.ps1 @@ -18,10 +18,25 @@ function Connect-MgxBenchmark { return } + # Certificate first. Each benchmark runs in a fresh pwsh process, so there is never a + # session to inherit, and the client-secret file below is the only other path - which is + # why this suite could not run at all once that file went away. The same three AZURE_* + # variables everything else in this repo uses, and no secret at rest. + if ($env:AZURE_TENANT_ID -and $env:AZURE_CLIENT_ID -and $env:AZURE_CLIENT_CERTIFICATE_PATH) { + $pfx = $env:AZURE_CLIENT_CERTIFICATE_PATH + if (-not (Test-Path $pfx)) { throw "AZURE_CLIENT_CERTIFICATE_PATH points at '$pfx', which does not exist." } + $cert = [System.Security.Cryptography.X509Certificates.X509Certificate2]::new( + $pfx, $env:AZURE_CLIENT_CERTIFICATE_PASSWORD) + Connect-MgGraph -TenantId $env:AZURE_TENANT_ID -ClientId $env:AZURE_CLIENT_ID ` + -Certificate $cert -NoWelcome + Write-Host "Connected app-only by certificate ($($env:AZURE_CLIENT_ID))" + return + } + $credPath = if ($env:MGX_BENCH_APP) { $env:MGX_BENCH_APP } else { Join-Path $HOME '.mgx-bench/app.json' } if (-not (Test-Path $credPath)) { - throw "No Graph session and no app credentials at '$credPath'. Connect-MgGraph first, or set MGX_BENCH_APP." + throw "No Graph session, no AZURE_* certificate variables, and no app credentials at '$credPath'." } $cfg = Get-Content $credPath -Raw | ConvertFrom-Json $cred = [pscredential]::new($cfg.appId, (ConvertTo-SecureString $cfg.clientSecret -AsPlainText -Force)) @@ -32,6 +47,47 @@ function Connect-MgxBenchmark { # Mints a raw app-only bearer token for the Invoke-RestMethod baselines, # which deliberately bypass every SDK/Mgx layer. function Get-BenchAppToken { + # The raw-REST contender needs a bearer token of its own. Under certificate auth there is + # no secret to POST, so mint one with a signed client assertion (private_key_jwt) using the + # same certificate the SDK session uses. + if ($env:AZURE_TENANT_ID -and $env:AZURE_CLIENT_ID -and $env:AZURE_CLIENT_CERTIFICATE_PATH) { + $cert = [System.Security.Cryptography.X509Certificates.X509Certificate2]::new( + $env:AZURE_CLIENT_CERTIFICATE_PATH, $env:AZURE_CLIENT_CERTIFICATE_PASSWORD) + $aud = "https://login.microsoftonline.com/$($env:AZURE_TENANT_ID)/oauth2/v2.0/token" + $now = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() + + # x5t is the SHA-1 thumbprint, base64url - Entra rejects the assertion without it. + $b64u = { param($bytes) [Convert]::ToBase64String($bytes).TrimEnd('=').Replace('+','-').Replace('/','_') } + $hdr = @{ alg = 'RS256'; typ = 'JWT'; x5t = (& $b64u $cert.GetCertHash()) } | ConvertTo-Json -Compress + $pay = @{ aud = $aud; iss = $env:AZURE_CLIENT_ID; sub = $env:AZURE_CLIENT_ID + jti = [guid]::NewGuid().ToString(); nbf = $now; exp = $now + 600 } | ConvertTo-Json -Compress + + $unsigned = "$(& $b64u ([Text.Encoding]::UTF8.GetBytes($hdr))).$(& $b64u ([Text.Encoding]::UTF8.GetBytes($pay)))" + # RSACertificateExtensions.GetRSAPrivateKey is an EXTENSION method; PowerShell cannot + # invoke it as $cert.GetRSAPrivateKey(), so call the static form explicitly. + $rsa = [System.Security.Cryptography.X509Certificates.RSACertificateExtensions]::GetRSAPrivateKey($cert) + if (-not $rsa) { throw "Certificate '$($env:AZURE_CLIENT_CERTIFICATE_PATH)' has no usable RSA private key." } + $sig = $rsa.SignData( + [Text.Encoding]::UTF8.GetBytes($unsigned), + [Security.Cryptography.HashAlgorithmName]::SHA256, + [Security.Cryptography.RSASignaturePadding]::Pkcs1) + $assertion = "$unsigned.$(& $b64u $sig)" + + foreach ($attempt in 1..3) { + try { + $resp = Invoke-RestMethod -Method POST -Uri $aud -TimeoutSec 30 -Body @{ + grant_type = 'client_credentials' + client_id = $env:AZURE_CLIENT_ID + client_assertion_type = 'urn:ietf:params:oauth:client-assertion-type:jwt-bearer' + client_assertion = $assertion + scope = 'https://graph.microsoft.com/.default' + } + return $resp.access_token + } + catch { if ($attempt -eq 3) { throw }; Start-Sleep -Seconds 5 } + } + } + $credPath = if ($env:MGX_BENCH_APP) { $env:MGX_BENCH_APP } else { Join-Path $HOME '.mgx-bench/app.json' } if (-not (Test-Path $credPath)) { throw "No app credentials at '$credPath' (set MGX_BENCH_APP)." } @@ -48,6 +104,57 @@ function Get-BenchAppToken { } } +function Get-BenchUserIds { + <# + .SYNOPSIS + Ids to run a benchmark against, preferring the seeded 'bench.u' cohort. + + .DESCRIPTION + Five benchmarks (02, 03, 04, 07, 08) filter on startsWith(userPrincipalName,'bench.u'), + and nothing in this repo creates those users - so on any tenant that was not hand-seeded + the suite died with "seed the tenant first" and no instructions. That is most of the + reason the benchmark results in results/ went stale: the suite simply would not run. + + Seeded users are still preferred, because they are disposable and a write benchmark can + safely PATCH them. When there are not enough, fall back to ordinary users and say so + loudly - a number measured against a different cohort is still a number, it just must not + be compared against a seeded run. Callers that WRITE must pass -RequireSeeded. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [int] $Count, + [string] $Prefix = 'bench.u', + [switch] $RequireSeeded + ) + + $ids = [System.Collections.Generic.List[string]]::new() + Invoke-MgxRequest /users -All -Filter "startsWith(userPrincipalName,'$Prefix')" -Property id -WarningAction SilentlyContinue | + Select-Object -First $Count | ForEach-Object { $ids.Add($_.id) } + + if ($ids.Count -ge $Count) { + Write-Host " using $($ids.Count) seeded '$Prefix' users" + return , $ids + } + + if ($RequireSeeded) { + throw ("This benchmark WRITES to the users it touches, so it will only run against the " + + "disposable '$Prefix' cohort. Found $($ids.Count) of $Count. Seed the tenant, or " + + "run a read-only benchmark instead.") + } + + Write-Warning ("Only $($ids.Count) '$Prefix' users; falling back to ordinary tenant users. " + + "Read-only, but do NOT compare this run against a seeded one.") + $ids.Clear() + Invoke-MgxRequest /users -All -Property id -WarningAction SilentlyContinue | + Select-Object -First $Count | ForEach-Object { $ids.Add($_.id) } + + if ($ids.Count -eq 0) { throw "No users at all in this tenant - nothing to benchmark." } + if ($ids.Count -lt $Count) { + Write-Warning "Tenant has only $($ids.Count) users; running at that size instead of $Count." + } + return , $ids +} + function Import-MgxLocal { # Users FIRST: it auto-loads its exactly-matching Microsoft.Graph.Authentication. # Importing Mgx (or Authentication) first pulls the newest Auth assembly, and a @@ -208,15 +315,87 @@ function Invoke-WatchdoggedContender { function Write-BenchResult { param( [Parameter(Mandatory)] [string] $Benchmark, - [Parameter(Mandatory)] [object] $Result + [Parameter(Mandatory)] [object] $Result, + # The run's wall clock. Only needed for the RU rate below; picked up from $Result.WallMs + # when the caller already records one. + [long] $WallMs = 0 ) $dir = Join-Path $PSScriptRoot 'results' if (-not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir | Out-Null } $file = Join-Path $dir "$Benchmark.json" $entries = @() if (Test-Path $file) { $entries = @(Get-Content $file -Raw | ConvertFrom-Json) } + # Resource units are the currency Graph actually throttles directory workloads in, so a + # benchmark that records only wall time describes half the cost. Captured from session + # telemetry, which accumulates x-ms-resource-unit across every response. + $telemetry = $null + $wall = if ($WallMs -gt 0) { $WallMs } + elseif ($Result.WallMs -and [long]$Result.WallMs -gt 0) { [long]$Result.WallMs } + else { 0 } + try { + $t = Get-MgxTelemetry -ErrorAction Stop + $telemetry = [pscustomobject]@{ + ResourceUnits = $t.ResourceUnitsConsumed + TotalRequests = $t.Requests + Succeeded = $t.Succeeded + Failed = $t.Failed + ThrottleRetries = $t.ThrottleRetries + OtherRetries = $t.OtherRetries + PacingWaitMs = $t.AdaptivePacingWaitMs + PacingActivations= $t.AdaptivePacingActivations + RateLimiterWaitMs= $t.RateLimiterWaitMs + RuPerRequest = $(if ($t.Requests -gt 0) { + [math]::Round($t.ResourceUnitsConsumed / $t.Requests, 2) + } else { 0 }) + # Per-workload state, parsed from the pacer's own description. RU itself is a single + # tenant-wide counter, but the buckets tell you WHICH workload was being paced when + # the units were spent - a directory fan-out and a drive pull draw on limits that are + # documented and measured as independent, so a total alone hides which one is near + # its ceiling. + PacingState = $t.PacingState + PacingBuckets = $( + if ($t.PacingState) { + @($t.PacingState -split ';' | Where-Object { $_ } | + ForEach-Object { + $name, $rest = $_ -split ':', 2 + [pscustomobject]@{ + Workload = $name.Trim() + State = if ($rest) { $rest.Trim() } else { '' } + } + }) + } else { @() } + ) + # -1 means Graph never sent x-ms-throttle-limit-percentage. Measured live it never + # arrives, even during active 429s, so recording it per run is how we would notice + # if that ever changed. + LastThrottlePct = $t.LastThrottlePercentage + # A rate is only meaningful against the run's WALL clock. Telemetry's TotalElapsedMs + # is the SUM of per-request durations, so a concurrent run exceeds its own wall time + # by roughly the concurrency factor - dividing by it yields RU per request-second and + # understates the budget draw by that factor (a concurrency-128 run measured at 182 + # RU/s reported 1.6). Callers that know their wall clock pass it; the rest record no + # rate at all, because a missing number is auditable and a wrong one is not. + WallMs = $(if ($wall -gt 0) { $wall } else { $null }) + RuPerSecond = $(if ($wall -gt 0) { + [math]::Round($t.ResourceUnitsConsumed / ($wall / 1000), 1) + } else { $null }) + # The documented budget is 8,000 RU per 10s per application+tenant pair for tenants + # above 500 users, i.e. 800 RU/s. Recorded as a ratio so a run that approaches the + # ceiling is obvious without re-deriving the arithmetic each time. + BudgetFraction = $(if ($wall -gt 0) { + [math]::Round(($t.ResourceUnitsConsumed / ($wall / 1000)) / 800, 3) + } else { $null }) + } + } + catch { + # A benchmark that does not load Mgx (the bare-SDK comparison arms) has no telemetry. + # That is expected; record its absence rather than failing the run. + $telemetry = $null + } + $meta = [pscustomobject]@{ Result = $Result + Telemetry = $telemetry MgxVersion = (Get-Module Mgx -ErrorAction SilentlyContinue)?.Version?.ToString() SdkVersion = (Get-Module Microsoft.Graph.Authentication -ErrorAction SilentlyContinue)?.Version?.ToString() PSVersion = $PSVersionTable.PSVersion.ToString() diff --git a/tests/benchmarks/run.ps1 b/tests/benchmarks/run.ps1 index 53400b6..a292695 100644 --- a/tests/benchmarks/run.ps1 +++ b/tests/benchmarks/run.ps1 @@ -27,7 +27,10 @@ $suite = @( @{ id = '08'; script = '08-delta-sync.ps1'; args = @(); cooldown = $false }, @{ id = '09'; script = '09-kill-resume.ps1'; args = @(); cooldown = $false }, @{ id = '07'; script = '07-adaptive-pacing.ps1'; args = @(); cooldown = $true }, - @{ id = '04'; script = '04-batch-create.ps1'; args = $(if ($IncludeSlow) { @() } else { @('-SkipBaselines') }); cooldown = $false } + @{ id = '04'; script = '04-batch-create.ps1'; args = $(if ($IncludeSlow) { @() } else { @('-SkipBaselines') }); cooldown = $true }, + # Last, and after a cooldown: it drives the tenant toward its resource-unit budget on + # purpose, so it must not color any earlier benchmark's numbers. + @{ id = '10'; script = '10-pacing-under-real-throttling.ps1'; args = @(); cooldown = $false } ) if ($Only) { diff --git a/examples/23-benchmark-resilience.ps1 b/tests/benchmarks/tenant-resilience-comparison.ps1 similarity index 100% rename from examples/23-benchmark-resilience.ps1 rename to tests/benchmarks/tenant-resilience-comparison.ps1 From c650c224ca5691e5ff051990084e12d03b3bc0f0 Mon Sep 17 00:00:00 2001 From: gromedev <61883549+gromedev@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:50:59 +0200 Subject: [PATCH 02/10] mgx 2.1.1 --- CHANGELOG.md | 12 + Directory.Build.props | 2 +- README.md | 253 ++++++++---------- module/mgx.psd1 | 64 +---- .../Configuration/EnableMgxResilience.cs | 62 ++++- .../Http/ResilientDelegatingHandler.cs | 55 ++++ tests/Live/Mgx.Throttle.Tests.ps1 | 135 ++++++++++ tests/Live/README.md | 28 ++ .../Engine/Http/ResilienceBridgeTests.cs | 249 +++++++++++++++++ .../Mgx.IntegrationTests.csproj | 14 + 10 files changed, 668 insertions(+), 206 deletions(-) create mode 100644 tests/Live/Mgx.Throttle.Tests.ps1 create mode 100644 tests/Mgx.IntegrationTests/Engine/Http/ResilienceBridgeTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index c4ba26c..fca3990 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 2.1.1 + +Patch release. One fix; no feature or API changes. + +### Fixed + +- Enable-MgxResilience ignored throttling: pacing never slowed, telemetry reported no retries, and a throttled request could fail outright. + +### Changed + +- Under Enable-MgxResilience, a 503 or 504 on a write is no longer retried. Throttling (429) is unaffected. + ## 2.1.0 Adds proactive throttle avoidance, ranged content downloads, and resumable enumeration. diff --git a/Directory.Build.props b/Directory.Build.props index 611c34a..bcd34af 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,7 +1,7 @@ - 2.1.0 + 2.1.1 Thomas Maillo Grome Mgx (c) 2026 Thomas Maillo Grome. All rights reserved. diff --git a/README.md b/README.md index 18295e5..a2186cb 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,16 @@ # Mgx -Microsoft Graph becomes difficult when a PowerShell process turns into a long-running, concurrent, stateful workload operating against a throttled and eventually consistent API. Sequential lookups are slow at scale, long-running requests can hang on dead connections, and scripts that do not explicitly handle throttling or transient failures can lose data or require manual recovery. -Given a large Graph workload, Mgx can execute it efficiently, preserve correctness, adapt to Graph's limits, survive transient failure and state changes, remain observable, and recover without forcing the caller to implement the machinery itself. +Microsoft Graph works well for ordinary scripts. It gets harder when a script becomes a long-running job making thousands of requests. -Mgx does this by adding concurrency, streaming pagination, batching, and resilience features to Microsoft Graph PowerShell while (optionally) using the existing `Connect-MgGraph` authentication context. +At that point, the problems are usually not authentication or the Graph API itself. They are pagination, throttling, concurrency, dead connections, retries, memory usage, and jobs that need to survive for hours without manual intervention. -In benchmarks, operations that benefit from concurrency and batching were **4–5× faster** than the Microsoft Graph SDK. On plain enumeration, where there is nothing to parallelize, it comes out about level with a tuned SDK - the difference being that it needs no tuning to get there. Mgx also provides resilience for throttling, transient failures, dead connections, and interrupted long-running operations: enumerating a 100,000-user tenant while its request budget was held at the ceiling, it returned all 100,000, where a plain `Invoke-RestMethod` loop returned 1,700.¹ +Mgx is a PowerShell client for Microsoft Graph that deals with those problems. It adds concurrent requests, streaming pagination, batching, rate limiting, retries, connection recovery, checkpointed exports, delta synchronization, and telemetry. It can use the authentication context established by `Connect-MgGraph`, so it does not require a separate authentication system. -¹ Tested under sustained throttle waves, injected 429/503 faults, dead sockets, and `kill -9` during export. See [`tests/benchmarks/`](tests/benchmarks/). +In benchmarks against a 100,000-user tenant, workloads involving many individual lookups were **4–5× faster** than the Microsoft Graph SDK. Plain enumeration, where there is little to parallelize, is roughly the same speed as a tuned SDK. + +The bigger difference is what happens when Graph starts throttling or the network stops cooperating. In one test, a 100,000-user enumeration was run while the application's request budget was held at its limit. Mgx returned all 100,000 users. A raw `Invoke-RestMethod` loop without equivalent retry and pacing logic returned only 1,700.¹ + +¹ Tested with sustained throttle waves, injected 429/503 responses, dead sockets, and process termination during export. See [`tests/benchmarks/`](tests/benchmarks/). ## Quick start @@ -18,130 +21,104 @@ Connect-MgGraph -Scopes "User.Read.All" Invoke-MgxRequest /users -All -Property displayName,mail ``` -## Key Capabilities - -* **Performance:** Streaming pagination, concurrent fan-out, and batched writes of up to 20 sub-requests per HTTP call. -* **Resilience:** Proactive rate limiting, exponential backoff with jitter, circuit breakers, adaptive per-workload request pacing, body-read timeouts, and connection recycling. -* **Operations:** Streamed JSONL exports with checkpoint/resume, delta sync token management, and resilience injection for existing Graph SDK scripts through `Enable-MgxResilience`. -* **Content:** File and media downloads with `Get-MgxContent`, whole or by byte range, over a redirect path validated against a host allowlist and followed without the bearer token. -* **Observability:** Execution metrics via `Get-MgxTelemetry`, including HTTP timing, throttle waits, retries, and resource consumption. +## What Mgx adds ---- +- Concurrent fan-out for workloads with many independent requests +- Streaming pagination instead of collecting the entire result set in memory +- Batch requests with up to 20 sub-requests per HTTP call +- Proactive rate limiting, retries with jitter, circuit breaking, and connection recycling +- Adaptive per-workload request pacing to avoid running into Graph limits unnecessarily +- JSONL exports with checkpoints and resume support +- Delta synchronization with persisted state and mid-run recovery +- Whole-file and ranged content downloads through a validated redirect path without forwarding the bearer token +- Telemetry for HTTP timing, throttling, retries, and resource consumption +- `Enable-MgxResilience` for adding Mgx's resilience handling to existing Microsoft Graph SDK scripts ## Benchmarks > **Environment:** Entra ID test tenant with 100k users and 15.8k groups. PowerShell 7.6.4 (.NET 10) with the Microsoft.Graph SDK, app-only via certificate. The module's supported floor is 7.4 (.NET 8), which is verified by the test suite but is not where these figures were measured. Every row below comes from one run of the suite against one tenant on one build; rows that could not be re-measured have been removed rather than carried forward. -### Performance & Throughput +### Performance & throughput -| Operation | Mgx | SDK (`Get-MgUser`) | Raw REST (`Invoke-RestMethod`) | Speedup vs SDK | -| ------------------------------------- | ---------: | -----------------: | -----------------------------: | -------------: | -| **List 100,000 users** | **47.1s** | 53.2s¹ | 58.5s | 1.1× | -| **Look up 5,000 users by ID** | **98.8s** | 521.0s | 985.3s | **5.3×** | -| **User report** *(1k users + groups)* | **23.9s** | 107.9s | 205.8s | **4.5×** | -| **Full delta enumeration** *(130,233 items)* | **145.6s** | - | - | - | +| Operation | Mgx | SDK (`Get-MgUser`) | Raw REST (`Invoke-RestMethod`) | Speedup vs SDK | +| --- | ---: | ---: | ---: | ---: | +| **List 100,000 users** | **47.1s** | 53.2s¹ | 58.5s | 1.1× | +| **Look up 5,000 users by ID** | **98.8s** | 521.0s | 985.3s | **5.3×** | +| **User report** *(1k users + groups)* | **23.9s** | 107.9s | 205.8s | **4.5×** | +| **Full delta enumeration** *(130,233 items)* | **145.6s** | - | - | - | -¹ Both figures are the SDK at `-PageSize 999`, which is what the benchmark runs. At the SDK's *default* page size the same enumeration takes **about 2.5-3x** as long - measured 3.0x (168.2s against 57.0s), 2.66x and 2.52x on separate runs of the same tenant, which is the spread this figure has — the practical difference is that mgx needs no tuning to be fast, not that it out-runs a tuned SDK on plain enumeration. +¹ Both figures are the SDK at `-PageSize 999`, which is what the benchmark runs. At the SDK's default page size the same enumeration takes about 2.5–3× as long. The practical difference is that Mgx needs no tuning to get close to the tuned SDK result. -Graph charges resource units per request, by the shape of the query rather than by which client -sent it - so every column above spends the same: about 102 units for the enumeration, 5,002 for -the lookups, 2,003 for the report. That equality holds only because all three issue equivalent -requests - the page size noted above changes the request count, and the cost with it. -What mgx adds is not a cheaper request but a visible one: it accumulates `x-ms-resource-unit` -across the session and paces against the budget they are charged to. See -[Resource units: what queries actually cost](#resource-units-what-queries-actually-cost). +Graph charges resource units per request based on the query shape, not on which client sent it. The three columns above therefore spend roughly the same amount when they issue equivalent requests: about 102 units for the enumeration, 5,002 for the lookups, and 2,003 for the report. -Measured from `x-ms-resource-unit`. `/users/{id}` costs 1 RU; `transitiveMembers` with `$select` and `$top` costs 3, matching the documented cost table exactly. At 5,002 RU the heaviest row above spends about 1.3% of the 8,000 RU / 10s Identity & Access budget for one application + tenant pair. The delta endpoint does not emit the header, so no figure is given. +Mgx does not make those requests cheaper. It makes the budget visible: it accumulates `x-ms-resource-unit` across the session and uses that information when pacing requests. See [Resource units: what queries actually cost](#resource-units-what-queries-actually-cost). + +Measured from `x-ms-resource-unit`. `/users/{id}` costs 1 RU; `transitiveMembers` with `$select` and `$top` costs 3, matching the documented cost table. At 5,002 RU the heaviest row above spends about 1.3% of the 8,000 RU / 10s Identity & Access budget for one application + tenant pair. The delta endpoint does not emit the header, so no figure is given. ### Resilience under throttling -Enumerating all 100,000 users of a test tenant while a second client held the application's -resource-unit budget at its ceiling. Ground truth is `/users/$count`, which the directory -serves from its own index rather than by walking pages, so it cannot inherit a pagination -fault from the thing it is being used to check. - -| Contender | Retrieved | Missing | Duplicates | Wall time | -| ---------------------------------- | --------------------: | ------: | ---------: | --------: | -| `Invoke-MgxRequest -All` | **100,000** / 100,000 | 0 | 0 | 715.2s | -| `Get-MgUser -All` | **100,000** / 100,000 | 0 | 0 | 729.9s | -| `Invoke-RestMethod` + fixed retry | 8,000 / 100,000 | 92,000 | 0 | 120.0s | -| `Invoke-RestMethod`, no retry | 1,700 / 100,000 | 98,300 | 0 | 5.5s | - -The bottom row finishes in five and a half seconds holding 1.7% of the tenant. Both the SDK -and Mgx return everything - the SDK's retry handler is correct here, and fifteen seconds -between them across twelve minutes is not a difference worth reading into. What separates the -table is whether a contender honors `Retry-After` at all, not which library it comes from. - -Run again with the budget untouched, all four return exactly 100,000 with no duplicates. -The gap is produced by throttling, not by enumeration. - -Adding `Enable-MgxResilience` to the SDK neither helps nor hurts here, because the SDK's own -retry handler already covers throttling. What it adds is not covered by this table: per-request -body-read timeouts and periodic connection recycling, so a dead socket surfaces as a retryable -error rather than an indefinite hang. The benchmark suite runs its SDK baselines inside -watchdogged child processes for exactly that reason - bare SDK cmdlets have no default -body-read timeout and have hung indefinitely during these runs. +This test enumerated all 100,000 users while a second client held the application's resource-unit budget at its ceiling. Ground truth is `/users/$count`, which the directory serves from its own index rather than by walking pages. + +| Contender | Retrieved | Missing | Duplicates | Wall time | +| --- | ---: | ---: | ---: | ---: | +| `Invoke-MgxRequest -All` | **100,000 / 100,000** | 0 | 0 | 715.2s | +| `Get-MgUser -All` | **100,000 / 100,000** | 0 | 0 | 729.9s | +| `Invoke-RestMethod` + fixed retry | 8,000 / 100,000 | 92,000 | 0 | 120.0s | +| `Invoke-RestMethod`, no retry | 1,700 / 100,000 | 98,300 | 0 | 5.5s | + +The raw REST clients finish sooner because they stop making useful progress as soon as the budget is exhausted. Mgx and the SDK take longer because they keep going until the enumeration is complete. The SDK's retry handling is already correct here; the useful distinction is whether the client actually honors throttling instead of treating the first wave of 429s as the end of the job. + +Running the same enumeration without the competing load produced exactly 100,000 users from all four clients. The difference above is caused by throttling, not pagination. + +`Enable-MgxResilience` does not replace the SDK's existing retry handler. The SDK already handles throttling. Mgx adds things that are not covered by that handler, including per-request body-read timeouts and periodic connection recycling, so a dead socket can become a retryable failure instead of an indefinite hang. ### Delta enumerations repeat themselves -A delta run does not paginate a snapshot, so one object can arrive on several pages of the -same enumeration. Against a tenant of 15,779 groups, over 1,135 pages: +A delta run does not paginate a snapshot. The same object can appear on several pages of one enumeration. Against a tenant containing 15,779 groups, over 1,135 pages produced these results: -| Contender | Emitted | Distinct | -| ----------------------------------------- | ------: | -------: | -| `Invoke-RestMethod` over `/groups/delta` | 172,740 | 17,467 | -| `Sync-MgxDelta` over the same resource | 172,740 | 17,467 | +| Contender | Emitted | Distinct | +| --- | ---: | ---: | +| `Invoke-RestMethod` over `/groups/delta` | 172,740 | 17,467 | +| `Sync-MgxDelta` over the same resource | 172,740 | 17,467 | -Ten objects handed back for every one the directory holds. Mgx does not deduplicate this: it -is 126 seconds quicker over the same pages and exactly as repetitive. Deduplicating correctly -means choosing which occurrence of an id wins when a single run reports the same object both -live and `@removed`, and which occurrence is last is not knowable while streaming without -retaining every id seen. Until that is settled, dedupe downstream on `id`. +Mgx is not silently deduplicating these objects. Correct deduplication requires a policy for cases where the same ID appears both live and as `@removed`, and the final occurrence is not knowable while streaming without retaining every ID seen. Until a policy is defined by the caller, dedupe downstream on `id`. -Distinct exceeds the group count because a full delta also returns tombstones for deleted -groups - 16,324 of the objects above carry `@removed`. +Distinct exceeds the group count because a full delta also returns tombstones for deleted groups; 16,324 of the objects above carry `@removed`. ### What adaptive pacing costs when nothing is throttling -It depends entirely on how long the run is, and the honest answer is "nothing on a long one, -a lot on a short one". +Adaptive pacing is enabled by default. It starts conservatively, increases the allowed rate as requests succeed, and backs off when throttling or latency changes indicate that the workload is approaching a limit. -The pacer opens cold at 4 rps and doubles each clean second until it reaches the ceiling, then -deactivates. A run long enough to finish ramping pays for the ramp once and never notices. A run -that finishes *during* it pays for the whole thing. Both measured on the 100k-user tenant: +That has almost no cost on a long-running workload because the initial ramp is paid once. It can be noticeable on a small workload that finishes before the ramp is complete. -| workload | pacing on *(default)* | `-NoAdaptivePacing` | measured by | +| Workload | Pacing on *(default)* | `-NoAdaptivePacing` | Measured by | | --- | ---: | ---: | --- | | 8,000 concurrent reads | 39.4s | 40.3s | [benchmark 07](tests/benchmarks/07-adaptive-pacing.ps1) | | 50 lookups, concurrency 8, cold session | 4.4s | 1.0s | [benchmark 14](tests/benchmarks/14-pacing-cold-cost.ps1) | -The large run is free - 0 pacing activations, 0 throttle retries, and the paced run is if -anything the faster of the two. The small one costs **4-6x** depending on the day (4.6x above; -separate runs have given 4.4x and about 6x), and the reason is visible in telemetry: roughly 30 -seconds of accumulated `AdaptivePacingWaitMs` spread across a fan-out that would otherwise take -one. If your work is short, interactive, and against a tenant you are not hammering, -`Set-MgxOption -NoAdaptivePacing` is the switch. +The large run had zero pacing activations and zero throttle retries; the paced run was slightly faster. The small run paid roughly 4–6× because the workload completed during the cold-start ramp. Telemetry exposes this as `AdaptivePacingWaitMs`. + +For short interactive workloads against a tenant you are not pushing hard, disable it with: -No 429s appeared at any concurrency the module permits. It caps at 128, and this tenant took -700 RU/s for 90s without refusing anything - a raw client only crosses the ceiling at around 200 -requests in flight, which is past what Mgx will issue. Graph absorbed the load by inflating latency -from 113ms to ~1s instead. That is why pacing reacts to `Retry-After` and latency drift rather than -to a header that never arrives. +```powershell +Set-MgxOption -NoAdaptivePacing +``` + +No 429s appeared at any concurrency the module permits. It caps at 128, and this tenant sustained roughly 700 RU/s for 90 seconds without refusing requests. A raw client only crossed the tenant's ceiling at around 200 requests in flight, which is beyond what Mgx will issue. Graph instead responded by increasing latency from about 113 ms to roughly 1 second. The pacing logic therefore reacts to `Retry-After` and latency drift rather than waiting for a throttle header that may never arrive. -### Memory Efficiency & Recovery +### Memory efficiency & recovery -Export 100,000 users to JSONL: +Exporting 100,000 users to JSONL: -| | Mgx `Export-MgxCollection` | Mgx `Invoke-MgxRequest \| file` | `Invoke-RestMethod` buffer+write | -| ------------------ | -------------------------: | ------------------------------: | -------------------------------: | -| Wall time | 53.0s | 47.0s | 64.0s | -| Peak working set | 305MB | **265MB** | **555MB** | -| Managed heap delta | +19.8MB | **+5.0MB** | **+415.7MB** | +| | Mgx `Export-MgxCollection` | Mgx `Invoke-MgxRequest \| file` | `Invoke-RestMethod` buffer+write | +| --- | ---: | ---: | ---: | +| Wall time | 53.0s | 47.0s | 64.0s | +| Peak working set | 305MB | **265MB** | **555MB** | +| Managed heap delta | +19.8MB | **+5.0MB** | **+415.7MB** | -* **Low memory footprint:** streaming to a file adds **5MB** to the managed heap over the whole export, against 416MB for the buffer-then-write approach. Peak working set is 265MB against 555MB. -* **Kill-safe resume:** `Export-MgxCollection` checkpoints progress continuously. After interruption, rerunning the export resumes from the checkpoint without duplicating objects. +Streaming directly to a file added 5MB to the managed heap over the whole export, compared with 416MB for the buffer-then-write approach. Peak working set was 265MB versus 555MB. ---- +`Export-MgxCollection` also checkpoints progress continuously. After an interruption, rerunning the export resumes from the checkpoint without duplicating objects. ## Examples @@ -172,7 +149,7 @@ Get-MgUser -All Disable-MgxResilience ``` -### Concurrent Fan-Out & Batching +### Concurrent fan-out & batching ```powershell # Concurrent fan-out lookups @@ -200,21 +177,17 @@ Invoke-MgxRequest "/me/drive/items/$folderId/children" -All | See [`examples/`](examples/) for additional examples. ---- - -## Microsoft.Graph Comparison - -| Common Operation | Standard `Microsoft.Graph` | `Mgx` | -| ---------------------------- | -------------------------------------------------- | ------------------------------------------------------------ | -| **Bulk Lookups** | `$ids \| ForEach-Object { Get-MgUser -UserId $_ }` | `$ids \| Invoke-MgxRequest '/users/{id}'` | -| **Bulk Updates** | `$ids \| ForEach-Object { Update-MgUser ... }` | `$urls \| Invoke-MgxBatchRequest -Method PATCH -Body @{...}` | -| **Exporting Data** | `$all = Get-MgUser -All; $all \| Export-Csv ...` | `Export-MgxCollection /users -OutputFile users.jsonl` | -| **Fault Protection** | Written per script | Built-in, or added to existing scripts with `Enable-MgxResilience` | -| **Observability** | Timed by the caller | `Get-MgxTelemetry` | -| **Dead Connection Handling** | Left to the default HTTP timeouts | Body-read timeouts + connection recycling | -| **Beta Endpoints** | Requires `Microsoft.Graph.Beta` | `-ApiVersion beta` | +## Microsoft.Graph comparison ---- +| Common operation | Standard `Microsoft.Graph` | `Mgx` | +| --- | --- | --- | +| **Bulk lookups** | `$ids \| ForEach-Object { Get-MgUser -UserId $_ }` | `$ids \| Invoke-MgxRequest '/users/{id}'` | +| **Bulk updates** | `$ids \| ForEach-Object { Update-MgUser ... }` | `$urls \| Invoke-MgxBatchRequest -Method PATCH -Body @{...}` | +| **Exporting data** | `$all = Get-MgUser -All; $all \| Export-Csv ...` | `Export-MgxCollection /users -OutputFile users.jsonl` | +| **Fault protection** | Written per script | Built in, or added to existing scripts with `Enable-MgxResilience` | +| **Observability** | Timed by the caller | `Get-MgxTelemetry` | +| **Dead connection handling** | Left to the default HTTP timeouts | Body-read timeouts + connection recycling | +| **Beta endpoints** | Requires `Microsoft.Graph.Beta` | `-ApiVersion beta` | ## Output shape @@ -243,25 +216,21 @@ To work with `PSCustomObject`s instead, request the raw payload: Invoke-MgxRequest /users -All -Raw | ConvertFrom-Json ``` ---- - ## Cmdlets -| Cmdlet | Description | -| ------------------------------------------------ | --------------------------------------------------------------------------------------------------------- | -| `Invoke-MgxRequest` | Executes Graph requests with automatic concurrency, retries, and rate limiting | -| `Invoke-MgxBatchRequest` | Batches up to 20 requests into a single HTTP POST call | -| `Export-MgxCollection` | Streams paginated API results directly to JSONL with checkpointing | -| `Expand-MgxRelation` | Performs concurrent fan-out lookups to expand related object attributes | -| `Sync-MgxDelta` | Manages stateful delta queries and state token storage, with mid-run crash resume | -| `Get-MgxContent` | Downloads content bytes (whole or ranged) with a token-free, host-validated download path | -| `Set-MgxOption` / `Get-MgxOption` | Configures global limits, retry counts, timeouts, and circuit-breaker thresholds | -| `Enable-MgxResilience` / `Disable-MgxResilience` | Injects or removes Mgx resilience policies from native Microsoft.Graph SDK cmdlets | -| `Get-MgxTelemetry` | Outputs execution statistics including HTTP duration, rate-limit delays, retry counts, and resource units | - ---- +| Cmdlet | Description | +| --- | --- | +| `Invoke-MgxRequest` | Executes Graph requests with automatic concurrency, retries, and rate limiting | +| `Invoke-MgxBatchRequest` | Batches up to 20 requests into a single HTTP POST call | +| `Export-MgxCollection` | Streams paginated API results directly to JSONL with checkpointing | +| `Expand-MgxRelation` | Performs concurrent fan-out lookups to expand related object attributes | +| `Sync-MgxDelta` | Manages stateful delta queries and state token storage, with mid-run crash resume | +| `Get-MgxContent` | Downloads content bytes, whole or ranged, through a token-free validated download path | +| `Set-MgxOption` / `Get-MgxOption` | Configures global limits, retry counts, timeouts, and circuit-breaker thresholds | +| `Enable-MgxResilience` / `Disable-MgxResilience` | Injects or removes Mgx resilience policies from native Microsoft.Graph SDK cmdlets | +| `Get-MgxTelemetry` | Outputs execution statistics including HTTP duration, rate-limit delays, retry counts, and resource units | -## Resilience Architecture +## Resilience architecture All HTTP operations pass through four layered [Polly 8.x](https://github.com/App-vNext/Polly) resilience strategies: @@ -272,40 +241,42 @@ All HTTP operations pass through four layered [Polly 8.x](https://github.com/App ### Resource units: what queries actually cost -Graph throttles directory workloads on a **resource-unit budget**, not on request count or bandwidth. Mgx reads `x-ms-resource-unit` on every response and accumulates it, so `Get-MgxTelemetry` reports what a session actually spent - see [examples/resilience-and-telemetry/resource-unit-budgeting.ps1](examples/resilience-and-telemetry/resource-unit-budgeting.ps1). +Graph throttles directory workloads on a **resource-unit budget**, not simply on request count or bandwidth. Mgx reads `x-ms-resource-unit` from responses and accumulates it, so `Get-MgxTelemetry` can show what a session actually spent. See [examples/resilience-and-telemetry/resource-unit-budgeting.ps1](examples/resilience-and-telemetry/resource-unit-budgeting.ps1). Measured against a 15,779-group test tenant: | Query shape | RU | -|---|---| +| --- | ---: | | `transitiveMembers?$top=5` | 4 | | `transitiveMembers?$top=5&$select=id` | **3** | | `groups/{id}` (single read) | 1 | -Both figures match the [documented cost table](https://learn.microsoft.com/en-us/graph/throttling-limits) exactly: `transitiveMembers` is published at 5 RU, `$select` takes one off, and `$top` under 20 takes another. Cost is a property of the query *shape*, not the number of objects returned. On a per-group fan-out across that tenant, `$select` alone is the difference between 47,337 and 63,116 resource units. +Both figures match the documented cost table: `transitiveMembers` is published at 5 RU, `$select` takes one off, and `$top` under 20 takes another. Cost is a property of the query shape, not the number of objects returned. On a per-group fan-out across that tenant, `$select` alone is the difference between 47,337 and 63,116 resource units. Three findings from pushing a single client until the tenant pushed back: -- **The budget behaves like a token bucket, and burst hides its rate.** The published limit for a tenant of this size is 8,000 RU per 10 s per *application + tenant pair* (800 RU/s). Held at a fixed rate for 90 s from a single client: 300 and 700 RU/s were never refused - both spending well past the bucket - while 1,200 RU/s began refusing after 19 s, and once saturated the tenant served about **730 RU/s** however hard it was pushed. A short run at any of those rates shows nothing - the bucket starts full, so roughly the first 20,000 RU are free regardless of how fast they are spent, which is why a burst measurement reads as a much higher sustained ceiling than the tenant actually has. The ceiling itself moves: the day before, the same probe was refused at 631 RU/s and saturated near 440. Re-measure with [`tests/benchmarks/13-resource-unit-rate.ps1`](tests/benchmarks/13-resource-unit-rate.ps1) rather than trusting these constants. -- **`x-ms-throttle-limit-percentage` was never emitted** - not at 1.5x the documented budget, and not while the tenant was actively returning 429s. Mgx therefore treats it as opportunistic and paces on 429 + `Retry-After` and latency drift, which are reliable. -- **Batching does not buy cheaper units.** Unbatched requests sustained a *higher* RU/s before throttling than the same requests inside `$batch`. Batching saves round-trips, not budget. +- **The budget behaves like a token bucket, and burst hides its rate.** The published limit for a tenant of this size is 8,000 RU per 10 s per *application + tenant pair* (800 RU/s). Held at a fixed rate for 90 s from a single client, 300 and 700 RU/s were never refused. 1,200 RU/s began refusing after 19 s, and once saturated the tenant served about 730 RU/s however hard it was pushed. The exact ceiling moves: the same probe was refused at 631 RU/s on the previous day and saturated near 440. Re-measure with [`tests/benchmarks/13-resource-unit-rate.ps1`](tests/benchmarks/13-resource-unit-rate.ps1) rather than treating these measurements as constants. +- **`x-ms-throttle-limit-percentage` was never emitted.** Mgx therefore treats it as opportunistic and paces on 429 + `Retry-After` and latency drift instead. +- **Batching does not buy cheaper units.** Unbatched requests sustained a higher RU/s before throttling than the same requests inside `$batch`. Batching saves round-trips, not budget. -Limits are scoped per **application + tenant pair**, and separately per service - Intune, Excel, Education and the rest each carry their own quota. That is why pacing state is partitioned by workload rather than pooled: a throttled directory fan-out says nothing about the budget remaining for Teams. +Limits are scoped per **application + tenant pair**, and separately per service. That is why pacing state is partitioned by workload rather than pooled: a throttled directory fan-out says nothing about the budget remaining for Teams. -A failed request costs 0 RU - so a fan-out that fails uniformly consumes no budget, triggers no throttling, and finishes fast while measuring nothing. Check status codes, not duration. +A failed request costs 0 RU, so a fan-out that fails uniformly consumes no budget, triggers no throttling, and finishes fast while measuring nothing. Check status codes, not duration. -Ahead of those four, an **adaptive pacing gate** spaces requests before they are sent. It is on by default and spaces requests on both the `Invoke-Mgx*` path and the `Enable-MgxResilience` SDK-bridge path. It learns a per-workload rate from throttling signals (additive increase, multiplicative decrease), keeps Drive, Directory and other workloads in separate buckets so one throttled workload does not slow the rest, and starts conservatively on a cold session. The learning happens only on the `Invoke-Mgx*` path: the SDK's own retry handler sits inside the wrapped chain and answers a 429 before the pipeline sees it, so a bridged session keeps spacing but never adapts. Batch outer POSTs are the one exemption - `GraphBatchClient` runs its own item-level AIMD, and stacking two controllers on one workload would compound their backoff. Disable with `Set-MgxOption -NoAdaptivePacing`; inspect the learned state in `Get-MgxTelemetry`. +Adaptive pacing runs ahead of the four resilience strategies above. It spaces requests before they are sent, learns a per-workload rate from throttling signals using additive increase / multiplicative decrease, and keeps Drive, Directory, and other workloads in separate buckets. It starts conservatively on a cold session. ---- +Batch outer POSTs are the one exemption. `GraphBatchClient` runs its own item-level AIMD controller; stacking two controllers on one workload would compound their backoff. -## Requirements & Building +Disable adaptive pacing with `Set-MgxOption -NoAdaptivePacing` and inspect the learned state in `Get-MgxTelemetry`. + +## Requirements & building ### Requirements -* **PowerShell:** 7.4+ -* **Authentication:** `Microsoft.Graph.Authentication` 2.10.0+ for token acquisition. It is not a declared module dependency. +- **PowerShell:** 7.4+ +- **Authentication:** `Microsoft.Graph.Authentication` 2.10.0+ for token acquisition. It is not a declared module dependency. -### Build from Source +### Build from source ```powershell ./build.ps1 @@ -313,8 +284,6 @@ Ahead of those four, an **adaptive pacing gate** spaces requests before they are Dependencies (`Microsoft.Graph.Core`, `Polly.Core`, and `System.Threading.RateLimiting`) are loaded into an isolated **Assembly Load Context** to avoid version conflicts with other PowerShell modules. ---- - ## License [MIT](LICENSE) diff --git a/module/mgx.psd1 b/module/mgx.psd1 index ccb713f..4c5a7dc 100644 --- a/module/mgx.psd1 +++ b/module/mgx.psd1 @@ -1,6 +1,6 @@ @{ RootModule = 'mgx.psm1' - ModuleVersion = '2.1.0' + ModuleVersion = '2.1.1' GUID = 'a3f7e8d2-5b4c-4a1e-9f6d-2c8b0e3a7d5f' Author = 'Thomas Maillo Grome' CompanyName = 'Mgx' @@ -52,66 +52,14 @@ LicenseUri = 'https://github.com/gromedev/mgx/blob/main/LICENSE' ProjectUri = 'https://github.com/gromedev/mgx' ReleaseNotes = @' -v2.1.0 -Adds proactive throttle avoidance, ranged content downloads, and resumable enumeration. - -Added -- Adaptive request pacing, on by default. Requests are spaced before they are sent, per workload; opt out with Set-MgxOption -NoAdaptivePacing. -- Get-MgxContent, to download file and media content whole or by byte range (-First). -- Sync-MgxDelta -CheckpointPath, to resume interrupted enumerations. -- Sync-MgxDelta -Latest, to baseline state without enumerating. -- Sync-MgxDelta -Prefer, to send drive delta Prefer tokens. -- Resource units consumed, in telemetry output, benchmarks and examples. +v2.1.1 +Patch release. One fix; no feature or API changes. Fixed -- -Latest was honored after a state invalidation, dropping every change since the last sync. -- Delta resume dropped the interrupted run's items when an earlier run had already completed, and advanced the delta token past them. -- Export resume dropped the interrupted run's items when an earlier export had already written the output file. -- Export resume wrote a page's items twice when a run was interrupted twice inside the same page. -- Delta resume kept the previous sync's rows in front of the interrupted sync's output. -- Export resume started the export over when the interrupted run had died on a transient error. -- A resume checkpoint naming a file that was never a temp consumed it as data. -- A refused delta token was reported as an endpoint not supporting delta queries. -- Sync-MgxDelta as a session's first cmdlet sent its request to the public cloud endpoint. -- A denied or read-only output file ended a delta sync with an unhandled error on Windows. -- A missing drive item suggested retrying in beta instead of reporting the item absent. -- -Top was ignored when combined with -All, returning the whole collection. -- Get-Help described output shapes and parameters from before 2.0.0. -- Enumeration returned part of a collection without error when a nextLink was refused. -- A delta sync stopped with an error when a page contained a non-object item. -- Get-MgxContent left an unwritable -OutFile as an unhandled error naming a temp path. -- A non-Graph JSON error body threw from inside the exception it was being used to build, surfacing a CDN failure as a security refusal. -- A $batch 429 set a pacing cap that nothing enforced and nothing cleared, so telemetry reported it forever. -- Get-MgxContent ignored Retry-After when a download host sent it as an HTTP date. -- Get-Help was missing the examples for -CheckpointPath, -Latest, -Prefer, adaptive pacing, and two Get-MgxContent endpoints, and did not document Set-MgxOption -BatchChunkConcurrency. -- Seven examples rendered blank tables, and one reported password secrets as certificates. -- Repeat 429s could raise the adaptive cap above -RateLimitPerSecond. -- Delta state did not record its API version, so runs omitting -ApiVersion silently synced the other one. -- -Debug wrote pre-authenticated download URLs verbatim, from redirect headers and from response bodies. -- Get-MgxContent -OutFile with piped input downloaded every item but kept only the last. -- -Offset was ignored when a server returned a whole-body 200. -- An offset past end of file overwrote an existing -OutFile with an empty one. -- Two-hop content downloads were reported as failed requests. -- Adaptive pacing fired synchronized bursts when sleep times were clamped. -- Enable-MgxResilience recorded no telemetry: request counts, HTTP time and rate-limiter wait all stayed 0. -- Invoke-MgGraphRequest with a relative URI failed while resilience was enabled. -- $batch envelopes were bucketed as Other rather than their target workload. -- Slow start opened above ceilings set below 4 rps. -- ResiliencePipelineFactory.Reset() reverted configuration and kept batch pacer state across credential changes. -- Ctrl-C during file cleanup raised disposed-object errors. - -Documentation -- Graph delta responses repeat objects across pages; deduplicate on id, or baseline with -Latest. -- Adaptive pacing applies to every request except batch outer POSTs. -- Measured throttling behavior: resource unit budgets are scoped per application and tenant, and x-ms-throttle-limit-percentage is not emitted. - -v2.0.1 -Patch release. Three fixes; no feature or API changes. -- The rate limiter was disposed on a timer while live clients held it, so any Set-MgxOption call broke Enable-MgxResilience sessions minutes later. -- Enable-MgxResilience and Disable-MgxResilience cancelled SDK requests still in flight. -- Sync-MgxDelta -Uri with a query string or trailing slash failed on the second run. +- Enable-MgxResilience ignored throttling: pacing never slowed, telemetry reported no retries, and a throttled request could fail outright. -Earlier releases: https://github.com/gromedev/Mgx/blob/main/CHANGELOG.md +Changed +- Under Enable-MgxResilience, a 503 or 504 on a write is no longer retried. Throttling (429) is unaffected. '@ } } diff --git a/src/Mgx.Cmdlets/Cmdlets/Configuration/EnableMgxResilience.cs b/src/Mgx.Cmdlets/Cmdlets/Configuration/EnableMgxResilience.cs index 8af04e9..5a521d7 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Configuration/EnableMgxResilience.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Configuration/EnableMgxResilience.cs @@ -232,6 +232,55 @@ internal static void RefreshInjectedClient(Action warn, Action v } } + /// + /// Turns off the SDK's own retry handler for requests that pass through the wrap. + /// + /// That handler sits inside the wrapped chain and answers 429 and 503 itself, so a throttle + /// never reached Mgx's pipeline: the adaptive pacer stayed in slow start through a live + /// throttle, Get-MgxTelemetry reported no throttle retries while accumulating two minutes of + /// retry delay under another name, and the two retriers compounded - four intended attempts + /// could reach the wire eight times, because the SDK sleeps Retry-After inside the call and + /// Mgx's own attempt timeout then fires. + /// + /// Kiota reads this option per request. The type is resolved reflectively so the engine and + /// this assembly need no reference to it; if the SDK ever moves or renames it, the wrap keeps + /// working exactly as it did before. + /// + /// Throws rather than warning on the paths it cannot satisfy. It runs from the handler's + /// option factory, on a request thread with no pipeline to write a warning to - the handler + /// catches this, notes it, and carries on with the inner retry handler left as it was. + /// + internal static IReadOnlyDictionary? BuildInnerRetryOverride() + { + const string OptionType = "Microsoft.Kiota.Http.HttpClientLibrary.Middleware.Options.RetryHandlerOption"; + + var type = MgxCmdletBase.FindType(OptionType) + ?? throw new InvalidOperationException( + "the Graph SDK's retry option type was not found, so its own retry handler stays " + + "active inside the wrap and throttling will not reach Mgx's pacer or telemetry " + + "on this path"); + + var option = Activator.CreateInstance(type); + var maxRetry = type.GetProperty("MaxRetry"); + if (option == null || maxRetry == null || !maxRetry.CanWrite) + throw new InvalidOperationException( + "the Graph SDK's retry option could not be configured, so its own retry handler " + + "stays active inside the wrap"); + + // MaxRetry is the only lever that removes a retry. The option also exposes ShouldRetry, + // which looks like a way to decline 429 alone and leave the handler's 503 and 504 + // retries intact - it is not: the handler ORs it with its own status check, so + // ShouldRetry can only add retries, never suppress one. Measured against 1.21.1. + // + // The cost is that the handler's 503/504 retries go too, including on writes, and Mgx's + // pipeline will not take those over: it refuses to retry a non-idempotent request on a + // 5xx because the write may already have been applied. 429 is unaffected - the pipeline + // retries that for every method - so throttled writes still complete. + maxRetry.SetValue(option, 0); + + return new Dictionary { [OptionType] = option }; + } + private static HttpClient? BuildResilientSdkClient(HttpClient sdkClient, Action warn) { try @@ -246,15 +295,18 @@ internal static void RefreshInjectedClient(Action warn, Action v // The bridge handler delegates SendAsync to the original SDK HttpClient, // which processes through its complete handler pipeline internally. // - // The SDK's built-in RetryHandler still runs inside, so - // a persistent 429 may retry internally (SDK: ~3 times) before our outer - // handler retries again. This is bounded by TotalTimeoutSeconds (300s) - // and circuit breaker. Both paths share the same pipeline, rate limiter, + // The SDK's built-in RetryHandler sits inside this wrap and would otherwise answer + // 429 and 503 itself, before the outer pipeline ever saw them - so the pacer never + // learned from a throttle and telemetry booked a throttled session as zero retries. + // AdditionalRequestOptionsFactory disarms it per request. If that cannot be arranged + // it stays armed and the session behaves as it did before, just without the + // measurement. Both paths share the same pipeline, rate limiter, // and circuit breaker to prevent cache thrashing and ensure consistent // failure detection across SDK and direct Mgx cmdlets. var resilientHandler = new ResilientDelegatingHandler(pipeline, rateLimiter) { - InnerHandler = new SdkClientBridgeHandler(sdkClient) + InnerHandler = new SdkClientBridgeHandler(sdkClient), + AdditionalRequestOptionsFactory = BuildInnerRetryOverride }; ActiveHandler = resilientHandler; diff --git a/src/Mgx.Engine/Http/ResilientDelegatingHandler.cs b/src/Mgx.Engine/Http/ResilientDelegatingHandler.cs index 47bc7f7..74422b5 100644 --- a/src/Mgx.Engine/Http/ResilientDelegatingHandler.cs +++ b/src/Mgx.Engine/Http/ResilientDelegatingHandler.cs @@ -24,6 +24,54 @@ public sealed class ResilientDelegatingHandler : DelegatingHandler /// public Action? VerboseWriter { get; set; } + /// + /// Options stamped onto every request this handler sends, in addition to the caller's own. + /// The SDK bridge uses it to disarm the retry handler sitting inside the wrapped chain: that + /// handler answers 429 and 503 before this pipeline ever sees them, so the pacer never learns + /// from a throttle and telemetry books a throttled session as zero. Kiota reads its retry + /// option per request, so setting it here is what reaches it. + /// + /// Keyed by string and typed as object so the engine needs no reference to the SDK; the + /// caller supplies whatever the inner handler expects. + /// + public IReadOnlyDictionary? AdditionalRequestOptions { get; init; } + + /// + /// Resolved on the first request rather than when the handler is built. The type it needs + /// belongs to the SDK and is not loaded until the SDK has sent something through its own + /// chain, so building the options eagerly found nothing and silently left the inner handler + /// armed. Called once; the result, including null, is kept. + /// + public Func?>? AdditionalRequestOptionsFactory { get; init; } + + private IReadOnlyDictionary? _resolvedOptions; + private bool _optionsResolved; + + private IReadOnlyDictionary? ResolveAdditionalOptions() + { + if (AdditionalRequestOptions != null) return AdditionalRequestOptions; + if (AdditionalRequestOptionsFactory == null) return null; + if (_optionsResolved) return _resolvedOptions; + + try + { + _resolvedOptions = AdditionalRequestOptionsFactory(); + } + catch (Exception ex) + { + // The factory runs on a request thread, long after the cmdlet that supplied it + // finished its pipeline - so anything it touches that expects to be on that + // pipeline throws here. Failing the request over an option that only annotates it + // would be worse than not setting the option: leaving it null just means the inner + // handler keeps the retry behavior it had before the override existed. + _pendingVerbose.Enqueue($"Could not configure the SDK's retry option: {ex.Message}"); + _resolvedOptions = null; + } + + _optionsResolved = true; + return _resolvedOptions; + } + public ResilientDelegatingHandler( ResiliencePipeline pipeline, TokenBucketRateLimiter? rateLimiter) @@ -101,6 +149,13 @@ protected override async Task SendAsync( #pragma warning disable CS8714 // nullability mismatch in IDictionary generic foreach (var option in request.Options) ((IDictionary)clone.Options)[option.Key] = option.Value; + + // After the caller's own, so a handler-level stamp wins: the point is to + // override what the inner chain would otherwise do on its own. + var extra = ResolveAdditionalOptions(); + if (extra != null) + foreach (var option in extra) + ((IDictionary)clone.Options)[option.Key] = option.Value; #pragma warning restore CS8714 if (contentBytes != null) diff --git a/tests/Live/Mgx.Throttle.Tests.ps1 b/tests/Live/Mgx.Throttle.Tests.ps1 new file mode 100644 index 0000000..54d696d --- /dev/null +++ b/tests/Live/Mgx.Throttle.Tests.ps1 @@ -0,0 +1,135 @@ +<# + The one thing the rest of the suite cannot reach: what Enable-MgxResilience does when Graph + is actually refusing requests. + + It matters because the failure is silent. The Graph SDK's own retry handler sits inside the + wrap and answers 429 itself, so a throttled session looks identical to a clean one - the + pacer never slows down, and telemetry reports zero retries while the SDK sleeps out of sight + inside the call. Mocks cannot catch that: a mock 429 goes wherever the test author points it. + + This needs a second machine. One client cannot drive a tenant's budget from a single NAT'd + connection - measured 587 req/s from a laptop against 4,641 from a VM on the same network, + same test, same minute - and below roughly 700 req/s the tenant never refuses anything, so + the run proves nothing. Point MGX_LIVE_THROTTLE_SSH at a host with real egress. + + Run: MGX_LIVE_THROTTLE_SSH='user@host' Invoke-Pester -Path ./tests/Live/Mgx.Throttle.Tests.ps1 + + NOT read-only in effect: it deliberately exhausts the tenant's request budget and leaves it + refusing requests for several minutes afterwards. Run it last, or on its own. It is skipped + unless MGX_LIVE_THROTTLE_SSH is set, because most people running this suite do not have a + second machine to hand. +#> + +BeforeAll { + $repo = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent + if (-not $env:AZURE_TENANT_ID -or -not $env:AZURE_CLIENT_ID -or -not $env:AZURE_CLIENT_CERTIFICATE_PATH) { + throw "Live tests need AZURE_TENANT_ID, AZURE_CLIENT_ID and AZURE_CLIENT_CERTIFICATE_PATH." + } + $script:Configured = [bool]$env:MGX_LIVE_THROTTLE_SSH + . "$repo/tests/benchmarks/common.ps1" + Import-MgxLocal + Connect-MgxBenchmark | Out-Null + + if ($script:Configured) { + $script:Target = $env:MGX_LIVE_THROTTLE_SSH + $script:SshArgs = @('-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8') + if ($env:MGX_LIVE_THROTTLE_KEY) { $script:SshArgs = @('-i', $env:MGX_LIVE_THROTTLE_KEY) + $script:SshArgs } + $script:Remote = '/tmp/mgx-throttle-drain.py' + + # Reads the token on stdin. Never as an argument - argv is world-readable in ps - and + # never written to disk on the remote host. The certificate itself never leaves here: + # the assertion is signed locally and only the resulting bearer token crosses. + $drainer = @' +import http.client, ssl, sys, threading, time +token = sys.stdin.readline().strip() +workers, seconds = int(sys.argv[1]), int(sys.argv[2]) +ctx = ssl.create_default_context(); hdrs = {"Authorization": "Bearer " + token} +stop = time.time() + seconds +def run(): + c = None + while time.time() < stop: + try: + if c is None: + c = http.client.HTTPSConnection("graph.microsoft.com", 443, context=ctx, timeout=20) + c.request("GET", "/v1.0/users?$top=1&$select=id", headers=hdrs) + r = c.getresponse(); r.read() + except Exception: + try: c.close() + except Exception: pass + c = None +ts = [threading.Thread(target=run, daemon=True) for _ in range(workers)] +for t in ts: t.start() +for t in ts: t.join() +'@ + $drainer | & ssh @script:SshArgs $script:Target "cat > $script:Remote" + if ($LASTEXITCODE -ne 0) { throw "Could not stage the load generator on $script:Target." } + + $script:Token = Get-BenchAppToken + $script:Drain = Start-Job -ScriptBlock { + param($sshArgs, $target, $remote, $token) + $token | & ssh @sshArgs $target "python3 $remote 200 300" + } -ArgumentList $script:SshArgs, $script:Target, $script:Remote, $script:Token + + # Confirm the tenant is refusing requests before asserting anything about the refusal. + # Without this the test can only distinguish "the bridge saw no throttle" from "there + # was no throttle to see" by guessing, and a green result would mean nothing. + $probe = 'https://graph.microsoft.com/v1.0/users?$top=1&$select=id' + $deadline = (Get-Date).AddSeconds(120) + $script:ThrottleConfirmedAt = $null + while ((Get-Date) -lt $deadline -and -not $script:ThrottleConfirmedAt) { + $r = Invoke-WebRequest -Uri $probe -Headers @{ Authorization = "Bearer $script:Token" } ` + -SkipHttpErrorCheck -ErrorAction SilentlyContinue + if ($r.StatusCode -eq 429) { $script:ThrottleConfirmedAt = Get-Date } + else { Start-Sleep -Seconds 3 } + } + } +} + +AfterAll { + if ($script:Configured) { + if ($script:Drain) { Stop-Job $script:Drain -ErrorAction SilentlyContinue; Remove-Job $script:Drain -Force -ErrorAction SilentlyContinue } + # Stopping the job kills the local ssh client, not the process on the far side - that + # keeps running to its own deadline, hammering the tenant long after the test reports + # green. Kill it where it lives. The bracket makes the pattern not match the command + # carrying it, which would otherwise kill this shell before the rm ran. + & ssh @script:SshArgs $script:Target "rm -f $script:Remote; pkill -f '[m]gx-throttle-drain'" 2>$null + Disable-MgxResilience -ErrorAction SilentlyContinue | Out-Null + } +} + +Describe 'Enable-MgxResilience' { + Context 'under a real throttle' { + + It 'drove the tenant into refusing requests' -Skip:(-not $env:MGX_LIVE_THROTTLE_SSH) { + # Fails rather than skips: the run was asked for, so not reaching a throttle is a + # result about the setup, not a reason to report a pass. + $script:ThrottleConfirmedAt | Should -Not -BeNullOrEmpty -Because ` + 'the load generator never pushed the tenant to 429 - check the host has real egress, and that 200 workers is enough for it' + } + + It 'sees the throttle the SDK would otherwise absorb' -Skip:(-not $env:MGX_LIVE_THROTTLE_SSH) { + Enable-MgxResilience | Out-Null + Get-MgxTelemetry -Reset | Out-Null + foreach ($i in 1..12) { Get-MgUser -Top 1 -ErrorAction SilentlyContinue | Out-Null } + $t = Get-MgxTelemetry + + # The whole point of the wrap. Zero here with the tenant refusing requests means the + # SDK's own retry handler answered them first and Mgx never learned a thing. + $t.ThrottleRetries | Should -BeGreaterThan 0 -Because ` + 'the SDK retry handler inside the wrap must be disarmed, or a throttled session is indistinguishable from a clean one' + } + + It 'lets the pacer learn from it' -Skip:(-not $env:MGX_LIVE_THROTTLE_SSH) { + # Retries alone are not the win - slowing down is. A pacer still in slow-start after a + # throttle is one that never saw it. + (Get-MgxTelemetry).PacingState | Should -Match 'capped' -Because ` + 'the pacer should cap its rate once it has met a 429, not stay in slow-start' + } + + It 'still completes the caller''s requests' -Skip:(-not $env:MGX_LIVE_THROTTLE_SSH) { + # Surfacing throttling must not mean surfacing failures: the retries exist so the + # caller's call still returns. + (Get-MgxTelemetry).Failed | Should -Be 0 + } + } +} diff --git a/tests/Live/README.md b/tests/Live/README.md index d764cb2..6fe8b8e 100644 --- a/tests/Live/README.md +++ b/tests/Live/README.md @@ -33,6 +33,34 @@ A tenant with no SharePoint answers every sites/drives call with `BadRequest: Te have a SPO license`, and no permission grant changes that — the content roles can be present and still inert. +## The throttle test + +`Mgx.Throttle.Tests.ps1` covers the one thing mocks cannot: what `Enable-MgxResilience` does +when Graph is actually refusing requests. The SDK's own retry handler sits inside the wrap and +answers `429` itself, so a throttled session looks identical to a clean one — the pacer stays in +slow-start and telemetry reports zero retries while the SDK sleeps out of sight inside the call. +With that handler left armed, a caller's request eventually fails outright. + +It needs a second machine. One client cannot exhaust a tenant's budget through a single NAT'd +connection — 587 req/s from a laptop against 4,641 from a VM on the same network, same test, +same minute — and below roughly 700 req/s the tenant never refuses anything, so the run proves +nothing at all. + +```bash +export MGX_LIVE_THROTTLE_SSH='user@host' # a host with real egress +export MGX_LIVE_THROTTLE_KEY="$HOME/.ssh/id_..." # optional identity file +pwsh -c 'Invoke-Pester -Path ./tests/Live/Mgx.Throttle.Tests.ps1' +``` + +The certificate never leaves this machine: the assertion is signed locally and only the +resulting bearer token crosses, on stdin, never as an argument and never to a file. + +Unlike the rest of this directory it is **not read-only in effect** — it deliberately exhausts +the tenant's request budget and leaves it refusing requests for several minutes afterwards. Run +it last, or alone. It skips when `MGX_LIVE_THROTTLE_SSH` is unset, because most people running +this suite have no second machine to hand; when it is set, failing to reach a throttle is a +failure rather than a skip, so a green run cannot mean "there was nothing to see". + ## Keeping them honest `tests/Unit/Mgx.LiveCoverage.Tests.ps1` fails if an exported cmdlet has no `Describe` block diff --git a/tests/Mgx.IntegrationTests/Engine/Http/ResilienceBridgeTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/ResilienceBridgeTests.cs new file mode 100644 index 0000000..2eb898c --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/ResilienceBridgeTests.cs @@ -0,0 +1,249 @@ +using System.Net; +using Mgx.Cmdlets.Cmdlets.Configuration; +using Mgx.Engine.Http; +using Microsoft.Kiota.Http.HttpClientLibrary.Middleware; +using Microsoft.Kiota.Http.HttpClientLibrary.Middleware.Options; +using Xunit; + +namespace Mgx.IntegrationTests.Engine.Http; + +/// +/// The Enable-MgxResilience bridge wraps the SDK's handler chain, and that chain already +/// contains Kiota's own RetryHandler. It answers 429 and 503 itself, so without intervention +/// the outer pipeline never sees a throttle: the pacer does not slow down and telemetry books +/// a throttled session as zero retries. +/// +/// These run against the real Kiota handler rather than a stand-in, because a stand-in would +/// only prove that a mock agrees with the assumption being tested. +/// +[Collection("Pipeline")] +public class ResilienceBridgeTests +{ + private const string RetryOptionKey = + "Microsoft.Kiota.Http.HttpClientLibrary.Middleware.Options.RetryHandlerOption"; + + /// Kiota's RetryHandler only retries a request it can rewind, so give it a body. + private static HttpRequestMessage BufferedPost() => + new(HttpMethod.Post, "https://graph.microsoft.com/v1.0/users") + { + Content = new StringContent("{}") + }; + + private static HttpClient BuildBridge( + ThrottlingMockHandler wire, + Func?>? factory, + out ResilientDelegatingHandler bridge) + { + // The real SDK shape: our handler on the outside, Kiota's retry handler inside it, + // the network at the bottom. + var kiota = new RetryHandler(new RetryHandlerOption { MaxRetry = 3, Delay = 0 }) + { + InnerHandler = wire + }; + + var (pipeline, _) = ResiliencePipelineFactory.GetOrCreate( + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 3 }); + + bridge = new ResilientDelegatingHandler(pipeline, null) + { + AdditionalRequestOptionsFactory = factory, + InnerHandler = kiota + }; + + return new HttpClient(bridge); + } + + [Fact] + public async Task Bridge_WithoutOverride_InnerRetryHandlerHidesTheThrottle() + { + // Pins the defect the override exists to fix. If this ever goes green on its own, + // the SDK stopped retrying internally and the override is no longer load-bearing. + MgxTelemetryCollector.Current.Reset(); + var wire = new ThrottlingMockHandler(throttleFirst: 1); + + using var client = BuildBridge(wire, factory: null, out _); + var response = await client.SendAsync(BufferedPost()); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(2, wire.Requests); // the retry happened... + var snapshot = MgxTelemetryCollector.Current.GetSummary(); + Assert.Equal(0, snapshot.ThrottleRetries); // ...but it happened where we cannot see it + + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task Bridge_WithOverride_ThrottleReachesTheOuterPipeline() + { + MgxTelemetryCollector.Current.Reset(); + var wire = new ThrottlingMockHandler(throttleFirst: 1); + + using var client = BuildBridge( + wire, + () => new Dictionary { [RetryOptionKey] = new RetryHandlerOption { MaxRetry = 0 } }, + out _); + + var response = await client.SendAsync(BufferedPost()); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(2, wire.Requests); + var snapshot = MgxTelemetryCollector.Current.GetSummary(); + Assert.True(snapshot.ThrottleRetries > 0, + $"the 429 must reach the outer pipeline; ThrottleRetries was {snapshot.ThrottleRetries}"); + + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task Bridge_ResolvesOptionsLazily_AndOnlyOnce() + { + // The option's type belongs to the SDK and is not loaded until the SDK has sent + // something through its own chain. Building it eagerly found nothing and left the + // inner handler armed, which is why the factory is called on first send, not at + // construction. + MgxTelemetryCollector.Current.Reset(); + var calls = 0; + var wire = new ThrottlingMockHandler(throttleFirst: 0); + + using var client = BuildBridge( + wire, + () => + { + calls++; + return new Dictionary { [RetryOptionKey] = new RetryHandlerOption { MaxRetry = 0 } }; + }, + out _); + + Assert.Equal(0, calls); // not at construction + + await client.SendAsync(BufferedPost()); + Assert.Equal(1, calls); + + await client.SendAsync(BufferedPost()); + Assert.Equal(1, calls); // cached, including a null result + + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task Bridge_WithTheCmdletsOwnOption_ThrottleReachesTheOuterPipeline() + { + // Covers the half a hand-written dictionary cannot: the cmdlet resolves the SDK's option + // type reflectively, and nothing checks its spelling at compile time. This feeds what the + // cmdlet actually produced to the real handler. + _ = typeof(RetryHandlerOption); // the SDK has loaded this before the wrap sends anything + + var built = EnableMgxResilience.BuildInnerRetryOverride(); + Assert.NotNull(built); + Assert.True(built!.ContainsKey(RetryOptionKey), + $"keyed as [{string.Join(", ", built.Keys)}], which is not what Kiota reads"); + + MgxTelemetryCollector.Current.Reset(); + var wire = new ThrottlingMockHandler(throttleFirst: 1); + + using var client = BuildBridge(wire, () => built, out _); + var response = await client.SendAsync(BufferedPost()); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + var summary = MgxTelemetryCollector.Current.GetSummary(); + Assert.True(summary.ThrottleRetries > 0, + $"the cmdlet's own option must disarm the inner handler; ThrottleRetries was {summary.ThrottleRetries}"); + + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task Bridge_FactoryThatThrows_LeavesTheRequestAlone() + { + // The factory resolves on a request thread, after the cmdlet that supplied it has + // finished its pipeline. A late WriteWarning throws there, and before the guard that + // exception failed the request the option was only meant to annotate - turning a + // best-effort optimization into an outage. + MgxTelemetryCollector.Current.Reset(); + var wire = new ThrottlingMockHandler(throttleFirst: 0); + + using var client = BuildBridge( + wire, + () => throw new InvalidOperationException("no pipeline to write to"), + out _); + + var response = await client.SendAsync(BufferedPost()); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(1, wire.Requests); + + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task Bridge_WithOverride_NoLongerRetriesA503OnAWrite() + { + // Disarming the SDK's retry handler is not only about 429: Kiota also retries 503 and + // 504, and it retries them on POST. Mgx's pipeline deliberately will not - a 5xx on a + // non-idempotent request may mean the write was partially applied, and retrying it + // risks doing the work twice. + // + // ShouldRetry cannot decline 429 on its own - the handler ORs it with its own status + // check - so disarming it takes the 503 and 504 retries with it. This pins the + // consequence: a 503 on a write now reaches the caller instead of being retried out of + // sight. That is the safer answer for a write, but it is a change, and it should fail + // here rather than surprise someone. + MgxTelemetryCollector.Current.Reset(); + var wire = new ThrottlingMockHandler(throttleFirst: 1, code: HttpStatusCode.ServiceUnavailable); + + using var client = BuildBridge( + wire, + () => new Dictionary { [RetryOptionKey] = new RetryHandlerOption { MaxRetry = 0 } }, + out _); + + var response = await client.SendAsync(BufferedPost()); + + Assert.Equal(HttpStatusCode.ServiceUnavailable, response.StatusCode); + Assert.Equal(1, wire.Requests); + } + + [Fact] + public async Task Bridge_WithoutOverride_TheSdkRetriesA503OnAWrite() + { + // The other half of the pair: left armed, Kiota answers the 503 itself and the write + // is repeated. This is what the override above stops. + MgxTelemetryCollector.Current.Reset(); + var wire = new ThrottlingMockHandler(throttleFirst: 1, code: HttpStatusCode.ServiceUnavailable); + + using var client = BuildBridge(wire, factory: null, out _); + var response = await client.SendAsync(BufferedPost()); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(2, wire.Requests); + } + + private sealed class ThrottlingMockHandler(int throttleFirst, HttpStatusCode code = HttpStatusCode.TooManyRequests) : HttpMessageHandler + { + private int _seen; + public int Requests => _seen; + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var n = Interlocked.Increment(ref _seen); + if (n <= throttleFirst) + { + var throttled = new HttpResponseMessage(code) + { + // The real socket handler always sets this, and Kiota's retry loop rebuilds + // the request from it. A mock that leaves it null makes the SDK look like it + // never retries, which is the opposite of what is being tested here. + RequestMessage = request + }; + throttled.Headers.TryAddWithoutValidation("Retry-After", "0"); + return Task.FromResult(throttled); + } + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent("{\"value\":[]}") + }); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Mgx.IntegrationTests.csproj b/tests/Mgx.IntegrationTests/Mgx.IntegrationTests.csproj index 731c7d1..583b507 100644 --- a/tests/Mgx.IntegrationTests/Mgx.IntegrationTests.csproj +++ b/tests/Mgx.IntegrationTests/Mgx.IntegrationTests.csproj @@ -23,6 +23,20 @@ + + + + + + + From 9386fa5e3efa6ee677bd5985cc71bc4887c5f3af Mon Sep 17 00:00:00 2001 From: gromedev <61883549+gromedev@users.noreply.github.com> Date: Mon, 24 Aug 2026 14:36:15 +0200 Subject: [PATCH 03/10] mgx 2.1.2 --- .github/workflows/build-and-test.yml | 2 +- CHANGELOG.md | 30 +- Directory.Build.props | 2 +- README.md | 337 ++++++++---------- module/mgx.psd1 | 12 +- src/Mgx.Cmdlets/Base/MgxCmdletBase.cs | 78 +++- .../Cmdlets/Batch/InvokeMgxBatchRequest.cs | 14 +- .../Cmdlets/Content/GetMgxContent.cs | 9 +- .../Cmdlets/Expand/ExpandMgxRelation.cs | 19 + .../Cmdlets/Export/ExportMgxCollection.cs | 19 + src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs | 174 +++++++-- .../Http/ResiliencePipelineFactory.cs | 5 +- .../Http/ResilientDelegatingHandler.cs | 9 + src/Mgx.Engine/Http/ResilientGraphClient.cs | 22 +- .../Cmdlets/BatchErrorSurfacingTests.cs | 148 ++++++++ .../Cmdlets/DiagnosticsDrainTests.cs | 60 ++++ .../Cmdlets/HeaderWireTests.cs | 63 ++++ .../Cmdlets/ResponseFidelityTests.cs | 186 ++++++++++ .../Cmdlets/UriEncodingTests.cs | 226 ++++++++++++ .../Engine/Http/HeaderFidelityTests.cs | 137 +++++++ .../Engine/Http/ResilienceBridgeTests.cs | 45 +++ .../Fakes/MockHttpHandler.cs | 84 ++++- .../Serialization/WireBodyTests.cs | 112 ++++++ tests/RELEASING.md | 6 + 24 files changed, 1563 insertions(+), 236 deletions(-) create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/BatchErrorSurfacingTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/DiagnosticsDrainTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/HeaderWireTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/ResponseFidelityTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Http/HeaderFidelityTests.cs create mode 100644 tests/Mgx.IntegrationTests/Serialization/WireBodyTests.cs diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 59d0886..b5ce217 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -55,7 +55,7 @@ jobs: - name: Assert xUnit actually ran shell: pwsh run: | - $floor = 530 + $floor = 650 $trx = Get-ChildItem TestResults -Filter *.trx | Select-Object -First 1 if (-not $trx) { throw "No .trx produced - the test run did not report results." } $c = ([xml](Get-Content $trx.FullName -Raw)).TestRun.ResultSummary.Counters diff --git a/CHANGELOG.md b/CHANGELOG.md index fca3990..8a32e55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,34 @@ # Changelog -## 2.1.1 +## 2.1.2 + +### Fixed -Patch release. One fix; no feature or API changes. +- Invoke-MgxBatchRequest wrote no error records for failed items unless a dead-letter write also failed. +- A 204 or empty response body on a GET ended the cmdlet with an unhandled error. +- An HTML or otherwise non-JSON response body surfaced as an unhandled exception instead of an error record. +- A JSON response body that did not parse crashed the pipeline instead of reporting what was received. +- A '#' in a request path silently dropped everything after it. +- Pre-encoded -Filter, -Search, -Property, -Sort and -ExpandProperty values were percent-encoded a second time. +- Enums in -Body were sent as numbers, which Graph does not accept. +- A byte array in -Body became a JSON array of integers instead of base64. +- A TimeSpan in -Body was not sent as an ISO-8601 duration. +- A DateTime without a kind was sent without an offset and refused. +- A PSCustomObject -Body dropped every property that was not a NoteProperty, sending {} in the worst case. +- Non-ASCII text in -Body was \u-escaped, making -Debug traces and dead-letter files unreadable. +- An array in -Headers went on the wire as the literal text System.String[]. +- Invoke-MgxRequest, Expand-MgxRelation and Export-MgxCollection mangled an absolute -Uri instead of rejecting it. +- Get-MgxContent did not escape drive and item ids piped into the download path. +- A Content-Type or other content header in -Headers was silently dropped. +- A client-request-id in -Headers had Mgx's own value appended after it. +- If-Match and if-match in -Headers went on the wire as two headers. +- A typed parameter duplicating a query option already written into -Uri sent it twice. +- An attempt that timed out during Ctrl-C could schedule another retry. +- Requests through Enable-MgxResilience carried no client-request-id to quote at Graph support. +- An unexpected failure discarded the buffered retry history that explained it. +- A request body over 4MB passed through Enable-MgxResilience unchecked and was buffered whole. + +## 2.1.1 ### Fixed diff --git a/Directory.Build.props b/Directory.Build.props index bcd34af..c014344 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,7 +1,7 @@ - 2.1.1 + 2.1.2 Thomas Maillo Grome Mgx (c) 2026 Thomas Maillo Grome. All rights reserved. diff --git a/README.md b/README.md index a2186cb..f26975c 100644 --- a/README.md +++ b/README.md @@ -1,193 +1,208 @@ # Mgx -Microsoft Graph works well for ordinary scripts. It gets harder when a script becomes a long-running job making thousands of requests. +High-performance, resilient Microsoft Graph access for PowerShell. -At that point, the problems are usually not authentication or the Graph API itself. They are pagination, throttling, concurrency, dead connections, retries, memory usage, and jobs that need to survive for hours without manual intervention. +`Mgx` is a PowerShell client for Microsoft Graph built for long-running and high-volume workloads. It handles the parts that become difficult when a script makes thousands of requests: pagination, concurrency, throttling, retries, dead connections, memory usage, and recovery from interruption. -Mgx is a PowerShell client for Microsoft Graph that deals with those problems. It adds concurrent requests, streaming pagination, batching, rate limiting, retries, connection recovery, checkpointed exports, delta synchronization, and telemetry. It can use the authentication context established by `Connect-MgGraph`, so it does not require a separate authentication system. +It adds concurrent fan-out, streaming pagination, batching, adaptive rate limiting, retry and connection recovery, checkpointed exports, delta synchronization, content downloads, and telemetry. -In benchmarks against a 100,000-user tenant, workloads involving many individual lookups were **4–5× faster** than the Microsoft Graph SDK. Plain enumeration, where there is little to parallelize, is roughly the same speed as a tuned SDK. - -The bigger difference is what happens when Graph starts throttling or the network stops cooperating. In one test, a 100,000-user enumeration was run while the application's request budget was held at its limit. Mgx returned all 100,000 users. A raw `Invoke-RestMethod` loop without equivalent retry and pacing logic returned only 1,700.¹ - -¹ Tested with sustained throttle waves, injected 429/503 responses, dead sockets, and process termination during export. See [`tests/benchmarks/`](tests/benchmarks/). - -## Quick start +Mgx can use the authentication context established by `Connect-MgGraph`, so it does not require a separate authentication system. ```powershell -Install-Module Microsoft.Graph.Authentication # Connect-MgGraph lives here +Install-Module Microsoft.Graph.Authentication Install-Module Mgx + Connect-MgGraph -Scopes "User.Read.All" + Invoke-MgxRequest /users -All -Property displayName,mail ``` ## What Mgx adds -- Concurrent fan-out for workloads with many independent requests -- Streaming pagination instead of collecting the entire result set in memory -- Batch requests with up to 20 sub-requests per HTTP call -- Proactive rate limiting, retries with jitter, circuit breaking, and connection recycling -- Adaptive per-workload request pacing to avoid running into Graph limits unnecessarily -- JSONL exports with checkpoints and resume support -- Delta synchronization with persisted state and mid-run recovery -- Whole-file and ranged content downloads through a validated redirect path without forwarding the bearer token -- Telemetry for HTTP timing, throttling, retries, and resource consumption -- `Enable-MgxResilience` for adding Mgx's resilience handling to existing Microsoft Graph SDK scripts +- **Concurrent fan-out** for workloads with many independent requests +- **Streaming pagination** without buffering entire collections in memory +- **Batch requests** with up to 20 sub-requests per HTTP call +- **Adaptive rate limiting** that learns the usable rate for each workload +- **Retries, circuit breaking, timeouts, and connection recovery** +- **Checkpointed JSONL exports** with crash-safe resume +- **Delta synchronization** with persisted state and mid-run recovery +- **Whole-file and ranged content downloads** +- **Telemetry** for HTTP timing, throttling, retries, pacing, and resource units +- **SDK resilience integration** through `Enable-MgxResilience` -## Benchmarks +The difference matters most when Graph stops behaving like a simple request/response API and starts behaving like a distributed system with quotas, transient failures, long-running connections, and partial progress. -> **Environment:** Entra ID test tenant with 100k users and 15.8k groups. PowerShell 7.6.4 (.NET 10) with the Microsoft.Graph SDK, app-only via certificate. The module's supported floor is 7.4 (.NET 8), which is verified by the test suite but is not where these figures were measured. Every row below comes from one run of the suite against one tenant on one build; rows that could not be re-measured have been removed rather than carried forward. +## Examples -### Performance & throughput +```powershell +# Enumerate all users, streamed +Invoke-MgxRequest /users -All -Property displayName,mail,department -| Operation | Mgx | SDK (`Get-MgUser`) | Raw REST (`Invoke-RestMethod`) | Speedup vs SDK | -| --- | ---: | ---: | ---: | ---: | -| **List 100,000 users** | **47.1s** | 53.2s¹ | 58.5s | 1.1× | -| **Look up 5,000 users by ID** | **98.8s** | 521.0s | 985.3s | **5.3×** | -| **User report** *(1k users + groups)* | **23.9s** | 107.9s | 205.8s | **4.5×** | -| **Full delta enumeration** *(130,233 items)* | **145.6s** | - | - | - | +# Filter +Invoke-MgxRequest /users ` + -Filter "department eq 'Engineering'" ` + -Property displayName,mail -¹ Both figures are the SDK at `-PageSize 999`, which is what the benchmark runs. At the SDK's default page size the same enumeration takes about 2.5–3× as long. The practical difference is that Mgx needs no tuning to get close to the tuned SDK result. +# Single request +Invoke-MgxRequest "/users/$userId" -Graph charges resource units per request based on the query shape, not on which client sent it. The three columns above therefore spend roughly the same amount when they issue equivalent requests: about 102 units for the enumeration, 5,002 for the lookups, and 2,003 for the report. +# Concurrent fan-out +@("id1", "id2", "id3") | + Invoke-MgxRequest '/users/{id}' -Mgx does not make those requests cheaper. It makes the budget visible: it accumulates `x-ms-resource-unit` across the session and uses that information when pacing requests. See [Resource units: what queries actually cost](#resource-units-what-queries-actually-cost). +# Expand related objects in parallel +Invoke-MgxRequest /users -Top 50 | + Expand-MgxRelation '/users/{id}/manager' -As Manager -Flatten -Measured from `x-ms-resource-unit`. `/users/{id}` costs 1 RU; `transitiveMembers` with `$select` and `$top` costs 3, matching the documented cost table. At 5,002 RU the heaviest row above spends about 1.3% of the 8,000 RU / 10s Identity & Access budget for one application + tenant pair. The delta endpoint does not emit the header, so no figure is given. +# Batch up to 20 requests per HTTP call +@("/users/id1", "/users/id2") | + Invoke-MgxBatchRequest -Method PATCH -Body @{ department = "HR" } -### Resilience under throttling +# Stream a collection to JSONL with checkpoint/resume +Export-MgxCollection /auditLogs/signIns ` + -OutputFile ./signins.jsonl ` + -CheckpointPath ./signins.cp ` + -All -This test enumerated all 100,000 users while a second client held the application's resource-unit budget at its ceiling. Ground truth is `/users/$count`, which the directory serves from its own index rather than by walking pages. +# Stateful delta synchronization +Sync-MgxDelta /users/delta ` + -DeltaPath ./users.delta.json ` + -Property displayName,mail -| Contender | Retrieved | Missing | Duplicates | Wall time | -| --- | ---: | ---: | ---: | ---: | -| `Invoke-MgxRequest -All` | **100,000 / 100,000** | 0 | 0 | 715.2s | -| `Get-MgUser -All` | **100,000 / 100,000** | 0 | 0 | 729.9s | -| `Invoke-RestMethod` + fixed retry | 8,000 / 100,000 | 92,000 | 0 | 120.0s | -| `Invoke-RestMethod`, no retry | 1,700 / 100,000 | 98,300 | 0 | 5.5s | +# Resumable delta export +Sync-MgxDelta /me/drive/root/delta ` + -DeltaPath ./drive.json ` + -CheckpointPath ./drive.cp ` + -OutputFile ./drive.jsonl -The raw REST clients finish sooner because they stop making useful progress as soon as the budget is exhausted. Mgx and the SDK take longer because they keep going until the enumeration is complete. The SDK's retry handling is already correct here; the useful distinction is whether the client actually honors throttling instead of treating the first wave of 429s as the end of the job. +# Read only the first 256 KiB of each file +Invoke-MgxRequest "/me/drive/items/$folderId/children" -All | + Where-Object { $_.file } | + Get-MgxContent -First 262144 -Running the same enumeration without the competing load produced exactly 100,000 users from all four clients. The difference above is caused by throttling, not pagination. +# Add Mgx resilience to existing Microsoft Graph SDK code +Enable-MgxResilience +Get-MgUser -All +Disable-MgxResilience +``` -`Enable-MgxResilience` does not replace the SDK's existing retry handler. The SDK already handles throttling. Mgx adds things that are not covered by that handler, including per-request body-read timeouts and periodic connection recycling, so a dead socket can become a retryable failure instead of an indefinite hang. +See [`examples/`](examples/) for additional examples. -### Delta enumerations repeat themselves +## Microsoft.Graph comparison -A delta run does not paginate a snapshot. The same object can appear on several pages of one enumeration. Against a tenant containing 15,779 groups, over 1,135 pages produced these results: +Mgx is not intended to replace the Microsoft Graph SDK's typed cmdlets for ordinary interactive use. It targets workloads where request orchestration, throughput, and recovery become the dominant problems. -| Contender | Emitted | Distinct | -| --- | ---: | ---: | -| `Invoke-RestMethod` over `/groups/delta` | 172,740 | 17,467 | -| `Sync-MgxDelta` over the same resource | 172,740 | 17,467 | +| Common operation | Microsoft.Graph | Mgx | +| --- | --- | --- | +| Bulk lookups | `ForEach-Object { Get-MgUser -UserId $_ }` | `Invoke-MgxRequest '/users/{id}'` | +| Bulk updates | Individual SDK calls | `Invoke-MgxBatchRequest` | +| Large exports | Buffer with `Get-MgUser -All` | `Export-MgxCollection` | +| Fault protection | Script-specific | Built in | +| Adaptive pacing | Not provided | Built in | +| Dead connection recovery | Default HTTP behavior | Body-read timeouts + connection recycling | +| Telemetry | Caller-managed | `Get-MgxTelemetry` | +| Beta endpoints | `Microsoft.Graph.Beta` | `-ApiVersion beta` | + +`Enable-MgxResilience` can add Mgx's resilience handling to existing SDK scripts without replacing the SDK's own request pipeline. -Mgx is not silently deduplicating these objects. Correct deduplication requires a policy for cases where the same ID appears both live and as `@removed`, and the final occurrence is not knowable while streaming without retaining every ID seen. Until a policy is defined by the caller, dedupe downstream on `id`. +## Benchmarks -Distinct exceeds the group count because a full delta also returns tombstones for deleted groups; 16,324 of the objects above carry `@removed`. +### Performance -### What adaptive pacing costs when nothing is throttling +Benchmarks were run against an Entra ID test tenant containing 100,000 users and 15,779 groups using PowerShell 7.6.4 (.NET 10), app-only authentication, and the Microsoft Graph SDK. -Adaptive pacing is enabled by default. It starts conservatively, increases the allowed rate as requests succeed, and backs off when throttling or latency changes indicate that the workload is approaching a limit. +| Operation | Mgx | SDK (`Get-MgUser`) | Raw REST | Speedup vs SDK | +| --- | ---: | ---: | ---: | ---: | +| **List 100,000 users** | **47.1s** | 53.2s | 58.5s | 1.1× | +| **Look up 5,000 users by ID** | **98.8s** | 521.0s | 985.3s | **5.3×** | +| **User report** *(1k users + groups)* | **23.9s** | 107.9s | 205.8s | **4.5×** | +| **Full delta enumeration** *(130,233 items)* | **145.6s** | — | — | — | -That has almost no cost on a long-running workload because the initial ramp is paid once. It can be noticeable on a small workload that finishes before the ramp is complete. +Sequential enumeration has little concurrency to exploit, so Mgx is close to a tuned SDK. Workloads with many independent requests benefit much more from concurrency and batching. -| Workload | Pacing on *(default)* | `-NoAdaptivePacing` | Measured by | -| --- | ---: | ---: | --- | -| 8,000 concurrent reads | 39.4s | 40.3s | [benchmark 07](tests/benchmarks/07-adaptive-pacing.ps1) | -| 50 lookups, concurrency 8, cold session | 4.4s | 1.0s | [benchmark 14](tests/benchmarks/14-pacing-cold-cost.ps1) | +The SDK enumeration result above uses `-PageSize 999`, matching the benchmark. At its default page size, the same enumeration takes roughly 2.5–3× as long. -The large run had zero pacing activations and zero throttle retries; the paced run was slightly faster. The small run paid roughly 4–6× because the workload completed during the cold-start ramp. Telemetry exposes this as `AdaptivePacingWaitMs`. +### Resilience under throttling -For short interactive workloads against a tenant you are not pushing hard, disable it with: +A second client was used to hold the application's resource-unit budget near its limit while all 100,000 users were enumerated. -```powershell -Set-MgxOption -NoAdaptivePacing -``` +| Contender | Retrieved | Missing | Duplicates | Wall time | +| --- | ---: | ---: | ---: | ---: | +| `Invoke-MgxRequest -All` | **100,000 / 100,000** | 0 | 0 | 715.2s | +| `Get-MgUser -All` | **100,000 / 100,000** | 0 | 0 | 729.9s | +| `Invoke-RestMethod` + fixed retry | 8,000 / 100,000 | 92,000 | 0 | 120.0s | +| `Invoke-RestMethod`, no retry | 1,700 / 100,000 | 98,300 | 0 | 5.5s | -No 429s appeared at any concurrency the module permits. It caps at 128, and this tenant sustained roughly 700 RU/s for 90 seconds without refusing requests. A raw client only crossed the tenant's ceiling at around 200 requests in flight, which is beyond what Mgx will issue. Graph instead responded by increasing latency from about 113 ms to roughly 1 second. The pacing logic therefore reacts to `Retry-After` and latency drift rather than waiting for a throttle header that may never arrive. +The raw REST loops finish sooner because they stop making useful progress once the budget is exhausted. Mgx and the SDK take longer because they continue until the enumeration is complete. -### Memory efficiency & recovery +Without the competing load, all four clients returned exactly 100,000 users. The difference above is caused by throttling, not pagination. + +### Memory efficiency Exporting 100,000 users to JSONL: -| | Mgx `Export-MgxCollection` | Mgx `Invoke-MgxRequest \| file` | `Invoke-RestMethod` buffer+write | +| | Mgx `Export-MgxCollection` | Mgx request → file | REST buffer → write | | --- | ---: | ---: | ---: | | Wall time | 53.0s | 47.0s | 64.0s | | Peak working set | 305MB | **265MB** | **555MB** | | Managed heap delta | +19.8MB | **+5.0MB** | **+415.7MB** | -Streaming directly to a file added 5MB to the managed heap over the whole export, compared with 416MB for the buffer-then-write approach. Peak working set was 265MB versus 555MB. +Streaming directly to a file keeps managed heap growth close to zero relative to a buffer-then-write implementation, and `Export-MgxCollection` persists progress so an interrupted export can resume without duplicating objects. -`Export-MgxCollection` also checkpoints progress continuously. After an interruption, rerunning the export resumes from the checkpoint without duplicating objects. +### Adaptive pacing -## Examples +Adaptive pacing is enabled by default. It starts conservatively, increases the allowed rate as requests succeed, and backs off when throttling or latency indicates that a workload is approaching a limit. + +| Workload | Pacing on | `-NoAdaptivePacing` | +| --- | ---: | ---: | +| 8,000 concurrent reads | 39.4s | 40.3s | +| 50 lookups, concurrency 8, cold session | 4.4s | 1.0s | + +The cold-start cost is most visible on very small workloads. For interactive workloads where the tenant is not being pushed hard, disable it with: ```powershell -# All users, streamed -Invoke-MgxRequest /users -All -Property displayName,mail,department +Set-MgxOption -NoAdaptivePacing +``` -# Filter -Invoke-MgxRequest /users -Filter "department eq 'Engineering'" -Property displayName,mail +## Resilience model -# Single user -Invoke-MgxRequest "/users/$userId" +Mgx combines request-level resilience with workload-level pacing. -# Pipe to CSV -Invoke-MgxRequest /users -All -Property displayName,mail,department | - Select-Object displayName,mail,department | - Export-Csv users.csv +The request pipeline provides: -# JSONL export with checkpoint/resume -Export-MgxCollection /auditLogs/signIns -OutputFile ./signins.jsonl -CheckpointPath ./cp.json -All +1. **Rate limiting** — token bucket with burst capacity and a configurable refill rate +2. **Retries** — exponential backoff with jitter for transient failures, including `429`, `500`, `502`, `503`, and `504`, while honoring `Retry-After` +3. **Circuit breaking** — stops repeatedly failing workloads and allows controlled half-open recovery +4. **Timeouts** — per-request and cumulative limits across retries +5. **Connection recycling** — periodically replaces long-lived HTTP connections so dead sockets do not remain stuck indefinitely -# Beta endpoints -Invoke-MgxRequest /users -ApiVersion beta -Top 10 +Adaptive pacing sits above those request-level strategies and learns a separate usable rate for each workload. A throttled Directory workload therefore does not unnecessarily slow an unrelated workload such as Drive. -# Add resilience to existing SDK scripts -Enable-MgxResilience -Get-MgUser -All -Disable-MgxResilience -``` +`Enable-MgxResilience` does not replace the SDK's own retry handler. It adds handling for failure modes outside that handler's scope, including body-read timeouts and periodic connection recycling. -### Concurrent fan-out & batching +## Resource units and throttling -```powershell -# Concurrent fan-out lookups -@("id1", "id2", "id3") | Invoke-MgxRequest '/users/{id}' +Microsoft Graph directory workloads use resource-unit budgets. The cost depends on query shape rather than on which client issued the request. -# Resolve nested relationships in parallel -Invoke-MgxRequest /users -Top 50 | - Expand-MgxRelation '/users/{id}/manager' -As Manager -Flatten +Mgx reads `x-ms-resource-unit` and exposes the accumulated values through `Get-MgxTelemetry`. It uses throttling signals and observed latency to pace requests before the workload repeatedly hits the limit. -# Batch up to 20 requests per HTTP call -@("/users/id1", "/users/id2") | - Invoke-MgxBatchRequest -Method PATCH -Body @{ department = "HR" } +Examples measured against the test tenant: -# Delta sync with automatic token handling -Sync-MgxDelta /users/delta -DeltaPath ./delta.json -Property displayName,mail +| Query shape | Resource units | +| --- | ---: | +| `transitiveMembers?$top=5` | 4 | +| `transitiveMembers?$top=5&$select=id` | **3** | +| `groups/{id}` | 1 | -# Enumerate a whole drive, resumable mid-run; later runs return only changes -Sync-MgxDelta /me/drive/root/delta -DeltaPath ./drive.json -CheckpointPath ./drive.cp -OutputFile drive.jsonl +Resource units are not reduced by changing clients. Mgx makes them visible and uses them when deciding how aggressively to issue requests. -# Ranged reads: first 256 KB of each file instead of the whole thing -Invoke-MgxRequest "/me/drive/items/$folderId/children" -All | - Where-Object { $_.file } | - Get-MgxContent -First 262144 -``` +See [`examples/resilience-and-telemetry/resource-unit-budgeting.ps1`](examples/resilience-and-telemetry/resource-unit-budgeting.ps1) and [`tests/benchmarks/13-resource-unit-rate.ps1`](tests/benchmarks/13-resource-unit-rate.ps1) for the measurement code. -See [`examples/`](examples/) for additional examples. +## Delta synchronization -## Microsoft.Graph comparison +Graph delta enumeration can return the same object more than once during a single run, including combinations of live objects and `@removed` tombstones. Mgx does not silently deduplicate the stream because doing so requires a caller policy for conflicting occurrences and retaining every ID seen during the enumeration. -| Common operation | Standard `Microsoft.Graph` | `Mgx` | -| --- | --- | --- | -| **Bulk lookups** | `$ids \| ForEach-Object { Get-MgUser -UserId $_ }` | `$ids \| Invoke-MgxRequest '/users/{id}'` | -| **Bulk updates** | `$ids \| ForEach-Object { Update-MgUser ... }` | `$urls \| Invoke-MgxBatchRequest -Method PATCH -Body @{...}` | -| **Exporting data** | `$all = Get-MgUser -All; $all \| Export-Csv ...` | `Export-MgxCollection /users -OutputFile users.jsonl` | -| **Fault protection** | Written per script | Built in, or added to existing scripts with `Enable-MgxResilience` | -| **Observability** | Timed by the caller | `Get-MgxTelemetry` | -| **Dead connection handling** | Left to the default HTTP timeouts | Body-read timeouts + connection recycling | -| **Beta endpoints** | Requires `Microsoft.Graph.Beta` | `-ApiVersion beta` | +`Sync-MgxDelta` handles delta-link persistence and checkpointed recovery while preserving the stream returned by Graph. Apply deduplication downstream when the application needs a specific conflict policy. ## Output shape @@ -200,9 +215,9 @@ $user.displayName $user['@odata.type'] ``` -`@odata.type` is the only annotation preserved. Other `@odata.*` transport metadata is stripped, including `@odata.etag`. Use `-Raw` when the original payload or `If-Match` tag is required. +`@odata.type` is preserved. Other `@odata.*` transport metadata is stripped, including `@odata.etag`. Use `-Raw` when the original payload or `If-Match` tag is required. -Hashtable keys have no defined order. Pin columns explicitly when output order matters: +Keys have no defined order. Pin output columns explicitly when order matters: ```powershell Invoke-MgxRequest /users -All | @@ -210,79 +225,41 @@ Invoke-MgxRequest /users -All | Export-Csv users.csv ``` -To work with `PSCustomObject`s instead, request the raw payload: +Use `-Raw` when you need the original JSON object: ```powershell -Invoke-MgxRequest /users -All -Raw | ConvertFrom-Json +Invoke-MgxRequest /users -All -Raw | + ConvertFrom-Json ``` ## Cmdlets -| Cmdlet | Description | +| Cmdlet | Purpose | | --- | --- | -| `Invoke-MgxRequest` | Executes Graph requests with automatic concurrency, retries, and rate limiting | -| `Invoke-MgxBatchRequest` | Batches up to 20 requests into a single HTTP POST call | -| `Export-MgxCollection` | Streams paginated API results directly to JSONL with checkpointing | -| `Expand-MgxRelation` | Performs concurrent fan-out lookups to expand related object attributes | -| `Sync-MgxDelta` | Manages stateful delta queries and state token storage, with mid-run crash resume | -| `Get-MgxContent` | Downloads content bytes, whole or ranged, through a token-free validated download path | -| `Set-MgxOption` / `Get-MgxOption` | Configures global limits, retry counts, timeouts, and circuit-breaker thresholds | -| `Enable-MgxResilience` / `Disable-MgxResilience` | Injects or removes Mgx resilience policies from native Microsoft.Graph SDK cmdlets | -| `Get-MgxTelemetry` | Outputs execution statistics including HTTP duration, rate-limit delays, retry counts, and resource units | - -## Resilience architecture - -All HTTP operations pass through four layered [Polly 8.x](https://github.com/App-vNext/Polly) resilience strategies: - -1. **Rate limiter:** Token bucket with 200 burst capacity and 50 requests/sec refill. -2. **Retry policy:** Up to 7 retries with exponential backoff and jitter for transient errors (`429`, `500`, `502`, `503`, `504`), honoring `Retry-After`. -3. **Circuit breaker:** Opens when error rates exceed 10%, with a half-open probe after 15 seconds. -4. **Timeout:** 30-second per-request limit and 300-second cumulative ceiling across retries. - -### Resource units: what queries actually cost - -Graph throttles directory workloads on a **resource-unit budget**, not simply on request count or bandwidth. Mgx reads `x-ms-resource-unit` from responses and accumulates it, so `Get-MgxTelemetry` can show what a session actually spent. See [examples/resilience-and-telemetry/resource-unit-budgeting.ps1](examples/resilience-and-telemetry/resource-unit-budgeting.ps1). - -Measured against a 15,779-group test tenant: - -| Query shape | RU | -| --- | ---: | -| `transitiveMembers?$top=5` | 4 | -| `transitiveMembers?$top=5&$select=id` | **3** | -| `groups/{id}` (single read) | 1 | - -Both figures match the documented cost table: `transitiveMembers` is published at 5 RU, `$select` takes one off, and `$top` under 20 takes another. Cost is a property of the query shape, not the number of objects returned. On a per-group fan-out across that tenant, `$select` alone is the difference between 47,337 and 63,116 resource units. - -Three findings from pushing a single client until the tenant pushed back: - -- **The budget behaves like a token bucket, and burst hides its rate.** The published limit for a tenant of this size is 8,000 RU per 10 s per *application + tenant pair* (800 RU/s). Held at a fixed rate for 90 s from a single client, 300 and 700 RU/s were never refused. 1,200 RU/s began refusing after 19 s, and once saturated the tenant served about 730 RU/s however hard it was pushed. The exact ceiling moves: the same probe was refused at 631 RU/s on the previous day and saturated near 440. Re-measure with [`tests/benchmarks/13-resource-unit-rate.ps1`](tests/benchmarks/13-resource-unit-rate.ps1) rather than treating these measurements as constants. -- **`x-ms-throttle-limit-percentage` was never emitted.** Mgx therefore treats it as opportunistic and paces on 429 + `Retry-After` and latency drift instead. -- **Batching does not buy cheaper units.** Unbatched requests sustained a higher RU/s before throttling than the same requests inside `$batch`. Batching saves round-trips, not budget. - -Limits are scoped per **application + tenant pair**, and separately per service. That is why pacing state is partitioned by workload rather than pooled: a throttled directory fan-out says nothing about the budget remaining for Teams. - -A failed request costs 0 RU, so a fan-out that fails uniformly consumes no budget, triggers no throttling, and finishes fast while measuring nothing. Check status codes, not duration. - -Adaptive pacing runs ahead of the four resilience strategies above. It spaces requests before they are sent, learns a per-workload rate from throttling signals using additive increase / multiplicative decrease, and keeps Drive, Directory, and other workloads in separate buckets. It starts conservatively on a cold session. - -Batch outer POSTs are the one exemption. `GraphBatchClient` runs its own item-level AIMD controller; stacking two controllers on one workload would compound their backoff. - -Disable adaptive pacing with `Set-MgxOption -NoAdaptivePacing` and inspect the learned state in `Get-MgxTelemetry`. - -## Requirements & building - -### Requirements +| `Invoke-MgxRequest` | Execute Graph requests with streaming, concurrency, retries, and rate limiting | +| `Invoke-MgxBatchRequest` | Send up to 20 sub-requests in one Graph batch | +| `Export-MgxCollection` | Stream paginated results to JSONL with checkpointing | +| `Expand-MgxRelation` | Perform concurrent fan-out lookups for related objects | +| `Sync-MgxDelta` | Manage delta state and checkpointed synchronization | +| `Get-MgxContent` | Download whole files or ranged content | +| `Set-MgxOption` / `Get-MgxOption` | Configure limits, retry counts, timeouts, and resilience thresholds | +| `Enable-MgxResilience` / `Disable-MgxResilience` | Add or remove Mgx resilience from Microsoft.Graph SDK calls | +| `Get-MgxTelemetry` | Report HTTP timing, retries, throttling, pacing, and resource-unit usage | + +## Requirements - **PowerShell:** 7.4+ -- **Authentication:** `Microsoft.Graph.Authentication` 2.10.0+ for token acquisition. It is not a declared module dependency. +- **Authentication:** `Microsoft.Graph.Authentication` 2.10.0+ for token acquisition + +The authentication module is intentionally not a declared dependency because Mgx can operate with the authentication context already established by `Connect-MgGraph`. -### Build from source +## Build ```powershell ./build.ps1 ``` -Dependencies (`Microsoft.Graph.Core`, `Polly.Core`, and `System.Threading.RateLimiting`) are loaded into an isolated **Assembly Load Context** to avoid version conflicts with other PowerShell modules. +`Microsoft.Graph.Core`, `Polly.Core`, and `System.Threading.RateLimiting` are loaded into an isolated **Assembly Load Context** to avoid assembly-version conflicts with other PowerShell modules. ## License diff --git a/module/mgx.psd1 b/module/mgx.psd1 index 4c5a7dc..de63877 100644 --- a/module/mgx.psd1 +++ b/module/mgx.psd1 @@ -1,6 +1,6 @@ @{ RootModule = 'mgx.psm1' - ModuleVersion = '2.1.1' + ModuleVersion = '2.1.2' GUID = 'a3f7e8d2-5b4c-4a1e-9f6d-2c8b0e3a7d5f' Author = 'Thomas Maillo Grome' CompanyName = 'Mgx' @@ -52,6 +52,16 @@ LicenseUri = 'https://github.com/gromedev/mgx/blob/main/LICENSE' ProjectUri = 'https://github.com/gromedev/mgx' ReleaseNotes = @' +v2.1.2 + +Fixed +- URI handling: a '#' in a path dropped everything after it, pre-encoded query values were encoded twice, piped drive and item ids went unescaped, an absolute -Uri was mangled rather than refused, and a typed parameter could repeat an option already in -Uri. +- Body serialization: enums went out as numbers, byte arrays as integer lists, TimeSpan and a Kind-less DateTime in forms Graph refuses, and a PSCustomObject body lost every property that was not a NoteProperty. +- Headers: content headers from -Headers were dropped, arrays were sent as System.String[], names merged case-sensitively, and a caller's client-request-id was doubled. +- Failed batch items wrote no error records, so -ErrorAction Stop did not stop. +- An empty, non-JSON or malformed response body ended the cmdlet with an unhandled error. +- An unexpected failure discarded the retry history explaining it, and a timed-out attempt could retry after Ctrl-C. + v2.1.1 Patch release. One fix; no feature or API changes. diff --git a/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs b/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs index 1625b7b..766c30c 100644 --- a/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs +++ b/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs @@ -6,6 +6,7 @@ using System.Reflection; using System.Security.Cryptography; using System.Text; +using System.Text.RegularExpressions; using Mgx.Engine.Http; using Mgx.Engine.Models; using Polly.CircuitBreaker; @@ -978,9 +979,65 @@ protected static bool TryAdoptOrphanedTemp(string outputPath, long itemCount) protected static string NormalizePath(string path) { if (string.IsNullOrEmpty(path)) return string.Empty; + // A request URI has no use for a fragment - the server never sees one - so a raw '#' + // is always data (a filename, a filter value). Left alone, System.Uri would treat + // everything after it as a fragment and silently drop it from the wire. + path = path.Replace("#", "%23"); return path.StartsWith('/') ? path : $"/{path}"; } + /// + /// Query-option names already present in a URL's own query string. Graph rejects a URL + /// carrying the same option twice, so a typed parameter defers to what the caller + /// already wrote into -Uri. + /// + protected static HashSet ExistingQueryOptions(string url) + { + var set = new HashSet(StringComparer.OrdinalIgnoreCase); + var q = url.IndexOf('?'); + if (q < 0) return set; + foreach (var part in url[(q + 1)..].Split('&', StringSplitOptions.RemoveEmptyEntries)) + set.Add(part.Split('=', 2)[0]); + return set; + } + + private static readonly Regex s_encodedTriplet = new("%[0-9A-Fa-f]{2}", RegexOptions.Compiled); + + /// + /// Escapes a query-option value, leaving anything already percent-encoded alone - a + /// pre-encoded filter must not be encoded twice (%27 becoming %2527), and a raw + /// apostrophe or space must still be escaped. Any %XX triplet counts as already + /// encoded; a value that means a literal percent-then-hex sequence has to arrive + /// pre-encoded (%25XX) to survive as data. + /// + protected static string EscapeQueryValue(string value) + { + var sb = new StringBuilder(value.Length + 16); + int i = 0; + foreach (Match m in s_encodedTriplet.Matches(value)) + { + sb.Append(Uri.EscapeDataString(value[i..m.Index])); + sb.Append(m.Value); + i = m.Index + m.Length; + } + sb.Append(Uri.EscapeDataString(value[i..])); + return sb.ToString(); + } + + /// + /// A header value from a Hashtable. An array (Prefer takes several values) joins with + /// ", " - HTTP's list form - instead of .NET's array ToString(), which would put the + /// literal text "System.String[]" on the wire. + /// + private static string HeaderValueToString(object? value) + { + if (value == null) return string.Empty; + if (value is string s) return s; + if (value is IEnumerable seq) + return string.Join(", ", seq.Cast().Select(v => v?.ToString() ?? string.Empty)); + return value.ToString() ?? string.Empty; + } + protected static Dictionary? BuildRequestHeaders( string? consistencyLevel, System.Collections.Hashtable? extraHeaders) { @@ -989,15 +1046,17 @@ protected static string NormalizePath(string path) // Apply extraHeaders first so dedicated parameters can override if (extraHeaders != null) { - headers = new Dictionary(); + // Case-insensitive: HTTP header names are, and a case-variant duplicate + // (If-Match and if-match) must not become two wire headers. + headers = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var key in extraHeaders.Keys) - headers[key.ToString()!] = extraHeaders[key]?.ToString() ?? string.Empty; + headers[key.ToString()!] = HeaderValueToString(extraHeaders[key]); } // Dedicated -ConsistencyLevel parameter always wins over -Headers key if (!string.IsNullOrEmpty(consistencyLevel)) { - headers ??= new Dictionary(); + headers ??= new Dictionary(StringComparer.OrdinalIgnoreCase); headers["ConsistencyLevel"] = consistencyLevel; } @@ -1028,13 +1087,13 @@ protected static string BuildListUrl(string versionedBaseUrl, string relativeUri } if (!string.IsNullOrEmpty(p.Filter)) - queryParams.Add($"$filter={Uri.EscapeDataString(p.Filter)}"); + queryParams.Add($"$filter={EscapeQueryValue(p.Filter)}"); if (p.Property is { Length: > 0 }) - queryParams.Add($"$select={Uri.EscapeDataString(string.Join(",", p.Property))}"); + queryParams.Add($"$select={EscapeQueryValue(string.Join(",", p.Property))}"); if (p.Sort is { Length: > 0 }) - queryParams.Add($"$orderby={Uri.EscapeDataString(string.Join(",", p.Sort))}"); + queryParams.Add($"$orderby={EscapeQueryValue(string.Join(",", p.Sort))}"); if (!string.IsNullOrEmpty(p.Search)) { @@ -1042,20 +1101,23 @@ protected static string BuildListUrl(string versionedBaseUrl, string relativeUri var searchValue = p.Search; if (!searchValue.StartsWith('"') || !searchValue.EndsWith('"')) searchValue = $"\"{searchValue}\""; - queryParams.Add($"$search={Uri.EscapeDataString(searchValue)}"); + queryParams.Add($"$search={EscapeQueryValue(searchValue)}"); } if (p.Skip > 0) queryParams.Add($"$skip={p.Skip}"); if (p.ExpandProperty is { Length: > 0 }) - queryParams.Add($"$expand={Uri.EscapeDataString(string.Join(",", p.ExpandProperty))}"); + queryParams.Add($"$expand={EscapeQueryValue(string.Join(",", p.ExpandProperty))}"); // $count=true: required explicitly via -CountVariable, or implicitly when $search is used // (Graph advanced query capabilities require $count=true alongside $search) if (p.IncludeCount || !string.IsNullOrEmpty(p.Search)) queryParams.Add("$count=true"); + var existing = ExistingQueryOptions(baseUrl); + queryParams.RemoveAll(qp => existing.Contains(qp.Split('=', 2)[0])); + if (queryParams.Count == 0) return baseUrl; diff --git a/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs b/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs index c2913d8..2ae3300 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs @@ -291,8 +291,14 @@ protected override void EndProcessing() catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { WriteWarning($"Failed to write dead-letter file '{resolvedDeadLetterPath}': {ex.Message}"); + } + + if (failedCount > 0) + WriteVerbose($"Wrote {failedCount} failed items to dead-letter file: {resolvedDeadLetterPath}"); + } - // Emit errors for failed items (enables -ErrorAction Stop, populates $Error) + // Emit errors for failed items (enables -ErrorAction Stop, populates $Error). + // After the dead-letter write, so -ErrorAction Stop cannot cut the file short. for (int i = 0; i < results.Count; i++) { var (_, item) = results[i]; @@ -317,12 +323,6 @@ protected override void EndProcessing() } } - } - - if (failedCount > 0) - WriteVerbose($"Wrote {failedCount} failed items to dead-letter file: {resolvedDeadLetterPath}"); - } - // Structured telemetry summary WriteBatchTelemetry(telemetry); } diff --git a/src/Mgx.Cmdlets/Cmdlets/Content/GetMgxContent.cs b/src/Mgx.Cmdlets/Cmdlets/Content/GetMgxContent.cs index 7c496e1..81c93e7 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Content/GetMgxContent.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Content/GetMgxContent.cs @@ -162,7 +162,7 @@ protected override void ProcessRecord() } FetchContent(downloadUrl: null, - relativeUri: $"/drives/{driveId}/items/{id}/content", + relativeUri: $"/drives/{System.Uri.EscapeDataString(driveId)}/items/{System.Uri.EscapeDataString(id)}/content", errorTarget: InputObject); } @@ -282,6 +282,13 @@ private void FetchContent(string? downloadUrl, string? relativeUri, object? erro DrainClientMessages(); WriteError(new ErrorRecord(ex, "IOError", ErrorCategory.WriteError, OutFile)); } + catch (Exception) + { + // Unexpected exception types skip the drains above; the buffered verbose and + // warning messages are the context that explains the failure. + DrainClientMessages(); + throw; + } } private void WriteToFile(GraphContentResult result, long? maxBytes, long skipBytes) diff --git a/src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs b/src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs index 534ee15..9be9a20 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs @@ -80,6 +80,18 @@ protected override void BeginProcessing() { base.BeginProcessing(); + // Reject absolute URLs (relative paths only); concatenation onto the versioned + // base URL would otherwise silently produce /v1.0/https:/... on the wire. + if (Uri.StartsWith("https://", StringComparison.OrdinalIgnoreCase) || + Uri.StartsWith("http://", StringComparison.OrdinalIgnoreCase)) + { + ThrowTerminatingError(new ErrorRecord( + new ArgumentException( + $"-Uri must be a relative path (e.g., /groups/{{id}}/members), not an absolute URL. Got: '{Uri}'"), + "AbsoluteUriNotAllowed", ErrorCategory.InvalidArgument, null)); + return; + } + if (!Uri.Contains("{id}", StringComparison.OrdinalIgnoreCase)) { ThrowTerminatingError(new ErrorRecord( @@ -129,6 +141,13 @@ protected override void EndProcessing() { WriteWarning("Expand-MgxRelation cancelled by user."); } + catch (Exception) + { + // Unexpected exception types skip the drains above; the buffered verbose and + // warning messages are the context that explains the failure. + DrainClientMessages(); + throw; + } base.EndProcessing(); } diff --git a/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs b/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs index 61bc07a..ec6b644 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs @@ -82,6 +82,18 @@ public class ExportMgxCollection : MgxCmdletBase protected override void BeginProcessing() { + // Reject absolute URLs (relative paths only); concatenation onto the versioned + // base URL would otherwise silently produce /v1.0/https:/... on the wire. + if (Uri.StartsWith("https://", StringComparison.OrdinalIgnoreCase) || + Uri.StartsWith("http://", StringComparison.OrdinalIgnoreCase)) + { + ThrowTerminatingError(new ErrorRecord( + new ArgumentException( + $"-Uri must be a relative path (e.g., /users), not an absolute URL. Got: '{Uri}'"), + "AbsoluteUriNotAllowed", ErrorCategory.InvalidArgument, null)); + return; + } + // $search requires ConsistencyLevel: eventual. Error if missing (data loss otherwise) if (!string.IsNullOrEmpty(Search) && string.IsNullOrEmpty(ConsistencyLevel)) { @@ -510,6 +522,13 @@ protected override void ProcessRecord() ErrorCategory.PermissionDenied, OutputFile)); return; } + catch (Exception) + { + // Unexpected exception types skip the drains above; the buffered verbose + // and warning messages are the context that explains the failure. + DrainClientMessages(); + throw; + } } } diff --git a/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs b/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs index 9059a4c..eddd69f 100644 --- a/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs +++ b/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs @@ -3,6 +3,7 @@ using System.Net; using System.Text; using System.Text.Json; +using System.Text.Json.Serialization; using Mgx.Cmdlets.Base; using Mgx.Engine.Http; using Mgx.Engine.Models; @@ -139,6 +140,18 @@ public class InvokeMgxRequest : MgxCmdletBase protected override void BeginProcessing() { + // Reject absolute URLs (relative paths only); concatenation onto the versioned + // base URL would otherwise silently produce /v1.0/https:/... on the wire. + if (Uri.StartsWith("https://", StringComparison.OrdinalIgnoreCase) || + Uri.StartsWith("http://", StringComparison.OrdinalIgnoreCase)) + { + ThrowTerminatingError(new ErrorRecord( + new ArgumentException( + $"-Uri must be a relative path (e.g., /users), not an absolute URL. Got: '{Uri}'"), + "AbsoluteUriNotAllowed", ErrorCategory.InvalidArgument, null)); + return; + } + _isFanOut = Uri.Contains("{id}", StringComparison.OrdinalIgnoreCase); // $search requires ConsistencyLevel: eventual. Error if missing (data loss otherwise) @@ -190,7 +203,17 @@ protected override void ProcessRecord() } // Direct mode (no fan-out): execute immediately - ExecuteRequest(Uri, sourceId: null); + try + { + ExecuteRequest(Uri, sourceId: null); + } + catch (Exception) + { + // Unexpected exception types skip the drains inside; the buffered verbose and + // warning messages are the context that explains the failure. + DrainClientMessages(); + throw; + } } /// @@ -230,6 +253,13 @@ protected override void EndProcessing() { WriteWarning("Request cancelled by user."); } + catch (Exception) + { + // Unexpected exception types skip the drains above; the buffered verbose and + // warning messages are the context that explains the failure. + DrainClientMessages(); + throw; + } finally { base.EndProcessing(); @@ -408,6 +438,15 @@ private void ExecuteList(string relativeUri, string? sourceId) WriteWarning("Request cancelled by user."); return; } + catch (JsonException ex) + { + // A page body that does not parse. Items from earlier pages have already been + // emitted; report the failure instead of crashing the pipeline. + WriteError(new ErrorRecord( + new InvalidOperationException($"A response page declared JSON but does not parse: {ex.Message}", ex), + "MalformedJsonResponse", ErrorCategory.InvalidData, relativeUri)); + return; + } catch (GraphServiceException ex) when (includeAutoCount && countAutoAdded && ex.StatusCode == HttpStatusCode.BadRequest) { // Auto-added $count=true rejected by this endpoint; retry without it @@ -456,11 +495,10 @@ private void ExecuteGet(string relativeUri, string? sourceId) throw new GraphServiceException(response.StatusCode, body); } - using var stream = response.Content.ReadAsStreamAsync(CancellationToken).GetAwaiter().GetResult(); - var json = JsonSerializer.DeserializeAsync(stream, cancellationToken: CancellationToken) - .AsTask().GetAwaiter().GetResult(); - - OutputPayload(json, sourceId); + var bodyBytes = response.Content.ReadAsByteArrayAsync(CancellationToken).GetAwaiter().GetResult(); + var json = ReadJsonPayload(response, bodyBytes, relativeUri); + if (json != null) + OutputPayload(json.Value, sourceId); } catch (OperationCanceledException) when (CancellationToken.IsCancellationRequested) { @@ -511,11 +549,9 @@ private void ExecuteWrite(HttpMethod method, string relativeUri, string? sourceI // stream.Length throws NotSupportedException on network/decompression streams. // Read as bytes to safely handle null ContentLength (chunked transfer) and empty bodies. var bodyBytes = response.Content.ReadAsByteArrayAsync(CancellationToken).GetAwaiter().GetResult(); - if (bodyBytes.Length > 0) - { - var jsonEl = JsonSerializer.Deserialize(bodyBytes); - OutputPayload(jsonEl, sourceId); - } + var jsonEl = ReadJsonPayload(response, bodyBytes, relativeUri); + if (jsonEl != null) + OutputPayload(jsonEl.Value, sourceId); } finally { @@ -907,10 +943,13 @@ private string BuildGetUrl(string relativeUri) var queryParams = new List(); if (Property is { Length: > 0 }) - queryParams.Add($"$select={System.Uri.EscapeDataString(string.Join(",", Property))}"); + queryParams.Add($"$select={EscapeQueryValue(string.Join(",", Property))}"); if (ExpandProperty is { Length: > 0 }) - queryParams.Add($"$expand={System.Uri.EscapeDataString(string.Join(",", ExpandProperty))}"); + queryParams.Add($"$expand={EscapeQueryValue(string.Join(",", ExpandProperty))}"); + + var existing = ExistingQueryOptions(baseUrl); + queryParams.RemoveAll(qp => existing.Contains(qp.Split('=', 2)[0])); if (queryParams.Count == 0) return baseUrl; @@ -932,6 +971,49 @@ private string BuildGetUrl(string relativeUri) /// action endpoints such as /directoryObjects/getByIds) is unwrapped into one item per element; /// anything else is emitted whole. /// + /// + /// A response body as JSON for output, or null when nothing further should be emitted: + /// an empty body (204, or 200 with no content), a non-JSON body (emitted as text under + /// -Raw, otherwise an error), or a body that declares JSON and does not parse. + /// + private JsonElement? ReadJsonPayload(HttpResponseMessage response, byte[] bodyBytes, string relativeUri) + { + if (bodyBytes.Length == 0) + { + WriteVerbose($"HTTP {(int)response.StatusCode}: response has no content; nothing to emit."); + return null; + } + + var contentType = response.Content.Headers.ContentType?.MediaType; + if (contentType != null && !contentType.EndsWith("json", StringComparison.OrdinalIgnoreCase)) + { + if (Raw.IsPresent) + { + WriteObject(Encoding.UTF8.GetString(bodyBytes)); + return null; + } + WriteError(new ErrorRecord( + new InvalidOperationException( + $"The response is {contentType}, not JSON. Use -Raw to receive it as text, or Get-MgxContent for file and media content."), + "NonJsonResponse", ErrorCategory.InvalidData, relativeUri)); + return null; + } + + try + { + return JsonSerializer.Deserialize(bodyBytes); + } + catch (JsonException) + { + var snippet = Encoding.UTF8.GetString(bodyBytes, 0, Math.Min(bodyBytes.Length, 200)); + WriteError(new ErrorRecord( + new InvalidOperationException( + $"The response declared JSON but does not parse. Body starts: {snippet}"), + "MalformedJsonResponse", ErrorCategory.InvalidData, relativeUri)); + return null; + } + } + private void OutputPayload(JsonElement json, string? sourceId) { var items = TryUnwrapCollection(json, out var truncated); @@ -1004,6 +1086,50 @@ private void OutputItem(JsonElement element, string? sourceId) return method != HttpMethod.Delete ? "{}" : null; } + /// + /// Options for -Body serialization, chosen for what Graph accepts rather than STJ's + /// defaults: enums as camelCase names (Graph never takes them numerically), TimeSpan as + /// an Edm.Duration string, a Kind-less DateTime pinned to UTC (Graph rejects a bare + /// timestamp), and readable non-ASCII instead of \uXXXX - both forms decode the same, + /// but dead-letter files and -Debug traces are read by people. + /// + internal static readonly JsonSerializerOptions BodyJsonOptions = new() + { + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + Converters = + { + new JsonStringEnumConverter(JsonNamingPolicy.CamelCase), + new GraphDurationConverter(), + new GraphDateTimeConverter(), + }, + }; + + /// Edm.Duration is ISO-8601 ("PT1H"); STJ's default "01:00:00" is refused. + private sealed class GraphDurationConverter : JsonConverter + { + public override TimeSpan Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + => System.Xml.XmlConvert.ToTimeSpan(reader.GetString()!); + + public override void Write(Utf8JsonWriter writer, TimeSpan value, JsonSerializerOptions options) + => writer.WriteStringValue(System.Xml.XmlConvert.ToString(value)); + } + + /// + /// A DateTime with Kind=Unspecified would serialize with no offset, which Graph rejects. + /// Assume UTC - the read side already resolves timestamps to UtcDateTime, so a value that + /// round-trips through mgx keeps its meaning. + /// + private sealed class GraphDateTimeConverter : JsonConverter + { + public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + => reader.GetDateTime(); + + public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options) + => writer.WriteStringValue(value.Kind == DateTimeKind.Unspecified + ? DateTime.SpecifyKind(value, DateTimeKind.Utc) + : value); + } + /// /// Serialize a -Body argument to the JSON that goes on the wire. A raw JSON string is passed /// through verbatim, everything else is serialized. @@ -1013,20 +1139,26 @@ internal static string SerializeBody(object body) var value = UnwrapPSObject(body); if (value is string s) return s; // Still a PSObject after unwrapping: a PSCustomObject, whose members live on the wrapper - if (value is PSObject pso) return JsonSerializer.Serialize(PSOToDict(pso)); - if (value is IDictionary dict) return JsonSerializer.Serialize(DictionaryToDict(dict)); + if (value is PSObject pso) return JsonSerializer.Serialize(PSOToDict(pso), BodyJsonOptions); + if (value is IDictionary dict) return JsonSerializer.Serialize(DictionaryToDict(dict), BodyJsonOptions); // Handle array body (object[], ArrayList, List, ... from PowerShell) - if (value is IEnumerable seq) return JsonSerializer.Serialize(EnumerableToArray(seq)); - return JsonSerializer.Serialize(value); + if (value is IEnumerable seq and not byte[]) return JsonSerializer.Serialize(EnumerableToArray(seq), BodyJsonOptions); + return JsonSerializer.Serialize(value, BodyJsonOptions); } internal static Dictionary PSOToDict(PSObject pso) { + // Every property kind, matching what ConvertTo-Json reads - so -Body $obj and + // -Body ($obj | ConvertTo-Json) produce the same members. ScriptProperties are + // evaluated; one whose getter throws serializes as null rather than failing the + // whole body. var dict = new Dictionary(); foreach (var prop in pso.Properties) { - if (prop.MemberType == PSMemberTypes.NoteProperty) - dict[prop.Name] = UnwrapValue(prop.Value); + object? raw; + try { raw = prop.Value; } + catch (GetValueException) { raw = null; } + dict[prop.Name] = UnwrapValue(raw); } return dict; } @@ -1057,7 +1189,9 @@ internal static string SerializeBody(object body) } if (value is IDictionary dict) return DictionaryToDict(dict); - if (value is IEnumerable seq and not string) + // byte[] is Edm.Binary and serializes as base64; flattening it into object?[] + // would emit a JSON array of integers instead. + if (value is IEnumerable seq and not (string or byte[])) return EnumerableToArray(seq); return value; } diff --git a/src/Mgx.Engine/Http/ResiliencePipelineFactory.cs b/src/Mgx.Engine/Http/ResiliencePipelineFactory.cs index 8b56f67..c2a269c 100644 --- a/src/Mgx.Engine/Http/ResiliencePipelineFactory.cs +++ b/src/Mgx.Engine/Http/ResiliencePipelineFactory.cs @@ -171,7 +171,10 @@ or HttpStatusCode.GatewayTimeout // AttemptTimeoutSeconds. Without this, requests fail permanently when // Graph returns Retry-After > AttemptTimeoutSeconds. // The outer TotalTimeout still bounds the overall operation. - if (args.Outcome.Exception is TimeoutRejectedException) + // Never after cancellation: an attempt timeout that fires while the caller + // is cancelling must not schedule another attempt. + if (args.Outcome.Exception is TimeoutRejectedException && + !args.Context.CancellationToken.IsCancellationRequested) return ValueTask.FromResult(isIdempotent); return ValueTask.FromResult(false); diff --git a/src/Mgx.Engine/Http/ResilientDelegatingHandler.cs b/src/Mgx.Engine/Http/ResilientDelegatingHandler.cs index 74422b5..6865b08 100644 --- a/src/Mgx.Engine/Http/ResilientDelegatingHandler.cs +++ b/src/Mgx.Engine/Http/ResilientDelegatingHandler.cs @@ -91,8 +91,13 @@ protected override async Task SendAsync( if (request.Content != null) { contentBytes = await request.Content.ReadAsByteArrayAsync(cancellationToken); + if (contentBytes.Length > ResilientGraphClient.MaxRequestBodyBytes) + throw new InvalidOperationException( + $"Request body size ({contentBytes.Length:N0} bytes) exceeds the {ResilientGraphClient.MaxRequestBodyBytes / (1024 * 1024)}MB limit. " + + "Graph API rejects bodies larger than 4MB on most endpoints."); contentHeaders = request.Content.Headers.ToList(); } + var clientRequestId = Guid.NewGuid().ToString(); RateLimitLease? lease = null; var context = ResilienceContextPool.Shared.Get(cancellationToken); @@ -144,6 +149,10 @@ protected override async Task SendAsync( foreach (var header in request.Headers) clone.Headers.TryAddWithoutValidation(header.Key, header.Value); clone.Headers.TryAddWithoutValidation("SdkVersion", MgxSdkVersion.Value); + // Parity with the owned client: one correlation id per logical request, + // shared across attempts - unless the SDK already stamped its own. + if (!clone.Headers.Contains("client-request-id")) + clone.Headers.TryAddWithoutValidation("client-request-id", clientRequestId); // Copy request options (used by SDK handlers for per-request metadata) #pragma warning disable CS8714 // nullability mismatch in IDictionary generic diff --git a/src/Mgx.Engine/Http/ResilientGraphClient.cs b/src/Mgx.Engine/Http/ResilientGraphClient.cs index d735e35..cd7563f 100644 --- a/src/Mgx.Engine/Http/ResilientGraphClient.cs +++ b/src/Mgx.Engine/Http/ResilientGraphClient.cs @@ -218,10 +218,26 @@ public async Task SendAsync( if (headers != null) { foreach (var (key, value) in headers) - request.Headers.TryAddWithoutValidation(key, value); + { + if (request.Headers.TryAddWithoutValidation(key, value)) continue; + // The request collection refuses content headers (Content-Type, + // Content-Disposition, ...); they belong on the content, replacing + // any default the buffered content carried. + if (request.Content != null) + { + request.Content.Headers.Remove(key); + if (request.Content.Headers.TryAddWithoutValidation(key, value)) continue; + } + if (attempt == 1) + _pendingWarnings.Enqueue($"Header '{key}' is not valid on this request and was not sent."); + } } - request.Headers.TryAddWithoutValidation("SdkVersion", MgxSdkVersion.Value); - request.Headers.TryAddWithoutValidation("client-request-id", clientRequestId); + // A caller-supplied value wins: appending a second one would put two + // values on the wire and defeat the correlation id the caller is logging. + if (headers == null || !headers.ContainsKey("SdkVersion")) + request.Headers.TryAddWithoutValidation("SdkVersion", MgxSdkVersion.Value); + if (headers == null || !headers.ContainsKey("client-request-id")) + request.Headers.TryAddWithoutValidation("client-request-id", clientRequestId); if (DebugEnabled) _pendingDebug.Enqueue(GraphRequestTracer.FormatRequest(request, contentBytes, attempt)); diff --git a/tests/Mgx.IntegrationTests/Cmdlets/BatchErrorSurfacingTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/BatchErrorSurfacingTests.cs new file mode 100644 index 0000000..5e330cf --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/BatchErrorSurfacingTests.cs @@ -0,0 +1,148 @@ +using System.Net; +using System.Reflection; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// A failed batch item is an error the pipeline can see: it lands in $Error, counts for +/// -ErrorVariable, and stops the pipeline under -ErrorAction Stop - with or without a +/// dead-letter file. (Corpus: M365DSC-7198, batch semantics.) +/// +[Collection("Pipeline")] +public class BatchErrorSurfacingTests +{ + private static readonly Type Base = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); + private const BindingFlags Static = BindingFlags.NonPublic | BindingFlags.Static; + + private static void InjectTransport(HttpMessageHandler wire) + { + Base.GetField("s_graphHttpClient", Static)!.SetValue(null, new HttpClient(wire)); + Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, + Mgx.Cmdlets.Base.MgxCmdletBase.BuildAuthFingerprint( + new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); + Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); + Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); + Base.GetField("s_clientOptions", Static)!.SetValue(null, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + ResiliencePipelineFactory.Reset(); + } + + private static void ResetTransport() + { + Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); + Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); + ResiliencePipelineFactory.Reset(); + } + + private static System.Management.Automation.PowerShell CreateShell() + { + var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + return ps; + } + + private const string OneOkOneNotFound = """ + { "responses": [ + { "id": "1", "status": 200, "body": { "id": "u1" } }, + { "id": "2", "status": 404, "body": { "error": { "code": "Request_ResourceNotFound", "message": "u2 does not exist" } } } + ] } + """; + + [Fact] + public void A_failed_item_writes_an_error_record_without_a_dead_letter_path() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, OneOkOneNotFound); + InjectTransport(wire); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new[] { "/users/u1", "/users/u2" }) + .AddParameter("Method", "GET"); + var output = ps.Invoke(); + + Assert.Equal(2, output.Count); + var error = Assert.Single(ps.Streams.Error); + Assert.StartsWith("BatchItemError", error.FullyQualifiedErrorId); + Assert.Contains("u2", error.Exception.Message); + Assert.Equal(System.Management.Automation.ErrorCategory.ObjectNotFound, + error.CategoryInfo.Category); + } + finally { ResetTransport(); } + } + + [Fact] + public void ErrorAction_Stop_stops_on_a_failed_item() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, OneOkOneNotFound); + InjectTransport(wire); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new[] { "/users/u1", "/users/u2" }) + .AddParameter("Method", "GET") + .AddParameter("ErrorAction", "Stop"); + var ex = Assert.ThrowsAny(() => ps.Invoke()); + Assert.Contains("u2", ex.Message); + } + finally { ResetTransport(); } + } + + /// First POST answers its chunk in full; every later POST is refused outright. + private sealed class FirstChunkOnlyHandler : HttpMessageHandler + { + private int _posts; + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + _posts++; + if (_posts == 1) + { + var body = "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, 20).Select(i => + $"{{ \"id\": \"{i}\", \"status\": 200, \"body\": {{ \"id\": \"u{i}\" }} }}")) + "] }"; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") + }); + } + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent( + "{\"error\":{\"code\":\"BadRequest\",\"message\":\"batch rejected\"}}", + System.Text.Encoding.UTF8, "application/json") + }); + } + } + + [Fact] + public void Items_a_failed_chunk_never_sent_each_write_an_error_record() + { + InjectTransport(new FirstChunkOnlyHandler()); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", Enumerable.Range(1, 25).Select(i => $"/users/u{i}").ToArray()) + .AddParameter("Method", "GET"); + ps.Invoke(); + + var errors = ps.Streams.Error.ToList(); + Assert.Equal(5, errors.Count(e => e.FullyQualifiedErrorId.StartsWith("BatchItemNotSent"))); + Assert.Equal(1, errors.Count(e => e.FullyQualifiedErrorId.StartsWith("BatchChunkFailed"))); + } + finally { ResetTransport(); } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DiagnosticsDrainTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DiagnosticsDrainTests.cs new file mode 100644 index 0000000..f9a28ec --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/DiagnosticsDrainTests.cs @@ -0,0 +1,60 @@ +using System.Net; +using System.Reflection; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// When a request dies on an exception type no catch filter expects, the buffered verbose +/// messages - the retry history that explains what led up to the failure - still reach the +/// caller instead of being discarded with the client. +/// +[Collection("Pipeline")] +public class DiagnosticsDrainTests +{ + private static readonly Type Base = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); + private const BindingFlags Static = BindingFlags.NonPublic | BindingFlags.Static; + + [Fact] + public void An_unexpected_exception_still_delivers_the_buffered_retry_history() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.ServiceUnavailable); + wire.QueueException(new InvalidOperationException("boom")); + + Base.GetField("s_graphHttpClient", Static)!.SetValue(null, new HttpClient(wire)); + Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, + Mgx.Cmdlets.Base.MgxCmdletBase.BuildAuthFingerprint( + new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); + Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); + Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); + Base.GetField("s_clientOptions", Static)!.SetValue(null, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 3, MaxRetryAfterSeconds = 1 }); + ResiliencePipelineFactory.Reset(); + try + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users/u1") + .AddParameter("Verbose", true); + Assert.ThrowsAny(() => ps.Invoke()); + + Assert.Contains(ps.Streams.Verbose, + v => v.Message.Contains("Retry attempt", StringComparison.OrdinalIgnoreCase)); + } + finally + { + Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); + Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); + ResiliencePipelineFactory.Reset(); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/HeaderWireTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/HeaderWireTests.cs new file mode 100644 index 0000000..c143037 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/HeaderWireTests.cs @@ -0,0 +1,63 @@ +using System.Net; +using System.Reflection; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// Header values from a -Headers hashtable reach the wire as HTTP header text. +/// +[Collection("Pipeline")] +public class HeaderWireTests +{ + private static readonly Type Base = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); + private const BindingFlags Static = BindingFlags.NonPublic | BindingFlags.Static; + + private static void InjectTransport(HttpMessageHandler wire) + { + Base.GetField("s_graphHttpClient", Static)!.SetValue(null, new HttpClient(wire)); + Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, + Mgx.Cmdlets.Base.MgxCmdletBase.BuildAuthFingerprint( + new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); + Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); + Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); + Base.GetField("s_clientOptions", Static)!.SetValue(null, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + ResiliencePipelineFactory.Reset(); + } + + private static void ResetTransport() + { + Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); + Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public void An_array_header_value_joins_as_an_http_list_not_a_type_name() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + InjectTransport(wire); + try + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("Invoke-MgxRequest -Uri /users/u1 -Headers @{ Prefer = @('outlook.timezone=\"UTC\"', 'IdType=\"ImmutableId\"') }"); + ps.Invoke(); + + var captured = wire.CapturedRequests; + Assert.NotEmpty(captured); + var prefer = Assert.Contains("Prefer", captured[0].Headers); + Assert.Equal("outlook.timezone=\"UTC\", IdType=\"ImmutableId\"", string.Join(", ", prefer)); + Assert.DoesNotContain("System.", string.Join(",", prefer)); + } + finally { ResetTransport(); } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ResponseFidelityTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ResponseFidelityTests.cs new file mode 100644 index 0000000..156edcc --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/ResponseFidelityTests.cs @@ -0,0 +1,186 @@ +using System.Net; +using System.Reflection; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// Responses that are not the JSON entity the caller expected - empty bodies, HTML error +/// pages from a proxy, truncated JSON - surface as verbose output or error records, never +/// as an unhandled exception. (Corpus: GraphSDK-1425/2088, non-JSON responses.) +/// +[Collection("Pipeline")] +public class ResponseFidelityTests +{ + private static readonly Type Base = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); + private const BindingFlags Static = BindingFlags.NonPublic | BindingFlags.Static; + + private static void InjectTransport(HttpMessageHandler wire) + { + Base.GetField("s_graphHttpClient", Static)!.SetValue(null, new HttpClient(wire)); + Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, + Mgx.Cmdlets.Base.MgxCmdletBase.BuildAuthFingerprint( + new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); + Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); + Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); + Base.GetField("s_clientOptions", Static)!.SetValue(null, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + ResiliencePipelineFactory.Reset(); + } + + private static void ResetTransport() + { + Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); + Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); + ResiliencePipelineFactory.Reset(); + } + + private static System.Management.Automation.PowerShell CreateShell() + { + var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + return ps; + } + + [Fact] + public void A_204_on_a_get_emits_nothing_and_no_error() + { + var wire = new MockHttpHandler(); + wire.QueueEmpty(HttpStatusCode.NoContent); + InjectTransport(wire); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1"); + var output = ps.Invoke(); + + Assert.Empty(output); + Assert.Empty(ps.Streams.Error); + } + finally { ResetTransport(); } + } + + [Fact] + public void A_200_with_a_zero_length_body_emits_nothing_and_no_error() + { + var wire = new MockHttpHandler(); + wire.QueueEmpty(HttpStatusCode.OK); + InjectTransport(wire); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1"); + var output = ps.Invoke(); + + Assert.Empty(output); + Assert.Empty(ps.Streams.Error); + } + finally { ResetTransport(); } + } + + [Fact] + public void An_html_body_is_an_error_record_naming_the_content_type() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "proxy says no", contentType: "text/html"); + InjectTransport(wire); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1"); + var output = ps.Invoke(); + + Assert.Empty(output); + var error = Assert.Single(ps.Streams.Error); + Assert.StartsWith("NonJsonResponse", error.FullyQualifiedErrorId); + Assert.Contains("text/html", error.Exception.Message); + } + finally { ResetTransport(); } + } + + [Fact] + public void Raw_passes_a_non_json_body_through_as_text() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "plain text payload", contentType: "text/plain"); + InjectTransport(wire); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1").AddParameter("Raw", true); + var output = ps.Invoke(); + + var item = Assert.Single(output); + Assert.Equal("plain text payload", item.BaseObject); + Assert.Empty(ps.Streams.Error); + } + finally { ResetTransport(); } + } + + [Fact] + public void Malformed_json_is_an_error_record_with_a_body_snippet() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "{\"id\": \"u1\", \"displayNa"); + InjectTransport(wire); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1"); + var output = ps.Invoke(); + + Assert.Empty(output); + var error = Assert.Single(ps.Streams.Error); + Assert.StartsWith("MalformedJsonResponse", error.FullyQualifiedErrorId); + Assert.Contains("displayNa", error.Exception.Message); + } + finally { ResetTransport(); } + } + + [Fact] + public void A_malformed_page_during_enumeration_is_an_error_record_not_a_crash() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "not a collection envelope at all"); + InjectTransport(wire); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users").AddParameter("All", true); + ps.Invoke(); + + var error = Assert.Single(ps.Streams.Error); + Assert.StartsWith("MalformedJsonResponse", error.FullyQualifiedErrorId); + } + finally { ResetTransport(); } + } + + [Fact] + public void A_write_answered_with_html_is_an_error_record() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "gateway", contentType: "text/html"); + InjectTransport(wire); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("Method", "POST") + .AddParameter("Body", new System.Collections.Hashtable { ["displayName"] = "x" }) + .AddParameter("Confirm", false); + var output = ps.Invoke(); + + Assert.Empty(output); + var error = Assert.Single(ps.Streams.Error); + Assert.StartsWith("NonJsonResponse", error.FullyQualifiedErrorId); + } + finally { ResetTransport(); } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs new file mode 100644 index 0000000..1a07372 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs @@ -0,0 +1,226 @@ +using System.Net; +using System.Reflection; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// What the caller wrote is what goes on the wire: a '#' in a path stays in the path +/// instead of vanishing as a URI fragment, hostile filter values are escaped once, and +/// pre-encoded input is not encoded twice. (Corpus: GraphSDK-1947, GraphSDK-2709/2942, +/// M365DSC-5354.) +/// +[Collection("Pipeline")] +public class UriEncodingTests +{ + private static readonly Type Base = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); + private const BindingFlags Static = BindingFlags.NonPublic | BindingFlags.Static; + + private static void InjectTransport(HttpMessageHandler wire) + { + Base.GetField("s_graphHttpClient", Static)!.SetValue(null, new HttpClient(wire)); + Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, + Mgx.Cmdlets.Base.MgxCmdletBase.BuildAuthFingerprint( + new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); + Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); + Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); + Base.GetField("s_clientOptions", Static)!.SetValue(null, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + ResiliencePipelineFactory.Reset(); + } + + private static void ResetTransport() + { + Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); + Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); + ResiliencePipelineFactory.Reset(); + } + + private static string WireUriFor(MockHttpHandler wire, Action configure) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + configure(ps); + ps.Invoke(); + var captured = wire.CapturedRequests; + Assert.NotEmpty(captured); + return captured[0].Uri; + } + + [Fact] + public void A_hash_in_a_path_reaches_the_wire_instead_of_becoming_a_fragment() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + InjectTransport(wire); + try + { + var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/me/drive/root:/reports/a#b.txt")); + + Assert.Contains("/reports/a%23b.txt", uri); + Assert.DoesNotContain("#", uri); + } + finally { ResetTransport(); } + } + + [Fact] + public void A_raw_filter_is_escaped_once() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); + InjectTransport(wire); + try + { + var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("Filter", "displayName eq 'O''Brien & Söhne'")); + + Assert.Contains("$filter=displayName%20eq%20%27O%27%27Brien%20%26%20S%C3%B6hne%27", uri); + } + finally { ResetTransport(); } + } + + [Fact] + public void A_pre_encoded_filter_is_not_encoded_twice() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); + InjectTransport(wire); + try + { + var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("Filter", "displayName%20eq%20%27Bob%27")); + + Assert.Contains("$filter=displayName%20eq%20%27Bob%27", uri); + Assert.DoesNotContain("%25", uri); + } + finally { ResetTransport(); } + } + + [Fact] + public void A_literal_percent_that_is_not_a_triplet_is_escaped() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); + InjectTransport(wire); + try + { + var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("Filter", "displayName eq '50% off'")); + + Assert.Contains("50%25%20off", uri); + } + finally { ResetTransport(); } + } + + [Fact] + public void A_piped_drive_item_id_is_escaped_into_the_content_path() + { + // Drive item ids can carry '!' and ':'; SharePoint site ids carry ','. The path is + // built from pipeline data, so the injection point escapes rather than trusts. + var wire = new MockHttpHandler(); + wire.QueueBytes(HttpStatusCode.OK, [1, 2, 3], "application/octet-stream"); + InjectTransport(wire); + // Get-MgxContent refuses borrowed transports (the SDK's RedirectHandler would + // auto-follow the content 302); this injected client is mgx-owned. An owned client + // is rebuilt when the cached timeout disagrees with options, so keep them equal. + Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, true); + Base.GetField("s_cachedTotalTimeoutSeconds", Static)!.SetValue(null, + new ResilientGraphClientOptions().TotalTimeoutSeconds); + try + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript(""" + [Hashtable]@{ + id = '01ABC!DEF:XYZ' + parentReference = @{ driveId = 'b!x_y' } + } | Get-MgxContent + """); + ps.Invoke(); + + var errText = string.Join(" | ", ps.Streams.Error.Select(e => e.FullyQualifiedErrorId + ": " + e.Exception.Message)); + var captured = wire.CapturedRequests; + Assert.True(captured.Count > 0, $"no request sent; errors: {errText}"); + Assert.Contains("/drives/b%21x_y/items/01ABC%21DEF%3AXYZ/content", captured[0].Uri); + } + finally { ResetTransport(); } + } + + [Fact] + public void A_top_already_in_the_uri_is_not_sent_twice() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); + InjectTransport(wire); + try + { + var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users?$top=5") + .AddParameter("All", true)); + + Assert.Equal(1, System.Text.RegularExpressions.Regex.Matches(uri, "top=").Count); + Assert.Contains("$top=5", uri); + } + finally { ResetTransport(); } + } + + [Fact] + public void A_filter_already_in_the_uri_wins_over_the_parameter() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); + InjectTransport(wire); + try + { + var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users?$filter=accountEnabled eq true") + .AddParameter("Filter", "displayName eq 'x'")); + + Assert.Equal(1, System.Text.RegularExpressions.Regex.Matches(uri, "filter=").Count); + Assert.Contains("accountEnabled", uri); + Assert.DoesNotContain("displayName", uri); + } + finally { ResetTransport(); } + } + + /// + /// An absolute -Uri concatenated onto the versioned base URL would silently produce + /// /v1.0/https:/... on the wire. Get-MgxContent and Sync-MgxDelta already refuse it; + /// the other three URI-taking cmdlets refuse it the same way. + /// + [Theory] + [InlineData("Invoke-MgxRequest -Uri 'https://graph.microsoft.com/v1.0/users'")] + [InlineData("[PSCustomObject]@{ id = 'g1' } | Expand-MgxRelation -Uri 'https://graph.microsoft.com/v1.0/groups/{id}/members' -As members")] + [InlineData("Export-MgxCollection -Uri 'https://graph.microsoft.com/v1.0/users' -OutputFile ([IO.Path]::GetTempFileName())")] + public void An_absolute_uri_is_refused_not_mangled(string command) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript(command); + var output = ps.Invoke(); + + Assert.Empty(output); + var error = Assert.Single(ps.Streams.Error); + Assert.StartsWith("AbsoluteUriNotAllowed", error.FullyQualifiedErrorId); + Assert.Contains("must be a relative path", error.Exception.Message); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Http/HeaderFidelityTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/HeaderFidelityTests.cs new file mode 100644 index 0000000..99fdcf5 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/HeaderFidelityTests.cs @@ -0,0 +1,137 @@ +using System.Net; +using System.Text; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// Caller headers reach the wire as given: content headers land on the content collection +/// instead of being silently dropped, a caller's correlation id is not doubled by mgx's +/// own, and the merge is case-insensitive like HTTP header names are. +/// (Corpus: GraphSDK-2328, missing required headers.) +/// +[Collection("Pipeline")] +public class HeaderFidelityTests +{ + private static readonly ResilientGraphClientOptions Options = new() + { + NoRateLimit = true, + MaxRetryAttempts = 3, + TotalTimeoutSeconds = 30, + AttemptTimeoutSeconds = 10 + }; + + private static (MockHttpHandler wire, ResilientGraphClient client) NewClient() + { + ResiliencePipelineFactory.Reset(); + var wire = new MockHttpHandler(); + var client = new ResilientGraphClient(new HttpClient(wire), Options); + return (wire, client); + } + + [Fact] + public async Task A_caller_content_type_replaces_the_default_instead_of_vanishing() + { + var (wire, client) = NewClient(); + using (client) + { + wire.QueueResponse(HttpStatusCode.OK, "{}"); + using var response = await client.SendAsync(HttpMethod.Put, + "https://graph.microsoft.com/v1.0/drives/d/items/i/content", + new StringContent("hello", Encoding.UTF8, "application/json"), + new Dictionary { ["Content-Type"] = "text/plain" }, + CancellationToken.None); + + var sent = Assert.Single(wire.CapturedRequests); + var contentType = Assert.Contains("Content-Type", sent.ContentHeaders); + Assert.Equal("text/plain", Assert.Single(contentType)); + } + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task A_content_header_survives_every_retry_attempt() + { + var (wire, client) = NewClient(); + using (client) + { + wire.QueueFailuresThenSuccess(1, HttpStatusCode.ServiceUnavailable, "{}"); + using var response = await client.SendAsync(HttpMethod.Put, + "https://graph.microsoft.com/v1.0/drives/d/items/i/content", + new StringContent("hello", Encoding.UTF8, "application/json"), + new Dictionary { ["Content-Type"] = "text/plain" }, + CancellationToken.None); + + Assert.Equal(2, wire.CapturedRequests.Count); + foreach (var sent in wire.CapturedRequests) + { + var contentType = Assert.Contains("Content-Type", sent.ContentHeaders); + Assert.Equal("text/plain", Assert.Single(contentType)); + } + } + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task A_header_no_collection_accepts_warns_instead_of_vanishing_silently() + { + var (wire, client) = NewClient(); + using (client) + { + wire.QueueResponse(HttpStatusCode.OK, TestData.SingleUser); + // A GET has no content, so a content header has nowhere to go. + using var response = await client.GetAsync( + "https://graph.microsoft.com/v1.0/users/u1", + CancellationToken.None, + new Dictionary { ["Content-Type"] = "text/plain" }); + + var warnings = new List(); + client.WarningWriter = warnings.Add; + client.DrainWarningMessages(); + Assert.Contains(warnings, w => w.Contains("Content-Type")); + } + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task A_caller_correlation_id_is_not_doubled() + { + var (wire, client) = NewClient(); + using (client) + { + wire.QueueResponse(HttpStatusCode.OK, TestData.SingleUser); + using var response = await client.GetAsync( + "https://graph.microsoft.com/v1.0/users/u1", + CancellationToken.None, + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["client-request-id"] = "caller-supplied-id" + }); + + var sent = Assert.Single(wire.CapturedRequests); + var ids = Assert.Contains("client-request-id", sent.Headers); + Assert.Equal("caller-supplied-id", Assert.Single(ids)); + } + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public void Case_variant_duplicate_keys_merge_to_one_header() + { + var headers = TestCmdlet.BuildHeaders(null, new System.Collections.Hashtable + { + ["If-Match"] = "\"etag-a\"", + }); + Assert.NotNull(headers); + Assert.True(headers!.ContainsKey("if-match")); + Assert.True(headers.ContainsKey("IF-MATCH")); + } + + /// BuildRequestHeaders is protected static; surfaced for assertion. + private sealed class TestCmdlet : Mgx.Cmdlets.Base.MgxCmdletBase + { + public static Dictionary? BuildHeaders( + string? consistencyLevel, System.Collections.Hashtable? extra) + => BuildRequestHeaders(consistencyLevel, extra); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Http/ResilienceBridgeTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/ResilienceBridgeTests.cs index 2eb898c..0b246c0 100644 --- a/tests/Mgx.IntegrationTests/Engine/Http/ResilienceBridgeTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Http/ResilienceBridgeTests.cs @@ -246,4 +246,49 @@ protected override Task SendAsync( }); } } + + [Fact] + public async Task Bridge_stamps_one_correlation_id_per_logical_request() + { + ResiliencePipelineFactory.Reset(); + var wire = new MockHttpHandler(); + wire.QueueFailuresThenSuccess(1, HttpStatusCode.ServiceUnavailable, "{}"); + + var (pipeline, _) = ResiliencePipelineFactory.GetOrCreate( + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 3 }); + using var handler = new ResilientDelegatingHandler(pipeline, null) { InnerHandler = wire }; + using var client = new HttpClient(handler); + + var response = await client.SendAsync( + new HttpRequestMessage(HttpMethod.Get, "https://graph.microsoft.com/v1.0/users")); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + var captured = wire.CapturedRequests; + Assert.Equal(2, captured.Count); + var first = Assert.Single(Assert.Contains("client-request-id", captured[0].Headers)); + var second = Assert.Single(Assert.Contains("client-request-id", captured[1].Headers)); + Assert.Equal(first, second); + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task Bridge_keeps_the_sdk_correlation_id_when_one_is_already_set() + { + ResiliencePipelineFactory.Reset(); + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "{}"); + + var (pipeline, _) = ResiliencePipelineFactory.GetOrCreate( + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 3 }); + using var handler = new ResilientDelegatingHandler(pipeline, null) { InnerHandler = wire }; + using var client = new HttpClient(handler); + + var request = new HttpRequestMessage(HttpMethod.Get, "https://graph.microsoft.com/v1.0/users"); + request.Headers.TryAddWithoutValidation("client-request-id", "sdk-set-id"); + await client.SendAsync(request); + + var sent = Assert.Single(wire.CapturedRequests); + Assert.Equal("sdk-set-id", Assert.Single(Assert.Contains("client-request-id", sent.Headers))); + ResiliencePipelineFactory.Reset(); + } } diff --git a/tests/Mgx.IntegrationTests/Fakes/MockHttpHandler.cs b/tests/Mgx.IntegrationTests/Fakes/MockHttpHandler.cs index 8b1e136..2f78599 100644 --- a/tests/Mgx.IntegrationTests/Fakes/MockHttpHandler.cs +++ b/tests/Mgx.IntegrationTests/Fakes/MockHttpHandler.cs @@ -11,6 +11,21 @@ namespace Mgx.IntegrationTests; [CollectionDefinition("Pipeline")] public class PipelineCollection; +/// +/// A request as it looked on the wire at send time. Captured eagerly because the +/// client may dispose the request content once the call completes, so reading +/// Requests[n].Content after the fact is unreliable. +/// +public sealed record CapturedRequest( + HttpMethod Method, + string Uri, + IReadOnlyDictionary Headers, + IReadOnlyDictionary ContentHeaders, + byte[]? Body) +{ + public string? BodyText => Body == null ? null : Encoding.UTF8.GetString(Body); +} + /// /// Mock HTTP handler that returns configurable responses. /// Tracks request count for verifying retry behavior. @@ -19,6 +34,7 @@ public class MockHttpHandler : HttpMessageHandler { private readonly Queue _responses = new(); private readonly List _requests = []; + private readonly List _captured = []; private readonly object _lock = new(); private MockResponse? _defaultResponse; @@ -32,14 +48,39 @@ public List Requests get { lock (_lock) { return [.. _requests]; } } } - public void QueueResponse(HttpStatusCode statusCode, string? body = null, Dictionary? headers = null) + /// + /// Requests buffered at send time: method, URI, headers, and body bytes. + /// Survives the client disposing the originals. + /// + public List CapturedRequests { - _responses.Enqueue(new MockResponse(statusCode, body, headers, null)); + get { lock (_lock) { return [.. _captured]; } } + } + + public void QueueResponse(HttpStatusCode statusCode, string? body = null, Dictionary? headers = null, string contentType = "application/json") + { + _responses.Enqueue(new MockResponse(statusCode, body, headers, null, contentType, null)); + } + + /// + /// Queue a response with a raw byte body, for binary and non-UTF8 cases. + /// + public void QueueBytes(HttpStatusCode statusCode, byte[] body, string contentType, Dictionary? headers = null) + { + _responses.Enqueue(new MockResponse(statusCode, null, headers, null, contentType, body)); + } + + /// + /// Queue a response with no content at all - a 204, or a 200 with a zero-length body. + /// + public void QueueEmpty(HttpStatusCode statusCode, Dictionary? headers = null) + { + _responses.Enqueue(new MockResponse(statusCode, null, headers, null, null, null)); } public void SetDefaultResponse(HttpStatusCode statusCode, string? body = null, Dictionary? headers = null) { - _defaultResponse = new MockResponse(statusCode, body, headers, null); + _defaultResponse = new MockResponse(statusCode, body, headers, null, "application/json", null); } /// @@ -58,26 +99,47 @@ public void QueueFailuresThenSuccess(int failCount, HttpStatusCode failStatus, s /// public void QueueException(Exception exception) { - _responses.Enqueue(new MockResponse(HttpStatusCode.OK, null, null, exception)); + _responses.Enqueue(new MockResponse(HttpStatusCode.OK, null, null, exception, null, null)); } - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { + byte[]? bodyBytes = null; + if (request.Content != null) + bodyBytes = await request.Content.ReadAsByteArrayAsync(cancellationToken); + + var captured = new CapturedRequest( + request.Method, + request.RequestUri?.OriginalString ?? string.Empty, + request.Headers.ToDictionary(h => h.Key, h => h.Value.ToArray()), + request.Content?.Headers.ToDictionary(h => h.Key, h => h.Value.ToArray()) + ?? new Dictionary(), + bodyBytes); + MockResponse mock; lock (_lock) { _requests.Add(request); - mock = _responses.Count > 0 ? _responses.Dequeue() : (_defaultResponse ?? new MockResponse(HttpStatusCode.OK, null, null, null)); + _captured.Add(captured); + mock = _responses.Count > 0 ? _responses.Dequeue() : (_defaultResponse ?? new MockResponse(HttpStatusCode.OK, null, null, null, null, null)); } if (mock.Exception != null) - return Task.FromException(mock.Exception); + throw mock.Exception; // Real transports (SocketsHttpHandler) set RequestMessage on the response; consumers // like the pacer's OnRetry hook read the request URI off it. Mirror that here. var response = new HttpResponseMessage(mock.StatusCode) { RequestMessage = request }; - if (mock.Body != null) - response.Content = new StringContent(mock.Body, Encoding.UTF8, "application/json"); + if (mock.BodyBytes != null) + { + response.Content = new ByteArrayContent(mock.BodyBytes); + if (mock.ContentType != null) + response.Content.Headers.TryAddWithoutValidation("Content-Type", mock.ContentType); + } + else if (mock.Body != null) + { + response.Content = new StringContent(mock.Body, Encoding.UTF8, mock.ContentType ?? "application/json"); + } if (mock.Headers != null) { @@ -85,10 +147,10 @@ protected override Task SendAsync(HttpRequestMessage reques response.Headers.TryAddWithoutValidation(key, value); } - return Task.FromResult(response); + return response; } - private record MockResponse(HttpStatusCode StatusCode, string? Body, Dictionary? Headers, Exception? Exception); + private record MockResponse(HttpStatusCode StatusCode, string? Body, Dictionary? Headers, Exception? Exception, string? ContentType, byte[]? BodyBytes); } public static class TestData diff --git a/tests/Mgx.IntegrationTests/Serialization/WireBodyTests.cs b/tests/Mgx.IntegrationTests/Serialization/WireBodyTests.cs new file mode 100644 index 0000000..37741b7 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Serialization/WireBodyTests.cs @@ -0,0 +1,112 @@ +using System.Net; +using System.Reflection; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// The JSON that actually goes on the wire for a -Body argument, asserted byte for byte - +/// not whether serialization merely succeeds. (Corpus: GraphSDK-3361 scalar/array, +/// GraphSDK-3654 PSObject wrappers, M365DSC-5306/7175 malformed bodies.) +/// +[Collection("Pipeline")] +public class WireBodyTests +{ + private static readonly Type Base = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); + private const BindingFlags Static = BindingFlags.NonPublic | BindingFlags.Static; + + private static void InjectTransport(HttpMessageHandler wire) + { + Base.GetField("s_graphHttpClient", Static)!.SetValue(null, new HttpClient(wire)); + Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, + Mgx.Cmdlets.Base.MgxCmdletBase.BuildAuthFingerprint( + new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); + Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); + Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); + Base.GetField("s_clientOptions", Static)!.SetValue(null, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + ResiliencePipelineFactory.Reset(); + } + + private static void ResetTransport() + { + Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); + Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); + ResiliencePipelineFactory.Reset(); + } + + /// Runs a POST built by the given script and returns the wire body text. + private static string WireBodyFor(string bodyExpression) + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.NoContent); + InjectTransport(wire); + try + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript($"Invoke-MgxRequest -Uri /widgets -Method POST -Confirm:$false -Body $({bodyExpression})"); + ps.Invoke(); + + var errors = string.Join(" | ", ps.Streams.Error.Select(e => e.Exception.Message)); + var captured = wire.CapturedRequests; + Assert.True(captured.Count > 0, $"no request sent; errors: {errors}"); + return captured[0].BodyText!; + } + finally { ResetTransport(); } + } + + [Fact] + public void An_enum_serializes_as_its_camel_case_name() + => Assert.Equal("""{"day":"monday"}""", WireBodyFor("@{ day = [System.DayOfWeek]::Monday }")); + + [Fact] + public void A_byte_array_serializes_as_base64() + => Assert.Equal("""{"key":"AQID"}""", WireBodyFor("@{ key = [byte[]](1,2,3) }")); + + [Fact] + public void A_timespan_serializes_as_an_iso_duration() + => Assert.Equal("""{"d":"PT1H30M"}""", WireBodyFor("@{ d = [TimeSpan]::new(1,30,0) }")); + + [Fact] + public void A_kindless_datetime_is_pinned_to_utc() + => Assert.Equal("""{"at":"2026-01-05T10:00:00Z"}""", + WireBodyFor("@{ at = [DateTime]::new(2026,1,5,10,0,0) }")); + + [Fact] + public void Non_ascii_text_is_sent_as_utf8_not_escape_sequences() + => Assert.Equal("""{"name":"Müller & Söhne "}""", + WireBodyFor("@{ name = 'Müller & Söhne ' }")); + + [Fact] + public void A_single_element_array_stays_an_array() + => Assert.Equal("""{"members":["a"]}""", WireBodyFor("@{ members = @('a') }")); + + [Fact] + public void A_script_property_and_an_alias_property_serialize_like_convertto_json() + { + var body = WireBodyFor(""" + $o = [PSCustomObject]@{ displayName = 'x' } + $o | Add-Member -MemberType ScriptProperty -Name mailNickname -Value { 'nick-' + $this.displayName } -PassThru | + Add-Member -MemberType AliasProperty -Name alias -Value displayName -PassThru + """); + var doc = System.Text.Json.JsonDocument.Parse(body); + Assert.Equal("x", doc.RootElement.GetProperty("displayName").GetString()); + Assert.Equal("nick-x", doc.RootElement.GetProperty("mailNickname").GetString()); + Assert.Equal("x", doc.RootElement.GetProperty("alias").GetString()); + } + + [Fact] + public void A_wrapped_and_an_unwrapped_hashtable_produce_identical_wire_json() + { + var direct = WireBodyFor("@{ a = 1; nested = @{ b = $true } }"); + var wrapped = WireBodyFor("[PSObject]@{ a = 1; nested = @{ b = $true } }"); + Assert.Equal(direct, wrapped); + } +} diff --git a/tests/RELEASING.md b/tests/RELEASING.md index c6acedb..e6ddb5c 100644 --- a/tests/RELEASING.md +++ b/tests/RELEASING.md @@ -51,6 +51,12 @@ git push origin main && git push origin vX.Y.Z The squash message is the public record of the release; the CHANGELOG carries the per-defect detail. Describe what the release *is*, not the churn that produced it. +Do not open it by restating what the version number already says. "Patch release", a count of +the fixes, "no feature or API changes", "the PowerShell 7.4 floor is unchanged" - a reader can +see all of that from the version and the entries beneath it. A feature, an API change or a +floor move is worth a sentence precisely because it is not the default; the absence of one is +not. Start with what changed. + ## After `Publish-Module` from the tagged `main`. From 932c9c0e53574549abf6df0dca08b0900e801c8d Mon Sep 17 00:00:00 2001 From: gromedev <61883549+gromedev@users.noreply.github.com> Date: Wed, 26 Aug 2026 21:00:13 +0200 Subject: [PATCH 04/10] mgx 2.1.3 --- .github/workflows/build-and-test.yml | 6 +- CHANGELOG.md | 190 +++++++++----- Directory.Build.props | 2 +- README.md | 4 +- examples/README.md | 69 ++--- .../assign-licenses.ps1 | 106 ++++++++ .../bulk-create-users-from-csv.ps1 | 86 ++++++ .../bulk-writes-and-batching/bulk-update.ps1 | 32 --- .../dead-letter-retry.ps1 | 68 ----- .../mixed-endpoint-batch.ps1 | 61 ----- .../offboard-users.ps1 | 83 ++++++ .../tenant-snapshot.ps1 | 53 ++++ examples/change-feeds/group-change-feed.ps1 | 50 ++++ .../tenant-change-feed.ps1 | 8 + examples/change-feeds/tenant-inventory.ps1 | 123 +++++++++ .../drive-content/download-user-onedrive.ps1 | 97 +++++++ .../drive-content/drive-content-triage.ps1 | 16 +- .../drive-delta-resumable.ps1 | 0 .../drive-content/partial-content-hash.ps1 | 48 ---- .../chained-relation-expansion.ps1 | 34 --- .../enrich-users-with-manager.ps1 | 32 --- .../get-managers-fan-out.ps1 | 36 --- .../group-members-multipage.ps1 | 26 -- examples/getting-started/beta-endpoint.ps1 | 34 --- .../delta-users-sync.ps1} | 0 .../getting-started/export-users-to-csv.ps1 | 44 ++++ .../conditional-access-export.ps1 | 45 ---- .../disabled-accounts-report.ps1 | 33 --- .../download-groups-and-members.ps1 | 153 +++++++++++ .../group-membership-report.ps1 | 84 ++++++ .../guest-users-report.ps1 | 33 --- .../inactive-users-report.ps1 | 67 +++++ .../reporting-and-audit/license-report.ps1 | 80 ++++++ .../stale-devices-report.ps1 | 37 --- .../user-directory-report.ps1 | 45 ++++ .../pacing-observability.ps1 | 178 ------------- .../progress-for-long-runs.ps1 | 81 ++++++ .../resilience-for-existing-scripts.ps1 | 24 -- .../resilience-status.ps1 | 8 +- .../resource-unit-budgeting.ps1 | 80 ++---- .../resilience-and-telemetry/telemetry.ps1 | 44 ---- .../tune-rate-limits.ps1 | 63 ----- module/en-US/Mgx.Cmdlets.dll-Help.xml | 4 + module/en-US/about_Mgx.help.txt | 1 + module/en-US/about_Mgx_Errors.help.txt | 99 +++++++ module/help/Invoke-MgxBatchRequest.md | 2 + module/help/Invoke-MgxRequest.md | 2 + module/mgx.psd1 | 29 ++- src/Mgx.Cmdlets/Base/MgxCmdletBase.cs | 83 +++++- src/Mgx.Cmdlets/Base/MgxErrorPresentation.cs | 77 ++++++ .../Cmdlets/Batch/InvokeMgxBatchRequest.cs | 40 ++- .../Cmdlets/Content/GetMgxContent.cs | 55 ++-- src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs | 29 ++- .../Cmdlets/Expand/ExpandMgxRelation.cs | 43 +-- .../Cmdlets/Export/ExportMgxCollection.cs | 52 ++-- src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs | 206 +++++++++++---- src/Mgx.Engine/Errors/MgxErrorClass.cs | 48 ++++ src/Mgx.Engine/Errors/MgxErrorClassifier.cs | 70 +++++ src/Mgx.Engine/Errors/MgxErrorInfo.cs | 16 ++ src/Mgx.Engine/Errors/MgxErrorPolicy.cs | 42 +++ src/Mgx.Engine/Errors/RetryAfterPolicy.cs | 57 ++++ src/Mgx.Engine/Http/AdaptiveRequestPacer.cs | 10 +- src/Mgx.Engine/Http/GraphBatchClient.cs | 24 +- src/Mgx.Engine/Http/GraphContentClient.cs | 78 ++++-- .../Http/ResiliencePipelineFactory.cs | 110 +++----- .../Http/ResilientDelegatingHandler.cs | 27 +- src/Mgx.Engine/Http/ResilientGraphClient.cs | 31 ++- tests/Live/Mgx.Live.Tests.ps1 | 6 +- tests/Live/Mgx.Throttle.Tests.ps1 | 6 +- tests/Live/README.md | 4 + .../Cmdlets/BatchErrorSurfacingTests.cs | 40 ++- .../Cmdlets/DiagnosticsDrainTests.cs | 3 + .../Cmdlets/HeaderWireTests.cs | 6 + .../Cmdlets/OutFileGuardTests.cs | 74 ++++++ .../Cmdlets/ResponseFidelityTests.cs | 95 +++++++ .../Cmdlets/SkipLogicTests.cs | 115 ++++++++ .../Cmdlets/UriEncodingTests.cs | 136 +++++++++- .../Engine/Errors/ErrorClassifierTests.cs | 52 ++++ .../Engine/Errors/ErrorPolicyParityTests.cs | 174 +++++++++++++ .../Engine/Errors/RetryAfterPolicyTests.cs | 61 +++++ .../Engine/Http/DownloadHeaderTests.cs | 110 ++++++++ .../Engine/Http/HeaderFidelityTests.cs | 67 ++++- .../Engine/Http/ResilienceBridgeTests.cs | 91 +++++++ .../Engine/Pagination/OpaqueLinkTests.cs | 54 ++++ .../PSWrapperEquivalenceTests.cs | 87 +++++++ .../Serialization/WireBodyTests.cs | 119 +++++++++ .../TestSetup/HostileInputs.cs | 48 ++++ tests/Unit/Mgx.ErrorContract.Tests.ps1 | 64 +++++ .../10-pacing-under-real-throttling.ps1 | 246 +++++++++++++++--- .../15-fanout-concurrency-scaling.ps1 | 233 +++++++++++++++++ tests/benchmarks/README.md | 9 +- tests/benchmarks/common.ps1 | 31 ++- 92 files changed, 4152 insertions(+), 1377 deletions(-) create mode 100644 examples/bulk-writes-and-batching/assign-licenses.ps1 create mode 100644 examples/bulk-writes-and-batching/bulk-create-users-from-csv.ps1 delete mode 100644 examples/bulk-writes-and-batching/bulk-update.ps1 delete mode 100644 examples/bulk-writes-and-batching/dead-letter-retry.ps1 delete mode 100644 examples/bulk-writes-and-batching/mixed-endpoint-batch.ps1 create mode 100644 examples/bulk-writes-and-batching/offboard-users.ps1 create mode 100644 examples/bulk-writes-and-batching/tenant-snapshot.ps1 create mode 100644 examples/change-feeds/group-change-feed.ps1 rename examples/{delta-and-sync => change-feeds}/tenant-change-feed.ps1 (90%) create mode 100644 examples/change-feeds/tenant-inventory.ps1 create mode 100644 examples/drive-content/download-user-onedrive.ps1 rename examples/{delta-and-sync => drive-content}/drive-delta-resumable.ps1 (100%) delete mode 100644 examples/drive-content/partial-content-hash.ps1 delete mode 100644 examples/fan-out-and-relations/chained-relation-expansion.ps1 delete mode 100644 examples/fan-out-and-relations/enrich-users-with-manager.ps1 delete mode 100644 examples/fan-out-and-relations/get-managers-fan-out.ps1 delete mode 100644 examples/fan-out-and-relations/group-members-multipage.ps1 delete mode 100644 examples/getting-started/beta-endpoint.ps1 rename examples/{delta-and-sync/delta-sync.ps1 => getting-started/delta-users-sync.ps1} (100%) create mode 100644 examples/getting-started/export-users-to-csv.ps1 delete mode 100644 examples/reporting-and-audit/conditional-access-export.ps1 delete mode 100644 examples/reporting-and-audit/disabled-accounts-report.ps1 create mode 100644 examples/reporting-and-audit/download-groups-and-members.ps1 create mode 100644 examples/reporting-and-audit/group-membership-report.ps1 delete mode 100644 examples/reporting-and-audit/guest-users-report.ps1 create mode 100644 examples/reporting-and-audit/inactive-users-report.ps1 create mode 100644 examples/reporting-and-audit/license-report.ps1 delete mode 100644 examples/reporting-and-audit/stale-devices-report.ps1 create mode 100644 examples/reporting-and-audit/user-directory-report.ps1 delete mode 100644 examples/resilience-and-telemetry/pacing-observability.ps1 create mode 100644 examples/resilience-and-telemetry/progress-for-long-runs.ps1 delete mode 100644 examples/resilience-and-telemetry/resilience-for-existing-scripts.ps1 delete mode 100644 examples/resilience-and-telemetry/telemetry.ps1 delete mode 100644 examples/resilience-and-telemetry/tune-rate-limits.ps1 create mode 100644 module/en-US/about_Mgx_Errors.help.txt create mode 100644 src/Mgx.Cmdlets/Base/MgxErrorPresentation.cs create mode 100644 src/Mgx.Engine/Errors/MgxErrorClass.cs create mode 100644 src/Mgx.Engine/Errors/MgxErrorClassifier.cs create mode 100644 src/Mgx.Engine/Errors/MgxErrorInfo.cs create mode 100644 src/Mgx.Engine/Errors/MgxErrorPolicy.cs create mode 100644 src/Mgx.Engine/Errors/RetryAfterPolicy.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/OutFileGuardTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/SkipLogicTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Errors/ErrorClassifierTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Errors/ErrorPolicyParityTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Errors/RetryAfterPolicyTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Http/DownloadHeaderTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Pagination/OpaqueLinkTests.cs create mode 100644 tests/Mgx.IntegrationTests/Serialization/PSWrapperEquivalenceTests.cs create mode 100644 tests/Mgx.IntegrationTests/TestSetup/HostileInputs.cs create mode 100644 tests/Unit/Mgx.ErrorContract.Tests.ps1 create mode 100644 tests/benchmarks/15-fanout-concurrency-scaling.ps1 diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index b5ce217..b758cdd 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -55,7 +55,7 @@ jobs: - name: Assert xUnit actually ran shell: pwsh run: | - $floor = 650 + $floor = 2700 $trx = Get-ChildItem TestResults -Filter *.trx | Select-Object -First 1 if (-not $trx) { throw "No .trx produced - the test run did not report results." } $c = ([xml](Get-Content $trx.FullName -Raw)).TestRun.ResultSummary.Counters @@ -83,7 +83,7 @@ jobs: throw "$($result.FailedCount) Pester test(s) failed." } # Same reasoning as the xUnit floor above: silence is not success. - $floor = 215 + $floor = 235 $ran = [int]$result.PassedCount + [int]$result.SkippedCount if ($ran -lt $floor) { throw "Only $ran Pester test(s) ran, expected at least $floor. The suite is not running." @@ -174,7 +174,7 @@ jobs: $r = Invoke-TestHarness -IgnoreCodeCoverage "$($r.PassedCount) passed, $($r.FailedCount) failed, $($r.SkippedCount) skipped" | Write-Host if ($r.FailedCount -gt 0) { throw "$($r.FailedCount) test(s) failed" } - $floor = 215 + $floor = 235 $ran = [int]$r.PassedCount + [int]$r.SkippedCount if ($ran -lt $floor) { throw "Only $ran test(s) ran on this host, expected at least $floor." diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a32e55..a832925 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,99 +1,145 @@ # Changelog +## 2.1.3 + +### Added + +- Added about_Mgx_Errors documentation detailing termination behavior, error record generation, and -ErrorAction/-ErrorVariable handling. +- Added a performance benchmark measuring throughput under -Concurrency against connection pool limits. +- Live test suite now supports client secret authentication alongside certificate-based authentication. + +### Fixed + +- SecureString, credential objects, and script blocks in -Body throw explicit errors rather than serializing as reflection noise. +- Property path details are now included in error records when NaN or Infinity values fail -Body serialization. +- Server errors, timeouts, and precondition failures assign specific ErrorCategory values instead of defaulting to NotSpecified. +- Broken circuit exceptions across fan-out execution paths surface consistent error guidance. +- Get-MgxContent forwards the full family of conditional headers (including If-None-Match) to content download hosts. +- Piped input to Get-MgxContent validates -OutFile constraints prior to requesting download URLs. +- BatchChunkFailed events now map underlying exception root causes to appropriate ErrorCategory values. +- Failed batch chunks under -ErrorAction Stop write dead-letter files prior to terminating pipeline execution. +- Typed parameters deferring to options already present in -Uri emit warnings rather than dropping silently. +- Sync-MgxDelta state comparison evaluates active -Property sets correctly when -Uri $select parameters are present. +- Embedded $count parameters in -Uri avoid triggering redundant degradation retries. +- Encoded %24top options in -Uri combined with -Top parameters no longer emit duplicate query parameters. +- Absolute -Uri inputs with leading whitespace are now rejected by the guard instead of being prepended to the versioned base URL. +- Lowercase conditional headers are preserved and forwarded across two-hop content download requests. +- client-request-id headers on two-hop content download paths avoid duplicate values. +- Requests with bodies now emit warnings on malformed header names rather than raising unhandled crashes. +- Caller-supplied Content-Length headers are ignored in favor of computed payload lengths, preventing request failures from length mismatches. +- Request bodies over 4 MB pass directly to the SDK chain under Enable-MgxResilience instead of being rejected. +- HTTP 304 Not Modified responses on conditional downloads complete cleanly without generating error records. +- If-Match headers are stripped when requesting content download URLs to prevent HTTP 412 Precondition Failed errors. +- Byte-level JSON parsing handles UTF-8 BOM prefixes correctly. +- Response bodies and error snippets respect declared response character sets during decoding. +- Export-MgxCollection handles malformed page responses cleanly by emitting error records instead of raw exceptions. +- Circular self-referencing objects in -Body raise validation errors instead of causing process stack overflows. +- Dead-letter file output preserves unescaped readable text formatting. +- Benchmark 10 workload scaling was adjusted to trigger true throttling and measure adaptive pacing accurately. + +### Changed + +- Unified failure classification logic across retries, circuit breaking, batch retry checks, content downloads, and adaptive pacing. + +### Documentation + +- Reorganized examples by task to cover end-to-end workflows rather than isolated cmdlet usage. +- Clarified that adaptive pacing responds directly to throttling signals, using observed latency for telemetry reporting rather than as a control input. +- Verbose output now identifies non-JSON string -Body inputs and notes Content-Type override usage. +- Updated cmdlet help to document -Body serialization contracts and prohibited parameter object types. + ## 2.1.2 ### Fixed -- Invoke-MgxBatchRequest wrote no error records for failed items unless a dead-letter write also failed. -- A 204 or empty response body on a GET ended the cmdlet with an unhandled error. -- An HTML or otherwise non-JSON response body surfaced as an unhandled exception instead of an error record. -- A JSON response body that did not parse crashed the pipeline instead of reporting what was received. -- A '#' in a request path silently dropped everything after it. -- Pre-encoded -Filter, -Search, -Property, -Sort and -ExpandProperty values were percent-encoded a second time. -- Enums in -Body were sent as numbers, which Graph does not accept. -- A byte array in -Body became a JSON array of integers instead of base64. -- A TimeSpan in -Body was not sent as an ISO-8601 duration. -- A DateTime without a kind was sent without an offset and refused. -- A PSCustomObject -Body dropped every property that was not a NoteProperty, sending {} in the worst case. -- Non-ASCII text in -Body was \u-escaped, making -Debug traces and dead-letter files unreadable. -- An array in -Headers went on the wire as the literal text System.String[]. -- Invoke-MgxRequest, Expand-MgxRelation and Export-MgxCollection mangled an absolute -Uri instead of rejecting it. -- Get-MgxContent did not escape drive and item ids piped into the download path. -- A Content-Type or other content header in -Headers was silently dropped. -- A client-request-id in -Headers had Mgx's own value appended after it. -- If-Match and if-match in -Headers went on the wire as two headers. -- A typed parameter duplicating a query option already written into -Uri sent it twice. -- An attempt that timed out during Ctrl-C could schedule another retry. -- Requests through Enable-MgxResilience carried no client-request-id to quote at Graph support. -- An unexpected failure discarded the buffered retry history that explained it. -- A request body over 4MB passed through Enable-MgxResilience unchecked and was buffered whole. +- `Invoke-MgxBatchRequest` wrote no error records for failed items unless a dead-letter write also failed. +- HTTP 204 and empty response bodies on GET requests complete cleanly instead of raising unhandled errors. +- HTML and non-JSON response bodies surface as proper error records instead of unhandled exceptions. +- Unparseable JSON response bodies report raw received text without crashing the pipeline. +- '#' characters in request paths no longer truncate subsequent path content. +- Pre-encoded values in `-Filter`, `-Search`, `-Property`, `-Sort`, and `-ExpandProperty` avoid double-percent-encoding. +- Enum values in `-Body` transmit as strings instead of numbers. +- Byte arrays in `-Body` serialize as base64 strings instead of JSON integer arrays. +- `TimeSpan` values in `-Body` format correctly as ISO-8601 durations. +- `DateTime` values without a `Kind` property include valid offsets to prevent endpoint rejection. +- `PSCustomObject` inputs to `-Body` retain non-NoteProperty members instead of serializing to `{}`. +- Non-ASCII text in `-Body` avoids `\u`-escaping to keep `-Debug` traces and dead-letter files readable. +- Array values in `-Headers` transmit properly instead of serializing to the literal string `System.String[]`. +- `Invoke-MgxRequest`, `Expand-MgxRelation`, and `Export-MgxCollection` preserve absolute `-Uri` inputs without mangling. +- `Get-MgxContent` properly escapes drive and item IDs piped into download paths. +- Content-Type and other content headers in `-Headers` are preserved instead of being silently dropped. +- Custom `client-request-id` values in `-Headers` no longer have internal Mgx IDs appended. +- Case-variant `If-Match` headers in `-Headers` avoid sending duplicate headers. +- Typed parameters duplicating query options already in `-Uri` no longer emit twice. +- Timed-out request attempts during Ctrl-C cancellation no longer schedule redundant retries. +- Requests routed through `Enable-MgxResilience` include required `client-request-id` headers. +- Unexpected failures preserve buffered retry histories for diagnostics. +- A request body over 4 MB passed through `Enable-MgxResilience` unchecked and was buffered whole. ## 2.1.1 ### Fixed -- Enable-MgxResilience ignored throttling: pacing never slowed, telemetry reported no retries, and a throttled request could fail outright. +- `Enable-MgxResilience` now respects throttling signals, properly slowing pacing, logging telemetry retries, and preventing throttled requests from failing outright. ### Changed -- Under Enable-MgxResilience, a 503 or 504 on a write is no longer retried. Throttling (429) is unaffected. +- HTTP 503 and 504 errors on write operations are no longer retried under `Enable-MgxResilience` (throttling 429 retries are unaffected). ## 2.1.0 -Adds proactive throttle avoidance, ranged content downloads, and resumable enumeration. - ### Added -- Adaptive request pacing, on by default. Requests are spaced before they are sent, per workload; opt out with Set-MgxOption -NoAdaptivePacing. -- Get-MgxContent, to download file and media content whole or by byte range (-First). -- Sync-MgxDelta -CheckpointPath, to resume interrupted enumerations. -- Sync-MgxDelta -Latest, to baseline state without enumerating. -- Sync-MgxDelta -Prefer, to send drive delta Prefer tokens. -- Resource units consumed, in telemetry output, benchmarks and examples. +- Added adaptive request pacing by default per workload, with opt-out via `Set-MgxOption -NoAdaptivePacing`. +- Added `Get-MgxContent` to download file and media content whole or by byte range (`-First`). +- Added `Sync-MgxDelta -CheckpointPath` to resume interrupted enumerations at page boundaries or every 500 items in JSONL mode. +- Added `Sync-MgxDelta -Latest` to baseline state from the current moment without enumerating. +- Added `Sync-MgxDelta -Prefer` to send drive delta Prefer tokens. +- Added Resource Units Consumed reporting to telemetry outputs, benchmark results, and examples. ### Fixed -- -Latest was honored after a state invalidation, dropping every change since the last sync. -- Delta resume dropped the interrupted run's items when an earlier run had already completed, and advanced the delta token past them. -- Export resume dropped the interrupted run's items when an earlier export had already written the output file. -- Export resume wrote a page's items twice when a run was interrupted twice inside the same page. -- Delta resume kept the previous sync's rows in front of the interrupted sync's output. -- Export resume started the export over when the interrupted run had died on a transient error. -- A resume checkpoint naming a file that was never a temp consumed it as data. -- A refused delta token was reported as an endpoint not supporting delta queries. -- Sync-MgxDelta as a session's first cmdlet sent its request to the public cloud endpoint. -- A denied or read-only output file ended a delta sync with an unhandled error on Windows. -- A missing drive item suggested retrying in beta instead of reporting the item absent. -- -Top was ignored when combined with -All, returning the whole collection. -- Get-Help described output shapes and parameters from before 2.0.0. -- Enumeration returned part of a collection without error when a nextLink was refused. -- A delta sync stopped with an error when a page contained a non-object item. -- Get-MgxContent left an unwritable -OutFile as an unhandled error naming a temp path. -- A non-Graph JSON error body threw from inside the exception it was being used to build, surfacing a CDN failure as a security refusal. -- A $batch 429 set a pacing cap that nothing enforced and nothing cleared, so telemetry reported it forever. -- Get-MgxContent ignored Retry-After when a download host sent it as an HTTP date. -- Get-Help was missing the examples for -CheckpointPath, -Latest, -Prefer, adaptive pacing, and two Get-MgxContent endpoints, and did not document Set-MgxOption -BatchChunkConcurrency. -- Seven examples rendered blank tables, and one reported password secrets as certificates. -- Repeat 429s could raise the adaptive cap above -RateLimitPerSecond. -- Delta state did not record its API version, so runs omitting -ApiVersion silently synced the other one. -- -Debug wrote pre-authenticated download URLs verbatim, from redirect headers and from response bodies. -- Get-MgxContent -OutFile with piped input downloaded every item but kept only the last. -- -Offset was ignored when a server returned a whole-body 200. -- An offset past end of file overwrote an existing -OutFile with an empty one. -- Two-hop content downloads were reported as failed requests. -- Adaptive pacing fired synchronized bursts when sleep times were clamped. -- Enable-MgxResilience recorded no telemetry: request counts, HTTP time and rate-limiter wait all stayed 0. -- Invoke-MgGraphRequest with a relative URI failed while resilience was enabled. -- $batch envelopes were bucketed as Other rather than their target workload. -- Slow start opened above ceilings set below 4 rps. -- ResiliencePipelineFactory.Reset() reverted configuration and kept batch pacer state across credential changes. -- Ctrl-C during file cleanup raised disposed-object errors. +- `-Latest` flag behavior preserved after state invalidation to avoid dropping changes since the previous sync. +- Delta resume retains interrupted run items and properly manages delta tokens when previous runs completed. +- Export resume retains interrupted run items when previous exports have written output files. +- Export resume prevents duplicate page item writes when runs are interrupted twice within the same page. +- Delta resume outputs interrupted sync data cleanly without prepending previous sync rows. +- Export resume recovers transient error interruptions without restarting exports from scratch. +- Resume checkpoints ignore non-temp files to prevent consuming them as data. +- Refused delta tokens report distinct error messages instead of claiming unsupported delta endpoints. +- `Sync-MgxDelta` targets correct cloud endpoints when run as the first cmdlet in a session. +- Access-denied or read-only output files raise clear errors on Windows without unhandled exceptions. +- Missing drive items report absent items directly without suggesting retry on beta endpoints. +- `-Top` parameter functions correctly when combined with `-All`. +- `Get-Help` reflects post-2.0.0 output shapes and parameters. +- Enumerations report errors cleanly when `nextLink` URLs are refused instead of returning partial collections. +- Delta syncs handle non-object items within pages without halting. +- `Get-MgxContent` reports unwritable `-OutFile` errors clearly against target paths rather than temporary paths. +- Non-Graph JSON error bodies surface CDN failures accurately without throwing nested exceptions. +- Batch 429 responses clear pacing caps properly without persisting in telemetry endlessly. +- `Get-MgxContent` respects HTTP date-formatted `Retry-After` headers from download hosts. +- `Get-Help` documentation updated with examples for `-CheckpointPath`, `-Latest`, `-Prefer`, adaptive pacing, `Get-MgxContent` endpoints, and `Set-MgxOption -BatchChunkConcurrency`. +- Example table rendering fixed across seven examples, and password secret examples updated to avoid misclassification as certificates. +- Adaptive caps respect `-RateLimitPerSecond` ceilings during repeated 429 throttling events. +- Delta state records API versions to prevent un-versioned runs from silently syncing alternate versions. +- `-Debug` tracing redacts pre-authenticated download URLs from redirect headers and response bodies. +- `Get-MgxContent -OutFile` with piped input appends/manages output files without overwriting all but the final item. +- `-Offset` parameter respected when servers return full-body HTTP 200 responses. +- Offsets past end-of-file preserve existing `-OutFile` targets instead of overwriting with empty files. +- Two-hop content downloads record as successful requests. +- Adaptive pacing prevents synchronized burst requests when queue sleep times are clamped. +- `Enable-MgxResilience` populates telemetry metrics for request counts, HTTP time, and rate-limiter waits. +- `Invoke-MgGraphRequest` supports relative URIs while resilience is enabled. +- `$batch` envelopes bucket under target workloads instead of `Other`. +- Slow start respects rate limit ceilings configured below 4 rps. +- `ResiliencePipelineFactory.Reset()` isolates configuration changes and resets pacer state across credential changes. +- Ctrl-C cancellation during file cleanup completes without throwing disposed-object errors. ### Documentation -- Graph delta responses repeat objects across pages; deduplicate on id, or baseline with -Latest. -- Adaptive pacing applies to every request except batch outer POSTs. -- Measured throttling behavior: resource unit budgets are scoped per application and tenant, and x-ms-throttle-limit-percentage is not emitted. +- Documented Graph delta response object duplication across pages, recommending `id` deduplication or `-Latest` baselining. +- Clarified that adaptive pacing applies to every request except batch outer POSTs. +- Documented measured throttling behavior: resource unit budgets are scoped per application and tenant, and `x-ms-throttle-limit-percentage` is not emitted. ## 2.0.1 @@ -110,7 +156,7 @@ Patch release. Three fixes; no feature or API changes. ### Breaking - Invoke-MgxRequest, Invoke-MgxBatchRequest, Expand-MgxRelation and Sync-MgxDelta now emit case-insensitive Hashtables instead of PSObjects, matching Invoke-MgGraphRequest. Use -Raw | ConvertFrom-Json for the old shape. -- @odata.type is preserved under its own key; all other @odata.* transport metadata is stripped, including @odata.etag. +- @odata.type is preserved under its own key; all other @odata.- transport metadata is stripped, including @odata.etag. - Removed the custom format views (Mgx.User, Mgx.Group, and the rest); dictionaries render with PowerShell's built-in Name/Value view. - Lowered the floor to PowerShell 7.4 (LTS) and .NET 8, down from 7.5. - Removed Microsoft.Graph.Authentication from RequiredModules. Cmdlets needing auth now report GraphAuthModuleNotLoaded with install instructions. diff --git a/Directory.Build.props b/Directory.Build.props index c014344..7d5f301 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,7 +1,7 @@ - 2.1.2 + 2.1.3 Thomas Maillo Grome Mgx (c) 2026 Thomas Maillo Grome. All rights reserved. diff --git a/README.md b/README.md index f26975c..538b23a 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ Streaming directly to a file keeps managed heap growth close to zero relative to ### Adaptive pacing -Adaptive pacing is enabled by default. It starts conservatively, increases the allowed rate as requests succeed, and backs off when throttling or latency indicates that a workload is approaching a limit. +Adaptive pacing is enabled by default. It starts conservatively, increases the allowed rate as requests succeed, and backs off when throttling indicates that a workload is approaching a limit. | Workload | Pacing on | `-NoAdaptivePacing` | | --- | ---: | ---: | @@ -184,7 +184,7 @@ Adaptive pacing sits above those request-level strategies and learns a separate Microsoft Graph directory workloads use resource-unit budgets. The cost depends on query shape rather than on which client issued the request. -Mgx reads `x-ms-resource-unit` and exposes the accumulated values through `Get-MgxTelemetry`. It uses throttling signals and observed latency to pace requests before the workload repeatedly hits the limit. +Mgx reads `x-ms-resource-unit` and exposes the accumulated values through `Get-MgxTelemetry`. It uses throttling signals to pace requests before the workload repeatedly hits the limit. Observed latency is reported alongside them but is not a pacing input. Examples measured against the test tenant: diff --git a/examples/README.md b/examples/README.md index cfd3022..37be230 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,8 +1,8 @@ # Mgx Examples -Runnable scripts covering common Microsoft Graph scenarios. Each script is self-contained and includes the required scopes in its header comment. +Runnable scripts for things people actually do against Microsoft Graph. Each one is self-contained and names the scopes it needs in its header comment. -Scripts are grouped by purpose - pick the directory that matches what you are trying to do. +Scripts are grouped by the job, not by the cmdlet - pick the directory that matches what you are trying to get done. Within a directory, the shorter script usually comes first and the longer one builds on it. ## Prerequisites @@ -20,78 +20,51 @@ Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All" # adjust scopes per |--------|-------------|--------| | [get-all-users.ps1](getting-started/get-all-users.ps1) | Stream all users to the console | `User.Read.All` | | [export-users-to-jsonl.ps1](getting-started/export-users-to-jsonl.ps1) | Export all users to a JSONL file with checkpoint/resume | `User.Read.All` | -| [beta-endpoint.ps1](getting-started/beta-endpoint.ps1) | Access beta endpoints without installing extra modules | `User.Read.All` | - -### Fan-Out & Relations (`fan-out-and-relations/`) - -| Script | Description | Scopes | -|--------|-------------|--------| -| [get-managers-fan-out.ps1](fan-out-and-relations/get-managers-fan-out.ps1) | Fetch every user's manager concurrently | `User.Read.All` | -| [enrich-users-with-manager.ps1](fan-out-and-relations/enrich-users-with-manager.ps1) | Attach manager as a property on each user object | `User.Read.All` | -| [chained-relation-expansion.ps1](fan-out-and-relations/chained-relation-expansion.ps1) | Enrich users with manager + licenses in one pass | `User.Read.All` | -| [group-members-multipage.ps1](fan-out-and-relations/group-members-multipage.ps1) | Stream all members from all groups concurrently | `Group.Read.All`, `User.Read.All` | +| [export-users-to-csv.ps1](getting-started/export-users-to-csv.ps1) | Export all users to CSV, flattening the nested properties | `User.Read.All` | +| [delta-users-sync.ps1](getting-started/delta-users-sync.ps1) | Full sync on first run, changed users only thereafter | `User.Read.All` | ### Reporting & Audit (`reporting-and-audit/`) | Script | Description | Scopes | |--------|-------------|--------| -| [disabled-accounts-report.ps1](reporting-and-audit/disabled-accounts-report.ps1) | Report all disabled accounts | `User.Read.All` | -| [guest-users-report.ps1](reporting-and-audit/guest-users-report.ps1) | Report all guest (external) accounts | `User.Read.All` | -| [stale-devices-report.ps1](reporting-and-audit/stale-devices-report.ps1) | Report devices inactive for 90+ days | `Device.Read.All` | +| [user-directory-report.ps1](reporting-and-audit/user-directory-report.ps1) | Every enabled user with manager and licenses attached | `User.Read.All` | +| [download-groups-and-members.ps1](reporting-and-audit/download-groups-and-members.ps1) | Download every group and all memberships, nesting resolved locally, with a progress bar | `Group.Read.All`, `User.Read.All` | +| [group-membership-report.ps1](reporting-and-audit/group-membership-report.ps1) | Direct vs effective membership of every group, nesting resolved server-side | `Group.Read.All`, `User.Read.All` | +| [inactive-users-report.ps1](reporting-and-audit/inactive-users-report.ps1) | Users not seen in 90 days, and those who never signed in | `User.Read.All`, `AuditLog.Read.All` | +| [license-report.ps1](reporting-and-audit/license-report.ps1) | Seats per SKU, who holds them, and which are on disabled accounts | `Organization.Read.All`, `User.Read.All` | | [app-secrets-expiry.ps1](reporting-and-audit/app-secrets-expiry.ps1) | Find app secrets and certificates expiring within 30 days | `Application.Read.All` | -| [conditional-access-export.ps1](reporting-and-audit/conditional-access-export.ps1) | Export all Conditional Access policies (beta endpoint) | `Policy.Read.All` | | [export-sign-in-logs.ps1](reporting-and-audit/export-sign-in-logs.ps1) | Export sign-in logs to JSONL with checkpoint/resume | `AuditLog.Read.All` | ### Bulk Writes & Batching (`bulk-writes-and-batching/`) | Script | Description | Scopes | |--------|-------------|--------| -| [bulk-update.ps1](bulk-writes-and-batching/bulk-update.ps1) | PATCH multiple users via `$batch` (20 per HTTP call) | `User.ReadWrite.All` | -| [dead-letter-retry.ps1](bulk-writes-and-batching/dead-letter-retry.ps1) | Bulk create users with dead-letter tracking for failures | `User.ReadWrite.All` | +| [tenant-snapshot.ps1](bulk-writes-and-batching/tenant-snapshot.ps1) | Users, groups, apps, and licenses in one HTTP round trip | `User.Read.All`, `Group.Read.All`, `Application.Read.All` | +| [bulk-create-users-from-csv.ps1](bulk-writes-and-batching/bulk-create-users-from-csv.ps1) | Create users from a CSV, keeping failures for replay | `User.ReadWrite.All` | +| [assign-licenses.ps1](bulk-writes-and-batching/assign-licenses.ps1) | License everyone who is missing one, seat-checked first | `User.ReadWrite.All`, `Organization.Read.All` | +| [offboard-users.ps1](bulk-writes-and-batching/offboard-users.ps1) | Block sign-in, revoke sessions, and return the licenses | `User.ReadWrite.All`, `Directory.ReadWrite.All` | | [bulk-delete-whatif.ps1](bulk-writes-and-batching/bulk-delete-whatif.ps1) | Bulk delete stale guests with `-WhatIf` preview | `User.ReadWrite.All` | -| [mixed-endpoint-batch.ps1](bulk-writes-and-batching/mixed-endpoint-batch.ps1) | Query users, groups, apps, and SKUs in one HTTP call | `User.Read.All`, `Group.Read.All`, `Application.Read.All` | -### Delta & Sync (`delta-and-sync/`) +### Change Feeds (`change-feeds/`) | Script | Description | Scopes | |--------|-------------|--------| -| [delta-sync.ps1](delta-and-sync/delta-sync.ps1) | Full sync on first run, incremental changes thereafter | `User.Read.All` | -| [drive-delta-resumable.ps1](delta-and-sync/drive-delta-resumable.ps1) | Resumable OneDrive/SharePoint delta sync with checkpointing | `Files.Read.All` | -| [tenant-change-feed.ps1](delta-and-sync/tenant-change-feed.ps1) | A resumable tenant-wide change feed, baselined with `-Latest` | `Group.Read.All`, `Application.Read.All` | +| [tenant-inventory.ps1](change-feeds/tenant-inventory.ps1) | Snapshot every root collection `$metadata` declares, resumable | whatever read scopes you have | +| [group-change-feed.ps1](change-feeds/group-change-feed.ps1) | Baseline one resource with `-Latest`, then log only what changes | `Group.Read.All` | +| [tenant-change-feed.ps1](change-feeds/tenant-change-feed.ps1) | The same across several resources, resumable, as a scheduled job | `Group.Read.All`, `Application.Read.All` | ### Drive Content (`drive-content/`) | Script | Description | Scopes | |--------|-------------|--------| -| [partial-content-hash.ps1](drive-content/partial-content-hash.ps1) | Identify files by hashing a byte range instead of downloading them whole | `Files.Read.All` | +| [drive-delta-resumable.ps1](drive-content/drive-delta-resumable.ps1) | Resumable OneDrive/SharePoint drive enumeration to JSONL | `Files.Read.All` | +| [download-user-onedrive.ps1](drive-content/download-user-onedrive.ps1) | Download a user's whole OneDrive, preserving the folder tree | `Files.Read.All` | | [drive-content-triage.ps1](drive-content/drive-content-triage.ps1) | Sniff 4 KB of each file, then download only what survives triage | `Files.Read.All` | ### Resilience & Telemetry (`resilience-and-telemetry/`) | Script | Description | Scopes | |--------|-------------|--------| -| [resilience-for-existing-scripts.ps1](resilience-and-telemetry/resilience-for-existing-scripts.ps1) | Add retry/circuit breaker to existing SDK scripts (zero code changes) | `User.Read.All` | -| [resilience-status.ps1](resilience-and-telemetry/resilience-status.ps1) | Check, enable, and disable SDK resilience injection | `User.Read.All` | -| [telemetry.ps1](resilience-and-telemetry/telemetry.ps1) | View request counts, retries, and throttle events for the session | `User.Read.All` | -| [tune-rate-limits.ps1](resilience-and-telemetry/tune-rate-limits.ps1) | Tune rate limiter, retry count, and timeouts at runtime | `User.Read.All` | +| [resilience-status.ps1](resilience-and-telemetry/resilience-status.ps1) | Add retry and circuit breaking to existing SDK scripts, then check and remove it | `User.Read.All` | +| [progress-for-long-runs.ps1](resilience-and-telemetry/progress-for-long-runs.ps1) | A live status line for a streamed fan-out: percent, rate, ETA, resource units | `Group.Read.All`, `User.Read.All` | | [resource-unit-budgeting.ps1](resilience-and-telemetry/resource-unit-budgeting.ps1) | Measure what query shapes cost in resource units, and size a fan-out from the measurement | `Group.Read.All`, `User.Read.All` | -| [pacing-observability.ps1](resilience-and-telemetry/pacing-observability.ps1) | Watch the adaptive pacer: slow start, per-workload buckets, and which throttle signals are real | `User.Read.All`, `Files.Read.All` | - -## Cmdlet Coverage - -Every Mgx cmdlet is demonstrated in at least one script: - -| Cmdlet | Scripts | -|--------|---------| -| `Invoke-MgxRequest` | get-all-users, beta-endpoint, get-managers-fan-out, enrich-users-with-manager, chained-relation-expansion, group-members-multipage, disabled-accounts-report, guest-users-report, stale-devices-report, app-secrets-expiry, conditional-access-export, bulk-delete-whatif, partial-content-hash, drive-content-triage, telemetry, tune-rate-limits, tenant-resilience-comparison, resource-unit-budgeting, pacing-observability | -| `Invoke-MgxBatchRequest` | bulk-update, dead-letter-retry, mixed-endpoint-batch | -| `Export-MgxCollection` | export-users-to-jsonl, export-sign-in-logs, tenant-resilience-comparison | -| `Expand-MgxRelation` | enrich-users-with-manager, chained-relation-expansion | -| `Sync-MgxDelta` | delta-sync, drive-delta-resumable, tenant-change-feed | -| `Enable-MgxResilience` | resilience-for-existing-scripts, resilience-status, tenant-resilience-comparison | -| `Disable-MgxResilience` | resilience-status, tenant-resilience-comparison | -| `Get-MgxResilience` | resilience-status | -| `Set-MgxOption` | tune-rate-limits, pacing-observability | -| `Get-MgxOption` | tune-rate-limits | -| `Get-MgxTelemetry` | telemetry, tenant-resilience-comparison, resource-unit-budgeting, pacing-observability, partial-content-hash, drive-content-triage | -| `Get-MgxContent` | partial-content-hash, drive-content-triage | diff --git a/examples/bulk-writes-and-batching/assign-licenses.ps1 b/examples/bulk-writes-and-batching/assign-licenses.ps1 new file mode 100644 index 0000000..23374b1 --- /dev/null +++ b/examples/bulk-writes-and-batching/assign-licenses.ps1 @@ -0,0 +1,106 @@ +# Assign a license to every enabled user who does not have one yet. +# +# Two things have to happen and both are batched: a user with no usageLocation +# cannot be licensed at all, so those get a PATCH first, then everyone gets an +# assignLicense POST. Invoke-MgxBatchRequest bundles 20 requests per HTTP call, +# so 400 users are 20 calls rather than 400. +# +# Seats are checked against /subscribedSkus before anything is written - Graph +# will happily refuse item 300 of 400 and leave you working out which ones landed. +# +# Requirements: Connect-MgGraph -Scopes "User.ReadWrite.All", +# "Organization.Read.All", "Directory.ReadWrite.All" + +param( + [string]$SkuPartNumber = 'ENTERPRISEPACK', # Office 365 E3 + [string]$DefaultUsageLocation = 'US', + [switch]$Execute # omit for a preview +) + +Import-Module Mgx + +$sku = Invoke-MgxRequest /subscribedSkus | + Where-Object { $_.skuPartNumber -eq $SkuPartNumber } | + Select-Object -First 1 +if (-not $sku) { throw "No subscription for $SkuPartNumber in this tenant." } + +$available = $sku.prepaidUnits.enabled - $sku.consumedUnits +Write-Host ("{0}: {1} of {2} seats used, {3} free" -f ` + $SkuPartNumber, $sku.consumedUnits, $sku.prepaidUnits.enabled, $available) + +# assignedLicenses is filterable server-side, but only as an advanced query +# ($count=true plus ConsistencyLevel eventual). Checking it here keeps the +# example to one query shape and costs nothing extra on a tenant this size. +$candidates = @(Invoke-MgxRequest /users -All ` + -Filter 'accountEnabled eq true' ` + -Property id,userPrincipalName,usageLocation,assignedLicenses | + Where-Object { $_.assignedLicenses.skuId -notcontains $sku.skuId }) + +Write-Host "$($candidates.Count) user(s) without $SkuPartNumber" +if ($candidates.Count -eq 0) { return } +if ($candidates.Count -gt $available) { + throw "$($candidates.Count) users need a license but only $available seats are free." +} + +$needsLocation = @($candidates | Where-Object { -not $_.usageLocation }) +if (-not $Execute) { + Write-Host "Preview only - re-run with -Execute to write." + Write-Host " would set usageLocation on $($needsLocation.Count) user(s)" + Write-Host " would license $($candidates.Count) user(s)" + $candidates | Select-Object userPrincipalName, usageLocation | Format-Table -AutoSize + return +} + +# Stage 1: usageLocation. Batches run their items concurrently, so anything +# order-dependent has to be a separate call - which is why this is not folded +# into the assignLicense batch below. +if ($needsLocation) { + $patched = $needsLocation | ForEach-Object { + [PSCustomObject]@{ + Url = "/users/$($_.id)" + Method = 'PATCH' + Body = @{ usageLocation = $DefaultUsageLocation } + } + } | Invoke-MgxBatchRequest + Write-Host "usageLocation set: $(@($patched | Where-Object { $_.Status -lt 400 }).Count)" +} + +# Stage 2: the licenses themselves. +$assigned = $candidates | ForEach-Object { + [PSCustomObject]@{ + Url = "/users/$($_.id)/assignLicense" + Method = 'POST' + Body = @{ + addLicenses = @(@{ skuId = $sku.skuId; disabledPlans = @() }) + removeLicenses = @() + } + } +} | Invoke-MgxBatchRequest -DeadLetterPath ./license-failures.jsonl + +$ok = @($assigned | Where-Object { $_.Status -lt 400 }).Count +$fail = @($assigned | Where-Object { $_.Status -ge 400 }).Count +Write-Host "Licensed: $ok Failed: $fail" +if ($fail) { Write-Host "Failed items are in ./license-failures.jsonl" } + +<# +Expected output (preview): + +ENTERPRISEPACK: 22 of 25 seats used, 3 free +3 user(s) without ENTERPRISEPACK +Preview only - re-run with -Execute to write. + would set usageLocation on 1 user(s) + would license 3 user(s) + +userPrincipalName usageLocation +----------------- ------------- +graceb@contoso.com US +henriettam@contoso.com US +irvins@contoso.com + +Expected output (-Execute): + +ENTERPRISEPACK: 22 of 25 seats used, 3 free +3 user(s) without ENTERPRISEPACK +usageLocation set: 1 +Licensed: 3 Failed: 0 +#> diff --git a/examples/bulk-writes-and-batching/bulk-create-users-from-csv.ps1 b/examples/bulk-writes-and-batching/bulk-create-users-from-csv.ps1 new file mode 100644 index 0000000..e6de71b --- /dev/null +++ b/examples/bulk-writes-and-batching/bulk-create-users-from-csv.ps1 @@ -0,0 +1,86 @@ +# Create users from a CSV, keeping the ones that fail. +# +# A batch of 200 new hires will not be 200 clean creates: a duplicate UPN, a +# rejected password, a domain that is not verified. -DeadLetterPath writes those +# items - request and response - to a JSONL file instead of losing them in the +# console, so the run can be fixed and replayed against only what failed. +# +# The CSV is expected to have DisplayName, MailNickname, UserPrincipalName and +# Department columns: +# +# DisplayName,MailNickname,UserPrincipalName,Department +# Grace Blake,graceb,graceb@contoso.com,Retail +# +# Requirements: Connect-MgGraph -Scopes "User.ReadWrite.All" + +param( + [string]$Path = './new-hires.csv', + [string]$DeadLetterPath = './failed-users.jsonl' +) + +Import-Module Mgx + +function New-Password { + # Cryptographically random, unlike Get-Random. A real script delivers this + # out of band rather than leaving it in a variable. + $chars = 'abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789!@#$%^&*' + -join (1..16 | ForEach-Object { + $chars[[System.Security.Cryptography.RandomNumberGenerator]::GetInt32($chars.Length)] + }) +} + +$rows = Import-Csv $Path +Write-Host "$($rows.Count) row(s) in $Path" + +$requests = $rows | ForEach-Object { + [PSCustomObject]@{ + Url = '/users' + Method = 'POST' + Body = @{ + displayName = $_.DisplayName + mailNickname = $_.MailNickname + userPrincipalName = $_.UserPrincipalName + department = $_.Department + accountEnabled = $true + passwordProfile = @{ + password = New-Password + forceChangePasswordNextSignIn = $true + } + } + } +} + +$results = $requests | Invoke-MgxBatchRequest -DeadLetterPath $DeadLetterPath + +$created = @($results | Where-Object { $_.Status -lt 400 }) +$failed = @($results | Where-Object { $_.Status -ge 400 }) +Write-Host "Created: $($created.Count) Failed: $($failed.Count)" + +# What went wrong, grouped - one bad domain looks like 40 unrelated failures +# until they are collapsed like this. +if ($failed) { + $failed | + Group-Object { $_.Body.error.message } | + Select-Object Count, Name | + Format-Table -AutoSize -Wrap + Write-Host "Failed items saved to $DeadLetterPath" +} + +# Replaying: fix the CSV, or edit the dead-letter file and feed it straight back. +# Each line is a complete request, so nothing has to be rebuilt. +# +# Get-Content $DeadLetterPath | ConvertFrom-Json | Invoke-MgxBatchRequest + +<# +Expected output: + +12 row(s) in ./new-hires.csv +WARNING: 2 of 12 batch items failed after all retry attempts. Check $Error for details on each failed item. +Created: 10 Failed: 2 + +Count Name +----- ---- + 2 Another object with the same value for property userPrincipalName already exists. + +Failed items saved to ./failed-users.jsonl +#> diff --git a/examples/bulk-writes-and-batching/bulk-update.ps1 b/examples/bulk-writes-and-batching/bulk-update.ps1 deleted file mode 100644 index 8718dc5..0000000 --- a/examples/bulk-writes-and-batching/bulk-update.ps1 +++ /dev/null @@ -1,32 +0,0 @@ -# Bulk-update multiple users via $batch (20 requests per HTTP call). -# -# Instead of one PATCH per user (N HTTP calls), Invoke-MgxBatchRequest -# bundles up to 20 requests per call automatically. Throttle-aware pacing -# prevents 429s under load. -# -# This example sets the department field on a list of users. -# -# Requirements: Connect-MgGraph -Scopes "User.ReadWrite.All" - -Import-Module Mgx - -# Replace these with real user IDs from your tenant. -$userIds = @( - "00000000-0000-0000-0000-000000000001" - "00000000-0000-0000-0000-000000000002" - "00000000-0000-0000-0000-000000000003" -) - -$results = $userIds ` - | ForEach-Object { "/users/$_" } ` - | Invoke-MgxBatchRequest -Method PATCH -Body @{ department = "Engineering" } - -$ok = ($results | Where-Object { $_.Status -lt 400 }).Count -$fail = ($results | Where-Object { $_.Status -ge 400 }).Count -Write-Host "Updated: $ok Failed: $fail" - -<# -Expected output: - -Updated: 3 Failed: 0 -#> diff --git a/examples/bulk-writes-and-batching/dead-letter-retry.ps1 b/examples/bulk-writes-and-batching/dead-letter-retry.ps1 deleted file mode 100644 index cd5e14e..0000000 --- a/examples/bulk-writes-and-batching/dead-letter-retry.ps1 +++ /dev/null @@ -1,68 +0,0 @@ -# Bulk-create users with dead-letter tracking. -# -# Failed batch items are written to a JSONL file instead of silently -# dropped. Re-run the retry block to attempt them again - useful for -# production scripts where partial failures must not be lost. -# -# Requirements: Connect-MgGraph -Scopes "User.ReadWrite.All" - -Import-Module Mgx - -$deadLetterFile = "./failed-users.jsonl" -$domain = "contoso.com" # replace with your tenant domain - -# Build 50 user creation requests -$requests = 1..50 | ForEach-Object { - [PSCustomObject]@{ - Url = "/users" - Method = "POST" - Body = @{ - displayName = "Sample User $_" - mailNickname = "sampleuser$_" - userPrincipalName = "sampleuser$_@$domain" - accountEnabled = $true - passwordProfile = @{ - password = "P@ss$(Get-Random -Minimum 1000 -Maximum 9999)!" - forceChangePasswordNextSignIn = $false - } - } - } -} - -# First attempt - failed items go to dead-letter file -$results = $requests | Invoke-MgxBatchRequest -DeadLetterPath $deadLetterFile -Write-Host "Created: $(($results | Where-Object { $_.Status -lt 400 }).Count)" -Write-Host "Failed: $(($results | Where-Object { $_.Status -ge 400 }).Count)" - -# Retry from dead-letter file -if (Test-Path $deadLetterFile) { - Write-Host "Retrying failed items..." - $retry = Get-Content $deadLetterFile | ConvertFrom-Json - $retry | Invoke-MgxBatchRequest - Remove-Item $deadLetterFile -} - -<# -Expected output: - -Invoke-MgxBatchRequest: /tmp/mgx-examples/dead-letter-retry.ps1:33 -Line | - 33 | ... = $requests | Invoke-MgxBatchRequest -DeadLetterPath $deadLetterFile - | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - | POST /users: Request_BadRequest: Another object with the same value for property - | userPrincipalName already exists. -... -WARNING: 10 of 50 batch items failed after all retry attempts. Check $Error for details on each failed item. -Created: 40 -Failed: 10 -Retrying failed items... - -Name Value ----- ----- -Status 400 -Url /users -Body {[error, System.Collections.Hashtable]} -Method POST -... -WARNING: 10 of 10 batch items failed after all retry attempts. Check $Error for details on each failed item. -#> diff --git a/examples/bulk-writes-and-batching/mixed-endpoint-batch.ps1 b/examples/bulk-writes-and-batching/mixed-endpoint-batch.ps1 deleted file mode 100644 index 4c0010d..0000000 --- a/examples/bulk-writes-and-batching/mixed-endpoint-batch.ps1 +++ /dev/null @@ -1,61 +0,0 @@ -# Query multiple Graph endpoints in a single HTTP round-trip. -# -# Invoke-MgxBatchRequest accepts mixed URLs, methods, and bodies -# in one batch. This avoids serial requests when you need data -# from unrelated endpoints (users, groups, apps, skus, etc.). -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" - -Import-Module Mgx - -# Mixed GET: pull a snapshot from four different endpoints at once -$results = @( - "/users?`$top=5&`$select=displayName,userPrincipalName" - "/groups?`$top=5&`$select=displayName,groupTypes" - "/applications?`$top=5&`$select=displayName,appId" - "/subscribedSkus" -) | Invoke-MgxBatchRequest - -Write-Host "Batch returned $($results.Count) responses" -# A batch result is a Hashtable of Url / Method / Status / Body - the entity itself lives under -# Body. Reading $_.displayName straight off the result finds nothing, and a Hashtable returns -# $null for a missing key rather than erroring, so the mistake prints a fallback string forever -# instead of failing loudly. -$results | ForEach-Object { - $count = if ($_.Body -and $_.Body.value) { @($_.Body.value).Count } else { 0 } - Write-Host (" {0,-3} {1,-46} {2} item(s)" -f $_.Status, $_.Url, $count) -} - -# Mixed methods: read some things from different endpoints, all in one call -$requests = @( - [PSCustomObject]@{ Url = "/me"; Method = "GET" } - [PSCustomObject]@{ Url = "/organization"; Method = "GET" } - [PSCustomObject]@{ Url = "/users?`$top=1"; Method = "GET" } -) -$mixed = $requests | Invoke-MgxBatchRequest - -Write-Host "`nMixed-method batch:" -$mixed | ForEach-Object { - # Single-entity responses put the object directly in Body; collections nest it under - # Body.value. Checking for value distinguishes the two without guessing per endpoint. - $summary = if ($_.Body -and $_.Body.value) { "$(@($_.Body.value).Count) item(s)" } - elseif ($_.Body -and $_.Body.displayName) { $_.Body.displayName } - elseif ($_.Body -and $_.Body.id) { $_.Body.id } - else { '(no body)' } - Write-Host (" {0,-6} {1,-3} {2,-24} {3}" -f $_.Method, $_.Status, $_.Url, $summary) -} - -<# -Expected output: - -Batch returned 4 responses - 200 /users?$top=5&$select=displayName,userPrincipalName 5 item(s) - 200 /groups?$top=5&$select=displayName,groupTypes 5 item(s) - 200 /applications?$top=5&$select=displayName,appId 5 item(s) - 200 /subscribedSkus 3 item(s) - -Mixed-method batch: - GET 200 /me MOD Administrator - GET 200 /organization 1 item(s) - GET 200 /users?$top=1 1 item(s) -#> diff --git a/examples/bulk-writes-and-batching/offboard-users.ps1 b/examples/bulk-writes-and-batching/offboard-users.ps1 new file mode 100644 index 0000000..a96d1a0 --- /dev/null +++ b/examples/bulk-writes-and-batching/offboard-users.ps1 @@ -0,0 +1,83 @@ +# Offboard departed users: block sign-in, kill live sessions, strip licenses. +# +# Three writes per user, and the order is not optional - blocking sign-in does +# nothing about a session that is already open, so the revoke has to follow it. +# A batch runs its items concurrently, so ordering means three batched stages +# rather than one batch of three requests per user. +# +# Stage 3 reads back each user's licenses first: assignLicense needs the skuIds +# to remove, and the account is already blocked by then, so nothing can change +# underneath it. +# +# Requirements: Connect-MgGraph -Scopes "User.ReadWrite.All", "Directory.ReadWrite.All" + +param( + [string[]]$UserPrincipalName, + [string]$Path, # or a CSV with a UserPrincipalName column + [switch]$Execute +) + +Import-Module Mgx + +$upns = if ($Path) { (Import-Csv $Path).UserPrincipalName } else { $UserPrincipalName } +if (-not $upns) { throw 'Nothing to offboard: pass -UserPrincipalName or -Path.' } + +# Resolve first: a typo in the list should surface here, not halfway through +# stage 2 with some accounts already blocked. +# {id} accepts bare strings as well as objects with an id, so UPNs pipe straight in. +$users = @($upns | Invoke-MgxRequest '/users/{id}' -SkipNotFound ` + -Property id,displayName,userPrincipalName,accountEnabled,assignedLicenses) + +Write-Host "$($users.Count) of $(@($upns).Count) account(s) resolved" +$users | Select-Object userPrincipalName, displayName, accountEnabled | Format-Table -AutoSize + +if (-not $Execute) { + Write-Host "Preview only - re-run with -Execute to write." + return +} + +# Stage 1: block sign-in. +$blocked = $users | ForEach-Object { + [PSCustomObject]@{ Url = "/users/$($_.id)"; Method = 'PATCH'; Body = @{ accountEnabled = $false } } +} | Invoke-MgxBatchRequest +Write-Host "blocked : $(@($blocked | Where-Object { $_.Status -lt 400 }).Count)" + +# Stage 2: revoke refresh tokens, ending sessions that are already open. +$revoked = $users | ForEach-Object { + [PSCustomObject]@{ Url = "/users/$($_.id)/revokeSignInSessions"; Method = 'POST' } +} | Invoke-MgxBatchRequest +Write-Host "revoked : $(@($revoked | Where-Object { $_.Status -lt 400 }).Count)" + +# Stage 3: return the licenses to the pool. Users with none are skipped - an +# empty removeLicenses array is a wasted request, not a no-op. +$licensed = @($users | Where-Object { $_.assignedLicenses.Count -gt 0 }) +if ($licensed) { + $stripped = $licensed | ForEach-Object { + [PSCustomObject]@{ + Url = "/users/$($_.id)/assignLicense" + Method = 'POST' + Body = @{ addLicenses = @(); removeLicenses = @($_.assignedLicenses.skuId) } + } + } | Invoke-MgxBatchRequest -DeadLetterPath ./offboard-failures.jsonl + Write-Host "unlicensed: $(@($stripped | Where-Object { $_.Status -lt 400 }).Count) of $($licensed.Count)" +} + +# Deliberately not done here: deleting the account. A blocked, unlicensed user +# keeps their OneDrive and mailbox reachable for whoever inherits the work - +# download-user-onedrive.ps1 is the other half of that. + +<# +Expected output: + +3 of 3 account(s) resolved + +userPrincipalName displayName accountEnabled +----------------- ----------- -------------- +graceb@contoso.com Grace Blake True +henriettam@contoso.com Henrietta Mueller True +irvins@contoso.com Irvin Sayers True + +blocked : 3 +revoked : 3 +unlicensed: 2 of 2 +#> diff --git a/examples/bulk-writes-and-batching/tenant-snapshot.ps1 b/examples/bulk-writes-and-batching/tenant-snapshot.ps1 new file mode 100644 index 0000000..3e9ebcb --- /dev/null +++ b/examples/bulk-writes-and-batching/tenant-snapshot.ps1 @@ -0,0 +1,53 @@ +# A one-call snapshot of the tenant: users, groups, apps, and licenses. +# +# Four unrelated endpoints in one HTTP round trip instead of four. Useful as the +# opening move of a script that needs a bit of everything, and as the thing to +# run when someone asks how big the tenant is. +# +# Requirements: Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" + +Import-Module Mgx + +# $count=true returns the total in @odata.count without paging anything, and +# -ConsistencyLevel rides along on each subrequest, which Graph requires for it. +$counts = @( + "/users?`$top=1&`$count=true" + "/groups?`$top=1&`$count=true" + "/applications?`$top=1&`$count=true" + "/servicePrincipals?`$top=1&`$count=true" +) | Invoke-MgxBatchRequest -ConsistencyLevel eventual + +Write-Host "`n=== Tenant size ===" -ForegroundColor Cyan +$counts | ForEach-Object { + " {0,-20} {1,6}" -f ($_.Url -replace '\?.*$', ''), $_.Body.'@odata.count' +} + +$orgUri = "/organization?`$select=displayName,verifiedDomains" +$samples = @($orgUri, '/subscribedSkus') | Invoke-MgxBatchRequest + +# A batch result is Url / Method / Status / Body - the entity itself lives under +# Body, and a collection nests it one deeper under Body.value. Results come back +# keyed by the URL that produced them rather than in the order they were sent. +$byUrl = @{} +foreach ($r in $samples) { $byUrl[$r.Url] = $r.Body } + +$tenant = $byUrl[$orgUri].value[0] +$skus = $byUrl['/subscribedSkus'].value + +Write-Host "`n=== $($tenant.displayName) ===" -ForegroundColor Cyan +" domains : $(($tenant.verifiedDomains.name) -join ', ')" +" licenses: $(($skus | ForEach-Object { '{0} {1}/{2}' -f $_.skuPartNumber, $_.consumedUnits, $_.prepaidUnits.enabled }) -join ' ')" + +<# +Expected output: + +=== Tenant size === + /users 28 + /groups 12 + /applications 9 + /servicePrincipals 47 + +=== Contoso Ltd === + domains : contoso.onmicrosoft.com, contoso.com + licenses: ENTERPRISEPACK 22/25 EMSPREMIUM 4/10 POWER_BI_STANDARD 7/100 +#> diff --git a/examples/change-feeds/group-change-feed.ps1 b/examples/change-feeds/group-change-feed.ps1 new file mode 100644 index 0000000..fef569e --- /dev/null +++ b/examples/change-feeds/group-change-feed.ps1 @@ -0,0 +1,50 @@ +# Watch one resource for changes: baseline now, then get only what changed. +# +# This is the smallest useful change feed, and the thing to read before +# tenant-change-feed.ps1, which does the same across several resources at once. +# +# Two ideas: +# * -Latest baselines the delta token from now, instantly, without enumerating +# the resource first. A change feed does not want the initial dump - it wants +# "what happened since I started watching". +# * Delta state is a file. Point later runs at the same -DeltaPath and they pick +# up exactly where the last one stopped, which makes this a cron job. +# +# Requirements: Connect-MgGraph -Scopes "Group.Read.All" + +Import-Module Mgx + +$deltaFile = "./groups.delta" +$feedFile = "./groups-feed.jsonl" + +# First run only: start the feed from now instead of enumerating every group. +if (-not (Test-Path $deltaFile)) { + Write-Host "First run: baselining from now (no initial enumeration)." + $null = Sync-MgxDelta /groups/delta -DeltaPath $deltaFile -Latest + Write-Host "Baselined. Change a group, then run this again." + return +} + +$changes = Sync-MgxDelta /groups/delta -DeltaPath $deltaFile + +foreach ($group in $changes) { + $group | ConvertTo-Json -Compress -Depth 10 | Add-Content $feedFile +} + +Write-Host "$($changes.Count) change(s) since the last run, appended to $feedFile" +$changes | + Select-Object @{ Name = 'Change'; Expression = { if ($_.'@removed') { 'removed' } else { 'changed' } } }, + id, displayName | + Format-Table -AutoSize + +<# +Expected output (second run): + +3 change(s) since the last run, appended to ./groups-feed.jsonl + +Change id displayName +------ -- ----------- +changed 5cf66b99-8a06-436a-8107-644f1a1c4b3e Mark 8 Project Team +changed 24ffeed6-38b4-4bee-ae29-f0d18b5b7a92 Engineering All Hands +removed 8b1f2c04-19ac-4c0b-9a58-7f5e4c1d2a11 +#> diff --git a/examples/delta-and-sync/tenant-change-feed.ps1 b/examples/change-feeds/tenant-change-feed.ps1 similarity index 90% rename from examples/delta-and-sync/tenant-change-feed.ps1 rename to examples/change-feeds/tenant-change-feed.ps1 index 9cb4756..afa57d8 100644 --- a/examples/delta-and-sync/tenant-change-feed.ps1 +++ b/examples/change-feeds/tenant-change-feed.ps1 @@ -1,5 +1,13 @@ # A tenant change feed: everything that changed since the last run, resumable. # +# group-change-feed.ps1 does this for a single resource and is the shorter read; +# this is the same idea across several resources, with the details that show up +# at tenant scale - checkpointing and duplicate collapsing. +# +# A feed baselined with -Latest deliberately skips what existed before it +# started. If you want that too, tenant-inventory.ps1 is the snapshot half: +# inventory once, then this feed keeps it current. +# # Baseline once with -Latest (instant - no initial enumeration), then each # scheduled run appends only the changes to a JSONL log. Delta state makes # it incremental; -CheckpointPath makes a killed run continue instead of diff --git a/examples/change-feeds/tenant-inventory.ps1 b/examples/change-feeds/tenant-inventory.ps1 new file mode 100644 index 0000000..7c6925d --- /dev/null +++ b/examples/change-feeds/tenant-inventory.ps1 @@ -0,0 +1,123 @@ +# Everything the tenant has: every root collection, each to its own JSONL. +# +# Graph has no /* - but $metadata is the machine-readable list of every root +# collection the API serves (41 of them on v1.0 today), and it needs no token. +# So "grab everything" is: ask the schema for the collections, export each one, +# and treat refusals as answers rather than failures. +# +# Re-running resumes. A set with a pending checkpoint continues mid-enumeration, +# a set that already finished is skipped, so a killed overnight run picks up +# where it died instead of starting over. Delete the output directory to redo. +# +# This is the snapshot half of a pair: run it once, then tenant-change-feed.ps1 +# appends what changes afterwards. Snapshot plus feed is the tenant as of now, +# without ever enumerating it a second time. +# +# Requirements: whatever read scopes you have. Directory.Read.All covers most of +# the directory tier; each extra scope (Sites.Read.All, Policy.Read.All, ...) +# unlocks more sets. A denied set is reported, not fatal. + +param( + [string]$OutputDirectory = './tenant-inventory', + # directoryObjects is the union of users, groups, service principals, and + # devices - exporting it re-transfers what other sets already cover. + [string[]]$Skip = @('directoryObjects') +) + +Import-Module Mgx + +New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null + +# The schema is the wildcard: every EntitySet is a root collection. +$csdl = Invoke-RestMethod 'https://graph.microsoft.com/v1.0/$metadata' +$sets = @($csdl.Edmx.DataServices.Schema.EntityContainer.EntitySet.Name | + Where-Object { $_ -notin $Skip } | Sort-Object) + +Write-Host "$($sets.Count) root collections declared by `$metadata" + +$results = foreach ($set in $sets) { + $outFile = Join-Path $OutputDirectory "$set.jsonl" + $checkpoint = Join-Path $OutputDirectory "$set.checkpoint" + + # Finished on a previous run: output present, no checkpoint pending. + if ((Test-Path $outFile) -and -not (Test-Path $checkpoint)) { + [pscustomobject]@{ Set = $set; Items = $null; Outcome = 'done (previous run)' } + continue + } + + try { + # Empty sets are normal here, so the 0-item warning (which suspects a + # single-entity URI) is suppressed - it does not apply to this workload. + $r = Export-MgxCollection "/$set" -All ` + -OutputFile $outFile -CheckpointPath $checkpoint ` + -WarningAction SilentlyContinue -ErrorAction Stop + [pscustomobject]@{ Set = $set; Items = $r.ItemCount; Outcome = 'exported' } + } + catch { + $msg = ($_.Exception.Message -split "`n")[0] + $outcome = switch -Regex ($msg) { + 'Authorization_RequestDenied|Forbidden' { 'denied - scope not granted' } + 'BadRequest|NotImplemented|not support' { 'not enumerable at the root' } + default { $msg } + } + [pscustomobject]@{ Set = $set; Items = $null; Outcome = $outcome } + } +} + +$results | Format-Table Set, Items, Outcome -AutoSize + +$exported = @($results | Where-Object Outcome -eq 'exported') +$totalItems = ($exported.Items | Measure-Object -Sum).Sum +$ru = (Get-MgxTelemetry).ResourceUnitsConsumed +Write-Host ("{0} of {1} sets exported, {2:N0} objects, {3:N0} resource units" -f ` + $exported.Count, $sets.Count, $totalItems, $ru) + +# Reading the outcomes: an exported set with 0 items is an answer - the +# collection exists and is empty. Only a denial says you cannot know. And the +# refusals are by design, not gaps: chats needs user context, shares needs +# sharing tokens, drives is enumerated per site or user, not tenant-wide. +# +# Properties: each object carries its type's DEFAULT property set. Deliberate - +# naming every declared property makes queries fragile (signInActivity, for +# one, fails the whole request without the license to read it). The full +# per-type property list is in the same $metadata if a set is worth the tuning. +# +# The other half of "everything" is the 31 singletons ($csdl ... .Singleton): +# auditLogs, deviceManagement, reports, security. Those are not collections - +# the interesting data hangs off them as navigation properties, each with its +# own scope, and that is a crawler rather than a loop. + +<# +Expected output: + +40 root collections declared by $metadata + +Set Items Outcome +--- ----- ------- +agreementAcceptances 0 exported +agreements 0 exported +applications 9 exported +applicationTemplates 1743 exported +appRoleAssignments not enumerable at the root +certificateBasedAuthConfiguration 0 exported +chats not enumerable at the root +connections 0 exported +contacts 0 exported +contracts 0 exported +devices 5 exported +directoryRoles 14 exported +domains 2 exported +groups 12 exported +oauth2PermissionGrants 19 exported +organization 1 exported +places 0 exported +servicePrincipals 47 exported +shares not enumerable at the root +sites denied - scope not granted +subscribedSkus 3 exported +teams 12 exported +users 28 exported +... + +34 of 40 sets exported, 1,934 objects, 78 resource units +#> diff --git a/examples/drive-content/download-user-onedrive.ps1 b/examples/drive-content/download-user-onedrive.ps1 new file mode 100644 index 0000000..95eb191 --- /dev/null +++ b/examples/drive-content/download-user-onedrive.ps1 @@ -0,0 +1,97 @@ +# Download a user's whole OneDrive to a local folder, preserving the tree. +# +# The enumeration is a delta call, which is the cheapest way to walk a drive: +# one paged sweep instead of a recursive /children crawl per folder. Each item +# carries parentReference.path, which is what rebuilds the directory structure +# locally. +# +# Downloads go through -OutFile, so bytes stream to disk and never sit in the +# pipeline. Where Graph gave the item a @microsoft.graph.downloadUrl, piping the +# item uses that pre-authenticated URL directly: no Graph round trip for the +# bytes, no resource units, and no bearer token sent to the download host. +# +# Re-running skips files whose local size already matches, so an interrupted run +# continues rather than starting over. +# +# Requirements: Connect-MgGraph -Scopes "Files.Read.All" (delegated: Files.Read +# for your own drive). Reading another user's drive needs the +# application permission Files.Read.All. + +param( + [Parameter(Mandatory)][string]$User, # UPN or object id + [string]$Destination = './onedrive-backup' +) + +Import-Module Mgx + +$root = Join-Path $Destination $User +New-Item -ItemType Directory -Path $root -Force | Out-Null + +$items = Sync-MgxDelta "/users/$User/drive/root/delta" ` + -DeltaPath (Join-Path $root '.delta.json') ` + -CheckpointPath (Join-Path $root '.delta.checkpoint') + +# Folders and deletions drop out here; a removed item arrives as @removed, or as a +# 'deleted' facet if the call asked for -Prefer deltashowremovedasdeleted. +$files = @($items | Where-Object { $_.file -and -not $_.deleted -and -not $_.'@removed' }) +Write-Host "$($files.Count) file(s) to consider under $User's drive" + +$downloaded = 0 +$skipped = 0 +foreach ($item in $files) { + # parentReference.path looks like "/drive/root:/Documents/Reports" - everything + # after the colon is the folder path inside the drive. + $relative = '' + if ($item.parentReference.path -match ':(?

.*)$') { + $relative = $Matches.p.TrimStart('/') + } + $targetDir = if ($relative) { Join-Path $root $relative } else { $root } + New-Item -ItemType Directory -Path $targetDir -Force | Out-Null + $target = Join-Path $targetDir $item.name + + # Size is the cheap check. For a real integrity check compare against + # $item.file.hashes.quickXorHash - Graph reports it in base64 where rclone + # and most other tools print hex, so convert before comparing: + # [System.Convert]::ToHexString([System.Convert]::FromBase64String($h)).ToLowerInvariant() + if ((Test-Path $target) -and (Get-Item $target).Length -eq $item.size) { + $skipped++ + continue + } + + try { + if ($item['@microsoft.graph.downloadUrl']) { + $item | Get-MgxContent -OutFile $target -ErrorAction Stop + } + else { + # No downloadUrl on this item (delta does not always include one) - + # fetch through Graph instead. + Get-MgxContent "/users/$User/drive/items/$($item.id)/content" ` + -OutFile $target -ErrorAction Stop + } + $downloaded++ + } + catch { + Write-Warning "$($item.name): $($_.Exception.Message -replace '\s+', ' ')" + } +} + +$t = Get-MgxTelemetry +Write-Host ("downloaded {0}, already current {1}, {2:N1} MB transferred" -f ` + $downloaded, $skipped, ($t.ContentBytesDownloaded / 1MB)) + +# The delta token is kept next to the files, so the next run enumerates only what +# changed and downloads only that. To back up every mailbox-sized drive in the +# tenant, loop over /users and call this once per user. + +<# +Expected output: + +11 file(s) to consider under adelev@contoso.com's drive +WARNING: Locked-Budget.xlsx: The resource you are attempting to access is locked +downloaded 10, already current 0, 4.7 MB transferred + +onedrive-backup/adelev@contoso.com/ + Documents/Q4-Revenue-Report.pdf + Documents/Reports/Meeting-Notes.txt + Team-Photo.png +#> diff --git a/examples/drive-content/drive-content-triage.ps1 b/examples/drive-content/drive-content-triage.ps1 index debb7d4..9119272 100644 --- a/examples/drive-content/drive-content-triage.ps1 +++ b/examples/drive-content/drive-content-triage.ps1 @@ -1,16 +1,8 @@ -# Triage a drive by reading 4 KB of each file, then download only what survives triage. +# Triage a drive by reading 4 KB of each file, then download only what survives. # -# -# Two properties of Get-MgxContent make this cheap and safe: -# -# * A ranged read is a Range request. -First 4096 transfers 4 KB, not 4 KB of a 2 GB download -# you abandon. Some endpoints ignore Range and answer 200 with the whole body (profile -# photos do); mgx detects that and truncates client-side, so you never receive more than you -# asked for either way. -# * Piping a DriveItem uses its own @microsoft.graph.downloadUrl - a pre-authenticated URL - -# so there is no Graph round trip at all for the bytes. No resource units, and no bearer -# token sent to the download host. That URL is itself a credential for the file, which is -# why mgx redacts it from -Debug output. +# -First 4096 is a Range request: it transfers 4 KB, not 4 KB of a 2 GB download +# you abandon. Enough to read magic bytes and decide, on a whole drive, for the +# cost of downloading one mid-sized file. # # Requirements: Connect-MgGraph -Scopes "Files.Read.All" # Read-only against Graph. Writes only to the local -OutputDirectory. diff --git a/examples/delta-and-sync/drive-delta-resumable.ps1 b/examples/drive-content/drive-delta-resumable.ps1 similarity index 100% rename from examples/delta-and-sync/drive-delta-resumable.ps1 rename to examples/drive-content/drive-delta-resumable.ps1 diff --git a/examples/drive-content/partial-content-hash.ps1 b/examples/drive-content/partial-content-hash.ps1 deleted file mode 100644 index f96e9d4..0000000 --- a/examples/drive-content/partial-content-hash.ps1 +++ /dev/null @@ -1,48 +0,0 @@ -# Identify files by content WITHOUT downloading them: ranged reads. -# -# Get-MgxContent: a metadata-extraction job -# over 20k drive files needed only each file's first 256 KB (magic bytes, -# EXIF, partial hash) - ranged reads moved ~5 GB instead of 63 GB. -# -# Requirements: Connect-MgGraph -Scopes "Files.Read.All" - -Import-Module Mgx - -$folderId = "01FOLDERID" # a folder's driveItem id - -Invoke-MgxRequest "/me/drive/items/$folderId/children" -All | - Where-Object { $_.file } | - ForEach-Object { - $head = $_ | Get-MgxContent -First 262144 - - [PSCustomObject]@{ - Name = $_.name - Size = $_.size - Magic = [BitConverter]::ToString($head[0..([Math]::Min(7, $head.Length - 1))]) - PartialSha = [BitConverter]::ToString( - [System.Security.Cryptography.SHA256]::HashData($head) - ).Replace('-', '').ToLowerInvariant() - # Graph reports quickXorHash in BASE64; rclone and most tools print - # HEX. Convert before comparing across tools: - QuickXorHex = if ($_.file.hashes.quickXorHash) { - [System.Convert]::ToHexString( - [System.Convert]::FromBase64String($_.file.hashes.quickXorHash) - ).ToLowerInvariant() - } - } - } | Format-Table -AutoSize - -# Bytes moved vs. bytes identified: -(Get-MgxTelemetry).ContentBytesDownloaded - -<# -Expected output: - -Name Size Magic PartialSha QuickXorHex ----- ---- ----- ---------- ----------- -Employee-Handbook.pdf 1204224 25-50-44-46-2D-31-2E-37 fcdec4adc77794abd24402810879c26beaee6f95f515b42be79fc032e69b6f5c e3bc4fcdfa266aadc7e2148a472720e3877b5de6 -README.txt 5120 61-62-63-64-65-66-67-68 a31ba217553a0e1b431276ab8ee0d0df43b958686c77041ec6d2eb859943e02d f9e21c1a070d2467204274dcec5d93d20613645d -Team-Photo.png 742912 89-50-4E-47-0D-0A-1A-0A bc3891906da735372e276ea14e9edcf82ee78e402a19ad70872b2b410d7fb2bf 51c7556a48958d776b8ff3ac36d2b5a5e8071fec - -529408 -#> diff --git a/examples/fan-out-and-relations/chained-relation-expansion.ps1 b/examples/fan-out-and-relations/chained-relation-expansion.ps1 deleted file mode 100644 index d9bdbf1..0000000 --- a/examples/fan-out-and-relations/chained-relation-expansion.ps1 +++ /dev/null @@ -1,34 +0,0 @@ -# Chain multiple Expand-MgxRelation stages to build rich objects in one pass. -# -# Each stage concurrently fetches a different relation for every piped object. -# The result has the original properties plus Manager and Licenses -# attached as nested objects, all fetched in parallel across users. -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All" - -Import-Module Mgx - -$enriched = Invoke-MgxRequest /users -Top 20 -Property id,displayName,mail | - Expand-MgxRelation '/users/{id}/manager' -As Manager -Flatten -SkipNotFound | - Expand-MgxRelation '/users/{id}/licenseDetails' -As Licenses -SkipNotFound - -foreach ($user in $enriched) { - $mgr = $user.Manager.displayName ?? "(none)" - $licenses = ($user.Licenses | Select-Object -ExpandProperty skuPartNumber) -join ", " - if (-not $licenses) { $licenses = "(none)" } - Write-Host "$($user.displayName) | Manager: $mgr | Licenses: $licenses" -} - -<# -Expected output: - -Adele Vance | Manager: Nestor Wilke | Licenses: SPE_E5 -Alex Wilber | Manager: Megan Bowen | Licenses: SPE_E5 -Allan Deyoung | Manager: Nestor Wilke | Licenses: SPE_E5, EMSPREMIUM -Brian Johnson | Manager: Lee Gu | Licenses: SPE_E3 -Cameron White | Manager: Adele Vance | Licenses: SPE_E3 -Christie Cline | Manager: Nestor Wilke | Licenses: SPE_E5 -Debra Berger | Manager: Nestor Wilke | Licenses: SPE_E5 -Delia Dennis | Manager: Allan Deyoung | Licenses: SPE_E5, EMSPREMIUM -... -#> diff --git a/examples/fan-out-and-relations/enrich-users-with-manager.ps1 b/examples/fan-out-and-relations/enrich-users-with-manager.ps1 deleted file mode 100644 index 29a648a..0000000 --- a/examples/fan-out-and-relations/enrich-users-with-manager.ps1 +++ /dev/null @@ -1,32 +0,0 @@ -# Enrich a list of users with their manager's display name. -# -# Expand-MgxRelation concurrently fetches the related data for each -# piped object and attaches it as a new property. This replaces a -# manual foreach loop with N serial HTTP calls. -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All" - -Import-Module Mgx - -Invoke-MgxRequest /users -Top 50 -Property id,displayName,mail | - Expand-MgxRelation '/users/{id}/manager' -As Manager -Flatten -SkipNotFound | - Select-Object displayName, mail, @{ n='Manager'; e={ $_.Manager.displayName } } | - Format-Table -AutoSize - -<# -Expected output: - -WARNING: Skipped 4 entities due to 404 (Not Found) responses. - -displayName mail Manager ------------ ---- ------- -Adele Vance adelev@contoso.com Nestor Wilke -Alex Wilber alexw@contoso.com Megan Bowen -Allan Deyoung allande@contoso.com Nestor Wilke -Brian Johnson brianj@contoso.com Lee Gu -Cameron White cameronw@contoso.com Adele Vance -Christie Cline christiec@contoso.com Nestor Wilke -Debra Berger debrab@contoso.com Nestor Wilke -Delia Dennis deliad@contoso.com Allan Deyoung -... -#> diff --git a/examples/fan-out-and-relations/get-managers-fan-out.ps1 b/examples/fan-out-and-relations/get-managers-fan-out.ps1 deleted file mode 100644 index 7fdefd8..0000000 --- a/examples/fan-out-and-relations/get-managers-fan-out.ps1 +++ /dev/null @@ -1,36 +0,0 @@ -# Get the manager for every user in the tenant using concurrent fan-out. -# -# The naive approach is a foreach loop: one HTTP call per user. -# With {id} template substitution, Invoke-MgxRequest dispatches all -# requests concurrently (default concurrency: 10) and streams results -# back as they complete. -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All" - -Import-Module Mgx - -# Get all users first, then fan-out to fetch each manager concurrently. -$users = Invoke-MgxRequest /users -All -Property id,displayName - -$users | - Invoke-MgxRequest '/users/{id}/manager' -SkipNotFound | - Select-Object id, displayName | - Format-Table -AutoSize - -<# -Expected output: - -WARNING: This endpoint may only be available in beta. Retry with -ApiVersion beta. -WARNING: Skipped 4 entities due to 404 (Not Found) responses. -id displayName --- ----------- -6426ded0-36f8-4bf7-9fe4-1353bd002dbe Nestor Wilke -2d871c04-0885-491c-bcbb-f14ab6f4a375 Megan Bowen -6426ded0-36f8-4bf7-9fe4-1353bd002dbe Nestor Wilke -c5151c70-bc16-4934-acf2-1a776694b6af Lee Gu -05c76d47-f457-4be9-bf0a-a75b57af73c1 Adele Vance -6426ded0-36f8-4bf7-9fe4-1353bd002dbe Nestor Wilke -6426ded0-36f8-4bf7-9fe4-1353bd002dbe Nestor Wilke -02109083-4f10-4e5e-ab8b-17ec81caa674 Allan Deyoung -... -#> diff --git a/examples/fan-out-and-relations/group-members-multipage.ps1 b/examples/fan-out-and-relations/group-members-multipage.ps1 deleted file mode 100644 index 732c6e2..0000000 --- a/examples/fan-out-and-relations/group-members-multipage.ps1 +++ /dev/null @@ -1,26 +0,0 @@ -# Get members of multiple groups concurrently, handling pagination per group. -# -# Each group can have hundreds of pages of members. Fan-out with -All -# handles pagination independently per group - the pipeline receives -# members from all groups interleaved as pages complete. -# -# Requirements: Connect-MgGraph -Scopes "Group.Read.All", "User.Read.All" - -Import-Module Mgx - -# Get all groups, then stream all members from all groups concurrently -$allMembers = Invoke-MgxRequest /groups -All -Property id,displayName | - Invoke-MgxRequest '/groups/{id}/members' -All -SkipNotFound -SkipForbidden - -Write-Host "Total members across all groups: $($allMembers.Count)" -$allMembers | Group-Object '@odata.type' | Format-Table Name, Count -AutoSize - -<# -Expected output: - -Total members across all groups: 30 - -Name Count ----- ----- -#microsoft.graph.user 30 -#> diff --git a/examples/getting-started/beta-endpoint.ps1 b/examples/getting-started/beta-endpoint.ps1 deleted file mode 100644 index 3ad801f..0000000 --- a/examples/getting-started/beta-endpoint.ps1 +++ /dev/null @@ -1,34 +0,0 @@ -# Access beta Graph endpoints without installing Microsoft.Graph.Beta.*. -# -# The -ApiVersion parameter switches between v1.0 (default) and beta -# on the same cmdlet, with the same resilience and pagination support. -# No extra module installs needed. -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All" - -Import-Module Mgx - -# Get users with beta-only properties (e.g. customSecurityAttributes) -$users = Invoke-MgxRequest /users ` - -ApiVersion beta ` - -Top 10 ` - -Property id,displayName,customSecurityAttributes - -$users | Select-Object id, displayName | Format-Table -AutoSize - -<# -Expected output: - -id displayName --- ----------- -05c76d47-f457-4be9-bf0a-a75b57af73c1 Adele Vance -b92d487c-5d51-460b-963a-c395ddc747ba Alex Wilber -02109083-4f10-4e5e-ab8b-17ec81caa674 Allan Deyoung -aea6b38c-19f7-4225-adb4-feeeea0b4f20 Brian Johnson -d1e24810-61ae-41fb-a250-88e2672cb4e1 Cameron White -27640233-6c77-4337-b2f9-693608bfd60f Christie Cline -af9691ee-cc4e-4748-8646-4605b2921a9c Debra Berger -29d7068e-0922-41e9-aa7a-7e97f477f67b Delia Dennis -f736ca7a-71d9-4708-8aae-3cfa2111a106 Diego Siciliani -26f05c3d-0bcd-4160-9789-cbea7b676305 Emily Braun -#> diff --git a/examples/delta-and-sync/delta-sync.ps1 b/examples/getting-started/delta-users-sync.ps1 similarity index 100% rename from examples/delta-and-sync/delta-sync.ps1 rename to examples/getting-started/delta-users-sync.ps1 diff --git a/examples/getting-started/export-users-to-csv.ps1 b/examples/getting-started/export-users-to-csv.ps1 new file mode 100644 index 0000000..2c06617 --- /dev/null +++ b/examples/getting-started/export-users-to-csv.ps1 @@ -0,0 +1,44 @@ +# Export all users to a CSV file. +# +# Export-MgxCollection writes JSONL because Graph objects are nested and JSONL +# survives that without losing anything. CSV is flat, so it needs a projection: +# name the columns you want and let Export-Csv stream them to disk as pages +# arrive. Memory stays flat - nothing accumulates in a variable first. +# +# Nested values (assignedLicenses, businessPhones, manager) need flattening by +# hand, as below, or they land in the file as type names. +# +# Requirements: Connect-MgGraph -Scopes "User.Read.All" + +Import-Module Mgx + +$outputFile = "./users.csv" + +$count = 0 +Invoke-MgxRequest /users -All ` + -Property id,displayName,mail,department,jobTitle,accountEnabled,businessPhones | + ForEach-Object { $count++; $_ } | + Select-Object id, displayName, mail, department, jobTitle, accountEnabled, + @{ Name = 'businessPhones'; Expression = { $_.businessPhones -join ';' } } | + Export-Csv -Path $outputFile -NoTypeInformation -Encoding utf8 + +Write-Host "Exported $count users to $outputFile" + +# Excel opens UTF-8 CSV correctly only when it starts with a BOM. If the file is +# for Excel rather than for another script, use -Encoding utf8BOM instead. +# +# For a resumable export over a large tenant, write JSONL with +# Export-MgxCollection -CheckpointPath (see export-users-to-jsonl.ps1) and +# convert afterwards: +# Get-Content ./users.jsonl | ConvertFrom-Json | Export-Csv ./users.csv -NoTypeInformation + +<# +Expected output: + +Exported 28 users to ./users.csv + +users.csv (first 3 lines): +"id","displayName","mail","department","jobTitle","accountEnabled","businessPhones" +"05c76d47-f457-4be9-bf0a-a75b57af73c1","Adele Vance","adelev@contoso.com","Retail","Retail Manager","True","+1 425 555 0109" +"b92d487c-5d51-460b-963a-c395ddc747ba","Alex Wilber","alexw@contoso.com","Marketing","Marketing Assistant","True","+1 858 555 0110" +#> diff --git a/examples/reporting-and-audit/conditional-access-export.ps1 b/examples/reporting-and-audit/conditional-access-export.ps1 deleted file mode 100644 index b6ce2e6..0000000 --- a/examples/reporting-and-audit/conditional-access-export.ps1 +++ /dev/null @@ -1,45 +0,0 @@ -# Export all Conditional Access policies to JSON. -# -# CA policies are only available on the beta endpoint. -ApiVersion beta -# accesses them without installing any extra modules. -# -# Requirements: Connect-MgGraph -Scopes "Policy.Read.All" - -Import-Module Mgx - -$policies = Invoke-MgxRequest /identity/conditionalAccess/policies ` - -All ` - -ApiVersion beta ` - -Property id,displayName,state,createdDateTime,modifiedDateTime - -Write-Host "Conditional Access policies: $($policies.Count)" -$policies | Sort-Object displayName | Select-Object displayName, state, modifiedDateTime | Format-Table -AutoSize - -# Export full policy details to JSON for backup/audit -$policies | ConvertTo-Json -Depth 10 | Out-File "./conditional-access-policies.json" -Write-Host "Full export saved to conditional-access-policies.json" - -<# -Expected output: - -Conditional Access policies: 4 - -displayName state modifiedDateTime ------------ ----- ---------------- -Block legacy authentication enabled 7/20/2026 12:48:56 PM -Block sign-in from unsupported countries disabled 5/15/2026 12:17:15 AM -Require compliant device for Exchange enabledForReportingButNotEnforced 8/16/2026 2:13:54 AM -Require MFA for all users enabled 8/11/2026 9:47:30 AM - -Full export saved to conditional-access-policies.json - -conditional-access-policies.json (first lines): -[ - { - "createdDateTime": "2025-07-04T09:20:08Z", - "state": "enabled", - "displayName": "Block legacy authentication", - "id": "d7d8695f-36ef-4a02-9045-cc9efadb0eb1", - "modifiedDateTime": "2026-07-20T12:48:56Z" -... -#> diff --git a/examples/reporting-and-audit/disabled-accounts-report.ps1 b/examples/reporting-and-audit/disabled-accounts-report.ps1 deleted file mode 100644 index 9f010fa..0000000 --- a/examples/reporting-and-audit/disabled-accounts-report.ps1 +++ /dev/null @@ -1,33 +0,0 @@ -# Report all disabled user accounts in the tenant. -# -# Uses server-side filtering so only matching records are transferred. -# -ConsistencyLevel eventual enables advanced query support required -# for certain filter expressions on large tenants. -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All" - -Import-Module Mgx - -$disabled = Invoke-MgxRequest /users ` - -All ` - -Filter "accountEnabled eq false" ` - -Property displayName,mail,userPrincipalName,createdDateTime ` - -ConsistencyLevel eventual - -Write-Host "Disabled accounts: $($disabled.Count)" -$disabled | Sort-Object createdDateTime | Select-Object displayName, mail, createdDateTime | Format-Table -AutoSize - -<# -Expected output: - -Disabled accounts: 6 - -displayName mail createdDateTime ------------ ---- --------------- -Brian Johnson brianj@contoso.com 5/21/2025 8:22:32 PM -Isaiah Langer isaiahl@contoso.com 6/24/2025 8:18:13 PM -Bianca Pisani bianca.pisani@fabrikam.com 7/3/2025 8:13:54 PM -Pradeep Gupta pradeepg@contoso.com 8/13/2025 12:23:01 AM -Cameron White cameronw@contoso.com 11/11/2025 5:13:25 PM -Raul Razo raul.razo@woodgrovebank.com 12/30/2025 11:43:39 AM -#> diff --git a/examples/reporting-and-audit/download-groups-and-members.ps1 b/examples/reporting-and-audit/download-groups-and-members.ps1 new file mode 100644 index 0000000..fbda55b --- /dev/null +++ b/examples/reporting-and-audit/download-groups-and-members.ps1 @@ -0,0 +1,153 @@ +# Download every group and ALL its members - nesting resolved - with a +# progress bar. +# +# Three JSONL files: +# groups.jsonl every group +# members-direct.jsonl every direct membership; rows whose @odata.type is a +# group are the nesting edges +# members.jsonl every membership with nesting resolved +# +# One fan-out, not two, and no transitiveMembers. Asking Graph to flatten the +# nesting per group re-transfers every deeply nested member once per ancestor +# group; the direct download already contains the whole graph, so the closure +# is computed locally instead - each membership crosses the wire exactly once, +# at 2 RU per request instead of 3. +# +# Requirements: Connect-MgGraph -Scopes "Group.Read.All", "User.Read.All" + +param( + [string]$OutputDirectory = './groups-export', + # Fan-out width. The default of 5 leaves a big tenant latency-bound; at + # ~2 RU per call, 16 concurrent stays far below the 800 RU/s budget floor + # (resource-unit-budgeting.ps1 measures your tenant's own numbers). + [int]$Concurrency = 16, + [int]$UpdateEvery = 100 +) + +Import-Module Mgx + +New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null +$directFile = Join-Path $OutputDirectory 'members-direct.jsonl' +$resolvedFile = Join-Path $OutputDirectory 'members.jsonl' + +# groups.jsonl is a single enumeration - exactly what Export-MgxCollection is +# for: request loop, JSONL writing, and resume in one cmdlet. The fan-out +# below cannot use it, because Export-MgxCollection drives its own requests +# from a URI rather than accepting piped objects; its rows arrive through the +# pipeline, so those files are written by hand. +$groupsFile = Join-Path $OutputDirectory 'groups.jsonl' +$null = Export-MgxCollection /groups -All ` + -Property id,displayName,description,mail,groupTypes,securityEnabled ` + -OutputFile $groupsFile +$groups = @(Get-Content $groupsFile | ConvertFrom-Json) +Write-Host "$($groups.Count) groups -> groups.jsonl" + +# --- pass 1: one fan-out, every direct membership straight to disk --- +$sw = [System.Diagnostics.Stopwatch]::StartNew() +$started = @{} +$parents = @{} # child group id -> list of parent group ids (built as rows stream by) +$rows = 0 +# One StreamWriter held open for the run - Add-Content would reopen the file +# once per row. +$writer = [System.IO.StreamWriter]::new($directFile, $false, [System.Text.UTF8Encoding]::new($false)) +try { + $groups | Invoke-MgxRequest '/groups/{id}/members?$select=id,displayName' ` + -All -Concurrency $Concurrency -SkipNotFound -SkipForbidden | + ForEach-Object { + $_['groupId'] = $_['_MgxSourceId'] + $_.Remove('_MgxSourceId') + $writer.WriteLine(($_ | ConvertTo-Json -Compress -Depth 10)) + + if ($_['@odata.type'] -eq '#microsoft.graph.group') { + if (-not $parents.ContainsKey($_['id'])) { + $parents[$_['id']] = [System.Collections.Generic.List[string]]::new() + } + $parents[$_['id']].Add($_['groupId']) + } + + $rows++ + $started[$_['groupId']] = $true + if ($rows % $UpdateEvery -eq 0) { + $done = $started.Count / $groups.Count + $t = Get-MgxTelemetry + Write-Progress -Activity 'Downloading direct memberships' ` + -Status ('{0} of {1} groups | {2:N0} rows | {3:N0}/s | {4} requests, {5:N0} RU' -f ` + $started.Count, $groups.Count, $rows, + ($rows / $sw.Elapsed.TotalSeconds), $t.Requests, $t.ResourceUnitsConsumed) ` + -PercentComplete (100 * $done) ` + -SecondsRemaining ([int]($sw.Elapsed.TotalSeconds * (1 - $done) / [Math]::Max($done, 0.01))) + } + } +} +finally { + $writer.Dispose() +} +Write-Progress -Activity 'Downloading direct memberships' -Completed +Write-Host ('{0:N0} direct membership row(s) in {1:mm\:ss} -> members-direct.jsonl' -f $rows, $sw.Elapsed) + +# --- pass 2: resolve the nesting locally - disk to disk, no requests --- +$ancestorCache = @{} +function Get-Ancestors { + # Every group transitively containing $GroupId. The visited set makes a + # membership cycle a non-event rather than a hang. + param([string]$GroupId) + if ($ancestorCache.ContainsKey($GroupId)) { return ,$ancestorCache[$GroupId] } + $seen = [System.Collections.Generic.HashSet[string]]::new() + $queue = [System.Collections.Generic.Queue[string]]::new() + if ($parents.ContainsKey($GroupId)) { foreach ($p in $parents[$GroupId]) { $queue.Enqueue($p) } } + while ($queue.Count -gt 0) { + $g = $queue.Dequeue() + if ($seen.Add($g) -and $parents.ContainsKey($g)) { + foreach ($p in $parents[$g]) { $queue.Enqueue($p) } + } + } + $ancestorCache[$GroupId] = @($seen) + return ,$ancestorCache[$GroupId] +} + +$resolved = 0 +$writer = [System.IO.StreamWriter]::new($resolvedFile, $false, [System.Text.UTF8Encoding]::new($false)) +try { + foreach ($line in [System.IO.File]::ReadLines($directFile)) { + $writer.WriteLine($line) # a member of a group is a member of that group + $resolved++ + $row = $line | ConvertFrom-Json + foreach ($ancestor in (Get-Ancestors $row.groupId)) { + $row.groupId = $ancestor + $writer.WriteLine(($row | ConvertTo-Json -Compress -Depth 10)) + $resolved++ + } + } +} +finally { + $writer.Dispose() +} +Write-Host ('{0:N0} resolved membership row(s) -> members.jsonl' -f $resolved) + +$t = Get-MgxTelemetry +Write-Host ('done in {0:mm\:ss} - {1} requests, {2:N0} resource units' -f ` + $sw.Elapsed, $t.Requests, $t.ResourceUnitsConsumed) + +# Reading it back - everyone in one group, including via nesting. Multiple +# nesting paths collapse to one row, but a member placed DIRECTLY in several +# groups under the same ancestor appears once per placement - dedup on id: +# Get-Content ./groups-export/members.jsonl | ConvertFrom-Json | +# Where-Object { $_.groupId -eq $id -and $_.'@odata.type' -eq '#microsoft.graph.user' } | +# Sort-Object id -Unique +# +# Notes: +# * The nesting edges are the group-typed rows of members-direct.jsonl: +# ... | ConvertFrom-Json | Where-Object '@odata.type' -eq '#microsoft.graph.group' +# * Empty groups contribute no rows, so the bar can top out just short of +# 100 before it clears (see progress-for-long-runs.ps1). +# * A killed run restarts pass 1: checkpoints belong to single enumerations, +# not fan-outs. Pass 2 is local and costs nothing to redo. + +<# +Expected output (final state; the bar redraws in place while running): + +12 groups -> groups.jsonl +1,214 direct membership row(s) in 00:19 -> members-direct.jsonl +1,532 resolved membership row(s) -> members.jsonl +done in 00:20 - 14 requests, 27 RU +#> diff --git a/examples/reporting-and-audit/group-membership-report.ps1 b/examples/reporting-and-audit/group-membership-report.ps1 new file mode 100644 index 0000000..5a3571a --- /dev/null +++ b/examples/reporting-and-audit/group-membership-report.ps1 @@ -0,0 +1,84 @@ +# Membership of every group: direct, and effective through nesting. +# +# A group's real membership is not its member list - groups contain groups. +# /transitiveMembers flattens the nesting server-side, so there is no recursion +# to write and no cycle handling to get wrong. The groups where effective and +# direct counts differ are the ones nesting hides members in. +# +# The memory model is half the point. Tenant-wide membership is too big to +# hold, so no member list is ever kept: the fan-out streams members out as +# pages arrive, each tagged with _MgxSourceId (the group it came from), and +# the loop reduces them to a count on the spot. RAM holds one row per group, +# not the membership of the tenant. +# +# Requirements: Connect-MgGraph -Scopes "Group.Read.All", "User.Read.All" + +Import-Module Mgx + +# One row per group is all that is ever kept. +$groups = @(Invoke-MgxRequest /groups -All -Property id,displayName) +$stats = @{} +foreach ($g in $groups) { + $stats[$g.id] = [pscustomobject]@{ + Group = $g.displayName + Direct = 0 + Effective = 0 + Nested = [System.Collections.Generic.List[string]]::new() + } +} +Write-Host "$($groups.Count) groups" + +# The type casts filter server-side: /members/microsoft.graph.user returns only +# user members, so nothing arrives that has to be discarded, and $select=id +# keeps each streamed object to the two fields the tally needs. +$groups | Invoke-MgxRequest '/groups/{id}/members/microsoft.graph.user?$select=id' ` + -All -SkipNotFound -SkipForbidden | + ForEach-Object { $stats[$_._MgxSourceId].Direct++ } + +$groups | Invoke-MgxRequest '/groups/{id}/members/microsoft.graph.group?$select=id,displayName' ` + -All -SkipNotFound -SkipForbidden | + ForEach-Object { $stats[$_._MgxSourceId].Nested.Add($_.displayName) } + +$groups | Invoke-MgxRequest '/groups/{id}/transitiveMembers/microsoft.graph.user?$select=id' ` + -All -SkipNotFound -SkipForbidden | + ForEach-Object { $stats[$_._MgxSourceId].Effective++ } + +$report = $stats.Values | + Select-Object Group, Direct, Effective, @{ n = 'Nested'; e = { $_.Nested -join ', ' } } | + Sort-Object { $_.Effective - $_.Direct }, Group -Descending + +$report | Format-Table -AutoSize +$hidden = @($report | Where-Object { $_.Effective -gt $_.Direct }) +Write-Host "$($hidden.Count) group(s) have members through nesting alone" + +# Notes: +# * Expand-MgxRelation would attach the member arrays to each group instead - +# right when the enriched objects ARE the deliverable (user-directory-report +# does exactly that), wrong here, where they would only be counted and +# thrown away. Expand holds all input and all fetched relations until its +# fan-out completes; this report only ever holds integers. +# * Nesting is a security-group phenomenon - Microsoft 365 groups do not +# accept groups as members, so their two counts always match. +# * transitiveMembers bills more resource units per call than members; +# resource-unit-budgeting.ps1 measures both shapes. +# * The inverse question - every group a user is effectively in, nesting +# resolved - is one relation: /users/{id}/transitiveMemberOf. +# * On a tenant where this runs for minutes, progress-for-long-runs.ps1 is +# the same fan-out with a live status line. + +<# +Expected output: + +12 groups + +Group Direct Effective Nested +----- ------ --------- ------ +Engineering All Hands 3 11 Engineering Leads, Engineering ICs +Retail 2 9 Retail East, Retail West +U.S. Sales 5 5 +Mark 8 Project Team 8 8 +sg-IT-Helpdesk 4 4 +... + +2 group(s) have members through nesting alone +#> diff --git a/examples/reporting-and-audit/guest-users-report.ps1 b/examples/reporting-and-audit/guest-users-report.ps1 deleted file mode 100644 index e690b1d..0000000 --- a/examples/reporting-and-audit/guest-users-report.ps1 +++ /dev/null @@ -1,33 +0,0 @@ -# Report all guest (external) users in the tenant. -# -# Guest accounts have userType eq 'Guest'. This exports them to JSONL -# for audit purposes and also prints a summary to the console. -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All" - -Import-Module Mgx - -$guests = Invoke-MgxRequest /users ` - -All ` - -Filter "userType eq 'Guest'" ` - -Property displayName,mail,userPrincipalName,createdDateTime,externalUserState ` - -ConsistencyLevel eventual - -Write-Host "Guest accounts: $($guests.Count)" - -$guests | - Sort-Object createdDateTime -Descending | - Select-Object displayName, mail, externalUserState, createdDateTime | - Format-Table -AutoSize - -<# -Expected output: - -Guest accounts: 3 - -displayName mail externalUserState createdDateTime ------------ ---- ----------------- --------------- -Gerhart Moller gerhart.moller@northwindtraders.com PendingAcceptance 6/23/2026 4:04:46 PM -Raul Razo raul.razo@woodgrovebank.com Accepted 12/30/2025 11:43:39 AM -Bianca Pisani bianca.pisani@fabrikam.com Accepted 7/3/2025 8:13:54 PM -#> diff --git a/examples/reporting-and-audit/inactive-users-report.ps1 b/examples/reporting-and-audit/inactive-users-report.ps1 new file mode 100644 index 0000000..bca159f --- /dev/null +++ b/examples/reporting-and-audit/inactive-users-report.ps1 @@ -0,0 +1,67 @@ +# Users who have not signed in for 90 days. +# +# accountEnabled only tells you the account is allowed to sign in, not that +# anyone does. signInActivity is the property that answers the question, and it +# is filterable server-side, so the tenant is never enumerated in full. +# +# Two caveats worth knowing before the result is trusted: +# * signInActivity needs a Microsoft Entra ID P1 license and the +# AuditLog.Read.All scope, on top of User.Read.All. +# * An account that has NEVER signed in has no signInActivity at all, so it +# does not match this filter. Those are the second query below. +# +# Requirements: Connect-MgGraph -Scopes "User.Read.All", "AuditLog.Read.All" + +Import-Module Mgx + +$cutoff = (Get-Date).AddDays(-90).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') + +$inactive = Invoke-MgxRequest /users -All ` + -Filter "signInActivity/lastSignInDateTime le $cutoff" ` + -Property id,displayName,userPrincipalName,accountEnabled,signInActivity + +Write-Host "Not seen since $($cutoff.Substring(0,10)): $($inactive.Count)" +$inactive | + Select-Object displayName, userPrincipalName, accountEnabled, + @{ n = 'LastSignIn'; e = { ([datetime]$_.signInActivity.lastSignInDateTime).ToString('yyyy-MM-dd') } } | + Sort-Object LastSignIn | + Format-Table -AutoSize + +# Never signed in at all - created and forgotten, which is the more interesting +# half of the report on most tenants. These have no signInActivity to filter on, +# so this one sweeps and checks locally: one enumeration, same behavior on every +# tenant, and no dependency on how Graph handles a null comparison. +$never = @(Invoke-MgxRequest /users -All ` + -Property id,displayName,userPrincipalName,createdDateTime,accountEnabled,signInActivity | + Where-Object { -not $_.signInActivity.lastSignInDateTime }) + +Write-Host "`nNever signed in: $($never.Count)" +$never | + Select-Object displayName, userPrincipalName, accountEnabled, + @{ n = 'Created'; e = { ([datetime]$_.createdDateTime).ToString('yyyy-MM-dd') } } | + Sort-Object Created | + Format-Table -AutoSize + +# license-report.ps1 answers the follow-up: which of these are holding a seat. +# The same shape reports stale devices: /devices, filtered on +# approximateLastSignInDateTime le $cutoff. + +<# +Expected output: + +Not seen since 2026-05-26: 4 + +displayName userPrincipalName accountEnabled LastSignIn +----------- ----------------- -------------- ---------- +Brian Johnson brianj@contoso.com False 2025-11-02 +Isaiah Langer isaiahl@contoso.com False 2026-01-14 +Pradeep Gupta pradeepg@contoso.com True 2026-02-28 +Raul Razo raul.razo@woodgrovebank.com True 2026-03-19 + +Never signed in: 2 + +displayName userPrincipalName accountEnabled Created +----------- ----------------- -------------- ------- +Sample Vendor vendor@contoso.com True 2025-08-13 +Bianca Pisani bianca.pisani@fabrikam.com True 2025-07-03 +#> diff --git a/examples/reporting-and-audit/license-report.ps1 b/examples/reporting-and-audit/license-report.ps1 new file mode 100644 index 0000000..ab062b1 --- /dev/null +++ b/examples/reporting-and-audit/license-report.ps1 @@ -0,0 +1,80 @@ +# Where the licenses went: seats per SKU, and who is holding them. +# +# /subscribedSkus gives what the tenant bought and how much is consumed; the +# user sweep gives the names behind that number. Both are cheap - assignedLicenses +# comes back with the user object, so no per-user call is needed. +# +# Requirements: Connect-MgGraph -Scopes "Organization.Read.All", "User.Read.All" + +Import-Module Mgx + +$skus = Invoke-MgxRequest /subscribedSkus + +Write-Host "`n=== Seats ===" -ForegroundColor Cyan +$skus | + ForEach-Object { + [pscustomobject]@{ + Sku = $_.skuPartNumber + Purchased = $_.prepaidUnits.enabled + Assigned = $_.consumedUnits + Free = $_.prepaidUnits.enabled - $_.consumedUnits + Warning = $_.prepaidUnits.warning # seats in the 30-day grace period + } + } | + Sort-Object Free | + Format-Table -AutoSize + +# skuId is what a user object carries; skuPartNumber is what a human reads. +$nameById = @{} +foreach ($sku in $skus) { $nameById[[string]$sku.skuId] = $sku.skuPartNumber } + +$users = Invoke-MgxRequest /users -All -Property id,displayName,userPrincipalName,assignedLicenses,accountEnabled + +Write-Host "=== Assignments ===" -ForegroundColor Cyan +$assignments = foreach ($user in $users) { + foreach ($license in $user.assignedLicenses) { + [pscustomobject]@{ + Sku = $nameById[[string]$license.skuId] ?? $license.skuId + User = $user.userPrincipalName + Enabled = $user.accountEnabled + } + } +} + +$assignments | Group-Object Sku | Sort-Object Count -Descending | + Select-Object @{ n = 'Sku'; e = { $_.Name } }, Count | Format-Table -AutoSize + +# The number worth acting on: seats held by accounts that cannot sign in. +$wasted = @($assignments | Where-Object { -not $_.Enabled }) +if ($wasted) { + Write-Host "$($wasted.Count) seat(s) assigned to disabled accounts:" -ForegroundColor Yellow + $wasted | Sort-Object Sku, User | Format-Table Sku, User -AutoSize +} + +<# +Expected output: + +=== Seats === + +Sku Purchased Assigned Free Warning +--- --------- -------- ---- ------- +ENTERPRISEPACK 25 22 3 0 +EMSPREMIUM 10 4 6 0 +POWER_BI_STANDARD 100 7 93 0 + +=== Assignments === + +Sku Count +--- ----- +ENTERPRISEPACK 22 +POWER_BI_STANDARD 7 +EMSPREMIUM 4 + +3 seat(s) assigned to disabled accounts: + +Sku User +--- ---- +EMSPREMIUM brianj@contoso.com +ENTERPRISEPACK brianj@contoso.com +ENTERPRISEPACK isaiahl@contoso.com +#> diff --git a/examples/reporting-and-audit/stale-devices-report.ps1 b/examples/reporting-and-audit/stale-devices-report.ps1 deleted file mode 100644 index 5af8922..0000000 --- a/examples/reporting-and-audit/stale-devices-report.ps1 +++ /dev/null @@ -1,37 +0,0 @@ -# Report devices that haven't checked in for 90+ days. -# -# Stale devices are candidates for cleanup. This filters server-side -# by approximateLastSignInDateTime to avoid pulling every device. -# -# Requirements: Connect-MgGraph -Scopes "Device.Read.All" - -Import-Module Mgx - -$cutoff = (Get-Date).AddDays(-90).ToString("yyyy-MM-ddTHH:mm:ssZ") - -$stale = Invoke-MgxRequest /devices ` - -All ` - -Filter "approximateLastSignInDateTime le $cutoff" ` - -Property displayName,operatingSystem,operatingSystemVersion,approximateLastSignInDateTime,accountEnabled ` - -ConsistencyLevel eventual - -Write-Host "Stale devices (90+ days): $($stale.Count)" - -$stale | - Sort-Object approximateLastSignInDateTime | - Select-Object displayName, operatingSystem, approximateLastSignInDateTime, accountEnabled | - Format-Table -AutoSize - -<# -Expected output: - -Stale devices (90+ days): 5 - -displayName operatingSystem approximateLastSignInDateTime accountEnabled ------------ --------------- ----------------------------- -------------- -ISAIAHL-LAPTOP Windows 9/13/2025 1:55:10 AM False -ADELEV-SURFACE Windows 1/17/2026 10:17:15 PM True -CONTOSO-KIOSK-02 Windows 2/14/2026 3:15:20 AM True -CONTOSO-MAC-04 macOS 4/10/2026 1:10:32 PM True -MEGANB-IPAD iOS 5/16/2026 5:23:58 AM True -#> diff --git a/examples/reporting-and-audit/user-directory-report.ps1 b/examples/reporting-and-audit/user-directory-report.ps1 new file mode 100644 index 0000000..7a3a936 --- /dev/null +++ b/examples/reporting-and-audit/user-directory-report.ps1 @@ -0,0 +1,45 @@ +# A user directory: every enabled user with manager and licenses attached. +# +# Neither relation comes back with the user object, so each needs a call per +# user. Stacked Expand-MgxRelation stages fetch each relation concurrently +# across users, and the second stage receives objects already carrying the +# first - two serial foreach loops collapse into one pass. +# +# Requirements: Connect-MgGraph -Scopes "User.Read.All" + +Import-Module Mgx + +$directory = Invoke-MgxRequest /users -All ` + -Filter 'accountEnabled eq true' ` + -Property id,displayName,department,jobTitle | + Expand-MgxRelation '/users/{id}/manager' -As Manager -Flatten -SkipNotFound | + Expand-MgxRelation '/users/{id}/licenseDetails' -As Licenses -SkipNotFound + +$directory | + Select-Object displayName, department, jobTitle, + @{ n = 'Manager'; e = { $_.Manager.displayName } }, + @{ n = 'Licenses'; e = { (@($_.Licenses.skuPartNumber) | Sort-Object) -join ', ' } } | + Sort-Object department, displayName | + Format-Table -AutoSize + +# The skipped-404 warning is expected: users at the top of the org have no +# manager. Swap Format-Table for Export-Csv to hand the file to whoever asked. + +<# +Expected output: + +WARNING: Skipped 4 entities due to 404 (Not Found) responses. + +displayName department jobTitle Manager Licenses +----------- ---------- -------- ------- -------- +Brian Johnson Engineering Senior Engineer Lee Gu SPE_E3 +Lee Gu Engineering Director SPE_E5 +Christie Cline Human Resources Benefits Specialist Nestor Wilke SPE_E5 +Allan Deyoung IT Corporate Security Officer Nestor Wilke EMSPREMIUM, SPE_E5 +Delia Dennis IT Support Engineer Allan Deyoung EMSPREMIUM, SPE_E5 +Alex Wilber Marketing Marketing Assistant Megan Bowen SPE_E5 +Debra Berger Operations Administrative Assistant Nestor Wilke SPE_E5 +Adele Vance Retail Retail Manager Nestor Wilke SPE_E5 +Cameron White Retail Store Associate Adele Vance SPE_E3 +... +#> diff --git a/examples/resilience-and-telemetry/pacing-observability.ps1 b/examples/resilience-and-telemetry/pacing-observability.ps1 deleted file mode 100644 index 97816fe..0000000 --- a/examples/resilience-and-telemetry/pacing-observability.ps1 +++ /dev/null @@ -1,178 +0,0 @@ -# What the adaptive pacer is doing, and why it is not doing what the docs suggest. -# -# The 2.1 pacer spaces requests BEFORE sending, learning a rate per workload. This script makes -# that state visible: what a cold session looks like, what slow start does to a fan-out, how -# workloads stay independent, and which signals the pacer can actually rely on. -# -# The last point is the one worth reading. Microsoft documents an x-ms-throttle-limit-percentage -# header that warns when you pass 0.8 of the resource-unit budget. Measured against a live -# tenant it was never emitted - not at 1.5x the documented budget, and not while the tenant was -# actively returning 429s. So the pacer treats it as opportunistic and relies on Retry-After and -# latency drift, which are real. This script shows you that for your own tenant. -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All","Files.Read.All" -# Read-only. - -Import-Module Mgx - -function Show-PacingState { - param([string]$Label) - $t = Get-MgxTelemetry - Write-Host "`n$Label" -ForegroundColor Cyan - " requests : $($t.Requests) ($($t.Succeeded) ok, $($t.Failed) failed)" - " pacing waits : $($t.AdaptivePacingWaitMs) ms over $($t.AdaptivePacingActivations) activations" - " throttle retries : $($t.ThrottleRetries) rate-limiter waits: $($t.RateLimiterWaitMs) ms" - " resource units : $($t.ResourceUnitsConsumed)" - # PacingState is one line per workload bucket that has seen traffic. A bucket with nothing - # to say is absent rather than zeroed, so the line grows as you touch more workloads. - " pacing state : $(if ($t.PacingState) { $t.PacingState } else { '(no bucket has seen traffic yet)' })" - " last throttle % : $($t.LastThrottlePercentage)" -} - -Write-Host "=== 1. A cold session has no pacing state ===" -ForegroundColor Green -Show-PacingState "before any request" - -Write-Host "`n=== 2. Slow start on a cold directory workload ===" -ForegroundColor Green -# A cold bucket opens at 4 rps and doubles each clean second. Request #1 is never delayed - the -# cap only bites once a fan-out is in flight, which is exactly the case that gets throttled -# before it has returned a single item. -$sw = [Diagnostics.Stopwatch]::StartNew() -$users = Invoke-MgxRequest /users -All -PageSize 5 -Top 25 -Property id,displayName -$sw.Stop() -" fetched $(@($users).Count) users in $($sw.Elapsed.TotalSeconds.ToString('F1'))s" -Show-PacingState "after a directory read" - -Write-Host "`n=== 3. A second workload gets its own independent bucket ===" -ForegroundColor Green -# Drive and directory are separate buckets. A throttled Teams or Drive workload must not slow an -# Entra fan-out, because the budgets really are separate: Microsoft documents limits per service -# and per application+tenant pair, and a two-identity A/B run confirmed one identity absorbing -# 8,203 throttles while another in the same tenant was never refused. -try { - $null = Invoke-MgxRequest '/me/drive?$select=id,driveType' -ErrorAction Stop - " drive reachable - watch a second bucket appear in the state line below" -} -catch { - " no drive on this account ($($_.Exception.Message -replace '\s+', ' '))" - " the directory bucket below should be unaffected either way - that is the point" -} -Show-PacingState "after touching a second workload" - -Write-Host "`n=== 4. Did Graph ever send the proximity header? ===" -ForegroundColor Green -$t = Get-MgxTelemetry -if ($t.LastThrottlePercentage -lt 0) { - @" - LastThrottlePercentage = $($t.LastThrottlePercentage) - - Negative means Graph has not sent x-ms-throttle-limit-percentage on any response in this - session. That is the expected result. In testing it was never emitted, including during runs - that were actively being throttled 8,203 times - so a control loop built on it would be dead - code that looks like a feature. - - What the pacer uses instead, in order of reliability: - 1. 429 + Retry-After - unambiguous, always honored - 2. latency drift - throughput fell ~19x under load before recovering - 3. this percentage - welcome when present, assumed absent -"@ -} -else { - " LastThrottlePercentage = $($t.LastThrottlePercentage) - Graph DID send it." - " Worth recording: that contradicts what was measured elsewhere. Above 0.8 means you are" - " near the budget and the pacer is damping; at 1.2 it spaces to ~2s per request." -} - -Write-Host "`n=== 5. A/B: the same fan-out with pacing off ===" -ForegroundColor Green -# The honest comparison. Pacing costs a little wall-clock on a cold sequential run and roughly -# nothing on a warm one; what it buys is not being throttled at scale. Do not read a single -# small run as evidence either way. -$ids = @($users | Select-Object -First 8 | ForEach-Object { $_.id }) -if ($ids.Count -lt 2) { - " need at least 2 users to compare; skipping" - return -} - -Set-MgxOption -NoAdaptivePacing -$swOff = [Diagnostics.Stopwatch]::StartNew() -$null = $ids | Invoke-MgxRequest -Uri '/users/{id}?$select=id' -Concurrency 4 -$swOff.Stop() - -Set-MgxOption -Reset -$swOn = [Diagnostics.Stopwatch]::StartNew() -$null = $ids | Invoke-MgxRequest -Uri '/users/{id}?$select=id' -Concurrency 4 -$swOn.Stop() - -" {0} lookups, pacing OFF : {1:F2}s" -f $ids.Count, $swOff.Elapsed.TotalSeconds -" {0} lookups, pacing ON : {1:F2}s" -f $ids.Count, $swOn.Elapsed.TotalSeconds -" difference : {0:F2}s" -f ($swOn.Elapsed.TotalSeconds - $swOff.Elapsed.TotalSeconds) -"" -" A cold bucket pays slow start; a warm one usually pays nothing. The cost that matters is the" -" one you avoid - a throttled fan-out that collapses from 322 rps to 17 rps and takes 15 minutes" -" to drain." - -Show-PacingState "final" -Set-MgxOption -Reset - -<# -Expected output: - -=== 1. A cold session has no pacing state === - -before any request - requests : 0 (0 ok, 0 failed) - pacing waits : 0 ms over 0 activations - throttle retries : 0 rate-limiter waits: 0 ms - resource units : 0 - pacing state : (no bucket has seen traffic yet) - last throttle % : -1 - -=== 2. Slow start on a cold directory workload === - fetched 28 users in 0.4s - -after a directory read - requests : 1 (1 ok, 0 failed) - pacing waits : 0 ms over 0 activations - throttle retries : 0 rate-limiter waits: 0 ms - resource units : 1 - pacing state : directory: slow-start 4 rps, latency 241ms (1.0x of 241ms baseline) - last throttle % : -1 - -=== 3. A second workload gets its own independent bucket === - drive reachable - watch a second bucket appear in the state line below - -after touching a second workload - requests : 2 (2 ok, 0 failed) - pacing waits : 0 ms over 0 activations - throttle retries : 0 rate-limiter waits: 0 ms - resource units : 2 - pacing state : drive: slow-start 4 rps; directory: slow-start 4 rps, latency 241ms (1.0x of 241ms baseline) - last throttle % : -1 - -=== 4. Did Graph ever send the proximity header? === - LastThrottlePercentage = -1 - - Negative means Graph has not sent x-ms-throttle-limit-percentage on any response in this - session. That is the expected result. In testing it was never emitted, including during runs - that were actively being throttled 8,203 times - so a control loop built on it would be dead - code that looks like a feature. - - What the pacer uses instead, in order of reliability: - 1. 429 + Retry-After - unambiguous, always honored - 2. latency drift - throughput fell ~19x under load before recovering - 3. this percentage - welcome when present, assumed absent - -=== 5. A/B: the same fan-out with pacing off === - 8 lookups, pacing OFF : 0.71s - 8 lookups, pacing ON : 1.93s - difference : 1.22s - - A cold bucket pays slow start; a warm one usually pays nothing. The cost that matters is the - one you avoid - a throttled fan-out that collapses from 322 rps to 17 rps and takes 15 minutes - to drain. - -final - requests : 18 (18 ok, 0 failed) - pacing waits : 6193 ms over 8 activations - throttle retries : 0 rate-limiter waits: 0 ms - resource units : 18 - pacing state : drive: slow-start 4 rps; directory: slow-start 8 rps, latency 198ms (0.9x of 224ms baseline) - last throttle % : -1 -#> diff --git a/examples/resilience-and-telemetry/progress-for-long-runs.ps1 b/examples/resilience-and-telemetry/progress-for-long-runs.ps1 new file mode 100644 index 0000000..6bff533 --- /dev/null +++ b/examples/resilience-and-telemetry/progress-for-long-runs.ps1 @@ -0,0 +1,81 @@ +# A live status line for a long run: how far along, how fast, what it costs. +# +# Streaming is what makes this possible at all. Fan-out results flow through +# the pipeline while the requests are still running, so a ForEach-Object in +# the middle can tally them and draw progress - percent, rate, ETA, and the +# resource units the run has consumed so far. A cmdlet that buffers until +# everything is done has nothing to hang a progress bar on. +# +# The demonstration workload is the transitive-members fan-out from +# group-membership-report.ps1 - the kind of run that takes long enough on a +# real tenant to make you wonder whether it is alive. +# +# Requirements: Connect-MgGraph -Scopes "Group.Read.All", "User.Read.All" + +param( + # Write-Progress is not free; redrawing every item can dominate a fast + # stream. Every 100th is smooth and costs nothing measurable. + [int]$UpdateEvery = 100 +) + +Import-Module Mgx + +$groups = @(Invoke-MgxRequest /groups -All -Property id,displayName) +Write-Host "$($groups.Count) groups to enumerate" + +$sw = [System.Diagnostics.Stopwatch]::StartNew() +$started = @{} # groups whose first member has arrived +$counts = @{} +$members = 0 + +$groups | Invoke-MgxRequest '/groups/{id}/transitiveMembers/microsoft.graph.user?$select=id' ` + -All -SkipNotFound -SkipForbidden | + ForEach-Object { + $members++ + $counts[$_._MgxSourceId] = ($counts[$_._MgxSourceId] ?? 0) + 1 + $started[$_._MgxSourceId] = $true + + if ($members % $UpdateEvery -eq 0) { + # "started", not "finished": a group joins the tally when its first + # member arrives. With the default concurrency of 5, at most 5 + # groups are in flight, so the two differ by at most 5. + $done = $started.Count / $groups.Count + $t = Get-MgxTelemetry + Write-Progress -Activity 'Transitive members, all groups' ` + -Status ('{0} of {1} groups | {2:N0} members | {3:N0}/s | {4} requests, {5:N0} RU' -f ` + $started.Count, $groups.Count, $members, + ($members / $sw.Elapsed.TotalSeconds), $t.Requests, $t.ResourceUnitsConsumed) ` + -PercentComplete (100 * $done) ` + -SecondsRemaining ([int]($sw.Elapsed.TotalSeconds * (1 - $done) / [Math]::Max($done, 0.01))) + } + } + +Write-Progress -Activity 'Transitive members, all groups' -Completed +Write-Host ('{0:N0} members across {1} groups in {2:mm\:ss}' -f $members, $groups.Count, $sw.Elapsed) + +Write-Host "`nLargest groups:" +$counts.GetEnumerator() | Sort-Object Value -Descending | Select-Object -First 5 | + ForEach-Object { + $g = $groups | Where-Object id -eq $_.Key + '{0,8:N0} {1}' -f $_.Value, $g.displayName + } + +# Honest limits of the numbers on the bar: +# * Empty groups never emit a member, so they never join the "started" count - +# the percentage can top out just short of 100 before the bar clears. +# * The ETA assumes groups are similar in size. They are not; one giant group +# at the end makes it optimistic. Treat it as a trend, not a promise. + +<# +Expected output (final state; the bar redraws in place while running): + +12 groups to enumerate +1,178 members across 12 groups in 00:41 + +Largest groups: + 412 Engineering All Hands + 306 All Employees + 144 Retail + 98 U.S. Sales + 67 sg-IT-Helpdesk +#> diff --git a/examples/resilience-and-telemetry/resilience-for-existing-scripts.ps1 b/examples/resilience-and-telemetry/resilience-for-existing-scripts.ps1 deleted file mode 100644 index 60d467f..0000000 --- a/examples/resilience-and-telemetry/resilience-for-existing-scripts.ps1 +++ /dev/null @@ -1,24 +0,0 @@ -# Add retry, circuit breaker, and rate limiting to existing Microsoft.Graph -# scripts without changing a single line of their code. -# -# Enable-MgxResilience injects a Polly pipeline into the SDK's HTTP transport. -# All SDK cmdlets (Get-MgUser, Get-MgGroup, etc.) then automatically retry -# on 429/5xx, honor Retry-After headers, and back off under load. -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All" - -Import-Module Microsoft.Graph.Users -Import-Module Mgx - -Enable-MgxResilience - -# These are unmodified Microsoft.Graph SDK calls - they now have full -# retry and circuit breaker protection. -$users = Get-MgUser -All -Property displayName, mail -Write-Host "Got $($users.Count) users via SDK (with Mgx resilience)" - -<# -Expected output: - -Got 28 users via SDK (with Mgx resilience) -#> diff --git a/examples/resilience-and-telemetry/resilience-status.ps1 b/examples/resilience-and-telemetry/resilience-status.ps1 index b432f36..d59ca40 100644 --- a/examples/resilience-and-telemetry/resilience-status.ps1 +++ b/examples/resilience-and-telemetry/resilience-status.ps1 @@ -1,7 +1,9 @@ -# Check and toggle SDK resilience injection. +# Add retry, circuit breaking, and rate limiting to existing Microsoft.Graph +# scripts without changing a line of their code - then check it and take it off. # -# Get-MgxResilience reports whether the Polly pipeline is currently -# injected into the Microsoft.Graph SDK HTTP transport. +# Enable-MgxResilience injects a Polly pipeline into the SDK's HTTP transport, so +# unmodified SDK cmdlets (Get-MgUser, Get-MgGroup, ...) retry on 429/5xx and honor +# Retry-After. Get-MgxResilience reports whether it is currently injected; # Disable-MgxResilience removes it and restores the original SDK behavior. # # Requirements: Connect-MgGraph -Scopes "User.Read.All" diff --git a/examples/resilience-and-telemetry/resource-unit-budgeting.ps1 b/examples/resilience-and-telemetry/resource-unit-budgeting.ps1 index 5a21ca2..fa94c5e 100644 --- a/examples/resilience-and-telemetry/resource-unit-budgeting.ps1 +++ b/examples/resilience-and-telemetry/resource-unit-budgeting.ps1 @@ -1,31 +1,28 @@ # Resource units: what a query actually costs, and how to size a fan-out before running it. # # Graph throttles directory workloads on a resource-unit budget, not on request count or -# bandwidth. Two queries that look alike can differ 5x in cost, and the difference is invisible -# unless you read x-ms-resource-unit. Mgx accumulates it for you: Get-MgxTelemetry reports -# ResourceUnitsConsumed for the session. -# -# This script measures the cost of the query shapes you are choosing between, then uses the -# measured cost - not a guess - to size a tenant-scale fan-out. +# bandwidth. Two queries that look alike can differ several times over in cost, and the +# difference is invisible unless you read x-ms-resource-unit. Mgx accumulates it for you: +# Get-MgxTelemetry reports ResourceUnitsConsumed for the session. # # Requirements: Connect-MgGraph -Scopes "Group.Read.All","User.Read.All" Import-Module Mgx function Measure-QueryCost { - <# - Cost of one call, measured rather than assumed. The documented RU table is directionally - right but not exact: transitiveMembers is published at 5 RU and bills 4 here, and the - documented "-1 RU for $select" discount does apply on top of that. - #> + # Cost of one call, measured rather than assumed. The documented RU table is directionally + # right but not exact - transitiveMembers is published at 5 RU and bills 4 here. param([string]$Label, [string]$Uri) $before = (Get-MgxTelemetry).ResourceUnitsConsumed try { $null = Invoke-MgxRequest $Uri -ErrorAction Stop } catch { Write-Warning "$Label : $($_.Exception.Message)"; return } - $cost = (Get-MgxTelemetry).ResourceUnitsConsumed - $before - [pscustomobject]@{ Query = $Label; ResourceUnits = $cost; Uri = $Uri } + [pscustomobject]@{ + Query = $Label + ResourceUnits = (Get-MgxTelemetry).ResourceUnitsConsumed - $before + Uri = $Uri + } } $sample = (Invoke-MgxRequest '/groups?$top=1&$select=id').id @@ -40,8 +37,8 @@ $costs = @( Measure-QueryCost 'single group read' "/groups/$sample" Measure-QueryCost 'users list ($top=25)' "/users?`$top=25" Measure-QueryCost 'users list ($select)' "/users?`$top=25&`$select=id,displayName" -) -$costs | Where-Object { $_ } | Format-Table Query, ResourceUnits -AutoSize +) | Where-Object { $_ } +$costs | Format-Table Query, ResourceUnits -AutoSize # --- size the fan-out from the measured cost --- # @@ -49,7 +46,8 @@ $costs | Where-Object { $_ } | Format-Table Query, ResourceUnits -AutoSize # throttling in one test tenant was appreciably higher (clean at 882 RU/s, 429s from ~1,200 # RU/s), so treat 800 as a conservative floor rather than a hard ceiling - and never as a target. $BudgetRuPerSecond = 800 -$chosen = $costs | Where-Object { $_ -and $_.Query -eq 'transitiveMembers ($select)' } | Select-Object -First 1 +$chosen = $costs | Where-Object { $_.Query -eq 'transitiveMembers ($select)' } | Select-Object -First 1 +$bare = $costs | Where-Object { $_.Query -eq 'transitiveMembers (bare)' } | Select-Object -First 1 if (-not $chosen -or $chosen.ResourceUnits -le 0) { Write-Warning 'No RU header observed - this tenant or endpoint may not be RU-metered. Skipping the projection.' return @@ -65,32 +63,15 @@ Write-Host "`n=== Sizing a per-group fan-out over the whole tenant ===" -Foregro " cost per group : {0} RU ({1})" -f $chosen.ResourceUnits, $chosen.Query " total cost : {0:N0} RU" -f $totalRu " floor at {0} RU/s : {1:N0}s ({2:N1} min) of pure budget consumption" -f $BudgetRuPerSecond, $seconds, ($seconds / 60) -'' -" Dropping `$select would cost {0:N0} RU more ({1:P0} increase)." -f ` - ($groupCount * 1), (1 / $chosen.ResourceUnits) - -# --- what the session actually spent --- -Write-Host "`n=== Session telemetry ===" -ForegroundColor Cyan -$t = Get-MgxTelemetry -" requests : $($t.Requests) ($($t.Succeeded) ok, $($t.Failed) failed)" -" resource units : $($t.ResourceUnitsConsumed)" -" throttle retries : $($t.ThrottleRetries)" -" pacing waits : $($t.AdaptivePacingWaitMs) ms over $($t.AdaptivePacingActivations) activations" -if ($t.Requests -gt 0) { - " average cost : {0:N2} RU per request" -f ($t.ResourceUnitsConsumed / $t.Requests) +if ($bare -and $bare.ResourceUnits -gt $chosen.ResourceUnits) { + $extra = $groupCount * ($bare.ResourceUnits - $chosen.ResourceUnits) + " dropping `$select : {0:N0} RU more ({1:P0} increase)" -f ` + $extra, ($extra / $totalRu) } -Write-Host @" - -Why this matters - * Cost is a property of the query SHAPE, not the object count. Adding `$select reduces it; - `$expand increases it. Measure the shape you are about to run 15,000 times. - * A 403 or 400 costs 0 RU. A fan-out that fails uniformly consumes no budget and triggers no - throttling - it looks fast and healthy while measuring nothing. Check status, not duration. - * x-ms-throttle-limit-percentage is documented as a proximity warning above 0.8 of budget. It - was never observed in testing, even while the tenant was actively returning 429s. Do not - build a control loop that depends on it; rely on 429 + Retry-After, which is reliable. -"@ -ForegroundColor DarkGray +# Cost is a property of the query SHAPE, not the object count: $select reduces it, $expand +# raises it. A 403 or 400 costs 0 RU, so a fan-out that fails uniformly looks fast and healthy +# while measuring nothing - check status, not duration. <# Expected output: @@ -113,22 +94,5 @@ users list ($select) 1 cost per group : 3 RU (transitiveMembers ($select)) total cost : 47,337 RU floor at 800 RU/s : 59s (1.0 min) of pure budget consumption - - Dropping $select would cost 15,779 RU more (33% increase). - -=== Session telemetry === - requests : 9 (9 ok, 0 failed) - resource units : 18 - throttle retries : 0 - pacing waits : 1496 ms over 8 activations - average cost : 2.00 RU per request - -Why this matters - * Cost is a property of the query SHAPE, not the object count. Adding $select reduces it; - $expand increases it. Measure the shape you are about to run 15,000 times. - * A 403 or 400 costs 0 RU. A fan-out that fails uniformly consumes no budget and triggers no - throttling - it looks fast and healthy while measuring nothing. Check status, not duration. - * x-ms-throttle-limit-percentage is documented as a proximity warning above 0.8 of budget. It - was never observed in testing, even while the tenant was actively returning 429s. Do not - build a control loop that depends on it; rely on 429 + Retry-After, which is reliable. + dropping $select : 15,779 RU more (33% increase) #> diff --git a/examples/resilience-and-telemetry/telemetry.ps1 b/examples/resilience-and-telemetry/telemetry.ps1 deleted file mode 100644 index 096b334..0000000 --- a/examples/resilience-and-telemetry/telemetry.ps1 +++ /dev/null @@ -1,44 +0,0 @@ -# View session telemetry after running Graph operations. -# -# Get-MgxTelemetry shows accumulated request counts, retry events, -# throttle waits, and timing across all Mgx cmdlet calls in the session. -# Useful for understanding what your script actually did. -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All" - -Import-Module Mgx - -# Run some operations -$users = Invoke-MgxRequest /users -All -Property id,displayName -Write-Host "Users: $($users.Count)" - -$groups = Invoke-MgxRequest /groups -All -Property id,displayName -Write-Host "Groups: $($groups.Count)" - -# See what happened -Get-MgxTelemetry - -<# -Expected output: - -Users: 28 -Groups: 6 - -Requests : 2 -Succeeded : 2 -Failed : 0 -Throttle Retries (429) : 0 -Other Retries (5xx) : 0 -Circuit Breaker Trips : 0 -Rate Limiter Wait (ms) : 0 -Retry Delay (ms) : 0 -HTTP Time (ms) : 486 -Total Elapsed (ms) : 704 -Resource Units : 2 -Batch Item Throttles : 0 -Pacing Wait (ms) : 201 -Pacing Activations : 1 -Last Throttle % : never seen -Pacing State : directory: slow-start 4 rps, latency 237ms (1.0x of 241ms baseline) -Content Bytes : 0 -#> diff --git a/examples/resilience-and-telemetry/tune-rate-limits.ps1 b/examples/resilience-and-telemetry/tune-rate-limits.ps1 deleted file mode 100644 index 1a3ba43..0000000 --- a/examples/resilience-and-telemetry/tune-rate-limits.ps1 +++ /dev/null @@ -1,63 +0,0 @@ -# Tune resilience options for a specific workload. -# -# Set-MgxOption lets you adjust the rate limiter, retry count, circuit -# breaker, and timeouts at runtime without restarting the session. -# Only the parameters you pass are changed; everything else stays. -# -# Use Get-MgxOption to inspect current settings. -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All" - -Import-Module Mgx - -# View current defaults -Get-MgxOption - -# Slow down for a tenant that throttles aggressively -Set-MgxOption -BatchItemsPerSecond 5 -MaxRetryAttempts 10 - -# Run your workload -$users = Invoke-MgxRequest /users -All -Property id,displayName -Write-Host "Users: $($users.Count)" - -# Restore all defaults when done -Set-MgxOption -Reset - -Get-MgxOption - -<# -Expected output: - -Rate Limit Burst : 200 -Rate Limit/sec : 50 -No Rate Limit : False -No Adaptive Pacing : False -Queue Limit : 500 -Max Retry Attempts : 7 -Max Retry-After (s) : 120 -Total Timeout (s) : 300 -Attempt Timeout (s) : 30 -CB Duration (s) : 15 -CB Failure Ratio : 0.1 -CB Min Throughput : 40 -CB Sampling (s) : 30 -Batch Chunk Concurrency : 1 -Batch Items/sec : 20 - -Users: 28 -Rate Limit Burst : 200 -Rate Limit/sec : 50 -No Rate Limit : False -No Adaptive Pacing : False -Queue Limit : 500 -Max Retry Attempts : 7 -Max Retry-After (s) : 120 -Total Timeout (s) : 300 -Attempt Timeout (s) : 30 -CB Duration (s) : 15 -CB Failure Ratio : 0.1 -CB Min Throughput : 40 -CB Sampling (s) : 30 -Batch Chunk Concurrency : 1 -Batch Items/sec : 20 -#> diff --git a/module/en-US/Mgx.Cmdlets.dll-Help.xml b/module/en-US/Mgx.Cmdlets.dll-Help.xml index c652c4b..df2afa5 100644 --- a/module/en-US/Mgx.Cmdlets.dll-Help.xml +++ b/module/en-US/Mgx.Cmdlets.dll-Help.xml @@ -1974,6 +1974,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP Body Request body for all requests when piping string URLs. Ignored when pipeline input carries its own Body member. + Bodies follow the same serialization contract as Invoke-MgxRequest -Body. An item whose body serialization refuses a value (a SecureString, NaN) fails on its own, like an item whose body is not valid JSON; the rest of the batch is sent. Object @@ -2096,6 +2097,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP Body Request body for all requests when piping string URLs. Ignored when pipeline input carries its own Body member. + Bodies follow the same serialization contract as Invoke-MgxRequest -Body. An item whose body serialization refuses a value (a SecureString, NaN) fails on its own, like an item whose body is not valid JSON; the rest of the batch is sent. Object @@ -2413,6 +2415,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Body Request body for write operations (POST, PATCH, PUT). Accepts a hashtable, PSObject or array, which is serialized to JSON, or a string already holding JSON, which is sent verbatim. Ignored (with a warning) on GET. + Enums serialize as camelCase names, byte arrays as base64, TimeSpan as an ISO-8601 duration, and a DateTime without a kind is treated as UTC. A SecureString, PSCredential, script block, certificate, NaN or Infinity anywhere in the body is refused before anything is sent, with an error naming the property. A string that is not JSON is sent verbatim; add a Content-Type to -Headers to declare its real type. Object @@ -2717,6 +2720,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Body Request body for write operations (POST, PATCH, PUT). Accepts a hashtable, PSObject or array, which is serialized to JSON, or a string already holding JSON, which is sent verbatim. Ignored (with a warning) on GET. + Enums serialize as camelCase names, byte arrays as base64, TimeSpan as an ISO-8601 duration, and a DateTime without a kind is treated as UTC. A SecureString, PSCredential, script block, certificate, NaN or Infinity anywhere in the body is refused before anything is sent, with an error naming the property. A string that is not JSON is sent verbatim; add a Content-Type to -Headers to declare its real type. Object diff --git a/module/en-US/about_Mgx.help.txt b/module/en-US/about_Mgx.help.txt index e951a42..1aa6c85 100644 --- a/module/en-US/about_Mgx.help.txt +++ b/module/en-US/about_Mgx.help.txt @@ -217,6 +217,7 @@ REQUIREMENTS is absent. SEE ALSO + about_Mgx_Errors about_Mgx_Tuning Invoke-MgxRequest Invoke-MgxBatchRequest diff --git a/module/en-US/about_Mgx_Errors.help.txt b/module/en-US/about_Mgx_Errors.help.txt new file mode 100644 index 0000000..84936aa --- /dev/null +++ b/module/en-US/about_Mgx_Errors.help.txt @@ -0,0 +1,99 @@ +TOPIC + about_Mgx_Errors + +SHORT DESCRIPTION + What terminates, what writes an error record, and how -ErrorAction and + -ErrorVariable behave across every Mgx cmdlet. + +LONG DESCRIPTION + Mgx follows one rule: only failures that make the whole invocation + meaningless terminate. Everything the service or transport says about an + individual request is a non-terminating error record, so a 10,000-item + fan-out that loses three items reports three errors and delivers 9,997 + results. + +TERMINATING ERRORS + + ThrowTerminatingError is reserved for pre-flight failures - the request + never went out, and no partial work exists: + + - Not connected, or Microsoft.Graph.Authentication missing + (NotConnected, GraphAuthModuleNotLoaded, HttpClientInitFailed) + - Invalid parameters or combinations: an absolute -Uri + (AbsoluteUriNotAllowed), -Search without ConsistencyLevel + (ConsistencyLevelRequired), a missing {id} placeholder, an unwritable + state or output file validated before the first request, a body value + with no JSON meaning (InvalidBodyValue), a second piped item against + one -OutFile (OutFileWithMultipleInputs) - refused before its content + is fetched + + Everything else is non-terminating. + +NON-TERMINATING ERRORS + + Every service and transport failure writes an ErrorRecord and continues: + + ErrorId When + ------- ---- + A Graph error with a code (e.g. + Request_ResourceNotFound); "GraphError" + when the body carried none + HttpError Transport failure with no response + CircuitBroken The circuit breaker is open; the record wraps + the guidance text + BatchItemError A batch item answered >= 400 + BatchItemNotSent A batch item skipped because an earlier chunk + failed + BatchChunkFailed The chunk POST itself failed + FanOutError A per-id GET failed during fan-out + BulkWriteError A per-id write failed during fan-out + BulkWriteInfraError A fan-out write died before an HTTP status + ExpandRelationError A relation fetch failed during expansion + NonJsonResponse The body is not JSON (use -Raw or + Get-MgxContent) + MalformedJsonResponse The body declared JSON and does not parse + IOError / AccessDenied An output, checkpoint, or state file failed + + The ErrorCategory derives from the failure: ObjectNotFound (404), + PermissionDenied (403), AuthenticationError (401), LimitsExceeded (429), + ResourceUnavailable (5xx, open circuit), ConnectionError (transport, + 408), InvalidArgument (400 family), ResourceExists (409, 412). + +-ERRORACTION + + Stop turns the first error record into a terminating error, ending the + pipeline. For batches this stops at the first failed item - after the + dead-letter file is written, so -DeadLetterPath always captures every + failure even under Stop. SilentlyContinue suppresses and continues; + records still land in $Error and -ErrorVariable. + +-ERRORVARIABLE + + Collects every non-terminating record: one per failed item, URL, or + page. A fan-out over N ids can produce up to N records. + +RETRIES AND ERRORS + + A failure that is retried is not an error until retries are exhausted: + a request that survives three 503s and succeeds writes nothing to the + error stream. The retry history is visible under -Verbose, and the + final record reflects the last failure. + +CANCELLATION + + Ctrl-C produces a warning, not an error record, because the pipeline is + already stopping and error records written during teardown are + unreliable. A cancelled run is therefore indistinguishable from an empty + result by $Error alone; checkpointed cmdlets write their checkpoint, and + -Verbose says the run was cancelled. + +SKIP SWITCHES + + -SkipNotFound and -SkipForbidden (Invoke-MgxRequest, Expand-MgxRelation) + downgrade 404 and 403 per-item failures to a counted warning summary. + Skipped items produce no error records - by design, since the caller + declared them expected. + +SEE ALSO + about_Mgx + about_Mgx_Tuning diff --git a/module/help/Invoke-MgxBatchRequest.md b/module/help/Invoke-MgxBatchRequest.md index 6dc3209..b1bde9b 100644 --- a/module/help/Invoke-MgxBatchRequest.md +++ b/module/help/Invoke-MgxBatchRequest.md @@ -147,6 +147,8 @@ Accept wildcard characters: False ### -Body Request body for all requests when piping string URLs. Ignored when pipeline input carries its own Body member. +Bodies follow the same serialization contract as Invoke-MgxRequest -Body. An item whose body serialization refuses a value (a SecureString, NaN) fails on its own, like an item whose body is not valid JSON; the rest of the batch is sent. + ```yaml Type: Object Parameter Sets: (All) diff --git a/module/help/Invoke-MgxRequest.md b/module/help/Invoke-MgxRequest.md index 342f4cd..ada75cd 100644 --- a/module/help/Invoke-MgxRequest.md +++ b/module/help/Invoke-MgxRequest.md @@ -183,6 +183,8 @@ Accept wildcard characters: False ### -Body Request body for write operations (POST, PATCH, PUT). Accepts a hashtable, PSObject or array, which is serialized to JSON, or a string already holding JSON, which is sent verbatim. Ignored (with a warning) on GET. +Enums serialize as camelCase names, byte arrays as base64, TimeSpan as an ISO-8601 duration, and a DateTime without a kind is treated as UTC. A SecureString, PSCredential, script block, certificate, NaN or Infinity anywhere in the body is refused before anything is sent, with an error naming the property. A string that is not JSON is sent verbatim; add a Content-Type to -Headers to declare its real type. + ```yaml Type: Object Parameter Sets: (All) diff --git a/module/mgx.psd1 b/module/mgx.psd1 index de63877..90e044d 100644 --- a/module/mgx.psd1 +++ b/module/mgx.psd1 @@ -1,6 +1,6 @@ @{ RootModule = 'mgx.psm1' - ModuleVersion = '2.1.2' + ModuleVersion = '2.1.3' GUID = 'a3f7e8d2-5b4c-4a1e-9f6d-2c8b0e3a7d5f' Author = 'Thomas Maillo Grome' CompanyName = 'Mgx' @@ -52,8 +52,24 @@ LicenseUri = 'https://github.com/gromedev/mgx/blob/main/LICENSE' ProjectUri = 'https://github.com/gromedev/mgx' ReleaseNotes = @' -v2.1.2 +v2.1.3 +Added +- about_Mgx_Errors documentation: termination behavior, error records, -ErrorAction/-ErrorVariable handling. +- Live test suite supports client secret authentication alongside certificates. +Fixed +- SecureString, credential objects, and script blocks in -Body throw explicit errors rather than serializing as reflection noise. +- Server errors, timeouts, and precondition failures assign specific ErrorCategory values instead of NotSpecified. +- Get-MgxContent forwards the conditional header family (including If-None-Match) to content download hosts. +- Failed batch chunks under -ErrorAction Stop write dead-letter files prior to terminating. +- Typed parameters deferring to options already present in -Uri emit warnings rather than dropping silently. +- Request bodies over 4 MB pass directly to the SDK chain under Enable-MgxResilience instead of being rejected. +- HTTP 304 on conditional downloads completes cleanly without an error record. +- Circular self-referencing objects in -Body raise validation errors instead of a process stack overflow. +Changed +- Unified failure classification across retries, circuit breaking, batch retry checks, content downloads, and adaptive pacing. +See CHANGELOG.md for the full list. +v2.1.2 Fixed - URI handling: a '#' in a path dropped everything after it, pre-encoded query values were encoded twice, piped drive and item ids went unescaped, an absolute -Uri was mangled rather than refused, and a typed parameter could repeat an option already in -Uri. - Body serialization: enums went out as numbers, byte arrays as integer lists, TimeSpan and a Kind-less DateTime in forms Graph refuses, and a PSCustomObject body lost every property that was not a NoteProperty. @@ -62,14 +78,7 @@ Fixed - An empty, non-JSON or malformed response body ended the cmdlet with an unhandled error. - An unexpected failure discarded the retry history explaining it, and a timed-out attempt could retry after Ctrl-C. -v2.1.1 -Patch release. One fix; no feature or API changes. - -Fixed -- Enable-MgxResilience ignored throttling: pacing never slowed, telemetry reported no retries, and a throttled request could fail outright. - -Changed -- Under Enable-MgxResilience, a 503 or 504 on a write is no longer retried. Throttling (429) is unaffected. +See CHANGELOG.md for the full history. '@ } } diff --git a/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs b/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs index 766c30c..bcaf1c1 100644 --- a/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs +++ b/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs @@ -997,10 +997,59 @@ protected static HashSet ExistingQueryOptions(string url) var q = url.IndexOf('?'); if (q < 0) return set; foreach (var part in url[(q + 1)..].Split('&', StringSplitOptions.RemoveEmptyEntries)) - set.Add(part.Split('=', 2)[0]); + // Decoded: %24top and $top are the same option to the server. + set.Add(Uri.UnescapeDataString(part.Split('=', 2)[0])); return set; } + private static readonly Dictionary s_optionParameterNames = new(StringComparer.OrdinalIgnoreCase) + { + ["$filter"] = "-Filter", ["$select"] = "-Property", ["$orderby"] = "-Sort", + ["$search"] = "-Search", ["$expand"] = "-ExpandProperty", ["$skip"] = "-Skip", + ["$top"] = "-Top", + }; + + ///

One warning line for typed parameters that deferred to options in -Uri. + protected static string DescribeDeferredOptions(IReadOnlyList deferred) + { + var names = deferred.Select(o => + s_optionParameterNames.TryGetValue(o, out var param) ? $"{param} ({o})" : o); + return $"{string.Join(", ", names)}: -Uri already carries the option, so the parameter's " + + "value was not added to the query. The -Uri value was sent."; + } + + /// The decoded value of a query option in a built URL, or null when absent. + protected static string? GetQueryOptionValue(string url, string name) + { + var q = url.IndexOf('?'); + if (q < 0) return null; + foreach (var part in url[(q + 1)..].Split('&', StringSplitOptions.RemoveEmptyEntries)) + { + var pieces = part.Split('=', 2); + if (string.Equals(Uri.UnescapeDataString(pieces[0]), name, StringComparison.OrdinalIgnoreCase)) + return pieces.Length > 1 ? Uri.UnescapeDataString(pieces[1]) : string.Empty; + } + return null; + } + + /// + /// Query-degradation retries, shared by the collection paths: some endpoints refuse an + /// auto-added $count=true with a bare 400, and some (e.g. /directoryRoles) refuse $top + /// with Request_UnsupportedQuery. Both recover by dropping the option and retrying. + /// + protected static bool IsCountRejection(GraphServiceException ex, bool countWasAutoAdded) + => countWasAutoAdded && ex.StatusCode == HttpStatusCode.BadRequest; + + protected static bool IsTopRejection(GraphServiceException ex, bool topSuppressed, bool noPageSize) + => !topSuppressed && !noPageSize + && ex.StatusCode == HttpStatusCode.BadRequest + && string.Equals(ex.ErrorCode, "Request_UnsupportedQuery", StringComparison.OrdinalIgnoreCase); + + protected const string CountRejectedVerbose = + "Endpoint rejected $count=true (HTTP 400). Retrying without count parameter."; + protected const string TopRejectedVerbose = + "Endpoint rejected $top (Request_UnsupportedQuery). Retrying without page size."; + private static readonly Regex s_encodedTriplet = new("%[0-9A-Fa-f]{2}", RegexOptions.Compiled); /// @@ -1076,6 +1125,17 @@ protected record ODataListParams( bool IncludeCount = false); protected static string BuildListUrl(string versionedBaseUrl, string relativeUri, ODataListParams p) + => BuildListUrl(versionedBaseUrl, relativeUri, p, out _); + + /// + /// reports typed parameters that were NOT sent because + /// -Uri already carries the option. Callers warn: silently preferring the -Uri value + /// turns "-Property displayName" into a different projection with no sign anything was + /// ignored. Excluded from the report: the automatic page-size $top (only an explicit + /// -Top counts) and the automatic $count. + /// + protected static string BuildListUrl(string versionedBaseUrl, string relativeUri, ODataListParams p, + out List deferredToUri) { var baseUrl = $"{versionedBaseUrl}{NormalizePath(relativeUri)}"; var queryParams = new List(); @@ -1116,7 +1176,13 @@ protected static string BuildListUrl(string versionedBaseUrl, string relativeUri queryParams.Add("$count=true"); var existing = ExistingQueryOptions(baseUrl); + var removed = queryParams.Where(qp => existing.Contains(qp.Split('=', 2)[0])) + .Select(qp => qp.Split('=', 2)[0]).ToList(); queryParams.RemoveAll(qp => existing.Contains(qp.Split('=', 2)[0])); + deferredToUri = removed.Where(name => + !string.Equals(name, "$count", StringComparison.OrdinalIgnoreCase) + && (!string.Equals(name, "$top", StringComparison.OrdinalIgnoreCase) || p.Top > 0)) + .ToList(); if (queryParams.Count == 0) return baseUrl; @@ -1167,16 +1233,11 @@ protected void WriteBetaHintIfApplicable(HttpStatusCode statusCode, string apiVe WriteWarning("This endpoint may only be available in beta. Retry with -ApiVersion beta."); } - protected static ErrorCategory MapStatusToCategory(HttpStatusCode statusCode) => statusCode switch - { - HttpStatusCode.NotFound => ErrorCategory.ObjectNotFound, - HttpStatusCode.Unauthorized => ErrorCategory.AuthenticationError, - HttpStatusCode.Forbidden => ErrorCategory.PermissionDenied, - HttpStatusCode.BadRequest => ErrorCategory.InvalidArgument, - HttpStatusCode.Conflict => ErrorCategory.ResourceExists, - (HttpStatusCode)429 => ErrorCategory.LimitsExceeded, - _ => ErrorCategory.NotSpecified - }; + // Derived from the failure class, so every status the classifier knows gets a real + // category: a 500 is ResourceUnavailable and a 408 ConnectionError where the old + // six-entry map left both NotSpecified. + protected static ErrorCategory MapStatusToCategory(HttpStatusCode statusCode) + => MgxErrorPresentation.CategoryForStatus(statusCode); /// /// Handles the three common terminal exception types (GraphServiceException, diff --git a/src/Mgx.Cmdlets/Base/MgxErrorPresentation.cs b/src/Mgx.Cmdlets/Base/MgxErrorPresentation.cs new file mode 100644 index 0000000..ad60e92 --- /dev/null +++ b/src/Mgx.Cmdlets/Base/MgxErrorPresentation.cs @@ -0,0 +1,77 @@ +using System.Management.Automation; +using System.Net; +using Mgx.Engine.Errors; +using Mgx.Engine.Models; +using Polly.CircuitBreaker; + +namespace Mgx.Cmdlets.Base; + +/// +/// How a classified failure surfaces to PowerShell: the ErrorCategory for a class, the +/// status inside an exception, and the shared presentation of infrastructure failures. +/// Before this existed, the category map covered six statuses and left a 500 in +/// NotSpecified, the status extraction existed verbatim in two cmdlets, and the fan-out +/// error switches disagreed on whether a broken circuit carried its inner status. +/// +internal static class MgxErrorPresentation +{ + /// The PowerShell ErrorCategory for a failure class. + internal static ErrorCategory Category(MgxErrorClass cls) => cls switch + { + MgxErrorClass.Throttle => ErrorCategory.LimitsExceeded, + MgxErrorClass.TransientServer => ErrorCategory.ResourceUnavailable, + MgxErrorClass.TransientTransport => ErrorCategory.ConnectionError, + MgxErrorClass.Authentication => ErrorCategory.AuthenticationError, + MgxErrorClass.Authorization => ErrorCategory.PermissionDenied, + MgxErrorClass.Consistency => ErrorCategory.ResourceUnavailable, + MgxErrorClass.InvalidRequest => ErrorCategory.InvalidArgument, + MgxErrorClass.NotFound => ErrorCategory.ObjectNotFound, + MgxErrorClass.Conflict => ErrorCategory.ResourceExists, + _ => ErrorCategory.NotSpecified, + }; + + /// The category for a status, through the classifier. + internal static ErrorCategory CategoryForStatus(HttpStatusCode statusCode) + => Category(MgxErrorClassifier.Classify((int)statusCode).Class); + + /// The HTTP status a thrown exception carries, when it carries one. + internal static HttpStatusCode? TryGetStatus(Exception ex) + { + if (ex is GraphServiceException gse) return gse.StatusCode; + if (ex is HttpRequestException hre && hre.StatusCode.HasValue) return hre.StatusCode.Value; + return null; + } + + /// Whether -SkipNotFound / -SkipForbidden apply to this status. + internal static bool ShouldSkip(HttpStatusCode? status, bool skipNotFound, bool skipForbidden) + => (skipNotFound && status == HttpStatusCode.NotFound) + || (skipForbidden && status == HttpStatusCode.Forbidden); + + /// + /// One presentation for a per-item failure: id, category, and the exception to report. + /// A broken circuit is rewrapped with the session's guidance text - previously one + /// fan-out path did this and the other did not, so the same failure surfaced + /// differently depending on the cmdlet. Always ResourceUnavailable: the breaker's + /// exception never carries a Graph error (those are constructed above the pipeline), + /// and "stop sending" is the message whatever opened it. + /// + internal static (string ErrorId, ErrorCategory Category, Exception Report) PresentItemFailure( + Exception ex, string fallbackErrorId, string circuitMessage) + { + return ex switch + { + BrokenCircuitException bce => ( + "CircuitBroken", + ErrorCategory.ResourceUnavailable, + new InvalidOperationException(circuitMessage, bce)), + HttpRequestException => ("HttpError", ErrorCategory.ConnectionError, ex), + System.Text.Json.JsonException => ( + "MalformedJsonResponse", ErrorCategory.InvalidData, + new InvalidOperationException($"A response declared JSON but does not parse: {ex.Message}", ex)), + _ => ( + fallbackErrorId, + TryGetStatus(ex) is { } status ? CategoryForStatus(status) : ErrorCategory.NotSpecified, + ex), + }; + } +} diff --git a/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs b/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs index 2ae3300..641a67f 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs @@ -25,6 +25,13 @@ namespace Mgx.Cmdlets.Cmdlets.Batch; [OutputType(typeof(Hashtable))] public class InvokeMgxBatchRequest : MgxCmdletBase { + /// Dead-letter lines are read by people; keep non-ASCII readable, matching + /// the request serializer's escaping. + private static readonly JsonSerializerOptions s_deadLetterJsonOptions = new() + { + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + /// /// Graph API URLs to batch. Accepts absolute URLs (https://graph.microsoft.com/v1.0/users/id) /// or relative URLs (/users/id). Also accepts Hashtables or PSObjects with Url/Method/Body members. @@ -187,13 +194,15 @@ protected override void EndProcessing() JsonElement? body = null; if (input.Body != null) { - var json = InvokeMgxRequest.SerializeBody(input.Body); try { + var json = InvokeMgxRequest.SerializeBody(input.Body); body = JsonSerializer.Deserialize(json); } - catch (JsonException ex) + catch (Exception ex) when (ex is JsonException or ArgumentException) { + // ArgumentException: a value serialization refuses (SecureString, NaN), + // named by property path. JsonException: a string body that is not JSON. WriteError(new ErrorRecord( new ArgumentException( $"Body for {input.Method} {input.Url} is not valid JSON: {ex.Message}", ex), @@ -241,16 +250,6 @@ protected override void EndProcessing() WriteObject(result); } - // A chunk's POST failed after earlier chunks were applied. Their results are above; - // this says why the rest never went, and NotSent names them. - if (batchResult.ChunkFailure != null) - { - var notSent = batchResult.NotSent.Count; - WriteError(new ErrorRecord(batchResult.ChunkFailure, "BatchChunkFailed", - ErrorCategory.NotSpecified, - $"{notSent} of {results.Count} operations were not sent")); - } - // Write failed items to dead-letter file (append mode) if (resolvedDeadLetterPath != null) { @@ -284,7 +283,7 @@ protected override void EndProcessing() if (errorMsg != null) deadLetter["Error"] = errorMsg; - writer.WriteLine(deadLetter.ToJsonString()); + writer.WriteLine(deadLetter.ToJsonString(s_deadLetterJsonOptions)); failedCount++; } } @@ -297,6 +296,21 @@ protected override void EndProcessing() WriteVerbose($"Wrote {failedCount} failed items to dead-letter file: {resolvedDeadLetterPath}"); } + // A chunk's POST failed after earlier chunks were applied. Their results are above; + // this says why the rest never went, and NotSent names them. After the dead-letter + // write, like the item errors below, so -ErrorAction Stop cannot cut the file short. + if (batchResult.ChunkFailure != null) + { + var notSent = batchResult.NotSent.Count; + // The id stays BatchChunkFailed; the category and wrapping come from the + // failure itself - a throttled chunk is LimitsExceeded, an open circuit + // ResourceUnavailable with the guidance text, not NotSpecified. + var (_, category, report) = MgxErrorPresentation.PresentItemFailure( + batchResult.ChunkFailure, "BatchChunkFailed", CircuitBreakerMessage); + WriteError(new ErrorRecord(report, "BatchChunkFailed", category, + $"{notSent} of {results.Count} operations were not sent")); + } + // Emit errors for failed items (enables -ErrorAction Stop, populates $Error). // After the dead-letter write, so -ErrorAction Stop cannot cut the file short. for (int i = 0; i < results.Count; i++) diff --git a/src/Mgx.Cmdlets/Cmdlets/Content/GetMgxContent.cs b/src/Mgx.Cmdlets/Cmdlets/Content/GetMgxContent.cs index 81c93e7..d824c0c 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Content/GetMgxContent.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Content/GetMgxContent.cs @@ -78,8 +78,8 @@ protected override void BeginProcessing() { if (ParameterSetName == "Uri") { - if (Uri.StartsWith("https://", StringComparison.OrdinalIgnoreCase) || - Uri.StartsWith("http://", StringComparison.OrdinalIgnoreCase)) + if (Uri.TrimStart().StartsWith("https://", StringComparison.OrdinalIgnoreCase) || + Uri.TrimStart().StartsWith("http://", StringComparison.OrdinalIgnoreCase)) { ThrowTerminatingError(new ErrorRecord( new ArgumentException( @@ -123,21 +123,36 @@ protected override void BeginProcessing() protected override void ProcessRecord() { - if (ParameterSetName == "Uri") + if (ParameterSetName != "Uri" && InputObject == null) { - FetchContent(downloadUrl: null, - relativeUri: Uri, - errorTarget: Uri); + // A null item was never a download; it must not trip the one-OutFile guard. + WriteVerbose("Skipping null pipeline input."); return; } - if (InputObject == null) + if (_wroteOutFile && !string.IsNullOrEmpty(OutFile)) { - WriteVerbose("Skipping null pipeline input."); + // Refuse rather than overwrite - and refuse BEFORE fetching, so the second + // item's content is not downloaded just to be discarded. Checked here, not in + // BeginProcessing, so the legitimate single-piped-item case still works. + ThrowTerminatingError(new ErrorRecord( + new InvalidOperationException( + "-OutFile writes a single file, but more than one item was piped in. " + + "Each item would overwrite the last. Pipe one item, or omit -OutFile " + + "and redirect the byte[] output per item."), + "OutFileWithMultipleInputs", ErrorCategory.InvalidOperation, OutFile)); return; } - var value = UnwrapPSObject(InputObject); + if (ParameterSetName == "Uri") + { + FetchContent(downloadUrl: null, + relativeUri: Uri, + errorTarget: Uri); + return; + } + + var value = UnwrapPSObject(InputObject!); // null handled above // Prefer the item's own pre-authenticated download URL: no hop-1 round trip, no // request-budget charge. Validated against the allowlist before any fetch. @@ -206,7 +221,8 @@ private void FetchContent(string? downloadUrl, string? relativeUri, object? erro { using var result = downloadUrl != null ? GraphContentClient.GetFromDownloadUrlAsync( - downloadUrl, range, client.BodyReadTimeout, CancellationToken) + downloadUrl, range, client.BodyReadTimeout, CancellationToken, + BuildRequestHeaders(null, Headers)) .GetAwaiter().GetResult() : client.GetContentAsync( $"{VersionedBaseUrl}{NormalizePath(relativeUri!)}", range, @@ -234,18 +250,6 @@ private void FetchContent(string? downloadUrl, string? relativeUri, object? erro if (_resolvedOutFile != null) { - if (_wroteOutFile) - { - // Refuse rather than overwrite. Checked here, not in BeginProcessing, so the - // legitimate single-piped-item case still works. - ThrowTerminatingError(new ErrorRecord( - new InvalidOperationException( - "-OutFile writes a single file, but more than one item was piped in. " - + "Each item would overwrite the last. Pipe one item, or omit -OutFile " - + "and redirect the byte[] output per item."), - "OutFileWithMultipleInputs", ErrorCategory.InvalidOperation, OutFile)); - return; - } WriteToFile(result, maxBytes, skipBytes); _wroteOutFile = true; } @@ -259,6 +263,13 @@ private void FetchContent(string? downloadUrl, string? relativeUri, object? erro DrainClientMessages(); WriteWarning("Content download cancelled."); } + catch (GraphServiceException ex) when (ex.StatusCode == HttpStatusCode.NotModified) + { + // The conditional request worked: the caller's If-None-Match/If-Modified-Since + // matched, and there is nothing newer to download. Not an error. + DrainClientMessages(); + WriteVerbose("Not modified; the content matches the caller's condition. Nothing downloaded."); + } catch (InvalidOperationException ex) { // Validator refusal or transport-redirect detection: a security boundary, not a diff --git a/src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs b/src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs index 77d97a8..ace6a1f 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs @@ -99,8 +99,8 @@ private static string NormalizeSelect(string? s) => protected override void BeginProcessing() { // Reject absolute URLs (relative paths only) - if (Uri.StartsWith("https://", StringComparison.OrdinalIgnoreCase) || - Uri.StartsWith("http://", StringComparison.OrdinalIgnoreCase)) + if (Uri.TrimStart().StartsWith("https://", StringComparison.OrdinalIgnoreCase) || + Uri.TrimStart().StartsWith("http://", StringComparison.OrdinalIgnoreCase)) { ThrowTerminatingError(new ErrorRecord( new ArgumentException( @@ -190,8 +190,14 @@ protected override void ProcessRecord() DeleteCheckpoint(resolvedCheckpointPath, "full sync requested"); } - // Normalize $select and Prefer for order-independent comparison - var normalizedSelect = NormalizeSelect(Property != null ? string.Join(",", Property) : null); + // Normalize $select and Prefer for order-independent comparison. The effective + // select is what will go on the wire: a $select already in -Uri wins over + // -Property (the builder defers to it, with a warning), and the state must + // record and compare the wire value or every later run trips the consistency + // check against a select that was never sent. + var normalizedSelect = NormalizeSelect( + GetQueryOptionValue(Uri, "$select") + ?? (Property != null ? string.Join(",", Property) : null)); var normalizedPrefer = NormalizeSelect(Prefer != null ? string.Join(",", Prefer) : null); var currentFilter = Filter; string requestUrl; @@ -261,7 +267,15 @@ protected override void ProcessRecord() // Normalized $select comparison (order-independent, deduplicated) var storedSelect = NormalizeSelect(existingState.Select); - if (!string.Equals(storedSelect, normalizedSelect, StringComparison.OrdinalIgnoreCase)) + // State written before the wire-value change recorded the -Property form even + // when -Uri carried the $select. If the stored value matches THAT form for + // the same invocation, nothing actually changed on the wire - accept it, and + // the state is rewritten in the new form on save. + var legacySelect = NormalizeSelect(Property != null ? string.Join(",", Property) : null); + var selectUnchanged = + string.Equals(storedSelect, normalizedSelect, StringComparison.OrdinalIgnoreCase) + || string.Equals(storedSelect, legacySelect, StringComparison.OrdinalIgnoreCase); + if (!selectUnchanged) { WriteWarning( "Property selection changed since last sync " @@ -377,7 +391,10 @@ protected override void ProcessRecord() else { requestUrl = BuildListUrl(VersionedBaseUrl, Uri, - new ODataListParams(false, Top, Top > 0 ? Top : 999, Filter, Property, null, null, 0, null)); + new ODataListParams(false, Top, Top > 0 ? Top : 999, Filter, Property, null, null, 0, null), + out var deferred); + if (deferred.Count > 0) + WriteWarning(DescribeDeferredOptions(deferred)); if (Latest.IsPresent && !honourLatest) { diff --git a/src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs b/src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs index 9be9a20..852b19e 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs @@ -82,8 +82,8 @@ protected override void BeginProcessing() // Reject absolute URLs (relative paths only); concatenation onto the versioned // base URL would otherwise silently produce /v1.0/https:/... on the wire. - if (Uri.StartsWith("https://", StringComparison.OrdinalIgnoreCase) || - Uri.StartsWith("http://", StringComparison.OrdinalIgnoreCase)) + if (Uri.TrimStart().StartsWith("https://", StringComparison.OrdinalIgnoreCase) || + Uri.TrimStart().StartsWith("http://", StringComparison.OrdinalIgnoreCase)) { ThrowTerminatingError(new ErrorRecord( new ArgumentException( @@ -381,7 +381,7 @@ private string BuildUrl(string id) // -Uri: Graph rejects a URL carrying $top twice, which turned -Top into a 400 for // every input object. // The client-side truncation in the lambda enforces -Top either way. - if (Top > 0 && !HasTopQueryOption(url)) + if (Top > 0 && !ExistingQueryOptions(url).Contains("$top")) { var separator = url.Contains('?') ? "&" : "?"; url = $"{url}{separator}$top={Top}"; @@ -390,19 +390,6 @@ private string BuildUrl(string id) return url; } - /// True when the URL already carries a $top query option. - private static bool HasTopQueryOption(string url) - { - var q = url.IndexOf('?'); - if (q < 0) return false; - foreach (var part in url[(q + 1)..].Split('&')) - { - var name = part.Split('=', 2)[0]; - if (string.Equals(name, "$top", StringComparison.OrdinalIgnoreCase)) return true; - } - return false; - } - private void HandleFanOutErrors( Dictionary errors, Dictionary urlToId) @@ -413,7 +400,7 @@ private void HandleFanOutErrors( foreach (var (url, ex) in errors) { var id = urlToId.GetValueOrDefault(url, url); - var statusCode = GetStatusCodeFromException(ex); + var statusCode = MgxErrorPresentation.TryGetStatus(ex); if (SkipNotFound.IsPresent && statusCode == HttpStatusCode.NotFound) { @@ -426,19 +413,9 @@ private void HandleFanOutErrors( continue; } - var (errorId, category) = ex switch - { - BrokenCircuitException bce => ("CircuitBroken", bce.InnerException is GraphServiceException inner - ? MapStatusToCategory(inner.StatusCode) - : ErrorCategory.ResourceUnavailable), - HttpRequestException => ("HttpError", ErrorCategory.ConnectionError), - _ => ("ExpandRelationError", statusCode.HasValue - ? MapStatusToCategory(statusCode.Value) - : ErrorCategory.NotSpecified) - }; - Exception reportEx = ex is BrokenCircuitException - ? new InvalidOperationException(CircuitBreakerMessage, ex) : ex; - WriteError(new ErrorRecord(reportEx, errorId, category, id)); + var (errorId, category, report) = + MgxErrorPresentation.PresentItemFailure(ex, "ExpandRelationError", CircuitBreakerMessage); + WriteError(new ErrorRecord(report, errorId, category, id)); } int skippedTotal = skipped404 + skipped403; @@ -451,10 +428,4 @@ private void HandleFanOutErrors( } } - private static HttpStatusCode? GetStatusCodeFromException(Exception ex) - { - if (ex is GraphServiceException gse) return gse.StatusCode; - if (ex is HttpRequestException hre && hre.StatusCode.HasValue) return hre.StatusCode.Value; - return null; - } } diff --git a/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs b/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs index ec6b644..ef4d8df 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs @@ -84,8 +84,8 @@ protected override void BeginProcessing() { // Reject absolute URLs (relative paths only); concatenation onto the versioned // base URL would otherwise silently produce /v1.0/https:/... on the wire. - if (Uri.StartsWith("https://", StringComparison.OrdinalIgnoreCase) || - Uri.StartsWith("http://", StringComparison.OrdinalIgnoreCase)) + if (Uri.TrimStart().StartsWith("https://", StringComparison.OrdinalIgnoreCase) || + Uri.TrimStart().StartsWith("http://", StringComparison.OrdinalIgnoreCase)) { ThrowTerminatingError(new ErrorRecord( new ArgumentException( @@ -180,7 +180,8 @@ protected override void ProcessRecord() // Track whether $count=true was auto-added (not user-requested). // If the endpoint rejects it with 400, retry without. - bool countAutoAdded = !string.IsNullOrEmpty(Filter); + bool countAutoAdded = !string.IsNullOrEmpty(Filter) + && !ExistingQueryOptions(Uri).Contains("$count"); bool includeAutoCount = countAutoAdded; bool suppressTop = false; @@ -484,22 +485,27 @@ protected override void ProcessRecord() WriteWarning($"Export cancelled. {resumeHint}"); return; } - catch (GraphServiceException ex) when (includeAutoCount && countAutoAdded && ex.StatusCode == HttpStatusCode.BadRequest) + catch (System.Text.Json.JsonException ex) { - // Auto-added $count=true rejected by this endpoint; retry without it + // A page body that does not parse. Items already exported stay in the file; + // report the failure instead of ending the pipeline with a raw exception. DrainClientMessages(); - WriteVerbose("Endpoint rejected $count=true (HTTP 400). Retrying without count parameter."); + WriteError(new ErrorRecord( + new InvalidOperationException($"A response page declared JSON but does not parse: {ex.Message}", ex), + "MalformedJsonResponse", ErrorCategory.InvalidData, Uri)); + return; + } + catch (GraphServiceException ex) when (IsCountRejection(ex, includeAutoCount && countAutoAdded)) + { + DrainClientMessages(); + WriteVerbose(CountRejectedVerbose); includeAutoCount = false; continue; } - catch (GraphServiceException ex) when ( - !suppressTop - && !NoPageSize.IsPresent - && ex.StatusCode == HttpStatusCode.BadRequest - && string.Equals(ex.ErrorCode, "Request_UnsupportedQuery", StringComparison.OrdinalIgnoreCase)) + catch (GraphServiceException ex) when (IsTopRejection(ex, suppressTop, NoPageSize.IsPresent)) { DrainClientMessages(); - WriteVerbose("Endpoint rejected $top (Request_UnsupportedQuery). Retrying without page size."); + WriteVerbose(TopRejectedVerbose); suppressTop = true; continue; } @@ -532,11 +538,23 @@ protected override void ProcessRecord() } } - private string BuildUrl(bool includeCount, bool suppressTop = false) => BuildListUrl( - VersionedBaseUrl, Uri, - new ODataListParams(NoPageSize.IsPresent || suppressTop, Top, PageSize, Filter, - Property, Sort, Search, Skip, ExpandProperty, - IncludeCount: includeCount)); + private bool _warnedDeferredOptions; + + private string BuildUrl(bool includeCount, bool suppressTop = false) + { + var url = BuildListUrl( + VersionedBaseUrl, Uri, + new ODataListParams(NoPageSize.IsPresent || suppressTop, Top, PageSize, Filter, + Property, Sort, Search, Skip, ExpandProperty, + IncludeCount: includeCount), + out var deferred); + if (deferred.Count > 0 && !_warnedDeferredOptions) + { + _warnedDeferredOptions = true; + WriteWarning(DescribeDeferredOptions(deferred)); + } + return url; + } private Dictionary? BuildHeaders() => BuildRequestHeaders(ConsistencyLevel, Headers); diff --git a/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs b/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs index eddd69f..a42b4c1 100644 --- a/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs +++ b/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs @@ -142,8 +142,8 @@ protected override void BeginProcessing() { // Reject absolute URLs (relative paths only); concatenation onto the versioned // base URL would otherwise silently produce /v1.0/https:/... on the wire. - if (Uri.StartsWith("https://", StringComparison.OrdinalIgnoreCase) || - Uri.StartsWith("http://", StringComparison.OrdinalIgnoreCase)) + if (Uri.TrimStart().StartsWith("https://", StringComparison.OrdinalIgnoreCase) || + Uri.TrimStart().StartsWith("http://", StringComparison.OrdinalIgnoreCase)) { ThrowTerminatingError(new ErrorRecord( new ArgumentException( @@ -295,9 +295,11 @@ private void ExecuteRequest(string relativeUri, string? sourceId) private void ExecuteList(string relativeUri, string? sourceId) { - // Track whether $count=true was auto-added (not user-requested via -CountVariable). - // If the endpoint rejects it with 400, retry without. - bool countAutoAdded = !string.IsNullOrEmpty(Filter) && string.IsNullOrEmpty(CountVariable); + // Track whether $count=true was auto-added (not user-requested via -CountVariable, + // and not already written into -Uri, where dropping it changes nothing and the + // "retry without count" rebuild would re-send a byte-identical request). + bool countAutoAdded = !string.IsNullOrEmpty(Filter) && string.IsNullOrEmpty(CountVariable) + && !ExistingQueryOptions(Uri).Contains("$count"); bool includeAutoCount = countAutoAdded; bool suppressTop = false; @@ -447,21 +449,15 @@ private void ExecuteList(string relativeUri, string? sourceId) "MalformedJsonResponse", ErrorCategory.InvalidData, relativeUri)); return; } - catch (GraphServiceException ex) when (includeAutoCount && countAutoAdded && ex.StatusCode == HttpStatusCode.BadRequest) + catch (GraphServiceException ex) when (IsCountRejection(ex, includeAutoCount && countAutoAdded)) { - // Auto-added $count=true rejected by this endpoint; retry without it - WriteVerbose("Endpoint rejected $count=true (HTTP 400). Retrying without count parameter."); + WriteVerbose(CountRejectedVerbose); includeAutoCount = false; continue; } - catch (GraphServiceException ex) when ( - !suppressTop - && !NoPageSize.IsPresent - && ex.StatusCode == HttpStatusCode.BadRequest - && string.Equals(ex.ErrorCode, "Request_UnsupportedQuery", StringComparison.OrdinalIgnoreCase)) + catch (GraphServiceException ex) when (IsTopRejection(ex, suppressTop, NoPageSize.IsPresent)) { - // Endpoint doesn't support $top (e.g., /directoryRoles). Retry without page size. - WriteVerbose("Endpoint rejected $top (Request_UnsupportedQuery). Retrying without page size."); + WriteVerbose(TopRejectedVerbose); suppressTop = true; continue; } @@ -519,13 +515,25 @@ private void ExecuteWrite(HttpMethod method, string relativeUri, string? sourceI if (!ShouldProcess(relativeUri, method.Method)) return; + string? serializedBody; + try + { + serializedBody = ResolveRequestBody(method); + } + catch (ArgumentException ex) + { + // Pre-flight: nothing was sent. Terminating, per about_Mgx_Errors. + ThrowTerminatingError(new ErrorRecord(ex, "InvalidBodyValue", + ErrorCategory.InvalidArgument, relativeUri)); + return; + } + NoteNonJsonStringBody(); + try { var url = $"{VersionedBaseUrl}{NormalizePath(relativeUri)}"; var client = GetClient(); var headers = BuildHeaders(); - - var serializedBody = ResolveRequestBody(method); HttpContent? content = serializedBody != null ? new StringContent(serializedBody, Encoding.UTF8, "application/json") : null; @@ -733,10 +741,22 @@ private void ExecuteWriteFanOut(ConcurrentFanOut fanOut, List uniqueIds, if (!ShouldProcess($"{httpMethod.Method} {uniqueIds.Count} items via {Uri}", "Bulk write")) return; + string? serializedBody; try { // Serialize body once (shared across all operations) - string? serializedBody = ResolveRequestBody(httpMethod); + serializedBody = ResolveRequestBody(httpMethod); + } + catch (ArgumentException ex) + { + ThrowTerminatingError(new ErrorRecord(ex, "InvalidBodyValue", + ErrorCategory.InvalidArgument, Uri)); + return; + } + NoteNonJsonStringBody(); + + try + { // Build operations list: (id, resolved URL) var operations = uniqueIds.Select(id => @@ -843,7 +863,7 @@ private void HandleFanOutErrors(Dictionary errors) bool has404 = false; foreach (var (key, ex) in errors) { - var statusCode = GetStatusCodeFromException(ex); + var statusCode = MgxErrorPresentation.TryGetStatus(ex); // Only a 404 that might mean "no such endpoint" is worth a beta hint. A 404 naming // a missing object says the path was fine, and hinting over it sends the caller to @@ -862,14 +882,9 @@ private void HandleFanOutErrors(Dictionary errors) continue; } - // Preserve diagnostic specificity for infrastructure exceptions - var (errorId, category) = ex switch - { - Polly.CircuitBreaker.BrokenCircuitException => ("CircuitBroken", ErrorCategory.ResourceUnavailable), - HttpRequestException => ("HttpError", ErrorCategory.ConnectionError), - _ => ("FanOutError", statusCode.HasValue ? MapStatusToCategory(statusCode.Value) : ErrorCategory.NotSpecified) - }; - WriteError(new ErrorRecord(ex, errorId, category, key)); + var (errorId, category, report) = + MgxErrorPresentation.PresentItemFailure(ex, "FanOutError", CircuitBreakerMessage); + WriteError(new ErrorRecord(report, errorId, category, key)); } if (has404) @@ -896,13 +911,6 @@ private string ResolveTemplate(string id) System.Text.RegularExpressions.RegexOptions.IgnoreCase); } - private static HttpStatusCode? GetStatusCodeFromException(Exception ex) - { - if (ex is GraphServiceException gse) return gse.StatusCode; - if (ex is HttpRequestException hre && hre.StatusCode.HasValue) return hre.StatusCode.Value; - return null; - } - /// /// Check if a GraphServiceException should be silently skipped based on /// -SkipNotFound / -SkipForbidden switches. Used by single-request paths @@ -931,11 +939,27 @@ private bool ShouldSkipGraphError(GraphServiceException ex) private string BuildCollectionUrl(string relativeUri) => BuildCollectionUrl(relativeUri, includeCount: !string.IsNullOrEmpty(CountVariable) || !string.IsNullOrEmpty(Filter)); - private string BuildCollectionUrl(string relativeUri, bool includeCount, bool noPageSize = false) => BuildListUrl( - VersionedBaseUrl, relativeUri, - new ODataListParams(NoPageSize.IsPresent || noPageSize, Top, PageSize, Filter, - Property, Sort, Search, Skip, ExpandProperty, - IncludeCount: includeCount)); + private int _warnedDeferredOptions; + + private void WarnDeferredOptions(List deferred) + { + // Interlocked: URL building runs inside fan-out lambdas that can resume off the + // pipeline thread; the warning must fire exactly once and from one caller. + if (deferred.Count == 0 || Interlocked.Exchange(ref _warnedDeferredOptions, 1) == 1) return; + WriteWarning(DescribeDeferredOptions(deferred)); + } + + private string BuildCollectionUrl(string relativeUri, bool includeCount, bool noPageSize = false) + { + var url = BuildListUrl( + VersionedBaseUrl, relativeUri, + new ODataListParams(NoPageSize.IsPresent || noPageSize, Top, PageSize, Filter, + Property, Sort, Search, Skip, ExpandProperty, + IncludeCount: includeCount), + out var deferred); + WarnDeferredOptions(deferred); + return url; + } private string BuildGetUrl(string relativeUri) { @@ -949,6 +973,8 @@ private string BuildGetUrl(string relativeUri) queryParams.Add($"$expand={EscapeQueryValue(string.Join(",", ExpandProperty))}"); var existing = ExistingQueryOptions(baseUrl); + WarnDeferredOptions(queryParams.Where(qp => existing.Contains(qp.Split('=', 2)[0])) + .Select(qp => qp.Split('=', 2)[0]).ToList()); queryParams.RemoveAll(qp => existing.Contains(qp.Split('=', 2)[0])); if (queryParams.Count == 0) @@ -971,6 +997,29 @@ private string BuildGetUrl(string relativeUri) /// action endpoints such as /directoryObjects/getByIds) is unwrapped into one item per element; /// anything else is emitted whole. /// + /// + /// A string -Body travels verbatim under Content-Type: application/json. When it is not + /// JSON that is usually intentional (a raw upload) - but the endpoint sees the wrong + /// content type unless -Headers overrides it, and the resulting 400 names neither. + /// A verbose note, not a warning: raw string bodies are a supported path. + /// + private void NoteNonJsonStringBody() + { + if (Body is null || UnwrapPSObject(Body) is not string s || string.IsNullOrWhiteSpace(s)) return; + if (Headers != null && Headers.Keys.Cast() + .Any(k => string.Equals(k.ToString(), "Content-Type", StringComparison.OrdinalIgnoreCase))) + return; + try + { + using var _ = JsonDocument.Parse(s); + } + catch (JsonException) + { + WriteVerbose("-Body is a string that is not JSON; it is sent verbatim with " + + "Content-Type: application/json. Add a Content-Type to -Headers to declare its real type."); + } + } + /// /// A response body as JSON for output, or null when nothing further should be emitted: /// an empty body (204, or 200 with no content), a non-JSON body (emitted as text under @@ -984,12 +1033,13 @@ private string BuildGetUrl(string relativeUri) return null; } + var encoding = ResolveEncoding(response.Content.Headers.ContentType?.CharSet); var contentType = response.Content.Headers.ContentType?.MediaType; if (contentType != null && !contentType.EndsWith("json", StringComparison.OrdinalIgnoreCase)) { if (Raw.IsPresent) { - WriteObject(Encoding.UTF8.GetString(bodyBytes)); + WriteObject(encoding.GetString(bodyBytes)); return null; } WriteError(new ErrorRecord( @@ -999,13 +1049,17 @@ private string BuildGetUrl(string relativeUri) return null; } + // Through a string, not the raw bytes: honors the declared charset. GetString keeps + // a BOM as U+FEFF, which the serializer refuses but the old stream path accepted - + // trim it explicitly. + var text = encoding.GetString(bodyBytes).TrimStart('\uFEFF'); try { - return JsonSerializer.Deserialize(bodyBytes); + return JsonSerializer.Deserialize(text); } catch (JsonException) { - var snippet = Encoding.UTF8.GetString(bodyBytes, 0, Math.Min(bodyBytes.Length, 200)); + var snippet = text[..Math.Min(text.Length, 200)]; WriteError(new ErrorRecord( new InvalidOperationException( $"The response declared JSON but does not parse. Body starts: {snippet}"), @@ -1014,6 +1068,15 @@ private string BuildGetUrl(string relativeUri) } } + /// The response's declared charset, defaulting to UTF-8. GetString strips a + /// matching BOM, which the byte-level Deserialize refused. + private static Encoding ResolveEncoding(string? charset) + { + if (string.IsNullOrEmpty(charset)) return Encoding.UTF8; + try { return Encoding.GetEncoding(charset.Trim('"')); } + catch (ArgumentException) { return Encoding.UTF8; } + } + private void OutputPayload(JsonElement json, string? sourceId) { var items = TryUnwrapCollection(json, out var truncated); @@ -1137,6 +1200,7 @@ public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializer internal static string SerializeBody(object body) { var value = UnwrapPSObject(body); + RefuseUnserializable(value, "-Body"); if (value is string s) return s; // Still a PSObject after unwrapping: a PSCustomObject, whose members live on the wrapper if (value is PSObject pso) return JsonSerializer.Serialize(PSOToDict(pso), BodyJsonOptions); @@ -1146,7 +1210,7 @@ internal static string SerializeBody(object body) return JsonSerializer.Serialize(value, BodyJsonOptions); } - internal static Dictionary PSOToDict(PSObject pso) + internal static Dictionary PSOToDict(PSObject pso, string path = "-Body") { // Every property kind, matching what ConvertTo-Json reads - so -Body $obj and // -Body ($obj | ConvertTo-Json) produce the same members. ScriptProperties are @@ -1158,7 +1222,7 @@ internal static string SerializeBody(object body) object? raw; try { raw = prop.Value; } catch (GetValueException) { raw = null; } - dict[prop.Name] = UnwrapValue(raw); + dict[prop.Name] = UnwrapValue(raw, $"{path}.{prop.Name}"); } return dict; } @@ -1167,11 +1231,11 @@ internal static string SerializeBody(object body) /// Flatten any IDictionary (Hashtable or ordered dictionary) into a serializable /// dictionary, unwrapping nested PowerShell values. /// - internal static Dictionary DictionaryToDict(IDictionary source) + internal static Dictionary DictionaryToDict(IDictionary source, string path = "-Body") { var dict = new Dictionary(); foreach (DictionaryEntry entry in source) - dict[entry.Key.ToString()!] = UnwrapValue(entry.Value); + dict[entry.Key.ToString()!] = UnwrapValue(entry.Value, $"{path}.{entry.Key}"); return dict; } @@ -1180,28 +1244,66 @@ internal static string SerializeBody(object body) /// A PSCustomObject keeps its members on the PSObject (its BaseObject is an empty /// marker), so it must be read through PSOToDict rather than its BaseObject. /// - internal static object? UnwrapValue(object? value) + private const int MaxBodyDepth = 64; + + internal static object? UnwrapValue(object? value, string path = "-Body") { + // Depth from the path: each nesting level appends a segment. A self-referencing + // hashtable ($h.self = $h) recursed to a StackOverflowException, which no catch + // can stop - the process died. 64 levels is far beyond any real Graph body. + if (path.Length > MaxBodyDepth * 8) + { + var depth = path.Count(c => c is '.' or '['); + if (depth > MaxBodyDepth) + throw new ArgumentException( + $"The value at '{path[..64]}...' nests deeper than {MaxBodyDepth} levels. " + + "Is the body self-referencing?"); + } if (value is PSObject pso) { var unwrapped = UnwrapPSObject(pso); - return ReferenceEquals(unwrapped, pso) ? PSOToDict(pso) : UnwrapValue(unwrapped); + return ReferenceEquals(unwrapped, pso) ? PSOToDict(pso, path) : UnwrapValue(unwrapped, path); } + RefuseUnserializable(value, path); if (value is IDictionary dict) - return DictionaryToDict(dict); + return DictionaryToDict(dict, path); // byte[] is Edm.Binary and serializes as base64; flattening it into object?[] // would emit a JSON array of integers instead. if (value is IEnumerable seq and not (string or byte[])) - return EnumerableToArray(seq); + return EnumerableToArray(seq, path); return value; } + /// + /// Values that would serialize to something other than what they mean. A SecureString + /// reflects to {"Length":8} - the request "succeeds" carrying garbage instead of the + /// secret, or worse, would carry the secret if it round-tripped. NaN and Infinity have + /// no JSON representation, and STJ's own refusal does not say which property. + /// + private static void RefuseUnserializable(object? value, string path) + { + switch (value) + { + case System.Security.SecureString: + case PSCredential: + case ScriptBlock: + case System.Security.Cryptography.X509Certificates.X509Certificate: + throw new ArgumentException( + $"The value at '{path}' is a {value.GetType().Name}, which does not serialize " + + "to JSON meaningfully. Convert it to what the endpoint expects before sending."); + case double d when double.IsNaN(d) || double.IsInfinity(d): + throw new ArgumentException($"The value at '{path}' is {d}; JSON has no representation for it."); + case float f when float.IsNaN(f) || float.IsInfinity(f): + throw new ArgumentException($"The value at '{path}' is {f}; JSON has no representation for it."); + } + } + /// /// Flatten any non-string sequence (object[], ArrayList, List<object>, ...) into an array of /// unwrapped values. /// - private static object?[] EnumerableToArray(IEnumerable source) => - source.Cast().Select(UnwrapValue).ToArray(); + private static object?[] EnumerableToArray(IEnumerable source, string path = "-Body") => + source.Cast().Select((v, i) => UnwrapValue(v, $"{path}[{i}]")).ToArray(); #endregion diff --git a/src/Mgx.Engine/Errors/MgxErrorClass.cs b/src/Mgx.Engine/Errors/MgxErrorClass.cs new file mode 100644 index 0000000..64a22a6 --- /dev/null +++ b/src/Mgx.Engine/Errors/MgxErrorClass.cs @@ -0,0 +1,48 @@ +namespace Mgx.Engine.Errors; + +/// +/// What went wrong, as one vocabulary shared by retry, circuit breaking, pacing, +/// telemetry, and the errors cmdlets surface. Classification happens in +/// ; what to do about a class happens in +/// . Nothing else in the codebase decides either question. +/// +public enum MgxErrorClass +{ + /// 429: the service is pacing this workload. Retryable for every method. + Throttle, + + /// 500/502/503/504: the service failed and may not have processed the request. + TransientServer, + + /// + /// The request may never have arrived: a connection failure, a timeout (408 or a + /// client-side attempt timeout), a reset mid-body. + /// + TransientTransport, + + /// 401: the token is missing, expired, or not valid here. + Authentication, + + /// 403: authenticated, but not permitted. + Authorization, + + /// + /// Declared for the read-after-write window (a 404 immediately following a successful + /// write). Nothing produces it yet; the consistency policy that will is future work, + /// and telemetry and category mappings are complete for it now so adding the producer + /// does not change any schema. + /// + Consistency, + + /// 400 and kin: the request is malformed and will fail identically every time. + InvalidRequest, + + /// 404: the path is valid and the object is not there. + NotFound, + + /// 409/412: the object changed underneath the request. + Conflict, + + /// Everything else, including cancellation: retrying cannot help. + Permanent, +} diff --git a/src/Mgx.Engine/Errors/MgxErrorClassifier.cs b/src/Mgx.Engine/Errors/MgxErrorClassifier.cs new file mode 100644 index 0000000..2bc018f --- /dev/null +++ b/src/Mgx.Engine/Errors/MgxErrorClassifier.cs @@ -0,0 +1,70 @@ +using System.Net.Http.Headers; +using Polly.CircuitBreaker; +using Polly.Timeout; + +namespace Mgx.Engine.Errors; + +/// +/// The one place a status code or exception becomes an . +/// Before this existed the same decision lived in the retry predicate, the circuit-breaker +/// predicate, the batch item loop, the content download pipeline, and the pacer - each with +/// its own list, and the lists disagreed in ways none of them documented. +/// +public static class MgxErrorClassifier +{ + /// Classify an HTTP status. For non-success codes; a 2xx/3xx is not a failure + /// and lands in only because nothing retries it. + public static MgxErrorInfo Classify(int statusCode, string? errorCode = null, TimeSpan? serverRetryAfter = null) + { + var cls = statusCode switch + { + 429 => MgxErrorClass.Throttle, + 500 or 502 or 503 or 504 => MgxErrorClass.TransientServer, + // A 408 is a timeout the request may not have survived - transport, not server: + // the server did not fail, it never saw the whole request. + 408 => MgxErrorClass.TransientTransport, + 401 => MgxErrorClass.Authentication, + 403 => MgxErrorClass.Authorization, + 404 => MgxErrorClass.NotFound, + 409 or 412 => MgxErrorClass.Conflict, + 400 or 405 or 411 or 413 or 414 or 415 or 416 or 422 or 501 => MgxErrorClass.InvalidRequest, + // Unknown 5xx (505, 507, ...) deliberately Permanent: mgx has never retried + // them, and classifying them TransientServer would silently start. + _ => MgxErrorClass.Permanent, + }; + return new MgxErrorInfo(cls, statusCode, errorCode, serverRetryAfter); + } + + /// Classify a response. Reads status and Retry-After; never the body - + /// retry decisions run at ResponseHeadersRead, so the body is not available. + public static MgxErrorInfo Classify(HttpResponseMessage response, string? errorCode = null) + => Classify((int)response.StatusCode, errorCode, + RetryAfterPolicy.ServerRequested(response.Headers.RetryAfter)); + + /// + /// Classify an exception. is the caller's own + /// token state: a cancellation-shaped exception after the caller cancelled is + /// Permanent - nothing may act on it - while the same exception without a cancelled + /// token is a timeout, which is transport. + /// + public static MgxErrorInfo Classify(Exception exception, bool cancellationRequested) + { + if (cancellationRequested && exception + is OperationCanceledException or TimeoutRejectedException) + return new MgxErrorInfo(MgxErrorClass.Permanent, 0); + + var cls = exception switch + { + HttpRequestException => MgxErrorClass.TransientTransport, + System.Net.Sockets.SocketException => MgxErrorClass.TransientTransport, + IOException => MgxErrorClass.TransientTransport, + // TaskCanceledException without user cancellation is HttpClient's own timeout. + OperationCanceledException => MgxErrorClass.TransientTransport, + TimeoutRejectedException => MgxErrorClass.TransientTransport, + // The breaker is open; retrying into it is the thing it exists to stop. + BrokenCircuitException => MgxErrorClass.Permanent, + _ => MgxErrorClass.Permanent, + }; + return new MgxErrorInfo(cls, 0); + } +} diff --git a/src/Mgx.Engine/Errors/MgxErrorInfo.cs b/src/Mgx.Engine/Errors/MgxErrorInfo.cs new file mode 100644 index 0000000..8c698b0 --- /dev/null +++ b/src/Mgx.Engine/Errors/MgxErrorInfo.cs @@ -0,0 +1,16 @@ +namespace Mgx.Engine.Errors; + +/// +/// The classified facts of a failure. Facts only - what to do with them +/// (retry, count against the breaker, surface) belongs to . +/// +/// The failure class. +/// HTTP status, or 0 when there was no response (transport, timeout, circuit). +/// Graph's error.code, when the body has been read. Retry-time +/// classification runs at ResponseHeadersRead and never has it. +/// The server's Retry-After, unclamped. Policy clamps. +public readonly record struct MgxErrorInfo( + MgxErrorClass Class, + int StatusCode, + string? ErrorCode = null, + TimeSpan? ServerRetryAfter = null); diff --git a/src/Mgx.Engine/Errors/MgxErrorPolicy.cs b/src/Mgx.Engine/Errors/MgxErrorPolicy.cs new file mode 100644 index 0000000..506874b --- /dev/null +++ b/src/Mgx.Engine/Errors/MgxErrorPolicy.cs @@ -0,0 +1,42 @@ +namespace Mgx.Engine.Errors; + +/// +/// What to do about a classified failure. Retry and circuit breaking legitimately +/// disagree - that disagreement lives here, in one visible place, instead of being +/// implied by two lists a hundred lines apart. +/// +public static class MgxErrorPolicy +{ + /// + /// Whether another attempt may help. Throttles retry for every method including POST + /// (matches the Kiota SDK); server and transport failures retry only when the method is + /// idempotent, because a 5xx or a dead connection may mean a write was already applied. + /// + public static bool ShouldRetry(MgxErrorClass cls, bool isIdempotent) => cls switch + { + MgxErrorClass.Throttle => true, + MgxErrorClass.TransientServer or MgxErrorClass.TransientTransport => isIdempotent, + _ => false, + }; + + /// + /// Whether the failure counts toward opening the circuit breaker. + /// 429 and 408 are excluded deliberately: 429 is the service pacing us, not failing - + /// Retry-After handles it, and counting it would open the circuit exactly when the + /// correct response is to slow down and keep going. 408 is a client-perceived timeout, + /// not a server-side failure indicator. + /// + public static bool CountsAsCircuitFailure(in MgxErrorInfo info) => + info.Class == MgxErrorClass.TransientServer + || (info.Class == MgxErrorClass.TransientTransport && info.StatusCode != 408); + + /// + /// The download-host retry filter, narrower than on purpose: + /// no 408 and no cancellation-shaped timeouts, because the content path handles stalls + /// with its own idle timeout and rebuilds hop 1 on an expired URL. Throttles, 5xx, and + /// connection failures retry; everything else surfaces. + /// + public static bool ShouldRetryDownload(in MgxErrorInfo info, Exception? exception) => + info.Class is MgxErrorClass.Throttle or MgxErrorClass.TransientServer + || exception is HttpRequestException; +} diff --git a/src/Mgx.Engine/Errors/RetryAfterPolicy.cs b/src/Mgx.Engine/Errors/RetryAfterPolicy.cs new file mode 100644 index 0000000..1ef12b8 --- /dev/null +++ b/src/Mgx.Engine/Errors/RetryAfterPolicy.cs @@ -0,0 +1,57 @@ +using System.Net.Http.Headers; + +namespace Mgx.Engine.Errors; + +/// +/// One reading of Retry-After. The header has two forms - delta-seconds and an HTTP date - +/// and before this existed the pipeline, the batch client, the content client, and the +/// pacer each parsed it themselves, two of them with their own caps. +/// +public static class RetryAfterPolicy +{ + /// The server's requested delay, unclamped: delta-seconds verbatim, or + /// date minus now (which may be negative for a past date). Null when absent. + public static TimeSpan? ServerRequested(RetryConditionHeaderValue? retryAfter) + => retryAfter?.Delta + ?? (retryAfter?.Date is { } date ? date - DateTimeOffset.UtcNow : null); + + /// + /// The delay to actually sleep: the server's request clamped to , + /// or null when the header is absent or asks for the past - callers fall back to their + /// exponential backoff. A delta is honored even at zero; a date is honored only in the + /// future. This is exactly what the pipeline and the content client each did alone. + /// + public static TimeSpan? Resolve(RetryConditionHeaderValue? retryAfter, TimeSpan cap) + { + if (retryAfter?.Delta is { } delta) + return delta > cap ? cap : delta; + if (retryAfter?.Date is { } date) + { + var delay = date - DateTimeOffset.UtcNow; + if (delay > TimeSpan.Zero) + return delay > cap ? cap : delay; + } + return null; + } + + /// + /// The batch form: whole seconds from a raw header string, rounded up, floored at zero, + /// clamped to . carries + /// what the server asked for so the caller can say when it clamped. + /// + public static bool TryResolveSeconds(string? headerValue, int capSeconds, + out int requestedSeconds, out int clampedSeconds) + { + requestedSeconds = 0; + clampedSeconds = 0; + if (headerValue == null || !RetryConditionHeaderValue.TryParse(headerValue, out var ra)) + return false; + + var delay = ra.Delta ?? (ra.Date.HasValue ? ra.Date.Value - DateTimeOffset.UtcNow : (TimeSpan?)null); + if (!delay.HasValue) return false; + + requestedSeconds = (int)Math.Ceiling(delay.Value.TotalSeconds); + clampedSeconds = Math.Min(Math.Max(requestedSeconds, 0), capSeconds); + return true; + } +} diff --git a/src/Mgx.Engine/Http/AdaptiveRequestPacer.cs b/src/Mgx.Engine/Http/AdaptiveRequestPacer.cs index f1bc093..3667f39 100644 --- a/src/Mgx.Engine/Http/AdaptiveRequestPacer.cs +++ b/src/Mgx.Engine/Http/AdaptiveRequestPacer.cs @@ -1,3 +1,4 @@ +using Mgx.Engine.Errors; using System.Diagnostics; using System.Globalization; @@ -355,12 +356,9 @@ internal static void RecordResponse(WorkloadBucket bucket, HttpResponseMessage r Volatile.Write(ref s_lastPercentageTicks[b], Stopwatch.GetTimestamp()); } - if ((int)response.StatusCode == 429) - { - TimeSpan? retryAfter = response.Headers.RetryAfter?.Delta - ?? (response.Headers.RetryAfter?.Date is { } d ? d - DateTimeOffset.UtcNow : null); - RecordThrottle(bucket, retryAfter); - } + var info = MgxErrorClassifier.Classify(response); + if (info.Class == MgxErrorClass.Throttle) + RecordThrottle(bucket, info.ServerRetryAfter); } /// diff --git a/src/Mgx.Engine/Http/GraphBatchClient.cs b/src/Mgx.Engine/Http/GraphBatchClient.cs index c06e0ff..d45bfa8 100644 --- a/src/Mgx.Engine/Http/GraphBatchClient.cs +++ b/src/Mgx.Engine/Http/GraphBatchClient.cs @@ -1,3 +1,4 @@ +using Mgx.Engine.Errors; using System.Collections.Concurrent; using System.Diagnostics; using System.Text; @@ -582,15 +583,10 @@ public async Task ExecuteBatchIndexedAsync( var retryAfterValue = item.Headers .FirstOrDefault(h => string.Equals(h.Key, "Retry-After", StringComparison.OrdinalIgnoreCase)) .Value; - if (retryAfterValue != null && - System.Net.Http.Headers.RetryConditionHeaderValue.TryParse(retryAfterValue, out var ra)) { - var delay = ra.Delta - ?? (ra.Date.HasValue ? ra.Date.Value - DateTimeOffset.UtcNow : (TimeSpan?)null); - if (delay.HasValue) + if (RetryAfterPolicy.TryResolveSeconds(retryAfterValue, _maxRetryAfterSeconds, + out var seconds, out var clamped)) { - var seconds = (int)Math.Ceiling(delay.Value.TotalSeconds); - var clamped = Math.Min(Math.Max(seconds, 0), _maxRetryAfterSeconds); if (clamped < seconds && attempt < MaxPerRequestRetries) { _pendingVerbose.Enqueue( @@ -660,14 +656,12 @@ public void DrainVerboseMessages() } /// - /// Determines if a batch response item should be retried. - /// POST is non-idempotent: only retry on 429 (matches Kiota SDK behavior), not on 5xx/408 (could create duplicates). - /// Other methods (GET, PATCH, PUT, DELETE) retry on 429/408/500/502/503/504 (aligned with ResiliencePipelineFactory). + /// Whether a batch response item should be retried: the same classifier and policy the + /// request pipeline uses, so the two cannot drift apart again. Batch items carry a + /// status and inline headers, never an exception - that is the whole difference. /// private static bool IsRetryable(int statusCode, string method) - { - if (statusCode == 429) return true; - if (string.Equals(method, "POST", StringComparison.OrdinalIgnoreCase)) return false; - return statusCode is 408 or 500 or 502 or 503 or 504; - } + => MgxErrorPolicy.ShouldRetry( + MgxErrorClassifier.Classify(statusCode).Class, + isIdempotent: !string.Equals(method, "POST", StringComparison.OrdinalIgnoreCase)); } diff --git a/src/Mgx.Engine/Http/GraphContentClient.cs b/src/Mgx.Engine/Http/GraphContentClient.cs index 433010e..f762712 100644 --- a/src/Mgx.Engine/Http/GraphContentClient.cs +++ b/src/Mgx.Engine/Http/GraphContentClient.cs @@ -1,3 +1,4 @@ +using Mgx.Engine.Errors; using System.Net; using System.Net.Http.Headers; using Polly; @@ -99,18 +100,10 @@ or HttpStatusCode.SeeOther or HttpStatusCode.TemporaryRedirect /// is the only bound on the sleep. /// internal static TimeSpan? ResolveRetryDelay(RetryConditionHeaderValue? retryAfter) - { - var cap = TimeSpan.FromSeconds(120); - if (retryAfter?.Delta is { } delta) - return delta > cap ? cap : delta; - if (retryAfter?.Date is { } date) - { - var delay = date - DateTimeOffset.UtcNow; - if (delay > TimeSpan.Zero) - return delay > cap ? cap : delay; - } - return null; - } + // The 120s cap is hardcoded because this pipeline is a type-initialized static and + // cannot see ResilientGraphClientOptions; making it options-driven is a redesign of + // this class, not a parameter change. + => RetryAfterPolicy.Resolve(retryAfter, TimeSpan.FromSeconds(120)); private static readonly ResiliencePipeline s_downloadPipeline = new ResiliencePipelineBuilder() @@ -123,13 +116,13 @@ or HttpStatusCode.SeeOther or HttpStatusCode.TemporaryRedirect MaxDelay = TimeSpan.FromSeconds(120), ShouldHandle = args => { - if (args.Outcome.Result?.StatusCode is (HttpStatusCode)429 - or HttpStatusCode.InternalServerError - or HttpStatusCode.BadGateway - or HttpStatusCode.ServiceUnavailable - or HttpStatusCode.GatewayTimeout) - return ValueTask.FromResult(true); - return ValueTask.FromResult(args.Outcome.Exception is HttpRequestException); + var info = args.Outcome.Result is { } response + ? MgxErrorClassifier.Classify(response) + : args.Outcome.Exception is { } ex + ? MgxErrorClassifier.Classify(ex, cancellationRequested: false) + : new MgxErrorInfo(MgxErrorClass.Permanent, 0); + return ValueTask.FromResult( + MgxErrorPolicy.ShouldRetryDownload(info, args.Outcome.Exception)); }, DelayGenerator = args => ValueTask.FromResult(ResolveRetryDelay(args.Outcome.Result?.Headers.RetryAfter)), @@ -153,9 +146,12 @@ public static async Task GetContentAsync( CancellationToken cancellationToken) { var graphHost = new Uri(requestUri).Host; + // Preserve case-insensitivity: Dictionary(IDictionary) silently swaps in the + // ordinal comparer, which made the hop-2 conditional forward and the + // client-request-id suppression casing-sensitive on this path only. var requestHeaders = headers != null - ? new Dictionary(headers) - : new Dictionary(); + ? new Dictionary(headers, StringComparer.OrdinalIgnoreCase) + : new Dictionary(StringComparer.OrdinalIgnoreCase); if (range != null) requestHeaders["Range"] = range.ToString(); @@ -201,7 +197,8 @@ public static async Task GetContentAsync( $"Download host '{TryGetHost(absolute)}' is not on the allowed list " + "(SharePoint/OneDrive download hosts only). Refusing to fetch content from it."); - var hop2 = await FetchFromDownloadHostAsync(validated, range, cancellationToken); + var hop2 = await FetchFromDownloadHostAsync(validated, range, cancellationToken, + FilterDownloadHeaders(requestHeaders)); if (hop2.StatusCode is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden && authAttempt == 0) @@ -239,14 +236,16 @@ public static async Task GetFromDownloadUrlAsync( string downloadUrl, RangeHeaderValue? range, TimeSpan bodyReadTimeout, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + Dictionary? headers = null) { var validated = DownloadUrlValidator.Validate(downloadUrl) ?? throw new InvalidOperationException( $"Download host '{TryGetHost(downloadUrl)}' is not on the allowed list " + "(SharePoint/OneDrive download hosts only). Refusing to fetch content from it."); - var response = await FetchFromDownloadHostAsync(validated, range, cancellationToken); + var response = await FetchFromDownloadHostAsync(validated, range, cancellationToken, + FilterDownloadHeaders(headers)); if (response.IsSuccessStatusCode) return await WrapAsync(response, fromDownloadHost: true, bodyReadTimeout, cancellationToken); @@ -256,12 +255,38 @@ public static async Task GetFromDownloadUrlAsync( throw new Models.GraphServiceException(status, body); } + /// + /// The caller headers that cross to the download host: the READ-conditional family + /// plus Accept. Everything else stays on hop 1 - the download host is token-free by + /// construction, and correlation or auth headers must never reach it. If-Match and + /// If-Unmodified-Since stay behind too: they are write validators, and a Graph etag + /// forwarded to a SharePoint blob host cannot match - it would turn a shared + /// -Headers splat into a 412 on every download. + /// + private static readonly string[] s_forwardableDownloadHeaders = + ["If-None-Match", "If-Modified-Since", "If-Range", "Accept"]; + + private static Dictionary? FilterDownloadHeaders(Dictionary? headers) + { + if (headers == null) return null; + Dictionary? filtered = null; + foreach (var (key, value) in headers) + { + // Compare by name, not by the source dictionary's comparer - a caller-built + // dictionary may be case-sensitive, and header names are not. + if (s_forwardableDownloadHeaders.Any(n => string.Equals(n, key, StringComparison.OrdinalIgnoreCase))) + (filtered ??= new Dictionary(StringComparer.OrdinalIgnoreCase))[key] = value; + } + return filtered; + } + /// /// Token-free fetch with manual, re-validated redirects. Every hop must pass the /// allowlist: an open redirect on an allowlisted host is not a pass. /// private static async Task FetchFromDownloadHostAsync( - string url, RangeHeaderValue? range, CancellationToken cancellationToken) + string url, RangeHeaderValue? range, CancellationToken cancellationToken, + Dictionary? conditionalHeaders = null) { var current = url; for (var redirects = 0; ; redirects++) @@ -277,6 +302,9 @@ private static async Task FetchFromDownloadHostAsync( var request = new HttpRequestMessage(HttpMethod.Get, target); if (range != null) request.Headers.Range = range; + if (conditionalHeaders != null) + foreach (var (name, value) in conditionalHeaders) + request.Headers.TryAddWithoutValidation(name, value); return await (DownloadClientForTests ?? s_downloadClient).SendAsync( request, HttpCompletionOption.ResponseHeadersRead, ctx.CancellationToken); }, diff --git a/src/Mgx.Engine/Http/ResiliencePipelineFactory.cs b/src/Mgx.Engine/Http/ResiliencePipelineFactory.cs index c2a269c..fc540fd 100644 --- a/src/Mgx.Engine/Http/ResiliencePipelineFactory.cs +++ b/src/Mgx.Engine/Http/ResiliencePipelineFactory.cs @@ -1,3 +1,4 @@ +using Mgx.Engine.Errors; using System.Net; using System.Net.Http.Headers; using System.Threading.RateLimiting; @@ -6,6 +7,8 @@ using Polly.Retry; using Polly.Timeout; + + namespace Mgx.Engine.Http; /// @@ -138,70 +141,26 @@ private static ResiliencePipeline BuildPipeline(ResilientGr MaxDelay = TimeSpan.FromSeconds(maxRetryAfterCap), ShouldHandle = args => { - // 429 is safe to retry for all methods including POST (matches Kiota SDK behavior) - if (args.Outcome.Result?.StatusCode == (HttpStatusCode)429) - return ValueTask.FromResult(true); - - // For non-idempotent methods (POST), only retry 429. - // 500/502/503/504 may mean the request was partially processed. + // Classification and the retry decision live in Errors/, shared with the + // batch client and the download pipeline; ErrorPolicyParityTests holds + // them to the decisions this predicate used to make inline. Per-attempt + // timeouts (TimeoutRejectedException) retry so the Enable-MgxResilience + // path survives the SDK honoring a Retry-After longer than + // AttemptTimeoutSeconds; the outer TotalTimeout still bounds the whole. var isIdempotent = args.Context.Properties.GetValue(IsIdempotentKey, true); - if (!isIdempotent) - return ValueTask.FromResult(false); - - if (args.Outcome.Result?.StatusCode is HttpStatusCode.InternalServerError - or HttpStatusCode.BadGateway - or HttpStatusCode.ServiceUnavailable - or HttpStatusCode.GatewayTimeout - or HttpStatusCode.RequestTimeout) - return ValueTask.FromResult(true); - - if (args.Outcome.Exception is HttpRequestException) - return ValueTask.FromResult(true); - - // Retry TaskCanceledException only if NOT caused by user cancellation (Ctrl+C). - // Polly's attempt timeout throws TimeoutRejectedException (not TaskCanceledException), - // so TaskCanceledException here means either user cancelled or HttpClient timeout. - if (args.Outcome.Exception is TaskCanceledException && - !args.Context.CancellationToken.IsCancellationRequested) - return ValueTask.FromResult(true); - - // Retry when Polly's per-attempt timeout fires (idempotent methods only). - // This is critical for the Enable-MgxResilience path: the SDK's internal - // RetryHandler may be honoring a Retry-After delay that exceeds - // AttemptTimeoutSeconds. Without this, requests fail permanently when - // Graph returns Retry-After > AttemptTimeoutSeconds. - // The outer TotalTimeout still bounds the overall operation. - // Never after cancellation: an attempt timeout that fires while the caller - // is cancelling must not schedule another attempt. - if (args.Outcome.Exception is TimeoutRejectedException && - !args.Context.CancellationToken.IsCancellationRequested) - return ValueTask.FromResult(isIdempotent); - - return ValueTask.FromResult(false); + var info = args.Outcome.Result is { } response + ? MgxErrorClassifier.Classify(response) + : args.Outcome.Exception is { } ex + ? MgxErrorClassifier.Classify(ex, args.Context.CancellationToken.IsCancellationRequested) + : new MgxErrorInfo(MgxErrorClass.Permanent, 0); + return ValueTask.FromResult(MgxErrorPolicy.ShouldRetry(info.Class, isIdempotent)); }, DelayGenerator = args => - { - // Respect Retry-After header from Graph API (429 responses) - if (args.Outcome.Result?.Headers.RetryAfter is RetryConditionHeaderValue retryAfter) - { - if (retryAfter.Delta.HasValue) - { - var delay = retryAfter.Delta.Value; - var cap = TimeSpan.FromSeconds(maxRetryAfterCap); - return ValueTask.FromResult(delay > cap ? cap : delay); - } - if (retryAfter.Date.HasValue) - { - var delay = retryAfter.Date.Value - DateTimeOffset.UtcNow; - if (delay > TimeSpan.Zero) - { - var cap = TimeSpan.FromSeconds(maxRetryAfterCap); - return ValueTask.FromResult(delay > cap ? cap : delay); - } - } - } - return ValueTask.FromResult(null); - }, + // Respect Retry-After from the service; null falls back to the + // exponential backoff configured above. + ValueTask.FromResult(RetryAfterPolicy.Resolve( + args.Outcome.Result?.Headers.RetryAfter, + TimeSpan.FromSeconds(maxRetryAfterCap))), OnRetry = args => { var response = args.Outcome.Result; @@ -258,28 +217,25 @@ or HttpStatusCode.GatewayTimeout return default; } }) - // Circuit breaker (5xx, not 429 or 408) - // 408 excluded: it's a client-perceived timeout, not a server-side failure indicator. - // Including it would cause proxy timeouts to trip the circuit breaker incorrectly. + // Circuit breaker. What counts as a circuit failure - and why 429 and 408 + // deliberately do not - lives in MgxErrorPolicy.CountsAsCircuitFailure, + // beside the retry policy it diverges from. .AddCircuitBreaker(new CircuitBreakerStrategyOptions { FailureRatio = options.CircuitBreakerFailureRatio, SamplingDuration = TimeSpan.FromSeconds(options.CircuitBreakerSamplingDurationSeconds), MinimumThroughput = options.CircuitBreakerMinThroughput, BreakDuration = TimeSpan.FromSeconds(options.CircuitBreakerDurationSeconds), - ShouldHandle = new PredicateBuilder() - .HandleResult(r => r.StatusCode == HttpStatusCode.InternalServerError) - .HandleResult(r => r.StatusCode == HttpStatusCode.BadGateway) - .HandleResult(r => r.StatusCode == HttpStatusCode.ServiceUnavailable) - .HandleResult(r => r.StatusCode == HttpStatusCode.GatewayTimeout) - .Handle() - // Exclude user cancellation (Ctrl+C) from circuit breaker failure counting. - // Only count non-user TaskCanceledException (e.g., HttpClient timeout). - .Handle(e => !e.CancellationToken.IsCancellationRequested) - // Count per-attempt timeouts as failures. Without this, repeated - // timeouts (e.g., downstream hung) never trip the circuit breaker, - // wasting MaxRetryAttempts * AttemptTimeoutSeconds before giving up. - .Handle(), + ShouldHandle = args => + { + var info = args.Outcome.Result is { } response + ? MgxErrorClassifier.Classify(response) + : args.Outcome.Exception is { } ex and not BrokenCircuitException + ? MgxErrorClassifier.Classify(ex, + (ex as OperationCanceledException)?.CancellationToken.IsCancellationRequested ?? false) + : new MgxErrorInfo(MgxErrorClass.Permanent, 0); + return ValueTask.FromResult(MgxErrorPolicy.CountsAsCircuitFailure(info)); + }, OnOpened = _ => { MgxTelemetryCollector.Current.RecordCircuitBreakerTrip(); diff --git a/src/Mgx.Engine/Http/ResilientDelegatingHandler.cs b/src/Mgx.Engine/Http/ResilientDelegatingHandler.cs index 6865b08..f363f53 100644 --- a/src/Mgx.Engine/Http/ResilientDelegatingHandler.cs +++ b/src/Mgx.Engine/Http/ResilientDelegatingHandler.cs @@ -86,15 +86,34 @@ protected override async Task SendAsync( // Buffer content upfront so retries can reconstruct a fresh request body. // Also snapshot all content headers (not just ContentType) to preserve // Content-Encoding, Content-Disposition, etc. on retry. + // What the pipeline cannot replay, it does not manage. A body whose declared + // length exceeds the replay-buffer cap (a drive-content upload runs to 250MB) + // passes to the SDK chain untouched: no clone, no retry-option stamp, no + // per-attempt timeout, no circuit counting - exactly what the SDK does without + // the wrap, which is the only promise the wrap can keep for a stream it cannot + // rewind. A body with no declared length is buffered whatever its size - the + // read is unavoidable to know, and once buffered it replays like any other. + if (request.Content?.Headers.ContentLength > ResilientGraphClient.MaxRequestBodyBytes) + { + var passSw = System.Diagnostics.Stopwatch.StartNew(); + var passSucceeded = false; + try + { + var passthrough = await base.SendAsync(request, cancellationToken); + passSucceeded = passthrough.IsSuccessStatusCode; + return passthrough; + } + finally + { + MgxTelemetryCollector.Current.RecordRequest(passSucceeded, passSw.ElapsedMilliseconds); + } + } + byte[]? contentBytes = null; List>>? contentHeaders = null; if (request.Content != null) { contentBytes = await request.Content.ReadAsByteArrayAsync(cancellationToken); - if (contentBytes.Length > ResilientGraphClient.MaxRequestBodyBytes) - throw new InvalidOperationException( - $"Request body size ({contentBytes.Length:N0} bytes) exceeds the {ResilientGraphClient.MaxRequestBodyBytes / (1024 * 1024)}MB limit. " + - "Graph API rejects bodies larger than 4MB on most endpoints."); contentHeaders = request.Content.Headers.ToList(); } var clientRequestId = Guid.NewGuid().ToString(); diff --git a/src/Mgx.Engine/Http/ResilientGraphClient.cs b/src/Mgx.Engine/Http/ResilientGraphClient.cs index cd7563f..607f4b0 100644 --- a/src/Mgx.Engine/Http/ResilientGraphClient.cs +++ b/src/Mgx.Engine/Http/ResilientGraphClient.cs @@ -24,7 +24,12 @@ public sealed class ResilientGraphClient : IDisposable private readonly ConcurrentQueue _pendingWarnings = new(); private readonly ConcurrentQueue _pendingDebug = new(); - /// Maximum request body size (4MB). Graph API rejects larger bodies on most endpoints. + /// + /// Maximum request body size (4MB). Bodies on this path are JSON to Graph endpoints, + /// where 4MB is the service's own cap, so refusing early is a favor. The resilience + /// wrap deliberately differs: it carries SDK requests (content uploads run to 250MB) + /// and passes an over-cap body to the SDK untouched instead of refusing it. + /// internal const int MaxRequestBodyBytes = 4 * 1024 * 1024; /// @@ -219,14 +224,32 @@ public async Task SendAsync( { foreach (var (key, value) in headers) { + // Content-Length is computed from the actual body; a caller value + // can only disagree with it, and the transport fails the request + // on either direction of the mismatch. + if (string.Equals(key, "Content-Length", StringComparison.OrdinalIgnoreCase)) + { + if (attempt == 1) + _pendingWarnings.Enqueue( + "Header 'Content-Length' is computed from the request body and was not overridden."); + continue; + } if (request.Headers.TryAddWithoutValidation(key, value)) continue; // The request collection refuses content headers (Content-Type, // Content-Disposition, ...); they belong on the content, replacing - // any default the buffered content carried. + // any default the buffered content carried. Remove throws on a + // malformed name where TryAdd merely returns false - a name both + // collections refuse must warn, not crash the request. if (request.Content != null) { - request.Content.Headers.Remove(key); - if (request.Content.Headers.TryAddWithoutValidation(key, value)) continue; + try + { + request.Content.Headers.Remove(key); + if (request.Content.Headers.TryAddWithoutValidation(key, value)) continue; + } + catch (Exception ex) when (ex is FormatException or ArgumentException) + { + } } if (attempt == 1) _pendingWarnings.Enqueue($"Header '{key}' is not valid on this request and was not sent."); diff --git a/tests/Live/Mgx.Live.Tests.ps1 b/tests/Live/Mgx.Live.Tests.ps1 index 248e7ce..4787a3e 100644 --- a/tests/Live/Mgx.Live.Tests.ps1 +++ b/tests/Live/Mgx.Live.Tests.ps1 @@ -15,8 +15,10 @@ BeforeAll { $repo = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent - if (-not $env:AZURE_TENANT_ID -or -not $env:AZURE_CLIENT_ID -or -not $env:AZURE_CLIENT_CERTIFICATE_PATH) { - throw "Live tests need AZURE_TENANT_ID, AZURE_CLIENT_ID and AZURE_CLIENT_CERTIFICATE_PATH. " + + if (-not $env:AZURE_TENANT_ID -or -not $env:AZURE_CLIENT_ID -or + (-not $env:AZURE_CLIENT_CERTIFICATE_PATH -and -not $env:AZURE_CLIENT_SECRET)) { + throw "Live tests need AZURE_TENANT_ID, AZURE_CLIENT_ID, and either " + + "AZURE_CLIENT_CERTIFICATE_PATH or AZURE_CLIENT_SECRET. " + "They are not skipped on purpose: a live suite that skips itself is a live suite nobody runs." } . "$repo/tests/benchmarks/common.ps1" diff --git a/tests/Live/Mgx.Throttle.Tests.ps1 b/tests/Live/Mgx.Throttle.Tests.ps1 index 54d696d..bd61299 100644 --- a/tests/Live/Mgx.Throttle.Tests.ps1 +++ b/tests/Live/Mgx.Throttle.Tests.ps1 @@ -22,8 +22,10 @@ BeforeAll { $repo = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent - if (-not $env:AZURE_TENANT_ID -or -not $env:AZURE_CLIENT_ID -or -not $env:AZURE_CLIENT_CERTIFICATE_PATH) { - throw "Live tests need AZURE_TENANT_ID, AZURE_CLIENT_ID and AZURE_CLIENT_CERTIFICATE_PATH." + if (-not $env:AZURE_TENANT_ID -or -not $env:AZURE_CLIENT_ID -or + (-not $env:AZURE_CLIENT_CERTIFICATE_PATH -and -not $env:AZURE_CLIENT_SECRET)) { + throw "Live tests need AZURE_TENANT_ID, AZURE_CLIENT_ID, and either " + + "AZURE_CLIENT_CERTIFICATE_PATH or AZURE_CLIENT_SECRET." } $script:Configured = [bool]$env:MGX_LIVE_THROTTLE_SSH . "$repo/tests/benchmarks/common.ps1" diff --git a/tests/Live/README.md b/tests/Live/README.md index 6fe8b8e..41cdc79 100644 --- a/tests/Live/README.md +++ b/tests/Live/README.md @@ -17,6 +17,10 @@ export AZURE_CLIENT_CERTIFICATE_PATH="$HOME/.certs/....pfx" pwsh -c 'Invoke-Pester -Path ./tests/Live' ``` +An app with a secret rather than a certificate works too - set `AZURE_CLIENT_SECRET` +instead of `AZURE_CLIENT_CERTIFICATE_PATH`. If both are set, the certificate wins. +Neither leaves anything on disk. + Missing credentials is a **failure**, not a skip. A live suite that skips itself when unconfigured is one that silently stops running. diff --git a/tests/Mgx.IntegrationTests/Cmdlets/BatchErrorSurfacingTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/BatchErrorSurfacingTests.cs index 5e330cf..f9435f0 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/BatchErrorSurfacingTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/BatchErrorSurfacingTests.cs @@ -30,8 +30,14 @@ private static void InjectTransport(HttpMessageHandler wire) private static void ResetTransport() { + // Restore every static InjectTransport touched - a later test in the collection + // that drives a cmdlet without injecting must not inherit this class's transport. Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); + Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); + Base.GetField("s_cachedTotalTimeoutSeconds", Static)!.SetValue(null, 0); + Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); + Base.GetField("s_clientOptions", Static)!.SetValue(null, new ResilientGraphClientOptions()); ResiliencePipelineFactory.Reset(); } @@ -127,6 +133,35 @@ protected override Task SendAsync( } } + [Fact] + public void Stop_preference_cannot_cut_the_dead_letter_file_short() + { + InjectTransport(new FirstChunkOnlyHandler()); + var deadLetter = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", Enumerable.Range(1, 25).Select(i => $"/users/u{i}").ToArray()) + .AddParameter("Method", "GET") + .AddParameter("DeadLetterPath", deadLetter) + .AddParameter("ErrorAction", "Stop"); + try { ps.Invoke(); } catch { /* Stop terminates - expected */ } + + // First chunk answered 20 items (all 200 in this fixture), second chunk's POST + // failed, so 5 items were never sent. Every failure must be on disk even though + // Stop terminated at the first error record. + Assert.True(File.Exists(deadLetter), "dead-letter file was not written"); + var lines = File.ReadAllLines(deadLetter); + Assert.Equal(5, lines.Length); + } + finally + { + File.Delete(deadLetter); + ResetTransport(); + } + } + [Fact] public void Items_a_failed_chunk_never_sent_each_write_an_error_record() { @@ -141,7 +176,10 @@ public void Items_a_failed_chunk_never_sent_each_write_an_error_record() var errors = ps.Streams.Error.ToList(); Assert.Equal(5, errors.Count(e => e.FullyQualifiedErrorId.StartsWith("BatchItemNotSent"))); - Assert.Equal(1, errors.Count(e => e.FullyQualifiedErrorId.StartsWith("BatchChunkFailed"))); + var chunk = Assert.Single(errors, e => e.FullyQualifiedErrorId.StartsWith("BatchChunkFailed")); + // The chunk died on a 400; the record carries the classified category, not NotSpecified. + Assert.Equal(System.Management.Automation.ErrorCategory.InvalidArgument, + chunk.CategoryInfo.Category); } finally { ResetTransport(); } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DiagnosticsDrainTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DiagnosticsDrainTests.cs index f9a28ec..cc47884 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/DiagnosticsDrainTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/DiagnosticsDrainTests.cs @@ -54,6 +54,9 @@ public void An_unexpected_exception_still_delivers_the_buffered_retry_history() { Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); + Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); + Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); + Base.GetField("s_clientOptions", Static)!.SetValue(null, new ResilientGraphClientOptions()); ResiliencePipelineFactory.Reset(); } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/HeaderWireTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/HeaderWireTests.cs index c143037..fb80fcd 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/HeaderWireTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/HeaderWireTests.cs @@ -28,8 +28,14 @@ private static void InjectTransport(HttpMessageHandler wire) private static void ResetTransport() { + // Restore every static InjectTransport touched - a later test in the collection + // that drives a cmdlet without injecting must not inherit this class's transport. Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); + Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); + Base.GetField("s_cachedTotalTimeoutSeconds", Static)!.SetValue(null, 0); + Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); + Base.GetField("s_clientOptions", Static)!.SetValue(null, new ResilientGraphClientOptions()); ResiliencePipelineFactory.Reset(); } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/OutFileGuardTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/OutFileGuardTests.cs new file mode 100644 index 0000000..163ca13 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/OutFileGuardTests.cs @@ -0,0 +1,74 @@ +using System.Net; +using System.Reflection; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// A second piped item against one -OutFile is refused BEFORE its content is fetched - +/// previously the whole body was downloaded and then thrown away with the refusal. +/// +[Collection("Pipeline")] +public class OutFileGuardTests +{ + private static readonly Type Base = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); + private const BindingFlags Static = BindingFlags.NonPublic | BindingFlags.Static; + + [Fact] + public void A_second_piped_item_is_refused_without_fetching_it() + { + var wire = new MockHttpHandler(); + wire.QueueBytes(HttpStatusCode.OK, [1, 2, 3, 4], "application/octet-stream"); + wire.QueueBytes(HttpStatusCode.OK, [5, 6, 7, 8], "application/octet-stream"); + + Base.GetField("s_graphHttpClient", Static)!.SetValue(null, new HttpClient(wire)); + Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, + Mgx.Cmdlets.Base.MgxCmdletBase.BuildAuthFingerprint( + new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); + Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, true); + Base.GetField("s_cachedTotalTimeoutSeconds", Static)!.SetValue(null, + new ResilientGraphClientOptions().TotalTimeoutSeconds); + Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); + Base.GetField("s_clientOptions", Static)!.SetValue(null, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + ResiliencePipelineFactory.Reset(); + + var outFile = Path.Combine(Path.GetTempPath(), $"mgx-outfile-{Guid.NewGuid():N}.bin"); + try + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript($$""" + @( + [Hashtable]@{ id = 'item1'; parentReference = @{ driveId = 'd1' } } + [Hashtable]@{ id = 'item2'; parentReference = @{ driveId = 'd1' } } + ) | Get-MgxContent -OutFile '{{outFile}}' + """); + ps.Invoke(); + + var error = Assert.Single(ps.Streams.Error); + Assert.StartsWith("OutFileWithMultipleInputs", error.FullyQualifiedErrorId); + + // The first item was fetched and written; the second was refused pre-fetch. + Assert.Single(wire.CapturedRequests); + Assert.Equal(new byte[] { 1, 2, 3, 4 }, File.ReadAllBytes(outFile)); + } + finally + { + File.Delete(outFile); + Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); + Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); + Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); + Base.GetField("s_cachedTotalTimeoutSeconds", Static)!.SetValue(null, 0); + Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); + Base.GetField("s_clientOptions", Static)!.SetValue(null, new ResilientGraphClientOptions()); + ResiliencePipelineFactory.Reset(); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ResponseFidelityTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ResponseFidelityTests.cs index 156edcc..fcdb829 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/ResponseFidelityTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/ResponseFidelityTests.cs @@ -30,8 +30,14 @@ private static void InjectTransport(HttpMessageHandler wire) private static void ResetTransport() { + // Restore every static InjectTransport touched - a later test in the collection + // that drives a cmdlet without injecting must not inherit this class's transport. Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); + Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); + Base.GetField("s_cachedTotalTimeoutSeconds", Static)!.SetValue(null, 0); + Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); + Base.GetField("s_clientOptions", Static)!.SetValue(null, new ResilientGraphClientOptions()); ResiliencePipelineFactory.Reset(); } @@ -161,6 +167,95 @@ public void A_malformed_page_during_enumeration_is_an_error_record_not_a_crash() finally { ResetTransport(); } } + [Fact] + public void A_bom_prefixed_json_body_parses() + { + var wire = new MockHttpHandler(); + var bom = new byte[] { 0xEF, 0xBB, 0xBF }; + var body = System.Text.Encoding.UTF8.GetBytes("""{"id":"u1","displayName":"BOM"}"""); + wire.QueueBytes(HttpStatusCode.OK, [.. bom, .. body], "application/json"); + InjectTransport(wire); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1"); + var output = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + var item = Assert.Single(output); + Assert.Equal("BOM", ((System.Collections.Hashtable)item.BaseObject)["displayName"]); + } + finally { ResetTransport(); } + } + + [Fact] + public void A_utf16_body_is_decoded_by_its_declared_charset() + { + var wire = new MockHttpHandler(); + wire.QueueBytes(HttpStatusCode.OK, + System.Text.Encoding.Unicode.GetBytes("plain utf-16 text"), + "text/plain; charset=utf-16"); + InjectTransport(wire); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1").AddParameter("Raw", true); + var output = ps.Invoke(); + + var item = Assert.Single(output); + Assert.Equal("plain utf-16 text", item.BaseObject); + } + finally { ResetTransport(); } + } + + [Fact] + public void A_malformed_export_page_is_an_error_record_not_a_crash() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "not a collection envelope"); + InjectTransport(wire); + var outFile = Path.Combine(Path.GetTempPath(), $"mgx-export-{Guid.NewGuid():N}.jsonl"); + try + { + using var ps = CreateShell(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", outFile); + ps.Invoke(); + + var error = Assert.Single(ps.Streams.Error); + Assert.StartsWith("MalformedJsonResponse", error.FullyQualifiedErrorId); + } + finally + { + File.Delete(outFile); + ResetTransport(); + } + } + + [Fact] + public void A_304_from_a_conditional_download_is_not_an_error() + { + var wire = new MockHttpHandler(); + wire.QueueEmpty((HttpStatusCode)304); + InjectTransport(wire); + Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, true); + Base.GetField("s_cachedTotalTimeoutSeconds", Static)!.SetValue(null, + new ResilientGraphClientOptions().TotalTimeoutSeconds); + try + { + using var ps = CreateShell(); + ps.AddScript(""" + Get-MgxContent '/me/drive/items/x/content' -Headers @{ 'If-None-Match' = '"etag"' } + """); + var output = ps.Invoke(); + + Assert.Empty(output); + Assert.Empty(ps.Streams.Error); + } + finally { ResetTransport(); } + } + [Fact] public void A_write_answered_with_html_is_an_error_record() { diff --git a/tests/Mgx.IntegrationTests/Cmdlets/SkipLogicTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/SkipLogicTests.cs new file mode 100644 index 0000000..71c07b7 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/SkipLogicTests.cs @@ -0,0 +1,115 @@ +using System.Net; +using System.Reflection; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// -SkipNotFound / -SkipForbidden behave identically on all four paths that implement +/// them: single request, GET fan-out, write fan-out, and relation expansion. The decision +/// is shared code now; these hold each path to it. +/// +[Collection("Pipeline")] +public class SkipLogicTests +{ + private static readonly Type Base = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); + private const BindingFlags Static = BindingFlags.NonPublic | BindingFlags.Static; + private const string NotFoundBody = + """{ "error": { "code": "Request_ResourceNotFound", "message": "missing" } }"""; + + private static void InjectTransport(HttpMessageHandler wire) + { + Base.GetField("s_graphHttpClient", Static)!.SetValue(null, new HttpClient(wire)); + Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, + Mgx.Cmdlets.Base.MgxCmdletBase.BuildAuthFingerprint( + new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); + Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); + Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); + Base.GetField("s_clientOptions", Static)!.SetValue(null, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + ResiliencePipelineFactory.Reset(); + } + + private static void ResetTransport() + { + // Restore every static InjectTransport touched - a later test in the collection + // that drives a cmdlet without injecting must not inherit this class's transport. + Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); + Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); + Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); + Base.GetField("s_cachedTotalTimeoutSeconds", Static)!.SetValue(null, 0); + Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); + Base.GetField("s_clientOptions", Static)!.SetValue(null, new ResilientGraphClientOptions()); + ResiliencePipelineFactory.Reset(); + } + + private static (int Output, List Errors, List Warnings) + Run(string script) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript(script); + var output = ps.Invoke(); + return (output.Count, ps.Streams.Error.ToList(), + ps.Streams.Warning.Select(w => w.Message).ToList()); + } + + [Theory] + [InlineData("Invoke-MgxRequest -Uri /users/u1 -SkipNotFound")] + [InlineData("'u1','u2' | Invoke-MgxRequest -Uri '/users/{id}' -SkipNotFound")] + [InlineData("'u1','u2' | Invoke-MgxRequest -Uri '/users/{id}' -Method PATCH -Body @{ x = 1 } -Confirm:$false -SkipNotFound")] + [InlineData("[pscustomobject]@{ id = 'u1' } | Expand-MgxRelation -Uri '/users/{id}/manager' -As manager -SkipNotFound")] + public void A_404_is_skipped_without_an_error_on_every_path(string script) + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.NotFound, NotFoundBody); + InjectTransport(wire); + try + { + var (_, errors, _) = Run(script); + Assert.Empty(errors); + } + finally { ResetTransport(); } + } + + [Theory] + [InlineData("Invoke-MgxRequest -Uri /users/u1")] + [InlineData("'u1','u2' | Invoke-MgxRequest -Uri '/users/{id}'")] + [InlineData("'u1','u2' | Invoke-MgxRequest -Uri '/users/{id}' -Method PATCH -Body @{ x = 1 } -Confirm:$false")] + [InlineData("[pscustomobject]@{ id = 'u1' } | Expand-MgxRelation -Uri '/users/{id}/manager' -As manager")] + public void The_same_404_is_an_error_without_the_switch(string script) + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.NotFound, NotFoundBody); + InjectTransport(wire); + try + { + var (_, errors, _) = Run(script); + Assert.NotEmpty(errors); + } + finally { ResetTransport(); } + } + + [Fact] + public void Skipped_items_are_counted_in_one_warning_summary() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.NotFound, NotFoundBody); + InjectTransport(wire); + try + { + var (output, errors, warnings) = Run( + "'u1','u2','u3' | Invoke-MgxRequest -Uri '/users/{id}' -SkipNotFound"); + Assert.Equal(0, output); + Assert.Empty(errors); + Assert.Contains(warnings, w => w.Contains("Skipped 3")); + } + finally { ResetTransport(); } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs index 1a07372..6fc66fa 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs @@ -31,8 +31,14 @@ private static void InjectTransport(HttpMessageHandler wire) private static void ResetTransport() { + // Restore every static InjectTransport touched - a later test in the collection + // that drives a cmdlet without injecting must not inherit this class's transport. Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); + Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); + Base.GetField("s_cachedTotalTimeoutSeconds", Static)!.SetValue(null, 0); + Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); + Base.GetField("s_clientOptions", Static)!.SetValue(null, new ResilientGraphClientOptions()); ResiliencePipelineFactory.Reset(); } @@ -162,6 +168,34 @@ public void A_piped_drive_item_id_is_escaped_into_the_content_path() finally { ResetTransport(); } } + [Fact] + public void Every_hostile_filter_round_trips_through_encoding() + { + // The corpus generalizes GraphSDK-2709/2942: the wire value must decode back to + // exactly what the caller wrote, whatever it contains. + foreach (var filter in HostileInputs.FilterValues) + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); + InjectTransport(wire); + try + { + var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("Filter", filter)); + // OriginalString keeps a raw '#' that the transport would then drop as a + // fragment - so round-tripping alone is not proof. No raw '#', '&' or '+' + // may survive in the encoded value; then the decode must give back the input. + var query = uri[(uri.IndexOf("$filter=", StringComparison.Ordinal) + "$filter=".Length)..]; + query = query.Split('&')[0]; + Assert.DoesNotContain('#', query); + Assert.DoesNotContain('+', query); + Assert.Equal(filter, System.Uri.UnescapeDataString(query)); + } + finally { ResetTransport(); } + } + } + [Fact] public void A_top_already_in_the_uri_is_not_sent_twice() { @@ -174,37 +208,129 @@ public void A_top_already_in_the_uri_is_not_sent_twice() .AddParameter("Uri", "/users?$top=5") .AddParameter("All", true)); - Assert.Equal(1, System.Text.RegularExpressions.Regex.Matches(uri, "top=").Count); + Assert.Single(System.Text.RegularExpressions.Regex.Matches(uri, "top=")); Assert.Contains("$top=5", uri); } finally { ResetTransport(); } } [Fact] - public void A_filter_already_in_the_uri_wins_over_the_parameter() + public void A_filter_already_in_the_uri_wins_over_the_parameter_with_a_warning() { var wire = new MockHttpHandler(); wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); InjectTransport(wire); try { - var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Invoke-MgxRequest") .AddParameter("Uri", "/users?$filter=accountEnabled eq true") - .AddParameter("Filter", "displayName eq 'x'")); + .AddParameter("Filter", "displayName eq 'x'"); + ps.Invoke(); - Assert.Equal(1, System.Text.RegularExpressions.Regex.Matches(uri, "filter=").Count); + var uri = wire.CapturedRequests[0].Uri; + Assert.Single(System.Text.RegularExpressions.Regex.Matches(uri, "filter=")); Assert.Contains("accountEnabled", uri); Assert.DoesNotContain("displayName", uri); + // Deferring must be loud: the caller's -Filter was not sent. + Assert.Contains(ps.Streams.Warning, w => w.Message.Contains("-Filter")); } finally { ResetTransport(); } } + [Fact] + public void An_encoded_option_name_in_the_uri_still_defers_the_parameter() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); + InjectTransport(wire); + try + { + var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users?%24top=10") + .AddParameter("All", true)); + + // %24top IS $top to the server; a second $top would draw a 400. + Assert.Single(System.Text.RegularExpressions.Regex.Matches(uri, "top=")); + } + finally { ResetTransport(); } + } + + [Fact] + public void A_caller_count_in_the_uri_does_not_arm_the_count_degradation_retry() + { + // Endpoint 400s; if mgx thinks its auto-$count caused it, the "retry without + // count" rebuild is byte-identical and burns a duplicate request. + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.BadRequest, + """{ "error": { "code": "BadRequest", "message": "no" } }"""); + InjectTransport(wire); + try + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("Invoke-MgxRequest '/users?$count=true' -Filter \"displayName eq 'x'\" -ErrorAction SilentlyContinue"); + ps.Invoke(); + + // One request (plus zero byte-identical degradation retries): the URI's own + // $count was never mgx's to drop. + Assert.Single(wire.CapturedRequests); + } + finally { ResetTransport(); } + } + + [Fact] + public void Every_hostile_path_segment_survives_id_templating() + { + // The PathSegments corpus, driven through the one path-injection mechanism a + // caller has: {id} templating. The captured request path must carry the fully + // escaped segment - no raw '#', '%', space, or comma damage. + foreach (var segment in HostileInputs.PathSegments) + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + InjectTransport(wire); + try + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/items/{id}"); + ps.Invoke(new[] { segment }); + + var sent = Assert.Single(wire.CapturedRequests); + Assert.Contains($"/items/{System.Uri.EscapeDataString(segment)}", sent.Uri); + Assert.DoesNotContain('#', sent.Uri); + } + finally { ResetTransport(); } + } + } + /// /// An absolute -Uri concatenated onto the versioned base URL would silently produce /// /v1.0/https:/... on the wire. Get-MgxContent and Sync-MgxDelta already refuse it; /// the other three URI-taking cmdlets refuse it the same way. /// [Theory] + [InlineData("Invoke-MgxRequest -Uri ' https://graph.microsoft.com/v1.0/users'")] [InlineData("Invoke-MgxRequest -Uri 'https://graph.microsoft.com/v1.0/users'")] [InlineData("[PSCustomObject]@{ id = 'g1' } | Expand-MgxRelation -Uri 'https://graph.microsoft.com/v1.0/groups/{id}/members' -As members")] [InlineData("Export-MgxCollection -Uri 'https://graph.microsoft.com/v1.0/users' -OutputFile ([IO.Path]::GetTempFileName())")] diff --git a/tests/Mgx.IntegrationTests/Engine/Errors/ErrorClassifierTests.cs b/tests/Mgx.IntegrationTests/Engine/Errors/ErrorClassifierTests.cs new file mode 100644 index 0000000..3206294 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Errors/ErrorClassifierTests.cs @@ -0,0 +1,52 @@ +using Mgx.Engine.Errors; + +namespace Mgx.IntegrationTests; + +public class ErrorClassifierTests +{ + [Theory] + [InlineData(429, MgxErrorClass.Throttle)] + [InlineData(500, MgxErrorClass.TransientServer)] + [InlineData(502, MgxErrorClass.TransientServer)] + [InlineData(503, MgxErrorClass.TransientServer)] + [InlineData(504, MgxErrorClass.TransientServer)] + [InlineData(408, MgxErrorClass.TransientTransport)] + [InlineData(401, MgxErrorClass.Authentication)] + [InlineData(403, MgxErrorClass.Authorization)] + [InlineData(404, MgxErrorClass.NotFound)] + [InlineData(409, MgxErrorClass.Conflict)] + [InlineData(412, MgxErrorClass.Conflict)] + [InlineData(400, MgxErrorClass.InvalidRequest)] + [InlineData(405, MgxErrorClass.InvalidRequest)] + [InlineData(413, MgxErrorClass.InvalidRequest)] + [InlineData(415, MgxErrorClass.InvalidRequest)] + [InlineData(422, MgxErrorClass.InvalidRequest)] + [InlineData(501, MgxErrorClass.InvalidRequest)] + [InlineData(410, MgxErrorClass.Permanent)] // delta resync belongs to a later phase + [InlineData(505, MgxErrorClass.Permanent)] // unknown 5xx: never retried, stays that way + [InlineData(507, MgxErrorClass.Permanent)] + [InlineData(418, MgxErrorClass.Permanent)] + public void Status_maps_to_its_class(int status, MgxErrorClass expected) + => Assert.Equal(expected, MgxErrorClassifier.Classify(status).Class); + + [Fact] + public void A_response_carries_status_and_unclamped_retry_after() + { + using var response = new HttpResponseMessage((System.Net.HttpStatusCode)429); + response.Headers.RetryAfter = new System.Net.Http.Headers.RetryConditionHeaderValue( + TimeSpan.FromSeconds(999)); + var info = MgxErrorClassifier.Classify(response); + Assert.Equal(MgxErrorClass.Throttle, info.Class); + Assert.Equal(429, info.StatusCode); + Assert.Equal(TimeSpan.FromSeconds(999), info.ServerRetryAfter); // facts unclamped; policy clamps + } + + [Fact] + public void Cancellation_is_permanent_regardless_of_exception_shape() + { + Assert.Equal(MgxErrorClass.Permanent, + MgxErrorClassifier.Classify(new TaskCanceledException(), cancellationRequested: true).Class); + Assert.Equal(MgxErrorClass.Permanent, + MgxErrorClassifier.Classify(new Polly.Timeout.TimeoutRejectedException(), cancellationRequested: true).Class); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Errors/ErrorPolicyParityTests.cs b/tests/Mgx.IntegrationTests/Engine/Errors/ErrorPolicyParityTests.cs new file mode 100644 index 0000000..34d83bf --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Errors/ErrorPolicyParityTests.cs @@ -0,0 +1,174 @@ +using System.Net.Sockets; +using Mgx.Engine.Errors; +using Polly.CircuitBreaker; +using Polly.Timeout; + +namespace Mgx.IntegrationTests; + +/// +/// Pins classifier + policy to the decisions the retry predicate, the circuit-breaker +/// predicate, the batch item loop, and the download pipeline each made on their own +/// before consolidation, across every status. The oracles below are transcriptions of +/// v2.1.2's inline code. Scope, stated honestly: the status tables are held exactly; +/// the batch theory drives the real IsRetryable; the pipeline predicates themselves are +/// lambdas exercised by the behavior suites (RetryTests, CircuitBreakerTests). Three +/// deliberate widenings in the exception dimension are pinned as INTENDED below: +/// SocketException, IOException (reachable as HttpIOException), and a bare +/// OperationCanceledException now classify as transport failures and retry/count where +/// v2.1.2 fell through to no-retry. +/// +public class ErrorPolicyParityTests +{ + // ResiliencePipelineFactory.cs retry predicate, as it stood. + private static bool OracleRetry(int status, bool isIdempotent) + { + if (status == 429) return true; + if (!isIdempotent) return false; + return status is 500 or 502 or 503 or 504 or 408; + } + + // ResiliencePipelineFactory.cs circuit-breaker predicate, as it stood. + private static bool OracleCircuit(int status) => status is 500 or 502 or 503 or 504; + + // GraphBatchClient.IsRetryable, as it stood. + private static bool OracleBatch(int status, string method) + { + if (status == 429) return true; + if (string.Equals(method, "POST", StringComparison.OrdinalIgnoreCase)) return false; + return status is 408 or 500 or 502 or 503 or 504; + } + + // GraphContentClient download pipeline ShouldHandle, as it stood. + private static bool OracleDownload(int status) => status is 429 or 500 or 502 or 503 or 504; + + public static TheoryData AllStatuses() + { + var d = new TheoryData(); + for (var s = 100; s < 600; s++) d.Add(s); + return d; + } + + [Theory] + [MemberData(nameof(AllStatuses))] + public void Retry_decision_matches_the_old_predicate_for_every_status(int status) + { + var cls = MgxErrorClassifier.Classify(status).Class; + Assert.Equal(OracleRetry(status, isIdempotent: true), MgxErrorPolicy.ShouldRetry(cls, isIdempotent: true)); + Assert.Equal(OracleRetry(status, isIdempotent: false), MgxErrorPolicy.ShouldRetry(cls, isIdempotent: false)); + } + + [Theory] + [MemberData(nameof(AllStatuses))] + public void Circuit_decision_matches_the_old_predicate_for_every_status(int status) + { + var info = MgxErrorClassifier.Classify(status); + Assert.Equal(OracleCircuit(status), MgxErrorPolicy.CountsAsCircuitFailure(info)); + } + + private static readonly System.Reflection.MethodInfo s_realBatchIsRetryable = + typeof(Mgx.Engine.Http.GraphBatchClient).GetMethod("IsRetryable", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!; + + [Theory] + [MemberData(nameof(AllStatuses))] + public void Batch_decision_matches_the_old_predicate_for_every_status(int status) + { + // The REAL private predicate, not a re-implementation of its glue - so a drift in + // GraphBatchClient.IsRetryable itself (say, its case-insensitive POST check) fails + // here rather than staying green behind a faithful copy. + foreach (var method in new[] { "GET", "POST", "PATCH", "PUT", "DELETE", "post" }) + { + var actual = (bool)s_realBatchIsRetryable.Invoke(null, [status, method])!; + Assert.Equal(OracleBatch(status, method), actual); + } + } + + [Theory] + [MemberData(nameof(AllStatuses))] + public void Download_decision_matches_the_old_predicate_for_every_status(int status) + { + var info = MgxErrorClassifier.Classify(status); + Assert.Equal(OracleDownload(status), MgxErrorPolicy.ShouldRetryDownload(info, exception: null)); + } + + [Fact] + public void Exception_decisions_match_the_old_predicates() + { + // Retry predicate: HttpRequestException retried (idempotent only, via the gate); + // TaskCanceled/TimeoutRejected retried unless the caller cancelled; everything else not. + static bool Retry(Exception ex, bool cancelled, bool idempotent) + => MgxErrorPolicy.ShouldRetry(MgxErrorClassifier.Classify(ex, cancelled).Class, idempotent); + + Assert.True(Retry(new HttpRequestException(), cancelled: false, idempotent: true)); + Assert.False(Retry(new HttpRequestException(), cancelled: false, idempotent: false)); + Assert.True(Retry(new TaskCanceledException(), cancelled: false, idempotent: true)); + Assert.False(Retry(new TaskCanceledException(), cancelled: true, idempotent: true)); + Assert.True(Retry(new TimeoutRejectedException(), cancelled: false, idempotent: true)); + Assert.False(Retry(new TimeoutRejectedException(), cancelled: true, idempotent: true)); + Assert.False(Retry(new BrokenCircuitException(), cancelled: false, idempotent: true)); + Assert.False(Retry(new InvalidOperationException(), cancelled: false, idempotent: true)); + + // Circuit predicate: transport failures count, except after cancellation. + static bool Circuit(Exception ex, bool cancelled) + => MgxErrorPolicy.CountsAsCircuitFailure(MgxErrorClassifier.Classify(ex, cancelled)); + + Assert.True(Circuit(new HttpRequestException(), cancelled: false)); + Assert.True(Circuit(new TaskCanceledException(), cancelled: false)); + Assert.False(Circuit(new TaskCanceledException(), cancelled: true)); + Assert.True(Circuit(new TimeoutRejectedException(), cancelled: false)); + Assert.False(Circuit(new BrokenCircuitException(), cancelled: false)); + + // Download pipeline: HttpRequestException only. + static bool Download(Exception ex) + => MgxErrorPolicy.ShouldRetryDownload(MgxErrorClassifier.Classify(ex, false), ex); + + Assert.True(Download(new HttpRequestException())); + Assert.False(Download(new TaskCanceledException())); + Assert.False(Download(new TimeoutRejectedException())); + Assert.False(Download(new SocketException())); + } + + [Fact] + public void The_intended_widenings_are_pinned_as_intended() + { + // v2.1.2's inline predicates fell through to no-retry for these; classification + // makes them transport failures, which retries (idempotent) and counts toward the + // breaker. Deliberate - a "response ended prematurely" HttpIOException is exactly + // as transient as the HttpRequestException that usually wraps it. If this test + // surprises you, the change log entry to read is 2.1.3's classifier note. + static (bool Retry, bool Circuit) Decide(Exception ex) + { + var info = MgxErrorClassifier.Classify(ex, cancellationRequested: false); + return (MgxErrorPolicy.ShouldRetry(info.Class, isIdempotent: true), + MgxErrorPolicy.CountsAsCircuitFailure(info)); + } + + Assert.Equal((true, true), Decide(new SocketException())); + Assert.Equal((true, true), Decide(new IOException())); + Assert.Equal((true, true), Decide(new OperationCanceledException())); + + // And the narrowing: a per-attempt timeout after the caller cancelled no longer + // retries (2.1.2's fix, preserved through classification). + var cancelled = MgxErrorClassifier.Classify(new TimeoutRejectedException(), cancellationRequested: true); + Assert.False(MgxErrorPolicy.ShouldRetry(cancelled.Class, isIdempotent: true)); + Assert.False(MgxErrorPolicy.CountsAsCircuitFailure(cancelled)); + } + + [Fact] + public void The_breaker_reads_cancellation_from_the_exceptions_own_token() + { + // The CB call site derives cancellationRequested from the exception's token - + // matching v2.1.2's Handle(e => !e.CancellationToken...) - + // while the retry predicate reads the context token. Pin the distinction: a TCE + // carrying a cancelled token must not count, one with a default token must. + using var cts = new CancellationTokenSource(); + cts.Cancel(); + var carried = new TaskCanceledException(null, null, cts.Token); + Assert.False(MgxErrorPolicy.CountsAsCircuitFailure( + MgxErrorClassifier.Classify(carried, carried.CancellationToken.IsCancellationRequested))); + + var bare = new TaskCanceledException(); + Assert.True(MgxErrorPolicy.CountsAsCircuitFailure( + MgxErrorClassifier.Classify(bare, bare.CancellationToken.IsCancellationRequested))); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Errors/RetryAfterPolicyTests.cs b/tests/Mgx.IntegrationTests/Engine/Errors/RetryAfterPolicyTests.cs new file mode 100644 index 0000000..7b4ac37 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Errors/RetryAfterPolicyTests.cs @@ -0,0 +1,61 @@ +using System.Net.Http.Headers; +using Mgx.Engine.Errors; + +namespace Mgx.IntegrationTests; + +public class RetryAfterPolicyTests +{ + private static readonly TimeSpan Cap = TimeSpan.FromSeconds(120); + + [Fact] + public void Delta_form_is_honored_and_clamped() + { + Assert.Equal(TimeSpan.FromSeconds(30), + RetryAfterPolicy.Resolve(new RetryConditionHeaderValue(TimeSpan.FromSeconds(30)), Cap)); + Assert.Equal(Cap, + RetryAfterPolicy.Resolve(new RetryConditionHeaderValue(TimeSpan.FromSeconds(600)), Cap)); + Assert.Equal(TimeSpan.Zero, + RetryAfterPolicy.Resolve(new RetryConditionHeaderValue(TimeSpan.Zero), Cap)); + } + + [Fact] + public void Date_form_is_honored_in_the_future_and_ignored_in_the_past() + { + var future = RetryAfterPolicy.Resolve( + new RetryConditionHeaderValue(DateTimeOffset.UtcNow.AddSeconds(40)), Cap); + Assert.NotNull(future); + Assert.InRange(future!.Value.TotalSeconds, 35, 41); + + Assert.Null(RetryAfterPolicy.Resolve( + new RetryConditionHeaderValue(DateTimeOffset.UtcNow.AddSeconds(-40)), Cap)); + + var farFuture = RetryAfterPolicy.Resolve( + new RetryConditionHeaderValue(DateTimeOffset.UtcNow.AddHours(2)), Cap); + Assert.Equal(Cap, farFuture); + } + + [Fact] + public void Absent_header_defers_to_backoff() + => Assert.Null(RetryAfterPolicy.Resolve(null, Cap)); + + [Fact] + public void Batch_seconds_form_rounds_up_floors_at_zero_and_clamps() + { + Assert.True(RetryAfterPolicy.TryResolveSeconds("30", 120, out var req, out var clamped)); + Assert.Equal(30, req); + Assert.Equal(30, clamped); + + Assert.True(RetryAfterPolicy.TryResolveSeconds("600", 120, out req, out clamped)); + Assert.Equal(600, req); + Assert.Equal(120, clamped); + + // A past HTTP date computes negative; the sleep floors at zero. + var past = DateTimeOffset.UtcNow.AddSeconds(-30).ToString("R"); + Assert.True(RetryAfterPolicy.TryResolveSeconds(past, 120, out req, out clamped)); + Assert.True(req <= 0); + Assert.Equal(0, clamped); + + Assert.False(RetryAfterPolicy.TryResolveSeconds("not-a-value", 120, out _, out _)); + Assert.False(RetryAfterPolicy.TryResolveSeconds(null, 120, out _, out _)); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Http/DownloadHeaderTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/DownloadHeaderTests.cs new file mode 100644 index 0000000..293f9f5 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/DownloadHeaderTests.cs @@ -0,0 +1,110 @@ +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// Conditional headers cross to the download host; auth and correlation headers never do. +/// Before this, hop 2 carried only Range, so a caller's If-None-Match silently vanished +/// and a conditional download re-fetched the full content every time. +/// +[Collection("Pipeline")] +public class DownloadHeaderTests : IDisposable +{ + private const string CdnUrl = "https://contoso-my.sharepoint.com/_layouts/15/download.aspx?UniqueId=abc&tempauth=TOKEN"; + + private readonly MockHttpHandler _cdnHandler = new(); + + public DownloadHeaderTests() + { + GraphContentClient.DownloadClientForTests = new HttpClient(_cdnHandler); + } + + public void Dispose() + { + GraphContentClient.DownloadClientForTests?.Dispose(); + GraphContentClient.DownloadClientForTests = null; + } + + [Fact] + public async Task Conditional_headers_reach_the_download_host() + { + _cdnHandler.QueueBytes(HttpStatusCode.NotModified, [], "application/octet-stream"); + + var headers = new Dictionary + { + ["If-None-Match"] = "\"etag-1\"", + ["If-Modified-Since"] = "Mon, 24 Aug 2026 10:00:00 GMT", + ["If-Match"] = "\"graph-etag\"", // write validator: stays on hop 1 + ["Authorization"] = "Bearer should-never-cross", + ["client-request-id"] = "should-never-cross", + ["ConsistencyLevel"] = "eventual", + }; + + // 304 is a non-success status; the client surfaces it as an error after reading the + // body - what matters here is what went on the wire. + try + { + using var result = await GraphContentClient.GetFromDownloadUrlAsync( + CdnUrl, range: null, TimeSpan.FromSeconds(5), CancellationToken.None, headers); + } + catch (Mgx.Engine.Models.GraphServiceException) { /* 304 surfaces as an error; fine */ } + + var sent = Assert.Single(_cdnHandler.CapturedRequests); + Assert.Equal("\"etag-1\"", Assert.Single(Assert.Contains("If-None-Match", sent.Headers))); + Assert.Contains("If-Modified-Since", sent.Headers); + Assert.DoesNotContain("Authorization", sent.Headers); + Assert.DoesNotContain("client-request-id", sent.Headers); + Assert.DoesNotContain("ConsistencyLevel", sent.Headers); + Assert.DoesNotContain("If-Match", sent.Headers); + } + + [Fact] + public async Task Lowercase_conditional_headers_forward_on_the_two_hop_path() + { + // The two-hop path copies the caller dictionary; the copy must keep case + // insensitivity or hop 2 silently drops non-canonical casings. + ResiliencePipelineFactory.Reset(); + var graphWire = new MockHttpHandler(); + graphWire.QueueResponse(HttpStatusCode.Found, headers: new Dictionary + { + ["Location"] = CdnUrl, + }); + _cdnHandler.QueueBytes(HttpStatusCode.OK, [1, 2, 3], "application/octet-stream"); + + using var client = new ResilientGraphClient(new HttpClient(graphWire), + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var headers = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["if-none-match"] = "\"lower-etag\"", + ["client-request-id"] = "caller-id", + }; + using var result = await client.GetContentAsync( + "https://graph.microsoft.com/v1.0/me/drive/items/x/content", + range: null, headers, CancellationToken.None); + + var hop2 = Assert.Single(_cdnHandler.CapturedRequests); + Assert.Equal("\"lower-etag\"", Assert.Single(Assert.Contains("If-None-Match", + new Dictionary(hop2.Headers, StringComparer.OrdinalIgnoreCase)))); + + // And hop 1 must carry the caller's correlation id exactly once. + var hop1 = Assert.Single(graphWire.CapturedRequests); + var ids = Assert.Contains("client-request-id", + new Dictionary(hop1.Headers, StringComparer.OrdinalIgnoreCase)); + Assert.Equal("caller-id", Assert.Single(ids)); + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task A_download_without_caller_headers_sends_none() + { + _cdnHandler.QueueBytes(HttpStatusCode.OK, [1, 2, 3], "application/octet-stream"); + + using var result = await GraphContentClient.GetFromDownloadUrlAsync( + CdnUrl, range: null, TimeSpan.FromSeconds(5), CancellationToken.None); + + var sent = Assert.Single(_cdnHandler.CapturedRequests); + Assert.DoesNotContain("If-None-Match", sent.Headers); + Assert.Equal(HttpStatusCode.OK, result.StatusCode); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Http/HeaderFidelityTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/HeaderFidelityTests.cs index 99fdcf5..3285387 100644 --- a/tests/Mgx.IntegrationTests/Engine/Http/HeaderFidelityTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Http/HeaderFidelityTests.cs @@ -118,15 +118,72 @@ public async Task A_caller_correlation_id_is_not_doubled() [Fact] public void Case_variant_duplicate_keys_merge_to_one_header() { - var headers = TestCmdlet.BuildHeaders(null, new System.Collections.Hashtable - { - ["If-Match"] = "\"etag-a\"", - }); + // A genuinely case-variant duplicate: an ordinal Hashtable holds both spellings; + // the merge must produce ONE entry (whichever value enumeration hands it last). + var duplicates = new System.Collections.Hashtable(); + duplicates["If-Match"] = "\"etag-a\""; + duplicates["if-match"] = "\"etag-b\""; + Assert.Equal(2, duplicates.Count); + + var headers = TestCmdlet.BuildHeaders(null, duplicates); Assert.NotNull(headers); - Assert.True(headers!.ContainsKey("if-match")); + Assert.Single(headers!.Keys, k => string.Equals(k, "If-Match", StringComparison.OrdinalIgnoreCase)); + Assert.True(headers.ContainsKey("if-match")); Assert.True(headers.ContainsKey("IF-MATCH")); } + [Fact] + public async Task A_malformed_header_name_warns_instead_of_crashing_a_write() + { + var (wire, client) = NewClient(); + using (client) + { + wire.QueueResponse(HttpStatusCode.OK, "{}"); + using var response = await client.SendAsync(HttpMethod.Post, + "https://graph.microsoft.com/v1.0/users", + new StringContent("{}", Encoding.UTF8, "application/json"), + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["Bad Header"] = "v", + }, + CancellationToken.None); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + var warnings = new List(); + client.WarningWriter = warnings.Add; + client.DrainWarningMessages(); + Assert.Contains(warnings, w => w.Contains("Bad Header")); + } + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task A_caller_content_length_is_ignored_with_a_warning() + { + var (wire, client) = NewClient(); + using (client) + { + wire.QueueResponse(HttpStatusCode.OK, "{}"); + using var response = await client.SendAsync(HttpMethod.Post, + "https://graph.microsoft.com/v1.0/users", + new StringContent("0123456789", Encoding.UTF8, "application/json"), + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["Content-Length"] = "3", + }, + CancellationToken.None); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + var sent = Assert.Single(wire.CapturedRequests); + Assert.Equal(10, sent.Body!.Length); // the real body, not a truncation + var warnings = new List(); + client.WarningWriter = warnings.Add; + client.DrainWarningMessages(); + Assert.Contains(warnings, w => w.Contains("Content-Length")); + } + ResiliencePipelineFactory.Reset(); + } + /// BuildRequestHeaders is protected static; surfaced for assertion. private sealed class TestCmdlet : Mgx.Cmdlets.Base.MgxCmdletBase { diff --git a/tests/Mgx.IntegrationTests/Engine/Http/ResilienceBridgeTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/ResilienceBridgeTests.cs index 0b246c0..e1dd027 100644 --- a/tests/Mgx.IntegrationTests/Engine/Http/ResilienceBridgeTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Http/ResilienceBridgeTests.cs @@ -271,6 +271,97 @@ public async Task Bridge_stamps_one_correlation_id_per_logical_request() ResiliencePipelineFactory.Reset(); } + [Fact] + public async Task A_known_oversized_body_passes_to_the_inner_chain_untouched() + { + // The pipeline cannot replay what it will not buffer, so it does not manage it: + // no clone, no retry stamp, no per-attempt timeout - the inner chain sees the + // ORIGINAL request exactly as the SDK built it. + ResiliencePipelineFactory.Reset(); + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "{}"); + + var (pipeline, _) = ResiliencePipelineFactory.GetOrCreate( + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 3 }); + using var handler = new ResilientDelegatingHandler(pipeline, null) { InnerHandler = wire }; + using var client = new HttpClient(handler); + + var big = new byte[5 * 1024 * 1024]; + var request = new HttpRequestMessage(HttpMethod.Put, "https://graph.microsoft.com/v1.0/drives/d/items/i/content") + { + Content = new ByteArrayContent(big), + }; + var response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + var sent = Assert.Single(wire.CapturedRequests); + Assert.Equal(big.Length, sent.Body!.Length); + // Untouched: none of the wrap's stamps. + Assert.DoesNotContain("SdkVersion", sent.Headers); + Assert.DoesNotContain("client-request-id", sent.Headers); + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task The_wrap_adds_no_retries_to_a_known_oversized_body() + { + // A 429 comes back as-is from the wrap's side; whether it retries is the inner + // SDK chain's decision, exactly as without Enable-MgxResilience. (The mock IS + // the inner chain here, so one request means the wrap added none.) + ResiliencePipelineFactory.Reset(); + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.TooManyRequests); + wire.QueueResponse(HttpStatusCode.OK, "{}"); + + var (pipeline, _) = ResiliencePipelineFactory.GetOrCreate( + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 3 }); + using var handler = new ResilientDelegatingHandler(pipeline, null) { InnerHandler = wire }; + using var client = new HttpClient(handler); + + var request = new HttpRequestMessage(HttpMethod.Put, "https://graph.microsoft.com/v1.0/drives/d/items/i/content") + { + Content = new ByteArrayContent(new byte[5 * 1024 * 1024]), + }; + var response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + Assert.Equal(1, wire.RequestCount); + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task An_unknown_length_body_keeps_full_retry_protection_whatever_its_size() + { + // No declared length means the read is unavoidable; once buffered, it replays. + ResiliencePipelineFactory.Reset(); + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.ServiceUnavailable); + wire.QueueResponse(HttpStatusCode.OK, "{}"); + + var (pipeline, _) = ResiliencePipelineFactory.GetOrCreate( + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 3 }); + using var handler = new ResilientDelegatingHandler(pipeline, null) { InnerHandler = wire }; + using var client = new HttpClient(handler); + + var big = new byte[5 * 1024 * 1024]; + var chunked = new StreamContent(new NonSeekableStream(big)); + var request = new HttpRequestMessage(HttpMethod.Put, "https://graph.microsoft.com/v1.0/drives/d/items/i/content") + { + Content = chunked, + }; + var response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(2, wire.RequestCount); + Assert.All(wire.CapturedRequests, r => Assert.Equal(big.Length, r.Body!.Length)); + ResiliencePipelineFactory.Reset(); + } + + private sealed class NonSeekableStream(byte[] data) : MemoryStream(data) + { + public override bool CanSeek => false; + } + [Fact] public async Task Bridge_keeps_the_sdk_correlation_id_when_one_is_already_set() { diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/OpaqueLinkTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/OpaqueLinkTests.cs new file mode 100644 index 0000000..e706108 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/OpaqueLinkTests.cs @@ -0,0 +1,54 @@ +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// A service-issued link is opaque: validation may refuse it, but a link that passes is +/// followed byte-identically - no re-encoding, no normalization, no re-parsing. A repaired +/// link is a different link, and Graph's skiptokens do not survive repair. +/// (GraphSDK-2488.) +/// +public class OpaqueLinkTests +{ + private static readonly Uri GraphHost = new("https://graph.microsoft.com/v1.0/users"); + + [Fact] + public void A_validated_link_is_the_original_string_byte_for_byte() + { + foreach (var link in HostileInputs.OpaqueLinks) + { + var validated = NextLinkValidator.Validate(link, GraphHost); + Assert.NotNull(validated); + Assert.Same(link, validated); // the same string instance - no rebuild happened + } + } + + [Fact] + public void ValidateOrThrow_returns_the_original_string_too() + { + foreach (var link in HostileInputs.OpaqueLinks) + Assert.Same(link, NextLinkValidator.ValidateOrThrow(link, GraphHost)); + } + + [Fact] + public void A_checkpoint_round_trips_its_link_byte_for_byte() + { + foreach (var link in HostileInputs.OpaqueLinks) + { + var path = Path.Combine(Path.GetTempPath(), $"mgx-oplink-{Guid.NewGuid():N}.json"); + try + { + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users", + NextLink = link, + ItemsCollected = 1, + }.Save(path); + var loaded = PaginationCheckpoint.Load(path); + Assert.NotNull(loaded); + Assert.Equal(link, loaded!.NextLink); + } + finally { File.Delete(path); } + } + } +} diff --git a/tests/Mgx.IntegrationTests/Serialization/PSWrapperEquivalenceTests.cs b/tests/Mgx.IntegrationTests/Serialization/PSWrapperEquivalenceTests.cs new file mode 100644 index 0000000..e909175 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Serialization/PSWrapperEquivalenceTests.cs @@ -0,0 +1,87 @@ +using System.Collections; +using System.Management.Automation; +using Mgx.Cmdlets.Cmdlets; + +namespace Mgx.IntegrationTests; + +/// +/// The acceptance criterion for body serialization: a PowerShell wrapper never alters the +/// wire JSON. Serialize(x) must be byte-identical to Serialize(PSObject.AsPSObject(x)), +/// with the wrapping at any depth. (GraphSDK-3654 PSObject wrappers, GraphSDK-3361 +/// scalar/array.) +/// +public class PSWrapperEquivalenceTests +{ + public static TheoryData Values() + { + var d = new TheoryData + { + { "bool", true }, + { "int", 42 }, + { "long-max", long.MaxValue }, + { "double", 3.5d }, + { "decimal", 79228162514264337593543950335m }, + { "guid", Guid.Parse("11111111-2222-3333-4444-555555555555") }, + { "datetime-utc", new DateTime(2026, 1, 5, 10, 0, 0, DateTimeKind.Utc) }, + { "datetime-unspecified", new DateTime(2026, 1, 5, 10, 0, 0, DateTimeKind.Unspecified) }, + { "datetimeoffset", new DateTimeOffset(2026, 1, 5, 10, 0, 0, TimeSpan.FromHours(2)) }, + { "timespan", new TimeSpan(1, 30, 0) }, + { "enum", DayOfWeek.Monday }, + { "string-unicode", "Müller & Söhne 日本語 😀" }, + { "string-quotes", "it's \"quoted\"" }, + { "bytes", new byte[] { 1, 2, 3 } }, + { "array", new object[] { 1, "two", true } }, + { "single-element-array", new object[] { "alone" } }, + { "empty-array", Array.Empty() }, + }; + return d; + } + + private static string Serialize(object body) => InvokeMgxRequest.SerializeBody(body); + + [Theory] + [MemberData(nameof(Values))] + public void Wrapping_at_depth_0_changes_nothing(string label, object value) + { + _ = label; + var direct = Serialize(new Hashtable { ["v"] = value }); + var wrapped = Serialize(new Hashtable { ["v"] = PSObject.AsPSObject(value) }); + Assert.Equal(direct, wrapped); + } + + [Theory] + [MemberData(nameof(Values))] + public void Wrapping_at_depth_1_changes_nothing(string label, object value) + { + _ = label; + var direct = Serialize(new Hashtable { ["outer"] = new Hashtable { ["v"] = value } }); + var wrapped = Serialize(PSObject.AsPSObject( + new Hashtable { ["outer"] = PSObject.AsPSObject(new Hashtable { ["v"] = PSObject.AsPSObject(value) }) })); + Assert.Equal(direct, wrapped); + } + + [Theory] + [MemberData(nameof(Values))] + public void Wrapping_at_depth_3_changes_nothing(string label, object value) + { + _ = label; + object Nest(object v) => new Hashtable { ["a"] = new Hashtable { ["b"] = new Hashtable { ["c"] = v } } }; + var direct = Serialize(Nest(value)); + var wrapped = Serialize(Nest(PSObject.AsPSObject(value))); + Assert.Equal(direct, wrapped); + } + + [Fact] + public void An_array_of_wrapped_elements_matches_the_unwrapped_array() + { + var direct = Serialize(new Hashtable { ["members"] = new object[] { "a", 1, true } }); + var wrapped = Serialize(new Hashtable + { + ["members"] = new object[] + { + PSObject.AsPSObject("a"), PSObject.AsPSObject(1), PSObject.AsPSObject(true) + } + }); + Assert.Equal(direct, wrapped); + } +} diff --git a/tests/Mgx.IntegrationTests/Serialization/WireBodyTests.cs b/tests/Mgx.IntegrationTests/Serialization/WireBodyTests.cs index 37741b7..43aeb6b 100644 --- a/tests/Mgx.IntegrationTests/Serialization/WireBodyTests.cs +++ b/tests/Mgx.IntegrationTests/Serialization/WireBodyTests.cs @@ -30,8 +30,14 @@ private static void InjectTransport(HttpMessageHandler wire) private static void ResetTransport() { + // Restore every static InjectTransport touched - a later test in the collection + // that drives a cmdlet without injecting must not inherit this class's transport. Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); + Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); + Base.GetField("s_cachedTotalTimeoutSeconds", Static)!.SetValue(null, 0); + Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); + Base.GetField("s_clientOptions", Static)!.SetValue(null, new ResilientGraphClientOptions()); ResiliencePipelineFactory.Reset(); } @@ -109,4 +115,117 @@ public void A_wrapped_and_an_unwrapped_hashtable_produce_identical_wire_json() var wrapped = WireBodyFor("[PSObject]@{ a = 1; nested = @{ b = $true } }"); Assert.Equal(direct, wrapped); } + + /// Runs a POST and returns (errors, requestCount) - for bodies that must refuse. + private static (List Errors, int Requests) RefusalFor(string bodyExpression) + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.NoContent); + InjectTransport(wire); + try + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript($"Invoke-MgxRequest -Uri /widgets -Method POST -Confirm:$false -Body $({bodyExpression})"); + ps.Invoke(); + return (ps.Streams.Error.ToList(), wire.CapturedRequests.Count); + } + finally { ResetTransport(); } + } + + [Fact] + public void A_secure_string_in_a_body_refuses_naming_the_property_path() + { + var (errors, requests) = RefusalFor("@{ profile = @{ password = (ConvertTo-SecureString 'x' -AsPlainText -Force) } }"); + Assert.Equal(0, requests); + var error = Assert.Single(errors); + Assert.StartsWith("InvalidBodyValue", error.FullyQualifiedErrorId); + Assert.Contains("-Body.profile.password", error.Exception.Message); + Assert.Contains("SecureString", error.Exception.Message); + } + + [Fact] + public void A_script_block_in_a_body_refuses() + { + var (errors, requests) = RefusalFor("@{ handler = { Get-Date } }"); + Assert.Equal(0, requests); + var error = Assert.Single(errors); + Assert.StartsWith("InvalidBodyValue", error.FullyQualifiedErrorId); + Assert.Contains("ScriptBlock", error.Exception.Message); + } + + [Fact] + public void NaN_in_a_body_refuses_naming_the_property_path() + { + var (errors, requests) = RefusalFor("@{ metrics = @{ score = [double]::NaN } }"); + Assert.Equal(0, requests); + var error = Assert.Single(errors); + Assert.StartsWith("InvalidBodyValue", error.FullyQualifiedErrorId); + Assert.Contains("-Body.metrics.score", error.Exception.Message); + } + + [Fact] + public void A_self_referencing_body_is_a_clean_error_not_a_stack_overflow() + { + var (errors, requests) = RefusalFor("$h = @{}; $h.self = $h; $h"); + Assert.Equal(0, requests); + var error = Assert.Single(errors); + Assert.StartsWith("InvalidBodyValue", error.FullyQualifiedErrorId); + Assert.Contains("self-referencing", error.Exception.Message); + } + + [Fact] + public void Scalar_wire_forms_hold_for_the_remaining_corpus_types() + { + // Exact wire values for the types the wrapper-equivalence theory covers only + // relationally: identical-on-both-sides is not the same as correct. + Assert.Equal("""{"v":79228162514264337593543950335}""", + WireBodyFor("@{ v = [decimal]::MaxValue }")); + Assert.Equal("""{"v":"11111111-2222-3333-4444-555555555555"}""", + WireBodyFor("@{ v = [Guid]'11111111-2222-3333-4444-555555555555' }")); + Assert.Equal("""{"v":9223372036854775807}""", + WireBodyFor("@{ v = [long]::MaxValue }")); + Assert.Equal("""{"v":"2026-01-05T10:00:00+02:00"}""", + WireBodyFor("@{ v = [DateTimeOffset]::new(2026,1,5,10,0,0,[TimeSpan]::FromHours(2)) }")); + Assert.Equal("""{"v":[]}""", WireBodyFor("@{ v = @() }")); + } + + [Fact] + public void A_refused_batch_body_fails_that_item_and_sends_the_rest() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, + """{ "responses": [ { "id": "1", "status": 204 } ] }"""); + InjectTransport(wire); + try + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript(""" + @( + @{ Url = '/users/u1'; Method = 'PATCH'; Body = @{ s = (ConvertTo-SecureString 'x' -AsPlainText -Force) } } + @{ Url = '/users/u2'; Method = 'PATCH'; Body = @{ displayName = 'ok' } } + ) | Invoke-MgxBatchRequest -Confirm:$false + """); + ps.Invoke(); + + var error = Assert.Single(ps.Streams.Error, + e => e.FullyQualifiedErrorId.StartsWith("InvalidBatchItemBody")); + Assert.Contains("SecureString", error.Exception.Message); + Assert.Single(wire.CapturedRequests); // the good item still went + } + finally { ResetTransport(); } + } } diff --git a/tests/Mgx.IntegrationTests/TestSetup/HostileInputs.cs b/tests/Mgx.IntegrationTests/TestSetup/HostileInputs.cs new file mode 100644 index 0000000..cea7c53 --- /dev/null +++ b/tests/Mgx.IntegrationTests/TestSetup/HostileInputs.cs @@ -0,0 +1,48 @@ +namespace Mgx.IntegrationTests; + +/// +/// The shared hostile-input corpus. Each group generalizes a real failure; the issue +/// numbers are provenance, never scope (M365DSC and Graph SDK issue trackers). +/// +public static class HostileInputs +{ + /// OData filter values that break naive encoding. + /// (GraphSDK-2709/2942 query corruption, GraphSDK-1947 '#'.) + public static readonly string[] FilterValues = + [ + "startsWith(displayName,'O''Brien')", // escaped apostrophe + "displayName eq 'A & B'", + "displayName eq 'C#'", + "displayName eq '50% off'", + "displayName eq 'a+b'", + "displayName eq 'x/y'", + "displayName eq '?'", + "displayName eq 'Müller'", + "displayName eq '日本語'", + "displayName eq '😀'", // surrogate pair + "mail eq 'a b@contoso.com'", + ]; + + /// Path segments that must survive into the request path. + /// (GraphSDK-1947, M365DSC-5354.) + public static readonly string[] PathSegments = + [ + "a#b.txt", + "a b.txt", + "100%.pdf", + "it's.txt", + "naïve.txt", + "01ABCDEF!ABC:XYZ", // drive-item id shape + "contoso.sharepoint.com,9f0e,ab12", // site id shape + ]; + + /// Service-issued links that must be followed byte-identically. + /// (GraphSDK-2488 nextLink handling.) + public static readonly string[] OpaqueLinks = + [ + "https://graph.microsoft.com/v1.0/users?$skiptoken=X%27443700...%27&$top=100", + "https://graph.microsoft.com/v1.0/users?$skiptoken=a+b=c==&$select=id", + "https://graph.microsoft.com/v1.0/users?$skiptoken=pre%20encoded%23frag", + "https://graph.microsoft.com/v1.0/drives/b!YZCE_x-y/items?$skiptoken=1", + ]; +} diff --git a/tests/Unit/Mgx.ErrorContract.Tests.ps1 b/tests/Unit/Mgx.ErrorContract.Tests.ps1 new file mode 100644 index 0000000..7e4fcbe --- /dev/null +++ b/tests/Unit/Mgx.ErrorContract.Tests.ps1 @@ -0,0 +1,64 @@ +#Requires -Modules Pester + +<# + Pins what about_Mgx_Errors documents, as far as it reaches without a transport: + pre-flight failures terminate with stable ids, land in -ErrorVariable, and set $? + false. Per-item wire behavior (BatchItemError, -ErrorAction Stop mid-batch) is pinned + by the xUnit suite, which fakes HTTP. +#> + +BeforeAll { + Import-Module -Name (Join-Path (Split-Path -Parent $PSScriptRoot) 'TestHarness.psm1') -Force + $script:Paths = Get-MgxTestPath + Import-Module -Name $script:Paths.ManifestPath -Force -ErrorAction Stop +} + +Describe 'about_Mgx_Errors' { + It 'ships and documents both halves of the contract' { + $topic = Get-Help about_Mgx_Errors -ErrorAction Stop | Out-String + $topic | Should -Match 'TERMINATING ERRORS' + $topic | Should -Match 'NON-TERMINATING ERRORS' + $topic | Should -Match 'CANCELLATION' + } +} + +Describe 'Pre-flight failures terminate' { + It ' terminates with ' -TestCases @( + @{ Name = 'an absolute Uri on Invoke-MgxRequest' + Script = { Invoke-MgxRequest -Uri 'https://graph.microsoft.com/v1.0/users' } + ErrorId = 'AbsoluteUriNotAllowed' } + @{ Name = 'an absolute Uri on Export-MgxCollection' + Script = { Export-MgxCollection -Uri 'https://graph.microsoft.com/v1.0/users' -OutputFile ([IO.Path]::GetTempFileName()) } + ErrorId = 'AbsoluteUriNotAllowed' } + @{ Name = 'an absolute Uri on Sync-MgxDelta' + Script = { Sync-MgxDelta -Uri 'https://graph.microsoft.com/v1.0/users/delta' -DeltaPath ([IO.Path]::GetTempFileName()) } + ErrorId = 'AbsoluteUriNotAllowed' } + @{ Name = '-Search without ConsistencyLevel' + Script = { Invoke-MgxRequest -Uri '/users' -Search 'displayName:x' } + ErrorId = 'ConsistencyLevelRequired' } + @{ Name = 'a -Uri without {id} on Expand-MgxRelation' + Script = { [pscustomobject]@{ id = 'x' } | Expand-MgxRelation -Uri '/users/x/manager' -As manager } + ErrorId = 'MissingIdPlaceholder' } + ) { + # A terminating error is catchable; a non-terminating one is not. Catching it IS + # the assertion that it terminates. + $caught = $null + try { & $Script } catch { $caught = $_ } + $caught | Should -Not -BeNullOrEmpty + $caught.FullyQualifiedErrorId | Should -Match "^$ErrorId" + } +} + +Describe 'about_Mgx_Errors vocabulary' { + It 'names every category the presentation map can produce' { + # A text check only: the categories themselves are asserted where records are + # actually produced (the xUnit wire suites). This holds the DOCUMENT to the + # vocabulary, not the code to the document. + $topic = Get-Help about_Mgx_Errors | Out-String + foreach ($category in 'ObjectNotFound', 'PermissionDenied', 'AuthenticationError', + 'LimitsExceeded', 'ResourceUnavailable', 'ConnectionError', + 'InvalidArgument', 'ResourceExists') { + $topic | Should -Match $category + } + } +} diff --git a/tests/benchmarks/10-pacing-under-real-throttling.ps1 b/tests/benchmarks/10-pacing-under-real-throttling.ps1 index e323a8a..7a197e9 100644 --- a/tests/benchmarks/10-pacing-under-real-throttling.ps1 +++ b/tests/benchmarks/10-pacing-under-real-throttling.ps1 @@ -5,47 +5,183 @@ # independent of send rate and backing off has no upside by construction. Real Graph throttles # HARDER the faster you push, which is the whole premise of pacing. # -# This drives the SAME read workload twice against a real tenant - once paced (2.1 default), -# once with -NoAdaptivePacing - hard enough to cross the resource-unit budget and provoke -# genuine 429s. Reads only: throttling is not write-specific, so this needs no write consent. +# The first version of this benchmark did not answer it either, and looked like it had. It drove +# 8,000 reads at concurrency 128, spent 8,009 RU at ~180 RU/s, and recorded zero throttle retries +# in BOTH arms. Benchmark 13 explains why: this tenant hands out ~23,000 RU of burst allowance +# before the first refusal and keeps serving ~730 RU/s while refusing, so that run was a third of +# the burst at a quarter of the rate. It could not have been throttled. A clean result there says +# nothing about pacing - it says the workload was small. # -# Each round runs in a FRESH pwsh process because the adapted rate is static process state, -# with a cooldown between so the second round does not inherit a drained budget. +# So this version states its admissibility, takes the thresholds from 13's recorded result rather +# than a constant someone has to remember to update, and records whether the run actually entered +# the throttled regime. If the unpaced arm was never refused there was nothing for pacing to +# avoid, and the comparison is reported as inconclusive rather than as a win for whichever arm +# happened to finish first. +# +# Reaching the regime needs more in flight than one pipeline holds: Invoke-MgxRequest caps +# -Concurrency at 128, and 128 in flight measured ~180 req/s. The work is therefore split across +# N pipelines in parallel runspaces of ONE process. The pacer, the telemetry collector and the +# HTTP client are all process-wide statics, so those runspaces share a single pacer - which is +# the point: one workload bucket, N producers, the shape a real fan-out has. +# +# Requests are /users/{id}?$select=id, 1 RU each, the same shape 13 calibrated the thresholds +# with, so request counts and unit counts are the same number. +# +# Reads only: throttling is not write-specific, so this needs no write consent. param( [ValidateSet('both', 'paced', 'unpaced')] [string] $Mode = 'both', - [int] $Count = 20000, - [int] $Concurrency = 60, + [int] $Count = 120000, + [int] $Pipelines = 8, + [ValidateRange(1, 128)] [int] $Concurrency = 128, + [int] $IdPool = 5000, [int] $CooldownSeconds = 300 ) . "$PSScriptRoot/common.ps1" +function Get-ThrottleThreshold { + <# + The bar this benchmark has to clear, measured rather than assumed. Benchmark 13 holds a + fixed send rate until the tenant refuses and records both numbers that matter: the burst + allowance (units served before the first 429) and the rate the tenant keeps serving at + once it is refusing. Reading them back here means a tenant with a different budget + recalibrates by running 13, not by editing a constant in this file. + + Without a recorded 13, fall back to the documented 8,000 RU per 10s for tenants above + 500 users - and say which one is in force, because a threshold that came from a document + and one that came from this tenant deserve different confidence. + #> + $file = Join-Path $PSScriptRoot 'results/13-resource-unit-rate.json' + if (Test-Path $file) { + $rounds = @(Get-Content $file -Raw | ConvertFrom-Json | ForEach-Object { + $_.Result.PSObject.Properties | Where-Object { $_.Name -match '^rate\d+$' } | + ForEach-Object { $_.Value } + }) + $refused = @($rounds | Where-Object { $null -ne $_.First429Seconds -and $_.ServedRateAfter429 }) + if ($refused) { + return @{ + BurstRu = [int](($refused | Measure-Object RuBeforeFirst429 -Maximum).Maximum) + Sustained= [int](($refused | Measure-Object ServedRateAfter429 -Minimum).Minimum) + Source = 'benchmark 13, this tenant' + } + } + } + @{ BurstRu = 8000; Sustained = 800; Source = 'documented default (8,000 RU / 10s)' } +} + +$threshold = Get-ThrottleThreshold + if ($Mode -eq 'both') { + Write-Host ("throttling threshold from {0}: burst {1} RU, sustained {2} RU/s" -f ` + $threshold.Source, $threshold.BurstRu, $threshold.Sustained) + if ($Count -lt $threshold.BurstRu * 3) { + Write-Host (" WARNING: -Count $Count is under 3x the burst allowance ($($threshold.BurstRu) RU).") -ForegroundColor Yellow + Write-Host (" The run may finish inside the allowance and measure nothing. Raise -Count.") -ForegroundColor Yellow + } + + $resultFile = Join-Path $PSScriptRoot 'results/10-pacing-under-real-throttling.json' + $arms = [ordered]@{} + $roundIndex = 0 foreach ($m in 'paced', 'unpaced') { + $roundIndex++ + if ($roundIndex -gt 1) { + # The bucket refills at roughly the sustained rate, so a full burst allowance is back + # in ~30s. 300s is an order of magnitude over that, which is the margin worth paying + # to be sure arm two is not measuring arm one's drained budget. + Write-Host "cooldown ${CooldownSeconds}s (let the resource-unit budget refill)..." + Start-Sleep -Seconds $CooldownSeconds + } Write-Host "=== round: $m ===" -ForegroundColor Cyan + # Start-Process rather than capturing stdout: each arm runs for minutes, and a round + # whose output only appears once it has finished is a round you cannot tell from a hang. + # The arm's own JSON record carries everything compared below, so nothing needs parsing + # out of the console. + $before = @(if (Test-Path $resultFile) { Get-Content $resultFile -Raw | ConvertFrom-Json }).Count $p = Start-Process pwsh -PassThru -Wait -NoNewWindow -ArgumentList ` - '-NoProfile', '-File', $PSCommandPath, '-Mode', $m, '-Count', $Count, '-Concurrency', $Concurrency + '-NoProfile', '-File', $PSCommandPath, '-Mode', $m, '-Count', $Count, ` + '-Pipelines', $Pipelines, '-Concurrency', $Concurrency, '-IdPool', $IdPool if ($p.ExitCode -ne 0) { throw "round '$m' failed (exit $($p.ExitCode))" } - if ($m -eq 'paced') { - Write-Host "cooldown ${CooldownSeconds}s (let the resource-unit budget refill)..." - Start-Sleep -Seconds $CooldownSeconds + + $entries = @(Get-Content $resultFile -Raw | ConvertFrom-Json) + $entry = @($entries | Select-Object -Skip $before | Where-Object { $_.Result.Mode -eq $m } | Select-Object -Last 1)[0] + if (-not $entry) { throw "round '$m' recorded no result in $resultFile" } + $r = $entry.Result + $arms[$m] = @{ + Seconds = [math]::Round($r.WallMs / 1000, 1) + Ok = [int]$r.Ok + Failed = [int]$r.Failed + ThrottleRetries = [int]$r.ThrottleRetries + RetryDelayMs = [long]$r.RetryDelayMs + PacingWaitMs = [long]$r.PacingWaitMs + ResourceUnits = [long]$r.ResourceUnits + RuPerSecond = if ($r.WallMs -gt 0) { [math]::Round($r.ResourceUnits / ($r.WallMs / 1000), 1) } else { 0 } + } + } + + Write-Host '' + Write-Host "=== ADAPTIVE PACING AGAINST REAL THROTTLING ($Count reads, $Pipelines x $Concurrency in flight) ===" + Write-Host ('{0,-28} {1,8} {2,8} {3,8} {4,10} {5,12} {6,10}' -f ` + 'Arm', 'Wall', 'Ok', 'Failed', '429 retries', 'RetryDelay', 'RU/s') + foreach ($k in 'paced', 'unpaced') { + $a = $arms[$k] + $label = if ($k -eq 'paced') { 'Adaptive pacing on (default)' } else { '-NoAdaptivePacing' } + Write-Host ('{0,-28} {1,7:F1}s {2,8} {3,8} {4,10} {5,11:F1}s {6,10:F1}' -f ` + $label, $a.Seconds, $a.Ok, $a.Failed, $a.ThrottleRetries, ($a.RetryDelayMs / 1000), $a.RuPerSecond) + } + + # The verdict, and the conditions under which there is not one. An unpaced arm that was never + # refused did not reach the regime this benchmark exists to measure, and the wall times below + # it are then a comparison of two unthrottled runs - which is benchmark 14's question, already + # answered there. Saying so is the difference between no result and a wrong one. + Write-Host '' + $conclusive = $arms.unpaced.ThrottleRetries -gt 0 + if (-not $conclusive) { + Write-Host 'INCONCLUSIVE: the unpaced arm was never throttled, so there was nothing for pacing to avoid.' -ForegroundColor Yellow + Write-Host (" It spent {0} RU at {1} RU/s; the tenant serves {2} RU/s while refusing and allows {3} RU of burst." -f ` + $arms.unpaced.ResourceUnits, $arms.unpaced.RuPerSecond, $threshold.Sustained, $threshold.BurstRu) -ForegroundColor Yellow + Write-Host ' Raise -Count (spend further past the burst) or -Pipelines (offer a higher rate), then re-run.' -ForegroundColor Yellow + } + else { + $delta = $arms.unpaced.Seconds - $arms.paced.Seconds + $verdict = if ($delta -gt 0) { "pacing finished {0:F1}s sooner" -f $delta } + elseif ($delta -lt 0) { "pacing finished {0:F1}s later" -f (-$delta) } + else { 'both arms finished level' } + Write-Host ("Unpaced took {0} throttle retries and {1:F1}s honoring Retry-After; {2}." -f ` + $arms.unpaced.ThrottleRetries, ($arms.unpaced.RetryDelayMs / 1000), $verdict) + if ($arms.unpaced.Failed -gt $arms.paced.Failed) { + Write-Host ("Unpaced also lost {0} request(s) that pacing completed." -f ` + ($arms.unpaced.Failed - $arms.paced.Failed)) } } + + Write-BenchResult -Benchmark '10-pacing-under-real-throttling' -Result ([pscustomobject]@{ + Count = $Count + Pipelines = $Pipelines + Concurrency = $Concurrency + Threshold = $threshold + Conclusive = $conclusive + Arms = $arms + }) Write-Host 'both rounds complete - see results/10-pacing-under-real-throttling.json' return } +# --- single arm --------------------------------------------------------------------------- + Import-MgxLocal Connect-MgxBenchmark -# A fixed, pre-materialised id list. Bounded work only: no loop that can run away. -Write-Host "collecting $Count ids..." -$ids = Get-BenchUserIds -Count $Count -Write-Host " got $($ids.Count)" +# A bounded id pool, cycled: the tenant does not need 120,000 users for the workload to draw +# 120,000 units, and a fixed pre-materialized list keeps the work bounded with no loop that can +# run away. +Write-Host "collecting up to $IdPool ids..." +$pool = Get-BenchUserIds -Count $IdPool +Write-Host " got $($pool.Count); cycling to $Count requests" -# The token-bucket limiter would cap us well under the RU budget and we would never reach -# the throttling regime this benchmark exists to measure. Lift it; the pacer (or its absence) -# is the variable under test. +# The token-bucket limiter would cap us well under the RU budget and we would never reach the +# throttling regime this benchmark exists to measure. Lift it; the pacer (or its absence) is the +# variable under test. Both settings are process-wide statics, so the parallel runspaces below +# inherit them without setting them again. Set-MgxOption -NoRateLimit if ($Mode -eq 'unpaced') { Set-MgxOption -NoAdaptivePacing @@ -54,29 +190,75 @@ if ($Mode -eq 'unpaced') { Write-Host ' adaptive pacing ON (2.1 default)' } +$script:slices = [System.Collections.Generic.List[object]]::new() +for ($i = 0; $i -lt $Pipelines; $i++) { $script:slices.Add([System.Collections.Generic.List[string]]::new()) } +for ($i = 0; $i -lt $Count; $i++) { $script:slices[$i % $Pipelines].Add($pool[$i % $pool.Count]) } + +$script:commonPath = Join-Path $PSScriptRoot 'common.ps1' +$script:armConcurrency = $Concurrency +$script:modulePath = (Get-Module Mgx).Path + $result = Measure-BenchPass -Name "reads $Mode" -Script { - $items = @($ids | Invoke-MgxRequest '/users/{id}' -Property id,displayName ` - -Concurrency $Concurrency -ErrorVariable errs -ErrorAction SilentlyContinue) - @{ ok = $items.Count; failed = @($errs).Count } + $tallies = $script:slices | ForEach-Object -Parallel { + # A fresh runspace has no modules, but the assembly - and with it the pacer, the + # telemetry collector and the HTTP client - loads once per PROCESS. Importing here + # attaches this runspace to that shared state rather than creating a second copy, and + # Connect-MgxBenchmark returns immediately because the Graph session is static too; + # each runspace saying so is the confirmation that they share one pacer. + # + # Deliberately not Import-MgxLocal: it imports -Force, and eight runspaces force-loading + # the same binary module at once is a race for no gain. The parent already imported it, + # which fixed the Users-before-Mgx assembly order this only has to preserve. + . $using:commonPath + if (-not (Get-Module Microsoft.Graph.Users)) { Import-Module Microsoft.Graph.Users } + Import-Module $using:modulePath + Connect-MgxBenchmark | Out-Null + $slice = $_ + $items = @($slice | Invoke-MgxRequest '/users/{id}' -Property id ` + -Concurrency $using:armConcurrency -ErrorVariable errs -ErrorAction SilentlyContinue) + [pscustomobject]@{ Ok = $items.Count; Failed = @($errs).Count } + } -ThrottleLimit $Pipelines + + @{ + ok = (@($tallies) | Measure-Object Ok -Sum).Sum + failed = (@($tallies) | Measure-Object Failed -Sum).Sum + } } $t = Get-MgxTelemetry +$wallSeconds = $result.ElapsedMs / 1000 Write-Host '' Write-Host ("=== $Mode ===") -Write-Host (" wall {0,8:F1}s" -f ($result.ElapsedMs / 1000)) +Write-Host (" wall {0,8:F1}s" -f $wallSeconds) Write-Host (" completed {0,8} failed {1}" -f $result.Output.ok, $result.Output.failed) Write-Host (" throttle retries {0,8}" -f $t.ThrottleRetries) -Write-Host (" retry delay {0,8:F1}s (time spent honouring Retry-After)" -f ($t.RetryDelayMs / 1000)) +Write-Host (" retry delay {0,8:F1}s (time spent honoring Retry-After)" -f ($t.RetryDelayMs / 1000)) Write-Host (" pacing wait {0,8:F1}s over {1} activations" -f ($t.AdaptivePacingWaitMs / 1000), $t.AdaptivePacingActivations) -Write-Host (" resource units {0,8} ({1:F1} RU/s)" -f $t.ResourceUnitsConsumed, ($t.ResourceUnitsConsumed / ($result.ElapsedMs / 1000))) +Write-Host (" resource units {0,8} ({1:F1} RU/s)" -f $t.ResourceUnitsConsumed, ($t.ResourceUnitsConsumed / $wallSeconds)) Write-Host (" pacing state {0}" -f $t.PacingState) -# Write-BenchResult attaches the telemetry snapshot itself, including resource units. +if ($Mode -eq 'unpaced' -and $t.ThrottleRetries -eq 0) { + Write-Host '' + Write-Host (" NOTE: no refusal at {0:F0} RU/s over {1} units. This arm did not reach the throttled" -f ` + ($t.ResourceUnitsConsumed / $wallSeconds), $t.ResourceUnitsConsumed) -ForegroundColor Yellow + Write-Host (" regime, so it sets no baseline for the paced arm to beat.") -ForegroundColor Yellow +} + +# WallMs is read back by Write-BenchResult for the RU rate: telemetry's own elapsed total is the +# SUM of per-request durations and would understate the budget draw by the concurrency factor. +# ThrottleRetries, RetryDelayMs and ResourceUnits are duplicated out of telemetry into the +# result itself: the parent process reads this record back to compare the arms, and it never +# loads Mgx, so it has no telemetry of its own to read them from. Write-BenchResult -Benchmark '10-pacing-under-real-throttling' -Result ([pscustomobject]@{ - Mode = $Mode - Count = $ids.Count - Concurrency = $Concurrency - WallMs = $result.ElapsedMs - Ok = $result.Output.ok - Failed = $result.Output.failed -}) + Mode = $Mode + Count = $Count + Pipelines = $Pipelines + Concurrency = $Concurrency + WallMs = $result.ElapsedMs + Ok = $result.Output.ok + Failed = $result.Output.failed + ThrottleRetries = $t.ThrottleRetries + RetryDelayMs = $t.RetryDelayMs + PacingWaitMs = $t.AdaptivePacingWaitMs + ResourceUnits = $t.ResourceUnitsConsumed +}) -WallMs ([long]$result.ElapsedMs) diff --git a/tests/benchmarks/15-fanout-concurrency-scaling.ps1 b/tests/benchmarks/15-fanout-concurrency-scaling.ps1 new file mode 100644 index 0000000..10405c3 --- /dev/null +++ b/tests/benchmarks/15-fanout-concurrency-scaling.ps1 @@ -0,0 +1,233 @@ +# Benchmark 15: does raising -Concurrency actually put more requests on the wire? +# +# Benchmark 10 recorded 8,000 reads at -Concurrency 128 and achieved ~180 req/s, with the pacer's +# per-attempt latency baseline sitting at 632ms. The same tenant at -Concurrency 20 recorded a +# 94ms baseline. If 128 requests were genuinely in flight at 94ms each the run would have moved +# ~1,360 req/s, so something between the fan-out semaphore and the socket is not absorbing them - +# and until that is known, raising Invoke-MgxRequest's documented 1-128 range would buy nothing. +# +# The recorded latency cannot settle it on its own. RecordHttpTime wraps HttpClient.SendAsync +# (ResilientGraphClient.cs:245-248), and SendAsync includes waiting for a free connection out of +# the pool, so queueing time is INSIDE the number. Little's law over it is circular: it returns +# the offered concurrency by construction whether or not the transport is the constraint. +# +# So the question is settled by control rather than inference. Two raw HttpClients, identical to +# the module's transport in every respect except MaxConnectionsPerServer - 20 (what +# TransportDefaults ships) against 512 - offered the same concurrency over the same requests. If +# the pool is the ceiling, 512 pulls away and the gap IS the cost of the default. If they match, +# 20 connections were never the constraint and the service is simply slower under load, which +# makes the current range correct as it stands. +# +# The module ladder above it answers the user-facing half: what a caller gets for each step up +# in -Concurrency. Rate limiting and pacing are both off - either one would be measured instead +# of the transport. +# +# Reads only, ~1 RU each, a few thousand units total: this stays far inside the burst allowance +# benchmark 13 measured, so nothing here provokes throttling. +param( + [int[]] $Ladder = @(8, 20, 40, 64, 128), + [int] $Count = 400, + [int] $IdPool = 400, + [int] $RawOffered = 128, + [int[]] $RawPools = @(20, 512) +) + +. "$PSScriptRoot/common.ps1" +Import-MgxLocal +Connect-MgxBenchmark + +Write-Host "collecting up to $IdPool ids..." +$ids = Get-BenchUserIds -Count $IdPool +Write-Host " got $($ids.Count)" +$work = @(0..($Count - 1) | ForEach-Object { $ids[$_ % $ids.Count] }) + +# --- Part A: what a caller gets per step of -Concurrency ------------------------------------ + +# Both gates off. The token bucket caps request rate outright and the pacer holds a cold +# workload at 4 rps doubling per clean second; either would be the thing measured. +Set-MgxOption -NoRateLimit +Set-MgxOption -NoAdaptivePacing + +# A cold connection pool would charge its handshakes to whichever arm ran first. Warm it, then +# run the ladder up and back down: if the two directions disagree, the ladder is measuring drift +# rather than concurrency, and the report says so instead of averaging the disagreement away. +Write-Host 'warming the connection pool...' +$null = @($work | Select-Object -First 40 | Invoke-MgxRequest '/users/{id}' -Property id -Concurrency 8 ` + -ErrorAction SilentlyContinue) + +function Measure-Ladder { + param([int[]] $Steps, [string] $Direction) + $out = [ordered]@{} + foreach ($c in $Steps) { + $pass = Measure-BenchPass -Name "concurrency $c ($Direction)" -Script { + $items = @($work | Invoke-MgxRequest '/users/{id}' -Property id -Concurrency $c ` + -ErrorVariable errs -ErrorAction SilentlyContinue) + @{ ok = $items.Count; failed = @($errs).Count } + } + $seconds = $pass.ElapsedMs / 1000 + $requests = if ($pass.MgxTelemetry) { $pass.MgxTelemetry.Requests } else { $pass.Output.ok } + $out["$c"] = [pscustomobject]@{ + Concurrency = $c + Seconds = [math]::Round($seconds, 2) + Ok = $pass.Output.ok + Failed = $pass.Output.failed + RequestsPerSecond = if ($seconds -gt 0) { [math]::Round($pass.Output.ok / $seconds, 1) } else { 0 } + # Mean time inside SendAsync, which includes any wait for a free connection. + MeanHttpMs = if ($pass.MgxTelemetry -and $requests -gt 0) { + [math]::Round($pass.MgxTelemetry.HttpMs / $requests, 1) + } else { $null } + } + Write-Host (" concurrency {0,4}: {1,7:F2}s {2,7:F1} req/s mean http {3,7} ms" -f ` + $c, $out["$c"].Seconds, $out["$c"].RequestsPerSecond, $out["$c"].MeanHttpMs) + } + $out +} + +Write-Host '' +Write-Host "=== Part A: Invoke-MgxRequest -Concurrency ladder ($Count reads per step) ===" +$up = Measure-Ladder -Steps $Ladder -Direction 'up' +Write-Host ' --- and back down (ordering check) ---' +$down = Measure-Ladder -Steps ($Ladder[($Ladder.Count - 1)..0]) -Direction 'down' + +# --- Part B: the transport control ---------------------------------------------------------- + +function Invoke-RawFanOut { + <# + The module's transport with one variable changed. Everything here mirrors + MgxCmdletBase's handler and TransportDefaults - HTTP/2 preferred with fallback allowed, + the same connect timeout, lifetime and decompression - so MaxConnectionsPerServer is the + only difference between the two arms, and the difference between their numbers has + exactly one cause. + #> + param( + [Parameter(Mandatory)] [string] $Token, + [Parameter(Mandatory)] [string[]] $Work, + [Parameter(Mandatory)] [int] $Offered, + [Parameter(Mandatory)] [int] $PoolSize, + [int] $DeadlineSeconds = 180 + ) + $handler = [System.Net.Http.SocketsHttpHandler]::new() + $handler.MaxConnectionsPerServer = $PoolSize + $handler.EnableMultipleHttp2Connections = $true + $handler.ConnectTimeout = [TimeSpan]::FromSeconds(10) + $handler.PooledConnectionLifetime = [TimeSpan]::FromMinutes(2) + $handler.AutomaticDecompression = [System.Net.DecompressionMethods]::GZip -bor + [System.Net.DecompressionMethods]::Deflate -bor + [System.Net.DecompressionMethods]::Brotli + $client = [System.Net.Http.HttpClient]::new($handler) + $client.Timeout = [TimeSpan]::FromSeconds(120) + $client.DefaultRequestVersion= [System.Net.HttpVersion]::Version20 + $client.DefaultVersionPolicy = [System.Net.Http.HttpVersionPolicy]::RequestVersionOrLower + $client.DefaultRequestHeaders.Authorization = + [System.Net.Http.Headers.AuthenticationHeaderValue]::new('Bearer', $Token) + + # Warm the pool at this size before timing: connection setup belongs to neither arm. + try { ($client.GetAsync("https://graph.microsoft.com/v1.0/users/$($Work[0])?`$select=id")).GetAwaiter().GetResult().Dispose() } catch { } + + $pending = [System.Collections.Generic.List[System.Threading.Tasks.Task[System.Net.Http.HttpResponseMessage]]]::new() + $sent = 0; $ok = 0; $failed = 0; $version = $null + + # Harvest is inline rather than a scriptblock. Benchmark 13 can use one because its counters + # live at script scope; in here they are function locals, and a scriptblock assigning to + # $script:ok would tick a different variable and report zero completions forever. + # A deadline, because the loop's exit condition is "everything landed". HttpClient.Timeout + # would eventually fault a hung request, but an arm that spends two minutes per straggler is + # not a measurement either - it is a hang with a stopwatch attached. + $sw = [Diagnostics.Stopwatch]::StartNew() + while (($sent -lt $Work.Count -or $pending.Count -gt 0) -and $sw.Elapsed.TotalSeconds -lt $DeadlineSeconds) { + while ($pending.Count -lt $Offered -and $sent -lt $Work.Count) { + $pending.Add($client.GetAsync("https://graph.microsoft.com/v1.0/users/$($Work[$sent])?`$select=id")) + $sent++ + } + + $still = [System.Collections.Generic.List[System.Threading.Tasks.Task[System.Net.Http.HttpResponseMessage]]]::new() + foreach ($t in $pending) { + if (-not $t.IsCompleted) { $still.Add($t); continue } + if ($t.Status -ne 'RanToCompletion') { $failed++; continue } + $r = $t.Result + if ([int]$r.StatusCode -ge 200 -and [int]$r.StatusCode -lt 300) { + $ok++ + if ($null -eq $version) { $version = $r.Version.ToString() } + } else { $failed++ } + $r.Dispose() + } + $pending = $still + + if ($pending.Count -ge $Offered -or $sent -ge $Work.Count) { Start-Sleep -Milliseconds 5 } + } + $sw.Stop() + $abandoned = $pending.Count + ($Work.Count - $sent) + $client.Dispose() + + if ($abandoned -gt 0) { + Write-Host (" WARNING: pool $PoolSize hit the ${DeadlineSeconds}s deadline with $abandoned request(s) unaccounted for;") -ForegroundColor Yellow + Write-Host (' its rate is a floor, not a measurement.') -ForegroundColor Yellow + } + + [pscustomobject]@{ + PoolSize = $PoolSize + Offered = $Offered + Seconds = [math]::Round($sw.Elapsed.TotalSeconds, 2) + Ok = $ok + Failed = $failed + Abandoned = $abandoned + RequestsPerSecond = if ($sw.Elapsed.TotalSeconds -gt 0) { [math]::Round($ok / $sw.Elapsed.TotalSeconds, 1) } else { 0 } + NegotiatedVersion = $version + } +} + +Write-Host '' +Write-Host "=== Part B: same transport, pool size varied (offered concurrency $RawOffered) ===" +$token = Get-BenchAppToken +$raw = [ordered]@{} +foreach ($pool in $RawPools) { + $r = Invoke-RawFanOut -Token $token -Work $work -Offered $RawOffered -PoolSize $pool + $raw["$pool"] = $r + Write-Host (" pool {0,4}: {1,7:F2}s {2,7:F1} req/s ok {3} failed {4} negotiated HTTP/{5}" -f ` + $r.PoolSize, $r.Seconds, $r.RequestsPerSecond, $r.Ok, $r.Failed, $r.NegotiatedVersion) +} + +# --- Verdict --------------------------------------------------------------------------------- + +Write-Host '' +$default = $raw["$($RawPools[0])"] +$wide = $raw["$($RawPools[-1])"] +$ratio = if ($default.RequestsPerSecond -gt 0) { $wide.RequestsPerSecond / $default.RequestsPerSecond } else { 0 } + +if ($default.Abandoned -gt 0 -or $wide.Abandoned -gt 0) { + Write-Host 'INCONCLUSIVE: an arm did not finish inside its deadline, so the two rates are not comparable.' -ForegroundColor Yellow + Write-Host ' Re-run with a smaller -Count, or a longer deadline, before reading anything into the ratio.' -ForegroundColor Yellow +} + +if ($default.NegotiatedVersion -and $default.NegotiatedVersion -notlike '2*') { + Write-Host ("HTTP/{0} was negotiated, not HTTP/2. TransportDefaults sizes the pool at 20 on the" -f $default.NegotiatedVersion) -ForegroundColor Yellow + Write-Host ' assumption of multiplexing, so without it 20 connections is 20 requests in flight.' -ForegroundColor Yellow +} +if ($ratio -ge 1.5) { + Write-Host ("POOL-BOUND: {0} connections served {1:F1} req/s, {2} served {3:F1} - {4:F1}x." -f ` + $default.PoolSize, $default.RequestsPerSecond, $wide.PoolSize, $wide.RequestsPerSecond, $ratio) + Write-Host ' MaxConnectionsPerServer is the fan-out ceiling, not -Concurrency. Its comment sizes it' + Write-Host ' for "fan-out concurrency of 5" - the default, not the 128 the parameter allows.' +} +else { + Write-Host ("NOT POOL-BOUND: {0} connections and {1} are within {2:F1}x of each other." -f ` + $default.PoolSize, $wide.PoolSize, [math]::Max($ratio, 1 / [math]::Max($ratio, 0.001))) + Write-Host ' The service, not the connection pool, is what limits this fan-out. Raising either' + Write-Host ' MaxConnectionsPerServer or the -Concurrency range would buy nothing.' +} + +$first = $up["$($Ladder[0])"] +$last = $up["$($Ladder[-1])"] +$ladderGain = if ($first.RequestsPerSecond -gt 0) { $last.RequestsPerSecond / $first.RequestsPerSecond } else { 0 } +Write-Host ("Ladder: {0:F1}x concurrency ({1} -> {2}) returned {3:F1}x throughput." -f ` + ($Ladder[-1] / $Ladder[0]), $Ladder[0], $Ladder[-1], $ladderGain) + +Write-BenchResult -Benchmark '15-fanout-concurrency-scaling' -Result ([pscustomobject]@{ + LadderUp = $up + LadderDown = $down + RawByPool = $raw + PoolRatio = [math]::Round($ratio, 2) + PoolBound = ($ratio -ge 1.5) + NegotiatedVersion = $default.NegotiatedVersion + Count = $Count +}) diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md index 0904c9d..3b459da 100644 --- a/tests/benchmarks/README.md +++ b/tests/benchmarks/README.md @@ -23,7 +23,7 @@ there is no session telemetry to read - expected, not a failure. - A seeded test tenant (~100k users, ~15k groups) for the tenant benchmarks. `06-fault-gauntlet.ps1` needs **no tenant** — it runs against a local mock. -Never point these scripts at a production tenant. Benchmarks 04 and 07 create and delete objects; 07 deliberately provokes throttling. +Never point these scripts at a production tenant. Benchmarks 04 and 07 create and delete objects; 07 and 10 deliberately provoke throttling. ## The benchmarks @@ -42,9 +42,12 @@ Never point these scripts at a production tenant. Benchmarks 04 and 07 create an | 11 | `11-throttle-accuracy.ps1` | Under throttling, retrieval hinges on honoring `Retry-After` | yes | | 12 | `12-delta-replay.ps1` | Delta enumerations repeat objects; replay factor vs ground truth | yes | | 13 | `13-resource-unit-rate.ps1` | Sustained RU ceiling vs burst allowance, at held send rates | yes | +| 14 | `14-pacing-cold-cost.ps1` | What pacing costs a short run that finishes during the ramp | yes | +| 15 | `15-fanout-concurrency-scaling.ps1` | What `-Concurrency` buys, and whether the connection pool is the ceiling | yes | -11-13 are not in `run.ps1`: each deliberately drives the tenant to 429s, so they run standalone -(10 also throttles on purpose, which is why `run.ps1` puts it last). +11-15 are not in `run.ps1`. 11-13 each deliberately drive the tenant to 429s, so they run +standalone (10 also throttles on purpose, which is why `run.ps1` puts it last). 14 and 15 are +cheap and answer one question each; they are read on their own rather than as part of a sweep. ## Methodology diff --git a/tests/benchmarks/common.ps1 b/tests/benchmarks/common.ps1 index cf43f7b..9041373 100644 --- a/tests/benchmarks/common.ps1 +++ b/tests/benchmarks/common.ps1 @@ -1,6 +1,9 @@ # Shared plumbing for the Mgx benchmark suite. Dot-source from each benchmark script: # . "$PSScriptRoot/common.ps1" -# Auth resolution order: existing Graph session, then $env:MGX_BENCH_APP, then ~/.mgx-bench/app.json. +# Auth resolution order: existing Graph session, then AZURE_* certificate variables, then +# AZURE_* secret variables, then $env:MGX_BENCH_APP / ~/.mgx-bench/app.json. With both a +# certificate path and a secret set, the certificate wins - and a certificate path that +# does not exist throws rather than falling through, so a stale path masks the secret. $ErrorActionPreference = 'Stop' @@ -33,10 +36,20 @@ function Connect-MgxBenchmark { return } + # A secret in the environment, for an app that has no certificate uploaded. Same three + # variables minus the certificate, and like the certificate path it leaves nothing at rest. + if ($env:AZURE_TENANT_ID -and $env:AZURE_CLIENT_ID -and $env:AZURE_CLIENT_SECRET) { + $cred = [pscredential]::new($env:AZURE_CLIENT_ID, + (ConvertTo-SecureString $env:AZURE_CLIENT_SECRET -AsPlainText -Force)) + Connect-MgGraph -TenantId $env:AZURE_TENANT_ID -ClientSecretCredential $cred -NoWelcome + Write-Host "Connected app-only by secret ($($env:AZURE_CLIENT_ID))" + return + } + $credPath = if ($env:MGX_BENCH_APP) { $env:MGX_BENCH_APP } else { Join-Path $HOME '.mgx-bench/app.json' } if (-not (Test-Path $credPath)) { - throw "No Graph session, no AZURE_* certificate variables, and no app credentials at '$credPath'." + throw "No Graph session, no AZURE_* certificate or secret variables, and no app credentials at '$credPath'." } $cfg = Get-Content $credPath -Raw | ConvertFrom-Json $cred = [pscredential]::new($cfg.appId, (ConvertTo-SecureString $cfg.clientSecret -AsPlainText -Force)) @@ -88,6 +101,20 @@ function Get-BenchAppToken { } } + if ($env:AZURE_TENANT_ID -and $env:AZURE_CLIENT_ID -and $env:AZURE_CLIENT_SECRET) { + foreach ($attempt in 1..3) { + try { + $resp = Invoke-RestMethod -Method POST -TimeoutSec 30 ` + -Uri "https://login.microsoftonline.com/$($env:AZURE_TENANT_ID)/oauth2/v2.0/token" ` + -Body @{ grant_type = 'client_credentials'; client_id = $env:AZURE_CLIENT_ID + client_secret = $env:AZURE_CLIENT_SECRET + scope = 'https://graph.microsoft.com/.default' } + return $resp.access_token + } + catch { if ($attempt -eq 3) { throw }; Start-Sleep -Seconds 5 } + } + } + $credPath = if ($env:MGX_BENCH_APP) { $env:MGX_BENCH_APP } else { Join-Path $HOME '.mgx-bench/app.json' } if (-not (Test-Path $credPath)) { throw "No app credentials at '$credPath' (set MGX_BENCH_APP)." } From ccda2772eea829186d0e3ec7e8aef51ef7209f99 Mon Sep 17 00:00:00 2001 From: gromedev <61883549+gromedev@users.noreply.github.com> Date: Tue, 1 Sep 2026 11:33:21 +0200 Subject: [PATCH 05/10] mgx 2.1.4 --- .github/workflows/build-and-test.yml | 79 +- .gitignore | 1 + CHANGELOG.md | 50 + Directory.Build.props | 2 +- README.md | 6 +- module/en-US/Mgx.Cmdlets.dll-Help.xml | 12 +- module/en-US/about_Mgx_Errors.help.txt | 17 +- module/help/Invoke-MgxBatchRequest.md | 6 +- module/help/Invoke-MgxRequest.md | 2 +- module/mgx.psd1 | 39 +- src/Mgx.Cmdlets/AlcInitializer.cs | 36 +- src/Mgx.Cmdlets/Base/MgxCmdletBase.cs | 574 +++++++-- .../Cmdlets/Batch/InvokeMgxBatchRequest.cs | 151 ++- .../Configuration/DisableMgxResilience.cs | 34 +- .../Configuration/EnableMgxResilience.cs | 123 +- src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs | 577 +++++++-- .../Cmdlets/Expand/ExpandMgxRelation.cs | 48 +- .../Cmdlets/Export/ExportMgxCollection.cs | 588 +++++++-- src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs | 223 +++- src/Mgx.Engine/Http/AdaptiveRequestPacer.cs | 4 +- src/Mgx.Engine/Http/GraphBatchClient.cs | 460 ++++++- .../Http/ResiliencePipelineFactory.cs | 14 + src/Mgx.Engine/Http/ResilientGraphClient.cs | 270 ++++- src/Mgx.Engine/Models/GraphResponse.cs | 7 + src/Mgx.Engine/Pagination/ConcurrentFanOut.cs | 32 +- .../Pagination/PaginationCheckpoint.cs | 11 + tests/CORPUS.md | 66 + tests/Ecosystem/Mgx.Ecosystem.Tests.ps1 | 100 ++ tests/Live/Mgx.Live.Tests.ps1 | 2 +- tests/Live/README.md | 82 -- .../Cmdlets/Base/AuthFingerprintTests.cs | 51 +- .../Cmdlets/Base/BetaHintTests.cs | 39 +- .../Cmdlets/Base/TransportSeamOrderTests.cs | 52 + .../Base/TypeCacheInvalidationTests.cs | 129 ++ .../Cmdlets/BatchErrorSurfacingTests.cs | 114 +- .../Cmdlets/BulkWriteErrorSurfacingTests.cs | 86 ++ .../Cmdlets/DeltaCheckpointIntegrityTests.cs | 223 +++- .../Cmdlets/DeltaQueryTests.cs | 134 +- .../Cmdlets/DeltaRecoveryTests.cs | 118 +- .../Cmdlets/DeltaSharedCheckpointTests.cs | 996 +++++++++++++++ .../DeltaUndeletableCheckpointTests.cs | 375 ++++++ .../Cmdlets/DiagnosticsDrainTests.cs | 27 +- .../Cmdlets/ErrorActionMatrixTests.cs | 262 ++++ .../Cmdlets/ExportCheckpointIntegrityTests.cs | 333 ++++- .../Cmdlets/ExportCorruptCheckpointTests.cs | 216 ++++ .../ExportDegradedQueryCheckpointTests.cs | 711 +++++++++++ .../Cmdlets/ExportSharedCheckpointTests.cs | 542 +++++++++ .../Cmdlets/ExportWhatIfCheckpointTests.cs | 291 +++++ .../Cmdlets/HeaderWireTests.cs | 34 +- .../NamedTempPromotionOwnershipTests.cs | 372 ++++++ .../Cmdlets/OrphanTempOwnershipTests.cs | 342 ++++++ .../Cmdlets/OutFileGuardTests.cs | 24 +- .../ResilienceInjectionScenarioTests.cs | 560 +++++++++ .../Cmdlets/ResilienceWrapTests.cs | 4 +- .../Cmdlets/ResponseFidelityTests.cs | 296 ++++- .../Cmdlets/SkipLogicTests.cs | 41 +- .../Cmdlets/StaleTempSweepOwnershipTests.cs | 145 +++ .../Cmdlets/StalledBodyTests.cs | 137 +++ .../Cmdlets/TopWithAllTests.cs | 34 +- .../Cmdlets/UriEncodingTests.cs | 80 +- .../Engine/Batch/BatchEdgeCaseTests.cs | 10 +- .../Engine/Batch/BatchWriteTests.cs | 44 +- .../Engine/Batch/BulkWriteTests.cs | 148 ++- .../Batch/ParallelChunkPartialResultTests.cs | 854 +++++++++++++ .../Engine/Batch/PartialBatchResultTests.cs | 1078 ++++++++++++++++- .../Engine/Http/AdaptiveRequestPacerTests.cs | 30 + .../Engine/Http/GraphContentClientTests.cs | 110 +- .../Engine/Http/HeaderCultureTests.cs | 92 ++ .../Engine/Http/RateLimiterLifetimeTests.cs | 87 ++ .../Engine/Http/SendStopTests.cs | 233 ++++ .../Engine/Http/TransportSwapTests.cs | 82 ++ .../Engine/Pagination/BoundarySizeTests.cs | 89 ++ .../Pagination/NamedTempPromotionTests.cs | 108 ++ .../Engine/Pagination/OpaqueLinkTests.cs | 2 +- .../Engine/Pagination/OrphanAdoptionTests.cs | 146 ++- .../Fakes/FakeGraphSession.cs | 42 + .../Fakes/GraphSessionScope.cs | 56 + .../Fakes/MockHttpHandler.cs | 11 + .../Mgx.IntegrationTests.csproj | 3 +- .../Serialization/WireBodyTests.cs | 160 ++- .../TestSetup/MgxTransportScope.cs | 112 ++ .../TestSetup/RegressionCorpusIndexTests.cs | 141 +++ tests/RELEASING.md | 65 - tests/Unit/Mgx.Behaviour.Tests.ps1 | 18 +- tests/Unit/Mgx.ErrorContract.Tests.ps1 | 76 +- tests/Unit/Mgx.HelpFreshness.Tests.ps1 | 1 + tests/benchmarks/14-pacing-cold-cost.ps1 | 4 +- 87 files changed, 12748 insertions(+), 1438 deletions(-) create mode 100644 tests/CORPUS.md create mode 100644 tests/Ecosystem/Mgx.Ecosystem.Tests.ps1 delete mode 100644 tests/Live/README.md create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/Base/TransportSeamOrderTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/Base/TypeCacheInvalidationTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/BulkWriteErrorSurfacingTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/DeltaSharedCheckpointTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/DeltaUndeletableCheckpointTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/ErrorActionMatrixTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/ExportCorruptCheckpointTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/ExportDegradedQueryCheckpointTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/ExportSharedCheckpointTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/ExportWhatIfCheckpointTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/NamedTempPromotionOwnershipTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/OrphanTempOwnershipTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/ResilienceInjectionScenarioTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/StaleTempSweepOwnershipTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/StalledBodyTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Batch/ParallelChunkPartialResultTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Http/HeaderCultureTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Http/RateLimiterLifetimeTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Http/SendStopTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Http/TransportSwapTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Pagination/BoundarySizeTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Pagination/NamedTempPromotionTests.cs create mode 100644 tests/Mgx.IntegrationTests/Fakes/FakeGraphSession.cs create mode 100644 tests/Mgx.IntegrationTests/Fakes/GraphSessionScope.cs create mode 100644 tests/Mgx.IntegrationTests/TestSetup/MgxTransportScope.cs create mode 100644 tests/Mgx.IntegrationTests/TestSetup/RegressionCorpusIndexTests.cs delete mode 100644 tests/RELEASING.md diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index b758cdd..130a3ad 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -50,12 +50,15 @@ jobs: --results-directory TestResults # A run that executes nothing exits 0 and looks identical to a run that executes - # everything, so assert on the count as well as the exit code. Raise the floor as the - # suite grows. + # everything, so assert on the count as well as the exit code. The floor is the exact + # measured total - a handful of tests early-return on hosts that cannot exercise them, + # but every test executes and counts on every leg, so each leg reports the same number, + # and slack in the floor is just that many tests free to disappear unnoticed. Raise it + # with the suite. - name: Assert xUnit actually ran shell: pwsh run: | - $floor = 2700 + $floor = 2995 $trx = Get-ChildItem TestResults -Filter *.trx | Select-Object -First 1 if (-not $trx) { throw "No .trx produced - the test run did not report results." } $c = ([xml](Get-Content $trx.FullName -Raw)).TestRun.ResultSummary.Counters @@ -82,8 +85,10 @@ jobs: if ($result.FailedCount -gt 0) { throw "$($result.FailedCount) Pester test(s) failed." } - # Same reasoning as the xUnit floor above: silence is not success. - $floor = 235 + # Silence is not success here either. This floor keeps a small margin rather than + # sitting on the measured total, because the number it has to hold on is the one the + # compat leg below reports and that host is not this one. Raise both together. + $floor = 240 $ran = [int]$result.PassedCount + [int]$result.SkippedCount if ($ran -lt $floor) { throw "Only $ran Pester test(s) ran, expected at least $floor. The suite is not running." @@ -174,7 +179,9 @@ jobs: $r = Invoke-TestHarness -IgnoreCodeCoverage "$($r.PassedCount) passed, $($r.FailedCount) failed, $($r.SkippedCount) skipped" | Write-Host if ($r.FailedCount -gt 0) { throw "$($r.FailedCount) test(s) failed" } - $floor = 235 + # The same margin the main leg carries, for the same reason: this is the host the + # Pester total has to be true on, and nothing measures it outside this job. + $floor = 240 $ran = [int]$r.PassedCount + [int]$r.SkippedCount if ($ran -lt $floor) { throw "Only $ran test(s) ran on this host, expected at least $floor." @@ -184,3 +191,63 @@ jobs: $env:MGX_EXPECT_PSVERSION = '${{ matrix.psversion }}' pwsh -NoProfile -File ./compat-surface-tests.ps1 if ($LASTEXITCODE -ne 0) { throw "surface tests failed on PowerShell ${{ matrix.psversion }}" } + + # Module load and ALC resolution are irreversible in-process, so tests/Ecosystem spawns a + # child pwsh per load-order permutation. The neighbors are real Gallery installs because the + # failure being hunted is a binding conflict against the assemblies those modules actually + # ship - Az.Accounts carries its own System.Text.Json - which no mock can reproduce. + ecosystem: + name: Ecosystem isolation + runs-on: windows-latest + needs: build-and-test + if: github.repository == 'gromedev/mgx' + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup .NET + uses: actions/setup-dotnet@v5 + with: + dotnet-version: | + 8.0.x + 10.0.x + + - name: Restore + run: dotnet restore Mgx.slnx + + - name: Install platyPS + shell: pwsh + run: Install-PSResource -Name platyPS -TrustRepository -Scope AllUsers + + - name: Build module + shell: pwsh + run: ./build.ps1 -Configuration Release + + - name: Install ecosystem modules + shell: pwsh + run: | + # The matrix throws rather than skips when one is missing, so a failed install + # fails here, loudly, not inside a permutation. + Install-PSResource -Name Az.Accounts, PnP.PowerShell, ExchangeOnlineManagement -TrustRepository -Scope AllUsers + + - name: Run ecosystem matrix + shell: pwsh + run: | + Import-Module ./tests/TestHarness.psm1 + $r = Invoke-TestHarness -TestPath ./tests/Ecosystem -TestResultsFile TestResults/ecosystem-results.xml + if ($r.FailedCount -gt 0) { throw "$($r.FailedCount) ecosystem permutation(s) failed." } + + # Nine permutations, one test each. The floor is exact: deleting a permutation + # must drop the count below it rather than pass as a smaller green run. + $floor = 9 + if ($r.PassedCount -lt $floor) { + throw "Only $($r.PassedCount) ecosystem permutation(s) ran, expected at least $floor. The matrix is not running." + } + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: ecosystem-results + path: TestResults/ecosystem-results.xml + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index 7609840..f8d8919 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ # tmp bin/ obj/ +tmp/ publish/ *.jsonl *.json diff --git a/CHANGELOG.md b/CHANGELOG.md index a832925..dfd44d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,55 @@ # Changelog +## 2.1.4 + +# Changelog + +### Fixed + +- `Export-MgxCollection` starts a fresh export when its resume checkpoint is unreadable rather than appending duplicate collection data. +- Resume checkpoints belonging to different exports, syncs, or modes are rejected without altering existing operation files. +- Checkpoints from earlier releases resume only when confirmed by corresponding files on disk. +- Endpoints rejecting `$top` parameters no longer trigger checkpoint refusals or appending to older files. +- `Sync-MgxDelta` and `Export-MgxCollection` preserve resume eligibility when `-Uri` or `-OutputFile` casing differs from interrupted runs. +- Orphaned temp adoption and stale temp sweeps restrict file matching to operations written by the export itself. +- Custom Graph endpoints no longer cause valid resume checkpoints to misclassify as external exports. +- Running `-WhatIf` on an interrupted export previews recovery actions without modifying state. +- Batch operations preserve applied server responses even if subsequent chunk retries fail. +- Batch processing halts immediately upon chunk refusal, canceling pending waits promptly. +- Batch summaries report never-sent items accurately rather than counting them as successful writes. +- Batch chunks refused with redirect or not-modified responses record as failures and write to the dead-letter file. +- Dead-letter files record only failed and never-sent operations, excluding applied server writes. +- Concurrent batch chunk failures report both failure events in output logs. +- Malformed batch responses preserve and report received results alongside the error payload. +- UTF-8 byte-order marks no longer discard fan-out entities, bulk-write results, or expanded relations. +- Stalled mid-stream response bodies fail at the body-read timeout threshold instead of hanging the pipeline. +- Stalled error body responses format consistently across all error handling paths. +- Unsupported response charsets (such as UTF-7 and quoted variants) fall back to UTF-8 decoding without terminating the pipeline. +- Non-string `-Headers` names and values render culture-invariantly to prevent sending comma-decimal formatted numbers in European locales. +- `Get-MgxTelemetry` outputs `PacingState` latency ratios culture-invariantly. +- Flags enum values in `-Body` serialize combination members using standalone item names, including attribute-defined aliases. +- Mid-enumeration transport swaps (`Connect-MgGraph`, `Set-MgxOption`, `Enable-MgxResilience`) maintain active paging without interrupting execution. +- Removing the module restores native SDK HTTP clients, and re-enabling resilience after re-importing avoids stacking redundant wrappers. +- `Disable-MgxResilience` purges untracked wrappers and reports installation status accurately. +- Telemetry tracks request counts uniquely across enable, remove, and re-import lifecycle events. +- Cancellation requests landing between retries prevent subsequent attempts from transmitting. +- Completed exports and sync operations purge partial files leftover from abandoned attempts. +- Resume logic ignores temp files currently being written by active operations. +- Un-deletable abandoned checkpoints are skipped to enforce full re-fetching of items. +- Expired delta tokens prevent stale checkpoints from overriding active operation writes. +- Refused checkpoints from earlier execution passes are identified cleanly without misattributing to external sync operations. + +### Added + +- Added ecosystem isolation matrix tests verifying nine load-order permutations across `Az.Accounts`, `PnP.PowerShell`, and `ExchangeOnlineManagement` in CI. +- Added regression corpus index (`tests/CORPUS.md`) mapping covered issue IDs to test classes with validation guards. +- Added error-action test coverage verifying termination behavior, `$Error` populating, `-ErrorVariable` handling, and error-stream counts across all failure contexts. + +### Documentation + +- Updated batch help documentation with answered, refused, and never-sent classifications alongside dead-letter replay semantics. +- Updated `-Body` help documentation detailing flags enum combination naming and recommending literal strings for leading-acronym members (e.g., iOS). + ## 2.1.3 ### Added diff --git a/Directory.Build.props b/Directory.Build.props index 7d5f301..b8d8598 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,7 +1,7 @@ - 2.1.3 + 2.1.4 Thomas Maillo Grome Mgx (c) 2026 Thomas Maillo Grome. All rights reserved. diff --git a/README.md b/README.md index 538b23a..d46afce 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ # Mgx -High-performance, resilient Microsoft Graph access for PowerShell. +High-performance, resilient Microsoft Graph access for PowerShell. + +```powershell +Install-Module Mgx +``` `Mgx` is a PowerShell client for Microsoft Graph built for long-running and high-volume workloads. It handles the parts that become difficult when a script makes thousands of requests: pagination, concurrency, throttling, retries, dead connections, memory usage, and recovery from interruption. diff --git a/module/en-US/Mgx.Cmdlets.dll-Help.xml b/module/en-US/Mgx.Cmdlets.dll-Help.xml index df2afa5..725ab84 100644 --- a/module/en-US/Mgx.Cmdlets.dll-Help.xml +++ b/module/en-US/Mgx.Cmdlets.dll-Help.xml @@ -1986,7 +1986,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP DeadLetterPath - Path to a JSONL file where failed batch items (status >= 400) are appended, along with any that were never sent because an earlier chunk failed. Each line contains Url, Method, Body (with sensitive fields redacted), Status, Error, and Timestamp. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Read it to decide what to resubmit, and supply the sensitive fields again yourself. + Path to a JSONL file where the batch's failed and never-sent items are appended: anything the server answered with a status >= 400, the items of a chunk whose own POST was refused, which carry that refusal's status, and the items of chunks that were never POSTed, which carry status 0. Nothing the server confirmed is ever written, so the file holds only work that is still outstanding. Each line contains Url, Method, Body (with sensitive fields redacted), Status, Error, and Timestamp. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Read it to decide what to resubmit, and supply the sensitive fields again yourself. A refused item's request did reach the server, so it may have been applied without an answer coming back; resending one is a decision about duplicates rather than a free retry. String @@ -2109,7 +2109,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP DeadLetterPath - Path to a JSONL file where failed batch items (status >= 400) are appended, along with any that were never sent because an earlier chunk failed. Each line contains Url, Method, Body (with sensitive fields redacted), Status, Error, and Timestamp. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Read it to decide what to resubmit, and supply the sensitive fields again yourself. + Path to a JSONL file where the batch's failed and never-sent items are appended: anything the server answered with a status >= 400, the items of a chunk whose own POST was refused, which carry that refusal's status, and the items of chunks that were never POSTed, which carry status 0. Nothing the server confirmed is ever written, so the file holds only work that is still outstanding. Each line contains Url, Method, Body (with sensitive fields redacted), Status, Error, and Timestamp. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Read it to decide what to resubmit, and supply the sensitive fields again yourself. A refused item's request did reach the server, so it may have been applied without an answer coming back; resending one is a decision about duplicates rather than a free retry. String @@ -2232,13 +2232,13 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP System.Collections.Hashtable - Per-request results with Url, Method, Status, and Body keys, plus NotSent on any operation that was never sent because an earlier chunk failed. Body here is the RESPONSE body - for a failure that is the error envelope, not the request - so piping results straight back resubmits the wrong thing. Use Url and Method to rebuild the requests you want to retry. + Per-request results with Url, Method, Status, and Body keys. Status says what became of the operation: an item the server answered keeps the status it answered with; an item whose chunk was POSTed and then refused carries the refusal's status (>= 400), because its request went out and may have been applied; only an item in a chunk that was never POSTed gets status 0, and only those carry NotSent - another chunk of the batch failed, not necessarily one before it, since chunks can run in parallel. Body here is the RESPONSE body - for a failure that is the error envelope, not the request - so piping results straight back resubmits the wrong thing. Use Url and Method to rebuild the requests you want to retry. - Each batch item is retried individually on 429 (throttled) or 5xx errors (for idempotent methods). POST requests only retry on 429 because POST is non-idempotent - retrying a failed POST on 5xx could create duplicates if the server processed the request before the error. This matches the Kiota SDK retry behavior. Source: Microsoft Graph error responses and resource types (https://learn.microsoft.com/en-us/graph/errors)Items that exhaust per-chunk retries get one additional batch-level retry pass. + Each batch item is retried individually on 429 (throttled) or 5xx errors (for idempotent methods). POST requests only retry on 429 because POST is non-idempotent - retrying a failed POST on 5xx could create duplicates if the server processed the request before the error. This matches the Kiota SDK retry behavior. Source: Microsoft Graph error responses and resource types (https://learn.microsoft.com/en-us/graph/errors)Items that exhaust per-chunk retries get one additional batch-level retry pass, except after a chunk failure: the run has stopped sending, and that pass is a send. It is skipped for every item it would have picked up, including items in chunks that were POSTed and answered in full, and they are handed back with the status the server gave them rather than resent. Use -Verbose to see retry counts, throttle encounters, and timing. Batching reduces HTTP round-trips (20 operations per request instead of 1), but Graph counts each item inside a batch individually against the server-side write quota (3,000 writes / 2.5 min per app+tenant). Sustained write throughput caps at ~20/sec regardless of batching. See Set-MgxOption (Set-MgxOption.md) for the full throttle limits table and tuning guidance. Source: [Microsoft Graph service-specific throttling limits](https://learn.microsoft.com/en-us/graph/throttling-limits)For best performance and stability, always pipe all items into a single Invoke-MgxBatchRequest call rather than calling it in a loop. At 200+ rapid invocations per second, PowerShell's internal pipeline thread safety can race between cmdlet instances. Piping all items into one call avoids this entirely and is significantly faster. @@ -2415,7 +2415,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Body Request body for write operations (POST, PATCH, PUT). Accepts a hashtable, PSObject or array, which is serialized to JSON, or a string already holding JSON, which is sent verbatim. Ignored (with a warning) on GET. - Enums serialize as camelCase names, byte arrays as base64, TimeSpan as an ISO-8601 duration, and a DateTime without a kind is treated as UTC. A SecureString, PSCredential, script block, certificate, NaN or Infinity anywhere in the body is refused before anything is sent, with an error naming the property. A string that is not JSON is sent verbatim; add a Content-Type to -Headers to declare its real type. + Enums serialize as camelCase names, byte arrays as base64, TimeSpan as an ISO-8601 duration, and a DateTime without a kind is treated as UTC. A [Flags] combination goes out in the comma-joined form OData expects, with no space: `ignoreCase,multiline`. Each member of a combination is named as it would be alone, so a name set by JsonStringEnumMemberName, or the choice between two members sharing a value, reads the same either way. camelCase lowercases a leading acronym whole, so a member named `IOS` is sent as `ios`; where Graph wants `iOS`, pass the string rather than the enum. A SecureString, PSCredential, script block, certificate, NaN or Infinity anywhere in the body is refused before anything is sent, with an error naming the property. A string that is not JSON is sent verbatim; add a Content-Type to -Headers to declare its real type. Object @@ -2720,7 +2720,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Body Request body for write operations (POST, PATCH, PUT). Accepts a hashtable, PSObject or array, which is serialized to JSON, or a string already holding JSON, which is sent verbatim. Ignored (with a warning) on GET. - Enums serialize as camelCase names, byte arrays as base64, TimeSpan as an ISO-8601 duration, and a DateTime without a kind is treated as UTC. A SecureString, PSCredential, script block, certificate, NaN or Infinity anywhere in the body is refused before anything is sent, with an error naming the property. A string that is not JSON is sent verbatim; add a Content-Type to -Headers to declare its real type. + Enums serialize as camelCase names, byte arrays as base64, TimeSpan as an ISO-8601 duration, and a DateTime without a kind is treated as UTC. A [Flags] combination goes out in the comma-joined form OData expects, with no space: `ignoreCase,multiline`. Each member of a combination is named as it would be alone, so a name set by JsonStringEnumMemberName, or the choice between two members sharing a value, reads the same either way. camelCase lowercases a leading acronym whole, so a member named `IOS` is sent as `ios`; where Graph wants `iOS`, pass the string rather than the enum. A SecureString, PSCredential, script block, certificate, NaN or Infinity anywhere in the body is refused before anything is sent, with an error naming the property. A string that is not JSON is sent verbatim; add a Content-Type to -Headers to declare its real type. Object diff --git a/module/en-US/about_Mgx_Errors.help.txt b/module/en-US/about_Mgx_Errors.help.txt index 84936aa..60040da 100644 --- a/module/en-US/about_Mgx_Errors.help.txt +++ b/module/en-US/about_Mgx_Errors.help.txt @@ -42,9 +42,10 @@ NON-TERMINATING ERRORS CircuitBroken The circuit breaker is open; the record wraps the guidance text BatchItemError A batch item answered >= 400 - BatchItemNotSent A batch item skipped because an earlier chunk - failed - BatchChunkFailed The chunk POST itself failed + BatchItemNotSent A batch item never sent because + another chunk of the batch failed + BatchChunkFailed A chunk's own $batch call failed, or + answered nothing that was sent FanOutError A per-id GET failed during fan-out BulkWriteError A per-id write failed during fan-out BulkWriteInfraError A fan-out write died before an HTTP status @@ -64,8 +65,14 @@ NON-TERMINATING ERRORS Stop turns the first error record into a terminating error, ending the pipeline. For batches this stops at the first failed item - after the dead-letter file is written, so -DeadLetterPath always captures every - failure even under Stop. SilentlyContinue suppresses and continues; - records still land in $Error and -ErrorVariable. + failure even under Stop. Continue writes the record and carries on. + SilentlyContinue suppresses the display and continues; records still land + in $Error and -ErrorVariable. + + Ignore discards the record: nothing is displayed, nothing reaches $Error, + and -ErrorVariable stays empty. It is a common parameter only - + $ErrorActionPreference cannot be set to Ignore - so it applies to the one + invocation that carries it. -ERRORVARIABLE diff --git a/module/help/Invoke-MgxBatchRequest.md b/module/help/Invoke-MgxBatchRequest.md index b1bde9b..ba28885 100644 --- a/module/help/Invoke-MgxBatchRequest.md +++ b/module/help/Invoke-MgxBatchRequest.md @@ -162,7 +162,7 @@ Accept wildcard characters: False ``` ### -DeadLetterPath -Path to a JSONL file where failed batch items (status >= 400) are appended, along with any that were never sent because an earlier chunk failed. Each line contains Url, Method, Body (with sensitive fields redacted), Status, Error, and Timestamp. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Read it to decide what to resubmit, and supply the sensitive fields again yourself. +Path to a JSONL file where the batch's failed and never-sent items are appended: anything the server answered with a status >= 400, the items of a chunk whose own POST was refused, which carry that refusal's status, and the items of chunks that were never POSTed, which carry status 0. Nothing the server confirmed is ever written, so the file holds only work that is still outstanding. Each line contains Url, Method, Body (with sensitive fields redacted), Status, Error, and Timestamp. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Read it to decide what to resubmit, and supply the sensitive fields again yourself. A refused item's request did reach the server, so it may have been applied without an answer coming back; resending one is a decision about duplicates rather than a free retry. ```yaml Type: String @@ -311,12 +311,12 @@ String URLs, or hashtables or PSCustomObjects with Url, Method, and Body members ## OUTPUTS ### System.Collections.Hashtable -Per-request results with Url, Method, Status, and Body keys, plus NotSent on any operation that was never sent because an earlier chunk failed. Body here is the RESPONSE body - for a failure that is the error envelope, not the request - so piping results straight back resubmits the wrong thing. Use Url and Method to rebuild the requests you want to retry. +Per-request results with Url, Method, Status, and Body keys. Status says what became of the operation: an item the server answered keeps the status it answered with; an item whose chunk was POSTed and then refused carries the refusal's status (>= 400), because its request went out and may have been applied; only an item in a chunk that was never POSTed gets status 0, and only those carry NotSent - another chunk of the batch failed, not necessarily one before it, since chunks can run in parallel. Body here is the RESPONSE body - for a failure that is the error envelope, not the request - so piping results straight back resubmits the wrong thing. Use Url and Method to rebuild the requests you want to retry. ## NOTES Each batch item is retried individually on 429 (throttled) or 5xx errors (for idempotent methods). POST requests only retry on 429 because POST is non-idempotent - retrying a failed POST on 5xx could create duplicates if the server processed the request before the error. This matches the Kiota SDK retry behavior. Source: [Microsoft Graph error responses and resource types](https://learn.microsoft.com/en-us/graph/errors) -Items that exhaust per-chunk retries get one additional batch-level retry pass. +Items that exhaust per-chunk retries get one additional batch-level retry pass, except after a chunk failure: the run has stopped sending, and that pass is a send. It is skipped for every item it would have picked up, including items in chunks that were POSTed and answered in full, and they are handed back with the status the server gave them rather than resent. Use -Verbose to see retry counts, throttle encounters, and timing. diff --git a/module/help/Invoke-MgxRequest.md b/module/help/Invoke-MgxRequest.md index ada75cd..37b75bd 100644 --- a/module/help/Invoke-MgxRequest.md +++ b/module/help/Invoke-MgxRequest.md @@ -183,7 +183,7 @@ Accept wildcard characters: False ### -Body Request body for write operations (POST, PATCH, PUT). Accepts a hashtable, PSObject or array, which is serialized to JSON, or a string already holding JSON, which is sent verbatim. Ignored (with a warning) on GET. -Enums serialize as camelCase names, byte arrays as base64, TimeSpan as an ISO-8601 duration, and a DateTime without a kind is treated as UTC. A SecureString, PSCredential, script block, certificate, NaN or Infinity anywhere in the body is refused before anything is sent, with an error naming the property. A string that is not JSON is sent verbatim; add a Content-Type to -Headers to declare its real type. +Enums serialize as camelCase names, byte arrays as base64, TimeSpan as an ISO-8601 duration, and a DateTime without a kind is treated as UTC. A [Flags] combination goes out in the comma-joined form OData expects, with no space: `ignoreCase,multiline`. Each member of a combination is named as it would be alone, so a name set by JsonStringEnumMemberName, or the choice between two members sharing a value, reads the same either way. camelCase lowercases a leading acronym whole, so a member named `IOS` is sent as `ios`; where Graph wants `iOS`, pass the string rather than the enum. A SecureString, PSCredential, script block, certificate, NaN or Infinity anywhere in the body is refused before anything is sent, with an error naming the property. A string that is not JSON is sent verbatim; add a Content-Type to -Headers to declare its real type. ```yaml Type: Object diff --git a/module/mgx.psd1 b/module/mgx.psd1 index 90e044d..1235670 100644 --- a/module/mgx.psd1 +++ b/module/mgx.psd1 @@ -1,6 +1,6 @@ @{ RootModule = 'mgx.psm1' - ModuleVersion = '2.1.3' + ModuleVersion = '2.1.4' GUID = 'a3f7e8d2-5b4c-4a1e-9f6d-2c8b0e3a7d5f' Author = 'Thomas Maillo Grome' CompanyName = 'Mgx' @@ -52,33 +52,20 @@ LicenseUri = 'https://github.com/gromedev/mgx/blob/main/LICENSE' ProjectUri = 'https://github.com/gromedev/mgx' ReleaseNotes = @' -v2.1.3 -Added -- about_Mgx_Errors documentation: termination behavior, error records, -ErrorAction/-ErrorVariable handling. -- Live test suite supports client secret authentication alongside certificates. +v2.1.4 Fixed -- SecureString, credential objects, and script blocks in -Body throw explicit errors rather than serializing as reflection noise. -- Server errors, timeouts, and precondition failures assign specific ErrorCategory values instead of NotSpecified. -- Get-MgxContent forwards the conditional header family (including If-None-Match) to content download hosts. -- Failed batch chunks under -ErrorAction Stop write dead-letter files prior to terminating. -- Typed parameters deferring to options already present in -Uri emit warnings rather than dropping silently. -- Request bodies over 4 MB pass directly to the SDK chain under Enable-MgxResilience instead of being rejected. -- HTTP 304 on conditional downloads completes cleanly without an error record. -- Circular self-referencing objects in -Body raise validation errors instead of a process stack overflow. -Changed -- Unified failure classification across retries, circuit breaking, batch retry checks, content downloads, and adaptive pacing. +- Interrupted exports resume safely: unreadable, foreign, legacy, and case-differing checkpoints are handled without data loss or duplication. +- Batch items the server answered keep their results through chunk failures; the dead-letter file never contains an applied write. +- Batch summaries count only sent operations, and any chunk refusal reads as a failure. +- A stalled response or error body fails at the body-read timeout in the error contract's terms. +- A UTF-8 byte-order mark no longer discards fan-out, bulk-write, or expanded-relation results. +- Headers, telemetry ratios, and flags enum combinations render culture-invariantly with their attribute-supplied names. +- Removing the module restores the SDK's own client; re-enabling resilience never stacks wrappers. +- A resuming run never takes a file a live run is writing, and never resumes a checkpoint it gave up on. +- A caller's stop keeps the next retry off the wire; completed runs leave no abandoned partial files. +Added +- Ecosystem isolation matrix over Az.Accounts, PnP.PowerShell, and ExchangeOnlineManagement, with its own CI job. See CHANGELOG.md for the full list. - -v2.1.2 -Fixed -- URI handling: a '#' in a path dropped everything after it, pre-encoded query values were encoded twice, piped drive and item ids went unescaped, an absolute -Uri was mangled rather than refused, and a typed parameter could repeat an option already in -Uri. -- Body serialization: enums went out as numbers, byte arrays as integer lists, TimeSpan and a Kind-less DateTime in forms Graph refuses, and a PSCustomObject body lost every property that was not a NoteProperty. -- Headers: content headers from -Headers were dropped, arrays were sent as System.String[], names merged case-sensitively, and a caller's client-request-id was doubled. -- Failed batch items wrote no error records, so -ErrorAction Stop did not stop. -- An empty, non-JSON or malformed response body ended the cmdlet with an unhandled error. -- An unexpected failure discarded the retry history explaining it, and a timed-out attempt could retry after Ctrl-C. - -See CHANGELOG.md for the full history. '@ } } diff --git a/src/Mgx.Cmdlets/AlcInitializer.cs b/src/Mgx.Cmdlets/AlcInitializer.cs index ba5d259..8760648 100644 --- a/src/Mgx.Cmdlets/AlcInitializer.cs +++ b/src/Mgx.Cmdlets/AlcInitializer.cs @@ -20,6 +20,12 @@ public class AlcInitializer : IModuleAssemblyInitializer, IModuleAssemblyCleanup public void OnImport() { AssemblyLoadContext.Default.Resolving += ResolveDependency; + + // Re-arms type-cache invalidation. The hook is attached from a static constructor that + // has long since run by the time a second import happens, and removal detaches it, so + // without this every FindType after the first removal would answer from entries resolved + // before it - including a GraphSession belonging to a module that has been replaced. + Base.MgxCmdletBase.AttachAssemblyLoadHandler(); } private static Assembly? ResolveDependency(AssemblyLoadContext defaultAlc, AssemblyName name) @@ -80,7 +86,7 @@ public void OnRemove(PSModuleInfo module) // pieces of static state are released by this single call. try { - Base.MgxCmdletBase.ResetHttpClient(); + ReleaseStaticState(); } catch (Exception ex) { @@ -96,4 +102,32 @@ public void OnRemove(PSModuleInfo module) // that the type cache should still observe). Base.MgxCmdletBase.DetachAssemblyLoadHandler(); } + + /// + /// Everything module removal releases except the assembly-load hook, which OnRemove + /// detaches itself. Two reasons, neither of them that the detach cannot be undone - it can, + /// AttachAssemblyLoadHandler subscribes again and an import calls it. It has to run after + /// ResetHttpClient, whose loads the type cache should still observe; and this is the seam + /// tests drive a removal through, which is not the same as asking them to unsubscribe the + /// process's only invalidation hook. + /// + /// ResetHttpClient releases mgx's own client and the pipeline factory. The resilience + /// injection is separate state: it is installed on GraphSession, which belongs to another + /// module and outlives this one, so releasing it means putting the session back on the + /// genuine SDK client and only then letting go. Dropping the references alone would leave + /// the wrapper installed and the SDK sending through a handler belonging to an unloaded + /// module; the bridge-target map survives removal, so a later re-import's + /// Disable-MgxResilience can still take it off - the restore here just makes that + /// recovery unnecessary on the normal path. + /// + /// + internal static void ReleaseStaticState() + { + Base.MgxCmdletBase.ResetHttpClient(); + Cmdlets.Configuration.EnableMgxResilience.ReleaseInjection(); + + // Last: both calls above resolve types through this cache, so clearing it earlier would + // only refill it with the entries the removal exists to drop. + Base.MgxCmdletBase.ClearTypeCache(); + } } diff --git a/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs b/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs index bcaf1c1..e56c9e2 100644 --- a/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs +++ b/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs @@ -6,6 +6,7 @@ using System.Reflection; using System.Security.Cryptography; using System.Text; +using System.Text.Json; using System.Text.RegularExpressions; using Mgx.Engine.Http; using Mgx.Engine.Models; @@ -23,9 +24,27 @@ public abstract class MgxCmdletBase : MgxCmdletCore { private ResilientGraphClient? _client; + // Which test-transport generation _client was built for. Only consulted while the seam + // below is armed; -1 means "built outside a scope", which never matches a live epoch. + private int _clientEpoch = -1; + private static readonly object s_initLock = new(); + + /// + /// The transport every ResilientGraphClient is built on. A replaced one is dropped, never + /// closed: each client captured it as a readonly field in its constructor and goes on + /// sending through it, one page at a time, for as long as its enumeration runs. It used to + /// be disposed TotalTimeoutSeconds after a swap - a per-REQUEST timeout used as a resource + /// lifetime, the same construction the shared rate limiter dropped for the reason recorded + /// in ResiliencePipelineFactory.GetOrCreate. No such bound is correct: an export may hold + /// its transport for hours, and a Connect-MgGraph to another identity, a Set-MgxOption, an + /// Enable-MgxResilience or a Remove-Module on another thread killed it underneath, so the + /// next page threw ObjectDisposedException mid-stream. Letting go is enough: the handler's + /// sockets are closed by its own idle and lifetime timeouts once nothing is sending + /// through them. + /// private static HttpClient? s_graphHttpClient; - private static bool s_ownsHttpClient; // false when using SDK fallback (don't dispose SDK's client) + private static bool s_ownsHttpClient; // false when using SDK fallback (the SDK's own client) // Identity the cached client was built for private static volatile string? s_cachedAuthFingerprint; @@ -40,6 +59,20 @@ public abstract class MgxCmdletBase : MgxCmdletCore internal static volatile string s_graphEndpoint = "https://graph.microsoft.com"; internal static volatile ResilientGraphClientOptions s_clientOptions = ResilientGraphClientOptions.Default; + // Test transport seam. The xUnit assembly arms these three through MgxTransportScope, and + // every shipping path leaves s_testTransport null. InternalsVisibleTo names that assembly + // and neither assembly is signed, so the grant is by simple name: it keeps the seam out of + // the public surface and out of the way, not out of reach of code that means to find it. + internal static volatile HttpClient? s_testTransport; + + // Whether an armed test transport presents as mgx-owned. Get-MgxContent fails closed over a + // transport it does not own, so a content test arms this and the rest of the suite does not. + internal static volatile bool s_testTransportOwned; + + // Bumped whenever the armed transport changes. A cmdlet instance that cached a client under + // an earlier scope rebuilds instead of answering from a transport the scope has retired. + internal static int s_testTransportEpoch; + /// /// Base URL for Graph API requests (e.g., "https://graph.microsoft.com/v1.0"). /// Respects sovereign clouds via GraphSession environment. @@ -54,23 +87,38 @@ public abstract class MgxCmdletBase : MgxCmdletCore /// protected ResilientGraphClient GetClient() { - if (_client != null) return _client; + // The seam is read before the auth probe below on purpose: a test process has no Graph + // session, so reaching GetCurrentAuthIdentity would take the IsGraphAuthLoaded branch + // and terminate instead of running the cmdlet under the mock wire. + // + // The epoch, then the transport, then the epoch again. A scope arms the transport before + // it bumps the epoch, so a pair read this way either spans no bump - and belongs together + // - or is read again. Taking the transport first can pair one a scope has already retired + // with the epoch that retired it, and this instance then answers from that transport for + // as long as it lives. + int epoch; + HttpClient? testTransport; + do + { + epoch = Volatile.Read(ref s_testTransportEpoch); + testTransport = s_testTransport; + } + while (epoch != Volatile.Read(ref s_testTransportEpoch)); + + // The epoch gates the cached client on both paths, not just the seam's. It never moves + // outside a test process, so this is the plain cache hit in every shipping path - but a + // cmdlet instance that outlives the scope which armed it would otherwise keep answering + // from a transport that scope has already disposed. + if (_client != null && _clientEpoch == epoch) return _client; + _clientEpoch = epoch; + + if (testTransport != null) + return ConfigureClient(testTransport, s_clientOptions); var identity = GetCurrentAuthIdentity(WriteVerbose); if (string.IsNullOrEmpty(identity.Fingerprint)) { - // Microsoft.Graph.Authentication is a soft dependency (see mgx.psd1), so an empty - // fingerprint has two distinct causes that need different advice. While it sat in - // RequiredModules the SDK was guaranteed present and "run Connect-MgGraph" was - // always the right answer; without it, that message sends someone to a cmdlet - // that does not exist in their session. - var (message, errorId) = IsGraphAuthLoaded() - ? ("Not connected to Microsoft Graph. Run Connect-MgGraph first.", - "NotConnected") - : ("Microsoft.Graph.Authentication is not loaded. Install it " - + "(Install-PSResource -Name Microsoft.Graph.Authentication) and run " - + "Connect-MgGraph, or supply your own transport via Enable-MgxResilience.", - "GraphAuthModuleNotLoaded"); + var (message, errorId) = DescribeMissingConnection(IsGraphAuthLoaded()); ThrowTerminatingError(new ErrorRecord( new InvalidOperationException(message), @@ -129,9 +177,7 @@ protected ResilientGraphClient GetClient() // Only on a real credential change: a same-identity reconnect should keep its // warm rate limiter instead of earning a fresh burst allowance. if (credentialChanged) ResiliencePipelineFactory.Reset(); - // Schedule delayed disposal: in-flight ResilientGraphClient instances - // may still hold a reference to the old client via their constructor - ScheduleDelayedHttpClientDispose(s_graphHttpClient, s_ownsHttpClient); + // The old client is let go, not closed - see the note on s_graphHttpClient. s_graphHttpClient = BuildCleanHttpClient(clientOptions.TotalTimeoutSeconds); if (s_graphHttpClient != null) { @@ -171,6 +217,16 @@ protected ResilientGraphClient GetClient() if (identityChanged) Cmdlets.Configuration.EnableMgxResilience.RefreshInjectedClient(WriteWarning, WriteVerbose); + return ConfigureClient(httpClient, clientOptions); + } + + /// + /// Wraps a transport in the ResilientGraphClient this cmdlet invocation uses and caches it. + /// Split out of GetClient so the transport seam above reaches the same configuration the + /// auth path produces, rather than a second copy of it that can drift. + /// + private ResilientGraphClient ConfigureClient(HttpClient httpClient, ResilientGraphClientOptions clientOptions) + { _client = new ResilientGraphClient(httpClient, clientOptions); _client.BodyReadTimeout = TimeSpan.FromSeconds(clientOptions.AttemptTimeoutSeconds); _client.VerboseWriter = msg => WriteVerbose(msg); @@ -304,6 +360,22 @@ private static string Shorten(string? fingerprint) => BindingFlags.Public | BindingFlags.Static)?.GetValue(null); } + /// + /// Why no token could be obtained, given whether Microsoft.Graph.Authentication is present. + /// It is a soft dependency (see mgx.psd1), so an empty fingerprint has two distinct causes + /// needing different advice: while it sat in RequiredModules the SDK was guaranteed present + /// and "run Connect-MgGraph" was always the right answer, and without it that message sends + /// someone to a cmdlet that does not exist in their session. + /// + internal static (string Message, string ErrorId) DescribeMissingConnection(bool graphAuthLoaded) => + graphAuthLoaded + ? ("Not connected to Microsoft Graph. Run Connect-MgGraph first.", + "NotConnected") + : ("Microsoft.Graph.Authentication is not loaded. Install it " + + "(Install-PSResource -Name Microsoft.Graph.Authentication) and run " + + "Connect-MgGraph, or supply your own transport via Enable-MgxResilience.", + "GraphAuthModuleNotLoaded"); + /// /// Whether Microsoft.Graph.Authentication is present in the session. Because mgx does not /// declare it in RequiredModules, "absent" is a real state that has to be told apart from @@ -336,14 +408,14 @@ internal static bool IsGraphAuthLoaded() /// /// The HttpClient the Microsoft.Graph SDK is currently using, or null when it is not - /// initialized. Used to detect that a borrowed SDK client has been replaced underneath us. + /// initialized. Used to detect that a borrowed SDK client has been replaced underneath us, + /// and to borrow from when mgx cannot build a client of its own. /// internal static HttpClient? TryGetSessionGraphHttpClient() { try { - var instance = TryGetGraphSessionInstance(); - return instance?.GetType().GetProperty("GraphHttpClient")?.GetValue(instance) as HttpClient; + return GenuineSessionClient(TryGetGraphSessionInstance()); } catch { @@ -351,6 +423,24 @@ internal static bool IsGraphAuthLoaded() } } + /// + /// GraphSession's client with any wrapper mgx installed there resolved away. + /// Enable-MgxResilience leaves a client of ours on that property, and it already carries a + /// resilience pipeline: borrowing it puts mgx's own pipeline on top of that one, and each + /// layer retries the one beneath it. The configured retry budget then multiplies on the + /// wire, telemetry counts each attempt once per layer, and the pacer halves per layer. + /// + /// The comparison that decides a borrowed client has gone stale reads through here too. It + /// is against the client mgx borrowed, so both sides have to be resolved the same way or a + /// session holding a wrapper reads as a replacement on every invocation. + /// + /// + private static HttpClient? GenuineSessionClient(object? graphSessionInstance) => + graphSessionInstance?.GetType().GetProperty("GraphHttpClient") + ?.GetValue(graphSessionInstance) is HttpClient client + ? Cmdlets.Configuration.EnableMgxResilience.ResolveGenuineSdkClient(client) + : null; + private static string? GetGraphEndpointFrom(object instance) { var env = instance.GetType().GetProperty("Environment")?.GetValue(instance); @@ -529,7 +619,7 @@ internal static void TryPreInitHttpClient(Action warn, Action ve if (client == null) return; // BuildCleanHttpClient already warned with ex.Message ResiliencePipelineFactory.Reset(); - ScheduleDelayedHttpClientDispose(s_graphHttpClient, s_ownsHttpClient); + // The old client is let go, not closed - see the note on s_graphHttpClient. s_graphHttpClient = client; s_ownsHttpClient = true; s_cachedAuthFingerprint = identity.Fingerprint; @@ -557,8 +647,8 @@ private static bool ClientIsStale(AuthIdentity identity, ResilientGraphClientOpt var instance = TryGetGraphSessionInstance(); if (instance == null) return null; - var httpClient = instance.GetType().GetProperty("GraphHttpClient") - ?.GetValue(instance) as HttpClient; + // Resolved, never taken verbatim - see GenuineSessionClient. + var httpClient = GenuineSessionClient(instance); if (httpClient != null) return httpClient; // Use detected endpoint instead of hardcoded graph.microsoft.com @@ -583,8 +673,7 @@ private static bool ClientIsStale(AuthIdentity identity, ResilientGraphClientOpt // Restore AzureADEndpoint on the NEW Environment object the probe created. RestoreAzureADEndpoint(instance, savedAadEndpoint, WriteVerbose); - return instance.GetType().GetProperty("GraphHttpClient") - ?.GetValue(instance) as HttpClient; + return GenuineSessionClient(instance); } catch (Exception ex) { @@ -618,11 +707,17 @@ internal static void RestoreAzureADEndpoint(object graphSessionInstance, string? // ConcurrentDictionary is safe for concurrent runspaces. // Only non-null results are cached: assemblies load lazily in PowerShell, // so a miss now may succeed after the user imports additional modules. - private static readonly ConcurrentDictionary s_typeCache = new(); + // Internal on the same terms as the transport seam above: the suite asserts on it. + internal static readonly ConcurrentDictionary s_typeCache = new(); + + // Whether OnAssemblyLoad is subscribed. A process can import the module more than once, and + // += is not idempotent: a second subscription would outlive the -= that removal does, and + // clear the cache once per subscription on every load thereafter. + private static int s_assemblyLoadHooked; static MgxCmdletBase() { - AppDomain.CurrentDomain.AssemblyLoad += OnAssemblyLoad; + AttachAssemblyLoadHandler(); } // A cached Type carries the identity of the assembly that defined it. Re-importing @@ -633,12 +728,34 @@ static MgxCmdletBase() // always tiny dictionary is cheaper than validating entries on the hot path. private static void OnAssemblyLoad(object? sender, AssemblyLoadEventArgs args) => s_typeCache.Clear(); + /// + /// Subscribes the assembly-load hook, once. Module import calls this: the static constructor + /// runs the first time anything touches this type and never again, so after a removal has + /// detached the hook, nothing would re-arm invalidation for the rest of the process. + /// + internal static void AttachAssemblyLoadHandler() + { + if (Interlocked.Exchange(ref s_assemblyLoadHooked, 1) == 1) return; + AppDomain.CurrentDomain.AssemblyLoad += OnAssemblyLoad; + } + /// /// Detaches the assembly-load hook. Called on module removal so the handler does not /// root this type after the module is gone. /// - internal static void DetachAssemblyLoadHandler() => + internal static void DetachAssemblyLoadHandler() + { + if (Interlocked.Exchange(ref s_assemblyLoadHooked, 0) == 0) return; AppDomain.CurrentDomain.AssemblyLoad -= OnAssemblyLoad; + } + + /// + /// Drops every type resolved so far. Module removal does this because the entries carry the + /// identity of the assemblies that defined them: a Microsoft.Graph.Authentication loading + /// after the removal would otherwise keep resolving to the first one's GraphSession, whose + /// Instance is a singleton nobody else is using. + /// + internal static void ClearTypeCache() => s_typeCache.Clear(); internal static Type? FindType(string fullName) { @@ -663,7 +780,7 @@ public static void ResetHttpClient() { lock (s_initLock) { - ScheduleDelayedHttpClientDispose(s_graphHttpClient, s_ownsHttpClient); + // The old client is let go, not closed - see the note on s_graphHttpClient. s_graphHttpClient = null; s_ownsHttpClient = false; s_cachedAuthFingerprint = null; @@ -672,32 +789,14 @@ public static void ResetHttpClient() } } - /// - /// Disposes an HttpClient after a delay. In-flight ResilientGraphClient instances - /// may still hold a reference to the old client, so we wait for the total timeout - /// window to ensure all in-flight requests complete before disposing. - /// Same pattern as ResiliencePipelineFactory.ScheduleDelayedDispose for rate limiters. - /// - private static void ScheduleDelayedHttpClientDispose(HttpClient? client, bool owned) - { - // Ownership is passed in rather than read off the static. Callers replace the static - // right after this call, so reading it here would silently follow whichever edit lands - // first - and disposing a borrowed SDK client kills Invoke-MgGraphRequest session-wide. - if (client == null || !owned) return; - var delaySeconds = s_clientOptions.TotalTimeoutSeconds; - _ = Task.Delay(TimeSpan.FromSeconds(delaySeconds)).ContinueWith(_ => - { - try { client.Dispose(); } catch { /* best-effort cleanup */ } - }, TaskScheduler.Default); - } - /// /// Whether the active transport is the mgx-owned clean client (AllowAutoRedirect off) /// rather than the borrowed SDK client. The content path requires ownership: the SDK /// client ships a RedirectHandler that auto-follows a content 302 to a host mgx never /// validated, so Get-MgxContent fails closed when this is false. /// - protected static bool TransportIsOwned => s_ownsHttpClient; + protected static bool TransportIsOwned => + s_testTransport != null ? s_testTransportOwned : s_ownsHttpClient; /// /// Drain buffered verbose messages from the resilience pipeline. @@ -752,10 +851,116 @@ private static bool CanTakeExclusively(string path) } } + /// + /// How two file names are compared here. Directory.EnumerateFiles matches its pattern the + /// way the filesystem does - so on NTFS and APFS "Users.jsonl" answers to "users.jsonl" and + /// the glob returns it - and a predicate filtering those results with an ordinal comparison + /// disagreed with the enumeration that produced them: an export whose -OutputFile differed + /// only in case abandoned its own temp and re-enumerated in silence. + /// + private static readonly StringComparison s_fileNameComparison = + OperatingSystem.IsLinux() ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; + + /// + /// Whether two paths name the same file, normalized and compared the way the filesystem + /// would answer. A checkpoint records the output it was collecting into so a run can tell + /// its own from someone else's, and a leaf name cannot carry that: "a/users.jsonl" and + /// "b/users.jsonl" are one name and two files. False for anything that is not a path - + /// a checkpoint is untrusted input once it is on disk, and an unusable name is not this + /// run's output. + /// + protected static bool SamePath(string? left, string? right) + { + if (left == null || right == null) return false; + try + { + return string.Equals(Path.GetFullPath(left), Path.GetFullPath(right), + s_fileNameComparison); + } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException + or PathTooLongException or IOException) + { + return false; + } + } + + /// + /// What a URL addresses, independently of where Graph is reached: the path from its API + /// version onward, plus the query. Everything in front of the version is the endpoint, and + /// a gateway prefix, a trailing slash on the configured endpoint and a sovereign cloud all + /// change that without changing which resource is named - so comparing whole URLs called a + /// checkpoint another run's for reasons that have nothing to do with what it enumerated. + /// Null when the argument is not a URL. + /// + protected static string? ResourceIdentity(string? url) + { + if (!Uri.TryCreate(url, UriKind.Absolute, out var parsed)) return null; + var segments = parsed.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries); + var version = Array.FindIndex(segments, seg => + string.Equals(seg, "v1.0", StringComparison.OrdinalIgnoreCase) + || string.Equals(seg, "beta", StringComparison.OrdinalIgnoreCase)); + // No version segment to anchor on: nothing can be told apart from the endpoint, so the + // whole path stands, which can only refuse where it would have refused before. + var resource = version < 0 ? segments : segments[version..]; + return "/" + string.Join('/', resource) + parsed.Query; + } + + /// + /// Whether two values name the same enumeration. The path is + /// compared without case: Graph answers "/users" and "/Users" from one collection, so an + /// ordinal comparison turned a -Uri typed differently between two runs into a different + /// export, and the resume was refused and the collection enumerated again. The query is + /// compared as written - a $filter value is the service's to interpret, and two spellings of + /// one are not demonstrably the same enumeration. A null identity is one that could not be + /// read as a URL, and matches nothing. + /// + protected static bool SameResourceIdentity(string? recorded, string? current) + { + if (recorded == null || current == null) return false; + var r = recorded.IndexOf('?'); + var c = current.IndexOf('?'); + return string.Equals(r < 0 ? recorded : recorded[..r], + c < 0 ? current : current[..c], StringComparison.OrdinalIgnoreCase) + && string.Equals(r < 0 ? string.Empty : recorded[r..], + c < 0 ? string.Empty : current[c..], StringComparison.Ordinal); + } + + /// + /// Whether the file a checkpoint counted its items into is one this run collects into. A + /// checkpoint that records the output answers directly. One written before that field + /// existed records only where the bytes are - a temp of the output, or the output itself - + /// and reading that silence as "mine" is what let an older release's checkpoint drive a + /// recovery against a file it had never been measured against. It is believed only where + /// the file it points at corroborates it: a named temp has to be one this output's own name + /// produces and hold at least the bytes counted, and with no temp named the output itself + /// has to be there and be at least that long. Nothing to corroborate is not evidence, and a + /// recorded length below zero is not a measurement. + /// + protected static bool RecordedOutputMatches( + string? recordedOutput, string? recordedTemp, long? dataLength, string outputPath) + { + if (recordedOutput != null) return SamePath(recordedOutput, outputPath); + if (dataLength is not { } length || length < 0) return false; + if (recordedTemp != null) return ResolveNamedTemp(outputPath, recordedTemp, length) != null; + try + { + var info = new FileInfo(outputPath); + return info.Exists && info.Length >= length; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return false; + } + } + /// /// Remove leftover "{outputPath}.{guid}.tmp" files. Called only when no resume is pending, /// where every such file is an orphan by definition - except one a live run still holds, - /// which CanTakeExclusively keeps out of reach. + /// which CanTakeExclusively keeps out of reach. "Such a file" is decided by name and not by + /// the glob alone: the glob reaches a longer output's temps too, and those describe work + /// this run knows nothing about. The name has to be one a run gives its own temp, 32 hex + /// digits and all: anything looser is a promise that no file mgx did not write is deleted, + /// made about a directory the caller also keeps their own "users.jsonl.backup.tmp" in. /// protected void DeleteStaleTemps(string outputPath) { @@ -763,7 +968,10 @@ protected void DeleteStaleTemps(string outputPath) { var dir = Path.GetDirectoryName(outputPath); if (string.IsNullOrEmpty(dir) || !Directory.Exists(dir)) return; - foreach (var stale in Directory.EnumerateFiles(dir, Path.GetFileName(outputPath) + ".*.tmp").ToList()) + var outputName = Path.GetFileName(outputPath); + foreach (var stale in Directory.EnumerateFiles(dir, outputName + ".*.tmp") + .Where(p => IsRunTempName(outputName, Path.GetFileName(p))) + .ToList()) { // "Orphan" is an assumption about a file this run did not create, and a second // export running against the same output right now owns a file matching the same @@ -823,13 +1031,13 @@ protected void DeleteStaleTemps(string outputPath) /// True when is a name a fresh run gives its temp: /// the output's own name, a dot, 32 lowercase hex digits (Guid "N"), and ".tmp". /// - private static bool IsRunTempName(string outputFileName, string candidate) + protected static bool IsRunTempName(string outputFileName, string candidate) { var prefix = outputFileName + "."; const string suffix = ".tmp"; if (candidate.Length != prefix.Length + 32 + suffix.Length) return false; - if (!candidate.StartsWith(prefix, StringComparison.Ordinal)) return false; - if (!candidate.EndsWith(suffix, StringComparison.Ordinal)) return false; + if (!candidate.StartsWith(prefix, s_fileNameComparison)) return false; + if (!candidate.EndsWith(suffix, s_fileNameComparison)) return false; for (var i = prefix.Length; i < prefix.Length + 32; i++) { if (candidate[i] is (>= '0' and <= '9') or (>= 'a' and <= 'f')) continue; @@ -838,6 +1046,20 @@ private static bool IsRunTempName(string outputFileName, string candidate) return true; } + /// + /// Whether the file's last byte ends a line. Every whole row a run writes goes out through + /// WriteLine, which terminates it, so a file that does not end in a newline ends in a row + /// that was cut short - and only the last row can be, since every row before it is followed + /// by its own terminator. + /// + private static bool EndsWithNewline(string path) + { + using var fs = new FileStream(path, FileMode.Open, FileAccess.Read); + if (fs.Length == 0) return false; + fs.Seek(-1, SeekOrigin.End); + return fs.ReadByte() == '\n'; + } + /// /// Replace the output with the first bytes of a named temp, /// then remove the temp. A fresh run that finishes moves its temp over whatever the output @@ -846,8 +1068,9 @@ private static bool IsRunTempName(string outputFileName, string candidate) /// the glob-and-newest form, this takes exactly the file the checkpoint recorded, so a /// leftover from an unrelated run cannot be merged in. Bytes rather than lines: the length /// was taken from the writer's own position, so it cannot disagree with itself about line - /// endings or a torn final line. Returns false when the temp is absent or shorter than the - /// checkpoint promised, which means the caller must not resume past the items it counted. + /// endings or a torn final line. Returns false when the temp is absent, shorter than the + /// checkpoint promised, or still held by the run writing it, which means the caller must + /// not resume past the items it counted. /// protected static bool TryPromoteNamedTemp(string outputPath, string tempFileName, long dataLength) { @@ -856,6 +1079,17 @@ protected static bool TryPromoteNamedTemp(string outputPath, string tempFileName var tempPath = ResolveNamedTemp(outputPath, tempFileName, dataLength); if (tempPath == null) return false; + // The claim adoption and the stale-temp sweep take, for the reason they take it: + // this copies the file's rows into the output and then unlinks it, so a run that is + // still writing that temp went on writing into an unlinked inode, lost everything it + // had fetched, and had its rows handed over as this run's. Naming the file changes + // nothing about that - a checkpoint names the temp of the enumeration that wrote it, + // and that enumeration is either over, in which case nothing holds the file, or + // running right now, in which case this is its live output. Taken before anything + // reads the file, so both platforms decide it here rather than Windows failing the + // unlink afterwards with the rows already copied. + if (!CanTakeExclusively(tempPath)) return false; + // Staged like the other forms, so the destination is replaced in one Move rather // than truncated and refilled in place. var adoptPath = outputPath + ".adopt"; @@ -919,16 +1153,127 @@ protected static bool TryTrimOutputToCheckpoint(string outputPath, long dataLeng } } + /// + /// Whether TryPromoteNamedTemp would find a temp to promote, without promoting it. The + /// ShouldProcess gate has to name the action a real run would take before the run is + /// allowed to take any, and -WhatIf has to leave every staged file exactly as it found it. + /// The claim is asked for here too, and released again: a temp another run holds is one a + /// real run would refuse, so answering from the file's existence alone reported a recovery + /// that would not have happened. + /// + protected static bool CanPromoteNamedTemp(string outputPath, string tempFileName, long dataLength) + { + try + { + var tempPath = ResolveNamedTemp(outputPath, tempFileName, dataLength); + return tempPath != null && CanTakeExclusively(tempPath); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { return false; } + } + + /// + /// Whether the temp a checkpoint names is one another run has open. It is the single reason + /// a promotion fails that says nothing is wrong with the files: every row the checkpoint + /// counted is in that temp, and the enumeration that counted them is still collecting into + /// it. The other failures - no temp, a temp shorter than the recorded length - mean those + /// items are in no file at all, which is what the callers delete the checkpoint over and + /// re-enumerate. A checkpoint whose temp is being written is the position a running export + /// resumes from, so deleting it, or reporting its items as not on disk, describes a run + /// that has not failed and takes away what it would have come back to. + /// + protected static bool NamedTempIsHeld(string outputPath, string tempFileName, long dataLength) + { + try + { + var tempPath = ResolveNamedTemp(outputPath, tempFileName, dataLength); + return tempPath != null && !CanTakeExclusively(tempPath); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { return false; } + } + + /// + /// Whether TryTrimOutputToCheckpoint would find the output the checkpoint describes, + /// without cutting it. + /// + protected static bool CanTrimOutputToCheckpoint(string outputPath, long dataLength) + { + try + { + if (dataLength < 0) return false; + var info = new FileInfo(outputPath); + return info.Exists && info.Length >= dataLength; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { return false; } + } + + /// + /// The newest "{output}.{32-hex}.tmp" beside the output that nothing else holds open, or + /// null when there is none. Adoption copies the file it picks into the output and then + /// unlinks it, so a temp a second export is writing into right now must not be picked at + /// all: that run went on writing into an unlinked inode, lost everything it had fetched, + /// and its rows were handed over as this export's. The claim is the one DeleteStaleTemps + /// takes before it deletes, and it settles the same question - FileShare.None holds + /// between processes on Windows and Unix alike, so a run with its own temp open makes it + /// fail, and a file nobody can be found holding is an orphan. A candidate that cannot be + /// claimed is passed over rather than ending the search: newest-first reaches a live run's + /// temp before an older one, and the next candidate down may be the orphan recovery exists + /// for. Taking the claim before anything reads the file also keeps Windows declining here, + /// where a live writer already conflicts with the reads below rather than with this. + /// + private static FileInfo? NewestUnheldTemp(string outputPath) + { + var dir = Path.GetDirectoryName(outputPath); + if (string.IsNullOrEmpty(dir) || !Directory.Exists(dir)) return null; + var outputName = Path.GetFileName(outputPath); + return Directory.EnumerateFiles(dir, outputName + ".*.tmp") + .Where(p => IsRunTempName(outputName, Path.GetFileName(p))) + .Select(p => new FileInfo(p)) + .OrderByDescending(f => f.LastWriteTimeUtc) + .FirstOrDefault(f => CanTakeExclusively(f.FullName)); + } + + /// + /// Whether TryAdoptOrphanedTemp would find a temp holding the counted items, without + /// merging it into anything. + /// + protected static bool CanAdoptOrphanedTemp(string outputPath, long itemCount) + { + try + { + if (itemCount <= 0) return false; + if (File.Exists(outputPath)) return false; + var temp = NewestUnheldTemp(outputPath); + if (temp == null) return false; + + // Whole rows only, as below. + var endsWhole = EndsWithNewline(temp.FullName); + long counted = 0; + using var reader = new StreamReader(temp.FullName); + while (counted < itemCount && reader.ReadLine() != null) counted++; + return counted >= itemCount && (endsWhole || reader.ReadLine() != null); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { return false; } + } + /// /// Recovers a fresh-run JSONL job that was interrupted before its temp file was promoted, - /// from a checkpoint that predates the recorded temp name and length. With only a line - /// count and the newest matching temp to go on, this is safe solely when no output exists: + /// from a checkpoint that predates the recorded temp name and length. The candidates are + /// the names a run gives its own temp - "{output}.{32-hex}.tmp" - and nothing else: the + /// glob that finds them also reaches a caller's own "users.jsonl.backup.tmp", and adoption + /// copies what it takes into the output and then deletes it. With only a line count and the + /// newest of those nothing else holds open to go on, this is safe solely when no output exists: /// everything the run wrote is then in its temp, and creating the output from it is what a /// finishing run's own promotion would have done. Against an existing output there is no /// way to tell whose items the temp holds, so the caller must re-enumerate instead. /// Copies exactly lines - content beyond the last flush may - /// be absent or torn - then removes the temp. Returns false when nothing usable exists, - /// leaving the caller to the stale-checkpoint path. + /// be absent or torn - then removes the temp. The itemCount-th of them has to be a whole + /// row: ReadLine hands back an unterminated tail as though it were a line, so a temp cut + /// inside a counted row copied the fragment into the output as an item, reported it as + /// recovered, and left the rest of the enumeration appended behind it. A run mgx interrupts + /// cannot leave that state - every checkpoint site flushes the writer before recording the + /// count, so the rows a checkpoint counts are always terminated - but a temp truncated + /// under it by something outside the run can. Returns false when nothing usable exists, + /// leaving the caller to the stale-checkpoint path, which re-enumerates and loses nothing. /// protected static bool TryAdoptOrphanedTemp(string outputPath, long itemCount) { @@ -936,14 +1281,15 @@ protected static bool TryAdoptOrphanedTemp(string outputPath, long itemCount) { if (itemCount <= 0) return false; if (File.Exists(outputPath)) return false; - var dir = Path.GetDirectoryName(outputPath); - if (string.IsNullOrEmpty(dir) || !Directory.Exists(dir)) return false; - var temp = Directory.EnumerateFiles(dir, Path.GetFileName(outputPath) + ".*.tmp") - .Select(p => new FileInfo(p)) - .OrderByDescending(f => f.LastWriteTimeUtc) - .FirstOrDefault(); + var temp = NewestUnheldTemp(outputPath); if (temp == null) return false; + // A file that ends mid-row can only end mid-row in its LAST one, so the counted + // rows are whole unless the count reaches that far - which is the case where + // another line still follows the itemCount-th. + var endsWhole = EndsWithNewline(temp.FullName); + var countedRowIsWhole = false; + // Staged so the output appears in one Move. A merge that dies halfway leaves only // the staging file behind, and the caller keeps its checkpoint - the safe direction. long copied = 0; @@ -957,10 +1303,12 @@ protected static bool TryAdoptOrphanedTemp(string outputPath, long itemCount) writer.WriteLine(line); copied++; } + countedRowIsWhole = endsWhole || reader.ReadLine() != null; } - if (copied < itemCount) + if (copied < itemCount || !countedRowIsWhole) { - // Temp holds less than the checkpoint promises - unusable. + // Temp holds less than the checkpoint promises, or ends inside the last row it + // promises - unusable either way. File.Delete(adoptPath); return false; } @@ -1083,10 +1431,24 @@ private static string HeaderValueToString(object? value) if (value == null) return string.Empty; if (value is string s) return s; if (value is IEnumerable seq) - return string.Join(", ", seq.Cast().Select(v => v?.ToString() ?? string.Empty)); - return value.ToString() ?? string.Empty; + return string.Join(", ", seq.Cast().Select(HeaderScalarToString)); + return HeaderScalarToString(value); } + /// + /// One header value, rendered the same way on every machine. object.ToString() formats + /// under the thread's culture, so a double or a DateTimeOffset in -Headers put "0,5" on + /// the wire under de-DE where en-US put "0.5" - the same script sending different bytes. + /// CA1305 does not see it: the receiver is typed object, not IFormattable. + /// + private static string HeaderScalarToString(object? value) => value switch + { + null => string.Empty, + string s => s, + IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture), + _ => value.ToString() ?? string.Empty + }; + protected static Dictionary? BuildRequestHeaders( string? consistencyLevel, System.Collections.Hashtable? extraHeaders) { @@ -1098,8 +1460,11 @@ private static string HeaderValueToString(object? value) // Case-insensitive: HTTP header names are, and a case-variant duplicate // (If-Match and if-match) must not become two wire headers. headers = new Dictionary(StringComparer.OrdinalIgnoreCase); + // The name is rendered the same way as the value: a Hashtable key is typed object + // too, and object.ToString() puts a numeric one on the wire under the operator's + // culture. foreach (var key in extraHeaders.Keys) - headers[key.ToString()!] = HeaderValueToString(extraHeaders[key]); + headers[HeaderScalarToString(key)] = HeaderValueToString(extraHeaders[key]); } // Dedicated -ConsistencyLevel parameter always wins over -Headers key @@ -1318,6 +1683,73 @@ protected void WriteCountDiscrepancyWarning( + $"were returned ({pct}% shortfall). {cause}."); } + /// What a response body turned out to be, once its declared charset and its + /// byte-order mark are accounted for. + internal enum JsonPayloadKind + { + /// holds the parsed body. + Parsed, + /// No body at all - a 204, or a 200 that sent nothing. + Empty, + /// The body declares a content type that is not JSON. Text is the decoded body. + NotJson, + /// The body declares JSON and does not parse. Text is the decoded body. + Malformed, + } + + /// + /// The read, without the reporting. is the decoded body: as it arrived + /// for a non-JSON one, with any byte-order mark removed for a body meant to parse. + /// + internal readonly record struct JsonPayload( + JsonPayloadKind Kind, JsonElement Json, string Text, string? MediaType); + + /// + /// Decode and parse a response body: the declared charset, the byte-order mark that + /// GetString leaves as U+FEFF and the serializer refuses, and whether the body claims to + /// be JSON at all. Every caller reads a body the same way; what each does about a body it + /// cannot use differs, and that stays with the caller - this writes to no stream, so a + /// fan-out worker can call it too. + /// + internal static JsonPayload ReadJsonPayloadCore( + System.Net.Http.Headers.MediaTypeHeaderValue? declaredType, byte[] bodyBytes) + { + var mediaType = declaredType?.MediaType; + if (bodyBytes.Length == 0) + return new JsonPayload(JsonPayloadKind.Empty, default, string.Empty, mediaType); + + var text = ResolveEncoding(declaredType?.CharSet).GetString(bodyBytes); + if (mediaType != null && !mediaType.EndsWith("json", StringComparison.OrdinalIgnoreCase)) + return new JsonPayload(JsonPayloadKind.NotJson, default, text, mediaType); + + text = text.TrimStart('\uFEFF'); + try + { + return new JsonPayload(JsonPayloadKind.Parsed, + JsonSerializer.Deserialize(text), text, mediaType); + } + catch (JsonException) + { + return new JsonPayload(JsonPayloadKind.Malformed, default, text, mediaType); + } + } + + /// The response's declared charset, defaulting to UTF-8. GetString strips a + /// matching BOM, which the byte-level Deserialize refused. + internal static Encoding ResolveEncoding(string? charset) + { + if (string.IsNullOrEmpty(charset)) return Encoding.UTF8; + // Any charset .NET will not construct falls back to UTF-8. An unknown or malformed name + // is an ArgumentException, but a name .NET knows and refuses to build - utf-7, disabled + // since SYSLIB0001 - is a NotSupportedException, and catching only the first let a + // response header end the pipeline with a .NET deprecation message. + try { return Encoding.GetEncoding(charset.Trim('"')); } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException) + { + return Encoding.UTF8; + } + } + /// /// Release the Graph client. Invoked by under /// the Interlocked guard, so this runs exactly once even when StopProcessing diff --git a/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs b/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs index 641a67f..80fd976 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs @@ -152,7 +152,14 @@ protected override void EndProcessing() { target = $"GET {_collected.Count} requests via $batch"; } - else if (writeOps.All(o => string.Equals(o.Method, writeOps[0].Method, StringComparison.OrdinalIgnoreCase))) + // Only when the batch is that method and nothing else. The count is the whole + // batch by the reasoning above, and a write verb welded onto it described requests + // that were not in it: one DELETE among nineteen GETs read "DELETE 20 requests via + // $batch" at the one surface whose purpose is saying what is about to happen. With + // reads present the writes are named as their own count, the way a batch of several + // write methods already names them. + else if (writeOps.Count == _collected.Count + && writeOps.All(o => string.Equals(o.Method, writeOps[0].Method, StringComparison.OrdinalIgnoreCase))) { target = $"{writeOps[0].Method} {_collected.Count} requests via $batch"; } @@ -224,6 +231,13 @@ protected override void EndProcessing() var results = batchResult.Results; var telemetry = batchResult.Telemetry; + // Every line this run prints about itself reads these: the dead-letter line, the + // chunk-failure target, the verbose summary and the warning. Counting for + // themselves, they drifted - the warning said the batch-level retry pass had been + // withheld over runs in which it had gone out, been answered, and been refused on a + // later chunk, while the verbose line beside it credited the run with the retries. + var summary = BatchRunSummary.Of(results, telemetry, batchResult.ChunkFailure != null); + // Output all results as Hashtables (success and failure) for (int i = 0; i < results.Count; i++) { @@ -240,9 +254,9 @@ protected override void EndProcessing() : null }; - // Status 0 means the operation was never sent, because a chunk before it failed. - // It is not a success and must not read as one - the caller has to be able to - // tell a write that may have landed from one that certainly did not. + // Status 0 means the operation was never sent, because another chunk failed + // first. It is not a success and must not read as one - the caller has to be + // able to tell a write that may have landed from one that certainly did not. if (item.Status == GraphBatchClient.NotSentStatus) result["NotSent"] = true; @@ -253,7 +267,7 @@ protected override void EndProcessing() // Write failed items to dead-letter file (append mode) if (resolvedDeadLetterPath != null) { - var failedCount = 0; + var wrote = false; try { using var writer = new StreamWriter(resolvedDeadLetterPath, append: true); @@ -284,31 +298,41 @@ protected override void EndProcessing() deadLetter["Error"] = errorMsg; writer.WriteLine(deadLetter.ToJsonString(s_deadLetterJsonOptions)); - failedCount++; } + wrote = true; } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { WriteWarning($"Failed to write dead-letter file '{resolvedDeadLetterPath}': {ex.Message}"); } - if (failedCount > 0) - WriteVerbose($"Wrote {failedCount} failed items to dead-letter file: {resolvedDeadLetterPath}"); + // The file takes both, and they are not the same thing to a caller deciding what + // to re-pipe: a refused write may already have been applied and one that was + // never sent certainly was not. Only after the write finished - a line stating + // what is in a file the run could not finish writing states it of nothing. + if (wrote && summary.Failed + summary.NotSent > 0) + { + var written = summary.NotSent > 0 + ? $"{summary.Failed} failed and {summary.NotSent} not-sent items" + : $"{summary.Failed} failed items"; + WriteVerbose($"Wrote {written} to dead-letter file: {resolvedDeadLetterPath}"); + } } - // A chunk's POST failed after earlier chunks were applied. Their results are above; - // this says why the rest never went, and NotSent names them. After the dead-letter - // write, like the item errors below, so -ErrorAction Stop cannot cut the file short. + // A chunk's POST failed while other chunks were being applied. Their results are + // above; this says why the rest never went, and NotSent names them. After the + // dead-letter write, like the item errors below, so -ErrorAction Stop cannot cut + // the file short. if (batchResult.ChunkFailure != null) { - var notSent = batchResult.NotSent.Count; // The id stays BatchChunkFailed; the category and wrapping come from the // failure itself - a throttled chunk is LimitsExceeded, an open circuit // ResourceUnavailable with the guidance text, not NotSpecified. var (_, category, report) = MgxErrorPresentation.PresentItemFailure( batchResult.ChunkFailure, "BatchChunkFailed", CircuitBreakerMessage); WriteError(new ErrorRecord(report, "BatchChunkFailed", category, - $"{notSent} of {results.Count} operations were not sent")); + $"{summary.Failed} of {summary.Total} operations failed, " + + $"{summary.NotSent} were not sent")); } // Emit errors for failed items (enables -ErrorAction Stop, populates $Error). @@ -321,7 +345,7 @@ protected override void EndProcessing() var skipped = submitted[i]; WriteError(new ErrorRecord( new InvalidOperationException( - $"{skipped.Method} {skipped.Url} was not sent: an earlier chunk failed."), + $"{skipped.Method} {skipped.Url} was not sent: another chunk of the batch failed."), "BatchItemNotSent", ErrorCategory.NotSpecified, skipped.Url)); } else if (item.Status >= 400) @@ -337,8 +361,7 @@ protected override void EndProcessing() } } - // Structured telemetry summary - WriteBatchTelemetry(telemetry); + WriteBatchTelemetry(telemetry, summary); } catch (Exception ex) when (ex is GraphServiceException or BrokenCircuitException or HttpRequestException) { @@ -490,7 +513,59 @@ internal static void RedactSensitiveFields(JsonNode? node) return null; } - private void WriteBatchTelemetry(BatchTelemetry telemetry) + /// + /// What the run did, computed once so that every line stating it states the same thing. + /// + /// Operations submitted. + /// Answered with a status in [200,400). + /// Answered with a status of 400 or above, refusals included. + /// Never POSTed, so certainly not applied. + /// Items the batch-level retry pass put on the wire. + /// What became of that pass. + private sealed record BatchRunSummary( + int Total, int Succeeded, int Failed, int NotSent, int BatchLevelRetries, RetryPass Pass) + { + internal static BatchRunSummary Of( + IReadOnlyList<(BatchOperation Operation, GraphBatchResponseItem Response)> results, + BatchTelemetry telemetry, + bool stopped) + { + // The not-sent count comes from the results rather than from telemetry, which + // counts what was answered. A refused item carries a status and an unsent one does + // not, so the results are what say which is which. + var notSent = results.Count(r => r.Response.Status == GraphBatchClient.NotSentStatus); + + // Which of the two ways a stopped run stopped is decided by what the pass put on + // the wire, not by the stop itself. The count is taken as each of the pass's chunks + // is sent, so nonzero means the pass went out; a run stopped by a chunk failure + // before the pass never reaches the pass at all, and leaves it at zero. + var pass = !stopped ? RetryPass.RanClean + : telemetry.BatchLevelRetries > 0 ? RetryPass.RanRefused + : RetryPass.Withheld; + + return new BatchRunSummary(telemetry.TotalRequests, telemetry.Succeeded, + telemetry.Failed, notSent, telemetry.BatchLevelRetries, pass); + } + } + + /// What became of the batch-level retry pass, which decides what the warning + /// may claim about the attempts an item had. + private enum RetryPass + { + /// Nothing stopped the run: every item had the attempts it qualified for, + /// whether or not any of them needed the pass. + RanClean, + + /// The pass went out and its own POST was refused. What it carried before + /// that may have been applied on the server. + RanRefused, + + /// A chunk failed before the pass, so it was skipped for every candidate and + /// nothing of it reached the wire. + Withheld, + } + + private void WriteBatchTelemetry(BatchTelemetry telemetry, BatchRunSummary summary) { // Propagate per-item 429 counts to session telemetry if (telemetry.ThrottleEncounters > 0) @@ -503,24 +578,40 @@ private void WriteBatchTelemetry(BatchTelemetry telemetry) // Always emit verbose summary with timing breakdown var elapsedSec = telemetry.TotalElapsedMs / 1000.0; - var throughput = telemetry.TotalElapsedMs > 0 ? telemetry.TotalRequests / elapsedSec : 0; - var summary = $"Batch: {telemetry.Succeeded} succeeded, {telemetry.Failed} failed out of {telemetry.TotalRequests} requests in {elapsedSec:F1}s ({throughput:F1}/sec)."; + var throughput = telemetry.TotalElapsedMs > 0 ? summary.Total / elapsedSec : 0; + var notSentPart = summary.NotSent > 0 ? $", {summary.NotSent} not sent" : string.Empty; + var line = $"Batch: {summary.Succeeded} succeeded, {summary.Failed} failed{notSentPart} out of {summary.Total} requests in {elapsedSec:F1}s ({throughput:F1}/sec)."; if (telemetry.ItemRetries > 0) - summary += $" Item retries: {telemetry.ItemRetries}."; + line += $" Item retries: {telemetry.ItemRetries}."; if (telemetry.ThrottleEncounters > 0) - summary += $" Throttle (429) encounters: {telemetry.ThrottleEncounters}."; - if (telemetry.BatchLevelRetries > 0) - summary += $" Batch-level retries: {telemetry.BatchLevelRetries}."; + line += $" Throttle (429) encounters: {telemetry.ThrottleEncounters}."; + if (summary.BatchLevelRetries > 0) + line += $" Batch-level retries: {summary.BatchLevelRetries}."; if (telemetry.TotalRetryDelayMs > 0) - summary += $" Time in retry delays: {telemetry.TotalRetryDelayMs / 1000.0:F1}s."; - WriteVerbose(summary); + line += $" Time in retry delays: {telemetry.TotalRetryDelayMs / 1000.0:F1}s."; + WriteVerbose(line); - // Warn if any items failed after all retry attempts - if (telemetry.Failed > 0) + // The one line a caller running without -Verbose sees at all, so it says both what + // failed and what never went out - and does not credit the run with retries it withheld, + // nor deny it the ones it sent. + if (summary.Failed > 0 || summary.NotSent > 0) { - WriteWarning( - $"{telemetry.Failed} of {telemetry.TotalRequests} batch items failed after all retry attempts. " - + "Check $Error for details on each failed item."); + var outcome = summary.NotSent > 0 + ? $"{summary.Failed} of {summary.Total} batch items failed and {summary.NotSent} were not sent." + : $"{summary.Failed} of {summary.Total} batch items failed."; + // "after all retry attempts" holds only of a run that ran them, and "withheld" only + // of a pass that never left. Between them is the pass that went out, was answered, + // and was refused on a later chunk: the items it carried may have been applied, and + // a caller told the pass never ran has no reason to check before resubmitting them. + var retries = summary.Pass switch + { + RetryPass.Withheld => + " A chunk failed, so the run stopped sending and the retry pass was withheld.", + RetryPass.RanRefused => + " The retry pass was sent and then refused, so the run stopped sending.", + _ => " They failed after all retry attempts.", + }; + WriteWarning(outcome + retries + " Check $Error for details on each item."); } } diff --git a/src/Mgx.Cmdlets/Cmdlets/Configuration/DisableMgxResilience.cs b/src/Mgx.Cmdlets/Cmdlets/Configuration/DisableMgxResilience.cs index d1c9ad1..e78777f 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Configuration/DisableMgxResilience.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Configuration/DisableMgxResilience.cs @@ -17,6 +17,23 @@ protected override void ProcessRecord() { if (!EnableMgxResilience.IsEnabled) { + // A wrapper can outlive the state that tracked it. Module removal takes the + // injection off the session before it drops its references, but a removal that + // ran while GraphSession was unreachable cannot, and what it leaves behind is a + // wrapper nothing points at. Reaching it here is the only way back to the + // genuine client: no other reference to it survives. + if (EnableMgxResilience.SessionHoldsInjectedWrapper()) + { + if (!ShouldProcess("Microsoft.Graph SDK HttpClient", + "Restore original SDK HttpClient (remove Polly resilience)")) + return; + + EnableMgxResilience.TryRestoreGenuineSdkClient(); + WriteVerbose("MgxResilience disabled. The wrapper an earlier import left on " + + "the SDK client is gone and SDK cmdlets are restored to original behavior."); + return; + } + WriteWarning("MgxResilience is not currently enabled."); return; } @@ -77,15 +94,26 @@ protected override void ProcessRecord() clientProp.SetValue(instance, originalClient); - // Dispose the resilient client to release the ResilientDelegatingHandler - // and its references to the Polly pipeline. + // These statics are the last references to the ResilientDelegatingHandler and to + // the client wrapped around it, and dropping them is what releases those. Not the + // Polly pipeline: the factory keeps one instance in a static of its own and hands + // the same one to every client it builds, so it outlives this teardown - which is + // the point of it. Releasing it would mean ResiliencePipelineFactory.Reset(), and + // Disable must not call that: the circuit-breaker history, the rate limiter and + // the learned pacing it holds are shared with mgx's own cmdlets, which keep running + // after the SDK injection comes off. var resilientClient = EnableMgxResilience.ResilientSdkClient; EnableMgxResilience.IsEnabled = false; EnableMgxResilience.OriginalSdkClient = null; EnableMgxResilience.ResilientSdkClient = null; + // The handler goes with them. It was left set here, so a disabled session still + // rooted the Polly pipeline and the bridge to the client we just stopped using. + EnableMgxResilience.ActiveHandler = null; // Not disposed. Restoring GraphSession.GraphHttpClient already stops new traffic; // disposing would cancel SDK requests still in flight - a paged read running when the - // user types Disable-MgxResilience would die mid-enumeration rather than finish. + // user types Disable-MgxResilience would die mid-enumeration rather than finish. It + // holds no connections of its own either: the SDK client it bridges to owns those, + // and that client's pool closes its sockets on its own timers. _ = resilientClient; WriteVerbose("MgxResilience disabled. SDK cmdlets restored to original behavior."); diff --git a/src/Mgx.Cmdlets/Cmdlets/Configuration/EnableMgxResilience.cs b/src/Mgx.Cmdlets/Cmdlets/Configuration/EnableMgxResilience.cs index 5a521d7..43e28d4 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Configuration/EnableMgxResilience.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Configuration/EnableMgxResilience.cs @@ -1,6 +1,7 @@ using System.Management.Automation; using System.Net; using System.Reflection; +using System.Runtime.CompilerServices; using Mgx.Engine.Http; using Mgx.Cmdlets.Base; @@ -33,6 +34,102 @@ public class EnableMgxResilience : PSCmdlet internal static bool IsEnabled { get; set; } internal static ResilientDelegatingHandler? ActiveHandler { get; set; } + // Every wrapper this module has installed, against the genuine SDK client underneath it. The + // keys are weak, so an entry lasts exactly as long as the wrapper somebody still holds and + // the table is not itself a reference the injection has to release. It is how a wrapper left + // by an earlier import is recognized after the statics that tracked it are gone: this + // assembly is never unloaded, so the table spans any number of import cycles. + private static readonly ConditionalWeakTable s_bridgeTargets = new(); + + /// + /// The genuine SDK client under , which is + /// itself unless mgx wrapped it. Wrapping a wrapper multiplies every layer: under a throttle + /// each layer retries the one beneath it, so the attempts reaching the wire go up per layer, + /// telemetry counts each attempt once per layer, and the pacer halves once per layer. + /// + internal static HttpClient ResolveGenuineSdkClient(HttpClient client) + { + while (s_bridgeTargets.TryGetValue(client, out var inner) && !ReferenceEquals(inner, client)) + client = inner; + return client; + } + + /// Whether this client is a wrapper mgx installed, in this import or an earlier one. + internal static bool IsInjectedWrapper(HttpClient? client) => + client != null && s_bridgeTargets.TryGetValue(client, out _); + + /// Whether GraphSession is holding a wrapper of ours right now. + internal static bool SessionHoldsInjectedWrapper() + { + try + { + var instance = MgxCmdletBase.TryGetGraphSessionInstance(); + var current = instance?.GetType().GetProperty("GraphHttpClient")?.GetValue(instance); + return IsInjectedWrapper(current as HttpClient); + } + catch (Exception ex) + { + System.Diagnostics.Debug.WriteLine($"[Mgx] Could not read GraphHttpClient: {ex.Message}"); + return false; + } + } + + /// + /// Puts GraphSession.GraphHttpClient back on the genuine SDK client when the client installed + /// there is a wrapper of ours, and reports whether it did. + /// + /// The wrapper is not disposed: an SDK request already in flight is sending through it, and + /// disposing cancels that request mid-enumeration. Restoring the property stops new traffic, + /// and the wrapper is collected once the requests still using it finish. + /// + /// + internal static bool TryRestoreGenuineSdkClient() + { + try + { + var instance = MgxCmdletBase.TryGetGraphSessionInstance(); + var clientProp = instance?.GetType().GetProperty("GraphHttpClient"); + if (instance == null || clientProp == null) return false; + + if (clientProp.GetValue(instance) is not HttpClient current || !IsInjectedWrapper(current)) + return false; + + clientProp.SetValue(instance, ResolveGenuineSdkClient(current)); + return true; + } + catch (Exception ex) + { + // Module teardown calls this: a reflection failure must not stop the module unloading. + System.Diagnostics.Debug.WriteLine($"[Mgx] Could not restore the SDK client: {ex.Message}"); + return false; + } + } + + /// + /// Unwinds the injection: the session goes back to the genuine SDK client before mgx lets go + /// of the wrapper. Nothing is disposed - see TryRestoreGenuineSdkClient. + /// + /// These statics outlive Remove-Module, so what is left here is what the next import finds. + /// Dropping the references without restoring the session strands the wrapper: it stays + /// installed, and every SDK request the process makes from then on goes through a handler + /// belonging to a module that is no longer loaded, until somebody imports mgx and disables + /// it. What the wrapper bridges to is not lost with these references - s_bridgeTargets is + /// deliberately not one of them - so a later Enable-MgxResilience still wraps the genuine + /// client rather than the wrapper, and Disable-MgxResilience still has it to restore. + /// + /// + internal static void ReleaseInjection() + { + lock (StateLock) + { + TryRestoreGenuineSdkClient(); + IsEnabled = false; + ActiveHandler = null; + ResilientSdkClient = null; + OriginalSdkClient = null; + } + } + protected override void ProcessRecord() { lock (StateLock) @@ -92,12 +189,13 @@ protected override void ProcessRecord() return; } // Our client was replaced (e.g., by Connect-MgGraph or Set-MgRequestContext). - // Dispose the old wrapped client to release its handler chain and sockets. WriteVerbose("MgxResilience was reset by SDK. Re-injecting resilience..."); // Not disposed: HttpClient.Dispose cancels its pending-request token source and // the bridge handler forwards that token inward, so SDK requests already in // flight die. Restoring GraphSession.GraphHttpClient stops new traffic; the old - // client is collected once the requests still using it finish. + // client is collected once the requests still using it finish. It holds no + // connections of its own to release either - it bridges to the SDK's client, + // and that client's pool closes its sockets on its own timers. _ = ResilientSdkClient; ResilientSdkClient = null; // Reset circuit breaker / rate limiter state from the previous tenant @@ -108,6 +206,11 @@ protected override void ProcessRecord() "Replace with Polly resilience pipeline (retry, circuit breaker, rate limiting)")) return; + // Wrap the genuine client, never a wrapper. The session can still be holding one from + // an earlier import - its statics went with that import - and a second layer over it + // is what OriginalSdkClient would then be restored to by Disable-MgxResilience. + currentClient = ResolveGenuineSdkClient(currentClient); + // Save the current SDK client AFTER we know build will be attempted OriginalSdkClient = currentClient; @@ -215,6 +318,8 @@ internal static void RefreshInjectedClient(Action warn, Action v return; } + currentClient = ResolveGenuineSdkClient(currentClient); + var refreshed = BuildResilientSdkClient(currentClient, warn); if (refreshed == null) { @@ -281,10 +386,14 @@ internal static void RefreshInjectedClient(Action warn, Action v return new Dictionary { [OptionType] = option }; } - private static HttpClient? BuildResilientSdkClient(HttpClient sdkClient, Action warn) + internal static HttpClient? BuildResilientSdkClient(HttpClient sdkClient, Action warn) { try { + // Bridge to the genuine client whatever the caller hands over: a wrapper wrapping a + // wrapper multiplies wire attempts, telemetry counts and pacer steps per layer. + sdkClient = ResolveGenuineSdkClient(sdkClient); + var (pipeline, rateLimiter) = ResiliencePipelineFactory.GetOrCreate(MgxCmdletBase.s_clientOptions); // Wrap the existing SDK client (preserving its full handler chain: @@ -314,11 +423,17 @@ internal static void RefreshInjectedClient(Action warn, Action v // against the active client's BaseAddress before any handler runs. Default // request headers are NOT copied - the bridge delegates to sdkClient.SendAsync, // which applies the original client's defaults to each request anyway. - return new HttpClient(resilientHandler) + var wrapper = new HttpClient(resilientHandler) { BaseAddress = sdkClient.BaseAddress, Timeout = sdkClient.Timeout }; + + // What the wrapper bridges to, recorded where it survives module removal. The + // teardown, Disable-MgxResilience and the next Enable-MgxResilience all need the + // genuine client back, and after a removal this is the only thing that still knows it. + s_bridgeTargets.AddOrUpdate(wrapper, sdkClient); + return wrapper; } catch (Exception ex) { diff --git a/src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs b/src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs index ace6a1f..7371303 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs @@ -455,6 +455,188 @@ private void DeleteCheckpoint(string? checkpointPath, string reason) } + /// + /// Whose the checkpoint on disk is, and where it is not this sync's, why not. Three things + /// have to agree before it is this run's. + /// + /// The output mode: a checkpoint that names an output file, names a temp of one, or records + /// a byte length was written by a sync collecting into a file, and its position counts items + /// that went there rather than down the pipeline. Resuming a pipeline sync from it emits + /// neither those items nor anything before them and then saves a delta token over the lot. + /// The length carries as much of that as the names do: a file-mode run records no temp once + /// it is appending, and none when a cancellation promoted the one it had, so a release that + /// did not yet record the output left file-mode checkpoints naming neither - and every one + /// of them was adopted here. A length is measured on a file, and a pipeline run has none to + /// measure. The reverse is refused on the same grounds. + /// + /// The output itself, on file-mode runs: two syncs of the same -Uri build the same resource, + /// so that comparison says nothing about WHICH file a recorded length was counted in, and + /// applied to another sync's it cut that file to this one's offset, mid-line. + /// + /// The resource, endpoint-independently and however -Uri was typed: what was enumerated, not + /// where Graph was reached, and not which spelling reached it - Graph answers "/users/delta" + /// and "/Users/delta" from one collection, so a case the caller typed differently between + /// two runs is the same enumeration and refusing it costs the run its own position. + /// + /// Nothing here decides differently than the bare predicate this replaced; what a refusal + /// can then say is what changes. A checkpoint recording no output was believed for as long + /// as the files beside it corroborated it, so losing that corroboration - a temp since + /// removed, an output since replaced - is one sync and no second run anywhere, and it is + /// the shape every release before this one wrote. + /// + private CheckpointOwnership OwnershipOf(PaginationCheckpoint checkpoint, string? outputPath, + string requestUrl) + { + if (outputPath == null) + { + if (checkpoint.OutputFile != null || checkpoint.TempFile != null + || checkpoint.DataLength != null) + { + return CheckpointOwnership.AnotherSyncs; + } + } + else if (!RecordedOutputMatches(checkpoint.OutputFile, checkpoint.TempFile, + checkpoint.DataLength, outputPath)) + { + return checkpoint.OutputFile != null + ? CheckpointOwnership.AnotherSyncs + : CheckpointOwnership.Uncorroborated; + } + + return SameResourceIdentity(ResourceIdentity(checkpoint.Resource), + ResourceIdentity(requestUrl)) + ? CheckpointOwnership.Mine + : CheckpointOwnership.AnotherSyncs; + } + + /// Which of the three a checkpoint on disk is. + private enum CheckpointOwnership + { + /// This sync's. + Mine, + + /// + /// Another sync's: it records another output, another enumeration, or the other + /// output mode. + /// + AnotherSyncs, + + /// + /// Records no output file, and the files beside this one no longer stand for it. Which + /// sync wrote it cannot be told from here, and telling the caller it was another one is + /// a diagnosis they can go and check and find nothing behind. + /// + Uncorroborated, + } + + /// + /// Whether the temp a refused checkpoint names is a file the stale-temp sweep would reach. + /// The name comes off a checkpoint, which is untrusted once it is on disk, and nothing here + /// opens it - all it decides is whether the sweep runs at all, so a name of some other shape + /// is answered no rather than refused: the sweep cannot delete it either, and skipping the + /// sweep for it would leave real orphans behind for nothing. + /// + /// "Some other shape" is the sweep's own test of a name and not a looser one kept here. + /// Matching on the prefix and the suffix alone answers yes for "users.jsonl.{guid}.tmp" + /// beside an output named "users" - a file the sweep passes over, because the glob's '*' + /// spans dots and the sweep's own filter does not - so a checkpoint left by the run next + /// door suppressed this output's sweep and its real orphans stayed on disk, which is the + /// case this comment says is answered no. + /// + private static bool RefusedTempIsOnDisk(string? tempFile, string outputPath) + { + if (tempFile == null || tempFile != Path.GetFileName(tempFile)) return false; + if (!IsRunTempName(Path.GetFileName(outputPath), tempFile)) return false; + var dir = Path.GetDirectoryName(outputPath); + return !string.IsNullOrEmpty(dir) && File.Exists(Path.Combine(dir, tempFile)); + } + + /// + /// What a caller can act on when the checkpoint on disk records an enumeration this run + /// cannot resume: what is known about the file, what this run does instead, and that a + /// -CheckpointPath belongs to one enumeration. Refusing it is the whole response - the file + /// is left alone, and so is any staging file beside it, because whatever wrote them may + /// still resume from exactly those. + /// + /// Which sync wrote it is not something this can tell, and asserting a second one sent the + /// caller looking for a run that in the commonest shape here does not exist: a checkpoint + /// saved during a delta enumeration, found by the same command line after the state + /// recording that delta was lost - corrupt, deleted by hand, a -DeltaPath since moved - so + /// the run builds a full-sync URL and refuses its own earlier position. There is no second + /// sync to find, and giving each its own -CheckpointPath is what the caller already did. + /// Both readings are offered and neither is asserted; what this run does about it is the + /// same either way. + /// + private static string ForeignCheckpointWarning(string checkpointPath, string? outputPath, string deltaPath) => + $"The resume checkpoint at '{checkpointPath}' records an enumeration this run cannot " + + $"resume from, so it is left as it is and this run enumerates from what '{deltaPath}' " + + "records" + + (outputPath != null ? $", into '{outputPath}'" : "") + + ". It is either another sync's, sharing this -CheckpointPath, or this sync's own from " + + "a pass it no longer makes - a delta enumeration whose state has since been lost " + + "leaves exactly this. Two syncs sharing one -CheckpointPath overwrite each other's " + + "resume position; give each its own."; + + /// + /// What a caller can act on when a checkpoint that records no output file is refused: it + /// was written before the output was recorded, nothing beside this run's output stands for + /// it any more, and this run enumerates from the delta token instead. Naming a different + /// sync there sends the caller looking for a second run over the same -CheckpointPath, and + /// there is none to find - a temp that has since been removed and an output that has since + /// been replaced reach the same refusal. + /// + private static string UncorroboratedCheckpointWarning( + string checkpointPath, string outputPath, string deltaPath) => + $"The resume checkpoint at '{checkpointPath}' records no output file, and the files " + + $"beside '{outputPath}' no longer corroborate it, so it is left as it is and this run " + + $"enumerates from the delta token in '{deltaPath}', into '{outputPath}'; no changes " + + "are lost."; + + /// + /// What this run tells the caller about a checkpoint it will not resume from, or null when + /// it is this sync's and there is nothing to tell. Uncorroborated is reachable only from + /// the branch that has an output to name; a pipeline run refuses on the output mode, which + /// is another sync's checkpoint however little else is known about it. + /// + private string? RefusalFor(PaginationCheckpoint checkpoint, string? outputPath, + string requestUrl, string checkpointPath, string deltaPath) + => OwnershipOf(checkpoint, outputPath, requestUrl) switch + { + CheckpointOwnership.Mine => null, + CheckpointOwnership.Uncorroborated when outputPath != null => + UncorroboratedCheckpointWarning(checkpointPath, outputPath, deltaPath), + _ => ForeignCheckpointWarning(checkpointPath, outputPath, deltaPath), + }; + + /// + /// What a reconcile leaves for the caller to do. Two of the three say do not resume, and + /// they are not the same answer: one is a checkpoint a live sync is still using, which is + /// left standing along with the temp it names, and the other is a checkpoint describing + /// items that are in no file, which is deleted wherever this run is able to delete it. + /// + private enum CheckpointRecovery + { + /// + /// The files hold what the checkpoint says they hold - promoted, trimmed or already + /// so - and the run resumes from the position it records. + /// + Resumable, + + /// + /// Another sync is writing the temp this checkpoint names. Nothing here is this run's + /// to recover or to remove: the checkpoint stays, its temp is spared this run's sweep, + /// and this run enumerates from the delta state. + /// + Refused, + + /// + /// The items the checkpoint counts are in no file this run can reach, so there is + /// nothing to resume from. The delete that goes with it is attempted and may fail; the + /// answer holds either way. + /// + Discarded, + } + /// /// Put the files into the state the checkpoint claims, or delete the checkpoint. A /// checkpoint records which file its items were written to and how many bytes of that file @@ -478,28 +660,44 @@ private void DeleteCheckpoint(string? checkpointPath, string reason) /// /// When the counted items turn out to be in no file, the delta link has not moved, so /// re-enumerating costs time and loses nothing while resuming past them loses them for good. + /// + /// A named temp open in another run is none of those: the files are exactly what the + /// checkpoint says they are, and the sync they belong to has not been interrupted. Its + /// items are that run's, and the checkpoint is the position it comes back to, so both are + /// left where they are - and the caller still must not resume, since a position whose items + /// are in a file this run cannot have leaves them out of the output while the delta token + /// advances past them. + /// + /// Whether the checkpoint may be resumed from is the answer, and not the file left on disk. + /// A bail-out that deletes the checkpoint it has just given up on can fail to delete it - a + /// checkpoint directory this account may read but not unlink from is answered false and + /// leaves the file - so a caller reading the refusal off File.Exists read the one thing a + /// failed delete gets wrong, resumed from the nextLink the warning had promised to + /// re-enumerate past, and advanced the delta token over items that are in no file at all. + /// Discarded binds either way; the file that outlives it is given up on again next run. /// - private void ReconcileCheckpointWithFiles(string checkpointPath, string outputPath, PaginationCheckpoint checkpoint) + private CheckpointRecovery ReconcileCheckpointWithFiles(string checkpointPath, string outputPath, PaginationCheckpoint checkpoint) { if (checkpoint.DataLength is not { } dataLength) { if (!File.Exists(outputPath)) { if (TryAdoptOrphanedTemp(outputPath, checkpoint.ItemsCollected)) - WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted sync's temp file. Resuming from checkpoint."); - else { - WriteWarning("Checkpoint found but output file is missing. Deleting stale checkpoint and starting fresh."); - PaginationCheckpoint.Delete(checkpointPath); + WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted sync's temp file. Resuming from checkpoint."); + return CheckpointRecovery.Resumable; } - return; + + WriteWarning("Checkpoint found but output file is missing. Deleting stale checkpoint and starting fresh."); + PaginationCheckpoint.Delete(checkpointPath); + return CheckpointRecovery.Discarded; } WriteWarning( "The resume checkpoint does not record which file the interrupted sync's items are in. " + "Re-enumerating from the last saved delta token; no changes are lost."); PaginationCheckpoint.Delete(checkpointPath); - return; + return CheckpointRecovery.Discarded; } if (checkpoint.TempFile != null) @@ -510,29 +708,46 @@ private void ReconcileCheckpointWithFiles(string checkpointPath, string outputPa // Those items are the output now. Repoint the checkpoint at it immediately, // so a second interruption cannot promote the same temp a second time. checkpoint.TempFile = null; + checkpoint.OutputFile = Path.GetFullPath(outputPath); checkpoint.DataLength = new FileInfo(outputPath).Length; try { checkpoint.Save(checkpointPath); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { WriteWarning($"Checkpoint save failed after recovery: {ex.Message}"); } - return; + return CheckpointRecovery.Resumable; + } + + // The temp is there and whole, and another run has it open - so the sync this + // checkpoint belongs to is not interrupted at all, it is collecting into that file + // right now. Its items are that run's, the checkpoint is the position it comes back + // to, and this run has nothing here to recover: deleting the checkpoint below, or + // resuming from it, would be the same mistake as unlinking the temp, one file over. + if (NamedTempIsHeld(outputPath, checkpoint.TempFile, dataLength)) + { + WriteWarning( + "Another sync is still writing the temp file this resume checkpoint names, so the " + + $"{checkpoint.ItemsCollected} items it records are that run's and are not recovered here. " + + "Both files are left as they are and this run re-enumerates from the last saved delta " + + "token; no changes are lost. Two syncs writing one -OutputFile replace each other's " + + "result; give each its own."); + return CheckpointRecovery.Refused; } WriteWarning( $"The interrupted sync's temp file is missing or incomplete, so the {checkpoint.ItemsCollected} items it " + "recorded are not on disk. Re-enumerating from the last saved delta token; no changes are lost."); PaginationCheckpoint.Delete(checkpointPath); - return; + return CheckpointRecovery.Discarded; } - if (!TryTrimOutputToCheckpoint(outputPath, dataLength)) - { - WriteWarning( - $"'{outputPath}' no longer holds the {checkpoint.ItemsCollected} items the resume checkpoint records. " - + "Re-enumerating from the last saved delta token; no changes are lost."); - PaginationCheckpoint.Delete(checkpointPath); - } + if (TryTrimOutputToCheckpoint(outputPath, dataLength)) return CheckpointRecovery.Resumable; + + WriteWarning( + $"'{outputPath}' no longer holds the {checkpoint.ItemsCollected} items the resume checkpoint records. " + + "Re-enumerating from the last saved delta token; no changes are lost."); + PaginationCheckpoint.Delete(checkpointPath); + return CheckpointRecovery.Discarded; } private void ExecuteDeltaSync( @@ -547,6 +762,32 @@ private void ExecuteDeltaSync( Stopwatch sw) { bool isFullResync = false; + // Whether this run has already decided the checkpoint on disk is another sync's, and + // the temp that checkpoint points at. Both outlive the attempt that refused, because + // the 410 door below sends the loop round again: an attempt that has forgotten the + // refusal deletes the position it was told to leave alone and sweeps away the temp + // holding its items, seconds after warning that neither would be touched. Ownership is + // still re-decided on every load - the checkpoint is a file, and the sync that owns it + // is still going - but only ever this way round, and never granted back: the retry + // rebuilds the request URL, so a checkpoint refused against the delta link can compare + // equal to the full re-sync one, and that is not evidence of anything. + var refusedCheckpoint = false; + string? refusedTemp = null; + // The temp an attempt of this run kept when it died, because a checkpoint counting its + // items was on disk. Nothing else records it: the sweep at the top of the next attempt + // is the only thing that reclaims a temp, and a refusal holds that off over every temp + // beside this output - so when the checkpoint stops naming the file, by being replaced + // with the next attempt's or deleted by the run that completed, this is what says which + // one is now a partial copy of the caller's changes that nothing refers to. + string? keptTemp = null; + // Whether an attempt of this run has written a checkpoint of its own over + // -CheckpointPath. The refusal above leaves the file where it is and then the very next + // page boundary saves over the same path, so what the refusal was protecting is gone + // and what is there is this run's - which is the one thing the 410 door has to know + // before it spares the file on the refusal's account. Run-scoped, unlike the + // per-attempt flag of the same reading further down: that one answers which attempt + // saved the checkpoint now on disk, and the keep it feeds needs exactly that. + var tookOverCheckpoint = false; for (int attempt = 0; attempt < 2; attempt++) { @@ -567,77 +808,151 @@ private void ExecuteDeltaSync( var currentFetchUrl = requestUrl; var appendOutput = false; - if (checkpointPath != null && File.Exists(checkpointPath)) + // Refusing is the whole response: the checkpoint is left where it is because + // the sync that wrote it resumes from exactly that, and the items it counted + // are in the temp it names. A refusal already made is not revisited - this run + // has nothing to resume from either way, and the two files stay out of its + // reach for the rest of it. + if (checkpointPath != null && File.Exists(checkpointPath) && !refusedCheckpoint) { - // JSONL crash: the checkpoint survives but the output was never promoted from - // its temp file. Promote the temp (trimmed to the checkpointed length) so - // resume appends to real data instead of declaring staleness. Without this - // the resume restarts at checkpoint.NextLink and the crashed run's items, - // sitting only in the temp, are never emitted - while the delta token - // advances past them on success. - if (outputPath != null) + // Ownership first, in both output modes. A sync writing to the pipeline used + // to skip this block entirely, so a checkpoint from a JSONL sync resumed it + // at that sync's nextLink: the pages before it went to the file and never to + // the pipeline, and the delta token was saved over them on success, which + // puts those changes permanently out of reach. + var orphanCp = PaginationCheckpoint.Load(checkpointPath); + if (orphanCp != null + && RefusalFor(orphanCp, outputPath, requestUrl, checkpointPath, deltaPath) + is { } refusal) { - var orphanCp = PaginationCheckpoint.Load(checkpointPath); - - // The resource is checked before anything is merged. The temp glob is - // shaped from the output path, so a checkpoint belonging to a different - // enumeration must never be allowed to pull a file into this one. The - // mismatch itself is handled a few lines below; here we only decline. - var resourceMatches = orphanCp != null - && string.Equals(orphanCp.Resource, requestUrl, StringComparison.Ordinal); - - if (resourceMatches && orphanCp!.NextLink != null) - { - ReconcileCheckpointWithFiles(checkpointPath, outputPath, orphanCp); - } - else if (!File.Exists(outputPath)) - { - WriteWarning("Checkpoint found but output file is missing. Deleting stale checkpoint and starting fresh."); - PaginationCheckpoint.Delete(checkpointPath); - } + // What the refusal leaves has to survive the sweep in this same run, + // and the 410 retry that runs it a second time. + refusedCheckpoint = true; + refusedTemp = orphanCp.TempFile; + WriteWarning(refusal); } - - if (File.Exists(checkpointPath)) + else { - var checkpoint = PaginationCheckpoint.Load(checkpointPath); - if (checkpoint?.NextLink == null) + // JSONL crash: the checkpoint survives but the output was never promoted + // from its temp file. Promote the temp (trimmed to the checkpointed + // length) so resume appends to real data instead of declaring staleness. + // Without this the resume restarts at checkpoint.NextLink and the crashed + // run's items, sitting only in the temp, are never emitted - while the + // delta token advances past them on success. + // + // A load that answers null reaches none of it. "Checkpoint found but + // output file is missing" is a reading of the checkpoint's contents, and + // a checkpoint nothing could read has none to go on: the delete below is + // for a file that says a sync completed, not for one this run failed to + // open. That case is answered where it is decided, a few lines down. + + // Whether the reconcile left a position worth reloading. Not the same + // question as whether the file is still there: a bail-out deletes the + // checkpoint it gave up on, that delete fails wherever the account cannot + // unlink from the checkpoint's directory, and the file it leaves is what + // File.Exists then answered yes to - so the run reloaded the position it + // had just warned it would re-enumerate past, appended this sync's rows + // onto the previous sync's, and saved a token past items held in no file. + // A checkpoint that outlives being given up on is given up on again next + // run. Resumable where there was nothing to reconcile: a pipeline sync + // has no output to put into any state, and a checkpoint that could not be + // read is answered on its own terms below. + var recovery = CheckpointRecovery.Resumable; + if (outputPath != null && orphanCp != null) { - WriteVerbose("Checkpoint is stale (corrupt or completed). Deleting."); - PaginationCheckpoint.Delete(checkpointPath); - } - else if (!string.Equals(checkpoint.Resource, requestUrl, StringComparison.Ordinal)) - { - // A checkpoint from a different enumeration (different deltaLink, - // different parameters, a completed sync in between). - WriteWarning("Checkpoint belongs to a different enumeration. Deleting checkpoint and starting fresh."); - PaginationCheckpoint.Delete(checkpointPath); + if (orphanCp.NextLink != null) + { + recovery = ReconcileCheckpointWithFiles( + checkpointPath, outputPath, orphanCp); + + // A temp another sync still holds is refused the way an + // ownership refusal is, and for the same two reasons: the + // reload below would otherwise resume from a position whose + // items are in a file this run cannot have - leaving them out + // of the output while the delta token advances past them - and + // the sweep in this same run would take that temp the moment + // its holder is gone, with the checkpoint naming it still on + // disk. + if (recovery == CheckpointRecovery.Refused) + { + refusedCheckpoint = true; + refusedTemp = orphanCp.TempFile; + } + } + else if (!File.Exists(outputPath)) + { + WriteWarning("Checkpoint found but output file is missing. Deleting stale checkpoint and starting fresh."); + PaginationCheckpoint.Delete(checkpointPath); + } } - else + + // The refusal above is a refusal of this checkpoint, not of one + // read of it: reloading and resuming from the file just left alone is + // the resume it was made to stop. And a checkpoint given up on is not + // resumed from either, however the delete that went with it fared. + if (recovery == CheckpointRecovery.Resumable + && !refusedCheckpoint && File.Exists(checkpointPath)) { - // SSRF validation: the checkpoint nextLink is untrusted (a file on disk) - var expectedHost = new System.Uri(requestUrl); - var validatedLink = NextLinkValidator.Validate(checkpoint.NextLink, expectedHost); - if (validatedLink != null - && checkpoint.ItemsCollected >= 0 - && checkpoint.PageItemsAlreadyWritten >= 0) + var checkpoint = PaginationCheckpoint.Load(checkpointPath); + if (checkpoint == null) { - resume = new ResumeState( - validatedLink, - checkpoint.PageItemsAlreadyWritten, - checkpoint.ItemsCollected); - currentFetchUrl = validatedLink; - resumedItemCount = checkpoint.ItemsCollected; - appendOutput = outputPath != null && File.Exists(outputPath); - WriteVerbose($"Resuming delta enumeration from checkpoint: {resumedItemCount} items already processed" - + (checkpoint.PageItemsAlreadyWritten > 0 - ? $", skipping {checkpoint.PageItemsAlreadyWritten} items on first page." - : ".")); + // Left where it is. Load answers null for a file torn by a crash + // and for one that is locked or that this account cannot open, + // and this run does the same thing either way - resume stays + // null, so the sync re-enumerates from the delta token. Deleting + // it changed nothing here and destroyed a position the next run, + // or another account, could still have resumed from. + WriteWarning( + "The resume checkpoint could not be read, so it cannot say how far the " + + "interrupted sync got. Re-enumerating from the last saved delta token; " + + "no changes are lost."); } - else + else if (checkpoint.NextLink == null) { - WriteWarning("Checkpoint nextLink failed validation. Deleting checkpoint and starting fresh."); + // A completion marker is this run's to remove, and the reading is + // the file's own: the sync that wrote it finished. Leaving it is + // not free - -Latest is suppressed by the checkpoint file merely + // existing - so a marker that outlives its sync costs the next + // -Latest run the baseline it asked for. + WriteVerbose("Checkpoint indicates the previous sync completed. Deleting stale checkpoint."); PaginationCheckpoint.Delete(checkpointPath); } + else if (RefusalFor(checkpoint, outputPath, requestUrl, + checkpointPath, deltaPath) is { } reloadedRefusal) + { + // Decided again on this load rather than carried over: the + // checkpoint is a file, and the sync that owns it is still going. + refusedCheckpoint = true; + refusedTemp = checkpoint.TempFile; + WriteWarning(reloadedRefusal); + } + else + { + // SSRF validation: the checkpoint nextLink is untrusted (a file on disk) + var expectedHost = new System.Uri(requestUrl); + var validatedLink = NextLinkValidator.Validate(checkpoint.NextLink, expectedHost); + if (validatedLink != null + && checkpoint.ItemsCollected >= 0 + && checkpoint.PageItemsAlreadyWritten >= 0) + { + resume = new ResumeState( + validatedLink, + checkpoint.PageItemsAlreadyWritten, + checkpoint.ItemsCollected); + currentFetchUrl = validatedLink; + resumedItemCount = checkpoint.ItemsCollected; + appendOutput = outputPath != null && File.Exists(outputPath); + WriteVerbose($"Resuming delta enumeration from checkpoint: {resumedItemCount} items already processed" + + (checkpoint.PageItemsAlreadyWritten > 0 + ? $", skipping {checkpoint.PageItemsAlreadyWritten} items on first page." + : ".")); + } + else + { + WriteWarning("Checkpoint nextLink failed validation. Deleting checkpoint and starting fresh."); + PaginationCheckpoint.Delete(checkpointPath); + } + } } } } @@ -660,6 +975,33 @@ private void ExecuteDeltaSync( // are. Set once the writer exists; null on the pipeline path, which has no file. string? checkpointTempFile = null; long? checkpointDataLength = null; + // Whether the checkpoint on disk is one THIS attempt saved. A checkpoint an + // earlier attempt left names an earlier temp, and the two are not + // interchangeable when it comes to deciding what a file on disk is still for. + var savedOwnCheckpoint = false; + + // A temp a failed attempt of this run kept is kept on one condition: a + // checkpoint counting its items is on disk. Saving a checkpoint here is what + // ends that - the file just written names this attempt's temp, or the output on + // a resumed run, and never the earlier one - so from that moment nothing on + // disk refers to it, no recovery can reach it, and the sweep is still being + // held off on behalf of the refusal that carried it this far. Left where it + // was, it outlived the sync that made it: a partial copy of the caller's + // changes beside the finished output, waiting for some later run's sweep. + // + // Never the temp the checkpoint now names, which is this attempt's own. Best + // effort otherwise: a file something else holds open is not worth failing a + // finished sync over. + void ReleaseKeptTemp() + { + var kept = keptTemp; + if (kept == null + || string.Equals(Path.GetFileName(kept), checkpointTempFile, + StringComparison.OrdinalIgnoreCase)) + return; + keptTemp = null; + try { if (File.Exists(kept)) File.Delete(kept); } catch { } + } void OnPageComplete(PageCompletedInfo info) { @@ -680,8 +1022,12 @@ void SaveBoundaryCheckpoint(PageCompletedInfo info) ItemsCollected = totalProcessed, PageItemsAlreadyWritten = 0, TempFile = checkpointTempFile, + OutputFile = outputPath != null ? Path.GetFullPath(outputPath) : null, DataLength = checkpointDataLength }.Save(checkpointPath); + savedOwnCheckpoint = true; + tookOverCheckpoint = true; + ReleaseKeptTemp(); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { @@ -705,7 +1051,24 @@ void SaveBoundaryCheckpoint(PageCompletedInfo info) // crash's checkpoint, and one success makes those rows permanent. // Sweeping here is what keeps "a temp exists only while a checkpoint // describing it exists" true across runs. - DeleteStaleTemps(outputPath); + // + // Which is exactly why a refusal is the one case it is wrong in: the + // checkpoint describing that temp IS on disk, left there deliberately a + // moment ago, and sweeping the temp took the one file that made it worth + // keeping - so the sync it belongs to came back to a position pointing at + // nothing and re-enumerated from the start. The sweep is all or nothing + // over this output's temps, so a run that has just refused leaves them to + // the next run that has not. + if (RefusedTempIsOnDisk(refusedTemp, outputPath)) + { + WriteVerbose( + $"Left the temp files beside '{outputPath}' alone: '{refusedTemp}' " + + "holds the items of the checkpoint this run refused."); + } + else + { + DeleteStaleTemps(outputPath); + } } var writePath = appendOutput ? outputPath : $"{outputPath}.{Guid.NewGuid():N}.tmp"; // A resumed run appends to the output itself, so there is no temp to name. @@ -763,8 +1126,12 @@ void SaveBoundaryCheckpoint(PageCompletedInfo info) ItemsCollected = totalProcessed, PageItemsAlreadyWritten = pageItemsWritten, TempFile = checkpointTempFile, + OutputFile = outputPath != null ? Path.GetFullPath(outputPath) : null, DataLength = checkpointDataLength }.Save(checkpointPath); + savedOwnCheckpoint = true; + tookOverCheckpoint = true; + ReleaseKeptTemp(); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { @@ -812,8 +1179,10 @@ void SaveBoundaryCheckpoint(PageCompletedInfo info) ItemsCollected = totalProcessed, PageItemsAlreadyWritten = pageItemsWritten, TempFile = null, + OutputFile = Path.GetFullPath(outputPath), DataLength = promotedLength }.Save(checkpointPath); + ReleaseKeptTemp(); } catch (Exception promoteEx) when (promoteEx is IOException or UnauthorizedAccessException) { @@ -833,8 +1202,24 @@ void SaveBoundaryCheckpoint(PageCompletedInfo info) // Keep the temp for the next run to promote. It is deleted by // promotion, by a later fresh run's own failure once the checkpoint // is gone, or on the missing-output path below. - var resumable = checkpointPath != null && File.Exists(checkpointPath); - if (!resumable) + // + // The checkpoint counting them has to be one this attempt saved. + // Any other is an earlier attempt's, naming an earlier temp, and + // this attempt's items are then counted by nothing: keeping the + // file left a page no recovery can reach, and the newest temp + // beside an output is what the pre-length adoption path picks up + // on a line count alone. + var resumable = checkpointPath != null && File.Exists(checkpointPath) + && savedOwnCheckpoint; + if (resumable) + { + // Named, because nothing else on disk will be: the sweep the + // next attempt reaches is held off by the refusal, not by + // this, and the checkpoint that makes the file worth keeping + // is replaced or deleted without a word about it. + keptTemp = writePath; + } + else { try { if (File.Exists(writePath)) File.Delete(writePath); } catch { } } @@ -889,6 +1274,13 @@ void SaveBoundaryCheckpoint(PageCompletedInfo info) // rather than a resumable position into a completed enumeration (wrong). DeleteCheckpoint(checkpointPath, "sync completed"); + // And with it goes the last thing that could have named a temp an earlier + // attempt kept - the attempt that finished need never have saved a checkpoint of + // its own to release it, if everything it had left fitted in one page. Only + // here, on the way out of a run that completed: an attempt that dies leaves the + // checkpoint and the temp it names for the next run to resume from. + ReleaseKeptTemp(); + // Save delta state ONLY after successful completion (Architect P0). // Zero-item responses still save the token (Adversarial P0). if (capturedDeltaLink != null) @@ -935,13 +1327,38 @@ void SaveBoundaryCheckpoint(PageCompletedInfo info) && ex.StatusCode == HttpStatusCode.Gone) { // 410 Gone: delta token expired (>7 days for directory objects). - // Delete delta state AND any checkpoint - both describe the dead + // Delete delta state AND any checkpoint of this run's - both describe the dead // enumeration - and restart with full sync. // Second attempt builds fresh URL (no delta token), so 410 won't recur. DrainClientMessages(); if (!DeltaState.Delete(deltaPath)) WriteVerbose("Could not delete expired delta state (file may be locked). It will be overwritten."); - DeleteCheckpoint(checkpointPath, "delta token expired (410 Gone)"); + // The refusal has to still hold over the file that is actually there. A + // checkpoint refused as another sync's is spared here because that sync resumes + // from exactly it - but the first page boundary of this run saves over the same + // path, and from then on the refused file is gone and the position on disk is + // this run's own. Sparing that one keeps a position into the enumeration the 410 + // has just declared dead: the retry re-enumerates in full, so the resource it + // records matches nothing this command line builds again, and every later run + // refuses it in turn - warning about a sync that was never there and holding off + // its stale-temp sweep on that account. It goes the way any other position into + // a dead enumeration goes. + if (refusedCheckpoint && !tookOverCheckpoint) + { + // The token that expired is this run's; the checkpoint is not. It was + // refused moments ago as another sync's, and that sync resumes from exactly + // it - so an expired token of this one's is no reason to delete it, and + // deleting it took the other sync's position away right after the refusal + // said it would be left where it was. + WriteVerbose( + "Left the resume checkpoint alone (delta token expired (410 Gone)): it " + + "records an enumeration this run refused, which this run's token says " + + "nothing about."); + } + else + { + DeleteCheckpoint(checkpointPath, "delta token expired (410 Gone)"); + } isFullResync = true; requestUrl = BuildListUrl(VersionedBaseUrl, Uri, new ODataListParams(false, Top, Top > 0 ? Top : 999, Filter, Property, null, null, 0, null)); diff --git a/src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs b/src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs index 852b19e..c666c18 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs @@ -211,13 +211,17 @@ private void ExecuteFanOut() if (!response.IsSuccessStatusCode) { - var errorBody = await response.Content.ReadAsStringAsync(ct); + var errorBody = await client.ReadBodyAsStringAsync(response, ct); throw new GraphServiceException(response.StatusCode, errorBody); } - using var stream = await response.Content.ReadAsStreamAsync(ct); - var json = await JsonSerializer.DeserializeAsync(stream, cancellationToken: ct); - var root = json.Clone(); + // The same read the rest of the module does: the declared charset, and the + // byte-order mark the byte-level deserialize refuses - which lost every + // relation to a per-URL error. A body this cannot use is this URL's failure, + // which is what the fan-out reports and where the streams are written from. + var bodyBytes = await client.ReadBodyAsBytesAsync(response, ct); + var payload = ReadJsonPayloadCore(response.Content.Headers.ContentType, bodyBytes); + var root = ReadRelationBody(payload); if (root.ValueKind == JsonValueKind.Object && root.TryGetProperty("value", out var valueEl) @@ -371,6 +375,42 @@ private void OutputBufferedObjects(Dictionary resultsById } } + /// + /// The relation carried by a body, whatever that body declares itself to be. A relation is + /// not always an entity: /$count answers text/plain with the number in the body, and this + /// cmdlet has no -Raw to fall back on, so the declared type decides how the bytes are + /// decoded and not whether they are read. A body that is no JSON value at all carries no + /// relation to attach, which is this URL's failure. + /// + private static JsonElement ReadRelationBody(JsonPayload payload) + { + if (payload.Kind == JsonPayloadKind.Parsed) + return payload.Json.Clone(); + + if (payload.Kind == JsonPayloadKind.NotJson) + { + try + { + // Decoded by the declared charset above; the mark GetString leaves behind is + // still the serializer's invalid start character. + return JsonSerializer.Deserialize(payload.Text.TrimStart('\uFEFF')); + } + catch (JsonException) { } + } + + throw new InvalidOperationException(DescribeUnusableBody(payload)); + } + + /// Why a body carries no relation to attach, for the per-URL error record. + private static string DescribeUnusableBody(JsonPayload payload) => payload.Kind switch + { + JsonPayloadKind.Empty => "The response has no content, so there is no relation to attach.", + JsonPayloadKind.NotJson => $"The response is {payload.MediaType} and does not parse as JSON. " + + "Body starts: " + payload.Text[..Math.Min(payload.Text.Length, 200)], + _ => "The response declared JSON but does not parse. Body starts: " + + payload.Text[..Math.Min(payload.Text.Length, 200)], + }; + private string BuildUrl(string id) { var resolved = IdPlaceholder.Replace(Uri, System.Uri.EscapeDataString(id)); diff --git a/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs b/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs index ef4d8df..96bde4f 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs @@ -149,57 +149,62 @@ protected override void ProcessRecord() defaultedToPageSize = true; } - // Checkpoint safety: a checkpoint without its output file usually means a - // first-run export died mid-flight. The data lives in an orphaned temp file - // (fresh runs write to ..tmp and rename on success), so try to - // adopt it: trim to exactly the item count the checkpoint recorded (content - // past the last flush may be missing or torn) and promote it to the output - // path. Only when no usable temp exists is the checkpoint truly stale. - if (cpPath != null && File.Exists(cpPath)) - { - var orphanCp = PaginationCheckpoint.Load(cpPath); - if (orphanCp?.NextLink != null) - { - ReconcileCheckpointWithFiles(cpPath, outputPath, orphanCp); - } - else if (!File.Exists(outputPath)) - { - WriteWarning("Checkpoint found but output file is missing. Deleting stale checkpoint and starting fresh."); - PaginationCheckpoint.Delete(cpPath); - } - } - - // ShouldProcess check (before requiring Graph connection, so -WhatIf works without auth) - var initialAppend = cpPath != null && File.Exists(cpPath) && File.Exists(outputPath); - var action = initialAppend ? "Append JSONL data" : "Export JSONL data"; - if (!ShouldProcess(outputPath, action)) - return; - - // Init client after ShouldProcess (populates s_graphEndpoint for sovereign clouds) - var client = GetClient(); - // Track whether $count=true was auto-added (not user-requested). - // If the endpoint rejects it with 400, retry without. + // If the endpoint rejects it with 400, retry without. Settled before the checkpoint is + // looked at: which URL this run fetches is half of whether the checkpoint on disk is + // about it, and none of this needs a Graph connection. bool countAutoAdded = !string.IsNullOrEmpty(Filter) && !ExistingQueryOptions(Uri).Contains("$count"); bool includeAutoCount = countAutoAdded; bool suppressTop = false; - // If checkpoint was saved during a previous retry (URL without $count=true), - // match the checkpoint's URL to avoid mismatch and data loss on resume - if (countAutoAdded && cpPath != null && File.Exists(cpPath) && File.Exists(outputPath)) + // Which form of that URL the checkpoint on disk was written under, when it is this + // export's. The loop below drops the auto-added $count on a bare 400 and the automatic + // $top on a Request_UnsupportedQuery, so an export interrupted after a rejection + // recorded a URL this run does not build until its own retry. Comparing the first form + // alone called that checkpoint another export's: the reconcile recovered nothing, the + // sweep below deleted the temp holding its items, and the retry - which does build that + // form - matched with the append already decided from the output merely existing, so + // the remainder of the enumeration was appended onto a previous export's file. Every + // form goes to the same comparison, and it settles which URL this run resumes before + // anything is promoted, trimmed or deleted. + if (cpPath != null && File.Exists(cpPath) + && AttemptFormOf(PaginationCheckpoint.Load(cpPath), outputPath, countAutoAdded) is { } form) { - var existingCp = PaginationCheckpoint.Load(cpPath); - if (existingCp?.Resource != null && !existingCp.Resource.Contains("$count=true")) - includeAutoCount = false; + includeAutoCount = form.IncludeCount; + suppressTop = form.SuppressTop; } + // ShouldProcess check (before requiring Graph connection, so -WhatIf works without auth). + // The action it names is what recovery below would do, worked out without doing any of + // it: recovery promotes, trims and deletes, and it used to run above this gate, so + // -WhatIf rewrote the output and removed checkpoints on the way to reporting that it + // would not. It stays above GetClient either way - "what would this do" is not a + // question that should need a Graph connection. + var wouldAppend = ReconcileCheckpoint(cpPath, outputPath, includeAutoCount, suppressTop, apply: false); + if (!ShouldProcess(outputPath, wouldAppend ? "Append JSONL data" : "Export JSONL data")) + return; + + // Whether this run appends to the output rather than exporting into a fresh temp, + // decided once, against every form of the URL the attempts below can build, and only + // ever withdrawn afterwards. Recomputed from a checkpoint and an output merely + // EXISTING, it came back true on the attempt after the reconcile had refused - and the + // rest of the enumeration went onto whatever file was sitting at -OutputFile. A + // refusal has to outlast the attempt that made it. + var appendToOutput = ReconcileCheckpoint(cpPath, outputPath, includeAutoCount, suppressTop, apply: true); + + // Init client after the gate (populates s_graphEndpoint for sovereign clouds) + var client = GetClient(); + for (int attempt = 0; attempt < 3; attempt++) { try { - // Recalculate append (checkpoint may have been invalidated on previous attempt) - var append = cpPath != null && File.Exists(cpPath) && File.Exists(outputPath); + // The checkpoint can go between attempts - a previous one deleted it as stale, + // or finished with it - so its existence is re-checked. Whose it is, is not + // re-decided: that answer is above, and it can only be taken away here. + appendToOutput = appendToOutput + && cpPath != null && File.Exists(cpPath) && File.Exists(outputPath); var url = BuildUrl(includeAutoCount, suppressTop); var headers = BuildHeaders(); @@ -209,19 +214,45 @@ protected override void ProcessRecord() long resumedItemCount = 0; string currentFetchUrl = url; - if (cpPath != null && append) + if (cpPath != null && appendToOutput) { var checkpoint = PaginationCheckpoint.Load(cpPath); if (checkpoint != null) { - if (checkpoint.NextLink == null) + if (!DescribesThisExport(checkpoint, outputPath, includeAutoCount, suppressTop)) + { + // Ownership is re-decided on every load, not carried over from the + // check above: the checkpoint is a file, the run that owns it is + // still going, and it can be replaced between the two reads. A + // checkpoint that is not this export's is left where it is - the + // export it does belong to needs it - and this run exports fresh. + // Only ever this way round: a refusal stands for the rest of the + // run, and no later attempt grants it back from the two files + // being where they were. + // + // And said out loud. This is the only refusal that lands on a + // checkpoint already accepted, so a resume has been reported and is + // being taken back: the collection is enumerated from the first page + // again, and the pages collected under the resume are collected a + // second time. Withdrawn in silence, the recovery message was the + // last thing on any stream, and a run that had quietly started over + // was indistinguishable from one that resumed. + _refusedCheckpointTemp = checkpoint.TempFile; + appendToOutput = false; + WriteWarning( + $"The resume checkpoint at '{cpPath}' no longer describes this export - the " + + "request was rebuilt without a query option the endpoint refused, or another " + + $"run replaced the file - so the resume is dropped and '{outputPath}' is " + + "exported from the beginning; nothing is lost."); + } + else if (checkpoint.NextLink == null) { // Completion marker: previous export finished, checkpoint is stale WriteVerbose("Checkpoint indicates previous export completed. Deleting stale checkpoint."); PaginationCheckpoint.Delete(cpPath); - append = false; + appendToOutput = false; } - else if (string.Equals(checkpoint.Resource, url, StringComparison.Ordinal)) + else { // Validate NextLink (SSRF protection) var expectedHost = new System.Uri(url); @@ -242,15 +273,17 @@ protected override void ProcessRecord() { WriteWarning("Checkpoint nextLink failed validation. Deleting checkpoint and starting fresh."); PaginationCheckpoint.Delete(cpPath); - append = false; + appendToOutput = false; } } - else - { - WriteWarning("Checkpoint resource mismatch. Deleting checkpoint and starting fresh."); - PaginationCheckpoint.Delete(cpPath); - append = false; - } + } + else + { + // The unreadable checkpoint above could not be deleted either - it is + // locked, or the account cannot touch it. Appending to the output on + // the strength of a file nothing can read is what has to stop, not the + // export: write a fresh run to a temp and let it replace the output. + appendToOutput = false; } } @@ -258,18 +291,56 @@ protected override void ProcessRecord() // This protects any pre-existing output file from truncation if // the Graph request fails on the first page. // Use GUID to prevent collision when multiple exports target the same file. - if (!append) + if (!appendToOutput) { // No resume is pending, so every "{output}.{guid}.tmp" on disk is an orphan. // Leaving them is not inert: the pre-length adoption path picks the NEWEST // match with only a line count to go on, so a survivor of an unrelated run is // adoptable by some later crash's checkpoint. - DeleteStaleTemps(outputPath); + // + // Except after a refusal, where one of them is not an orphan at all: a + // checkpoint describing it is on disk, left there deliberately a moment ago, + // and the items it counts are in that temp. Sweeping it took the one file + // that made the checkpoint worth keeping, so the export it belongs to came + // back to a position pointing at nothing and re-enumerated from the first + // page - the cost the refusal was written to avoid. The sweep is all or + // nothing over this output's temps, so a run that has just refused leaves + // them to the next run that has not. + // + // And once more on a retry, where the checkpoint standing over the temp is + // this run's own. An endpoint that refuses the auto-added $count, or $top, + // sends the attempt loop round again after pages have already been + // collected and checkpointed, and the attempt that died kept its temp for + // exactly that checkpoint to resume from. Sweeping here deleted it while + // the checkpoint naming it was still on disk - the one combination neither + // this policy nor the retry's intends - and the next invocation found a + // position pointing at nothing and exported from the first page. + if (RefusedTempIsOnDisk(_refusedCheckpointTemp, outputPath)) + { + WriteVerbose( + $"Left the temp files beside '{outputPath}' alone: '{_refusedCheckpointTemp}' " + + "holds the items of the checkpoint this run refused."); + } + else if (_keptTempPath != null && File.Exists(_keptTempPath)) + { + WriteVerbose( + $"Left the temp files beside '{outputPath}' alone: " + + $"'{Path.GetFileName(_keptTempPath)}' holds the items of this run's own " + + "checkpoint, which a previous attempt left on disk."); + } + else + { + DeleteStaleTemps(outputPath); + } } - var writePath = append ? outputPath : $"{outputPath}.{Guid.NewGuid():N}.tmp"; + var writePath = appendToOutput ? outputPath : $"{outputPath}.{Guid.NewGuid():N}.tmp"; // What the checkpoint sites below should say about WHERE the counted items are. // A resumed run appends to the output itself, so it has no temp to name. - string? checkpointTempFile = append ? null : Path.GetFileName(writePath); + string? checkpointTempFile = appendToOutput ? null : Path.GetFileName(writePath); + // Whether the checkpoint on disk is one THIS attempt saved. A checkpoint an + // earlier attempt left names an earlier temp, and the two are not + // interchangeable when it comes to deciding what a file on disk is still for. + var savedOwnCheckpoint = false; long? checkpointDataLength = null; long itemCount = 0; // Seeded from the resume skip, not 0. PageIterator drops the skipped items before @@ -281,9 +352,31 @@ protected override void ProcessRecord() long totalWritten = resumedItemCount; long? reportedODataCount = null; + // A temp a failed attempt of this run kept is kept on one condition: a + // checkpoint counting its items is on disk. Saving a checkpoint here is what + // ends that - the file just written describes this attempt's temp, or the + // output on a resumed run, and never the earlier one - so from that moment + // nothing on disk refers to it, no recovery can reach it, and the sweep at the + // top of the next attempt is still being held off on its behalf. Left where it + // was, it outlived the export that made it: a partial copy of the caller's + // data sitting beside the finished file, waiting for some later run's sweep. + // + // Never the file this attempt is writing into, which on a resumed run is the + // output itself. Best effort otherwise: a temp something else holds open is + // not worth failing an export over. + void ReleaseKeptTemp() + { + var kept = _keptTempPath; + if (kept == null + || string.Equals(kept, writePath, StringComparison.OrdinalIgnoreCase)) + return; + _keptTempPath = null; + try { if (File.Exists(kept)) File.Delete(kept); } catch { } + } + try { - using (var writer = new StreamWriter(writePath, append)) + using (var writer = new StreamWriter(writePath, appendToOutput)) { var iterator = new PageIterator(client); @@ -309,8 +402,11 @@ protected override void ProcessRecord() ItemsCollected = totalWritten, PageItemsAlreadyWritten = 0, TempFile = checkpointTempFile, + OutputFile = Path.GetFullPath(outputPath), DataLength = checkpointDataLength }.Save(cpPath); + savedOwnCheckpoint = true; + ReleaseKeptTemp(); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { @@ -352,8 +448,11 @@ protected override void ProcessRecord() ItemsCollected = totalWritten, PageItemsAlreadyWritten = pageItemsWritten, TempFile = checkpointTempFile, + OutputFile = Path.GetFullPath(outputPath), DataLength = checkpointDataLength }.Save(cpPath); + savedOwnCheckpoint = true; + ReleaseKeptTemp(); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { @@ -369,19 +468,38 @@ protected override void ProcessRecord() } } - // Writer closed. Delete checkpoint before moving temp file. - // If crash after delete but before move: temp file orphaned, user re-runs fresh. - if (cpPath != null) PaginationCheckpoint.Delete(cpPath); - - // Move temp file to final path on success. - if (!append) + // Writer closed. Promote the temp, and only then let go of the checkpoint. + // + // The move is the one step that can still fail with every page already + // fetched: the output held open by another process, a read-only + // destination, a share that dropped between the last write and the rename. + // Deleting the checkpoint first made that failure destroy the run - the + // catch below read "no checkpoint" as "nothing to resume", deleted the temp + // holding every item, and the next invocation enumerated from the first + // page. This way round a failed move leaves the checkpoint and the temp on + // disk exactly as an interruption does, and the next run promotes the temp + // and resumes from the last page the checkpoint recorded. + if (!appendToOutput) { File.Move(writePath, outputPath, overwrite: true); } + + // The items are in the output now, so the position describing them is + // spent. A crash in the gap costs a re-enumeration: the checkpoint names a + // temp that is no longer there, which recovery reads as unusable. + if (cpPath != null) PaginationCheckpoint.Delete(cpPath); + + // And with it goes the last thing that could have named a temp an earlier + // attempt kept - the attempt that finished need never have saved a + // checkpoint of its own to release it, if everything it had left fitted in + // one page. Only here, on the way out of a run that promoted: a promotion + // that failed leaves the checkpoint and the temp for the next run to resume + // from, and both are still exactly what that run needs. + ReleaseKeptTemp(); } catch (Exception attemptEx) { - if (!append) + if (!appendToOutput) { // User cancellation of a checkpointed fresh run: promote the temp // file (the using block already flushed it on unwind) and save a @@ -410,8 +528,10 @@ protected override void ProcessRecord() ItemsCollected = totalWritten, PageItemsAlreadyWritten = pageItemsWritten, TempFile = null, + OutputFile = Path.GetFullPath(outputPath), DataLength = promotedLength }.Save(cpPath); + ReleaseKeptTemp(); } catch (Exception promoteEx) when (promoteEx is IOException or UnauthorizedAccessException) { @@ -429,9 +549,24 @@ protected override void ProcessRecord() // is the common way a long export dies. Keep the temp for the next // run to promote; it is deleted by promotion or by the stale-temp // sweep once the checkpoint is gone. - var resumable = cpPath != null && File.Exists(cpPath); - if (!resumable) + // + // The checkpoint counting them has to be one this attempt saved. + // Any other is an earlier attempt's, naming an earlier temp, and + // this attempt's items are then counted by nothing: keeping the + // file left a partial page no recovery can reach, and the newest + // temp beside an output is what the pre-length adoption path picks + // up on a line count alone. + var resumable = cpPath != null && File.Exists(cpPath) && savedOwnCheckpoint; + if (resumable) { + // Named for the sweep above, which the next attempt reaches + // before this checkpoint has been resumed from or replaced. + _keptTempPath = writePath; + } + else + { + // Whatever is named stays named: the checkpoint on disk is + // still the earlier attempt's, and so is the temp it counts. try { if (File.Exists(writePath)) File.Delete(writePath); } catch { } } } @@ -560,9 +695,182 @@ private string BuildUrl(bool includeCount, bool suppressTop = false) BuildRequestHeaders(ConsistencyLevel, Headers); /// - /// Put the files into the state the checkpoint claims, or delete the checkpoint. A - /// checkpoint records which file its items were written to and how many bytes of that file - /// they occupy, which makes three cases decidable instead of guessed. + /// What a caller can act on when the checkpoint on disk turns out to be another export's: + /// which two files disagree, what this run does instead, and that a -CheckpointPath belongs + /// to one export. Refusing it is the whole response - the file is left alone, and so is any + /// staging file beside it, because the export that wrote it resumes from exactly those. + /// + private static string ForeignCheckpointWarning(string checkpointPath, string outputPath) => + $"The resume checkpoint at '{checkpointPath}' belongs to a different export, so it is " + + $"left as it is and '{outputPath}' is exported from the beginning. Two exports sharing " + + "one -CheckpointPath overwrite each other's resume position; give each its own."; + + /// + /// What a caller can act on when a checkpoint that records no output file is refused: it + /// was written before the output was recorded, nothing beside this one's output stands for + /// it any more, and this run exports from the beginning. Naming a different export there + /// sends the caller looking for a second run over the same -CheckpointPath, and there is + /// none to find - a temp that has since been removed and an output that has since been + /// replaced reach the same refusal. + /// + private static string UncorroboratedCheckpointWarning(string checkpointPath, string outputPath) => + $"The resume checkpoint at '{checkpointPath}' records no output file, and the files " + + $"beside '{outputPath}' no longer corroborate it, so it is left as it is and " + + $"'{outputPath}' is exported from the beginning; nothing is lost."; + + /// + /// What a caller can act on when the temp a checkpoint names is open in another run. Not + /// phrased as a fault: nothing has gone wrong with either file, two exports are writing one + /// -OutputFile, and the only thing the caller has to decide is which of them they meant to + /// keep - the one that finishes last is the one whose rows the output ends up with. + /// + private static string HeldTempWarning(string checkpointPath, string outputPath, long items) => + $"Another export is still writing the temp file the resume checkpoint at '{checkpointPath}' " + + $"names, so the {items} items it records are that run's and are not recovered here. Both " + + $"files are left as they are and '{outputPath}' is exported from the beginning. Two " + + "exports writing one -OutputFile replace each other's result; give each its own."; + + /// + /// The form of this export's URL a checkpoint on disk was written under - whether the + /// auto-added $count was on it and whether the automatic $top was - or null when no form of + /// it is this export's. The attempt loop reaches each of these in turn, dropping $count on a + /// bare 400 and $top on a Request_UnsupportedQuery, and a checkpoint records whichever URL + /// was current when it was saved. Every form is put to the same comparison the loop makes, + /// so a URL the loop can legitimately build is recognized as this export's and one it cannot + /// is still refused. Ordered as the loop reaches them, so an unambiguous match is the one + /// this run would have built first. + /// + private (bool IncludeCount, bool SuppressTop)? AttemptFormOf( + PaginationCheckpoint? checkpoint, string outputPath, bool countAutoAdded) + { + if (checkpoint == null) return null; + bool[] counts = countAutoAdded ? [true, false] : [false]; + bool[] tops = [false, true]; + foreach (var includeCount in counts) + { + foreach (var suppressTop in tops) + { + if (DescribesThisExport(checkpoint, outputPath, includeCount, suppressTop)) + return (includeCount, suppressTop); + } + } + return null; + } + + /// + /// True when a checkpoint on disk is about the export this run is making. The reconcile + /// below promotes and trims files before any other part of the checkpoint is looked at, and + /// a byte offset applies to whatever file it is handed - so a -CheckpointPath shared with a + /// second export cut a file that checkpoint knows nothing about, mid-line, and the resumed + /// pages were appended onto the torn byte. Two things have to agree: the output the writing + /// run named - the whole path, since two exports to "users.jsonl" in different directories + /// agree on the name and on nothing else, and on checkpoints too old to record one, the + /// files on disk standing in for it - and the resource, path and query both. The query is + /// half of what a resource is: -Top, -Filter and -Select change which items come back and + /// in what order, so a checkpoint recorded under one of them counts a different enumeration + /// than the one this run is making. Leaving it out put the exact comparison after the + /// promoting and trimming instead of before it, which is not a comparison that can refuse. + /// + private bool DescribesThisExport(PaginationCheckpoint checkpoint, string outputPath, + bool includeCount, bool suppressTop) + => OwnershipOf(checkpoint, outputPath, includeCount, suppressTop) == CheckpointOwnership.Mine; + + /// + /// The temp file a checkpoint this run refused still points at, or null. Refusing is the + /// whole response: the checkpoint is left where it is because the export that wrote it + /// resumes from exactly that, and the items it counted are in the temp it names. + /// + private string? _refusedCheckpointTemp; + + /// + /// The temp a failed attempt of this run left on disk for a checkpoint of its own, or null. + /// The attempt loop comes back round on a query the endpoint refused, and the sweep it + /// reaches on the way is the one thing that can take that temp away while the checkpoint + /// naming it is still there. + /// + /// It says nothing once that checkpoint has been replaced by one naming another file, or + /// deleted by the run that finished: the temp is then a partial copy of the caller's data + /// that nothing refers to, and it is deleted where the checkpoint changes rather than left + /// beside the output for a later run's sweep. + /// + private string? _keptTempPath; + + /// + /// Whether that temp is a file the stale-temp sweep would reach. The name comes off a + /// checkpoint, which is untrusted once it is on disk, and nothing here opens it - all it + /// decides is whether the sweep runs at all, so a name of some other shape is answered no + /// rather than refused: the sweep cannot delete it either, and skipping the sweep for it + /// would leave real orphans behind for nothing. + /// + /// "Some other shape" is the sweep's own test of a name and not a looser one kept here. + /// Matching on the prefix and the suffix alone answers yes for "users.jsonl.{guid}.tmp" + /// beside an output named "users" - a file the sweep passes over, because the glob's '*' + /// spans dots and the sweep's own filter does not - so a checkpoint left by the run next + /// door suppressed this output's sweep and its real orphans stayed on disk, which is the + /// case this comment says is answered no. + /// + private static bool RefusedTempIsOnDisk(string? tempFile, string outputPath) + { + if (tempFile == null || tempFile != Path.GetFileName(tempFile)) return false; + if (!IsRunTempName(Path.GetFileName(outputPath), tempFile)) return false; + var dir = Path.GetDirectoryName(outputPath); + return !string.IsNullOrEmpty(dir) && File.Exists(Path.Combine(dir, tempFile)); + } + + /// Whose the checkpoint on disk is, and where it is not this export's, why not. + private enum CheckpointOwnership + { + /// This export's. + Mine, + + /// Another export's: it records another output, or another enumeration. + AnotherExports, + + /// + /// Records no output file, and the files beside this one no longer stand for it. Which + /// export wrote it cannot be told from here, and telling the caller it was another one + /// is a diagnosis they can go and check and find nothing behind. + /// + Uncorroborated, + } + + /// + /// Which of the three the checkpoint is. A checkpoint recording neither an output nor a + /// length has nothing here to be weighed: RecordedOutputMatches needs a length to hold a + /// file against, and answering "not yours" for want of one put the whole pre-2.1.0 shape + /// behind a refusal, where the stale-temp sweep deleted the items it was pointing at. That + /// shape is decided on the evidence it does have - a temp carrying this output's own name, + /// beside an output that is not there - by the reconcile, which refuses it wherever an + /// output exists. A recorded resource that is not a URL says nothing that can be matched + /// and is refused with the enumerations that do not match. + /// + private CheckpointOwnership OwnershipOf(PaginationCheckpoint checkpoint, string outputPath, + bool includeCount, bool suppressTop) + { + if ((checkpoint.OutputFile != null || checkpoint.DataLength != null) + && !RecordedOutputMatches(checkpoint.OutputFile, checkpoint.TempFile, + checkpoint.DataLength, outputPath)) + { + return checkpoint.OutputFile != null + ? CheckpointOwnership.AnotherExports + : CheckpointOwnership.Uncorroborated; + } + + return SameResourceIdentity(ResourceIdentity(checkpoint.Resource), + ResourceIdentity(BuildUrl(includeCount, suppressTop))) + ? CheckpointOwnership.Mine + : CheckpointOwnership.AnotherExports; + } + + /// + /// Put the files into the state the checkpoint claims, or leave them alone, and answer + /// whether the run that follows appends to the output rather than exporting into a fresh + /// temp. With false nothing is written, deleted or warned about: + /// the ShouldProcess gate has to name the action before the run is allowed to take any, and + /// -WhatIf has to leave every file it found exactly as it found it. + /// + /// A checkpoint records which file its items were written to and how many bytes of that + /// file they occupy, which makes three cases decidable instead of guessed. /// /// A temp is named, so the interrupted run was fresh and its items are in that temp while /// the output still holds a PREVIOUS export. An export is a snapshot and a fresh run that @@ -579,8 +887,59 @@ private string BuildUrl(bool includeCount, bool suppressTop = false) /// has moved, so starting over costs a pass and loses nothing, while resuming past them /// would drop them from the output for good. /// - private void ReconcileCheckpointWithFiles(string checkpointPath, string outputPath, PaginationCheckpoint checkpoint) + private bool ReconcileCheckpoint(string? checkpointPath, string outputPath, + bool includeCount, bool suppressTop, bool apply) { + if (checkpointPath == null || !File.Exists(checkpointPath)) return false; + + var checkpoint = PaginationCheckpoint.Load(checkpointPath); + if (checkpoint == null) + { + // Load answers null for a checkpoint that does not deserialize and for one that + // cannot be read at all. Either way it says nothing about how much of the output is + // already there, and the run below decides to append from the file merely EXISTING - + // which appended a second complete export onto the first. A checkpoint that cannot + // be read is no checkpoint. + // + // It is left where it is. The loop below forces a fresh export whenever the load + // answers null, so deleting the file changes nothing this run does - and "cannot be + // read" covers a lock and a denying ACL as well as a torn file, so it threw away a + // position that the next run, or another account, could still have resumed from. + if (apply) + { + WriteWarning( + "The resume checkpoint could not be read, so it cannot say what the output already holds. " + + "Exporting again from the beginning; nothing is lost."); + } + return false; + } + + var ownership = OwnershipOf(checkpoint, outputPath, includeCount, suppressTop); + if (ownership != CheckpointOwnership.Mine) + { + if (apply) + { + // What the refusal leaves has to survive the sweep in this same run. + _refusedCheckpointTemp = checkpoint.TempFile; + WriteWarning(ownership == CheckpointOwnership.AnotherExports + ? ForeignCheckpointWarning(checkpointPath, outputPath) + : UncorroboratedCheckpointWarning(checkpointPath, outputPath)); + } + return false; + } + + if (checkpoint.NextLink == null) + { + // Completion marker. With the output beside it the loop below deletes it and + // exports fresh; without one it describes nothing at all. + if (!File.Exists(outputPath) && apply) + { + WriteWarning("Checkpoint found but output file is missing. Deleting stale checkpoint and starting fresh."); + PaginationCheckpoint.Delete(checkpointPath); + } + return false; + } + if (checkpoint.DataLength is not { } dataLength) { // Written before any of this was recorded. Adoption then has only a line count and @@ -588,14 +947,21 @@ private void ReconcileCheckpointWithFiles(string checkpointPath, string outputPa // output it could be merged into - exactly the case this path used to be limited to. if (!File.Exists(outputPath)) { - if (TryAdoptOrphanedTemp(outputPath, checkpoint.ItemsCollected)) - WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted export's temp file. Resuming from checkpoint."); - else + if (apply + ? TryAdoptOrphanedTemp(outputPath, checkpoint.ItemsCollected) + : CanAdoptOrphanedTemp(outputPath, checkpoint.ItemsCollected)) + { + if (apply) + WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted export's temp file. Resuming from checkpoint."); + return true; + } + + if (apply) { WriteWarning("Checkpoint found but output file is missing. Deleting stale checkpoint and starting fresh."); PaginationCheckpoint.Delete(checkpointPath); } - return; + return false; } // The output exists and the checkpoint cannot say whether its items are in it. Both @@ -606,43 +972,81 @@ private void ReconcileCheckpointWithFiles(string checkpointPath, string outputPa // enumeration onto the earlier export - a 100,000-row file coming back with 163,037. // Undecidable means start over: an export re-runs from the first page and replaces // the output, which costs a pass and cannot leave a wrong file behind. - WriteWarning( - "The resume checkpoint does not record which file the interrupted export's items are in. " - + "Exporting again from the beginning; nothing is lost."); - PaginationCheckpoint.Delete(checkpointPath); - return; + if (apply) + { + WriteWarning( + "The resume checkpoint does not record which file the interrupted export's items are in. " + + "Exporting again from the beginning; nothing is lost."); + PaginationCheckpoint.Delete(checkpointPath); + } + return false; } if (checkpoint.TempFile != null) { - if (TryPromoteNamedTemp(outputPath, checkpoint.TempFile, dataLength)) + if (apply + ? TryPromoteNamedTemp(outputPath, checkpoint.TempFile, dataLength) + : CanPromoteNamedTemp(outputPath, checkpoint.TempFile, dataLength)) { - WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted export's temp file. Resuming from checkpoint."); - // Those items are the output now. Repoint the checkpoint at it before anything - // else can fail, so a second interruption cannot promote the same temp again. - checkpoint.TempFile = null; - checkpoint.DataLength = new FileInfo(outputPath).Length; - try { checkpoint.Save(checkpointPath); } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + if (apply) { - WriteWarning($"Checkpoint save failed after recovery: {ex.Message}"); + WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted export's temp file. Resuming from checkpoint."); + // Those items are the output now. Repoint the checkpoint at it before + // anything else can fail, so a second interruption cannot promote the same + // temp again. + checkpoint.TempFile = null; + checkpoint.OutputFile = Path.GetFullPath(outputPath); + checkpoint.DataLength = new FileInfo(outputPath).Length; + try { checkpoint.Save(checkpointPath); } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + WriteWarning($"Checkpoint save failed after recovery: {ex.Message}"); + } } - return; + return true; } - WriteWarning( - $"The interrupted export's temp file is missing or incomplete, so the {checkpoint.ItemsCollected} items it " - + "recorded are not on disk. Exporting again from the beginning; nothing is lost."); - PaginationCheckpoint.Delete(checkpointPath); - return; + // The temp is there and whole, and another run has it open - so the export this + // checkpoint belongs to is not interrupted at all, it is collecting into that file + // right now. Nothing here is a recovery: the items are that run's, the checkpoint is + // the position it comes back to, and this run exports from the beginning into a temp + // of its own. Deleting the checkpoint below would be the same mistake as unlinking + // the temp, one file over. + if (NamedTempIsHeld(outputPath, checkpoint.TempFile, dataLength)) + { + if (apply) + { + // What the refusal leaves has to survive the sweep in this same run: the + // holder can be gone by the time the sweep runs, and the checkpoint left + // standing over that temp is what makes it recoverable rather than stale. + _refusedCheckpointTemp = checkpoint.TempFile; + WriteWarning(HeldTempWarning(checkpointPath, outputPath, checkpoint.ItemsCollected)); + } + return false; + } + + if (apply) + { + WriteWarning( + $"The interrupted export's temp file is missing or incomplete, so the {checkpoint.ItemsCollected} items it " + + "recorded are not on disk. Exporting again from the beginning; nothing is lost."); + PaginationCheckpoint.Delete(checkpointPath); + } + return false; } - if (!TryTrimOutputToCheckpoint(outputPath, dataLength)) + if (apply + ? TryTrimOutputToCheckpoint(outputPath, dataLength) + : CanTrimOutputToCheckpoint(outputPath, dataLength)) + return true; + + if (apply) { WriteWarning( $"'{outputPath}' no longer holds the {checkpoint.ItemsCollected} items the resume checkpoint records. " + "Exporting again from the beginning; nothing is lost."); PaginationCheckpoint.Delete(checkpointPath); } + return false; } } diff --git a/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs b/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs index a42b4c1..44d3622 100644 --- a/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs +++ b/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs @@ -1,6 +1,7 @@ using System.Collections; using System.Management.Automation; using System.Net; +using System.Net.Http.Headers; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; @@ -487,11 +488,11 @@ private void ExecuteGet(string relativeUri, string? sourceId) if (!response.IsSuccessStatusCode) { - var body = response.Content.ReadAsStringAsync(CancellationToken).GetAwaiter().GetResult(); + var body = client.ReadBodyAsStringAsync(response, CancellationToken).GetAwaiter().GetResult(); throw new GraphServiceException(response.StatusCode, body); } - var bodyBytes = response.Content.ReadAsByteArrayAsync(CancellationToken).GetAwaiter().GetResult(); + var bodyBytes = client.ReadBodyAsBytesAsync(response, CancellationToken).GetAwaiter().GetResult(); var json = ReadJsonPayload(response, bodyBytes, relativeUri); if (json != null) OutputPayload(json.Value, sourceId); @@ -546,7 +547,7 @@ private void ExecuteWrite(HttpMethod method, string relativeUri, string? sourceI if (!response.IsSuccessStatusCode) { - var body = response.Content.ReadAsStringAsync(CancellationToken).GetAwaiter().GetResult(); + var body = client.ReadBodyAsStringAsync(response, CancellationToken).GetAwaiter().GetResult(); throw new GraphServiceException(response.StatusCode, body); } @@ -556,7 +557,7 @@ private void ExecuteWrite(HttpMethod method, string relativeUri, string? sourceI // stream.Length throws NotSupportedException on network/decompression streams. // Read as bytes to safely handle null ContentLength (chunked transfer) and empty bodies. - var bodyBytes = response.Content.ReadAsByteArrayAsync(CancellationToken).GetAwaiter().GetResult(); + var bodyBytes = client.ReadBodyAsBytesAsync(response, CancellationToken).GetAwaiter().GetResult(); var jsonEl = ReadJsonPayload(response, bodyBytes, relativeUri); if (jsonEl != null) OutputPayload(jsonEl.Value, sourceId); @@ -692,34 +693,33 @@ private void ExecuteEntityFanOut(ConcurrentFanOut fanOut, List uniqueIds if (!response.IsSuccessStatusCode) { - var body = await response.Content.ReadAsStringAsync(ct); + var body = await client.ReadBodyAsStringAsync(response, ct); throw new GraphServiceException(response.StatusCode, body); } - using var stream = await response.Content.ReadAsStreamAsync(ct); - var json = await JsonSerializer.DeserializeAsync(stream, cancellationToken: ct); + // The body crosses to the cmdlet thread undecoded. ReadJsonPayload answers + // the charset, the empty body, the non-JSON body and the parse failure, and + // every one of those answers is a verbose, output or error record - none of + // which a worker thread may write. + var bodyBytes = await client.ReadBodyAsBytesAsync(response, ct); + var contentType = response.Content.Headers.ContentType; - // Clone JsonElement to detach from the parent JsonDocument's buffer. - // Without Clone(), the JsonElement holds a reference to the document's internal - // memory, which could become invalid after the response stream is disposed. - var cloned = json.Clone(); - - // Must marshal back to the cmdlet thread for WriteObject - // ConcurrentFanOut collects results; we output them after - // Store in a thread-safe structure lock (_entityFanOutResults) { - _entityFanOutResults.Add((id, cloned)); + _entityFanOutResults.Add((id, response.StatusCode, contentType, bodyBytes)); } }, CancellationToken).GetAwaiter().GetResult(); DrainClientMessages(); // Output results on the cmdlet thread - foreach (var (sourceId, json) in _entityFanOutResults) + foreach (var (sourceId, status, contentType, bodyBytes) in _entityFanOutResults) { + var json = ReadJsonPayload(status, contentType, bodyBytes, ResolveTemplate(sourceId)); + if (json == null) + continue; totalItems++; - OutputItem(json, sourceId); + OutputItem(json.Value, sourceId); } HandleFanOutErrors(errors); @@ -730,7 +730,8 @@ private void ExecuteEntityFanOut(ConcurrentFanOut fanOut, List uniqueIds } } - private readonly List<(string sourceId, JsonElement json)> _entityFanOutResults = []; + private readonly List<(string sourceId, HttpStatusCode status, + MediaTypeHeaderValue? contentType, byte[] bodyBytes)> _entityFanOutResults = []; /// /// Write fan-out: execute POST/PATCH/PUT/DELETE for each piped ID concurrently. @@ -843,7 +844,10 @@ private void HandleBulkWriteErrors(IReadOnlyList errors) } var ex = new InvalidOperationException($"HTTP {error.StatusCode} for '{error.Id}': {error.Message}"); - // StatusCode 0 = infrastructure exception (network, circuit breaker, deserialization) + // StatusCode 0 = a write that reached no HTTP status at all: the network, an open + // circuit. Not a body that stalls or does not parse - the server answered those, and + // they carry the status it answered with, so they take the branch below and read as + // what they are rather than as a request that never arrived. var (errorId, category) = error.StatusCode == 0 ? ("BulkWriteInfraError", ErrorCategory.ConnectionError) : ("BulkWriteError", MapStatusToCategory(statusCode)); @@ -1026,55 +1030,45 @@ private void NoteNonJsonStringBody() /// -Raw, otherwise an error), or a body that declares JSON and does not parse. /// private JsonElement? ReadJsonPayload(HttpResponseMessage response, byte[] bodyBytes, string relativeUri) + => ReadJsonPayload(response.StatusCode, response.Content.Headers.ContentType, bodyBytes, relativeUri); + + /// + /// The same read for a caller that no longer holds the response: the entity fan-out reads + /// the body on a worker thread and decodes it here, where writing to the streams is legal. + /// + private JsonElement? ReadJsonPayload(HttpStatusCode statusCode, MediaTypeHeaderValue? declaredType, + byte[] bodyBytes, string relativeUri) { - if (bodyBytes.Length == 0) + var payload = ReadJsonPayloadCore(declaredType, bodyBytes); + switch (payload.Kind) { - WriteVerbose($"HTTP {(int)response.StatusCode}: response has no content; nothing to emit."); - return null; - } + case JsonPayloadKind.Empty: + WriteVerbose($"HTTP {(int)statusCode}: response has no content; nothing to emit."); + return null; - var encoding = ResolveEncoding(response.Content.Headers.ContentType?.CharSet); - var contentType = response.Content.Headers.ContentType?.MediaType; - if (contentType != null && !contentType.EndsWith("json", StringComparison.OrdinalIgnoreCase)) - { - if (Raw.IsPresent) - { - WriteObject(encoding.GetString(bodyBytes)); + case JsonPayloadKind.NotJson: + if (Raw.IsPresent) + { + WriteObject(payload.Text); + return null; + } + WriteError(new ErrorRecord( + new InvalidOperationException( + $"The response is {payload.MediaType}, not JSON. Use -Raw to receive it as text, or Get-MgxContent for file and media content."), + "NonJsonResponse", ErrorCategory.InvalidData, relativeUri)); return null; - } - WriteError(new ErrorRecord( - new InvalidOperationException( - $"The response is {contentType}, not JSON. Use -Raw to receive it as text, or Get-MgxContent for file and media content."), - "NonJsonResponse", ErrorCategory.InvalidData, relativeUri)); - return null; - } - // Through a string, not the raw bytes: honors the declared charset. GetString keeps - // a BOM as U+FEFF, which the serializer refuses but the old stream path accepted - - // trim it explicitly. - var text = encoding.GetString(bodyBytes).TrimStart('\uFEFF'); - try - { - return JsonSerializer.Deserialize(text); - } - catch (JsonException) - { - var snippet = text[..Math.Min(text.Length, 200)]; - WriteError(new ErrorRecord( - new InvalidOperationException( - $"The response declared JSON but does not parse. Body starts: {snippet}"), - "MalformedJsonResponse", ErrorCategory.InvalidData, relativeUri)); - return null; - } - } + case JsonPayloadKind.Malformed: + var snippet = payload.Text[..Math.Min(payload.Text.Length, 200)]; + WriteError(new ErrorRecord( + new InvalidOperationException( + $"The response declared JSON but does not parse. Body starts: {snippet}"), + "MalformedJsonResponse", ErrorCategory.InvalidData, relativeUri)); + return null; - /// The response's declared charset, defaulting to UTF-8. GetString strips a - /// matching BOM, which the byte-level Deserialize refused. - private static Encoding ResolveEncoding(string? charset) - { - if (string.IsNullOrEmpty(charset)) return Encoding.UTF8; - try { return Encoding.GetEncoding(charset.Trim('"')); } - catch (ArgumentException) { return Encoding.UTF8; } + default: + return payload.Json; + } } private void OutputPayload(JsonElement json, string? sourceId) @@ -1161,12 +1155,115 @@ private void OutputItem(JsonElement element, string? sourceId) Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, Converters = { + // Claims [Flags] enums only; every other enum falls through to the converter below. + new GraphFlagsEnumConverter(), new JsonStringEnumConverter(JsonNamingPolicy.CamelCase), new GraphDurationConverter(), new GraphDateTimeConverter(), }, }; + /// + /// OData spells a flags combination "ignoreCase,multiline"; JsonStringEnumConverter writes + /// ", " between the names, which Graph will not parse back into a flags-typed enum. + /// + /// Only that multi-name form is rewritten, and only its separator. Each name in it is + /// written by the converter STJ would otherwise have used, one member at a time, so a + /// combination spells its members exactly as they spell themselves alone. Resolving them + /// here from Enum.ToString did not: it picks the other name when two members share a value, + /// and it cannot see [JsonStringEnumMemberName], so a member sent as "read-only" alone went + /// out as "read" in a combination, which is not a name the service was ever given. + /// + /// Everything else - a single member, an alias, a combination with a name of its own, a + /// value with bits no member covers - is handed to that converter whole, so name resolution + /// and the numeric fallback stay byte-for-byte what they were. Formatting the number here + /// instead would carry the current culture's negative sign, which sv-SE writes as U+2212. + /// + /// Components are taken largest first, so a member covering several bits wins over the bits + /// themselves, and are written in ascending value order: {A=1,B=2,C=4,BC=6} at 7 is "a,bc". + /// + private sealed class GraphFlagsEnumConverter : JsonConverterFactory + { + public override bool CanConvert(Type typeToConvert) + => typeToConvert.IsEnum && typeToConvert.IsDefined(typeof(FlagsAttribute), inherit: false); + + public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options) + => (JsonConverter)Activator.CreateInstance( + typeof(FlagsConverter<>).MakeGenericType(typeToConvert), + new JsonStringEnumConverter(JsonNamingPolicy.CamelCase).CreateConverter(typeToConvert, options))!; + + private sealed class FlagsConverter : JsonConverter where T : struct, Enum + { + /// Every declared value with bits of its own, largest first. + private static readonly ulong[] s_members = Enum.GetValues() + .Select(ToBits).Where(bits => bits != 0).Distinct().OrderByDescending(bits => bits).ToArray(); + + private readonly JsonConverter _inner; + + public FlagsConverter(JsonConverter inner) => _inner = inner; + + public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + => _inner.Read(ref reader, typeToConvert, options); + + public override T ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + => _inner.ReadAsPropertyName(ref reader, typeToConvert, options); + + public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) + { + if (JoinNames(value, options) is { } joined) writer.WriteStringValue(joined); + else _inner.Write(writer, value, options); + } + + public override void WriteAsPropertyName(Utf8JsonWriter writer, T value, JsonSerializerOptions options) + { + if (JoinNames(value, options) is { } joined) writer.WritePropertyName(joined); + else _inner.WriteAsPropertyName(writer, value, options); + } + + /// The comma-joined form, or null when the value is not a multi-name one. + private string? JoinNames(T value, JsonSerializerOptions options) + { + var remaining = ToBits(value); + var names = new List(); + + foreach (var member in s_members) + { + if (remaining == 0) break; + if ((remaining & member) != member) continue; + if (NameOf(member, options) is not { } name) return null; + + // Chosen largest first, written smallest first. + names.Insert(0, name); + remaining &= ~member; + } + + // One name is not this converter's business - a lone member, an alias, a + // combination named in its own right, zero - and neither is a value with bits + // left over, which has no names at all and belongs in the numeric fallback. + return remaining == 0 && names.Count > 1 ? string.Join(",", names) : null; + } + + /// What the inner converter writes for one member alone, unquoted. + private string? NameOf(ulong member, JsonSerializerOptions options) + { + var buffer = new System.Buffers.ArrayBufferWriter(initialCapacity: 32); + using (var writer = new Utf8JsonWriter(buffer)) + _inner.Write(writer, (T)Enum.ToObject(typeof(T), member), options); + + var reader = new Utf8JsonReader(buffer.WrittenSpan); + return reader.Read() && reader.TokenType == JsonTokenType.String ? reader.GetString() : null; + } + + /// The value's bits, sign-extended rather than refused for a signed enum. + private static ulong ToBits(T value) => ((IConvertible)value).GetTypeCode() switch + { + TypeCode.Byte or TypeCode.UInt16 or TypeCode.UInt32 or TypeCode.UInt64 + => ((IConvertible)value).ToUInt64(provider: null), + _ => unchecked((ulong)((IConvertible)value).ToInt64(provider: null)), + }; + } + } + /// Edm.Duration is ISO-8601 ("PT1H"); STJ's default "01:00:00" is refused. private sealed class GraphDurationConverter : JsonConverter { diff --git a/src/Mgx.Engine/Http/AdaptiveRequestPacer.cs b/src/Mgx.Engine/Http/AdaptiveRequestPacer.cs index 3667f39..a23f041 100644 --- a/src/Mgx.Engine/Http/AdaptiveRequestPacer.cs +++ b/src/Mgx.Engine/Http/AdaptiveRequestPacer.cs @@ -429,7 +429,9 @@ internal static double LastThrottlePercentage if (s_latencyBaselineMs[i] > 0) { var ratio = (double)s_lastLatencyMs[i] / s_latencyBaselineMs[i]; - (facts ??= []).Add($"latency {s_lastLatencyMs[i]}ms ({ratio:0.0}x of {s_latencyBaselineMs[i]}ms baseline)"); + // Parsed by the benchmark harness, so the separator must not follow the ambient culture. + var ratioText = ratio.ToString("0.0", CultureInfo.InvariantCulture); + (facts ??= []).Add($"latency {s_lastLatencyMs[i]}ms ({ratioText}x of {s_latencyBaselineMs[i]}ms baseline)"); } } if (facts != null) diff --git a/src/Mgx.Engine/Http/GraphBatchClient.cs b/src/Mgx.Engine/Http/GraphBatchClient.cs index d45bfa8..c73d6a5 100644 --- a/src/Mgx.Engine/Http/GraphBatchClient.cs +++ b/src/Mgx.Engine/Http/GraphBatchClient.cs @@ -52,6 +52,24 @@ public sealed class GraphBatchClient /// public Dictionary? ItemHeaders { get; set; } + /// + /// Runs once a parallel run has stopped sending: a chunk's own POST failed, the failure is + /// recorded and the stop its siblings read is canceled. Null on every shipping path: the + /// suite holds a wire answer here so that what a sibling does next happens on the far side + /// of the stop, rather than a moment either way of it. Per client rather than static, so a + /// test arms only the run it built. + /// + internal Action? SendingStopped { get; set; } + + /// + /// Runs as a chunk enters the wait between two of its own attempts, with the stop already + /// linked into that wait. Null on every shipping path: the suite answers a peer chunk here + /// so a refusal lands on a backoff that is running, which is the only state the question of + /// whether the stop ends it can be asked from. Per client rather than static, so a test arms + /// only the run it built. + /// + internal Action? RetryWaitStarted { get; set; } + private readonly int _batchChunkConcurrency; private readonly int _batchItemsPerSecond; @@ -127,13 +145,124 @@ public async Task> ExecuteBatchAsync( /// retryable status codes are retried once more as a follow-up batch. /// /// - /// Status recorded for an operation that was never sent, because a chunk before it failed. - /// Distinct from any HTTP status so a caller can tell "not attempted" from "attempted and - /// refused" - the difference between a write that may have landed and one that certainly - /// did not. + /// Status recorded for an operation that was never sent, because another chunk failed + /// first - not necessarily one before it, since chunks can run in parallel. Distinct from + /// any HTTP status so a caller can tell "not attempted" from "attempted and refused" - the + /// difference between a write that may have landed and one that certainly did not. /// public const int NotSentStatus = 0; + /// + /// Status recorded for the items of a chunk whose own POST failed without a status of its + /// own - a stalled body, an open circuit. The request went out, so those writes may have + /// been applied; NotSentStatus would claim the opposite. + /// + private const int RefusedWithoutStatus = 503; + + /// + /// What a chunk spent on its way to whatever happened to it. Held by the caller of the + /// attempt loop rather than returned from it, so a chunk that fails still reports what it + /// cost - the throttles it met are the reason it failed, not a detail lost with it. + /// + private sealed class ChunkAttemptCounts + { + public int ItemRetries; + public int ThrottleEncounters; + public long RetryDelayMs; + } + + /// + /// A chunk's own $batch call failed. Carries the per-item results the earlier attempts of + /// that same chunk already collected - the server answered those items, and after the + /// failure they are the only record of what it applied - and what those attempts cost. + /// + private sealed class ChunkSendFailedException : Exception + { + public ChunkSendFailedException( + Exception cause, + (BatchOperation Operation, GraphBatchResponseItem Response)?[] partial, + int refusedStatus, + ChunkAttemptCounts counts) + : base(cause.Message, cause) + { + Cause = cause; + Partial = partial; + Counts = counts; + // Below 400 is not a status a refused item may carry: a success is counted in + // [200,400) and the dead-letter file keeps nothing under 400, so a 304 from a proxy + // or a captive portal reported twenty refused writes as twenty applied ones. Graph + // answers $batch with 200 or an error; anything else came from something between. + RefusedStatus = refusedStatus >= 400 ? refusedStatus : RefusedWithoutStatus; + } + + /// The failure itself, as the caller is to see it. + public Exception Cause { get; } + + /// By chunk-relative index; null where no sub-response arrived. + public (BatchOperation Operation, GraphBatchResponseItem Response)?[] Partial { get; } + + /// Status for the items of this chunk that have no sub-response. + public int RefusedStatus { get; } + + /// The retries, throttles and delay this chunk had spent when it failed. + public ChunkAttemptCounts Counts { get; } + } + + /// + /// Writes a refused chunk's own slots: an item the server answered keeps that answer, the + /// rest carry the refusal. The POST went out for all of them, so none of them may be + /// reported as never sent - the caller has to be able to tell a write that may have landed + /// from one that certainly did not. + /// + private static void FillRefusedChunk( + (BatchOperation Operation, GraphBatchResponseItem Response)?[] results, + BatchOperation[] chunk, int offset, ChunkSendFailedException failure) + { + for (int i = 0; i < chunk.Length; i++) + { + var answered = i < failure.Partial.Length ? failure.Partial[i] : null; + results[offset + i] = answered ?? (chunk[i], new GraphBatchResponseItem + { + Id = (offset + i).ToString(), + Status = failure.RefusedStatus + }); + } + } + + /// + /// Writes back what the batch-level retry pass got answers for before its own POST was + /// refused. Every item the pass carries already holds a status the first pass gave it, so + /// unlike a refused first-pass chunk there is nothing to fill in for the rest: an item this + /// pass got no answer of its own for keeps the one it came in with. Left out, an item the + /// pass confirmed is reported with the failure that sent it here - for a write the server + /// applied, that is an instruction to apply it a second time. + /// + private static void FillAnsweredRetryChunk( + (BatchOperation Operation, GraphBatchResponseItem Response)?[] results, + IReadOnlyList<(int OriginalIndex, BatchOperation Op)> candidates, int offset, + ChunkSendFailedException failure) + { + for (int i = 0; i < failure.Partial.Length; i++) + { + var answered = failure.Partial[i]; + if (answered != null) + results[candidates[offset + i].OriginalIndex] = answered; + } + } + + /// + /// A failed chunk's retries, throttles and delay. They are the run's, not the chunk's: the + /// chunk that met the throttles is often the one that then failed, and reading them only + /// off the chunks that succeeded reported a batch throttled into refusal as an untroubled + /// one that happened to end badly. + /// + private static void RecordChunkCosts(BatchTelemetry telemetry, ChunkSendFailedException failure) + { + telemetry.AddItemRetries(failure.Counts.ItemRetries); + telemetry.AddThrottleEncounters(failure.Counts.ThrottleEncounters); + telemetry.AddRetryDelayMs(failure.Counts.RetryDelayMs); + } + public async Task ExecuteBatchIndexedAsync( IReadOnlyList operations, CancellationToken cancellationToken = default) @@ -240,21 +369,23 @@ public async Task ExecuteBatchIndexedAsync( (chunkResults, throttleDelay, chunkRetries, chunkThrottles, chunkRetryDelayMs) = await SendBatchWithRetryAsync(chunk, cancellationToken); } - // Only an HTTP-level failure of the chunk itself. A malformed envelope - wrong - // response count, unknown ids - is a protocol violation rather than a chunk that - // did not land, and it still throws: nothing about the run can be trusted after it. - catch (GraphServiceException ex) + // Any way this chunk's own call can fail, an envelope that answers nothing that + // was sent included. Each of them is a failure of this chunk and is reported as + // one; none of them is a reason to lose the chunks applied before it. + catch (ChunkSendFailedException ex) { // The chunks before this one were applied on the server. Letting this // exception out discards their results, which is the only record of what // landed - and for writes that is the thing the caller most needs. Stop // here, keep them, and hand the failure back alongside. - // - // The unsent slots are filled rather than dropped: callers line results up - // against their own input list by position, so a short list would misattribute - // every error after the gap. - chunkFailure = ex; - for (int i = globalOffset; i < operations.Count; i++) + chunkFailure = ex.Cause; + RecordChunkCosts(telemetry, ex); + FillRefusedChunk(results, chunk, globalOffset, ex); + + // Only the chunks after this one were never sent. The unsent slots are filled + // rather than dropped: callers line results up against their own input list + // by position, so a short list would misattribute every error after the gap. + for (int i = globalOffset + chunk.Length; i < operations.Count; i++) { notSent.Add(operations[i]); results[i] = (operations[i], new GraphBatchResponseItem @@ -322,16 +453,30 @@ public async Task ExecuteBatchIndexedAsync( } using var semaphore = new SemaphoreSlim(_batchChunkConcurrency); + Exception? parallelFailure = null; + // Cancelled when a chunk's own POST fails. The chunks still running read it to stop + // before their next attempt, and wait on it alongside their backoff - a chunk part + // way through a Retry-After has stopped sending, and holding the call open for the + // rest of a delay it will not act on serves nobody. + using var stopSource = new CancellationTokenSource(); var tasks = chunks.Select(async (chunk, chunkIndex) => { bool acquired = false; try { + // A chunk has already failed its own POST. The sequential path stops there + // instead of sending the rest, and so does this one: what is in flight + // finishes and is kept, nothing further goes out. Read before queueing as + // well as after acquiring - a chunk that waited out the queue must not send + // on the strength of a check it made before the refusal happened. + if (Volatile.Read(ref parallelFailure) != null) return; await semaphore.WaitAsync(cancellationToken); acquired = true; + if (Volatile.Read(ref parallelFailure) != null) return; var (chunkResults, _, chunkRetries, chunkThrottles, chunkRetryDelayMs) = - await SendBatchWithRetryAsync(chunk, cancellationToken); + await SendBatchWithRetryAsync(chunk, cancellationToken, + stopRequested: stopSource.Token); telemetry.AddItemRetries(chunkRetries); telemetry.AddThrottleEncounters(chunkThrottles); telemetry.AddRetryDelayMs(chunkRetryDelayMs); @@ -340,6 +485,22 @@ public async Task ExecuteBatchIndexedAsync( for (int i = 0; i < chunkResults.Count; i++) results[offset + i] = chunkResults[i]; } + // A failure of the chunk's own call, as in the sequential branch. Task.WhenAll + // would rethrow it and take the populated results array with it - the results of + // the chunks that DID land, which for writes is the record the caller most needs. + catch (ChunkSendFailedException ex) + { + // Chunks fail concurrently, and only the first can be the one failure the + // caller is handed. The others are still things that happened to the run, + // and dropping them leaves a chunk whose items all failed with no statement + // anywhere of why. + if (Interlocked.CompareExchange(ref parallelFailure, ex.Cause, null) != null) + _pendingVerbose.Enqueue($"Batch chunk {chunkIndex} also failed: {ex.Cause.Message}"); + stopSource.Cancel(); + SendingStopped?.Invoke(); + RecordChunkCosts(telemetry, ex); + FillRefusedChunk(results, chunk, chunkOffsets[chunkIndex], ex); + } finally { if (acquired) semaphore.Release(); @@ -347,23 +508,49 @@ public async Task ExecuteBatchIndexedAsync( }).ToArray(); await Task.WhenAll(tasks); + + if (parallelFailure != null) + { + chunkFailure = parallelFailure; + // A failed chunk filled its own slots above, so every slot still empty belongs + // to a chunk that was never sent. Filled rather than dropped, for the same reason + // as above: callers line results up against their own input list by position. + for (int i = 0; i < results.Length; i++) + { + if (results[i] != null) continue; + notSent.Add(operations[i]); + results[i] = (operations[i], new GraphBatchResponseItem + { + Id = i.ToString(), + Status = NotSentStatus + }); + } + } } // Batch-level retry for items that exhausted per-chunk retries // but have retryable status codes. Throttle pressure may have subsided. + // + // Not after a chunk failure. The run stopped sending and this pass is a send, so it is + // skipped for every candidate rather than for the refused chunk's own. For a candidate + // a refusal carried, resending is the duplicate write the stop exists to prevent; for + // one a chunk that was POSTed and answered in full left failing, it is an extra attempt + // the run no longer has. Both are handed back as they stand, for the caller to resubmit + // knowing what it is resubmitting. var failedRetryable = new List<(int OriginalIndex, BatchOperation Op)>(); - for (int i = 0; i < results.Length; i++) + if (chunkFailure is null) { - if (results[i] == null) continue; - var (op, response) = results[i]!.Value; - if (IsRetryable(response.Status, op.Method)) - failedRetryable.Add((i, op)); + for (int i = 0; i < results.Length; i++) + { + if (results[i] == null) continue; + var (op, response) = results[i]!.Value; + if (IsRetryable(response.Status, op.Method)) + failedRetryable.Add((i, op)); + } } if (failedRetryable.Count > 0) { - telemetry.BatchLevelRetries = failedRetryable.Count; - // Backpressure delay before the batch-level retry pass. // Minimum 2s pause to let throttle pressure subside before retrying failed items. var backpressureDelay = 2; @@ -402,8 +589,35 @@ public async Task ExecuteBatchIndexedAsync( } var p2ChunkSw = Stopwatch.StartNew(); - var (chunkResults, chunkThrottleDelay, chunkRetries, chunkThrottles, chunkRetryDelayMs) = - await SendBatchWithRetryAsync(chunk, cancellationToken); + IReadOnlyList<(BatchOperation Operation, GraphBatchResponseItem Response)> chunkResults; + int chunkThrottleDelay, chunkRetries, chunkThrottles; + long chunkRetryDelayMs; + // Counted as each chunk goes out, for the same reason item retries are: the + // pass can be refused on its first chunk, and the candidates queued behind that + // chunk are then never POSTed. Read off the candidate list before the pass ran, + // the count reported forty batch-level retries over a wire that carried twenty. + // Here rather than after the call, so a chunk the server refused still counts - + // its POST went out, and what it carried may have been applied. + telemetry.BatchLevelRetries += chunk.Length; + try + { + (chunkResults, chunkThrottleDelay, chunkRetries, chunkThrottles, chunkRetryDelayMs) = + await SendBatchWithRetryAsync(chunk, cancellationToken); + } + // This pass exists to improve results that are already in hand. Its own POST + // failing does not unmake them: the items it carried keep the statuses the + // server gave them, which is what the caller has to act on. Thrown, it takes + // the whole array with it - every chunk that landed included - and the run ends + // with nothing said about writes the server applied. What its earlier attempts + // did get answers for is written back before it stops: those items were answered + // in this pass, and a later attempt of it being refused does not unmake that. + catch (ChunkSendFailedException ex) + { + chunkFailure = ex.Cause; + RecordChunkCosts(telemetry, ex); + FillAnsweredRetryChunk(results, failedRetryable, retryOffset, ex); + break; + } phase2PrevChunkMs = p2ChunkSw.ElapsedMilliseconds; phase2ThrottleDelay = chunkThrottleDelay; telemetry.AddItemRetries(chunkRetries); @@ -430,8 +644,10 @@ public async Task ExecuteBatchIndexedAsync( var finalResults = results.Select(r => r!.Value).ToList(); - // Compute success/failure counts and total elapsed - telemetry.Succeeded = finalResults.Count(r => r.Response.Status < 400); + // Compute success/failure counts and total elapsed. An operation that was never sent has + // no status and is neither: counting it as a success reported a run that stopped after + // one chunk as one in which everything worked. + telemetry.Succeeded = finalResults.Count(r => r.Response.Status is >= 200 and < 400); telemetry.Failed = finalResults.Count(r => r.Response.Status >= 400); telemetry.TotalElapsedMs = batchSw.ElapsedMilliseconds; @@ -457,27 +673,88 @@ public async Task ExecuteBatchIndexedAsync( /// ItemRetries: total individual item retries in this chunk. /// ThrottleEncounters: number of 429 responses seen across all attempts. /// RetryDelayMs: total milliseconds spent in retry delays within this chunk. + /// , once cancelled, ends the chunk before its next attempt + /// goes out and ends any backoff it is already waiting out. The items still pending keep + /// the last status the server gave them. /// private async Task<(IReadOnlyList<(BatchOperation Operation, GraphBatchResponseItem Response)> Results, int ThrottleDelaySeconds, int ItemRetries, int ThrottleEncounters, long RetryDelayMs)> SendBatchWithRetryAsync( BatchOperation[] operations, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + CancellationToken stopRequested = default) { - // Track results by original index position + // Results by original index position, held out here so that however the send fails it + // fails with them: they are the record of what the server applied, and no shape of + // failure is a reason to discard it. var results = new (BatchOperation Operation, GraphBatchResponseItem Response)?[operations.Length]; + // Held out here for the same reason as the results: a chunk that fails has still spent + // its retries and met its throttles, and the run's telemetry is where that is reported. + var counts = new ChunkAttemptCounts(); + try + { + return await SendChunkAttemptsAsync(operations, results, counts, cancellationToken, stopRequested); + } + // The caller cancelled. Not a chunk failure, and reporting it as one would bury it. + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (ChunkSendFailedException) + { + throw; + } + // Every other shape - a stalled response body, an open circuit, an envelope that does + // not answer what was sent - is this chunk's failure and not the whole run's. Letting + // it out would take the results array with it, and every other chunk's outcome in it. + catch (Exception ex) + { + throw new ChunkSendFailedException(ex, results, RefusalStatus(ex), counts); + } + } + + /// + /// The status the items of a failed chunk take when the failure carries none of its own. + /// + private static int RefusalStatus(Exception ex) => ex switch + { + GraphServiceException graph => (int)graph.StatusCode, + HttpRequestException { StatusCode: { } status } => (int)status, + _ => RefusedWithoutStatus, + }; + + private async Task<(IReadOnlyList<(BatchOperation Operation, GraphBatchResponseItem Response)> Results, int ThrottleDelaySeconds, int ItemRetries, int ThrottleEncounters, long RetryDelayMs)> SendChunkAttemptsAsync( + BatchOperation[] operations, + (BatchOperation Operation, GraphBatchResponseItem Response)?[] results, + ChunkAttemptCounts counts, + CancellationToken cancellationToken, + CancellationToken stopRequested) + { // Track which indices are still pending var pendingIndices = Enumerable.Range(0, operations.Length).ToList(); // Track last-seen throttle delay for cross-chunk backpressure int lastThrottleDelaySeconds = 0; - int totalItemRetries = 0; - int totalThrottleEncounters = 0; - long totalRetryDelayMs = 0; for (int attempt = 0; attempt <= MaxPerRequestRetries; attempt++) { if (pendingIndices.Count == 0) break; + // A chunk that was already sending when another chunk was refused must not send + // again: after the refusal the run is stopping, and a POST into it is a write + // nothing will report on. Read at the first attempt as well as the later ones - + // the refusal can land in the window between the permit and the first POST. + if (stopRequested.IsCancellationRequested) + { + // At the first attempt nothing has gone out at all. Returning no results leaves + // the chunk's slots to the run's never-sent handling, which is what happened to + // it; a status here would claim its writes were attempted. + if (attempt == 0) return ([], 0, 0, 0, 0); + break; + } var batchRequest = new GraphBatchRequest(); var idToIndex = new Dictionary(); + // What the previous attempt got for the items this one is about to re-send. Put back + // if this attempt is stopped before it reaches the wire: an attempt that never went + // out does not unmake the answer before it. + var withheld = new (BatchOperation Operation, GraphBatchResponseItem Response)?[pendingIndices.Count]; for (int i = 0; i < pendingIndices.Count; i++) { @@ -486,6 +763,12 @@ public async Task ExecuteBatchIndexedAsync( var id = (i + 1).ToString(); idToIndex[id] = originalIndex; + // In flight again, so an earlier attempt's answer is no longer the last thing + // that happened to it. If this attempt fails the item is attempted-unknown, + // not whatever it was before. + withheld[i] = results[originalIndex]; + results[originalIndex] = null; + var item = new GraphBatchRequestItem { Id = id, @@ -506,13 +789,46 @@ public async Task ExecuteBatchIndexedAsync( batchRequest.Requests.Add(item); } + // Counted here rather than when the retry was decided on: between the two the run + // can stop, and a retry counted in front of a backoff that the stop then ends is a + // request the summary claims went out over one that never did. + if (attempt > 0) counts.ItemRetries += pendingIndices.Count; + var json = JsonSerializer.Serialize(batchRequest, JsonOptions); var content = new StringContent(json, Encoding.UTF8, "application/json"); var httpSw = Stopwatch.StartNew(); // paceGate: false - this class owns batch throughput (cross-call pacing, item-level // AIMD). Running the request pacer's gate on top would stack two independent AIMD // controllers on one workload. Outer responses still feed pacer signal state. - using var response = await _client.PostAsync(_batchUrl, content, cancellationToken, paceGate: false); + // + // stopRetries carries the same stop the loop above reads. The transport retries this + // POST on its own for a throttled envelope, and it did so after the run had stopped + // sending: the chunk waited out the Retry-After the server asked for and then put + // its writes back on the wire. It reaches only the decision to send again, never the + // attempt in flight - that one finishes and keeps its answer. + HttpResponseMessage response; + try + { + response = await _client.PostAsync(_batchUrl, content, cancellationToken, + paceGate: false, stopRetries: stopRequested); + } + // The stop reached this attempt while it was still waiting for its turn to send - a + // permit from the token bucket - so nothing of it went out. The same situation as + // the check at the top of the loop, a moment later, and answered the same way. + catch (RequestNotSentException) + { + // At the first attempt nothing of the chunk has gone out at all, so its slots + // belong to the run's never-sent handling rather than to a status here. + if (attempt == 0) return ([], 0, 0, 0, 0); + + // A later attempt was counted above in front of a POST that then never left. The + // run reports the requests that went out, not the ones it had ready. + counts.ItemRetries -= pendingIndices.Count; + for (int i = 0; i < pendingIndices.Count; i++) + results[pendingIndices[i]] = withheld[i]; + break; + } + using var responseScope = response; var httpMs = httpSw.ElapsedMilliseconds; if (httpMs > 5000) _pendingVerbose.Enqueue($"Batch HTTP slow: {httpMs}ms for {pendingIndices.Count} items (attempt {attempt + 1})"); @@ -520,15 +836,20 @@ public async Task ExecuteBatchIndexedAsync( if (!response.IsSuccessStatusCode) { using var errCts = _client.CreateBodyReadCts(cancellationToken); + Exception refusal; try { var errorBody = await response.Content.ReadAsStringAsync(errCts.Token); - throw new GraphServiceException(response.StatusCode, errorBody); + refusal = new GraphServiceException(response.StatusCode, errorBody); } catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) { - throw new HttpRequestException($"Response body read timed out for error response (HTTP {(int)response.StatusCode})."); + refusal = new HttpRequestException(ResilientGraphClient.BodyReadTimedOutMessage); } + // A refusal on a later attempt does not unmake the sub-responses the earlier + // attempts got: those items were answered, and for a write that answer is the + // record of what the server did. They travel with the failure. + throw new ChunkSendFailedException(refusal, results, (int)response.StatusCode, counts); } GraphBatchResponse? batchResponse; @@ -541,7 +862,7 @@ public async Task ExecuteBatchIndexedAsync( catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) { // Body read timed out but user didn't cancel - treat as transient network error - throw new HttpRequestException("Response body read timed out. The server sent headers but the body stream stalled."); + throw new HttpRequestException(ResilientGraphClient.BodyReadTimedOutMessage); } // RD-H3: Validate batch response is not null/empty @@ -564,16 +885,31 @@ public async Task ExecuteBatchIndexedAsync( var retryIndices = new List(); int maxRetryAfterSeconds = 0; + // Sub-responses the envelope gave no status of its own. Collected rather than + // recorded, and answered for below. + List? statusless = null; foreach (var item in batchResponse.Responses) { if (!idToIndex.TryGetValue(item.Id, out var originalIndex)) continue; var op = operations[originalIndex]; + // The deserializer cannot tell a "status" member the envelope left out from one + // set to 0, and neither of those is a status a server answers with - while 0 is + // exactly NotSentStatus. A sub-response exists only for a request that went out, + // so recording it would hand back an answered write as one that certainly never + // reached the server, with a dead-letter line inviting the caller to send it + // again. Left out of the results, for the refusal below to fill. + if (item.Status <= 0) + { + (statusless ??= []).Add(item.Id); + continue; + } + // Count all 429s regardless of retry budget if (item.Status == 429) { - totalThrottleEncounters++; + counts.ThrottleEncounters++; // Parse Retry-After for delay calculation. Handles both formats: // - delta-seconds: "120" (integer seconds) @@ -598,31 +934,62 @@ public async Task ExecuteBatchIndexedAsync( } } + // Recorded even when it is about to be retried: the next attempt overwrites it, + // and if there is no next attempt this is what the server last said about the + // item - which is not the same thing as never having sent it. + results[originalIndex] = (op, item); if (IsRetryable(item.Status, op.Method) && attempt < MaxPerRequestRetries) - { retryIndices.Add(originalIndex); - } - else - { - results[originalIndex] = (op, item); - } + } + + // An envelope that answers a request without saying what happened to it has not + // answered it, and is reported the way the count and id checks above report an + // envelope that answers the wrong requests: as this chunk's failure. The items it + // did answer keep those answers, and the rest take the refusal status - which says + // the write may have landed, not that it certainly did not. + if (statusless != null) + { + throw new InvalidOperationException( + $"Graph $batch answered {statusless.Count} of {batchResponse.Responses.Count} requests " + + $"without a status (id {string.Join(", ", statusless)}) " + + $"(attempt {attempt + 1}/{MaxPerRequestRetries + 1}). " + + "The response may have been truncated or rewritten in transit."); } pendingIndices = retryIndices; - totalItemRetries += retryIndices.Count; lastThrottleDelaySeconds = Math.Max(lastThrottleDelaySeconds, maxRetryAfterSeconds); if (retryIndices.Count > 0 && attempt < MaxPerRequestRetries) { + // Waiting out a backoff for an attempt that will not be sent holds the whole + // batch open for a Retry-After the run has already stopped honoring. + if (stopRequested.IsCancellationRequested) break; + var baseDelaySeconds = maxRetryAfterSeconds > 0 ? maxRetryAfterSeconds : (int)Math.Pow(2, attempt); // C4: Add 0-50% jitter to prevent thundering herd on batch retries var jitter = baseDelaySeconds * Random.Shared.NextDouble() * 0.5; var retrySw = Stopwatch.StartNew(); - await Task.Delay(TimeSpan.FromSeconds(baseDelaySeconds + jitter), cancellationToken); - totalRetryDelayMs += retrySw.ElapsedMilliseconds; + // A refusal that lands mid-wait ends the wait. Checked only before it started, + // a chunk that stopped sending 300ms into a peer's Retry-After still held the + // call open for the rest of it - a minute, and up to the cap plus jitter. + using (var delayCts = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, stopRequested)) + { + try + { + RetryWaitStarted?.Invoke(); + await Task.Delay(TimeSpan.FromSeconds(baseDelaySeconds + jitter), delayCts.Token); + } + // The stop, not the caller's cancellation. The loop continues to the checks + // at the top of the next attempt, which end the chunk where it stands. + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + } + } + counts.RetryDelayMs += retrySw.ElapsedMilliseconds; } } @@ -636,7 +1003,8 @@ public async Task ExecuteBatchIndexedAsync( $"Batch response missing result for request at index {nullIndex} (URL: {missingOp.Url}). " + "Graph returned the expected number of responses but with non-matching IDs."); } - return (results.Select(r => r!.Value).ToList(), lastThrottleDelaySeconds, totalItemRetries, totalThrottleEncounters, totalRetryDelayMs); + return (results.Select(r => r!.Value).ToList(), lastThrottleDelaySeconds, + counts.ItemRetries, counts.ThrottleEncounters, counts.RetryDelayMs); } /// diff --git a/src/Mgx.Engine/Http/ResiliencePipelineFactory.cs b/src/Mgx.Engine/Http/ResiliencePipelineFactory.cs index fc540fd..e6ea951 100644 --- a/src/Mgx.Engine/Http/ResiliencePipelineFactory.cs +++ b/src/Mgx.Engine/Http/ResiliencePipelineFactory.cs @@ -37,6 +37,14 @@ public static class ResiliencePipelineFactory /// internal static readonly ResiliencePropertyKey?> VerboseWriterKey = new("VerboseWriter"); + /// + /// Property key for a caller's stop signal. Once canceled, no further attempt is decided + /// on for this request. It deliberately does not cancel the request: the attempt in flight + /// finishes and its answer is returned, because that answer is the only record of what the + /// server did with it. What stops is the decision to send another one. + /// + internal static readonly ResiliencePropertyKey StopRetriesKey = new("StopRetries"); + /// /// Get or create a shared resilience pipeline and rate limiter. /// Rebuilds when options change (detected by reference equality, since @@ -141,6 +149,12 @@ private static ResiliencePipeline BuildPipeline(ResilientGr MaxDelay = TimeSpan.FromSeconds(maxRetryAfterCap), ShouldHandle = args => { + // The caller has stopped sending. Asked before the classification because + // it is not about this outcome: whatever the server said, the run this + // request belongs to is not sending anything else. + if (args.Context.Properties.GetValue(StopRetriesKey, default).IsCancellationRequested) + return ValueTask.FromResult(false); + // Classification and the retry decision live in Errors/, shared with the // batch client and the download pipeline; ErrorPolicyParityTests holds // them to the decisions this predicate used to make inline. Per-attempt diff --git a/src/Mgx.Engine/Http/ResilientGraphClient.cs b/src/Mgx.Engine/Http/ResilientGraphClient.cs index 607f4b0..c2682c1 100644 --- a/src/Mgx.Engine/Http/ResilientGraphClient.cs +++ b/src/Mgx.Engine/Http/ResilientGraphClient.cs @@ -40,6 +40,27 @@ public sealed class ResilientGraphClient : IDisposable /// public TimeSpan BodyReadTimeout { get; set; } = TimeSpan.FromSeconds(30); + /// + /// Where one request is, as a single word. Entering an attempt and the caller's stop each + /// take it with one exchange, so the two orderings are the only ones that exist: the stop + /// arrives while the request is between attempts and ends the wait, or it finds the attempt + /// already on the wire and leaves it - and its answer - alone. + /// + private static class SendState + { + /// Nothing has gone out. The first attempt is the caller's own to guard. + public const int BeforeFirstAttempt = 0; + + /// An attempt is on the wire, or its answer is still being handed back. + public const int InAttempt = 1; + + /// An attempt is behind it and another may be waiting out a backoff. + public const int BetweenAttempts = 2; + + /// The stop took the request between attempts. Nothing further goes out. + public const int Stopped = 3; + } + private static readonly JsonSerializerOptions JsonOptions = new() { PropertyNameCaseInsensitive = true, @@ -89,6 +110,15 @@ public ResilientGraphClient(HttpClient httpClient, ResilientGraphClientOptions? /// public bool DebugEnabled { get; set; } + /// + /// Runs at the head of each attempt, before it claims the request, and is given the attempt + /// number. Null on every shipping path: the suite holds an attempt here to sit inside the + /// stretch between the retry decision and the claim, where a stop and the attempt about to + /// go out contend for the same word - a window too narrow to enter any other way. Per client + /// rather than static, so a test arms only the transport it built. + /// + internal Func? AttemptEntryGate { get; set; } + /// Drain buffered verbose messages. Must be called on the pipeline thread. public void DrainVerboseMessages() { @@ -152,7 +182,8 @@ public async Task SendAsync( int permitCount = 1, bool paceGate = true, bool traceResponseBody = true, - bool redirectIsSuccess = false) + bool redirectIsSuccess = false, + CancellationToken stopRetries = default) { // Buffer content bytes before pipeline so retries reconstruct fresh HttpContent. // Snapshot ALL content headers (not just ContentType) to preserve @@ -170,7 +201,58 @@ public async Task SendAsync( } RateLimitLease? lease = null; - var context = ResilienceContextPool.Shared.Get(cancellationToken); + + // A caller that has stopped sending - a batch chunk refused while its siblings are still + // going. It ends the retry loop without touching the attempt on the wire: the pipeline + // asks the stop before deciding on another attempt, and the token below ends a backoff + // that is already running, so the call comes back instead of waiting out a Retry-After + // it will not act on. Canceling the request itself would take the answer to the attempt + // in flight with it, and that answer is the record of what the server applied - the very + // thing the caller stops in order to keep. So the link is canceled only between + // attempts: never before the first, which has not gone out and is the caller's own to + // guard, and never during one. + // + // Where the request is lives in one word, and the stop and the attempt about to go out + // contend for it with an exchange each. Two words read one after the other cannot say + // where the request is: the pair can be read in a state neither of them was ever in. + var sendState = SendState.BeforeFirstAttempt; + CancellationTokenSource? stopLink = null; + var stopReg = default(CancellationTokenRegistration); + var pipelineToken = cancellationToken; + if (stopRetries.CanBeCanceled) + { + stopLink = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + pipelineToken = stopLink.Token; + stopReg = stopRetries.Register(() => + { + // One exchange decides it. Between attempts the stop takes the request and ends + // the backoff; from any other state the exchange fails and the stop is left to + // the retry predicate, which refuses the next attempt without reaching this one. + if (Interlocked.CompareExchange( + ref sendState, SendState.Stopped, SendState.BetweenAttempts) + == SendState.BetweenAttempts) + { + stopLink.Cancel(); + } + }); + } + + // The waits ahead of the first attempt: the pacer's gate and a permit from the token + // bucket. Either can park a request for seconds, and a caller that stopped sending + // during one of them cannot make the check again - the one it made before calling is + // stale by the time the wait ends, and the request would go out on the strength of it. + // The stop reaches these two waits and no others: past them the request is going out, + // and its answer is the thing the caller stopped in order to keep. + CancellationTokenSource? preSendLink = null; + var preSendToken = cancellationToken; + if (stopRetries.CanBeCanceled) + { + preSendLink = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, stopRetries); + preSendToken = preSendLink.Token; + } + + var context = ResilienceContextPool.Shared.Get(pipelineToken); + context.Properties.Set(ResiliencePipelineFactory.StopRetriesKey, stopRetries); context.Properties.Set(ResiliencePipelineFactory.IsIdempotentKey, IsIdempotent(method)); // Always set VerboseWriterKey to buffer messages (prevents stale writers from pooled contexts) context.Properties.Set(ResiliencePipelineFactory.VerboseWriterKey, @@ -181,6 +263,9 @@ public async Task SendAsync( var totalSw = Stopwatch.StartNew(); bool succeeded = false; + // The last status an attempt got, for a stop that ends the request between attempts: + // the answer is gone by then, but what it said is not. + HttpStatusCode? lastStatus = null; var bucket = AdaptivePacing.Classify(requestUri); try { @@ -189,7 +274,7 @@ public async Task SendAsync( // GraphBatchClient owns batch throughput - but their responses still feed signals. if (paceGate) { - var pacedMs = await AdaptiveRequestPacer.WaitAsync(bucket, cancellationToken); + var pacedMs = await AdaptiveRequestPacer.WaitAsync(bucket, preSendToken); if (pacedMs > 0) { MgxTelemetryCollector.Current.RecordPacingWait(pacedMs); @@ -200,7 +285,7 @@ public async Task SendAsync( if (_rateLimiter != null) { var limiterSw = Stopwatch.StartNew(); - lease = await _rateLimiter.AcquireAsync(permitCount, cancellationToken); + lease = await _rateLimiter.AcquireAsync(permitCount, preSendToken); MgxTelemetryCollector.Current.RecordRateLimiterWait(limiterSw.ElapsedMilliseconds); if (!lease.IsAcquired) throw new InvalidOperationException("Rate limit exceeded. Too many concurrent requests. Reduce -Concurrency on fan-out cmdlets, increase the queue with Set-MgxOption -RateLimitQueueLimit, or disable with Set-MgxOption -NoRateLimit."); @@ -211,6 +296,7 @@ public async Task SendAsync( async ctx => { attempt++; + if (AttemptEntryGate is { } gate) await gate(attempt); var request = new HttpRequestMessage(method, requestUri); if (contentBytes != null) { @@ -266,28 +352,63 @@ public async Task SendAsync( _pendingDebug.Enqueue(GraphRequestTracer.FormatRequest(request, contentBytes, attempt)); var httpSw = Stopwatch.StartNew(); - // Stream response headers immediately instead of buffering entire body - var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ctx.CancellationToken); - MgxTelemetryCollector.Current.RecordHttpTime(httpSw.ElapsedMilliseconds); - // Per-attempt network time feeds the latency baseline (telemetry-only: - // makes the SPO soft-clamp visible; not a pacing input in 2.1). - AdaptiveRequestPacer.RecordLatency(bucket, httpSw.ElapsedMilliseconds); - - if (DebugEnabled) + // From here to the answer is the window a stop must not touch: the request + // is going out, and canceling it would lose what the server did with it. + // The window is claimed rather than announced - a stop that took the request + // first keeps it, and this attempt ends here instead of going out on the + // strength of having overwritten the stop. + int seen; + do { - if (traceResponseBody) - { - await TraceResponseAsync(response, httpSw.ElapsedMilliseconds, ctx.CancellationToken); - } - else + seen = Volatile.Read(ref sendState); + // The stop reached the request before this attempt claimed it, so this + // one does not go out. Ended here rather than left to meet the canceled + // token below: the stop takes the word and cancels the link as two + // steps, and between them lies every instruction from here to the send. + // An attempt that read the word and went anyway is on the transport + // before the token arrives - the transport has the request and the + // cancel lands on a send already made - and the caller is then told + // that nothing further was sent over a request that was. + if (seen == SendState.Stopped) + throw new OperationCanceledException(stopRetries); + } + while (Interlocked.CompareExchange(ref sendState, SendState.InAttempt, seen) != seen); + try + { + // Stream response headers immediately instead of buffering entire body + var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ctx.CancellationToken); + lastStatus = response.StatusCode; + MgxTelemetryCollector.Current.RecordHttpTime(httpSw.ElapsedMilliseconds); + // Per-attempt network time feeds the latency baseline (telemetry-only: + // makes the SPO soft-clamp visible; not a pacing input in 2.1). + AdaptiveRequestPacer.RecordLatency(bucket, httpSw.ElapsedMilliseconds); + + if (DebugEnabled) { - // Content path: buffering a multi-megabyte download to trace it - // would defeat the streaming read. Headers-only line instead. - _pendingDebug.Enqueue(GraphRequestTracer.FormatResponse(response, httpSw.ElapsedMilliseconds, null)); + if (traceResponseBody) + { + await TraceResponseAsync(response, httpSw.ElapsedMilliseconds, ctx.CancellationToken); + } + else + { + // Content path: buffering a multi-megabyte download to trace it + // would defeat the streaming read. Headers-only line instead. + _pendingDebug.Enqueue(GraphRequestTracer.FormatResponse(response, httpSw.ElapsedMilliseconds, null)); + } } - } - return response; + return response; + } + finally + { + // One transition out, so there is no instant at which the state says the + // attempt is over while its response is still being handed back unread. + // A stop that landed during the attempt is not resumed here: the retry it + // would have stopped is already refused by the retry predicate, and + // canceling now would take the answer the attempt was kept alive for. + Interlocked.CompareExchange( + ref sendState, SendState.BetweenAttempts, SendState.InAttempt); + } }, context); // A redirect counts as success only for the caller that expects one. Graph answers @@ -319,9 +440,26 @@ public async Task SendAsync( return result; } + // The caller stopped this request's retries while it was waiting to be retried, and the + // wait ended with them. What the server last said is what the request is reported as: a + // retry that never went out does not turn a throttle into a call that got no answer. + catch (OperationCanceledException) when (stopRetries.IsCancellationRequested + && !cancellationToken.IsCancellationRequested) + { + // Nothing of this request went out: the stop reached it while it was still waiting + // for its turn to send. Said in its own words rather than as a refusal - a refusal + // means the server may have applied the writes, and whether the caller sends them + // again turns on exactly that difference. + if (Volatile.Read(ref sendState) == SendState.BeforeFirstAttempt) + throw new RequestNotSentException(); + throw new HttpRequestException(RetriesStoppedMessage, null, lastStatus); + } finally { MgxTelemetryCollector.Current.RecordRequest(succeeded, totalSw.ElapsedMilliseconds); + stopReg.Dispose(); + stopLink?.Dispose(); + preSendLink?.Dispose(); ResilienceContextPool.Shared.Return(context); lease?.Dispose(); } @@ -345,8 +483,10 @@ public Task PostAsync( CancellationToken cancellationToken = default, Dictionary? headers = null, int permitCount = 1, - bool paceGate = true) - => SendAsync(HttpMethod.Post, requestUri, content, headers, cancellationToken, permitCount, paceGate); + bool paceGate = true, + CancellationToken stopRetries = default) + => SendAsync(HttpMethod.Post, requestUri, content, headers, cancellationToken, permitCount, paceGate, + stopRetries: stopRetries); /// /// Fetch content bytes ($value / /content endpoints), optionally a byte range. @@ -380,7 +520,60 @@ public async Task GetCollectionPageAsync( } catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) { - throw new HttpRequestException("Response body read timed out. The server sent headers but the body stream stalled."); + throw new HttpRequestException(BodyReadTimedOutMessage); + } + } + + /// + /// What a caller sees when it stopped this request's retries while it was waiting to be + /// retried. Nothing further was sent; the status the exception carries is the last one the + /// server gave, so a stopped retry does not read as a request that never got an answer. + /// + internal const string RetriesStoppedMessage = + "The caller stopped retrying this request. No further attempt was sent."; + + /// + /// What a caller sees when a server sends headers and then stops sending the body. + /// + internal const string BodyReadTimedOutMessage = + "Response body read timed out. The server sent headers but the body stream stalled."; + + /// + /// Read a response body as bytes under . Requests are sent + /// with HttpCompletionOption.ResponseHeadersRead, and under that option HttpClient.Timeout + /// does not bound the content read at all - nor does the pipeline's attempt timeout, which + /// ends when the headers arrive. A read on the caller's token alone therefore never returns + /// against a stream that stalls without closing. + /// + public async Task ReadBodyAsBytesAsync( + HttpResponseMessage response, CancellationToken cancellationToken = default) + { + using var bodyCts = CreateBodyReadCts(cancellationToken); + try + { + return await response.Content.ReadAsByteArrayAsync(bodyCts.Token); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new HttpRequestException(BodyReadTimedOutMessage); + } + } + + /// + /// Read a response body as text under , for the same reason as + /// . An error body stalls exactly as a success body does. + /// + public async Task ReadBodyAsStringAsync( + HttpResponseMessage response, CancellationToken cancellationToken = default) + { + using var bodyCts = CreateBodyReadCts(cancellationToken); + try + { + return await response.Content.ReadAsStringAsync(bodyCts.Token); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new HttpRequestException(BodyReadTimedOutMessage); } } @@ -425,7 +618,17 @@ private async Task ThrowIfGraphErrorAsync(HttpResponseMessage response, Cancella { if (response.IsSuccessStatusCode) return; using var bodyCts = CreateBodyReadCts(ct); - var body = await response.Content.ReadAsStringAsync(bodyCts.Token); + string body; + try + { + body = await response.Content.ReadAsStringAsync(bodyCts.Token); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + // An error body stalls as a success body does, and the caller's contract has no + // room for the raw cancellation: it catches HttpRequestException. + throw new HttpRequestException(BodyReadTimedOutMessage); + } throw new GraphServiceException(response.StatusCode, body); } @@ -486,3 +689,18 @@ public void Dispose() // Don't dispose _httpClient either: it's owned by the caller (MgxCmdletBase). } } + +/// +/// A request the caller's stop ended before any attempt of it went out. Distinct from +/// , which ends a request that had +/// already been sent at least once and carries the status the server last gave it: this one +/// carries no status because there is nothing to report, and it is the only shape of stop +/// after which a caller can resubmit without asking whether the write already landed. +/// +internal sealed class RequestNotSentException : Exception +{ + public RequestNotSentException() + : base("The caller stopped sending before this request went out. Nothing was sent.") + { + } +} diff --git a/src/Mgx.Engine/Models/GraphResponse.cs b/src/Mgx.Engine/Models/GraphResponse.cs index 45b6662..9a488e4 100644 --- a/src/Mgx.Engine/Models/GraphResponse.cs +++ b/src/Mgx.Engine/Models/GraphResponse.cs @@ -88,6 +88,13 @@ public sealed class GraphBatchResponseItem [JsonPropertyName("id")] public string Id { get; set; } = string.Empty; + /// + /// The status the server answered this item with. The deserializer leaves it at 0 for a + /// sub-response carrying no "status" member at all, and 0 is + /// - so a value here is a status only after + /// has checked the envelope actually supplied one. Nothing + /// else may read a 0 on a deserialized item as "the request was never sent". + /// [JsonPropertyName("status")] public int Status { get; set; } diff --git a/src/Mgx.Engine/Pagination/ConcurrentFanOut.cs b/src/Mgx.Engine/Pagination/ConcurrentFanOut.cs index 25c8a7d..1237277 100644 --- a/src/Mgx.Engine/Pagination/ConcurrentFanOut.cs +++ b/src/Mgx.Engine/Pagination/ConcurrentFanOut.cs @@ -197,6 +197,10 @@ public async Task BulkWriteAsync( var tasks = operations.Select(async op => { bool acquired = false; + // The status this operation was answered with, for a failure that happens after + // the answer arrived. 0 until the server answers: the code for a write that never + // reached one. + int answered = 0; try { await semaphore.WaitAsync(cancellationToken); @@ -209,20 +213,23 @@ public async Task BulkWriteAsync( try { using var response = await _client.SendAsync(method, op.url, content, headers, cancellationToken); + answered = (int)response.StatusCode; if (!response.IsSuccessStatusCode) { - string body; + string errorMessage; using var errCts = _client.CreateBodyReadCts(cancellationToken); try { - body = await response.Content.ReadAsStringAsync(errCts.Token); + var body = await response.Content.ReadAsStringAsync(errCts.Token); + errorMessage = TryExtractGraphError(body) ?? $"HTTP {(int)response.StatusCode}"; } catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) { - body = $"Response body read timed out (HTTP {(int)response.StatusCode})"; + // Reported as it stands: a stall is not a Graph error document, and + // running it through TryExtractGraphError leaves a bare status line. + errorMessage = ResilientGraphClient.BodyReadTimedOutMessage; } - var errorMessage = TryExtractGraphError(body) ?? $"HTTP {(int)response.StatusCode}"; errorBag.Add(new BulkWriteError(op.id, (int)response.StatusCode, errorMessage)); var errCurrent = Volatile.Read(ref succeeded) + errorBag.Count; @@ -233,10 +240,16 @@ public async Task BulkWriteAsync( // Capture response body for POST/PATCH (created/updated entities) if (response.StatusCode != HttpStatusCode.NoContent) { - var bodyBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken); + var bodyBytes = await _client.ReadBodyAsBytesAsync(response, cancellationToken); if (bodyBytes.Length > 0) { - var json = JsonSerializer.Deserialize(bodyBytes); + // Read as a stream, which skips a UTF-8 byte-order mark - the way + // the batch client already reads its bodies. The byte-level overload + // refuses one, and refused it as a failure of the write: a 201 the + // server had applied came back as a status-0 transport error. + using var bodyStream = new MemoryStream(bodyBytes); + var json = await JsonSerializer.DeserializeAsync( + bodyStream, cancellationToken: cancellationToken); responseBag.Add((op.id, json)); } } @@ -259,7 +272,12 @@ public async Task BulkWriteAsync( } catch (Exception ex) { - var statusCode = ex is GraphServiceException gse ? (int)gse.StatusCode : 0; + // A body that stalls or does not parse is a failure of the read, not of the + // write: the server answered, and on a POST it may have applied it. The record + // carries the status it answered with and names the body problem, so a caller + // cannot mistake an applied write for a request that never reached the service + // and repeat it. Only a write with no answer at all is a 0. + var statusCode = ex is GraphServiceException gse ? (int)gse.StatusCode : answered; errorBag.Add(new BulkWriteError(op.id, statusCode, ex.Message)); var current = Volatile.Read(ref succeeded) + errorBag.Count; diff --git a/src/Mgx.Engine/Pagination/PaginationCheckpoint.cs b/src/Mgx.Engine/Pagination/PaginationCheckpoint.cs index ac00968..4e364df 100644 --- a/src/Mgx.Engine/Pagination/PaginationCheckpoint.cs +++ b/src/Mgx.Engine/Pagination/PaginationCheckpoint.cs @@ -37,6 +37,17 @@ public sealed class PaginationCheckpoint [JsonPropertyName("tempFile")] public string? TempFile { get; set; } + /// + /// Path of the output the writing run was collecting into. DataLength is a byte offset into + /// "the output" and nothing else here says WHICH one, so a checkpoint path used by two runs + /// let one run's offset cut the other run's file. The whole path and not the name: two + /// exports to "users.jsonl" in different directories share a name and share nothing else. + /// Null on checkpoints written before this was recorded, which say nothing about ownership + /// and are read as evidence of it only where the files on disk corroborate them. + /// + [JsonPropertyName("outputFile")] + public string? OutputFile { get; set; } + /// /// Byte length of that file at the moment this checkpoint was saved, captured after the /// writer was flushed. Null on checkpoints written before this was recorded, which are diff --git a/tests/CORPUS.md b/tests/CORPUS.md new file mode 100644 index 0000000..64f0474 --- /dev/null +++ b/tests/CORPUS.md @@ -0,0 +1,66 @@ +# Regression corpus + +Every entry below started as a real failure reported against Microsoft365DSC or the Graph +PowerShell SDK. The issue numbers are provenance, never scope: mgx does not reproduce the +reported case, it generalizes the class of failure and pins the guarantee the tests hold. +`tests/Mgx.IntegrationTests/TestSetup/HostileInputs.cs` carries the same provenance on the input +groups themselves; this file is the index over the whole suite. + +**Class** is the `MgxErrorClass` the failure surfaces as, from the classifier's own vocabulary in +`src/Mgx.Engine/Errors/MgxErrorClass.cs`. `n/a` is not a gap in the index: several of these +failures never reach the classifier at all, because nothing errors - a truncated enumeration, a +body the service accepts and misreads, a module that fails to load before a request exists. + +**Test type** is `wire` for xUnit over the mock transport, `unit` for xUnit with no transport, +`pester` for `tests/Unit`, and `live` for `tests/Live`, which needs a tenant. + +Test paths are relative to `tests/`. Where no test exists the row names the release that brings +one rather than pointing at something adjacent: an index that overstates coverage is worse than a +gap. + +## Covered + +| Issue | Source | Class | Guarantee | Test type | Subsystem | Tests | +| --- | --- | --- | --- | --- | --- | --- | +| M365DSC-7273 | Microsoft365DSC | Throttle | Sustained throttling is absorbed: Retry-After is honored, the rate halves on 429, and it climbs back afterwards. A 429 that outlives its retries reaches the caller under every `-ErrorAction`. | wire, live | pacing, retry | `Mgx.IntegrationTests/Engine/Http/RetryTests.cs`, `Mgx.IntegrationTests/Engine/Http/AdaptivePacingTests.cs`, `Mgx.IntegrationTests/Engine/Http/AdaptiveRequestPacerTests.cs`, `Mgx.IntegrationTests/Engine/Http/RetryAfterDateFormTests.cs`, `Mgx.IntegrationTests/Engine/Http/BatchPacingStateTests.cs`, `Mgx.IntegrationTests/Cmdlets/ErrorActionMatrixTests.cs` (throttling cells), `Live/Mgx.Throttle.Tests.ps1` | +| M365DSC-7274 | Microsoft365DSC | n/a | A large collection enumerates complete - through every nextLink, across page seams, and after a resume. | wire | pagination | `Mgx.IntegrationTests/Engine/Pagination/PaginationTests.cs`, `Mgx.IntegrationTests/Engine/Pagination/BoundarySizeTests.cs`, `Mgx.IntegrationTests/Engine/Pagination/CheckpointTests.cs`, `Mgx.IntegrationTests/Engine/Pagination/FanOutTests.cs`, `Mgx.IntegrationTests/Cmdlets/TopWithAllTests.cs`, `Mgx.IntegrationTests/Cmdlets/ExportCheckpointIntegrityTests.cs` | +| GraphSDK-3654 | Graph PowerShell SDK | InvalidRequest | A PowerShell wrapper never changes the wire JSON: `Serialize(x)` is byte-identical to `Serialize(PSObject.AsPSObject(x))`, at any nesting depth. | wire, unit | serialization | `Mgx.IntegrationTests/Serialization/PSWrapperEquivalenceTests.cs`, `Mgx.IntegrationTests/Serialization/WireBodyTests.cs` | +| GraphSDK-2942, GraphSDK-2709 | Graph PowerShell SDK | n/a | Hostile filter values are escaped exactly once, and the wire value decodes back to what the caller wrote. | wire | URI | `Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs`, `Mgx.IntegrationTests/TestSetup/HostileInputs.cs` (`FilterValues`) | +| GraphSDK-2488 | Graph PowerShell SDK | NotFound | A service-issued link is followed byte for byte - no re-encoding, no normalization - including across a checkpoint save and load. Validation may refuse a link, but never repairs one. | wire, unit | pagination | `Mgx.IntegrationTests/Engine/Pagination/OpaqueLinkTests.cs`, `Mgx.IntegrationTests/Engine/Pagination/SsrfValidationTests.cs`, `Mgx.IntegrationTests/TestSetup/HostileInputs.cs` (`OpaqueLinks`) | +| GraphSDK-1947 | Graph PowerShell SDK | NotFound | A `#` in a path stays in the path instead of being read as a URI fragment. | wire | URI | `Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs`, `Mgx.IntegrationTests/TestSetup/HostileInputs.cs` (`FilterValues`, `PathSegments`) | +| GraphSDK-2328 | Graph PowerShell SDK | InvalidRequest | Caller headers reach the wire as given: content headers land on the content collection rather than being dropped, and the merge is case-insensitive like HTTP header names are. | wire, unit | headers | `Mgx.IntegrationTests/Engine/Http/HeaderFidelityTests.cs`, `Mgx.IntegrationTests/Cmdlets/HeaderWireTests.cs` | +| GraphSDK-1425, GraphSDK-2088 | Graph PowerShell SDK | Permanent | A response that is not the JSON entity the caller expected - empty, HTML from a proxy, truncated - surfaces as verbose output or an error record, never an unhandled exception. | wire, unit | transport | `Mgx.IntegrationTests/Cmdlets/ResponseFidelityTests.cs`, `Mgx.IntegrationTests/Engine/ErrorBodyShapeTests.cs` | +| GraphSDK-3361 | Graph PowerShell SDK | InvalidRequest | Scalars and arrays serialize as themselves - a single-element array stays an array, a scalar does not become one. | wire, unit | serialization | `Mgx.IntegrationTests/Serialization/WireBodyTests.cs`, `Mgx.IntegrationTests/Serialization/PSWrapperEquivalenceTests.cs` | +| M365DSC-5306, M365DSC-7175 | Microsoft365DSC | InvalidRequest | A body that cannot be serialized is refused with a message naming the property path, and inside a batch it fails that item while the rest are sent. | wire | serialization, batch | `Mgx.IntegrationTests/Serialization/WireBodyTests.cs` | +| M365DSC-5354 | Microsoft365DSC | NotFound | Path segments carrying spaces, apostrophes, percent signs and non-ASCII survive into the request path unaltered. | wire | URI | `Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs`, `Mgx.IntegrationTests/TestSetup/HostileInputs.cs` (`PathSegments`) | +| M365DSC-7198 | Microsoft365DSC | NotFound | A failed batch item is an error the pipeline can see: it lands in `$Error`, counts for `-ErrorVariable`, and stops the pipeline under `-ErrorAction Stop`, with or without a dead-letter file. Results from chunks already applied are not lost when a later chunk fails. | wire | batch | `Mgx.IntegrationTests/Cmdlets/BatchErrorSurfacingTests.cs`, `Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs`, `Mgx.IntegrationTests/Cmdlets/ErrorActionMatrixTests.cs` (batch cells) | + +## Partly covered + +| Issue | Source | Class | Guarantee | Test type | Subsystem | Covered now / still planned | +| --- | --- | --- | --- | --- | --- | --- | +| M365DSC-4426 | Microsoft365DSC | Authentication | A 401 is classified as authentication rather than retried as transient, and a changed auth context re-keys the cached client instead of reusing the old token. | wire, unit | auth | Covered: classification and the fingerprint, in `Mgx.IntegrationTests/Engine/Errors/ErrorClassifierTests.cs`, `Mgx.IntegrationTests/Engine/Errors/ErrorPolicyParityTests.cs`, `Mgx.IntegrationTests/Cmdlets/Base/AuthFingerprintTests.cs`, `Mgx.IntegrationTests/Cmdlets/ResilienceInjectionScenarioTests.cs`. Planned: the mid-run expiry-and-reconnect drill - 2.1.5 fault injection. | +| GraphSDK-2148 | Graph PowerShell SDK | Permanent | mgx keeps working whatever else the session has loaded and in whatever order: no assembly-binding failure, no accidental SDK dependency. | wire | module load | Covered: the in-process half - injection, re-injection and teardown against a stand-in GraphSession, in `Mgx.IntegrationTests/Cmdlets/ResilienceInjectionScenarioTests.cs`. Covered: the load-order permutations, one child pwsh per permutation against installed Az.Accounts, PnP.PowerShell and ExchangeOnlineManagement, in `tests/Ecosystem/Mgx.Ecosystem.Tests.ps1`. | + +## Planned + +| Issue | Source | Class | Guarantee | Test type | Subsystem | Brought by | +| --- | --- | --- | --- | --- | --- | --- | +| M365DSC-7010 | Microsoft365DSC | Consistency | A read that immediately follows a write tolerates Graph's replication delay instead of reporting the object missing. | wire | consistency | 2.3.0. `MgxErrorClass.Consistency` is declared and mapped; nothing produces it yet. | + +## Unmapped + +| Issue | Source | Why it is here | +| --- | --- | --- | +| GraphSDK-2644 | Graph PowerShell SDK | No test claims this id and no failure class is recorded for it. The source issue needs reading before a row can say what it guarantees. Left in the index so the gap stays visible rather than disappearing with the row. | + +## Adding an entry + +A new provenance id gets a row here and a `(Corpus: , .)` line in the summary of +the test class that holds it, so the mapping reads the same from either end. A row whose +guarantee has no test says which release is meant to bring it. + +The mapping is guarded one way only, by +`tests/Mgx.IntegrationTests/TestSetup/RegressionCorpusIndexTests.cs`: every issue id written +anywhere under `tests/` has to appear here. The reverse cannot be guarded, because a row that +names the release bringing its test is exactly a row with no file to point at. diff --git a/tests/Ecosystem/Mgx.Ecosystem.Tests.ps1 b/tests/Ecosystem/Mgx.Ecosystem.Tests.ps1 new file mode 100644 index 0000000..9254326 --- /dev/null +++ b/tests/Ecosystem/Mgx.Ecosystem.Tests.ps1 @@ -0,0 +1,100 @@ +<# + Module load and assembly-load-context resolution are irreversible within a process, so + every permutation here runs in its own child pwsh. What a permutation can assert with no + tenant: the import works, the cmdlets resolve, the engine types JIT against whatever + assemblies the neighboring modules already loaded, and teardown leaves nothing armed. + A real Graph call under load order stays with tests/Live and the benchmarks. + + Provenance: GraphSDK-2148 (module isolation) - see tests/CORPUS.md. Az.Accounts ships its + own System.Text.Json; the interesting failure is a TypeLoadException the first time an + engine type JITs, which is why the option round-trip and telemetry calls are the probe. + + A missing module is a failure, not a skip: a suite that skips itself when unconfigured + silently stops running. CI installs all three in the ecosystem job. +#> + +BeforeDiscovery { + $script:RequiredModules = 'Az.Accounts', 'PnP.PowerShell', 'ExchangeOnlineManagement' + foreach ($m in $script:RequiredModules) { + if (-not (Get-Module -ListAvailable $m)) { + throw "Ecosystem tests need the $m module installed. Install-PSResource $m, or run the ecosystem CI job." + } + } +} + +BeforeAll { + $repo = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent + $script:Manifest = Join-Path $repo 'module/mgx.psd1' + if (-not (Test-Path $script:Manifest)) { throw "Built module not found at $script:Manifest - run ./build.ps1 first." } + + # The battery runs after the permutation's imports. Order inside matters: the side-load + # check must precede the Get-MgContext stub, and resilience and fingerprint run after the + # remove/re-import so they exercise the re-imported module, not the first import. + $script:Battery = @' +$results = [ordered]@{} +function Check([string]$name, [scriptblock]$body) { + try { $results[$name] = @{ ok = [bool](& $body); detail = '' } } + catch { $results[$name] = @{ ok = $false; detail = $_.Exception.Message } } +} +Check 'cmdletsResolve' { (Get-Command -Module mgx -CommandType Cmdlet).Count -eq 12 } +Check 'noGraphSideLoad' { @(Get-Module 'Microsoft.Graph*').Count -eq 0 } +Check 'optionRoundTrip' { Set-MgxOption -MaxRetryAttempts 4; (Get-MgxOption).MaxRetryAttempts -eq 4 } +Check 'telemetry' { $null -ne (Get-MgxTelemetry) } +Check 'depsResolve' { + # Polly and the rate limiter load lazily; Assembly.Load drives the module's own + # Resolving hook, which is the mechanism whose neighbors could break it. + $polly = [Reflection.Assembly]::Load('Polly.Core') + $rate = [Reflection.Assembly]::Load('System.Threading.RateLimiting') + $stj = [Reflection.Assembly]::Load('System.Text.Json') + ($null -ne $polly) -and ($null -ne $rate) -and ($null -ne $stj) +} +Check 'removeReimport' { + Remove-Module mgx -Force + Import-Module '__MANIFEST__' -ErrorAction Stop + (Get-Command -Module mgx -CommandType Cmdlet).Count -eq 12 -and $null -ne (Get-MgxOption) +} +Check 'resilienceNamesTheSession' { + # ThrowTerminatingError is statement-terminating; no -ErrorAction downgrades it, + # so the record arrives as a caught exception carrying the id. + try { Enable-MgxResilience; $false } + catch { $_.FullyQualifiedErrorId -like 'GraphSessionNotFound*' } +} +Check 'fingerprintFromStub' { + function global:Get-MgContext { [PSCustomObject]@{ TenantId = 'eco-tenant'; ClientId = 'eco-client'; AuthType = 'AppOnly'; Environment = 'Global' } } + $base = (Get-Command Invoke-MgxRequest).ImplementingType.BaseType + $m = $base.GetMethod('BuildAuthFingerprint', [Reflection.BindingFlags]'Static,NonPublic,Public') + $fp = $m.Invoke($null, @((Get-MgContext), $null)) + -not [string]::IsNullOrEmpty($fp) +} +[pscustomobject]@{ battery = $results } | ConvertTo-Json -Depth 4 -Compress +'@ -replace '__MANIFEST__', $script:Manifest + + $script:Permutations = [ordered]@{ + 'az-before-mgx' = @('Import-Module Az.Accounts', 'Import-Module __MANIFEST__') + 'az-after-mgx' = @('Import-Module __MANIFEST__', 'Import-Module Az.Accounts') + 'pnp-before-mgx' = @('Import-Module PnP.PowerShell', 'Import-Module __MANIFEST__') + 'pnp-after-mgx' = @('Import-Module __MANIFEST__', 'Import-Module PnP.PowerShell') + 'exo-before-mgx' = @('Import-Module ExchangeOnlineManagement', 'Import-Module __MANIFEST__') + 'exo-after-mgx' = @('Import-Module __MANIFEST__', 'Import-Module ExchangeOnlineManagement') + 'stack-mgx-last' = @('Import-Module Az.Accounts', 'Import-Module PnP.PowerShell', 'Import-Module ExchangeOnlineManagement', 'Import-Module __MANIFEST__') + 'stack-mgx-first' = @('Import-Module __MANIFEST__', 'Import-Module Az.Accounts', 'Import-Module PnP.PowerShell', 'Import-Module ExchangeOnlineManagement') + 'interleaved' = @('Import-Module Az.Accounts', 'Import-Module __MANIFEST__', 'Set-MgxOption -MaxRetryAttempts 3', 'Import-Module PnP.PowerShell', 'Import-Module ExchangeOnlineManagement') + } + + function Invoke-Permutation([string[]]$imports) { + $lines = ($imports -replace '__MANIFEST__', $script:Manifest) + $script:Battery + $out = & pwsh -NoProfile -NonInteractive -Command ($lines -join "`n") 2>&1 + $json = @($out) | Where-Object { "$_" -match '^\{' } | Select-Object -Last 1 + if (-not $json) { throw "Permutation produced no result. Output: $(($out | Select-Object -First 5) -join ' | ')" } + ($json | ConvertFrom-Json).battery + } +} + +Describe 'Ecosystem isolation' { + It 'holds under <_>' -ForEach @('az-before-mgx', 'az-after-mgx', 'pnp-before-mgx', 'pnp-after-mgx', 'exo-before-mgx', 'exo-after-mgx', 'stack-mgx-last', 'stack-mgx-first', 'interleaved') { + $battery = Invoke-Permutation $script:Permutations[$_] + $failed = $battery.PSObject.Properties | Where-Object { -not $_.Value.ok } | + ForEach-Object { "$($_.Name): $($_.Value.detail)" } + $failed -join '; ' | Should -BeNullOrEmpty + } +} diff --git a/tests/Live/Mgx.Live.Tests.ps1 b/tests/Live/Mgx.Live.Tests.ps1 index 4787a3e..7004ed0 100644 --- a/tests/Live/Mgx.Live.Tests.ps1 +++ b/tests/Live/Mgx.Live.Tests.ps1 @@ -10,7 +10,7 @@ Run: Invoke-Pester -Path ./tests/Live Needs AZURE_TENANT_ID / AZURE_CLIENT_ID / AZURE_CLIENT_CERTIFICATE_PATH for a directory - tenant, and optionally MGX_LIVE_CONTENT_* for the content tests (see tests/Live/README.md). + tenant, and optionally MGX_LIVE_CONTENT_* for the content tests. #> BeforeAll { diff --git a/tests/Live/README.md b/tests/Live/README.md deleted file mode 100644 index 41cdc79..0000000 --- a/tests/Live/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# Live tests - -The rest of the suite mocks HTTP. That proves the code does what it was written to do; it -cannot prove the request leaving the process is one Graph accepts. `Enable-MgxResilience` -shipped a wrapper client with no `BaseAddress` — every relative-URI SDK call threw before -reaching the wire — against a fully green suite, because every mocked test passed an absolute -URI. These tests exist for that class of defect. - -They are read-only. Nothing here writes to a tenant. - -## Running them - -```bash -export AZURE_TENANT_ID='...' -export AZURE_CLIENT_ID='...' -export AZURE_CLIENT_CERTIFICATE_PATH="$HOME/.certs/....pfx" -pwsh -c 'Invoke-Pester -Path ./tests/Live' -``` - -An app with a secret rather than a certificate works too - set `AZURE_CLIENT_SECRET` -instead of `AZURE_CLIENT_CERTIFICATE_PATH`. If both are set, the certificate wins. -Neither leaves anything on disk. - -Missing credentials is a **failure**, not a skip. A live suite that skips itself when -unconfigured is one that silently stops running. - -The tests size themselves from the tenant, so a 19-user directory and a 100,000-user one both -work. **Prefer a tenant with a SharePoint licence**: `Get-MgxContent` is the only cmdlet that -needs one, and without it those two tests skip, so a tenant that has it is the only place the -suite covers everything. Point it at any drive item: - -```bash -export MGX_LIVE_CONTENT_URI='/drives//items//content' -``` - -A tenant with no SharePoint answers every sites/drives call with `BadRequest: Tenant does not -have a SPO license`, and no permission grant changes that — the content roles can be present and -still inert. - -## The throttle test - -`Mgx.Throttle.Tests.ps1` covers the one thing mocks cannot: what `Enable-MgxResilience` does -when Graph is actually refusing requests. The SDK's own retry handler sits inside the wrap and -answers `429` itself, so a throttled session looks identical to a clean one — the pacer stays in -slow-start and telemetry reports zero retries while the SDK sleeps out of sight inside the call. -With that handler left armed, a caller's request eventually fails outright. - -It needs a second machine. One client cannot exhaust a tenant's budget through a single NAT'd -connection — 587 req/s from a laptop against 4,641 from a VM on the same network, same test, -same minute — and below roughly 700 req/s the tenant never refuses anything, so the run proves -nothing at all. - -```bash -export MGX_LIVE_THROTTLE_SSH='user@host' # a host with real egress -export MGX_LIVE_THROTTLE_KEY="$HOME/.ssh/id_..." # optional identity file -pwsh -c 'Invoke-Pester -Path ./tests/Live/Mgx.Throttle.Tests.ps1' -``` - -The certificate never leaves this machine: the assertion is signed locally and only the -resulting bearer token crosses, on stdin, never as an argument and never to a file. - -Unlike the rest of this directory it is **not read-only in effect** — it deliberately exhausts -the tenant's request budget and leaves it refusing requests for several minutes afterwards. Run -it last, or alone. It skips when `MGX_LIVE_THROTTLE_SSH` is unset, because most people running -this suite have no second machine to hand; when it is set, failing to reach a throttle is a -failure rather than a skip, so a green run cannot mean "there was nothing to see". - -## Keeping them honest - -`tests/Unit/Mgx.LiveCoverage.Tests.ps1` fails if an exported cmdlet has no `Describe` block -here. It parses this directory rather than running it, so it works in CI with no credentials. -Adding a cmdlet without a live test breaks the build. - -If a cmdlet genuinely cannot be exercised live, still add the `Describe` with a `-Skip` and a -reason. The point is that the decision is recorded rather than absent — which is how seven of -twelve cmdlets came to have no live coverage at all without anyone noticing. - -## What they will not catch - -They cover the happy path of each cmdlet. Failure modes, resume, throttling and accuracy are -covered by `tests/Mgx.IntegrationTests` (mocked, deterministic) and `tests/benchmarks` (live, -measured). A green run here means "the cmdlets work against Graph", not "the module is correct". diff --git a/tests/Mgx.IntegrationTests/Cmdlets/Base/AuthFingerprintTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/Base/AuthFingerprintTests.cs index df7b900..b7ab00c 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/Base/AuthFingerprintTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/Base/AuthFingerprintTests.cs @@ -7,17 +7,56 @@ namespace Mgx.IntegrationTests.Cmdlets; /// The cached Graph HttpClient is keyed on this fingerprint. Keying it on tenant id alone /// meant that reconnecting the same tenant with a different app registration silently reused /// the previous application's token, so every one of these cases is a stale-credential bug. +/// (Corpus: M365DSC-4426, stale credentials.) /// +[Collection("Pipeline")] public class AuthFingerprintTests { [Fact] - public void Reports_the_graph_sdk_as_absent_when_it_is_not_loaded() + public void A_session_with_no_context_reads_as_disconnected_not_as_a_missing_module() { - // Microsoft.Graph.Authentication is a soft dependency, so its absence is a state the - // module has to detect rather than assume away: it selects GraphAuthModuleNotLoaded - // over "run Connect-MgGraph", which would name a cmdlet the session does not have. - // This xUnit process loads no Graph assemblies, so absence is the honest answer here. - Assert.False(MgxCmdletBase.IsGraphAuthLoaded()); + // Microsoft.Graph.Authentication is a soft dependency, so "absent" and "present but + // disconnected" are different states needing different advice: naming Connect-MgGraph + // to someone who does not have the module sends them to a cmdlet that does not exist. + // + // Only the disconnected half is reachable through the cmdlet in this process: the suite + // declares a stand-in GraphSession for the resilience-injection scenarios and FindType + // resolves a type by full name, so IsGraphAuthLoaded answers true from the type alone + // whether or not a test has armed a session. The absent half is held by the test below, + // which asks the choice itself rather than the state it is made from. + using var scope = GraphSessionScope.Arm(); + + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1"); + var thrown = Record.Exception(() => ps.Invoke()); + + var ids = ps.Streams.Error.Select(e => e.FullyQualifiedErrorId).ToList(); + if (thrown is CmdletInvocationException invocation) + ids.Add(invocation.ErrorRecord.FullyQualifiedErrorId); + + Assert.Contains(ids, id => id.StartsWith("NotConnected", StringComparison.Ordinal)); + Assert.DoesNotContain(ids, id => id.StartsWith("GraphAuthModuleNotLoaded", StringComparison.Ordinal)); + } + + [Fact] + public void An_absent_module_and_a_disconnected_session_give_different_advice() + { + // Naming Connect-MgGraph to someone without the module is the failure this splits, so + // the two messages have to differ in the instruction, not only in the error id. + var disconnected = MgxCmdletBase.DescribeMissingConnection(graphAuthLoaded: true); + var absent = MgxCmdletBase.DescribeMissingConnection(graphAuthLoaded: false); + + Assert.Equal("NotConnected", disconnected.ErrorId); + Assert.Contains("Connect-MgGraph", disconnected.Message, StringComparison.Ordinal); + Assert.DoesNotContain("Install-PSResource", disconnected.Message, StringComparison.Ordinal); + + Assert.Equal("GraphAuthModuleNotLoaded", absent.ErrorId); + Assert.Contains("Install-PSResource -Name Microsoft.Graph.Authentication", + absent.Message, StringComparison.Ordinal); } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/Base/BetaHintTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/Base/BetaHintTests.cs index 5b1c48b..91c46a0 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/Base/BetaHintTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/Base/BetaHintTests.cs @@ -1,6 +1,5 @@ using System.Management.Automation; using System.Net; -using System.Reflection; using System.Text; using Mgx.Cmdlets.Base; using Mgx.Engine.Http; @@ -34,46 +33,11 @@ protected override Task SendAsync( } } - private static void InjectMock(HttpMessageHandler handler) - { - ResiliencePipelineFactory.Reset(); - var t = typeof(MgxCmdletBase); - t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, new HttpClient(handler)); - t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - // Get-MgxContent refuses to run over a transport mgx does not own (the SDK client - // auto-follows redirects, defeating download-host validation), so the mock must claim - // ownership or the cmdlet errors before any HTTP happens and the test is vacuous. - t.GetField("s_ownsHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, true); - t.GetField("s_graphEndpoint", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! - .SetValue(null, "https://graph.microsoft.com"); - var options = new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }; - t.GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! - .SetValue(null, options); - // Owning the client makes GetClient() check the cached timeout; a mismatch forces a - // rebuild that cannot succeed under a mock. Cache the value the options carry. - t.GetField("s_cachedTotalTimeoutSeconds", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, options.TotalTimeoutSeconds); - ResiliencePipelineFactory.Reset(); - } - - private static void CleanupMock() - { - var t = typeof(MgxCmdletBase); - t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - ResiliencePipelineFactory.Reset(); - } - private static (List Warnings, List Errors) Run( string cmdlet, string uri, HttpStatusCode status, string body) { var handler = new SingleResponseHandler(status, body); - InjectMock(handler); - try + using (MgxTransportScope.Inject(handler, owned: true)) { using var ps = PowerShell.Create(); ps.AddCommand("Import-Module") @@ -98,7 +62,6 @@ private static (List Warnings, List Errors) Run( errors.Add($"(silent; {handler.Requests.Count} HTTP requests: {string.Join(", ", handler.Requests)})"); return (warnings, errors); } - finally { CleanupMock(); } } private const string ItemNotFound = diff --git a/tests/Mgx.IntegrationTests/Cmdlets/Base/TransportSeamOrderTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/Base/TransportSeamOrderTests.cs new file mode 100644 index 0000000..7fc37cf --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/Base/TransportSeamOrderTests.cs @@ -0,0 +1,52 @@ +using Mgx.Cmdlets.Base; +using Mgx.IntegrationTests.Fakes; + +namespace Mgx.IntegrationTests; + +/// +/// The transport seam is a pair. GetClient takes the epoch, then the transport, then the epoch +/// again and reads both again on a change; a scope arms the transport before it bumps the epoch. +/// Either half on its own is wrong: a reader that takes the transport first can pair one the +/// scope has already retired with the epoch that retired it, and then answers from that +/// transport for as long as the cmdlet instance lives. +/// +/// The interleaving cannot be observed without racing the two, so what is pinned here is the +/// write order the reader depends on, seen from between the two writes. +/// +[Collection("Pipeline")] +public class TransportSeamOrderTests +{ + [Fact] + public void A_scope_arms_its_transport_before_it_bumps_the_epoch() + { + using var outer = MgxTransportScope.Inject(new StubHttpMessageHandler()); + var outerTransport = MgxCmdletBase.s_testTransport; + Assert.NotNull(outerTransport); + var epochBefore = Volatile.Read(ref MgxCmdletBase.s_testTransportEpoch); + + HttpClient? transportInWindow = null; + var epochInWindow = 0; + var windows = 0; + + using (var inner = MgxTransportScope.Inject(new StubHttpMessageHandler())) + { + // Arming happened in the constructor: the transport is the inner scope's and the + // epoch has moved once, so a client cached over the outer one no longer matches. + Assert.NotSame(outerTransport, MgxCmdletBase.s_testTransport); + Assert.Equal(epochBefore + 1, Volatile.Read(ref MgxCmdletBase.s_testTransportEpoch)); + + inner.BetweenPublishWrites = () => + { + windows++; + transportInWindow = MgxCmdletBase.s_testTransport; + epochInWindow = Volatile.Read(ref MgxCmdletBase.s_testTransportEpoch); + }; + } + + Assert.Equal(1, windows); + Assert.Same(outerTransport, transportInWindow); // the transport goes back first + Assert.Equal(epochBefore + 1, epochInWindow); // and only then does the epoch move + Assert.Equal(epochBefore + 2, Volatile.Read(ref MgxCmdletBase.s_testTransportEpoch)); + Assert.Same(outerTransport, MgxCmdletBase.s_testTransport); + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/Base/TypeCacheInvalidationTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/Base/TypeCacheInvalidationTests.cs new file mode 100644 index 0000000..9b40d21 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/Base/TypeCacheInvalidationTests.cs @@ -0,0 +1,129 @@ +using System.Reflection; +using System.Reflection.Emit; +using System.Runtime.Loader; +using Mgx.Cmdlets; +using Mgx.Cmdlets.Base; + +namespace Mgx.IntegrationTests; + +/// +/// FindType caches by full name, and a cached Type carries the identity of the assembly that +/// defined it. Invalidation is an AssemblyLoad hook that module removal detaches, so what has to +/// hold across a removal is that an import arms it again - once - and that the removal itself +/// leaves nothing resolved behind. +/// +/// In the Pipeline collection because every test that imports the module assembly is, and +/// importing it runs the initializer these tests drive directly. +/// +[Collection("Pipeline")] +public class TypeCacheInvalidationTests +{ + private const string GraphSessionTypeName = "Microsoft.Graph.PowerShell.Authentication.GraphSession"; + + /// + /// An entry only this test writes and only this test reads. The cache is process-global and + /// the collections running alongside this one both resolve types into it and load assemblies + /// that clear it, so neither "empty" nor "not empty" is this test's own doing - but an entry + /// put here and gone afterwards can only have been cleared. + /// + private const string Sentinel = "Mgx.IntegrationTests.TypeCacheInvalidationTests+Sentinel"; + + /// + /// What this test resolved, or put there itself, before a removal it expects to drop both. + /// The releases a removal makes resolve types of their own on the way through, so the entry + /// this test wrote is not enough on its own: it is cleared by any load, including one those + /// releases trigger, while the entries they then resolve are exactly what the removal's own + /// clear has to take out afterwards. + /// + private static readonly string[] Resolved = [GraphSessionTypeName, Sentinel]; + + [Fact] + public void An_assembly_loading_after_a_remove_and_re_import_clears_the_type_cache() + { + // Resolve something first: "cleared afterwards" has to mean the removal cleared the + // cache, not that nothing had resolved a type yet. + Assert.NotNull(MgxCmdletBase.FindType(GraphSessionTypeName)); + MgxCmdletBase.s_typeCache[Sentinel] = typeof(TypeCacheInvalidationTests); + + try + { + // A removal is ReleaseStaticState and then the detach; an import is OnImport. + AlcInitializer.ReleaseStaticState(); + MgxCmdletBase.DetachAssemblyLoadHandler(); + Assert.False(AnyStillCached(), "module removal left resolved types cached"); + + new AlcInitializer().OnImport(); + + // A load from a collection running alongside this one clears the cache too, so the + // resolved entry only has to end up there, not be there on the first attempt. + var primed = false; + for (var attempt = 0; attempt < 10 && !primed; attempt++) + { + Assert.NotNull(MgxCmdletBase.FindType(GraphSessionTypeName)); + primed = MgxCmdletBase.s_typeCache.ContainsKey(GraphSessionTypeName); + } + Assert.True(primed, "resolving a type after the import cached nothing"); + + MgxCmdletBase.s_typeCache[Sentinel] = typeof(TypeCacheInvalidationTests); + LoadAnAssembly("MgxTypeCacheProbe"); + + Assert.False(AnyStillCached(), + "a type resolved before the load is still cached, so a Microsoft.Graph.Authentication " + + "loading now would keep resolving to the session of the one it replaced"); + } + finally + { + MgxCmdletBase.s_typeCache.TryRemove(Sentinel, out _); + MgxCmdletBase.AttachAssemblyLoadHandler(); + } + } + + [Fact] + public void A_second_import_leaves_one_subscription_for_a_removal_to_detach() + { + try + { + new AlcInitializer().OnImport(); + new AlcInitializer().OnImport(); + Assert.Equal(1, InvalidationSubscriptions()); + + // One removal has to be enough. A second subscription would survive this and go on + // clearing the cache from a module that is gone. + MgxCmdletBase.DetachAssemblyLoadHandler(); + Assert.Equal(0, InvalidationSubscriptions()); + } + finally + { + MgxCmdletBase.AttachAssemblyLoadHandler(); + } + } + + /// Whether anything this test resolved, or wrote itself, is still cached. + private static bool AnyStillCached() => Resolved.Any(MgxCmdletBase.s_typeCache.ContainsKey); + + /// + /// How many times mgx's invalidation hook is subscribed. AppDomain.AssemblyLoad forwards to a + /// static event on AssemblyLoadContext, and its invocation list is the only place the count + /// shows: an extra subscription is silent until a removal detaches one and the other stays. + /// + private static int InvalidationSubscriptions() + { + var field = typeof(AssemblyLoadContext).GetField("AssemblyLoad", BindingFlags.Static | BindingFlags.NonPublic); + Assert.True(field != null, "AssemblyLoadContext no longer keeps its subscribers in a static field"); + var subscribers = (Delegate?)field!.GetValue(null); + return subscribers?.GetInvocationList().Count(d => d.Method.DeclaringType == typeof(MgxCmdletBase)) ?? 0; + } + + /// + /// Loads an assembly, which is what invalidation keys off. A dynamic one raises the event + /// exactly as a file-backed load does, and leaves nothing behind that the rest of the suite + /// has to be able to resolve. + /// + private static void LoadAnAssembly(string name) + { + var assembly = AssemblyBuilder.DefineDynamicAssembly(new AssemblyName(name), AssemblyBuilderAccess.Run); + assembly.DefineDynamicModule(name) + .DefineType($"{name}.Marker", TypeAttributes.Public) + .CreateType(); + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/BatchErrorSurfacingTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/BatchErrorSurfacingTests.cs index f9435f0..f38df48 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/BatchErrorSurfacingTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/BatchErrorSurfacingTests.cs @@ -1,5 +1,4 @@ using System.Net; -using System.Reflection; using Mgx.Engine.Http; namespace Mgx.IntegrationTests; @@ -12,35 +11,6 @@ namespace Mgx.IntegrationTests; [Collection("Pipeline")] public class BatchErrorSurfacingTests { - private static readonly Type Base = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); - private const BindingFlags Static = BindingFlags.NonPublic | BindingFlags.Static; - - private static void InjectTransport(HttpMessageHandler wire) - { - Base.GetField("s_graphHttpClient", Static)!.SetValue(null, new HttpClient(wire)); - Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, - Mgx.Cmdlets.Base.MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); - Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); - Base.GetField("s_clientOptions", Static)!.SetValue(null, - new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); - } - - private static void ResetTransport() - { - // Restore every static InjectTransport touched - a later test in the collection - // that drives a cmdlet without injecting must not inherit this class's transport. - Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); - Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); - Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); - Base.GetField("s_cachedTotalTimeoutSeconds", Static)!.SetValue(null, 0); - Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); - Base.GetField("s_clientOptions", Static)!.SetValue(null, new ResilientGraphClientOptions()); - ResiliencePipelineFactory.Reset(); - } - private static System.Management.Automation.PowerShell CreateShell() { var ps = System.Management.Automation.PowerShell.Create(); @@ -66,8 +36,7 @@ public void A_failed_item_writes_an_error_record_without_a_dead_letter_path() { var wire = new MockHttpHandler(); wire.QueueResponse(HttpStatusCode.OK, OneOkOneNotFound); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { using var ps = CreateShell(); ps.AddCommand("Invoke-MgxBatchRequest") @@ -82,7 +51,6 @@ public void A_failed_item_writes_an_error_record_without_a_dead_letter_path() Assert.Equal(System.Management.Automation.ErrorCategory.ObjectNotFound, error.CategoryInfo.Category); } - finally { ResetTransport(); } } [Fact] @@ -90,8 +58,7 @@ public void ErrorAction_Stop_stops_on_a_failed_item() { var wire = new MockHttpHandler(); wire.QueueResponse(HttpStatusCode.OK, OneOkOneNotFound); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { using var ps = CreateShell(); ps.AddCommand("Invoke-MgxBatchRequest") @@ -101,7 +68,6 @@ public void ErrorAction_Stop_stops_on_a_failed_item() var ex = Assert.ThrowsAny(() => ps.Invoke()); Assert.Contains("u2", ex.Message); } - finally { ResetTransport(); } } /// First POST answers its chunk in full; every later POST is refused outright. @@ -136,7 +102,7 @@ protected override Task SendAsync( [Fact] public void Stop_preference_cannot_cut_the_dead_letter_file_short() { - InjectTransport(new FirstChunkOnlyHandler()); + using var transport = MgxTransportScope.Inject(new FirstChunkOnlyHandler()); var deadLetter = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); try { @@ -148,9 +114,9 @@ public void Stop_preference_cannot_cut_the_dead_letter_file_short() .AddParameter("ErrorAction", "Stop"); try { ps.Invoke(); } catch { /* Stop terminates - expected */ } - // First chunk answered 20 items (all 200 in this fixture), second chunk's POST - // failed, so 5 items were never sent. Every failure must be on disk even though - // Stop terminated at the first error record. + // First chunk answered 20 items (all 200 in this fixture), second chunk's POST was + // refused, so its 5 items failed. Every failure must be on disk even though Stop + // terminated at the first error record. Assert.True(File.Exists(deadLetter), "dead-letter file was not written"); var lines = File.ReadAllLines(deadLetter); Assert.Equal(5, lines.Length); @@ -158,15 +124,13 @@ public void Stop_preference_cannot_cut_the_dead_letter_file_short() finally { File.Delete(deadLetter); - ResetTransport(); } } [Fact] - public void Items_a_failed_chunk_never_sent_each_write_an_error_record() + public void Items_of_a_refused_chunk_each_write_an_error_record() { - InjectTransport(new FirstChunkOnlyHandler()); - try + using (MgxTransportScope.Inject(new FirstChunkOnlyHandler())) { using var ps = CreateShell(); ps.AddCommand("Invoke-MgxBatchRequest") @@ -175,12 +139,70 @@ public void Items_a_failed_chunk_never_sent_each_write_an_error_record() ps.Invoke(); var errors = ps.Streams.Error.ToList(); - Assert.Equal(5, errors.Count(e => e.FullyQualifiedErrorId.StartsWith("BatchItemNotSent"))); + // The second chunk's five items went out and were refused, so each reads as a failed + // request. None of them is "not sent": that is reserved for a chunk never POSTed. + Assert.Equal(5, errors.Count(e => e.FullyQualifiedErrorId.StartsWith("BatchItemError"))); + Assert.DoesNotContain(errors, e => e.FullyQualifiedErrorId.StartsWith("BatchItemNotSent")); var chunk = Assert.Single(errors, e => e.FullyQualifiedErrorId.StartsWith("BatchChunkFailed")); // The chunk died on a 400; the record carries the classified category, not NotSpecified. Assert.Equal(System.Management.Automation.ErrorCategory.InvalidArgument, chunk.CategoryInfo.Category); + // Nothing was left unsent here - the refused chunk was the last one. The record has + // to say what did happen to its five operations, not only what did not. + Assert.Equal("5 of 25 operations failed, 0 were not sent", chunk.TargetObject); + } + } + + /// One chunk, both items answered, the second sub-response missing its status. + private const string OneOkOneWithoutAStatus = """ + { "responses": [ + { "id": "1", "status": 204, "body": null }, + { "id": "2", "body": null } + ] } + """; + + /// + /// "Not sent" is the cmdlet's word for a write that certainly did not reach the server, and + /// it carries a NotSent flag on the output, an error record telling the caller another chunk + /// failed, and a dead-letter line the help documents as re-pipeable. A PATCH the server + /// answered must reach none of those. + /// + [Fact] + public void A_write_answered_without_a_status_is_not_offered_back_for_resubmission() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, OneOkOneWithoutAStatus); + var deadLetter = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + try + { + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new[] { "/users/u1", "/users/u2" }) + .AddParameter("Method", "PATCH") + .AddParameter("Body", new System.Collections.Hashtable { ["x"] = 1 }) + .AddParameter("DeadLetterPath", deadLetter); + var output = ps.Invoke(); + + var second = Assert.IsType(output[1].BaseObject); + Assert.Equal("/users/u2", second["Url"]); + Assert.NotEqual(0, Assert.IsType(second["Status"])); + Assert.False(second.ContainsKey("NotSent"), + "a write the server answered was flagged as one that never went out"); + + Assert.DoesNotContain(ps.Streams.Error, + e => e.FullyQualifiedErrorId.StartsWith("BatchItemNotSent", StringComparison.Ordinal)); + } + + var line = Assert.Single(File.ReadAllLines(deadLetter), l => l.Contains("/users/u2", StringComparison.Ordinal)); + // Compact, as the writer emits it: the dead-letter serializer does not indent, so a + // needle with a space after the colon matches nothing the file can ever hold. + Assert.DoesNotContain("\"Status\":0", line, StringComparison.Ordinal); + } + finally + { + if (File.Exists(deadLetter)) File.Delete(deadLetter); } - finally { ResetTransport(); } } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/BulkWriteErrorSurfacingTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/BulkWriteErrorSurfacingTests.cs new file mode 100644 index 0000000..e8f8177 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/BulkWriteErrorSurfacingTests.cs @@ -0,0 +1,86 @@ +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// The two error ids a write fan-out can report are a statement about what reached the server. +/// BulkWriteInfraError is for a write that got no HTTP status at all - the network, an open +/// circuit - and it is the one a caller may repeat. Everything else carries the status the +/// server answered with, a body that failed to read after that answer included: the write may +/// have been applied, and repeating it is a decision about duplicates rather than a free retry. +/// +[Collection("Pipeline")] +public class BulkWriteErrorSurfacingTests +{ + private static (List Errors, int Output) Run(string script) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript(script); + var output = ps.Invoke(); + return (ps.Streams.Error.ToList(), output.Count); + } + + /// + /// A 201 whose body does not parse. The POST reached the server and the server applied it; + /// only the read of what it created failed. Reported under the id reserved for a write that + /// got no status, it would read as the one failure a caller can safely repeat - and each + /// repeat creates the entity a second time. + /// + [Fact] + public void A_body_that_does_not_parse_after_a_201_keeps_the_status_the_server_answered() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.Created, "THIS IS NOT JSON"); + using (MgxTransportScope.Inject(wire)) + { + var (errors, _) = Run( + "'u1','u2' | Invoke-MgxRequest -Uri '/users/{id}' -Method POST -Body @{ x = 1 } -Confirm:$false"); + + Assert.Equal(2, errors.Count); + Assert.DoesNotContain(errors, + e => e.FullyQualifiedErrorId.StartsWith("BulkWriteInfraError", StringComparison.Ordinal)); + Assert.All(errors, e => + Assert.StartsWith("BulkWriteError", e.FullyQualifiedErrorId, StringComparison.Ordinal)); + Assert.All(errors, e => + Assert.Contains("HTTP 201", e.Exception.Message, StringComparison.Ordinal)); + } + } + + /// Fails every request before a status exists, the way a dead route does. + private sealed class NeverConnectsHandler : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + => throw new HttpRequestException("no route to host"); + } + + /// + /// The other half of the same statement: a write that never reached a status is the one that + /// takes the infrastructure branch, and it still does. + /// + [Fact] + public void A_write_that_reached_no_status_is_the_one_reported_as_infrastructure() + { + using (MgxTransportScope.Inject(new NeverConnectsHandler(), options: new ResilientGraphClientOptions + { + NoRateLimit = true, + MaxRetryAttempts = 1 + })) + { + var (errors, _) = Run( + "'u1','u2' | Invoke-MgxRequest -Uri '/users/{id}' -Method POST -Body @{ x = 1 } -Confirm:$false"); + + Assert.NotEmpty(errors); + Assert.All(errors, e => + Assert.StartsWith("BulkWriteInfraError", e.FullyQualifiedErrorId, StringComparison.Ordinal)); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DeltaCheckpointIntegrityTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DeltaCheckpointIntegrityTests.cs index abe1b8a..4cc8c96 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/DeltaCheckpointIntegrityTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/DeltaCheckpointIntegrityTests.cs @@ -1,6 +1,5 @@ using System.Management.Automation; using System.Net; -using System.Reflection; using Mgx.Cmdlets.Base; using Mgx.Engine.Http; using Mgx.Engine.Pagination; @@ -32,32 +31,6 @@ public class DeltaCheckpointIntegrityTests """; private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; - private static void InjectMock(MockHttpHandler handler) - { - ResiliencePipelineFactory.Reset(); - var t = typeof(MgxCmdletBase); - t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, new HttpClient(handler)); - t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - t.GetField("s_ownsHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, false); - t.GetField("s_graphEndpoint", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! - .SetValue(null, "https://graph.microsoft.com"); - t.GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! - .SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); - } - - private static void CleanupMock() - { - var t = typeof(MgxCmdletBase); - t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - ResiliencePipelineFactory.Reset(); - } - private static PowerShell Shell() { var ps = PowerShell.Create(); @@ -108,6 +81,21 @@ private static void Sync(Env env) private static string[] Ids(string outputPath) => File.Exists(outputPath) ? File.ReadAllLines(outputPath) : []; + private static void SyncToPipeline(Env env) + { + using var ps = Shell(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", env.DeltaPath) + .AddParameter("CheckpointPath", env.CheckpointPath); + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + } + + /// A checkpoint torn between the write and the rename: no position in it at all. + private const string TornCheckpoint = + """{"resource":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D1","nextLi"""; + // ---------------------------------------------------------------- D5 /// @@ -127,7 +115,7 @@ public void A_torn_temp_does_not_let_the_token_advance_past_what_it_lost() handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 2 re-enumerates handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); Sync(env); var temp = Directory.GetFiles(env.Dir, "out.jsonl.*.tmp").Single(); @@ -146,7 +134,7 @@ public void A_torn_temp_does_not_let_the_token_advance_past_what_it_lost() ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], Ids(env.OutputPath)); } - finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + finally { try { Directory.Delete(env.Dir, true); } catch { } } } // ---------------------------------------------------------------- D6 @@ -171,7 +159,7 @@ public void The_checkpoints_own_staging_file_is_never_adopted_as_data() handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); Sync(env); var temp = Directory.GetFiles(env.Dir, "out.jsonl.*.tmp") @@ -189,7 +177,7 @@ public void The_checkpoints_own_staging_file_is_never_adopted_as_data() ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], Ids(env.OutputPath)); } - finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + finally { try { Directory.Delete(env.Dir, true); } catch { } } } // ---------------------------------------------------------------- D7 @@ -215,7 +203,7 @@ public void A_truncated_output_is_not_mistaken_for_the_baseline_it_describes() handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 3 re-enumerates handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); handler.QueueResponse(HttpStatusCode.OK, ChangesPage3); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); Sync(env); Sync(env); @@ -234,7 +222,7 @@ public void A_truncated_output_is_not_mistaken_for_the_baseline_it_describes() ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}", "{\"id\":\"b4\"}"], Ids(env.OutputPath)); } - finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + finally { try { Directory.Delete(env.Dir, true); } catch { } } } // ---------------------------------------------------------------- D8 @@ -258,7 +246,7 @@ public void Items_written_after_the_last_checkpoint_are_not_duplicated_on_resume handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.OK, ChangesPage3); // run 3: b4 - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); Sync(env); Sync(env); @@ -276,7 +264,7 @@ public void Items_written_after_the_last_checkpoint_are_not_duplicated_on_resume ids); Assert.Equal(ids.Length, ids.Distinct().Count()); } - finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + finally { try { Directory.Delete(env.Dir, true); } catch { } } } // ---------------------------------------------------------------- D9 @@ -300,7 +288,7 @@ public void A_temp_that_survives_its_own_adoption_is_not_merged_twice() handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // run 2: adopts, then dies handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); // run 3: b3 - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); Sync(env); var temp = Directory.GetFiles(env.Dir, "out.jsonl.*.tmp").Single(); @@ -319,7 +307,7 @@ public void A_temp_that_survives_its_own_adoption_is_not_merged_twice() ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], Ids(env.OutputPath)); } - finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + finally { try { Directory.Delete(env.Dir, true); } catch { } } } /// @@ -345,7 +333,7 @@ public void A_checkpoint_naming_the_output_as_its_own_temp_does_not_consume_it() var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); Sync(env); @@ -354,7 +342,7 @@ public void A_checkpoint_naming_the_output_as_its_own_temp_does_not_consume_it() ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], Ids(env.OutputPath)); } - finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + finally { try { Directory.Delete(env.Dir, true); } catch { } } } /// @@ -382,7 +370,7 @@ public void A_checkpoint_naming_a_file_that_is_not_a_temp_does_not_consume_it() var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); Sync(env); @@ -391,7 +379,7 @@ public void A_checkpoint_naming_a_file_that_is_not_a_temp_does_not_consume_it() ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], Ids(env.OutputPath)); } - finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + finally { try { Directory.Delete(env.Dir, true); } catch { } } } // ---------------------------------------------------------------- D10 @@ -411,7 +399,7 @@ public void Adoption_takes_the_temp_its_checkpoint_describes_not_the_newest_one( handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); Sync(env); // leaves its own temp with b1,b2 var own = Directory.GetFiles(env.Dir, "out.jsonl.*.tmp").Single(); @@ -428,7 +416,7 @@ public void Adoption_takes_the_temp_its_checkpoint_describes_not_the_newest_one( Assert.Contains("{\"id\":\"b1\"}", ids); Assert.Contains("{\"id\":\"b2\"}", ids); } - finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + finally { try { Directory.Delete(env.Dir, true); } catch { } } } // ---------------------------------------------------------------- D11 @@ -447,7 +435,7 @@ public void A_non_object_item_in_a_page_does_not_terminate_the_run() var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, """{"value":[{"id":"b1"},"not-an-object",{"id":"b2"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D2"}"""); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); Exception? escaped = null; using (var ps = Shell()) @@ -466,7 +454,7 @@ public void A_non_object_item_in_a_page_does_not_terminate_the_run() Assert.Contains("{\"id\":\"b1\"}", ids); Assert.Contains("{\"id\":\"b2\"}", ids); } - finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + finally { try { Directory.Delete(env.Dir, true); } catch { } } } // ---------------------------------------------------------------- D12 @@ -514,7 +502,7 @@ static string Page(string nextLink) => handler.QueueResponse(HttpStatusCode.OK, Page(Page2Link)); handler.QueueResponse(HttpStatusCode.OK, """{"value":[{"id":"tail"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D2"}"""); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); Sync(env); @@ -531,6 +519,147 @@ static string Page(string nextLink) => Assert.Equal(1001, ids.Length); Assert.Equal("{\"id\":\"tail\"}", ids[^1]); } - finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + finally { try { Directory.Delete(env.Dir, true); } catch { } } + } + + /// + /// PaginationCheckpoint.Load answers null for a file torn by a crash and for one that is + /// locked or that this account cannot open, and the sync does the same thing either way: + /// resume stays null, so it re-enumerates from the delta token. Deleting the file changed + /// nothing about the run and destroyed a position the next run, or another account, could + /// still have resumed from. In file mode the delete came out of the branch that reads + /// "output file is missing" - a reading of contents, off a checkpoint with none to read. + /// + [Fact] + public void A_checkpoint_this_sync_cannot_read_is_left_for_a_sync_that_can() + { + var env = NewEnv(); + try + { + File.WriteAllText(env.CheckpointPath, TornCheckpoint); + var before = File.ReadAllBytes(env.CheckpointPath); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + + // The interrupted run never promoted its output, so this is the branch that used to + // call the checkpoint stale for it. The sync then fails on its first page, so + // nothing on the success path can be what removes the file. + Assert.False(File.Exists(env.OutputPath)); + Sync(env); + + Assert.True(File.Exists(env.CheckpointPath), + "a position this sync could not read was deleted anyway"); + Assert.Equal(before, File.ReadAllBytes(env.CheckpointPath)); + } + finally { try { Directory.Delete(env.Dir, true); } catch { } } + } + + /// + /// The same checkpoint against a sync writing to the pipeline, which reaches the second of + /// the two deletes and took the file with no warning on any stream. + /// + [Fact] + public void A_checkpoint_a_pipeline_sync_cannot_read_is_left_for_a_sync_that_can() + { + var env = NewEnv(); + try + { + File.WriteAllText(env.CheckpointPath, TornCheckpoint); + var before = File.ReadAllBytes(env.CheckpointPath); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + + SyncToPipeline(env); + + Assert.True(File.Exists(env.CheckpointPath), + "a position this sync could not read was deleted anyway"); + Assert.Equal(before, File.ReadAllBytes(env.CheckpointPath)); + } + finally { try { Directory.Delete(env.Dir, true); } catch { } } + } + + /// + /// A checkpoint recording no nextLink says something an unreadable one cannot: the sync + /// that wrote it got to the end. That one is still deleted by a run that gets nowhere, + /// because -Latest is suppressed by the checkpoint file merely existing - a marker left + /// lying there costs the next -Latest run the baseline it asked for. + /// + [Fact] + public void A_checkpoint_that_says_the_previous_sync_finished_is_deleted() + { + var env = NewEnv(); + try + { + new PaginationCheckpoint + { + Resource = DeltaLink1, + NextLink = null, + ItemsCollected = 3, + PageItemsAlreadyWritten = 0, + TempFile = null, + DataLength = null, + }.Save(env.CheckpointPath); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + + SyncToPipeline(env); + + Assert.False(File.Exists(env.CheckpointPath)); + } + finally { try { Directory.Delete(env.Dir, true); } catch { } } + } + + /// + /// The last step of a completed sync is the move that puts its temp over the output, and it + /// can fail with every page already fetched: a destination another process holds open, a + /// read-only file, a share that dropped. The checkpoint goes only once that move has + /// landed, so a failure leaves both files exactly as an interruption leaves them and the + /// next run resumes from them instead of re-enumerating from the delta token. + /// + [Fact] + public void A_failed_promotion_keeps_the_checkpoint_and_the_temp_holding_the_sync() + { + var env = NewEnv(); + try + { + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 1: b1,b2 + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); // run 1: b3, then the deltaLink + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); // run 2, resumed + using var transport = MgxTransportScope.Inject(handler); + + // Something at the output path the promotion cannot replace, so both pages are + // fetched and written and the move at the end of the run is what fails. + Directory.CreateDirectory(env.OutputPath); + + Sync(env); + + Assert.True(File.Exists(env.CheckpointPath), + "the failed promotion deleted the position the next run resumes from"); + var cp = PaginationCheckpoint.Load(env.CheckpointPath); + Assert.NotNull(cp); + Assert.NotNull(cp!.TempFile); + var temp = Path.Combine(env.Dir, cp.TempFile!); + Assert.True(File.Exists(temp), + "the failed promotion deleted the temp holding every item the sync fetched"); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(temp)); + + // With the occupier gone: the temp is promoted, one page is fetched - the one the + // checkpoint recorded - and the enumeration does not start over. + Directory.Delete(env.OutputPath); + var before = handler.RequestCount; + Sync(env); + + Assert.Equal(1, handler.RequestCount - before); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], Ids(env.OutputPath)); + } + finally { try { Directory.Delete(env.Dir, true); } catch { } } } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DeltaQueryTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DeltaQueryTests.cs index cc4a305..a3225f6 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/DeltaQueryTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/DeltaQueryTests.cs @@ -926,43 +926,6 @@ public async Task Delta_NoDeltaLink_WhenEndpointReturnsNone() // --- Cmdlet-level tests via PowerShell.Create() --- - /// - /// Injects a mock HttpClient into MgxCmdletBase's static fields so Sync-MgxDelta - /// can run without a real Graph connection. Uses reflection because the fields - /// are private static. - /// - private static void InjectMockHttpClient(MockHttpHandler handler) - { - ResiliencePipelineFactory.Reset(); - var baseType = typeof(MgxCmdletBase); - var httpClientField = baseType.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!; - var fingerprintField = baseType.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)!; - var ownsClientField = baseType.GetField("s_ownsHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!; - - var httpClient = new HttpClient(handler); - var endpointField = baseType.GetField("s_graphEndpoint", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)!; - var optionsField = baseType.GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)!; - - httpClientField.SetValue(null, httpClient); - // The cmdlet recomputes the identity from the mocked Get-MgContext (a PSCustomObject - // carrying only TenantId), so the cached fingerprint must be built from an equivalent - // context or GetClient() sees a credential change and discards the injected mock. - fingerprintField.SetValue(null, MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - ownsClientField.SetValue(null, false); - endpointField.SetValue(null, "https://graph.microsoft.com"); - optionsField.SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true }); - } - - private static void CleanupMockHttpClient() - { - var baseType = typeof(MgxCmdletBase); - baseType.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - baseType.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - baseType.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - ResiliencePipelineFactory.Reset(); - } - private static PowerShell CreateTestShell() { var ps = PowerShell.Create(); @@ -993,7 +956,8 @@ public void Cmdlet_410Gone_DeletesStateAndResyncs() // Second attempt (fresh URL) succeeds with items + deltaLink handler.QueueResponse(HttpStatusCode.OK, DeltaPage2WithToken); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-410-{Guid.NewGuid()}.json"); try @@ -1026,7 +990,6 @@ public void Cmdlet_410Gone_DeletesStateAndResyncs() finally { DeltaState.Delete(deltaPath); - CleanupMockHttpClient(); } } @@ -1040,11 +1003,9 @@ public void Cmdlet_OutputFile_TempFileCleanedUpOnError() handler.QueueResponse(HttpStatusCode.InternalServerError, errorBody); // Page 2 fails handler.QueueResponse(HttpStatusCode.InternalServerError, errorBody); // Polly retry also fails - InjectMockHttpClient(handler); - // Override options to limit retries (default 7 would need 8 error responses) - var optField = typeof(MgxCmdletBase).GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)!; - optField.SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); + // Limit retries: the default would need eight error responses queued. + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-tempclean-{Guid.NewGuid()}.json"); var outputPath = Path.Combine(Path.GetTempPath(), $"cmdlet-tempclean-{Guid.NewGuid()}.jsonl"); @@ -1070,7 +1031,6 @@ public void Cmdlet_OutputFile_TempFileCleanedUpOnError() { DeltaState.Delete(deltaPath); if (File.Exists(outputPath)) File.Delete(outputPath); - CleanupMockHttpClient(); } } @@ -1084,18 +1044,16 @@ public void Cmdlet_BrokenCircuit_WritesErrorRecord() for (int i = 0; i < 20; i++) handler.QueueResponse(HttpStatusCode.InternalServerError); - InjectMockHttpClient(handler); - // Use aggressive CB settings so it trips quickly - var optionsField = typeof(MgxCmdletBase).GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)!; - optionsField.SetValue(null, new ResilientGraphClientOptions - { - NoRateLimit = true, - CircuitBreakerMinThroughput = 2, - CircuitBreakerFailureRatio = 0.5, - CircuitBreakerDurationSeconds = 30, - MaxRetryAttempts = 1 - }); - ResiliencePipelineFactory.Reset(); + // Aggressive circuit-breaker settings so it trips quickly. + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions + { + NoRateLimit = true, + CircuitBreakerMinThroughput = 2, + CircuitBreakerFailureRatio = 0.5, + CircuitBreakerDurationSeconds = 30, + MaxRetryAttempts = 1 + }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-cb-{Guid.NewGuid()}.json"); @@ -1122,7 +1080,6 @@ public void Cmdlet_BrokenCircuit_WritesErrorRecord() finally { DeltaState.Delete(deltaPath); - CleanupMockHttpClient(); } } @@ -1134,14 +1091,8 @@ public void Cmdlet_HttpRequestException_WritesErrorRecord() handler.QueueException(new HttpRequestException("Connection refused")); handler.QueueException(new HttpRequestException("Connection refused")); // For Polly retry - InjectMockHttpClient(handler); - var optionsField2 = typeof(MgxCmdletBase).GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)!; - optionsField2.SetValue(null, new ResilientGraphClientOptions - { - NoRateLimit = true, - MaxRetryAttempts = 1 - }); - ResiliencePipelineFactory.Reset(); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-http-{Guid.NewGuid()}.json"); @@ -1164,7 +1115,6 @@ e.Exception is HttpRequestException || finally { DeltaState.Delete(deltaPath); - CleanupMockHttpClient(); } } @@ -1231,7 +1181,8 @@ public void Cmdlet_Prefer_SentOnEveryPage_AndPersistedNormalized() handler.QueueResponse(HttpStatusCode.OK, DeltaPage1); handler.QueueResponse(HttpStatusCode.OK, DeltaPage2WithToken); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-prefer-{Guid.NewGuid()}.json"); try @@ -1259,7 +1210,6 @@ public void Cmdlet_Prefer_SentOnEveryPage_AndPersistedNormalized() finally { DeltaState.Delete(deltaPath); - CleanupMockHttpClient(); } } @@ -1269,7 +1219,8 @@ public void Cmdlet_PreferDrift_ForcesFullResync_AndDeletesCheckpoint() var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, DeltaPage2WithToken); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-preferdrift-{Guid.NewGuid()}.json"); var cpPath = Path.Combine(Path.GetTempPath(), $"cmdlet-preferdrift-{Guid.NewGuid()}.checkpoint"); @@ -1310,7 +1261,6 @@ public void Cmdlet_PreferDrift_ForcesFullResync_AndDeletesCheckpoint() { DeltaState.Delete(deltaPath); PaginationCheckpoint.Delete(cpPath); - CleanupMockHttpClient(); } } @@ -1320,7 +1270,8 @@ public void Cmdlet_Latest_UsesDeltatokenForm_ForDirectoryResources() var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, EmptyDeltaWithToken); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-latest-dir-{Guid.NewGuid()}.json"); try @@ -1342,7 +1293,6 @@ public void Cmdlet_Latest_UsesDeltatokenForm_ForDirectoryResources() finally { DeltaState.Delete(deltaPath); - CleanupMockHttpClient(); } } @@ -1359,7 +1309,8 @@ public void Cmdlet_Latest_UsesTokenForm_ForDrives() } """); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-latest-drive-{Guid.NewGuid()}.json"); try @@ -1380,7 +1331,6 @@ public void Cmdlet_Latest_UsesTokenForm_ForDrives() finally { DeltaState.Delete(deltaPath); - CleanupMockHttpClient(); } } @@ -1390,7 +1340,8 @@ public void Cmdlet_Latest_IgnoredWhenStateExists() var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, EmptyDeltaWithToken); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-latest-ignored-{Guid.NewGuid()}.json"); try @@ -1417,7 +1368,6 @@ public void Cmdlet_Latest_IgnoredWhenStateExists() finally { DeltaState.Delete(deltaPath); - CleanupMockHttpClient(); } } @@ -1433,10 +1383,8 @@ public void Cmdlet_Checkpoint_SurvivesCrash_ThenResumesAndDeletesOnSuccess() handler.QueueResponse(HttpStatusCode.InternalServerError, errorBody); handler.QueueResponse(HttpStatusCode.InternalServerError, errorBody); - InjectMockHttpClient(handler); - var optField = typeof(MgxCmdletBase).GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)!; - optField.SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-cp-resume-{Guid.NewGuid()}.json"); var cpPath = Path.Combine(Path.GetTempPath(), $"cmdlet-cp-resume-{Guid.NewGuid()}.checkpoint"); @@ -1486,7 +1434,6 @@ public void Cmdlet_Checkpoint_SurvivesCrash_ThenResumesAndDeletesOnSuccess() { DeltaState.Delete(deltaPath); PaginationCheckpoint.Delete(cpPath); - CleanupMockHttpClient(); } } @@ -1499,7 +1446,8 @@ public void Cmdlet_Checkpoint_JsonlOrphanedTemp_IsAdopted() var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, DeltaPage2WithToken); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var stem = $"cmdlet-cp-adopt-{Guid.NewGuid()}"; var deltaPath = Path.Combine(Path.GetTempPath(), $"{stem}.json"); var cpPath = Path.Combine(Path.GetTempPath(), $"{stem}.checkpoint"); @@ -1515,7 +1463,8 @@ public void Cmdlet_Checkpoint_JsonlOrphanedTemp_IsAdopted() Resource = "https://graph.microsoft.com/v1.0/users/delta?$top=999", NextLink = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=page2", ItemsCollected = 2, - PageItemsAlreadyWritten = 0 + PageItemsAlreadyWritten = 0, + OutputFile = outputPath }.Save(cpPath); File.WriteAllLines(tempPath, [ @@ -1545,7 +1494,6 @@ public void Cmdlet_Checkpoint_JsonlOrphanedTemp_IsAdopted() PaginationCheckpoint.Delete(cpPath); if (File.Exists(outputPath)) File.Delete(outputPath); if (File.Exists(tempPath)) File.Delete(tempPath); - CleanupMockHttpClient(); } } @@ -1559,7 +1507,8 @@ public void Cmdlet_410Gone_DeletesCheckpointToo() """{"error":{"code":"deltaTokenExpired","message":"Delta token has expired"}}"""); handler.QueueResponse(HttpStatusCode.OK, DeltaPage2WithToken); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-410cp-{Guid.NewGuid()}.json"); var cpPath = Path.Combine(Path.GetTempPath(), $"cmdlet-410cp-{Guid.NewGuid()}.checkpoint"); @@ -1595,7 +1544,6 @@ public void Cmdlet_410Gone_DeletesCheckpointToo() { DeltaState.Delete(deltaPath); PaginationCheckpoint.Delete(cpPath); - CleanupMockHttpClient(); } } @@ -1605,7 +1553,8 @@ public void Cmdlet_RemovedItems_AreCountedAndPassedThrough() var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, DeltaPageWithRemoved); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-removed-{Guid.NewGuid()}.json"); try @@ -1628,7 +1577,6 @@ public void Cmdlet_RemovedItems_AreCountedAndPassedThrough() finally { DeltaState.Delete(deltaPath); - CleanupMockHttpClient(); } } @@ -1667,7 +1615,8 @@ public void Cmdlet_Checkpoint_MidPage_BoundsProgressLostWhenARunDiesInsideAPage( var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, BuildSingleDeltaPage(ItemsInPage)); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var stem = $"cmdlet-cp-midpage-{Guid.NewGuid()}"; var deltaPath = Path.Combine(Path.GetTempPath(), $"{stem}.json"); var cpPath = Path.Combine(Path.GetTempPath(), $"{stem}.checkpoint"); @@ -1710,7 +1659,6 @@ public void Cmdlet_Checkpoint_MidPage_BoundsProgressLostWhenARunDiesInsideAPage( { try { File.Delete(leftover); } catch { } } - CleanupMockHttpClient(); } } [Fact] @@ -1724,7 +1672,8 @@ public void Latest_is_refused_when_the_previous_state_was_discarded() // triggers it. var handler = new MockHttpHandler(); handler.SetDefaultResponse(HttpStatusCode.OK, DeltaPage2WithToken); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var deltaPath = Path.Combine(Path.GetTempPath(), $"latest-discard-{Guid.NewGuid()}.json"); try @@ -1754,7 +1703,6 @@ public void Latest_is_refused_when_the_previous_state_was_discarded() finally { DeltaState.Delete(deltaPath); - CleanupMockHttpClient(); } } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DeltaRecoveryTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DeltaRecoveryTests.cs index 2b5a144..b8d65d8 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/DeltaRecoveryTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/DeltaRecoveryTests.cs @@ -1,6 +1,5 @@ using System.Management.Automation; using System.Net; -using System.Reflection; using Mgx.Cmdlets.Base; using Mgx.Engine.Http; using Mgx.Engine.Pagination; @@ -32,32 +31,6 @@ public class DeltaRecoveryTests """; private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; - private static void InjectMock(MockHttpHandler handler) - { - ResiliencePipelineFactory.Reset(); - var t = typeof(MgxCmdletBase); - t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, new HttpClient(handler)); - t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - t.GetField("s_ownsHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, false); - t.GetField("s_graphEndpoint", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! - .SetValue(null, "https://graph.microsoft.com"); - t.GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! - .SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); - } - - private static void CleanupMock() - { - var t = typeof(MgxCmdletBase); - t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - ResiliencePipelineFactory.Reset(); - } - private static PowerShell Shell() { var ps = PowerShell.Create(); @@ -109,7 +82,7 @@ public void A_transient_error_midrun_does_not_strand_the_pages_the_checkpoint_co handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // ... and its retry handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); // run 3 resumes - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); try { Sync(deltaPath, checkpointPath, outputPath); @@ -136,7 +109,7 @@ public void A_transient_error_midrun_does_not_strand_the_pages_the_checkpoint_co Assert.False(File.Exists(checkpointPath)); Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); } - finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + finally { try { Directory.Delete(dir, true); } catch { } } } /// @@ -158,7 +131,7 @@ public void A_completed_sync_replaces_the_previous_syncs_output() handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 2: b1,b2 handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); // run 2: b3 - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); try { Sync(deltaPath, checkpointPath, outputPath); @@ -169,7 +142,7 @@ public void A_completed_sync_replaces_the_previous_syncs_output() ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], File.ReadAllLines(outputPath)); } - finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + finally { try { Directory.Delete(dir, true); } catch { } } } /// @@ -195,7 +168,7 @@ public void A_checkpoint_that_does_not_name_its_file_reenumerates_rather_than_gu handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 3 re-enumerates handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); try { Sync(deltaPath, checkpointPath, outputPath); @@ -216,7 +189,7 @@ public void A_checkpoint_that_does_not_name_its_file_reenumerates_rather_than_gu File.ReadAllLines(outputPath)); Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); } - finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + finally { try { Directory.Delete(dir, true); } catch { } } } /// @@ -239,7 +212,7 @@ public void A_first_run_that_dies_midway_still_recovers_its_pages() handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); try { Sync(deltaPath, checkpointPath, outputPath); @@ -253,7 +226,53 @@ public void A_first_run_that_dies_midway_still_recovers_its_pages() Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); } - finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The same guard through the sync's own adoption call. A checkpoint too old to record a + /// length leaves adoption a line count and nothing else, and a temp cut inside the row that + /// count reaches hands the fragment back as a line like any other - so it went into the + /// JSONL output as a change, was reported as a recovery, and the rest of the enumeration + /// was appended behind it. Nothing is lost by refusing: the delta token has not moved. + /// + [Fact] + public void A_sync_does_not_adopt_a_temp_cut_inside_the_row_the_checkpoint_counts() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-recovery-torn-{Guid.NewGuid():N}")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var outputPath = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // A first sync collects page one into its temp and dies on page two. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, outputPath); + var checkpoint = PaginationCheckpoint.Load(checkpointPath)!; + var temp = Path.Combine(dir, checkpoint.TempFile!); + + // The same position as a release that recorded no length left it, and a temp that + // lost its tail after the two rows the checkpoint counts had been written. + File.WriteAllText(temp, "{\"id\":\"b1\"}\n{\"id\":\"b"); + checkpoint.TempFile = null; + checkpoint.DataLength = null; + checkpoint.Save(checkpointPath); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + Sync(deltaPath, checkpointPath, outputPath); + + var lines = File.ReadAllLines(outputPath); + foreach (var line in lines) + System.Text.Json.JsonDocument.Parse(line).Dispose(); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], lines); + } + finally { try { Directory.Delete(dir, true); } catch { } } } /// @@ -278,7 +297,7 @@ public void Latest_is_ignored_while_a_resume_checkpoint_exists() handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // ... and its retry handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); // run 2 resumes - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); try { Sync(deltaPath, checkpointPath, outputPath); @@ -302,7 +321,7 @@ public void Latest_is_ignored_while_a_resume_checkpoint_exists() Assert.Contains("$deltatoken=D2", token); Assert.DoesNotContain("token=latest", token); } - finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + finally { try { Directory.Delete(dir, true); } catch { } } } /// @@ -328,7 +347,7 @@ public void A_denied_output_ends_the_resume_as_an_error_record_not_an_unhandled_ handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); // run 3 resumes - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); try { Sync(deltaPath, checkpointPath, outputPath); @@ -361,7 +380,6 @@ public void A_denied_output_ends_the_resume_as_an_error_record_not_an_unhandled_ } finally { - CleanupMock(); try { if (OperatingSystem.IsWindows()) @@ -392,7 +410,7 @@ public void A_run_that_dies_with_no_checkpoint_leaves_no_temp() handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); try { Sync(deltaPath, checkpointPath, outputPath); @@ -400,7 +418,7 @@ public void A_run_that_dies_with_no_checkpoint_leaves_no_temp() Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); Assert.False(File.Exists(outputPath)); } - finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + finally { try { Directory.Delete(dir, true); } catch { } } } /// @@ -424,7 +442,7 @@ public void FullSync_with_Latest_still_baselines_from_now_despite_a_checkpoint() handler.QueueResponse(HttpStatusCode.OK, """{"value":[],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=FROMNOW"}"""); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); try { Sync(deltaPath, checkpointPath, outputPath); @@ -443,7 +461,7 @@ public void FullSync_with_Latest_still_baselines_from_now_despite_a_checkpoint() Assert.Contains("$deltatoken=FROMNOW", DeltaState.Load(deltaPath)!.DeltaLink); Assert.Contains("deltatoken=latest", handler.Requests[^1].RequestUri!.ToString()); } - finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + finally { try { Directory.Delete(dir, true); } catch { } } } /// @@ -463,15 +481,15 @@ public void A_fresh_run_clears_temps_orphaned_by_an_earlier_enumeration() var checkpointPath = Path.Combine(dir, "run.checkpoint"); var outputPath = Path.Combine(dir, "out.jsonl"); - // An orphan from an enumeration that no longer exists, long enough to satisfy any - // later checkpoint's line count. - var foreign = $"{outputPath}.deadbeef.tmp"; + // An orphan from an enumeration that no longer exists, named the way a run names its + // own temp, and long enough to satisfy any later checkpoint's line count. + var foreign = $"{outputPath}.deadbeefdeadbeefdeadbeefdeadbeef.tmp"; File.WriteAllLines(foreign, Enumerable.Range(0, 50).Select(i => $"{{\"id\":\"foreign{i}\"}}")); var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, Baseline); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); try { Sync(deltaPath, checkpointPath, outputPath); @@ -482,7 +500,7 @@ public void A_fresh_run_clears_temps_orphaned_by_an_earlier_enumeration() Assert.Equal(["{\"id\":\"a1\"}", "{\"id\":\"a2\"}"], lines); Assert.DoesNotContain(lines, l => l.Contains("foreign")); } - finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + finally { try { Directory.Delete(dir, true); } catch { } } } /// @@ -521,7 +539,7 @@ public void Resuming_from_a_midpage_checkpoint_repeats_no_items() var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, page2); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); try { Sync(deltaPath, checkpointPath, outputPath); @@ -532,6 +550,6 @@ public void Resuming_from_a_midpage_checkpoint_repeats_no_items() Assert.Equal("{\"id\":\"q0\"}", lines[999]); Assert.Equal("{\"id\":\"q1\"}", lines[1000]); } - finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + finally { try { Directory.Delete(dir, true); } catch { } } } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DeltaSharedCheckpointTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DeltaSharedCheckpointTests.cs new file mode 100644 index 0000000..0e8d2fe --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/DeltaSharedCheckpointTests.cs @@ -0,0 +1,996 @@ +using System.Management.Automation; +using System.Net; +using System.Text; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// Two syncs of the same -Uri build the same resource string, so the resource comparison that +/// guards recovery cannot tell them apart. What separates them is the file they collect into, +/// and a checkpoint that records a byte length used to record nothing about which file that +/// length was measured in - so a shared -CheckpointPath let one sync cut the other's output +/// back to its own offset, mid-line, and append the resumed page onto the torn byte. +/// +[Collection("Pipeline")] +public class DeltaSharedCheckpointTests +{ + private const string ChangesPage1 = """ + {"value":[{"id":"b1"},{"id":"b2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2"} + """; + private const string ChangesPage2 = """ + {"value":[{"id":"b3"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D2"} + """; + private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + + private const string GroupsPage = """ + {"value":[{"id":"g1"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/groups/delta?$deltatoken=G1"} + """; + // A first page with a continuation, so an attempt can reach a page boundary - and save a + // checkpoint of its own naming its own temp - before the request after it answers 410. + private const string GroupsPageOne = """ + {"value":[{"id":"g1"},{"id":"g2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/groups/delta?$skiptoken=G2"} + """; + private const string GroupsResync = """ + {"value":[{"id":"g9"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/groups/delta?$deltatoken=G1"} + """; + private const string TokenExpired = + """{"error":{"code":"deltaTokenExpired","message":"Delta token has expired"}}"""; + + private static readonly string[] OldSync = + ["{\"id\":\"old-0000001\"}", "{\"id\":\"old-0000002\"}", "{\"id\":\"old-0000003\"}"]; + + /// A groups sync holding a token older than Graph keeps them. + private static void StaleState(string deltaPath) => + new DeltaState + { + DeltaLink = "https://graph.microsoft.com/v1.0/groups/delta?$deltatoken=stale", + Resource = "/groups/delta", + GraphEndpoint = "https://graph.microsoft.com", + Select = "", + ApiVersion = "v1.0", + }.Save(deltaPath); + + /// A users sync partway through, holding the token its last run was issued. + private static void UsersState(string deltaPath) => + new DeltaState + { + DeltaLink = "https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D0", + Resource = "/users/delta", + GraphEndpoint = "https://graph.microsoft.com", + Select = "", + ApiVersion = "v1.0", + }.Save(deltaPath); + + private static void Sync(string deltaPath, string checkpointPath, string outputPath, + string uri = "/users/delta") + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", uri) + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("OutputFile", outputPath); + // A run that dies is part of the scenario, so a terminating error is expected. + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + } + + private static string[] SyncWarnings(string deltaPath, string checkpointPath, string outputPath, + string uri = "/users/delta") + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", uri) + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("OutputFile", outputPath); + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + return [.. ps.Streams.Warning.Select(w => w.Message)]; + } + + /// + /// Sync A is interrupted twice, so the checkpoint it leaves records a length with no temp + /// to explain it. Sync B, its own delta state and its own output, must not have that + /// length applied to its file. + /// + [Fact] + public void A_checkpoint_from_another_sync_does_not_cut_this_ones_output() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-shared-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var outA = Path.Combine(dir, "outA.jsonl"); + var outB = Path.Combine(dir, "outB.jsonl"); + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // A run 1, page 1 + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // A run 1, page 2 + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // A run 2 promotes, then dies + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // B, page 1 + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); // B, page 2 + using var transport = MgxTransportScope.Inject(handler); + try + { + Sync(deltaA, checkpointPath, outA); + Sync(deltaA, checkpointPath, outA); + + var cp = PaginationCheckpoint.Load(checkpointPath)!; + Assert.Null(cp.TempFile); + Assert.Equal(new FileInfo(outA).Length, cp.DataLength); + + // B has its own delta state and its own output, which already holds an earlier sync. + File.WriteAllLines(outB, OldSync); + Sync(deltaB, checkpointPath, outB); + + var lines = File.ReadAllLines(outB); + Assert.All(lines, l => Assert.StartsWith("{\"id\":\"", l)); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], lines); + + // A's own output is untouched by B. + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(outA)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same collision between two directories rather than two names. What distinguishes the + /// syncs is which file they collect into, and a leaf comparison cannot see that: sync B + /// adopted A's position and cut B's own output to A's byte count. + /// + [Fact] + public void A_checkpoint_for_the_same_name_in_another_directory_does_not_cut_this_ones_output() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-shared-dir-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var outA = Path.Combine(Directory.CreateDirectory(Path.Combine(dir, "a")).FullName, "out.jsonl"); + var outB = Path.Combine(Directory.CreateDirectory(Path.Combine(dir, "b")).FullName, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // A run 1, page 1 + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // A run 1, page 2 + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // A run 2 promotes, then dies + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // B, page 1 + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); // B, page 2 + using var transport = MgxTransportScope.Inject(handler); + try + { + Sync(deltaA, checkpointPath, outA); + Sync(deltaA, checkpointPath, outA); + + var cp = PaginationCheckpoint.Load(checkpointPath)!; + Assert.Null(cp.TempFile); + Assert.Equal(new FileInfo(outA).Length, cp.DataLength); + + File.WriteAllLines(outB, OldSync); + Sync(deltaB, checkpointPath, outB); + + var lines = File.ReadAllLines(outB); + Assert.All(lines, l => Assert.StartsWith("{\"id\":\"", l)); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], lines); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(outA)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same sync, resumed with -Uri typed differently. Graph answers "/users/delta" and + /// "/Users/delta" from one collection, so both runs enumerate the same thing - but the + /// recorded resource was compared ordinally, which made the second run a different sync: + /// its own position was refused as another's, the temp holding the changes the first run + /// had already collected was swept, and the enumeration started over. + /// + [Fact] + public void A_checkpoint_written_under_another_spelling_of_the_resource_still_resumes() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-spelling-{Guid.NewGuid():N}")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // Page one reaches the temp; page two fails, so the position survives. + Sync(deltaPath, checkpointPath, output, "/users/delta"); + Assert.NotNull(PaginationCheckpoint.Load(checkpointPath)); + Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + + // The resume, typed the other way, with only the page the dead run never got. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var first = handler.CapturedRequests.Count; + Sync(deltaPath, checkpointPath, output, "/Users/delta"); + + Assert.Contains("skiptoken=B2", handler.CapturedRequests[first].Uri); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + var state = DeltaState.Load(deltaPath); + Assert.NotNull(state); + Assert.Contains("$deltatoken=D2", state!.DeltaLink); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A sync writing to the pipeline was outside the ownership check entirely, so a checkpoint + /// left by a sync collecting into a file resumed it at that sync's nextLink. Every page + /// before that one went to the file and never to the pipeline, and the delta token was + /// saved over them on success - which is the one failure a delta sync cannot recover from, + /// since there is no re-fetch once the token moves. + /// + [Fact] + public void A_pipeline_sync_does_not_resume_from_a_checkpoint_that_collected_into_a_file() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-mode-{Guid.NewGuid():N}")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + try + { + // The other sync's position: page 1 is already in its file, page 2 is what remains. + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users/delta?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = Path.Combine(dir, "elsewhere.jsonl"), + DataLength = 24, + }.Save(checkpointPath); + + using var transport = MgxTransportScope.Inject(new ByUrlHandler()); + + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpointPath); + var emitted = ps.Invoke() + .Select(r => ((System.Collections.Hashtable)r.BaseObject)["id"]!.ToString()!) + .ToArray(); + + // Every change, not just the ones after the other sync's position. + Assert.Equal(["b1", "b2", "b3"], emitted); + // Only now may a token stand for this enumeration. + var state = DeltaState.Load(deltaPath); + Assert.NotNull(state); + Assert.Contains("$deltatoken=D2", state!.DeltaLink); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same checkpoint from a release that did not record the output file yet. A file-mode + /// run names no temp while it is appending, and none once a cancellation has promoted the + /// one it had, so all such a checkpoint says about where its items went is the length it + /// counted. A pipeline sync adopted every one of those: it emitted the tail of the + /// enumeration, saved the delta token over everything before it, and there is no re-fetch + /// once the token has moved. A pipeline run measures no file, so a length is the file mode. + /// + [Fact] + public void A_pipeline_sync_does_not_resume_from_a_checkpoint_that_records_only_a_length() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-legacy-{Guid.NewGuid():N}")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + try + { + // The other sync's position, in the shape 2.1.3 and before wrote it. + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users/delta?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = null, + DataLength = 24, + }.Save(checkpointPath); + + using var transport = MgxTransportScope.Inject(new ByUrlHandler()); + + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpointPath); + var emitted = ps.Invoke() + .Select(r => ((System.Collections.Hashtable)r.BaseObject)["id"]!.ToString()!) + .ToArray(); + + // Every change, not just the ones after the other sync's position. + Assert.Equal(["b1", "b2", "b3"], emitted); + // Only now may a token stand for this enumeration. + var state = DeltaState.Load(deltaPath); + Assert.NotNull(state); + Assert.Contains("$deltatoken=D2", state!.DeltaLink); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// How far "it is left as it is" has to reach. The refused checkpoint's items are in the + /// temp it names, and the stale-temp sweep a few lines later deleted every temp beside the + /// output - including that one. The sync it belongs to then came back to a position + /// pointing at a file that is gone and re-enumerated from the delta token, which is the + /// whole cost the refusal was written to avoid. + /// + [Fact] + public void A_refused_checkpoints_temp_survives_the_run_that_refused_it() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-refused-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // Sync A dies on page 2: its two items are in its temp, the output was never + // promoted, and the checkpoint names both. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + + var cp = PaginationCheckpoint.Load(checkpointPath)!; + Assert.NotNull(cp.TempFile); + var temp = Path.Combine(dir, cp.TempFile!); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(temp)); + var before = File.ReadAllBytes(checkpointPath); + + // Sync B: another collection, its own delta state, the same -OutputFile and + // -CheckpointPath, failing before its first page boundary. + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.Equal(before, File.ReadAllBytes(checkpointPath)); + Assert.True(File.Exists(temp), + "the refused checkpoint's items were swept away by the run that refused it"); + + // A, back where it left off, rather than at the delta token. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + Sync(deltaA, checkpointPath, output); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// How far "spare the temps a refused checkpoint names" reaches. The sweep only ever + /// deletes names a run gives its own temp - the output's name, a dot, 32 hex digits, + /// ".tmp" - so a refused checkpoint naming "out.jsonl.{guid}.tmp" beside an output called + /// "out" names a file this sweep could not touch either way. Skipping on it bought that + /// file nothing and cost this output its own orphans, which the pre-length adoption path + /// then picks up on a line count alone. + /// + [Fact] + public void A_refused_temp_this_sweep_could_never_reach_does_not_spare_this_outputs_orphans() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-sweep-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var mine = Path.Combine(dir, "out"); + var theirs = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // The sync into "out" collects page one into its temp and dies on page two. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, mine); + var orphan = Assert.Single(Directory.GetFiles(dir, "out.*.tmp")); + + // The sync next door takes the shared checkpoint over: it refuses what it finds, + // saves its own position at its first page boundary, and dies on page two. The + // first sync's temp is now an orphan - nothing on disk describes it. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaB, checkpointPath, theirs); + var theirTemp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + + // "out" again. It refuses a checkpoint naming a temp beside a different output, + // and that name must not stand in the way of its own sweep. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + Sync(deltaA, checkpointPath, mine); + + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(mine)); + Assert.False(File.Exists(orphan), + "a temp name the sweep could never reach suppressed this output's sweep"); + Assert.True(File.Exists(theirTemp), + "the refused checkpoint's own staging file was deleted"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// What each refusal says. A checkpoint recording another sync's output is a second sync + /// over one -CheckpointPath, and naming that is what the caller acts on. One recording no + /// output at all is not: it was believed while the files beside it corroborated it, and a + /// single sync whose temp has since gone, or whose output has been replaced, reaches the + /// same refusal. Every release before this one recorded no output, so that is the ordinary + /// upgrade path - and it was told there was a second sync somewhere and to give each its + /// own -CheckpointPath, which fixes nothing it has. + /// + [Fact] + public void A_refusal_reports_the_cause_it_can_show() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-refusal-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + File.WriteAllLines(output, OldSync); + + // The shape an interrupted fresh sync left before outputs were recorded, whose temp + // is no longer beside it. + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users/delta?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = $"out.jsonl.{Guid.NewGuid():N}.tmp", + OutputFile = null, + DataLength = 24, + }.Save(checkpointPath); + + var uncorroborated = SyncWarnings(deltaPath, checkpointPath, output); + Assert.Contains(uncorroborated, w => w.Contains("no longer corroborate")); + Assert.DoesNotContain(uncorroborated, + w => w.Contains("records an enumeration this run cannot resume from")); + + // The same position, recording an output this run is not collecting into. + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users/delta?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = Path.Combine(dir, "elsewhere.jsonl"), + DataLength = 24, + }.Save(checkpointPath); + + Assert.Contains(SyncWarnings(deltaPath, checkpointPath, output), + w => w.Contains("records an enumeration this run cannot resume from")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And the cause it cannot show. A run interrupted partway through an incremental sync + /// leaves a checkpoint recording the delta URL it was enumerating. Lose the state under it - + /// corrupt, deleted by hand, a -DeltaPath moved - and the same command line comes back + /// building a full-sync URL, which is not that one, so it refuses its own earlier position. + /// The refusal is right; the reading was not. It named a different sync, over a + /// -CheckpointPath no second sync has ever touched, and told the caller to give each of them + /// its own - which is the arrangement they were already running. + /// + [Fact] + public void A_refusal_does_not_name_a_second_sync_it_cannot_see() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-selfrefusal-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // One sync, one -CheckpointPath: interrupted on page 2 of its incremental pass, so + // what it leaves records the delta URL it was reading. + UsersState(deltaPath); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + Assert.Contains("$deltatoken=D0", PaginationCheckpoint.Load(checkpointPath)!.Resource); + + // The state goes, and the same command line enumerates in full. + File.Delete(deltaPath); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var refusal = Assert.Single( + SyncWarnings(deltaPath, checkpointPath, output), + w => w.Contains(checkpointPath)); + + Assert.Contains("records an enumeration this run cannot resume from", refusal); + Assert.Contains("either another sync's, sharing this -CheckpointPath", refusal); + Assert.Contains("this sync's own from a pass it no longer makes", refusal); + Assert.DoesNotContain("belongs to a different sync", refusal); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A 410 says the token this run holds is dead. It says nothing about a checkpoint this + /// same run has already decided belongs to another sync, and deleting that one took the + /// other sync's position away seconds after warning that it would be left alone - while the + /// retry, running the stale-temp sweep a second time with the refusal forgotten, took the + /// temp holding its items too. Both halves of the other sync's progress, on the one path + /// that promised to touch neither. + /// + [Fact] + public void An_expired_token_does_not_delete_the_checkpoint_this_run_refused() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410refused-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // Sync A dies on page 2: its two items are in its temp, and the checkpoint names + // both files. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + + var cp = PaginationCheckpoint.Load(checkpointPath)!; + var temp = Path.Combine(dir, cp.TempFile!); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(temp)); + var before = File.ReadAllBytes(checkpointPath); + + // Sync B: another collection over the same -CheckpointPath and -OutputFile, holding + // a token Graph answers 410 for. It refuses A's checkpoint, re-syncs in full, and + // dies before a page boundary of its own. + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.True(File.Exists(checkpointPath), + "the expired token deleted a checkpoint this run had refused as another sync's"); + Assert.Equal(before, File.ReadAllBytes(checkpointPath)); + Assert.True(File.Exists(temp), + "the retry swept away the temp the refusal a moment earlier had spared"); + + // A, back where it left off, rather than at the delta token. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + Sync(deltaA, checkpointPath, output); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And the run that refused still gets its own sync done: the 410 retry enumerates in full, + /// collects into the output and saves the token it was issued, with the other sync's temp + /// still where the refusal left it. + /// + [Fact] + public void A_full_resync_after_a_refusal_still_completes_its_own_sync() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410done-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var temp = Path.Combine(dir, PaginationCheckpoint.Load(checkpointPath)!.TempFile!); + + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + handler.QueueResponse(HttpStatusCode.OK, GroupsPage); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.Equal(["{\"id\":\"g1\"}"], File.ReadAllLines(output)); + Assert.Contains("$deltatoken=G1", DeltaState.Load(deltaB)!.DeltaLink); + Assert.True(File.Exists(temp), + "the retry swept away the temp the refusal a moment earlier had spared"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// What the run that refused leaves beside its own output. The attempt a 410 sends round the + /// loop again keeps its temp - a checkpoint counting those items is on disk - and the retry + /// spares it, because the sweep is held off over every temp beside this output for as long + /// as the refusal stands. What ends the holding off was never written: the checkpoint is + /// replaced by the retry's, or deleted by the run that completed, without a word about the + /// file it used to name. So a sync that promoted its output and reported success left a + /// partial copy of the changes it had already collected beside the finished file, for some + /// later sync's sweep to reclaim. + /// + [Fact] + public void A_full_resync_after_a_refusal_leaves_no_temp_of_its_own() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410kept-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // Sync A dies on page 2: the checkpoint names the temp holding its two items. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + + // Sync B refuses that checkpoint, collects a page of its own into a temp, and loses + // its token to a 410 on the request after it. The retry re-syncs in full. + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + handler.QueueResponse(HttpStatusCode.OK, GroupsResync); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.Equal(["{\"id\":\"g9\"}"], File.ReadAllLines(output)); + Assert.Contains("$deltatoken=G1", DeltaState.Load(deltaB)!.DeltaLink); + + // The one temp beside the output is the refused checkpoint's, which this run had no + // business touching either way. + var left = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(theirs, Path.GetFileName(left)); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(left)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same compound with the 410 on the first request of all, so the attempt that dies has + /// saved no checkpoint and its temp is empty. The keep asked only whether a checkpoint was + /// on disk, and the one there is the refused sync's: it counts nothing this run wrote and + /// names a temp of its own, so an empty file was kept on a foreign file's account and left + /// beside the output the retry went on to finish. + /// + [Fact] + public void An_attempt_that_saved_no_checkpoint_keeps_no_temp_on_a_refused_ones_account() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410empty-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + handler.QueueResponse(HttpStatusCode.OK, GroupsResync); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.Equal(["{\"id\":\"g9\"}"], File.ReadAllLines(output)); + var left = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(theirs, Path.GetFileName(left)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And the temp goes at the moment it stops being anything, not at the end of the run. The + /// retry saves its own first page-boundary checkpoint, which names its own temp - so the + /// one the attempt before it left is from then on counted by nothing - and then dies. What + /// is beside the output is the temp the checkpoint names, and the refused sync's. + /// + [Fact] + public void A_retry_deletes_the_temp_its_own_checkpoint_has_stopped_naming() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410release-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + + // The retry reaches a page boundary of its own and dies on the page after it. + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + var cp = PaginationCheckpoint.Load(checkpointPath); + Assert.NotNull(cp); + Assert.NotNull(cp!.TempFile); + Assert.NotEqual(theirs, cp.TempFile); + + var temps = Directory.GetFiles(dir, "out.jsonl.*.tmp") + .Select(Path.GetFileName).Order().ToArray(); + Assert.Equal(new[] { theirs, cp.TempFile }.Order().ToArray(), temps); + Assert.Equal(["{\"id\":\"g1\"}", "{\"id\":\"g2\"}"], + File.ReadAllLines(Path.Combine(dir, cp.TempFile!))); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other end of it: the retry dies before a page boundary of its own. The temp the + /// attempt before it filled is what an interrupted run leaves and stays where it is, while + /// the retry's own, which nothing counts, is the one that goes. The position that counted + /// the first goes with the token it belongs to - the 410 door deletes it, because the + /// attempt that took the path over wrote it, and it enumerates what Graph has just declared + /// gone. + /// + [Fact] + public void A_retry_that_dies_leaves_the_temp_its_own_attempt_filled() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410dead-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.False(File.Exists(checkpointPath), + "the expired token spared a position of this run's own into the dead enumeration"); + + var temps = Directory.GetFiles(dir, "out.jsonl.*.tmp") + .Select(Path.GetFileName).Order().ToArray(); + Assert.Equal(2, temps.Length); + var mine = temps.Single(t => t != theirs); + Assert.Equal(["{\"id\":\"g1\"}", "{\"id\":\"g2\"}"], + File.ReadAllLines(Path.Combine(dir, mine!))); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And what the run after it finds. The refusal is decided against the file that was on + /// disk when the run started, and the very next page boundary saves this run's own position + /// over that path - so by the time the 410 door reads the refusal, the file it spares is + /// this run's, recording the enumeration Graph has just declared gone. The retry then dies + /// before saving a position of its own, and the door has already deleted the delta state, + /// so the next invocation of the same command line started with nothing to resume from and + /// found that checkpoint waiting. It refused it - correctly; it enumerates something this + /// command line can no longer build - and warned about a sync that was never there, while + /// holding off its stale-temp sweep for a second run over files nothing counted any more. + /// + [Fact] + public void A_refused_position_this_run_wrote_over_goes_with_the_expired_token() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410tookover-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // Sync A dies on page 2: the checkpoint names the temp holding its two items. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + + // Sync B refuses that checkpoint, reaches a page boundary of its own - which saves + // over the same path - loses its token to a 410, and dies again in the re-sync + // before it can record a position of the enumeration it is now making. + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.False(File.Exists(checkpointPath), + "the 410 door spared a position this run had written over the refused one"); + Assert.True(File.Exists(Path.Combine(dir, theirs)), + "the retry swept away the temp the refusal a moment earlier had spared"); + Assert.False(File.Exists(deltaB), "the expired delta state outlived the 410"); + + // The next invocation of B's command line, with nothing of its own left to resume + // from: no checkpoint to refuse, so no warning about a sync that was never there, + // and the sweep it no longer holds off reclaims what the interrupted run left. + handler.QueueResponse(HttpStatusCode.OK, GroupsResync); + var warnings = SyncWarnings(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.DoesNotContain(warnings, w => w.Contains("resume checkpoint")); + Assert.Equal(["{\"id\":\"g9\"}"], File.ReadAllLines(output)); + Assert.Contains("$deltatoken=G1", DeltaState.Load(deltaB)!.DeltaLink); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other branch of the same door, which is the refusal's promise where it still means + /// something: the 410 arrives on the first request of all, so no attempt of this run has + /// saved anything over the path and what is there is still the position that was refused. + /// It is left exactly as it was found, byte for byte, and it is still the other sync's + /// enumeration rather than this one's. + /// + [Fact] + public void An_expired_token_spares_the_refused_position_no_attempt_wrote_over() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410intact-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + var before = File.ReadAllBytes(checkpointPath); + + // The same compound as above, differing only in reaching no page boundary before + // the 410 - so the door decides over a path this run has not written to. + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.Equal(before, File.ReadAllBytes(checkpointPath)); + var cp = PaginationCheckpoint.Load(checkpointPath); + Assert.NotNull(cp); + Assert.Contains("/users/delta", cp!.Resource); + Assert.Equal(theirs, cp.TempFile); + Assert.True(File.Exists(Path.Combine(dir, theirs)), + "the retry swept away the temp the refusal a moment earlier had spared"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// Answers by URL rather than in order, so what the sync asks for first is what decides + /// which items it receives. + /// + private sealed class ByUrlHandler : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var body = request.RequestUri!.ToString().Contains("skiptoken=B2", StringComparison.Ordinal) + ? ChangesPage2 + : ChangesPage1; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DeltaUndeletableCheckpointTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DeltaUndeletableCheckpointTests.cs new file mode 100644 index 0000000..7f5ef70 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/DeltaUndeletableCheckpointTests.cs @@ -0,0 +1,375 @@ +using System.Management.Automation; +using System.Net; +using System.Text; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// A recovery that gives up on the checkpoint has to give up on it whether or not the file goes +/// away. Every bail-out in the reconcile deletes the checkpoint and says what it does instead - +/// "Re-enumerating from the last saved delta token; no changes are lost" - and that delete can +/// fail: a checkpoint directory this account may read but not unlink from is the ordinary case, +/// and PaginationCheckpoint.Delete answers false for exactly it. The refusal used to be recorded +/// by nothing but the file's absence, so a failed delete left the run reloading the checkpoint it +/// had just given up on, resuming from its nextLink, putting the previous sync's rows in front of +/// this one's, and saving a delta token past items that are then in no file at all - under a +/// warning promising the opposite. +/// +/// Export-MgxCollection never had this: every bail-out in its reconcile returns false, and that +/// bool is what decides whether the run appends. These pin the same shape for the delta sync. +/// +[Collection("Pipeline")] +public class DeltaUndeletableCheckpointTests +{ + private const string Baseline = """ + {"value":[{"id":"a1"},{"id":"a2"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D1"} + """; + private const string ChangesPage1 = """ + {"value":[{"id":"b1"},{"id":"b2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2"} + """; + private const string ChangesPage2 = """ + {"value":[{"id":"b3"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D2"} + """; + private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + + private static readonly string[] ThisSyncsRows = + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"]; + + /// + /// Answers by URL rather than in order, so what the sync asks for first is what decides + /// which items it receives: a resume from the checkpoint's skiptoken gets what that + /// skiptoken addresses and nothing before it, which is the whole difference between + /// resuming and re-enumerating. Dies on that page while is set, + /// which is how the interrupted run that leaves the checkpoint dies. + /// + private sealed class ByUrlHandler : HttpMessageHandler + { + private readonly List _uris = []; + + public bool DieOnResume { get; set; } + + public List Uris { get { lock (_uris) { return [.. _uris]; } } } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var uri = request.RequestUri!.ToString(); + lock (_uris) { _uris.Add(uri); } + + var (status, body) = uri.Contains("skiptoken=B2", StringComparison.Ordinal) + ? DieOnResume + ? (HttpStatusCode.InternalServerError, ServerError) + : (HttpStatusCode.OK, ChangesPage2) + : uri.Contains("$deltatoken=D1", StringComparison.Ordinal) + ? (HttpStatusCode.OK, ChangesPage1) + : (HttpStatusCode.OK, Baseline); + + return Task.FromResult(new HttpResponseMessage(status) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } + + private static PowerShell Shell() + { + var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + return ps; + } + + /// The warnings the run wrote. A run that dies is setup here, not a failure. + private static List Sync(string deltaPath, string checkpointPath, string outputPath) + { + using var ps = Shell(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("OutputFile", outputPath); + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + return [.. ps.Streams.Warning.Select(w => w.Message)]; + } + + private static string NewDir() => Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-undeletable-{Guid.NewGuid():N}")).FullName; + + /// + /// Run 1 completes and records the D1 token; run 2 dies on the second page, leaving a + /// checkpoint that counts two items into the temp it names. That is the state every test + /// here starts from. + /// + private static void BaselineThenDeath( + ByUrlHandler handler, string deltaPath, string checkpointPath, string outputPath) + { + Sync(deltaPath, checkpointPath, outputPath); + handler.DieOnResume = true; + Sync(deltaPath, checkpointPath, outputPath); + handler.DieOnResume = false; + } + + /// + /// Make the checkpoint's directory one this account can read but not unlink from, which is + /// what makes PaginationCheckpoint.Delete answer false. Answers false itself where the host + /// does not enforce that - root ignores the mode bits, and there is no undeletable + /// checkpoint to be had there, so the caller returns rather than asserting something the + /// host cannot show. + /// + private static bool TrySeal(string dir) + { + if (OperatingSystem.IsWindows()) return false; + var canary = Path.Combine(dir, "deletable.probe"); + File.WriteAllText(canary, ""); + File.SetUnixFileMode(dir, UnixFileMode.UserRead | UnixFileMode.UserExecute); + try { File.Delete(canary); } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { return true; } + Unseal(dir); + return false; + } + + private static void Unseal(string dir) + { + if (OperatingSystem.IsWindows()) return; + File.SetUnixFileMode(dir, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + + /// + /// The counted items are in no file - the temp holding them is gone - and the checkpoint + /// naming it cannot be deleted. The bail-out's warning says this run re-enumerates from the + /// delta token, so that is what has to go on the wire: the two items the temp lost come + /// back, and the output holds this sync's rows rather than the previous sync's with the + /// remainder of this one behind them. + /// + [Fact] + public void A_missing_temp_bailout_reenumerates_when_the_checkpoint_cannot_be_deleted() + { + // The checkpoint sits in a directory the run cannot write, which is what makes its + // deletion fail. Windows expresses that through ACLs, not a mode. + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var vault = Directory.CreateDirectory(Path.Combine(dir, "vault")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(vault, "run.checkpoint"); + var outputPath = Path.Combine(dir, "out.jsonl"); + + var handler = new ByUrlHandler(); + using var transport = MgxTransportScope.Inject(handler); + try + { + BaselineThenDeath(handler, deltaPath, checkpointPath, outputPath); + + var checkpoint = PaginationCheckpoint.Load(checkpointPath)!; + Assert.Contains("skiptoken=B2", checkpoint.NextLink); + Assert.Equal(2, checkpoint.ItemsCollected); + Assert.NotNull(checkpoint.TempFile); + + // The temp holding b1 and b2 goes: a tmp sweeper, a container restart, a hand. The + // checkpoint still names it and still counts them. + foreach (var t in Directory.GetFiles(dir, "out.jsonl.*.tmp")) File.Delete(t); + if (!TrySeal(vault)) return; + + var before = handler.Uris.Count; + var warnings = Sync(deltaPath, checkpointPath, outputPath); + Unseal(vault); + + Assert.Contains(warnings, w => w.Contains("temp file is missing or incomplete")); + Assert.True(File.Exists(checkpointPath), + "the checkpoint was deletable after all, so no undeletable one was tested"); + + var wire = handler.Uris.Skip(before).ToList(); + Assert.True(wire[0].Contains("$deltatoken=D1", StringComparison.Ordinal), + $"the run resumed the checkpoint it gave up on; first request: {wire[0]}"); + Assert.DoesNotContain("skiptoken=B2", wire[0]); + + // "No changes are lost": b1 and b2 are back, and the previous sync's rows are not + // in front of them. + Assert.Equal(ThisSyncsRows, File.ReadAllLines(outputPath)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { Unseal(vault); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other bail-out a sync reaches with a checkpoint it wrote for itself: the checkpoint + /// counts its items into the output, and the output no longer holds them, so there is + /// nothing to trim back to. Same undeletable checkpoint, same promise, same properties - + /// and here a resume appends the remainder of the enumeration onto a file that never held + /// its first pages. + /// + [Fact] + public void A_trim_bailout_reenumerates_when_the_checkpoint_cannot_be_deleted() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var vault = Directory.CreateDirectory(Path.Combine(dir, "vault")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(vault, "run.checkpoint"); + var outputPath = Path.Combine(dir, "out.jsonl"); + + var handler = new ByUrlHandler(); + using var transport = MgxTransportScope.Inject(handler); + try + { + BaselineThenDeath(handler, deltaPath, checkpointPath, outputPath); + + // The shape a promoted cancellation leaves: the items are in the output itself, and + // the checkpoint records how many bytes of it they occupy. + File.WriteAllText(outputPath, "{\"id\":\"b1\"}\n{\"id\":\"b2\"}\n"); + foreach (var t in Directory.GetFiles(dir, "out.jsonl.*.tmp")) File.Delete(t); + var checkpoint = PaginationCheckpoint.Load(checkpointPath)!; + checkpoint.TempFile = null; + checkpoint.OutputFile = Path.GetFullPath(outputPath); + checkpoint.DataLength = new FileInfo(outputPath).Length; + checkpoint.Save(checkpointPath); + + // And the output is replaced by one that does not hold them. + File.WriteAllText(outputPath, ""); + if (!TrySeal(vault)) return; + + var before = handler.Uris.Count; + var warnings = Sync(deltaPath, checkpointPath, outputPath); + Unseal(vault); + + Assert.Contains(warnings, w => w.Contains("no longer holds the 2 items")); + Assert.True(File.Exists(checkpointPath), + "the checkpoint was deletable after all, so no undeletable one was tested"); + + var wire = handler.Uris.Skip(before).ToList(); + Assert.True(wire[0].Contains("$deltatoken=D1", StringComparison.Ordinal), + $"the run resumed the checkpoint it gave up on; first request: {wire[0]}"); + Assert.DoesNotContain("skiptoken=B2", wire[0]); + + Assert.Equal(ThisSyncsRows, File.ReadAllLines(outputPath)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { Unseal(vault); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The control, in the same undeletable directory: a checkpoint whose temp is where it says + /// it is is still resumed from, and the resume is the same one it has always been - one + /// request, for the page the checkpoint stopped at. What decides is the reconcile's answer + /// and not what is left on disk, so a run that can delete nothing at all resumes exactly as + /// it does when it can. + /// + [Fact] + public void A_healthy_checkpoint_still_resumes_when_it_cannot_be_deleted() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var vault = Directory.CreateDirectory(Path.Combine(dir, "vault")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(vault, "run.checkpoint"); + var outputPath = Path.Combine(dir, "out.jsonl"); + + var handler = new ByUrlHandler(); + using var transport = MgxTransportScope.Inject(handler); + try + { + BaselineThenDeath(handler, deltaPath, checkpointPath, outputPath); + + // Nothing is missing this time: the temp the checkpoint names holds the two items + // it counts. + Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + if (!TrySeal(vault)) return; + + var before = handler.Uris.Count; + var warnings = Sync(deltaPath, checkpointPath, outputPath); + Unseal(vault); + + Assert.Contains(warnings, w => w.Contains("Recovered 2 items")); + var wire = handler.Uris.Skip(before).ToList(); + Assert.Single(wire); + Assert.Contains("skiptoken=B2", wire[0]); + + Assert.Equal(ThisSyncsRows, File.ReadAllLines(outputPath)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { Unseal(vault); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The steady state a checkpoint nothing can remove leaves: it is still there next run, and + /// next run gives up on it again. Noisy - the warning repeats every time, which is what the + /// delete-failure warning already tells the caller to end by hand - and correct. What must + /// not happen is a later run reading the survivor as a position to resume from. + /// + [Fact] + public void A_second_run_against_the_surviving_checkpoint_gives_up_on_it_again() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var vault = Directory.CreateDirectory(Path.Combine(dir, "vault")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(vault, "run.checkpoint"); + var outputPath = Path.Combine(dir, "out.jsonl"); + + var handler = new ByUrlHandler(); + using var transport = MgxTransportScope.Inject(handler); + try + { + BaselineThenDeath(handler, deltaPath, checkpointPath, outputPath); + foreach (var t in Directory.GetFiles(dir, "out.jsonl.*.tmp")) File.Delete(t); + if (!TrySeal(vault)) return; + + // Run 3 gives up on the checkpoint, cannot delete it, cannot save one of its own + // over it either, and dies where the run before it died - so run 4 meets the file + // run 3 met, unchanged, with the delta token still where it was. + handler.DieOnResume = true; + var beforeThird = handler.Uris.Count; + var third = Sync(deltaPath, checkpointPath, outputPath); + handler.DieOnResume = false; + + Assert.Contains(third, w => w.Contains("temp file is missing or incomplete")); + Assert.True(handler.Uris[beforeThird].Contains("$deltatoken=D1", StringComparison.Ordinal), + $"the run resumed the checkpoint it gave up on; first request: {handler.Uris[beforeThird]}"); + Assert.True(File.Exists(checkpointPath)); + Assert.Contains("skiptoken=B2", PaginationCheckpoint.Load(checkpointPath)!.NextLink); + + var beforeFourth = handler.Uris.Count; + var fourth = Sync(deltaPath, checkpointPath, outputPath); + Unseal(vault); + + Assert.Contains(fourth, w => w.Contains("temp file is missing or incomplete")); + var wire = handler.Uris.Skip(beforeFourth).ToList(); + Assert.True(wire[0].Contains("$deltatoken=D1", StringComparison.Ordinal), + $"the run resumed the checkpoint it gave up on; first request: {wire[0]}"); + Assert.DoesNotContain("skiptoken=B2", wire[0]); + + // Nothing doubled, and nothing from the runs before it in front of it. + Assert.Equal(ThisSyncsRows, File.ReadAllLines(outputPath)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { Unseal(vault); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DiagnosticsDrainTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DiagnosticsDrainTests.cs index cc47884..87dc342 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/DiagnosticsDrainTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/DiagnosticsDrainTests.cs @@ -1,5 +1,4 @@ using System.Net; -using System.Reflection; using Mgx.Engine.Http; namespace Mgx.IntegrationTests; @@ -12,9 +11,6 @@ namespace Mgx.IntegrationTests; [Collection("Pipeline")] public class DiagnosticsDrainTests { - private static readonly Type Base = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); - private const BindingFlags Static = BindingFlags.NonPublic | BindingFlags.Static; - [Fact] public void An_unexpected_exception_still_delivers_the_buffered_retry_history() { @@ -22,16 +18,10 @@ public void An_unexpected_exception_still_delivers_the_buffered_retry_history() wire.QueueResponse(HttpStatusCode.ServiceUnavailable); wire.QueueException(new InvalidOperationException("boom")); - Base.GetField("s_graphHttpClient", Static)!.SetValue(null, new HttpClient(wire)); - Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, - Mgx.Cmdlets.Base.MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); - Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); - Base.GetField("s_clientOptions", Static)!.SetValue(null, - new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 3, MaxRetryAfterSeconds = 1 }); - ResiliencePipelineFactory.Reset(); - try + using (MgxTransportScope.Inject(wire, options: new ResilientGraphClientOptions + { + NoRateLimit = true, MaxRetryAttempts = 3, MaxRetryAfterSeconds = 1 + })) { using var ps = System.Management.Automation.PowerShell.Create(); ps.AddCommand("Import-Module") @@ -50,14 +40,5 @@ public void An_unexpected_exception_still_delivers_the_buffered_retry_history() Assert.Contains(ps.Streams.Verbose, v => v.Message.Contains("Retry attempt", StringComparison.OrdinalIgnoreCase)); } - finally - { - Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); - Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); - Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); - Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); - Base.GetField("s_clientOptions", Static)!.SetValue(null, new ResilientGraphClientOptions()); - ResiliencePipelineFactory.Reset(); - } } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ErrorActionMatrixTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ErrorActionMatrixTests.cs new file mode 100644 index 0000000..6824491 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/ErrorActionMatrixTests.cs @@ -0,0 +1,262 @@ +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// about_Mgx_Errors states the -ErrorAction and -ErrorVariable contract in prose and names only +/// Stop and SilentlyContinue. This drives all twenty-four preference-by-context cells and asserts, +/// per cell, whether the pipeline terminates, whether a record reaches $Error, whether +/// -ErrorVariable collects it, and whether anything is displayed. +/// +/// What was covered before this file, and by what: BatchErrorSurfacingTests pinned the batch +/// context under Stop twice (termination alone, with and without a dead-letter path) and under the +/// default preference once (one record on the error stream). Three cells of twenty-four, none of +/// them across all three properties; -ErrorVariable was asserted nowhere and Ignore appeared +/// nowhere. +/// +/// All four preferences are driven as common parameters. Ignore cannot be set as a preference +/// variable, so a preference-variable form could not cover the same grid. +/// +/// (Corpus: M365DSC-7273 sustained throttling - the throttling cells generalize it to what a +/// caller sees when a 429 outlives its retries; M365DSC-7198 batch semantics - the batch cells +/// extend it from Stop to all four preferences.) +/// +[Collection("Pipeline")] +public class ErrorActionMatrixTests +{ + private const string NotFoundBody = + """{"error":{"code":"Request_ResourceNotFound","message":"u1 does not exist"}}"""; + private const string ServerErrorBody = + """{"error":{"code":"InternalServerError","message":"boom"}}"""; + private const string ThrottledBody = + """{"error":{"code":"TooManyRequests","message":"slow down"}}"""; + private const string OneOkOneNotFound = """ + { "responses": [ + { "id": "1", "status": 200, "body": { "id": "u1" } }, + { "id": "2", "status": 404, "body": { "error": { "code": "Request_ResourceNotFound", "message": "u2 does not exist" } } } + ] } + """; + + private static readonly string[] Contexts = + [ + "DirectRequest", "ExhaustedRetry", "Batch", "Pagination", "CircuitBreaker", "Throttling" + ]; + + private static readonly string[] Preferences = + [ + "Stop", "Continue", "SilentlyContinue", "Ignore" + ]; + + public static TheoryData Cells + { + get + { + var cells = new TheoryData(); + foreach (var context in Contexts) + foreach (var preference in Preferences) + cells.Add(context, preference); + return cells; + } + } + + /// The wire, the options, an optional priming run, and the pipeline under test. + private sealed record Cell( + MockHttpHandler Wire, + ResilientGraphClientOptions Options, + string? Prime, + string Pipeline); + + private static ResilientGraphClientOptions Fast => new() + { + NoRateLimit = true, + MaxRetryAttempts = 1, + MaxRetryAfterSeconds = 1 + }; + + private static Cell BuildCell(string context) + { + var wire = new MockHttpHandler(); + switch (context) + { + // A 404 the retry policy does not retry: one request, one record. + case "DirectRequest": + wire.SetDefaultResponse(HttpStatusCode.NotFound, NotFoundBody); + return new Cell(wire, Fast, null, "Invoke-MgxRequest -Uri /users/u1"); + + // Every attempt fails, so the record is written after the last retry rather than + // on the first failure - the case about_Mgx_Errors calls out under RETRIES. + case "ExhaustedRetry": + wire.SetDefaultResponse(HttpStatusCode.ServiceUnavailable, ServerErrorBody); + return new Cell(wire, Fast, null, "Invoke-MgxRequest -Uri /users/u1"); + + // The chunk POST succeeds and one item inside it answers 404. + case "Batch": + wire.QueueResponse(HttpStatusCode.OK, OneOkOneNotFound); + return new Cell(wire, Fast, null, + "Invoke-MgxBatchRequest -Uri @('/users/u1','/users/u2') -Method GET"); + + // Page one is delivered, so output exists before the failure the record describes. + case "Pagination": + wire.QueueResponse(HttpStatusCode.OK, TestData.UsersPage1); + wire.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerErrorBody); + return new Cell(wire, Fast, null, "Invoke-MgxRequest -Uri /users -All"); + + // The priming run's two failed attempts meet the minimum throughput at a failure + // ratio of 1.0, so the breaker is open before the measured pipeline runs. + case "CircuitBreaker": + wire.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerErrorBody); + return new Cell( + wire, + new ResilientGraphClientOptions + { + NoRateLimit = true, + MaxRetryAttempts = 1, + MaxRetryAfterSeconds = 1, + CircuitBreakerMinThroughput = 2, + CircuitBreakerFailureRatio = 0.5, + CircuitBreakerDurationSeconds = 30 + }, + "Invoke-MgxRequest -Uri /users/u1 -ErrorAction SilentlyContinue | Out-Null", + "Invoke-MgxRequest -Uri /users/u1"); + + case "Throttling": + wire.SetDefaultResponse(HttpStatusCode.TooManyRequests, ThrottledBody, + new Dictionary { ["Retry-After"] = "0" }); + return new Cell(wire, Fast, null, "Invoke-MgxRequest -Uri /users/u1"); + + default: + throw new ArgumentOutOfRangeException(nameof(context), context, "unknown failure context"); + } + } + + private static System.Management.Automation.PowerShell CreateShell() + { + var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + return ps; + } + + private sealed record Outcome( + bool Terminated, + int ErrorCount, + int VariableCount, + string FirstErrorId, + int ErrorStreamCount, + bool FailureServed); + + /// + /// Runs one cell inside PowerShell so the preference is interpreted where it is interpreted + /// in production - by the engine, not by a caller reading a C# property. + /// + private static Outcome Drive(string context, string preference) + { + var cell = BuildCell(context); + using var transport = MgxTransportScope.Inject(cell.Wire, options: cell.Options); + using var ps = CreateShell(); + + // The priming run goes through its own Invoke so the error stream can be emptied after + // it. That stream is the only thing telling Continue from SilentlyContinue - both leave + // a record in $Error and in -ErrorVariable, and only Continue displays one. + if (cell.Prime != null) + { + ps.AddScript(cell.Prime); + ps.Invoke(); + ps.Commands.Clear(); + } + + ps.Streams.Error.Clear(); + + // $Error is cleared after any priming run, so the count measures the pipeline under test. + var script = $$""" + $Error.Clear() + $ev = $null + $terminated = $false + try { {{cell.Pipeline}} -ErrorAction {{preference}} -ErrorVariable ev | Out-Null } + catch { $terminated = $true } + [pscustomobject]@{ + Terminated = $terminated + ErrorCount = $Error.Count + VariableCount = @($ev).Where({ $null -ne $_ }).Count + FirstErrorId = if ($Error.Count) { @($Error)[-1].FullyQualifiedErrorId } else { '' } + } + """; + + ps.AddScript(script); + var results = ps.Invoke(); + var outcome = Assert.Single(results); + return new Outcome( + (bool)outcome.Properties["Terminated"].Value, + (int)outcome.Properties["ErrorCount"].Value, + (int)outcome.Properties["VariableCount"].Value, + (string)outcome.Properties["FirstErrorId"].Value, + ps.Streams.Error.Count, + FailureServed(context, cell.Wire)); + } + + /// + /// That the wire produced the failure this context is about. Under Ignore every count the + /// grid reads is zero whether the request failed or succeeded, so the cell needs a signal + /// that does not come from the pipeline. A batch failure travels inside a 200, so there the + /// signal is the chunk going out and the queued response still carrying a failed item. + /// + private static bool FailureServed(string context, MockHttpHandler wire) => context == "Batch" + ? wire.RequestCount > 0 && OneOkOneNotFound.Contains("\"status\": 404", StringComparison.Ordinal) + : wire.ServedStatusCodes.Exists(status => (int)status >= 400); + + /// + /// The record each context is supposed to produce. Without this a context that quietly stopped + /// failing the way it claims - a 404 answered before pagination started, say - would still + /// satisfy every count in the grid. + /// + private static string ExpectedErrorId(string context) => context switch + { + "DirectRequest" => "Request_ResourceNotFound", + "ExhaustedRetry" => "InternalServerError", + "Batch" => "BatchItemError", + "Pagination" => "InternalServerError", + "CircuitBreaker" => "CircuitBroken", + "Throttling" => "TooManyRequests", + _ => throw new ArgumentOutOfRangeException(nameof(context), context, "unknown failure context") + }; + + [Theory] + [MemberData(nameof(Cells))] + public void A_preference_decides_termination_the_error_stack_and_the_variable( + string context, string preference) + { + var outcome = Drive(context, preference); + + Assert.Equal(preference == "Stop", outcome.Terminated); + + // Only Continue displays the record. SilentlyContinue keeps it, in $Error and in + // -ErrorVariable, and shows nothing - which is the whole difference between them, and + // the one about_Mgx_Errors describes. + Assert.Equal(preference == "Continue", outcome.ErrorStreamCount > 0); + + if (preference == "Ignore") + { + // Ignore is the one preference that leaves no trace: nothing to find afterwards, + // by either route. Every count below holds just as well for a request that + // succeeded, so the wire has to say it produced the failure this cell is about. + Assert.True(outcome.FailureServed, + $"{context}/{preference}: the wire never served a failure"); + Assert.Equal(0, outcome.ErrorCount); + Assert.Equal(0, outcome.VariableCount); + } + else + { + Assert.True(outcome.ErrorCount > 0, + $"{context}/{preference}: no record reached $Error"); + Assert.True(outcome.VariableCount > 0, + $"{context}/{preference}: -ErrorVariable collected nothing"); + Assert.StartsWith(ExpectedErrorId(context), outcome.FirstErrorId); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ExportCheckpointIntegrityTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ExportCheckpointIntegrityTests.cs index af80c78..cf6967a 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/ExportCheckpointIntegrityTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/ExportCheckpointIntegrityTests.cs @@ -1,6 +1,5 @@ using System.Management.Automation; using System.Net; -using System.Reflection; using System.Text; using Mgx.Cmdlets.Base; using Mgx.Engine.Http; @@ -53,33 +52,8 @@ protected override Task SendAsync( } } - private static void InjectMock(HttpMessageHandler handler) - { - ResiliencePipelineFactory.Reset(); - var t = typeof(MgxCmdletBase); - t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, new HttpClient(handler)); - t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - t.GetField("s_ownsHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, false); - t.GetField("s_graphEndpoint", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! - .SetValue(null, "https://graph.microsoft.com"); - t.GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! - .SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); - } - - private static void CleanupMock() - { - var t = typeof(MgxCmdletBase); - t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - ResiliencePipelineFactory.Reset(); - } - - private static long? Export(string outputPath, string? checkpointPath, bool all, int top = 0) + private static long? Export(string outputPath, string? checkpointPath, bool all, int top = 0, + string uri = "/users") { using var ps = PowerShell.Create(); ps.AddCommand("Import-Module") @@ -91,7 +65,7 @@ private static void CleanupMock() ps.Commands.Clear(); var cmd = ps.AddCommand("Export-MgxCollection") - .AddParameter("Uri", "/users") + .AddParameter("Uri", uri) .AddParameter("OutputFile", outputPath); if (checkpointPath != null) cmd.AddParameter("CheckpointPath", checkpointPath); if (all) cmd.AddParameter("All"); @@ -127,7 +101,7 @@ public void A_resume_does_not_count_an_earlier_exports_output_as_this_ones_items try { var handler = new ScriptedHandler(); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); // Run 1: a one-item probe export. Completes, no checkpoint. handler.Queue(HttpStatusCode.OK, Page1); @@ -154,7 +128,6 @@ public void A_resume_does_not_count_an_earlier_exports_output_as_this_ones_items } finally { - CleanupMock(); try { Directory.Delete(dir, true); } catch { } } } @@ -175,7 +148,7 @@ public void A_transient_failure_keeps_the_temp_the_checkpoint_names() try { var handler = new ScriptedHandler(); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); handler.Queue(HttpStatusCode.OK, Page1); handler.Queue(HttpStatusCode.InternalServerError, ServerError); @@ -189,7 +162,6 @@ public void A_transient_failure_keeps_the_temp_the_checkpoint_names() } finally { - CleanupMock(); try { Directory.Delete(dir, true); } catch { } } } @@ -207,7 +179,7 @@ public void An_export_resumes_after_a_transient_failure_rather_than_starting_ove try { var handler = new ScriptedHandler(); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); handler.Queue(HttpStatusCode.OK, Page1); handler.Queue(HttpStatusCode.InternalServerError, ServerError); @@ -225,7 +197,65 @@ public void An_export_resumes_after_a_transient_failure_rather_than_starting_ove } finally { - CleanupMock(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The last step of a completed export is the move that puts its temp over the output, and + /// that move can fail with every page already fetched: a destination another process holds + /// open, a read-only file, a share that dropped. Deleting the checkpoint before it made + /// that failure read as "nothing was resumable", and the temp holding the whole export was + /// deleted with it - hours of enumeration, at the one moment all of it was on disk. The two + /// files have to be left exactly as an interruption leaves them, and the next run has to + /// resume from them. + /// + [Fact] + public void A_failed_promotion_keeps_the_checkpoint_and_the_temp_holding_the_export() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // Something at the output path the promotion cannot replace, so both pages are + // fetched and written and the move at the end of the run is what fails. + Directory.CreateDirectory(output); + + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + Assert.Null(Export(output, checkpoint, all: true)); + + Assert.True(File.Exists(checkpoint), + "the failed promotion deleted the position the next run resumes from"); + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.NotNull(cp!.TempFile); + var temp = Path.Combine(dir, cp.TempFile!); + Assert.True(File.Exists(temp), + "the failed promotion deleted the temp holding every item the export fetched"); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(temp)); + + // With the occupier gone: the temp is promoted, one page is fetched - the one the + // checkpoint recorded - and the export is not enumerated again from the start. + Directory.Delete(output); + handler.Queue(HttpStatusCode.OK, Page2); + var before = handler.Requests.Count; + var reported = Export(output, checkpoint, all: true); + + Assert.Equal(1, handler.Requests.Count - before); + Assert.Contains("skiptoken=P2", handler.Requests[before]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], Lines(output)); + Assert.Equal(3, reported); + Assert.False(File.Exists(checkpoint)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { try { Directory.Delete(dir, true); } catch { } } } @@ -243,7 +273,7 @@ public void A_resume_replaces_the_previous_export_rather_than_appending_to_it() try { var handler = new ScriptedHandler(); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); handler.Queue(HttpStatusCode.OK, Page1); handler.Queue(HttpStatusCode.OK, Page2); @@ -263,7 +293,6 @@ public void A_resume_replaces_the_previous_export_rather_than_appending_to_it() } finally { - CleanupMock(); try { Directory.Delete(dir, true); } catch { } } } @@ -282,7 +311,7 @@ public void A_surviving_temp_is_promoted_over_an_earlier_exports_output() try { var handler = new ScriptedHandler(); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); handler.Queue(HttpStatusCode.OK, Page1); Assert.Equal(1, Export(output, checkpointPath: null, all: false, top: 1)); @@ -298,6 +327,7 @@ public void A_surviving_temp_is_promoted_over_an_earlier_exports_output() ItemsCollected = 2, PageItemsAlreadyWritten = 0, TempFile = Path.GetFileName(temp), + OutputFile = output, DataLength = new FileInfo(temp).Length }.Save(checkpoint); @@ -310,7 +340,6 @@ public void A_surviving_temp_is_promoted_over_an_earlier_exports_output() } finally { - CleanupMock(); try { Directory.Delete(dir, true); } catch { } } } @@ -331,7 +360,7 @@ public void A_checkpoint_that_cannot_say_where_its_items_are_exports_again_rathe try { var handler = new ScriptedHandler(); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); // A previous export completed and left its output behind. File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n{\"id\":\"old3\"}\n"); @@ -358,7 +387,6 @@ public void A_checkpoint_that_cannot_say_where_its_items_are_exports_again_rathe } finally { - CleanupMock(); try { Directory.Delete(dir, true); } catch { } } } @@ -386,7 +414,7 @@ static string BigPage(string nextLink) => try { var handler = new ScriptedHandler(); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); // One real run, only so the checkpoint on disk carries the exact URL the cmdlet // builds for these parameters. @@ -428,7 +456,7 @@ static string BigPage(string nextLink) => Assert.Equal(lines.Length, lines.Distinct().Count()); Assert.Equal(1001, lines.Length); } - finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + finally { try { Directory.Delete(dir, true); } catch { } } } /// @@ -446,7 +474,7 @@ public void A_checkpoint_recording_a_negative_length_does_not_wedge_the_export() try { var handler = new ScriptedHandler(); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); File.WriteAllText(output, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); new PaginationCheckpoint @@ -470,7 +498,6 @@ public void A_checkpoint_recording_a_negative_length_does_not_wedge_the_export() } finally { - CleanupMock(); try { Directory.Delete(dir, true); } catch { } } } @@ -490,7 +517,7 @@ public void A_fresh_export_does_not_delete_a_temp_another_run_is_writing() try { var handler = new ScriptedHandler(); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); // Another export holds its temp open, exactly as StreamWriter does. using (var held = new FileStream(live, FileMode.Create, FileAccess.Write, FileShare.Read)) @@ -510,7 +537,219 @@ public void A_fresh_export_does_not_delete_a_temp_another_run_is_writing() } finally { - CleanupMock(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A checkpoint written by a release that recorded no output file, against the output it + /// was in fact collecting into. The files corroborate it - the output is there and holds + /// the bytes it counted - so an upgrade partway through an export still resumes. + /// + [Fact] + public void A_checkpoint_from_before_outputs_were_recorded_resumes_against_its_own_output() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + File.WriteAllText(output, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + DataLength = new FileInfo(output).Length, + }.Save(checkpoint); + + handler.Queue(HttpStatusCode.OK, Page2); + var before = handler.Requests.Count; + var reported = Export(output, checkpoint, all: true); + + Assert.Contains("skiptoken=P2", handler.Requests[before]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], Lines(output)); + Assert.Equal(3, reported); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The layout a pre-2.1.0 release left when a fresh export was killed: a checkpoint naming + /// no temp and recording no length, its items in a temp beside an output that was never + /// promoted. Deciding ownership from what such a checkpoint records refuses it for want of + /// anything recorded, and the export then swept the temp holding those items and enumerated + /// the collection again. What is on disk decides it instead - a temp carrying this output's + /// own name, and no output for its items to be confused with. + /// + [Fact] + public void A_checkpoint_from_before_temps_were_recorded_adopts_the_temp_holding_its_items() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // A fresh run's first page, in the temp it died holding. + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0 + }.Save(checkpoint); + + handler.Queue(HttpStatusCode.OK, Page2); + var before = handler.Requests.Count; + var reported = Export(output, checkpoint, all: true); + + // Resumed onto the recovered items rather than fetching them a second time. + Assert.Contains("skiptoken=P2", handler.Requests[before]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], Lines(output)); + Assert.Equal(3, reported); + Assert.False(File.Exists(temp), "the temp its items came from was left behind"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same shape of checkpoint, against an output it was never measured against. Nothing + /// in it names a file, and a null read as "mine" let a second export act on the first + /// one's position - here by declaring the temp it names unusable and deleting the + /// checkpoint, which is the one thing that made the first export's items unreachable. + /// + [Fact] + public void A_checkpoint_from_before_outputs_were_recorded_is_not_this_exports_by_default() + { + var dir = NewDir(); + var mine = Path.Combine(dir, "users.jsonl"); + var theirs = Path.Combine(dir, "other.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // An export to users.jsonl collects page 1 into its temp and dies on page 2. + handler.Queue(HttpStatusCode.OK, Page1); + Export(mine, checkpoint, all: true); + var temp = Assert.Single(Directory.GetFiles(dir, "users.jsonl.*.tmp")); + + // Its checkpoint, as the release before this one would have written it. + var upgraded = PaginationCheckpoint.Load(checkpoint)!; + upgraded.OutputFile = null; + upgraded.Save(checkpoint); + + // A second export shares the checkpoint path and fails on its first page. + Export(theirs, checkpoint, all: true); + + Assert.True(File.Exists(checkpoint), "the other export's position was deleted"); + Assert.True(File.Exists(temp), "the other export's temp was taken"); + + // The first export still resumes onto what it had already collected. + handler.Queue(HttpStatusCode.OK, Page2); + var reported = Export(mine, checkpoint, all: true); + + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], Lines(mine)); + Assert.Equal(3, reported); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same export, resumed with -Uri typed differently. Graph answers "/users" and + /// "/Users" from one collection, so both runs enumerate the same thing - but the recorded + /// resource was compared ordinally, which made the second run a different export: the + /// resume was refused, the collection was fetched again from its first page, and the + /// caller was told their checkpoint belonged to something else. + /// + [Fact] + public void A_checkpoint_written_under_another_spelling_of_the_resource_still_resumes() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + handler.Queue(HttpStatusCode.OK, Page1); + Export(output, checkpoint, all: true, uri: "/users"); + + handler.Queue(HttpStatusCode.OK, Page2); + var before = handler.Requests.Count; + var reported = Export(output, checkpoint, all: true, uri: "/Users"); + + Assert.Contains("skiptoken=P2", handler.Requests[before]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], Lines(output)); + Assert.Equal(3, reported); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The endpoint a session reports is not stable between runs: it comes back with a trailing + /// slash, through a gateway prefix, or from a sovereign cloud. None of that changes which + /// resource an export enumerates, but comparing whole URLs made it change whose checkpoint + /// the file on disk was - so the resume was refused and the collection re-enumerated from + /// the first page. + /// + [Fact] + public void A_checkpoint_written_under_another_spelling_of_the_endpoint_still_resumes() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + + using (MgxTransportScope.Inject(handler, endpoint: "https://graph.microsoft.com/")) + { + handler.Queue(HttpStatusCode.OK, Page1); + Export(output, checkpoint, all: true); + } + + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.Contains("//v1.0/users", cp!.Resource); + + using (MgxTransportScope.Inject(handler, endpoint: "https://graph.microsoft.com")) + { + handler.Queue(HttpStatusCode.OK, Page2); + var before = handler.Requests.Count; + var reported = Export(output, checkpoint, all: true); + + Assert.Contains("skiptoken=P2", handler.Requests[before]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], Lines(output)); + Assert.Equal(3, reported); + } + } + finally + { try { Directory.Delete(dir, true); } catch { } } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ExportCorruptCheckpointTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ExportCorruptCheckpointTests.cs new file mode 100644 index 0000000..6c3deac --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/ExportCorruptCheckpointTests.cs @@ -0,0 +1,216 @@ +using System.Management.Automation; +using System.Net; +using System.Text; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// A checkpoint file that cannot be read is not a resume. PaginationCheckpoint.Load answers +/// null for a truncated file, an empty one, a field of the wrong JSON type and a file the +/// account cannot open - and an export that decided to append from the checkpoint merely +/// EXISTING wrote a second complete copy of the collection onto the first, reporting the +/// item count of one pass while the file held two. +/// +[Collection("Pipeline")] +public class ExportCorruptCheckpointTests +{ + private const string Page1 = """ + {"value":[{"id":"u1"},{"id":"u2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2"} + """; + private const string Page2 = """ + {"value":[{"id":"u3"}]} + """; + private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + + private sealed class ScriptedHandler : HttpMessageHandler + { + private readonly Queue<(HttpStatusCode Status, string Body)> _steps = new(); + private readonly object _lock = new(); + + public void Queue(HttpStatusCode status, string body) + { + lock (_lock) _steps.Enqueue((status, body)); + } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + (HttpStatusCode Status, string Body) step; + lock (_lock) + step = _steps.Count > 0 ? _steps.Dequeue() : (HttpStatusCode.InternalServerError, ServerError); + return Task.FromResult(new HttpResponseMessage(step.Status) + { + RequestMessage = request, + Content = new StringContent(step.Body, Encoding.UTF8, "application/json") + }); + } + } + + private static long? Export(string outputPath, string? checkpointPath) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + var cmd = ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", outputPath) + .AddParameter("All"); + if (checkpointPath != null) cmd.AddParameter("CheckpointPath", checkpointPath); + try + { + foreach (var r in ps.Invoke()) + if (r?.BaseObject is Mgx.Cmdlets.Models.MgxExportResult summary) + return summary.ItemCount; + } + catch (CmdletInvocationException) { } + return null; + } + + private static string NewDir() => + Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-export-corrupt-{Guid.NewGuid():N}")).FullName; + + private static void ExportsOnceOver(string checkpointContent) + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // A completed export, with no checkpoint of its own. + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + Assert.Equal(3, Export(output, checkpointPath: null)); + + File.WriteAllText(checkpoint, checkpointContent); + + // Same export again, now pointed at a checkpoint nothing can read. + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + var reported = Export(output, checkpoint); + + var lines = File.ReadAllLines(output); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], lines); + Assert.Equal(3, reported); + Assert.Equal(lines.Length, reported); + Assert.False(File.Exists(checkpoint)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + [Fact] + public void A_truncated_checkpoint_does_not_append_a_whole_second_export() => + ExportsOnceOver("""{"resource":"https://graph.microsoft.com/v1.0/users?$top=999","nextLi"""); + + [Fact] + public void An_empty_checkpoint_does_not_append_a_whole_second_export() => + ExportsOnceOver(""); + + [Fact] + public void A_checkpoint_with_a_field_of_the_wrong_type_does_not_append_a_whole_second_export() => + ExportsOnceOver(""" + {"resource":"https://graph.microsoft.com/v1.0/users?$top=999", + "nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + "itemsCollected":"two","pageItemsAlreadyWritten":0,"tempFile":null,"dataLength":24} + """); + + /// + /// The same branch, reached by a file that is torn rather than closed: the crash that left + /// it got between the write and the rename. This is the one shape of "does not load" that + /// needs nothing of the filesystem to produce, so it is the shape that can be asserted + /// where a Unix file mode cannot go - Windows denies a read through an ACL, and the test + /// below returns before its first assertion there. + /// + [Fact] + public void A_checkpoint_this_run_cannot_parse_is_left_for_a_run_that_can() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + File.WriteAllText(checkpoint, + """{"resource":"https://graph.microsoft.com/v1.0/users?$top=999","nextLi"""); + var before = File.ReadAllBytes(checkpoint); + + // The export fails on its first page, so the success path - which deletes the + // checkpoint whatever it holds - is not what this measures. + Assert.Null(Export(output, checkpoint)); + + Assert.True(File.Exists(checkpoint), + "a position this run could not read was deleted anyway"); + Assert.Equal(before, File.ReadAllBytes(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// "Cannot be read" is a lock or a denying ACL as often as it is a torn file. Deleting the + /// checkpoint on that reading changed nothing about this run - a load that answers null + /// already forces a fresh export - while destroying a position the next run, or another + /// account, could still have resumed from. + /// + [Fact] + public void A_checkpoint_this_run_cannot_read_is_left_for_a_run_that_can() + { + // A file the account cannot open. Windows expresses that through ACLs, not a mode. + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = output, + DataLength = 24, + }.Save(checkpoint); + var before = File.ReadAllBytes(checkpoint); + File.SetUnixFileMode(checkpoint, UnixFileMode.None); + + // The export fails on its first page, so nothing that runs on success can be what + // removes the checkpoint. + Assert.Null(Export(output, checkpoint)); + + var survived = File.Exists(checkpoint); + if (survived) + { + File.SetUnixFileMode(checkpoint, UnixFileMode.UserRead | UnixFileMode.UserWrite); + Assert.Equal(before, File.ReadAllBytes(checkpoint)); + } + Assert.True(survived, "a position this run could not read was deleted anyway"); + } + finally + { + try { File.SetUnixFileMode(checkpoint, UnixFileMode.UserRead | UnixFileMode.UserWrite); } + catch { } + try { Directory.Delete(dir, true); } catch { } + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ExportDegradedQueryCheckpointTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ExportDegradedQueryCheckpointTests.cs new file mode 100644 index 0000000..77c96bf --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/ExportDegradedQueryCheckpointTests.cs @@ -0,0 +1,711 @@ +using System.Management.Automation; +using System.Net; +using System.Text; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// An endpoint that refuses $top, or the auto-added $count, sends the export round its attempt +/// loop again without that option - and the checkpoint it saves afterwards records the URL it +/// actually fetched. The recovery that runs before the loop compared only the URL the first +/// attempt builds, so an export's own checkpoint read as another export's: nothing was +/// recovered, the sweep deleted the temp holding its items, and the retry - which does build +/// that URL - appended the rest of the enumeration onto whatever file was at -OutputFile. +/// +[Collection("Pipeline")] +public class ExportDegradedQueryCheckpointTests +{ + private const string Page1 = """ + {"value":[{"id":"u1"},{"id":"u2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2"} + """; + private const string Page2 = """ + {"value":[{"id":"u3"}]} + """; + private const string TopUnsupported = """ + {"error":{"code":"Request_UnsupportedQuery","message":"$top is not supported on this endpoint"}} + """; + private const string CountUnsupported = """ + {"error":{"code":"BadRequest","message":"$count is not supported on this endpoint"}} + """; + private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + // A bare 400 off the resumed nextLink, with nothing to do with $count - which is all + // IsCountRejection needs to drop the auto-added one and send the loop round again. + private const string ExpiredSkipToken = """{"error":{"code":"BadRequest","message":"skip token expired"}}"""; + + private static readonly string[] OldExport = + ["{\"id\":\"old1\"}", "{\"id\":\"old2\"}", "{\"id\":\"old3\"}"]; + + /// + /// Answers by URL rather than in order: the option this endpoint refuses is refused every + /// time it appears, and page two fails until the second run asks for it. + /// + private sealed class RefusingHandler : HttpMessageHandler + { + private readonly object _lock = new(); + private readonly string _refused; + private readonly string _refusal; + + public RefusingHandler(string refused, string refusal) + { + _refused = refused; + _refusal = refusal; + } + + public List Requests { get; } = []; + public bool PageTwoFails { get; set; } = true; + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + HttpStatusCode status; + string body; + lock (_lock) + { + Requests.Add(url); + if (url.Contains(_refused, StringComparison.Ordinal)) + (status, body) = (HttpStatusCode.BadRequest, _refusal); + else if (!url.Contains("skiptoken=P2", StringComparison.Ordinal)) + (status, body) = (HttpStatusCode.OK, Page1); + else if (PageTwoFails) + (status, body) = (HttpStatusCode.InternalServerError, ServerError); + else + (status, body) = (HttpStatusCode.OK, Page2); + } + return Task.FromResult(new HttpResponseMessage(status) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } + + + /// + /// Refuses the option only once the first page has been collected: page one answers with + /// $top on it, and the nextLink that follows comes back Request_UnsupportedQuery. That is + /// what sends the attempt loop round again with a page-boundary checkpoint already on disk. + /// + private sealed class RefusingOnPageTwoHandler : HttpMessageHandler + { + private readonly object _lock = new(); + + public List Requests { get; } = []; + public bool PageTwoAnswers { get; set; } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + HttpStatusCode status; + string body; + lock (_lock) + { + Requests.Add(url); + if (url.Contains("skiptoken=P2", StringComparison.Ordinal)) + (status, body) = PageTwoAnswers + ? (HttpStatusCode.OK, Page2) + : (HttpStatusCode.BadRequest, TopUnsupported); + else if (url.Contains("$top=", StringComparison.Ordinal)) + (status, body) = (HttpStatusCode.OK, Page1); + else + // The retry, without the option the endpoint refused. It dies before its + // own first page boundary, so the checkpoint on disk is still the one the + // attempt before it saved. + (status, body) = (HttpStatusCode.InternalServerError, ServerError); + } + return Task.FromResult(new HttpResponseMessage(status) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } + + private static long? Export(string outputPath, string checkpointPath, string? filter = null) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + var cmd = ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("All"); + if (filter != null) cmd.AddParameter("Filter", filter); + try + { + foreach (var r in ps.Invoke()) + if (r?.BaseObject is Mgx.Cmdlets.Models.MgxExportResult summary) + return summary.ItemCount; + } + catch (CmdletInvocationException) { } + return null; + } + + /// + /// Answers the resumed nextLink with one bare 400 before it will serve page two. Page one + /// is served whatever form the URL is in, so the run that follows the refusal enumerates + /// the collection from the beginning. + /// + private sealed class RefusingTheResumedLinkHandler : HttpMessageHandler + { + private readonly object _lock = new(); + + public List Requests { get; } = []; + public bool PageTwoFails { get; set; } = true; + public bool RefuseTheResumedLinkOnce { get; set; } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + HttpStatusCode status; + string body; + lock (_lock) + { + Requests.Add(url); + if (!url.Contains("skiptoken=P2", StringComparison.Ordinal)) + (status, body) = (HttpStatusCode.OK, Page1); + else if (PageTwoFails) + (status, body) = (HttpStatusCode.InternalServerError, ServerError); + else if (RefuseTheResumedLinkOnce) + { + RefuseTheResumedLinkOnce = false; + (status, body) = (HttpStatusCode.BadRequest, ExpiredSkipToken); + } + else + (status, body) = (HttpStatusCode.OK, Page2); + } + return Task.FromResult(new HttpResponseMessage(status) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } + + private static (long? Count, List Warnings) ExportReporting( + string outputPath, string checkpointPath, string filter) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("Filter", filter) + .AddParameter("All"); + long? count = null; + try + { + foreach (var r in ps.Invoke()) + if (r?.BaseObject is Mgx.Cmdlets.Models.MgxExportResult summary) + count = summary.ItemCount; + } + catch (CmdletInvocationException) { } + return (count, [.. ps.Streams.Warning.Select(w => w.Message)]); + } + + private static string NewDir() => + Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-export-degraded-{Guid.NewGuid():N}")).FullName; + + // Page one with no nextLink at all: an endpoint that pages the request carrying the option + // and answers the one without it in a single page. + private const string WholeCollection = """ + {"value":[{"id":"u1"},{"id":"u2"}]} + """; + + /// + /// Serves page one whatever form the URL is in, and hands back a nextLink carrying the same + /// options - so the refusal can land on the CONTINUATION, which is the shape that leaves a + /// page-boundary checkpoint and the temp it names on disk before the attempt loop goes + /// round again. The refusal on $count says nothing about $count: a bare 400 is all a run + /// that auto-added it asks for, and an expired skiptoken is the same 400. + /// + private sealed class RefusingTheContinuationHandler : HttpMessageHandler + { + private readonly object _lock = new(); + + public List Requests { get; } = []; + public bool RefuseCount { get; set; } = true; + public bool RefuseTop { get; set; } + /// A continuation with nothing left to refuse dies instead of answering. + public bool ContinuationFails { get; set; } + /// The retry's page one is the whole collection, so it saves no checkpoint. + public bool RetryTakesOnePage { get; set; } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + var hasCount = url.Contains("$count=true", StringComparison.Ordinal); + var hasTop = url.Contains("$top=", StringComparison.Ordinal); + HttpStatusCode status; + string body; + lock (_lock) + { + Requests.Add(url); + if (url.Contains("skiptoken=P2", StringComparison.Ordinal)) + { + if (RefuseCount && hasCount) + (status, body) = (HttpStatusCode.BadRequest, CountUnsupported); + else if (RefuseTop && hasTop) + (status, body) = (HttpStatusCode.BadRequest, TopUnsupported); + else if (ContinuationFails) + (status, body) = (HttpStatusCode.InternalServerError, ServerError); + else + (status, body) = (HttpStatusCode.OK, Page2); + } + else if (RetryTakesOnePage && !hasCount) + (status, body) = (HttpStatusCode.OK, WholeCollection); + else + { + var link = "https://graph.microsoft.com/v1.0/users?" + + (hasCount ? "$count=true&" : "") + + (hasTop ? "$top=100&" : "") + + "$skiptoken=P2"; + body = "{\"value\":[{\"id\":\"u1\"},{\"id\":\"u2\"}],\"@odata.nextLink\":\"" + link + "\"}"; + status = HttpStatusCode.OK; + } + } + return Task.FromResult(new HttpResponseMessage(status) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } + + /// + /// The endpoint refuses $top, and the -OutputFile already holds a completed export. The + /// interrupted run's items are in a temp; the run that follows has to recover them and + /// resume, and what it must never do is treat the earlier export's rows as its own first + /// pages. + /// + [Fact] + public void An_export_resumes_its_own_checkpoint_after_the_endpoint_refused_top() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new RefusingHandler("$top=", TopUnsupported); + using var transport = MgxTransportScope.Inject(handler); + + // A completed export is already sitting at -OutputFile. + File.WriteAllLines(output, OldExport); + + // Run one: refused $top, retried without it, collected page one into its temp and + // died on page two. + Export(output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.DoesNotContain("$top=", cp!.Resource); + Assert.NotNull(cp.TempFile); + Assert.Equal(OldExport, File.ReadAllLines(output)); + + // Run two: the same export again, with page two answering this time. + handler.PageTwoFails = false; + var reported = Export(output, checkpoint); + + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(3, reported); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same defect on the other option, in the shape where it costs the resume rather than + /// the file: the endpoint refuses the auto-added $count, and the run that died left no + /// output - which is what the accommodation this replaces needed to see before it looked at + /// the checkpoint's URL at all. + /// + [Fact] + public void An_export_resumes_its_own_checkpoint_after_the_endpoint_refused_count() + { + const string filter = "startsWith(displayName,'a')"; + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new RefusingHandler("$count=true", CountUnsupported); + using var transport = MgxTransportScope.Inject(handler); + + Export(output, checkpoint, filter); + Assert.False(File.Exists(output), "a fresh run promotes nothing when it dies"); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.DoesNotContain("$count=true", cp!.Resource); + + handler.PageTwoFails = false; + var first = handler.Requests.Count; + var reported = Export(output, checkpoint, filter); + + // Resumed: the first thing the second run asks for is the page the first never got. + Assert.Contains("skiptoken=P2", handler.Requests[first]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(3, reported); + Assert.False(File.Exists(temp), "the temp its items came from was left behind"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The retry's own sweep, against a checkpoint that is this run's. Dropping $count or $top + /// sends the attempt loop round again, and the attempt that died kept its temp precisely + /// because the checkpoint counting those items survived it. The sweep at the top of the + /// next attempt then deleted the file that checkpoint names, so the next invocation found a + /// position pointing at nothing and enumerated from the first page - unbounded work thrown + /// away, since a checkpoint is saved at every page boundary. + /// + [Fact] + public void A_retry_does_not_sweep_the_temp_this_runs_own_checkpoint_names() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new RefusingOnPageTwoHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // Page one lands in a temp and is checkpointed; the nextLink is refused, the loop + // retries without $top, and that attempt dies before a boundary of its own. + Assert.Null(Export(output, checkpoint)); + + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.NotNull(cp!.TempFile); + Assert.True(File.Exists(Path.Combine(dir, cp.TempFile!)), + "the retry's sweep deleted the temp this run's own checkpoint names"); + + // The next invocation resumes where the first left off rather than starting over. + handler.PageTwoAnswers = true; + var first = handler.Requests.Count; + var reported = Export(output, checkpoint); + + Assert.Contains("skiptoken=P2", handler.Requests[first]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(3, reported); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other end of the same rebuild. The run recovers the interrupted export's temp, + /// reports that it is resuming, and then the resumed nextLink comes back 400 - any 400 on a + /// run that auto-added $count is read as the endpoint refusing it, so the loop goes round + /// with the URL rebuilt without it. The checkpoint was written under the previous form and + /// is no longer recognized, and the resume just announced is withdrawn. Every sibling + /// refusal on this path says so; this one restarted the enumeration with nothing on any + /// stream but the recovery message, so the run looked from the outside like one that had + /// resumed. + /// + [Fact] + public void An_export_says_so_when_a_rebuilt_request_costs_it_the_resume_it_announced() + { + const string filter = "id ne null"; + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new RefusingTheResumedLinkHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // Run one: page one collected under the auto-added $count, page two dies. + Export(output, checkpoint, filter); + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.Contains("$count=true", cp!.Resource); + Assert.False(File.Exists(output), "a fresh run promotes nothing when it dies"); + + // Run two: the temp is recovered and the resume announced, then the nextLink is + // refused and the URL is rebuilt into a form the checkpoint was not written under. + handler.PageTwoFails = false; + handler.RefuseTheResumedLinkOnce = true; + var (reported, warnings) = ExportReporting(output, checkpoint, filter); + + Assert.Contains(warnings, w => w.Contains("Resuming from checkpoint")); + Assert.Contains(warnings, w => w.Contains("no longer describes this export")); + + // And the run really did start over, which is what the warning is about. + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(3, reported); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// What a retry leaves beside the output once it succeeds. The attempt that died kept its + /// temp because a checkpoint counting those items survived it; the attempt that followed + /// saved a checkpoint of its own over that one, naming its own temp, and from that moment + /// nothing on disk referred to the first. It was spared anyway - the sweep is held off for + /// as long as this run remembers keeping it - so a reported-successful export left a + /// partial copy of the caller's data sitting beside the finished file, for some later + /// export's sweep to reclaim. + /// + [Fact] + public void A_successful_retry_leaves_no_temp_from_the_attempt_it_abandoned() + { + const string filter = "startsWith(displayName,'a')"; + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new RefusingTheContinuationHandler(); + using var transport = MgxTransportScope.Inject(handler); + + var reported = Export(output, checkpoint, filter); + + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(3, reported); + Assert.False(File.Exists(checkpoint)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same, chained: the endpoint refuses the auto-added $count on the continuation and + /// then $top on the next one, so two attempts are abandoned with a temp each before the + /// third finishes. + /// + [Fact] + public void A_second_retry_leaves_no_temp_from_either_attempt_before_it() + { + const string filter = "startsWith(displayName,'a')"; + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new RefusingTheContinuationHandler { RefuseTop = true }; + using var transport = MgxTransportScope.Inject(handler); + + var reported = Export(output, checkpoint, filter); + + // Page one and a refused continuation, twice, and then the pair that answered. + Assert.Equal(6, handler.Requests.Count); + Assert.DoesNotContain("$count=true", handler.Requests[5]); + Assert.DoesNotContain("$top=", handler.Requests[5]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(3, reported); + Assert.False(File.Exists(checkpoint)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And it goes at the moment it stops being anything, not at the end of the run. The retry + /// saves its first page-boundary checkpoint - which names its own temp, so the one before + /// it is now counted by nothing - and then dies, promoting nothing. What is left is the one + /// temp the checkpoint on disk names, and the run after it resumes from exactly that, so + /// what went was never something a resume needed. + /// + [Fact] + public void A_retry_deletes_the_temp_its_own_checkpoint_has_stopped_naming() + { + const string filter = "startsWith(displayName,'a')"; + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new RefusingTheContinuationHandler { ContinuationFails = true }; + using var transport = MgxTransportScope.Inject(handler); + + Assert.Null(Export(output, checkpoint, filter)); + + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.NotNull(cp!.TempFile); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(cp.TempFile, Path.GetFileName(temp)); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"], File.ReadAllLines(temp)); + + handler.ContinuationFails = false; + var first = handler.Requests.Count; + var reported = Export(output, checkpoint, filter); + + Assert.Contains("skiptoken=P2", handler.Requests[first]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(3, reported); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The retry that never saves a checkpoint at all: the endpoint pages the request carrying + /// the option it refuses and answers the one without it in a single page, so the attempt + /// that finishes reaches no page boundary and writes over nothing. The temp the attempt + /// before it left is released by the run finishing instead. + /// + [Fact] + public void A_retry_that_saves_no_checkpoint_of_its_own_leaves_no_temp_behind() + { + const string filter = "startsWith(displayName,'a')"; + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new RefusingTheContinuationHandler { RetryTakesOnePage = true }; + using var transport = MgxTransportScope.Inject(handler); + + var reported = Export(output, checkpoint, filter); + + // Page one, the continuation it refused, and the retry's single page: nothing the + // attempt that finished saved can have been what tidied up after the one before it. + Assert.Equal(3, handler.Requests.Count); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"], File.ReadAllLines(output)); + Assert.Equal(2, reported); + Assert.False(File.Exists(checkpoint)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other end of it: a retry that fetched every page and then could not move its temp + /// over the output. Nothing is promoted and nothing is finished, so the checkpoint and the + /// temp it names have to be left exactly as an interruption leaves them - the abandoned + /// attempt's temp is the only thing that goes - and the next run resumes from them. + /// + [Fact] + public void A_failed_promotion_after_a_retry_keeps_what_the_next_run_resumes_from() + { + const string filter = "startsWith(displayName,'a')"; + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new RefusingTheContinuationHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // Something at the output path the promotion cannot replace, so every page is + // fetched and written and the move at the end of the retry is what fails. + Directory.CreateDirectory(output); + + Assert.Null(Export(output, checkpoint, filter)); + + Assert.True(File.Exists(checkpoint), + "the failed promotion deleted the position the next run resumes from"); + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.NotNull(cp!.TempFile); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(cp.TempFile, Path.GetFileName(temp)); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(temp)); + + // With the occupier gone: the temp is promoted, the page the checkpoint recorded is + // the one page fetched, and the collection is not enumerated again from the start. + Directory.Delete(output); + var first = handler.Requests.Count; + var reported = Export(output, checkpoint, filter); + + Assert.Equal(1, handler.Requests.Count - first); + Assert.Contains("skiptoken=P2", handler.Requests[first]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(3, reported); + Assert.False(File.Exists(checkpoint)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The scenario above, carried through to the invocation that finishes. The retry that dies + /// before a boundary of its own has a temp too, and the checkpoint on disk - the attempt + /// before it saved that - counts nothing in it: keeping it left a file no recovery can + /// reach beside the one that is resumed from, and the invocation that did resume appended + /// to the output, so no sweep ever came for it. A finished export leaves its output. + /// + [Fact] + public void A_retry_that_dies_before_a_boundary_leaves_only_the_temp_its_checkpoint_names() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new RefusingOnPageTwoHandler(); + using var transport = MgxTransportScope.Inject(handler); + + Assert.Null(Export(output, checkpoint)); + + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.NotNull(cp!.TempFile); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(cp.TempFile, Path.GetFileName(temp)); + + handler.PageTwoAnswers = true; + var reported = Export(output, checkpoint); + + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(3, reported); + Assert.False(File.Exists(checkpoint)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ExportSharedCheckpointTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ExportSharedCheckpointTests.cs new file mode 100644 index 0000000..24b729d --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/ExportSharedCheckpointTests.cs @@ -0,0 +1,542 @@ +using System.Management.Automation; +using System.Net; +using System.Text; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// A checkpoint's recorded length is a byte offset into "the output", and the recovery path +/// applies it to whatever -OutputFile the run names. Nothing in a checkpoint used to say which +/// file those bytes were counted in, so one -CheckpointPath reused by two exports cut a file +/// the checkpoint knew nothing about - mid-line, with the resumed pages appended onto the torn +/// byte, and no warning on any stream. +/// +[Collection("Pipeline")] +public class ExportSharedCheckpointTests +{ + private const string UsersPage1 = """ + {"value":[{"id":"u1"},{"id":"u2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2"} + """; + private const string UsersPage2 = """ + {"value":[{"id":"u3"}]} + """; + private const string GroupsPage1 = """ + {"value":[{"id":"g1"},{"id":"g2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/groups?$skiptoken=P2"} + """; + private const string GroupsPage2 = """ + {"value":[{"id":"g3"}]} + """; + private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + + private static readonly string[] OldExport = + ["{\"id\":\"old-0000001\"}", "{\"id\":\"old-0000002\"}", "{\"id\":\"old-0000003\"}"]; + + private sealed class ScriptedHandler : HttpMessageHandler + { + private readonly Queue<(HttpStatusCode Status, string Body)> _steps = new(); + private readonly object _lock = new(); + + public void Queue(HttpStatusCode status, string body) + { + lock (_lock) _steps.Enqueue((status, body)); + } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + (HttpStatusCode Status, string Body) step; + lock (_lock) + step = _steps.Count > 0 ? _steps.Dequeue() : (HttpStatusCode.InternalServerError, ServerError); + return Task.FromResult(new HttpResponseMessage(step.Status) + { + RequestMessage = request, + Content = new StringContent(step.Body, Encoding.UTF8, "application/json") + }); + } + } + + private static long? Export(string uri, string outputPath, string checkpointPath) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", uri) + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("All"); + try + { + foreach (var r in ps.Invoke()) + if (r?.BaseObject is Mgx.Cmdlets.Models.MgxExportResult summary) + return summary.ItemCount; + } + catch (CmdletInvocationException) { } + return null; + } + + private static void ExportNoCheckpoint(string uri, string outputPath) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", uri) + .AddParameter("OutputFile", outputPath) + .AddParameter("All"); + ps.Invoke(); + } + + private static string[] ExportWarnings(string uri, string outputPath, string checkpointPath) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", uri) + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("All"); + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + return [.. ps.Streams.Warning.Select(w => w.Message)]; + } + + private static string NewDir() => + Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-export-shared-{Guid.NewGuid():N}")).FullName; + + /// + /// What each refusal says. A checkpoint recording another export's output is a second + /// export over one -CheckpointPath, and naming that is what the caller acts on. One + /// recording no output at all is not: it was believed while the files beside it + /// corroborated it, and a single export whose temp has since gone, or whose output has been + /// replaced, reaches the same refusal. Reporting a different export there names a cause the + /// caller can go and check and find nothing behind. + /// + [Fact] + public void A_refusal_reports_the_cause_it_can_show() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + File.WriteAllLines(output, OldExport); + + // Written before outputs were recorded, counting more bytes than this output holds. + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = null, + DataLength = new FileInfo(output).Length + 4096, + }.Save(checkpoint); + + var uncorroborated = ExportWarnings("/users", output, checkpoint); + Assert.Contains(uncorroborated, w => w.Contains("no longer corroborate")); + Assert.DoesNotContain(uncorroborated, w => w.Contains("belongs to a different export")); + + // The same position, recording an output this run is not writing to. + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = Path.Combine(dir, "elsewhere.jsonl"), + DataLength = 24, + }.Save(checkpoint); + + Assert.Contains(ExportWarnings("/users", output, checkpoint), + w => w.Contains("belongs to a different export")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// Two exports of the same resource to different output files, sharing one checkpoint path. + /// The second run's output must not be cut back to the first run's byte count. + /// + [Fact] + public void A_checkpoint_from_another_export_does_not_cut_this_ones_output() + { + var dir = NewDir(); + var outA = Path.Combine(dir, "outA.jsonl"); + var outB = Path.Combine(dir, "outB.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // Export A dies on page 2, twice: the second run promotes the temp, so the + // checkpoint it leaves records a length with no temp to explain it. + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", outA, checkpoint); + Export("/users", outA, checkpoint); + + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + Assert.Equal(new FileInfo(outA).Length, cp.DataLength); + + // Export B: same resource, its own output, which already holds an earlier export. + File.WriteAllLines(outB, OldExport); + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + var reported = Export("/users", outB, checkpoint); + + var lines = File.ReadAllLines(outB); + Assert.All(lines, l => Assert.StartsWith("{\"id\":\"", l)); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], lines); + Assert.Equal(3, reported); + + // A's own output is untouched by B. + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"], File.ReadAllLines(outA)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A checkpoint left by an export of a DIFFERENT resource, against the same output name. + /// The resource is compared before anything is cut, so a run that then fails leaves the + /// output it never replaced exactly as it found it. + /// + [Fact] + public void A_checkpoint_for_another_resource_does_not_destroy_an_output_the_run_never_replaces() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // An interrupted export of /groups, resumed once so its checkpoint names no temp. + handler.Queue(HttpStatusCode.OK, GroupsPage1); + Export("/groups", output, checkpoint); + Export("/groups", output, checkpoint); + Assert.Equal(2, File.ReadAllLines(output).Length); + + // A later export replaces the output without touching that checkpoint. + File.WriteAllLines(output, OldExport); + + // An export of /users to the same file, which fails on its first page. + var reported = Export("/users", output, checkpoint); + + Assert.Null(reported); + Assert.Equal(OldExport, File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A checkpoint that is not this export's is refused, and the refusal used to depend on + /// deleting it - an unchecked delete, so a checkpoint in a directory this account cannot + /// write survived, the loop below decided to append from it merely existing, and the other + /// export's remaining pages were appended onto a file that had never held its first ones. + /// + [Fact] + public void A_refused_checkpoint_that_cannot_be_deleted_appends_nothing_foreign() + { + // The checkpoint sits in a directory the run cannot write, which is what makes its + // deletion fail. Windows expresses that through ACLs, not a mode. + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var vault = Directory.CreateDirectory(Path.Combine(dir, "vault")).FullName; + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(vault, "run.checkpoint"); + try + { + File.WriteAllLines(output, OldExport); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = Path.Combine(dir, "elsewhere.jsonl"), + DataLength = new FileInfo(output).Length + }.Save(checkpoint); + File.SetUnixFileMode(vault, + UnixFileMode.UserRead | UnixFileMode.UserExecute); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + var reported = Export("/users", output, checkpoint); + + var lines = File.ReadAllLines(output); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], lines); + Assert.Equal(3, reported); + Assert.DoesNotContain(lines, l => l.Contains("old-")); + } + finally + { + try + { + File.SetUnixFileMode(vault, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// What the refusal leaves behind. The checkpoint and the staging file beside it are the + /// other export's resume position, and deleting them cost that export its progress while + /// this run went on to recreate the same collision at its first page boundary. + /// + [Fact] + public void A_refused_checkpoint_and_its_staging_file_are_left_where_they_are() + { + var dir = NewDir(); + var outA = Path.Combine(dir, "outA.jsonl"); + var outB = Path.Combine(dir, "outB.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // Export A dies on page 2 and leaves its position behind. + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", outA, checkpoint); + var before = File.ReadAllBytes(checkpoint); + // A save that died between writing the staging file and renaming it. + var staged = checkpoint + ".tmp"; + File.WriteAllBytes(staged, before); + + // Export B, its own output, sharing A's checkpoint path, failing on its first page. + Export("/users", outB, checkpoint); + + Assert.True(File.Exists(checkpoint), "the other export's resume position was deleted"); + Assert.Equal(before, File.ReadAllBytes(checkpoint)); + Assert.True(File.Exists(staged), "the other export's staging file was deleted"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// Two exports of the same resource to files that share a name in different directories, + /// over one checkpoint path. The name is all a leaf comparison sees, so the second export + /// adopted the first one's position and cut its file to the first one's byte count. + /// + [Fact] + public void A_checkpoint_for_the_same_name_in_another_directory_does_not_cut_this_ones_output() + { + var dir = NewDir(); + var outA = Path.Combine(Directory.CreateDirectory(Path.Combine(dir, "a")).FullName, "users.jsonl"); + var outB = Path.Combine(Directory.CreateDirectory(Path.Combine(dir, "b")).FullName, "users.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // Export A dies on page 2, twice: the second run promotes the temp, so the + // checkpoint it leaves records a length with no temp to explain it. + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", outA, checkpoint); + Export("/users", outA, checkpoint); + + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + Assert.Equal(new FileInfo(outA).Length, cp.DataLength); + + // Export B: same resource, same file name, its own directory, and its own earlier + // export already sitting there. + File.WriteAllLines(outB, OldExport); + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + var reported = Export("/users", outB, checkpoint); + + var lines = File.ReadAllLines(outB); + Assert.All(lines, l => Assert.StartsWith("{\"id\":\"", l)); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], lines); + Assert.Equal(3, reported); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"], File.ReadAllLines(outA)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A checkpoint for the same path under different query options - another -Top, another + /// -Filter - counts a different enumeration. Comparing paths and not queries let it through + /// the guard, so the output was cut to its byte count before the exact comparison that + /// refuses ever ran, and a run that then failed left the cut behind. + /// + [Fact] + public void A_checkpoint_whose_query_differs_does_not_cut_the_output_before_it_is_refused() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + File.WriteAllLines(output, OldExport); + var before = File.ReadAllBytes(output); + new PaginationCheckpoint + { + // Same path, half the page: an export of /users this one is not making. + Resource = "https://graph.microsoft.com/v1.0/users?$top=500", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = output, + DataLength = 24, + }.Save(checkpoint); + var checkpointBefore = File.ReadAllBytes(checkpoint); + + // This export fails on its first page, so anything the output loses, it loses here. + var reported = Export("/users", output, checkpoint); + + Assert.Null(reported); + Assert.Equal(before, File.ReadAllBytes(output)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// How far "it is left as it is" has to reach. The refused checkpoint's items are in the + /// temp it names, and the stale-temp sweep a few lines later deleted every temp beside the + /// output - including that one. The export it belongs to then came back to a position + /// pointing at a file that is gone and enumerated the collection again from the first page, + /// which for a long export is the whole cost the refusal was written to avoid. + /// + [Fact] + public void A_refused_checkpoints_temp_survives_the_run_that_refused_it() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // Export A dies on page 2: its two items are in its temp, the output was never + // promoted, and the checkpoint names both. + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", output, checkpoint); + + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.NotNull(cp.TempFile); + var temp = Path.Combine(dir, cp.TempFile); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"], File.ReadAllLines(temp)); + var before = File.ReadAllBytes(checkpoint); + + // Export B: another collection, the same -OutputFile and -CheckpointPath, failing + // before its first page boundary - so nothing it does on a boundary or at + // completion is what answers here. + Assert.Null(Export("/groups", output, checkpoint)); + + Assert.Equal(before, File.ReadAllBytes(checkpoint)); + Assert.True(File.Exists(temp), + "the refused checkpoint's items were swept away by the run that refused it"); + + // A, back where it left off, rather than at page 1. + handler.Queue(HttpStatusCode.OK, UsersPage2); + Assert.Equal(3, Export("/users", output, checkpoint)); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// How far "spare the temps a refused checkpoint names" reaches. The sweep only ever + /// deletes names a run gives its own temp - the output's name, a dot, 32 hex digits, + /// ".tmp" - so a refused checkpoint naming "users.jsonl.{guid}.tmp" beside an output called + /// "users" names a file this sweep could not touch either way. Skipping on it bought that + /// file nothing and cost this output its own orphans, which the pre-length adoption path + /// then picks up on a line count alone. + /// + [Fact] + public void A_refused_temp_this_sweep_could_never_reach_does_not_spare_this_outputs_orphans() + { + var dir = NewDir(); + var mine = Path.Combine(dir, "users"); + var theirs = Path.Combine(dir, "users.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // The export to "users" collects page one into its temp and dies on page two. + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", mine, checkpoint); + var orphan = Assert.Single(Directory.GetFiles(dir, "users.*.tmp")); + + // The export next door takes the shared checkpoint over: it refuses what it finds, + // saves its own position at its first page boundary, and dies on page two. The + // first export's temp is now an orphan - nothing on disk describes it. + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", theirs, checkpoint); + var theirTemp = Assert.Single(Directory.GetFiles(dir, "users.jsonl.*.tmp")); + + // "users" again. It refuses a checkpoint naming a temp beside a different output, + // and that name must not stand in the way of its own sweep. + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + Assert.Equal(3, Export("/users", mine, checkpoint)); + + Assert.False(File.Exists(orphan), + "a temp name the sweep could never reach suppressed this output's sweep"); + Assert.True(File.Exists(theirTemp), + "the refused checkpoint's own staging file was deleted"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ExportWhatIfCheckpointTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ExportWhatIfCheckpointTests.cs new file mode 100644 index 0000000..ee94e7f --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/ExportWhatIfCheckpointTests.cs @@ -0,0 +1,291 @@ +using System.Globalization; +using System.Management.Automation; +using System.Management.Automation.Host; +using System.Management.Automation.Runspaces; +using System.Net; +using System.Security; +using System.Text; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// -WhatIf on an export with a resume checkpoint. Recovery promotes temp files over outputs, +/// cuts outputs back to a recorded length and deletes checkpoints, and it ran above the +/// ShouldProcess gate - so the run that reported it would change nothing had already rewritten +/// the output and removed the position it was resuming from. The gate also has to name the +/// action a real run would take, which it cannot do from "a checkpoint file exists". +/// +[Collection("Pipeline")] +public class ExportWhatIfCheckpointTests +{ + private const string Page2 = """ + {"value":[{"id":"u3"}]} + """; + + private sealed class CountingHandler : HttpMessageHandler + { + public int Requests; + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref Requests); + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(Page2, Encoding.UTF8, "application/json") + }); + } + } + + /// The lines -WhatIf writes go to the host, so a test that reads them needs one. + private sealed class RecordingHost : PSHost + { + private readonly Guid _id = Guid.NewGuid(); + public RecordingHostUI Recorder { get; } = new(); + public override string Name => "MgxTestHost"; + public override Version Version => new(1, 0); + public override Guid InstanceId => _id; + public override PSHostUserInterface UI => Recorder; + public override CultureInfo CurrentCulture => CultureInfo.InvariantCulture; + public override CultureInfo CurrentUICulture => CultureInfo.InvariantCulture; + public override void EnterNestedPrompt() { } + public override void ExitNestedPrompt() { } + public override void NotifyBeginApplication() { } + public override void NotifyEndApplication() { } + public override void SetShouldExit(int exitCode) { } + } + + private sealed class RecordingHostUI : PSHostUserInterface + { + public List Lines { get; } = []; + public override PSHostRawUserInterface? RawUI => null; + public override void Write(string value) => Lines.Add(value); + public override void Write(ConsoleColor f, ConsoleColor b, string value) => Lines.Add(value); + public override void WriteLine(string value) => Lines.Add(value); + public override void WriteErrorLine(string value) => Lines.Add(value); + public override void WriteDebugLine(string value) => Lines.Add(value); + public override void WriteVerboseLine(string value) => Lines.Add(value); + public override void WriteWarningLine(string value) => Lines.Add(value); + public override void WriteProgress(long sourceId, ProgressRecord record) { } + public override string ReadLine() => string.Empty; + public override SecureString ReadLineAsSecureString() => new(); + public override Dictionary Prompt( + string caption, string message, System.Collections.ObjectModel.Collection descriptions) => []; + public override int PromptForChoice( + string caption, string message, + System.Collections.ObjectModel.Collection choices, int defaultChoice) => defaultChoice; + public override PSCredential PromptForCredential( + string caption, string message, string userName, string targetName) => PSCredential.Empty; + public override PSCredential PromptForCredential( + string caption, string message, string userName, string targetName, + PSCredentialTypes allowedCredentialTypes, PSCredentialUIOptions options) => PSCredential.Empty; + } + + private static string NewDir() => + Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-whatif-{Guid.NewGuid():N}")).FullName; + + /// Runs the export under -WhatIf and returns every line the gate wrote. + private static (List Lines, bool HadErrors) WhatIf(string output, string checkpoint) + { + var host = new RecordingHost(); + using var runspace = RunspaceFactory.CreateRunspace(host); + runspace.Open(); + using var ps = PowerShell.Create(); + ps.Runspace = runspace; + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", output) + .AddParameter("CheckpointPath", checkpoint) + .AddParameter("All") + .AddParameter("WhatIf", true); + ps.Invoke(); + return (host.Recorder.Lines, ps.HadErrors); + } + + private static void Export(string output, string checkpoint) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", output) + .AddParameter("CheckpointPath", checkpoint) + .AddParameter("All"); + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + } + + /// + /// What a hard kill leaves partway through a fresh checkpointed export: the previous + /// export's output, this run's items in a temp beside it, and a checkpoint naming both. + /// A real run promotes the temp over the output and appends the rest; -WhatIf has to say + /// so and touch none of the three. It also runs above the client, so it needs no session. + /// + [Fact] + public void WhatIf_promotes_nothing_and_names_the_append_a_real_run_would_do() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + try + { + File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n{\"id\":\"old3\"}\n"); + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = Path.GetFileName(temp), + OutputFile = output, + DataLength = new FileInfo(temp).Length, + }.Save(checkpoint); + + var outputBefore = File.ReadAllBytes(output); + var tempBefore = File.ReadAllBytes(temp); + var checkpointBefore = File.ReadAllBytes(checkpoint); + + // No transport and no Get-MgContext: reaching the client would end the run here. + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf asked for a Graph connection"); + Assert.Contains(lines, l => l.Contains("Append JSONL data") && l.Contains(output)); + Assert.Equal(outputBefore, File.ReadAllBytes(output)); + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.False(File.Exists(output + ".adopt")); + + // And the run it described does exactly that. + var handler = new CountingHandler(); + using var transport = MgxTransportScope.Inject(handler); + Export(output, checkpoint); + + Assert.Equal( + ["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same gate over a checkpoint that is another export's. A real run refuses it and + /// exports from the beginning, so that is what -WhatIf has to report - and the checkpoint, + /// which the refusal used to delete, is still there afterwards. + /// + [Fact] + public void WhatIf_over_another_exports_checkpoint_names_an_export_and_deletes_nothing() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = Path.Combine(dir, "elsewhere.jsonl"), + DataLength = 12, + }.Save(checkpoint); + + var outputBefore = File.ReadAllBytes(output); + var checkpointBefore = File.ReadAllBytes(checkpoint); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf asked for a Graph connection"); + Assert.Contains(lines, l => l.Contains("Export JSONL data") && l.Contains(output)); + Assert.DoesNotContain(lines, l => l.Contains("Append JSONL data")); + Assert.Equal(outputBefore, File.ReadAllBytes(output)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same gate over a checkpoint whose temp another export is still writing. A real run + /// takes nothing from that file and exports from the beginning, so reporting an append + /// described a recovery that would not have happened - and the report is the only thing a + /// caller running -WhatIf gets to act on. + /// + [Fact] + public void WhatIf_over_a_temp_a_running_export_holds_names_an_export() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + try + { + File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n"); + + // Opened the way an export opens its own temp, with two rows flushed and the run + // that wrote them still going. + using var live = new StreamWriter(temp, append: false); + live.WriteLine("{\"id\":\"u1\"}"); + live.WriteLine("{\"id\":\"u2\"}"); + live.Flush(); + + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = Path.GetFileName(temp), + OutputFile = output, + DataLength = new FileInfo(temp).Length, + }.Save(checkpoint); + + var outputBefore = File.ReadAllBytes(output); + var tempBefore = ReadShared(temp); + var checkpointBefore = File.ReadAllBytes(checkpoint); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf asked for a Graph connection"); + Assert.Contains(lines, l => l.Contains("Export JSONL data") && l.Contains(output)); + Assert.DoesNotContain(lines, l => l.Contains("Append JSONL data")); + Assert.Equal(outputBefore, File.ReadAllBytes(output)); + Assert.Equal(tempBefore, ReadShared(temp)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.False(File.Exists(output + ".adopt")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + // Reads a file the test's own live writer still holds open: on Windows the reader + // must offer ReadWrite sharing or the holder's write access denies the open. + private static byte[] ReadShared(string path) + { + using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + using var ms = new MemoryStream(); + fs.CopyTo(ms); + return ms.ToArray(); + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/HeaderWireTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/HeaderWireTests.cs index fb80fcd..6147360 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/HeaderWireTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/HeaderWireTests.cs @@ -1,5 +1,4 @@ using System.Net; -using System.Reflection; using Mgx.Engine.Http; namespace Mgx.IntegrationTests; @@ -10,42 +9,12 @@ namespace Mgx.IntegrationTests; [Collection("Pipeline")] public class HeaderWireTests { - private static readonly Type Base = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); - private const BindingFlags Static = BindingFlags.NonPublic | BindingFlags.Static; - - private static void InjectTransport(HttpMessageHandler wire) - { - Base.GetField("s_graphHttpClient", Static)!.SetValue(null, new HttpClient(wire)); - Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, - Mgx.Cmdlets.Base.MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); - Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); - Base.GetField("s_clientOptions", Static)!.SetValue(null, - new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); - } - - private static void ResetTransport() - { - // Restore every static InjectTransport touched - a later test in the collection - // that drives a cmdlet without injecting must not inherit this class's transport. - Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); - Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); - Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); - Base.GetField("s_cachedTotalTimeoutSeconds", Static)!.SetValue(null, 0); - Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); - Base.GetField("s_clientOptions", Static)!.SetValue(null, new ResilientGraphClientOptions()); - ResiliencePipelineFactory.Reset(); - } - [Fact] public void An_array_header_value_joins_as_an_http_list_not_a_type_name() { var wire = new MockHttpHandler(); wire.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { using var ps = System.Management.Automation.PowerShell.Create(); ps.AddCommand("Import-Module") @@ -64,6 +33,5 @@ public void An_array_header_value_joins_as_an_http_list_not_a_type_name() Assert.Equal("outlook.timezone=\"UTC\", IdType=\"ImmutableId\"", string.Join(", ", prefer)); Assert.DoesNotContain("System.", string.Join(",", prefer)); } - finally { ResetTransport(); } } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/NamedTempPromotionOwnershipTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/NamedTempPromotionOwnershipTests.cs new file mode 100644 index 0000000..0954fb7 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/NamedTempPromotionOwnershipTests.cs @@ -0,0 +1,372 @@ +using System.Management.Automation; +using System.Net; +using System.Text; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// Recovery from a checkpoint that NAMES its temp - every checkpoint this release writes. +/// The temp is taken by name rather than by the glob adoption searches with, which says which +/// file holds the counted items but nothing at all about whether anyone is still writing it: +/// a run whose export dies partway leaves that state on disk, and so does one that is still +/// going. Promotion copied the rows into its own output and unlinked the file underneath the +/// run that was filling it, which then wrote into an unlinked inode and failed its own +/// promotion naming the OUTPUT. A scheduled export overlapping itself reaches this with +/// nothing misconfigured. +/// +[Collection("Pipeline")] +public class NamedTempPromotionOwnershipTests +{ + private const string Page1 = """ + {"value":[{"id":"u1"},{"id":"u2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2"} + """; + private const string Page2 = """ + {"value":[{"id":"u3"}]} + """; + private const string Baseline = """ + {"value":[{"id":"a1"},{"id":"a2"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D1"} + """; + private const string ChangesPage1 = """ + {"value":[{"id":"b1"},{"id":"b2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2"} + """; + private const string ChangesPage2 = """ + {"value":[{"id":"b3"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D2"} + """; + private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + + /// + /// Answers what it was queued with, and 500s once the queue is empty - so a run dying on + /// the wire needs no assumption about how many times the client retries first. + /// + private sealed class ScriptedHandler : HttpMessageHandler + { + private readonly Queue<(HttpStatusCode Status, string Body)> _steps = new(); + private readonly object _lock = new(); + + public void Queue(HttpStatusCode status, string body) + { + lock (_lock) _steps.Enqueue((status, body)); + } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + (HttpStatusCode Status, string Body) step; + lock (_lock) + step = _steps.Count > 0 ? _steps.Dequeue() : (HttpStatusCode.InternalServerError, ServerError); + return Task.FromResult(new HttpResponseMessage(step.Status) + { + RequestMessage = request, + Content = new StringContent(step.Body, Encoding.UTF8, "application/json") + }); + } + } + + private static string NewDir() => + Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-promote-owner-{Guid.NewGuid():N}")).FullName; + + private sealed record RunResult(List Warnings, List Output); + + private static PowerShell Shell(bool withContext = false) + { + var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + if (withContext) + { + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + } + return ps; + } + + private static RunResult Export(string output, string checkpoint) + { + using var ps = Shell(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", output) + .AddParameter("CheckpointPath", checkpoint) + .AddParameter("All"); + // A run that dies is the subject here, so a terminating error is an expected outcome. + List results = []; + try { results = [.. ps.Invoke()]; } + catch (CmdletInvocationException) { } + return new RunResult([.. ps.Streams.Warning.Select(w => w.Message)], results); + } + + private static RunResult Sync(string deltaPath, string checkpoint, string output) + { + using var ps = Shell(withContext: true); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpoint) + .AddParameter("OutputFile", output); + List results = []; + try { results = [.. ps.Invoke()]; } + catch (CmdletInvocationException) { } + return new RunResult([.. ps.Streams.Warning.Select(w => w.Message)], results); + } + + /// + /// Reopens a temp the way the cmdlet opens its own - new StreamWriter(path, append: false), + /// which asks for FileShare.Read - with the bytes the checkpoint counted already in it. + /// This is the run that has not died: it is between pages, and its next flush is still to + /// come. + /// + private static StreamWriter HoldLikeALiveRun(string tempPath) + { + var counted = File.ReadAllText(tempPath); + var writer = new StreamWriter(tempPath, append: false); + writer.Write(counted); + writer.Flush(); + return writer; + } + + /// + /// The reproduction. A run dies after its first page, leaving a temp and a checkpoint + /// naming it; the same export is running again against that -OutputFile before the first + /// one has finished. The second run must not take the file the first is writing: it copied + /// those rows into its own output, unlinked the temp, and the first run's closing move then + /// failed with a FileNotFoundException reported against the output it never got to write. + /// + [Fact] + public void An_export_does_not_promote_a_temp_a_running_export_still_holds() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // The first run: one page collected, then the wire dies. A real temp, and a real + // checkpoint naming it with the flushed length. + handler.Queue(HttpStatusCode.OK, Page1); + Export(output, checkpoint); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.False(File.Exists(output)); + Assert.Equal(2, PaginationCheckpoint.Load(checkpoint)!.ItemsCollected); + + // It is not dead. It still holds that temp, with its two rows in it. + using var live = HoldLikeALiveRun(temp); + + // The second run, same -OutputFile and same -CheckpointPath, which finishes. + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + var second = Export(output, checkpoint); + + Assert.True(File.Exists(temp), "a running export's temp was unlinked under it"); + Assert.DoesNotContain(second.Warnings, w => w.Contains("Recovered")); + Assert.Contains(second.Warnings, w => + w.Contains("Another export is still writing the temp file") + && w.Contains(checkpoint) + && w.Contains("exported from the beginning")); + + // Its output is what it enumerated itself, not the other run's rows. + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + + // And the first run goes on: the page it fetched since its last save lands, and + // the move it ends with finds the file it has been writing all along. + live.WriteLine("{\"id\":\"u3-live\"}"); + live.Flush(); + Assert.True(File.Exists(temp)); + live.Dispose(); + File.Move(temp, output, overwrite: true); + Assert.Equal( + ["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3-live\"}"], + File.ReadAllLines(output)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// What the refusal leaves behind. A temp that is missing or short means the counted items + /// are in no file, and the checkpoint describing them is deleted on the way to exporting + /// again - but a temp another run is writing is the position that run comes back to, and + /// deleting it is the same mistake as unlinking the temp, one file over. Both files are + /// still there, and still say exactly what they said. + /// + [Fact] + public void A_refused_promotion_leaves_the_running_exports_checkpoint_and_temp_alone() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + handler.Queue(HttpStatusCode.OK, Page1); + Export(output, checkpoint); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + + var checkpointBefore = File.ReadAllBytes(checkpoint); + var tempBefore = File.ReadAllBytes(temp); + + using (HoldLikeALiveRun(temp)) + { + // The second run gets nothing off the wire, so it never saves a checkpoint of + // its own: whatever is on disk afterwards is what the refusal left. + Export(output, checkpoint); + + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.Equal(tempBefore, ReadShared(temp)); + Assert.False(File.Exists(output), "the other run's rows were published as an output"); + Assert.False(File.Exists(output + ".adopt")); + Assert.Equal([temp], Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The ordinary resume, which is the same state on disk minus the holder: the run that + /// wrote the temp is gone, so its items are recovered into the output and the enumeration + /// carries on from the page the checkpoint records. A claim that cannot tell the two apart + /// would cost every resume this release writes. + /// + [Fact] + public void An_export_promotes_the_named_temp_of_a_run_that_is_gone() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + handler.Queue(HttpStatusCode.OK, Page1); + Export(output, checkpoint); + Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + + handler.Queue(HttpStatusCode.OK, Page2); + var resumed = Export(output, checkpoint); + + Assert.Contains(resumed.Warnings, w => + w.Contains("Recovered 2 items from an interrupted export's temp file")); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(3L, Assert.IsType( + Assert.Single(resumed.Output).BaseObject).ItemCount); + Assert.Empty(Directory.GetFiles(dir, "*.tmp")); + Assert.False(File.Exists(checkpoint), "a completed export deletes its checkpoint"); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// Where the claim must not go. A checkpoint written before the output file was recorded is + /// diagnosed as this export's by the temp it names corroborating it - the same resolution + /// the promotion below uses - so a claim taken during that reading would answer "this names + /// no file of mine" for a temp that is merely open at the moment, and a settled ownership + /// diagnosis would swing on which second the export started in. The checkpoint is still + /// this export's; it is the promotion that refuses. + /// + [Fact] + public void A_held_temp_does_not_change_who_a_checkpoint_without_an_output_belongs_to() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + try + { + using var live = new StreamWriter(temp, append: false); + live.WriteLine("{\"id\":\"u1\"}"); + live.WriteLine("{\"id\":\"u2\"}"); + live.Flush(); + + // No "outputFile" field: the shape a release that did not record one leaves. + File.WriteAllText(checkpoint, $$""" + {"resource":"https://graph.microsoft.com/v1.0/users?$top=999", + "nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + "itemsCollected":2,"pageItemsAlreadyWritten":0, + "tempFile":"{{Path.GetFileName(temp)}}", + "dataLength":{{new FileInfo(temp).Length}}} + """); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + var run = Export(output, checkpoint); + + Assert.Contains(run.Warnings, w => w.Contains("Another export is still writing the temp file")); + Assert.DoesNotContain(run.Warnings, w => w.Contains("no longer corroborate it")); + Assert.True(File.Exists(temp), "a running export's temp was unlinked under it"); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The same helper, reached the same way, from the other cmdlet. A sync has one thing more + /// to lose: taking the live run's temp and then resuming past its items advances the delta + /// token over changes that are in no file this run can produce, and a delta token that has + /// moved cannot be asked for them again. + /// + [Fact] + public void A_sync_does_not_promote_a_temp_a_running_sync_still_holds() + { + var dir = NewDir(); + var delta = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // A baseline sync, then one that dies after its first page of changes. + handler.Queue(HttpStatusCode.OK, Baseline); + Sync(delta, checkpoint, output); + handler.Queue(HttpStatusCode.OK, ChangesPage1); + Sync(delta, checkpoint, output); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(temp)); + + using var live = HoldLikeALiveRun(temp); + + // The overlapping sync, which re-enumerates from the delta token instead. + handler.Queue(HttpStatusCode.OK, ChangesPage1); + handler.Queue(HttpStatusCode.OK, ChangesPage2); + var second = Sync(delta, checkpoint, output); + + Assert.True(File.Exists(temp), "a running sync's temp was unlinked under it"); + Assert.DoesNotContain(second.Warnings, w => w.Contains("Recovered")); + Assert.Contains(second.Warnings, w => + w.Contains("Another sync is still writing the temp file") + && w.Contains("re-enumerates from the last saved delta token")); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + + live.WriteLine("{\"id\":\"b3-live\"}"); + live.Flush(); + live.Dispose(); + Assert.Equal( + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3-live\"}"], + File.ReadAllLines(temp)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + // Reads a file the test's own live writer still holds open: on Windows the reader + // must offer ReadWrite sharing or the holder's write access denies the open. + private static byte[] ReadShared(string path) + { + using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + using var ms = new MemoryStream(); + fs.CopyTo(ms); + return ms.ToArray(); + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/OrphanTempOwnershipTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/OrphanTempOwnershipTests.cs new file mode 100644 index 0000000..44ae8e2 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/OrphanTempOwnershipTests.cs @@ -0,0 +1,342 @@ +using System.Management.Automation; +using System.Net; +using System.Reflection; +using System.Text; +using Mgx.Cmdlets.Base; + +namespace Mgx.IntegrationTests; + +/// +/// Which temp files belong to an output. The recovery paths search for them with +/// "{output}.*.tmp", which Directory.EnumerateFiles reads as a glob - and '*' spans dots, so +/// an export to "users" matched a file left by an export to "users.jsonl". Adopting one is how +/// another export's rows end up in this one's output. +/// +[Collection("Pipeline")] +public class OrphanTempOwnershipTests +{ + private const string Page1 = """ + {"value":[{"id":"u1"},{"id":"u2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2"} + """; + private const string Page2 = """ + {"value":[{"id":"u3"}]} + """; + private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + + private static readonly MethodInfo Adopt = + typeof(MgxCmdletBase).GetMethod( + "TryAdoptOrphanedTemp", BindingFlags.Static | BindingFlags.NonPublic)!; + + private sealed class ScriptedHandler : HttpMessageHandler + { + private readonly Queue<(HttpStatusCode Status, string Body)> _steps = new(); + private readonly object _lock = new(); + + public void Queue(HttpStatusCode status, string body) + { + lock (_lock) _steps.Enqueue((status, body)); + } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + (HttpStatusCode Status, string Body) step; + lock (_lock) + step = _steps.Count > 0 ? _steps.Dequeue() : (HttpStatusCode.InternalServerError, ServerError); + return Task.FromResult(new HttpResponseMessage(step.Status) + { + RequestMessage = request, + Content = new StringContent(step.Body, Encoding.UTF8, "application/json") + }); + } + } + + private static string NewDir() => + Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-temp-owner-{Guid.NewGuid():N}")).FullName; + + /// + /// Whether this directory's filesystem answers to a name in any case. Where it does not, + /// two spellings are two files and refusing the other one's temp is the right answer, so + /// the case test below has nothing to assert. + /// + private static bool NamesAreCaseInsensitive(string dir) + { + var probe = Path.Combine(dir, $"Case-{Guid.NewGuid():N}"); + File.WriteAllText(probe, ""); + try { return File.Exists(Path.Combine(dir, Path.GetFileName(probe).ToLowerInvariant())); } + finally { try { File.Delete(probe); } catch { } } + } + + private static void Export(string output, string checkpoint) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", output) + .AddParameter("CheckpointPath", checkpoint) + .AddParameter("All"); + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + } + + /// + /// Plain names, no glob characters anywhere: "users" and "users.jsonl" are two outputs in + /// one directory, and only the second one's export wrote that temp. + /// + [Fact] + public void A_temp_left_by_a_longer_named_export_is_not_adopted() + { + var dir = NewDir(); + try + { + var mine = Path.Combine(dir, "users"); + var theirs = Path.Combine(dir, $"users.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllLines(theirs, ["{\"id\":\"THEIRS-1\"}", "{\"id\":\"THEIRS-2\"}"]); + + Assert.False((bool)Adopt.Invoke(null, [mine, 2L])!); + + Assert.False(File.Exists(mine)); + Assert.True(File.Exists(theirs)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The same thing through the cmdlet, from the state that reaches adoption: a checkpoint + /// written before the temp name and length were recorded, and no output file. + /// + [Fact] + public void An_export_does_not_adopt_a_longer_named_exports_temp() + { + var dir = NewDir(); + var output = Path.Combine(dir, "users"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var theirs = Path.Combine(dir, $"users.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllLines(theirs, ["{\"id\":\"THEIRS-1\"}", "{\"id\":\"THEIRS-2\"}"]); + File.WriteAllText(checkpoint, """ + {"resource":"https://graph.microsoft.com/v1.0/users?$top=999", + "nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + "itemsCollected":2,"pageItemsAlreadyWritten":0} + """); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", output) + .AddParameter("CheckpointPath", checkpoint) + .AddParameter("All"); + ps.Invoke(); + + var lines = File.ReadAllLines(output); + Assert.DoesNotContain(lines, l => l.Contains("THEIRS")); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], lines); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A file the caller keeps beside their output. The glob that finds temps reaches it, and + /// nothing about the name says mgx wrote it - a run's own temp is the output's name, a dot, + /// 32 hex digits and ".tmp". Adoption copies what it takes into the output and then deletes + /// it, so a backup taken by hand was consumed as an interrupted export's items. + /// + [Fact] + public void A_file_that_is_not_a_run_temp_is_not_adopted() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "users.jsonl"); + var backup = Path.Combine(dir, "users.jsonl.backup.tmp"); + File.WriteAllLines(backup, ["{\"id\":\"BACKUP-1\"}", "{\"id\":\"BACKUP-2\"}"]); + + Assert.False((bool)Adopt.Invoke(null, [output, 2L])!); + + Assert.False(File.Exists(output)); + Assert.True(File.Exists(backup)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The same thing through the cmdlet, from the state that reaches adoption: a checkpoint + /// written before the temp name and length were recorded, and no output file. + /// + [Fact] + public void An_export_does_not_adopt_a_file_it_never_wrote() + { + var dir = NewDir(); + var output = Path.Combine(dir, "users.jsonl"); + var backup = Path.Combine(dir, "users.jsonl.backup.tmp"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + File.WriteAllLines(backup, ["{\"id\":\"BACKUP-1\"}", "{\"id\":\"BACKUP-2\"}"]); + File.WriteAllText(checkpoint, """ + {"resource":"https://graph.microsoft.com/v1.0/users?$top=999", + "nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + "itemsCollected":2,"pageItemsAlreadyWritten":0} + """); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + + Export(output, checkpoint); + + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(["{\"id\":\"BACKUP-1\"}", "{\"id\":\"BACKUP-2\"}"], + File.ReadAllLines(backup)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The same export, resumed with -OutputFile spelled differently. On a filesystem that + /// answers to either spelling the temp is this export's own, and the glob that finds it + /// matches case-insensitively - so a predicate comparing ordinally threw away the run's + /// own work and re-enumerated the collection from the first page without saying so. + /// + [Fact] + public void An_export_adopts_its_own_temp_when_the_output_is_spelled_differently() + { + var dir = NewDir(); + try + { + if (!NamesAreCaseInsensitive(dir)) return; + + var upper = Path.Combine(dir, "Users.jsonl"); + var lower = Path.Combine(dir, "users.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // Page 1 reaches the temp; page 2 fails, so the temp and its checkpoint survive. + handler.Queue(HttpStatusCode.OK, Page1); + Export(upper, checkpoint); + Assert.Single(Directory.GetFiles(dir, "Users.jsonl.*.tmp")); + + // The resume, typed the other way. Only the page the dead run never reached. + handler.Queue(HttpStatusCode.OK, Page2); + Export(lower, checkpoint); + + Assert.Equal( + ["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(lower)); + Assert.Empty(Directory.GetFiles(dir, "*.tmp")); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// A second export running against the same output right now. Adoption picks the newest + /// matching temp with only a line count to go on, and the file a live run is writing into + /// is always the newest one - so recovering a pre-2.1.0 checkpoint copied that run's rows + /// into this run's deliverable and unlinked the file underneath it, reported on the warning + /// stream as a successful recovery with nothing on the error stream. The sweep beside it + /// spares the same file; adoption has to. + /// + [Fact] + public void An_export_does_not_adopt_a_temp_a_running_export_still_holds() + { + var dir = NewDir(); + var output = Path.Combine(dir, "users.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var live = Path.Combine(dir, $"users.jsonl.{Guid.NewGuid():N}.tmp"); + try + { + File.WriteAllText(checkpoint, """ + {"resource":"https://graph.microsoft.com/v1.0/users?$top=999", + "nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + "itemsCollected":2,"pageItemsAlreadyWritten":0} + """); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + + // The other run, holding its temp open the way an export does, two rows flushed. + using (var writer = new StreamWriter(live, append: false)) + { + writer.WriteLine("{\"id\":\"OTHER-1\"}"); + writer.WriteLine("{\"id\":\"OTHER-2\"}"); + writer.Flush(); + + Export(output, checkpoint); + + Assert.True(File.Exists(live), "a running export's temp was unlinked under it"); + var lines = File.ReadAllLines(output); + Assert.DoesNotContain(lines, l => l.Contains("OTHER")); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], lines); + + writer.WriteLine("{\"id\":\"OTHER-3\"}"); + } + + Assert.Equal( + ["{\"id\":\"OTHER-1\"}", "{\"id\":\"OTHER-2\"}", "{\"id\":\"OTHER-3\"}"], + File.ReadAllLines(live)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// What reaches the caller's file. The checkpoint predates the recorded length, so adoption + /// has only a line count to go on - and a temp cut inside the row that count reaches hands + /// the fragment back as a line like any other. It went into the JSONL output as an item, + /// was reported as a recovery, and the rest of the enumeration was appended behind it, + /// leaving a row nothing can parse in the middle of the deliverable. + /// + [Fact] + public void An_export_does_not_adopt_a_temp_cut_inside_the_row_the_checkpoint_counts() + { + var dir = NewDir(); + var output = Path.Combine(dir, "users.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + File.WriteAllText(Path.Combine(dir, $"users.jsonl.{Guid.NewGuid():N}.tmp"), + "{\"id\":\"t1\"}\n{\"id\":\"t2\"}\n{\"id\":\"to"); + File.WriteAllText(checkpoint, """ + {"resource":"https://graph.microsoft.com/v1.0/users?$top=999", + "nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + "itemsCollected":3,"pageItemsAlreadyWritten":0} + """); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + + Export(output, checkpoint); + + var lines = File.ReadAllLines(output); + foreach (var line in lines) + System.Text.Json.JsonDocument.Parse(line).Dispose(); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], lines); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/OutFileGuardTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/OutFileGuardTests.cs index 163ca13..316db94 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/OutFileGuardTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/OutFileGuardTests.cs @@ -1,5 +1,4 @@ using System.Net; -using System.Reflection; using Mgx.Engine.Http; namespace Mgx.IntegrationTests; @@ -11,9 +10,6 @@ namespace Mgx.IntegrationTests; [Collection("Pipeline")] public class OutFileGuardTests { - private static readonly Type Base = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); - private const BindingFlags Static = BindingFlags.NonPublic | BindingFlags.Static; - [Fact] public void A_second_piped_item_is_refused_without_fetching_it() { @@ -21,17 +17,8 @@ public void A_second_piped_item_is_refused_without_fetching_it() wire.QueueBytes(HttpStatusCode.OK, [1, 2, 3, 4], "application/octet-stream"); wire.QueueBytes(HttpStatusCode.OK, [5, 6, 7, 8], "application/octet-stream"); - Base.GetField("s_graphHttpClient", Static)!.SetValue(null, new HttpClient(wire)); - Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, - Mgx.Cmdlets.Base.MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, true); - Base.GetField("s_cachedTotalTimeoutSeconds", Static)!.SetValue(null, - new ResilientGraphClientOptions().TotalTimeoutSeconds); - Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); - Base.GetField("s_clientOptions", Static)!.SetValue(null, - new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); + // Get-MgxContent refuses a transport mgx does not own, so this one claims ownership. + using var transport = MgxTransportScope.Inject(wire, owned: true); var outFile = Path.Combine(Path.GetTempPath(), $"mgx-outfile-{Guid.NewGuid():N}.bin"); try @@ -62,13 +49,6 @@ public void A_second_piped_item_is_refused_without_fetching_it() finally { File.Delete(outFile); - Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); - Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); - Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); - Base.GetField("s_cachedTotalTimeoutSeconds", Static)!.SetValue(null, 0); - Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); - Base.GetField("s_clientOptions", Static)!.SetValue(null, new ResilientGraphClientOptions()); - ResiliencePipelineFactory.Reset(); } } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ResilienceInjectionScenarioTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ResilienceInjectionScenarioTests.cs new file mode 100644 index 0000000..ede3fed --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/ResilienceInjectionScenarioTests.cs @@ -0,0 +1,560 @@ +using System.Net; +using System.Reflection; +using Mgx.Cmdlets; +using Mgx.Cmdlets.Base; +using Mgx.Cmdlets.Cmdlets.Configuration; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// The existing suites cover request-level behavior through the wrap - disarming Kiota's inner +/// retry handler across 429 and 503, correlation-id stamping, oversized bodies, lazy one-shot +/// option resolution - and two properties of the wrapper client. None of the injection +/// scenarios is covered as a scenario, because each needs a GraphSession and this process has +/// no Graph SDK. GraphSessionScope supplies one. +/// +/// Coexistence with Az/PnP/Exchange is not here: module load and ALC resolution are +/// irreversible within a process, so that belongs to the out-of-process ecosystem matrix. +/// +[Collection("Pipeline")] +public class ResilienceInjectionScenarioTests +{ + private static HttpClient? BuildWrapper(HttpClient sdkClient, List warnings) => + EnableMgxResilience.BuildResilientSdkClient(sdkClient, warnings.Add); + + /// Puts the process in the state Enable-MgxResilience leaves behind on success. + private static HttpClient Enable(GraphSessionScope scope, HttpClient sdkClient) + { + var wrapper = BuildWrapper(sdkClient, [])!; + EnableMgxResilience.OriginalSdkClient = sdkClient; + EnableMgxResilience.ResilientSdkClient = wrapper; + EnableMgxResilience.IsEnabled = true; + scope.Session.GraphHttpClient = wrapper; + return wrapper; + } + + /// + /// Asserts that is exactly one resilience layer over + /// . A second layer answers requests as happily as one, so nothing + /// above the wire shows it: each layer retries the one beneath it, so a throttled request + /// multiplies its attempts, telemetry counts each attempt once per layer, and the pacer + /// halves once per layer. + /// + private static void AssertOneLayerOver(HttpClient wrapper, HttpClient genuine, string context = "") + { + var (layers, innermost) = MeasureWrap(wrapper); + Assert.True(layers == 1, $"{context}the wrap is {layers} layers deep"); + Assert.True(ReferenceEquals(genuine, innermost), + $"{context}the wrap bridges to something other than the genuine SDK client"); + } + + /// + /// Walks the handler chain to the client at the bottom of it, counting resilience layers on + /// the way. The object graph, not mgx's bookkeeping about it: what the wire sees is the chain. + /// + private static (int Layers, HttpClient Innermost) MeasureWrap(HttpClient client) + { + var layers = 0; + while (HandlerOf(client) is ResilientDelegatingHandler resilient) + { + layers++; + var bridge = resilient.InnerHandler; + Assert.NotNull(bridge); + var target = bridge!.GetType().GetField("_sdkClient", BindingFlags.Instance | BindingFlags.NonPublic); + Assert.True(target != null, $"{bridge.GetType().Name} is not the SDK bridge handler"); + client = (HttpClient)target!.GetValue(bridge)!; + } + return (layers, client); + } + + /// The handler an HttpClient sends through; it keeps it in a private field. + private static HttpMessageHandler? HandlerOf(HttpClient client) + { + var field = typeof(HttpMessageInvoker).GetField("_handler", BindingFlags.Instance | BindingFlags.NonPublic); + Assert.True(field != null, "HttpMessageInvoker no longer keeps its handler in _handler"); + return (HttpMessageHandler?)field!.GetValue(client); + } + + private sealed class OkHandler : HttpMessageHandler + { + public int Requests { get; private set; } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Requests++; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent("""{"value":[]}""", + System.Text.Encoding.UTF8, "application/json") + }); + } + } + + // --- 1. the SDK replaced our client --- + + [Fact] + public void A_replaced_graph_http_client_is_wrapped_again_rather_than_left_raw() + { + using var scope = GraphSessionScope.Arm(); + using var first = new HttpClient(new OkHandler()) { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + var wrapper = Enable(scope, first); + + // Connect-MgGraph builds a fresh SDK client and hands it to the session, discarding ours. + using var replacement = new HttpClient(new OkHandler()) { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + scope.Session.GraphHttpClient = replacement; + + EnableMgxResilience.RefreshInjectedClient(_ => { }, _ => { }); + + var active = scope.Session.GraphHttpClient; + Assert.True(EnableMgxResilience.IsEnabled, "the wrap was dropped instead of re-injected"); + Assert.NotSame(replacement, active); // the caller is not left on the raw SDK client + Assert.NotSame(wrapper, active); // nor on the wrapper that bridged to the old one + Assert.Same(active, EnableMgxResilience.ResilientSdkClient); + Assert.Same(replacement, EnableMgxResilience.OriginalSdkClient); + } + + // --- 2. reconnect versus credential change --- + + [Fact] + public void A_same_identity_reconnect_keeps_the_limiter_a_credential_change_resets_it() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + using var sdkClient = new HttpClient(wire); + + const string tenant = "11111111-1111-1111-1111-111111111111"; + const string appA = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"; + const string appB = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"; + + using var scope = GraphSessionScope.Arm(sdkClient, GraphSessionScope.AuthContextFor(tenant, appA)); + + // No transport seam here: this is about the identity check inside GetClient, which the + // seam sits above. BuildCleanHttpClient cannot run without the SDK's auth helpers, so + // GetClient falls back to the session's own client - the borrowed-transport path. + // A real limiter, not NoRateLimit: the property under test is that the warm limiter + // instance survives, and NoRateLimit leaves the factory with none to keep. The burst + // covers every request this test sends, so nothing waits on it. + var options = new ResilientGraphClientOptions + { + MaxRetryAttempts = 1, + RateLimitBurst = 50, + RateLimitPerSecond = 50, + NoAdaptivePacing = true + }; + MgxCmdletBase.SetClientOptions(options); + ResiliencePipelineFactory.Reset(); + + Run(); + var afterFirst = ResiliencePipelineFactory.GetOrCreate(options).RateLimiter; + + // Reconnect: a new AuthContext object carrying the same credentials. The instance + // changed, so the client is rebuilt - but the identity did not, so the limiter that has + // already learned this tenant's ceiling must survive rather than earn a fresh burst. + scope.Session.AuthContext = GraphSessionScope.AuthContextFor(tenant, appA); + Run(); + Assert.Same(afterFirst, ResiliencePipelineFactory.GetOrCreate(options).RateLimiter); + + // A different application on the same tenant is a different credential. + scope.Session.AuthContext = GraphSessionScope.AuthContextFor(tenant, appB); + Run(); + Assert.NotSame(afterFirst, ResiliencePipelineFactory.GetOrCreate(options).RateLimiter); + + void Run() + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users/u1") + .AddParameter("WarningAction", System.Management.Automation.ActionPreference.SilentlyContinue); + var output = ps.Invoke(); + Assert.True(output.Count > 0, + "the request did not reach the session's client: " + + string.Join(" | ", ps.Streams.Error.Select(e => e.FullyQualifiedErrorId))); + } + } + + // --- 3. one wrapper at a time --- + + [Fact] + public void A_second_sdk_client_leaves_exactly_one_wrapper_armed() + { + using var scope = GraphSessionScope.Arm(); + using var first = new HttpClient(new OkHandler()); + Enable(scope, first); + var firstHandler = EnableMgxResilience.ActiveHandler; + Assert.NotNull(firstHandler); + + using var second = new HttpClient(new OkHandler()); + scope.Session.GraphHttpClient = second; + EnableMgxResilience.RefreshInjectedClient(_ => { }, _ => { }); + + // ActiveHandler is what Disable-MgxResilience and module teardown reach for. Two armed + // handlers would mean one of them keeps a pipeline and a bridge alive unreachably. + Assert.NotNull(EnableMgxResilience.ActiveHandler); + Assert.NotSame(firstHandler, EnableMgxResilience.ActiveHandler); + Assert.Same(scope.Session.GraphHttpClient, EnableMgxResilience.ResilientSdkClient); + Assert.Same(second, EnableMgxResilience.OriginalSdkClient); + } + + // --- 4. module removal --- + + [Fact] + public void Module_removal_puts_the_session_back_on_the_genuine_client() + { + using var scope = GraphSessionScope.Arm(); + using var sdkClient = new HttpClient(new OkHandler()) + { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + var wrapper = Enable(scope, sdkClient); + var handlerBeforeRemoval = EnableMgxResilience.ActiveHandler; + Assert.NotNull(handlerBeforeRemoval); + + // The teardown itself, not a copy of it. AlcInitializer.OnRemove is this call plus + // DetachAssemblyLoadHandler. + AlcInitializer.ReleaseStaticState(); + + Assert.Null(EnableMgxResilience.ActiveHandler); + Assert.Null(EnableMgxResilience.ResilientSdkClient); + Assert.Null(EnableMgxResilience.OriginalSdkClient); + Assert.False(EnableMgxResilience.IsEnabled); + + // What the teardown leaves installed is what the SDK goes on sending through, so the + // session has to be back on the genuine client before the references go. A wrapper left + // there routes every SDK request through a handler belonging to a module that is no + // longer loaded, for as long as nobody imports mgx again. + Assert.Same(sdkClient, scope.Session.GraphHttpClient); + + var warnings = EnableThroughTheCmdlet(); + + Assert.Empty(warnings); + Assert.True(EnableMgxResilience.IsEnabled); + Assert.NotNull(EnableMgxResilience.ActiveHandler); + Assert.NotSame(handlerBeforeRemoval, EnableMgxResilience.ActiveHandler); + Assert.NotSame(wrapper, EnableMgxResilience.ResilientSdkClient); + Assert.Same(scope.Session.GraphHttpClient, EnableMgxResilience.ResilientSdkClient); + Assert.Same(sdkClient, EnableMgxResilience.OriginalSdkClient); + AssertOneLayerOver(EnableMgxResilience.ResilientSdkClient!, sdkClient); + } + + [Fact] + public void Three_remove_and_re_enable_cycles_leave_the_wrap_one_layer_deep() + { + using var scope = GraphSessionScope.Arm(); + using var sdkClient = new HttpClient(new OkHandler()) + { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + scope.Session.GraphHttpClient = sdkClient; + + for (var cycle = 1; cycle <= 3; cycle++) + { + var where = $"cycle {cycle}: "; + Assert.Empty(EnableThroughTheCmdlet()); + + var wrapper = EnableMgxResilience.ResilientSdkClient; + Assert.True(wrapper != null, $"{where}resilience was not injected"); + Assert.True(ReferenceEquals(wrapper, scope.Session.GraphHttpClient), + $"{where}the session is not on the wrapper mgx built"); + Assert.True(ReferenceEquals(sdkClient, EnableMgxResilience.OriginalSdkClient), + $"{where}the client Disable-MgxResilience would restore is not the genuine one"); + AssertOneLayerOver(wrapper!, sdkClient, where); + + // Remove-Module. The next iteration's Enable-MgxResilience is the re-import: it runs + // the cmdlet against whatever this teardown left on the session. + AlcInitializer.ReleaseStaticState(); + Assert.True(ReferenceEquals(sdkClient, scope.Session.GraphHttpClient), + $"{where}removal left something other than the genuine client on the session"); + } + } + + /// + /// Runs Enable-MgxResilience where PowerShell runs it. The cmdlet reads the session through + /// the same reflection a re-imported module would, so what the previous import left behind is + /// what it decides against. + /// + private static IReadOnlyList EnableThroughTheCmdlet() => RunCmdlet("Enable-MgxResilience"); + + private static IReadOnlyList DisableThroughTheCmdlet() => RunCmdlet("Disable-MgxResilience"); + + private static IReadOnlyList RunCmdlet(string name) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(EnableMgxResilience).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand(name); + ps.Invoke(); + + Assert.Empty(ps.Streams.Error.Select(e => e.FullyQualifiedErrorId)); + return [.. ps.Streams.Warning.Select(w => w.Message)]; + } + + // --- 5. taking the injection off again --- + + [Fact] + public void Disable_after_a_module_removal_says_so_and_leaves_the_genuine_client() + { + using var scope = GraphSessionScope.Arm(); + using var sdkClient = new HttpClient(new OkHandler()) + { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + Enable(scope, sdkClient); + + AlcInitializer.ReleaseStaticState(); + + var warnings = DisableThroughTheCmdlet(); + + // Removal already unwound the injection, so there is nothing left to disable and saying + // so is the honest answer - but only because the session is on the genuine client. + Assert.Contains(warnings, w => w.Contains("not currently enabled", StringComparison.Ordinal)); + Assert.Same(sdkClient, scope.Session.GraphHttpClient); + } + + [Fact] + public void Disable_takes_off_a_wrapper_that_no_state_points_at_any_more() + { + using var scope = GraphSessionScope.Arm(); + using var sdkClient = new HttpClient(new OkHandler()) + { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + var wrapper = Enable(scope, sdkClient); + + // What a removal that could not reach GraphSession leaves: the wrapper is installed and + // every reference mgx held to it, and to the client under it, is gone. + EnableMgxResilience.IsEnabled = false; + EnableMgxResilience.ActiveHandler = null; + EnableMgxResilience.ResilientSdkClient = null; + EnableMgxResilience.OriginalSdkClient = null; + Assert.Same(wrapper, scope.Session.GraphHttpClient); + + var warnings = DisableThroughTheCmdlet(); + + Assert.Empty(warnings); + Assert.Same(sdkClient, scope.Session.GraphHttpClient); + } + + [Fact] + public void Disable_after_a_re_import_restores_the_genuine_client_not_the_previous_wrapper() + { + using var scope = GraphSessionScope.Arm(); + using var sdkClient = new HttpClient(new OkHandler()) + { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + var firstWrapper = Enable(scope, sdkClient); + + AlcInitializer.ReleaseStaticState(); + Assert.Empty(EnableThroughTheCmdlet()); + Assert.NotSame(firstWrapper, scope.Session.GraphHttpClient); + + var warnings = DisableThroughTheCmdlet(); + + Assert.Empty(warnings); + Assert.Same(sdkClient, scope.Session.GraphHttpClient); + Assert.False(EnableMgxResilience.IsEnabled); + Assert.Null(EnableMgxResilience.ResilientSdkClient); + Assert.Null(EnableMgxResilience.OriginalSdkClient); + Assert.Null(EnableMgxResilience.ActiveHandler); + } + + // --- 6. the option factory the cmdlet installs --- + + [Fact] + public async Task The_wrap_the_cmdlet_builds_sends_the_request_its_option_factory_annotates() + { + // The chain the cmdlet builds is ResilientDelegatingHandler -> bridge -> SDK client, and + // nothing in it consumes the retry option when the SDK's handlers are not there. The + // factory is the cmdlet's own, so whatever it does on this path - answer, or throw over a + // type the process has not loaded - is what the handler has to absorb, and the request + // has to go out either way. + var wire = new OkHandler(); + using var sdkClient = new HttpClient(wire) { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + var warnings = new List(); + + var wrapper = BuildWrapper(sdkClient, warnings); + try + { + Assert.NotNull(wrapper); + Assert.Empty(warnings); + + // The factory the cmdlet installs is BuildInnerRetryOverride itself, so the guard + // that already covers a throwing factory covers the production chain too. + var handler = EnableMgxResilience.ActiveHandler; + Assert.NotNull(handler); + var factory = handler.AdditionalRequestOptionsFactory; + Assert.NotNull(factory); + Assert.Equal(nameof(EnableMgxResilience.BuildInnerRetryOverride), factory.Method.Name); + + // Which outcome this run gets is not this file's to choose: BuildInnerRetryOverride + // resolves the SDK's option type across the loaded assemblies, so it depends on + // whether anything has pulled Kiota in yet. Asserting only the request would pass + // under either without saying which happened, so the handler's own note decides. + var notes = new List(); + handler.VerboseWriter = notes.Add; + var resolved = Record.Exception(() => factory()); + + using var response = await wrapper!.GetAsync("users"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(1, wire.Requests); + + var absorbed = notes.Where(note => + note.StartsWith("Could not configure the SDK's retry option", StringComparison.Ordinal)).ToList(); + if (resolved == null) + { + Assert.Empty(absorbed); + Assert.NotEmpty(factory()!); + } + else + { + Assert.Single(absorbed); + Assert.Contains(resolved.Message, absorbed[0], StringComparison.Ordinal); + } + } + finally + { + wrapper?.Dispose(); + EnableMgxResilience.ActiveHandler = null; + ResiliencePipelineFactory.Reset(); + } + } + + // --- 7. mgx's own requests while the injection is armed --- + + /// + /// Enable-MgxResilience leaves a wrapper of ours on GraphSession.GraphHttpClient. When + /// BuildCleanHttpClient cannot run - the Graph SDK's auth helpers are not in this process, and + /// in production any SDK build that moved them, or any throw out of provider construction - + /// GetClient borrows the session's client instead, and borrowing that wrapper puts mgx's own + /// pipeline on top of the one already inside it. + /// + /// Two pipelines answer a request as happily as one, so nothing above the wire says which is + /// running. What multiplies is the retry budget the caller configured: each layer retries the + /// one beneath it, so a throttled request sends the square of its attempts at a service that + /// is already throttling, and telemetry books each attempt once per layer. + /// + /// + [Fact] + public void The_borrowed_session_client_is_the_genuine_one_not_the_wrapper_over_it() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.TooManyRequests, + """{"error":{"code":"TooManyRequests","message":"slow down"}}""", + new Dictionary { ["Retry-After"] = "0" }); + using var sdkClient = new HttpClient(wire) + { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + + using var scope = GraphSessionScope.Arm( + sdkClient, + GraphSessionScope.AuthContextFor( + "11111111-1111-1111-1111-111111111111", "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")); + + // Two retries, so one pipeline serves three attempts to a request that keeps being + // refused and two serve nine - far enough apart that no timing or ordering detail reads + // as the other. The breaker never evaluates at either count, so the second invocation + // reaches the wire on the same terms as the first. Options before Enable: the wrapper + // captures a pipeline when it is built, and it has to be the one under test. + MgxCmdletBase.SetClientOptions(new ResilientGraphClientOptions + { + MaxRetryAttempts = 2, + MaxRetryAfterSeconds = 1, + NoRateLimit = true, + NoAdaptivePacing = true, + CircuitBreakerMinThroughput = 1000 + }); + ResiliencePipelineFactory.Reset(); + + var wrapper = Enable(scope, sdkClient); + + // No transport seam armed: the borrow is inside GetClient, which the seam sits above. + var warnings = new List(); + Invoke(warnings); + var firstRun = wire.RequestCount; + Invoke(warnings); + var secondRun = wire.RequestCount - firstRun; + + Assert.Same(wrapper, scope.Session.GraphHttpClient); // the injection is still armed + Assert.Equal(3, firstRun); + Assert.Equal(3, secondRun); + + // The transport GetClient built its ResilientGraphClient on, measured on the object graph + // rather than on mgx's bookkeeping about it. That client is the one resilience layer this + // request path is meant to have, so anything under the transport is a second one. + var (layers, innermost) = MeasureWrap(BorrowedTransport()); + Assert.True(layers == 0, $"mgx borrowed a transport {layers} resilience layers deep"); + Assert.Same(sdkClient, innermost); + + // The borrowed client is compared against the session's on every invocation, to catch a + // Connect-MgGraph that swapped it. Comparing the wrapper the session holds against the + // genuine client mgx borrowed reads as a swap every time: it rebuilds, and warns again, + // for the rest of the session. + Assert.Equal(1, warnings.Count(w => + w.Contains("Falling back to SDK client", StringComparison.Ordinal))); + + void Invoke(List collected) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("Invoke-MgxRequest -Uri /users/u1 -ErrorAction SilentlyContinue | Out-Null"); + ps.Invoke(); + collected.AddRange(ps.Streams.Warning.Select(w => w.Message)); + } + } + + /// + /// What the teardown releases, and what it leaves standing. The handler and the client + /// wrapped around it go with the statics; the Polly pipeline does not - the factory keeps + /// one instance and hands the same one to every client it builds, so it survives an SDK + /// injection being taken off. Releasing it would mean Reset(), and that would take the + /// circuit-breaker history, the rate limiter and the learned pacing that mgx's own cmdlets + /// go on using. + /// + [Fact] + public void Disable_leaves_the_pipeline_the_factory_shares_with_mgxs_own_cmdlets() + { + using var scope = GraphSessionScope.Arm(); + using var sdkClient = new HttpClient(new OkHandler()) + { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + Enable(scope, sdkClient); + + var options = new ResilientGraphClientOptions + { + MaxRetryAttempts = 1, + NoRateLimit = true, + NoAdaptivePacing = true + }; + var before = ResiliencePipelineFactory.GetOrCreate(options).Pipeline; + + try + { + DisableThroughTheCmdlet(); + + Assert.Null(EnableMgxResilience.ResilientSdkClient); + Assert.Null(EnableMgxResilience.ActiveHandler); + Assert.Same(sdkClient, scope.Session.GraphHttpClient); + Assert.Same(before, ResiliencePipelineFactory.GetOrCreate(options).Pipeline); + } + finally + { + ResiliencePipelineFactory.Reset(); + } + } + + /// + /// The transport GetClient last built a client on. MgxCmdletBase keeps it private, and it is + /// read the way MeasureWrap reads a handler chain: what mgx sends through is the property + /// under test, and no output above it tells one transport from another. + /// + private static HttpClient BorrowedTransport() + { + var field = typeof(MgxCmdletBase).GetField( + "s_graphHttpClient", BindingFlags.Static | BindingFlags.NonPublic); + Assert.True(field != null, "MgxCmdletBase no longer keeps its transport in s_graphHttpClient"); + var transport = field!.GetValue(null) as HttpClient; + Assert.True(transport != null, "GetClient did not build a transport"); + return transport!; + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ResilienceWrapTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ResilienceWrapTests.cs index 5e6695b..fb6a27b 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/ResilienceWrapTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/ResilienceWrapTests.cs @@ -15,11 +15,9 @@ public class ResilienceWrapTests { private static HttpClient? BuildWrapper(HttpClient sdkClient, List warnings) { - var build = typeof(EnableMgxResilience).GetMethod( - "BuildResilientSdkClient", BindingFlags.NonPublic | BindingFlags.Static)!; try { - return (HttpClient?)build.Invoke(null, [sdkClient, (Action)warnings.Add]); + return EnableMgxResilience.BuildResilientSdkClient(sdkClient, warnings.Add); } finally { diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ResponseFidelityTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ResponseFidelityTests.cs index fcdb829..60ead76 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/ResponseFidelityTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/ResponseFidelityTests.cs @@ -1,5 +1,4 @@ using System.Net; -using System.Reflection; using Mgx.Engine.Http; namespace Mgx.IntegrationTests; @@ -12,35 +11,6 @@ namespace Mgx.IntegrationTests; [Collection("Pipeline")] public class ResponseFidelityTests { - private static readonly Type Base = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); - private const BindingFlags Static = BindingFlags.NonPublic | BindingFlags.Static; - - private static void InjectTransport(HttpMessageHandler wire) - { - Base.GetField("s_graphHttpClient", Static)!.SetValue(null, new HttpClient(wire)); - Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, - Mgx.Cmdlets.Base.MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); - Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); - Base.GetField("s_clientOptions", Static)!.SetValue(null, - new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); - } - - private static void ResetTransport() - { - // Restore every static InjectTransport touched - a later test in the collection - // that drives a cmdlet without injecting must not inherit this class's transport. - Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); - Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); - Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); - Base.GetField("s_cachedTotalTimeoutSeconds", Static)!.SetValue(null, 0); - Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); - Base.GetField("s_clientOptions", Static)!.SetValue(null, new ResilientGraphClientOptions()); - ResiliencePipelineFactory.Reset(); - } - private static System.Management.Automation.PowerShell CreateShell() { var ps = System.Management.Automation.PowerShell.Create(); @@ -59,8 +29,7 @@ public void A_204_on_a_get_emits_nothing_and_no_error() { var wire = new MockHttpHandler(); wire.QueueEmpty(HttpStatusCode.NoContent); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { using var ps = CreateShell(); ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1"); @@ -69,7 +38,6 @@ public void A_204_on_a_get_emits_nothing_and_no_error() Assert.Empty(output); Assert.Empty(ps.Streams.Error); } - finally { ResetTransport(); } } [Fact] @@ -77,8 +45,7 @@ public void A_200_with_a_zero_length_body_emits_nothing_and_no_error() { var wire = new MockHttpHandler(); wire.QueueEmpty(HttpStatusCode.OK); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { using var ps = CreateShell(); ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1"); @@ -87,7 +54,6 @@ public void A_200_with_a_zero_length_body_emits_nothing_and_no_error() Assert.Empty(output); Assert.Empty(ps.Streams.Error); } - finally { ResetTransport(); } } [Fact] @@ -95,8 +61,7 @@ public void An_html_body_is_an_error_record_naming_the_content_type() { var wire = new MockHttpHandler(); wire.QueueResponse(HttpStatusCode.OK, "proxy says no", contentType: "text/html"); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { using var ps = CreateShell(); ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1"); @@ -107,7 +72,6 @@ public void An_html_body_is_an_error_record_naming_the_content_type() Assert.StartsWith("NonJsonResponse", error.FullyQualifiedErrorId); Assert.Contains("text/html", error.Exception.Message); } - finally { ResetTransport(); } } [Fact] @@ -115,8 +79,7 @@ public void Raw_passes_a_non_json_body_through_as_text() { var wire = new MockHttpHandler(); wire.QueueResponse(HttpStatusCode.OK, "plain text payload", contentType: "text/plain"); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { using var ps = CreateShell(); ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1").AddParameter("Raw", true); @@ -126,7 +89,6 @@ public void Raw_passes_a_non_json_body_through_as_text() Assert.Equal("plain text payload", item.BaseObject); Assert.Empty(ps.Streams.Error); } - finally { ResetTransport(); } } [Fact] @@ -134,8 +96,7 @@ public void Malformed_json_is_an_error_record_with_a_body_snippet() { var wire = new MockHttpHandler(); wire.QueueResponse(HttpStatusCode.OK, "{\"id\": \"u1\", \"displayNa"); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { using var ps = CreateShell(); ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1"); @@ -146,7 +107,6 @@ public void Malformed_json_is_an_error_record_with_a_body_snippet() Assert.StartsWith("MalformedJsonResponse", error.FullyQualifiedErrorId); Assert.Contains("displayNa", error.Exception.Message); } - finally { ResetTransport(); } } [Fact] @@ -154,8 +114,7 @@ public void A_malformed_page_during_enumeration_is_an_error_record_not_a_crash() { var wire = new MockHttpHandler(); wire.QueueResponse(HttpStatusCode.OK, "not a collection envelope at all"); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { using var ps = CreateShell(); ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users").AddParameter("All", true); @@ -164,7 +123,6 @@ public void A_malformed_page_during_enumeration_is_an_error_record_not_a_crash() var error = Assert.Single(ps.Streams.Error); Assert.StartsWith("MalformedJsonResponse", error.FullyQualifiedErrorId); } - finally { ResetTransport(); } } [Fact] @@ -174,8 +132,7 @@ public void A_bom_prefixed_json_body_parses() var bom = new byte[] { 0xEF, 0xBB, 0xBF }; var body = System.Text.Encoding.UTF8.GetBytes("""{"id":"u1","displayName":"BOM"}"""); wire.QueueBytes(HttpStatusCode.OK, [.. bom, .. body], "application/json"); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { using var ps = CreateShell(); ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1"); @@ -185,7 +142,173 @@ public void A_bom_prefixed_json_body_parses() var item = Assert.Single(output); Assert.Equal("BOM", ((System.Collections.Hashtable)item.BaseObject)["displayName"]); } - finally { ResetTransport(); } + } + + /// + /// The fan-out read is the same read. It deserialized the raw bytes, so the BOM that the + /// single-entity path accepts lost every entity to a per-ID FanOutError, and the declared + /// charset, the empty body and -Raw were decided differently there than everywhere else. + /// + [Fact] + public void A_bom_prefixed_body_parses_on_the_entity_fan_out_too() + { + var wire = new MockHttpHandler(); + var bom = new byte[] { 0xEF, 0xBB, 0xBF }; + var body = System.Text.Encoding.UTF8.GetBytes("""{"id":"u1","displayName":"BOM"}"""); + // Both IDs are answered the same way: the two requests race, so the queue order + // cannot say which entity gets which response. + wire.QueueBytes(HttpStatusCode.OK, [.. bom, .. body], "application/json"); + wire.QueueBytes(HttpStatusCode.OK, [.. bom, .. body], "application/json"); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddScript("'u1','u2' | Invoke-MgxRequest -Uri '/users/{id}'"); + var output = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + Assert.Equal(2, output.Count); + Assert.All(output, item => + Assert.Equal("BOM", ((System.Collections.Hashtable)item.BaseObject)["displayName"])); + } + } + + [Fact] + public void An_entity_fan_out_body_is_decoded_by_its_declared_charset() + { + var wire = new MockHttpHandler(); + var body = System.Text.Encoding.Unicode.GetBytes("""{"id":"u1","displayName":"charset"}"""); + wire.QueueBytes(HttpStatusCode.OK, body, "application/json; charset=utf-16"); + wire.QueueBytes(HttpStatusCode.OK, body, "application/json; charset=utf-16"); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddScript("'u1','u2' | Invoke-MgxRequest -Uri '/users/{id}'"); + var output = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + Assert.Equal(2, output.Count); + Assert.All(output, item => + Assert.Equal("charset", ((System.Collections.Hashtable)item.BaseObject)["displayName"])); + } + } + + [Fact] + public void Raw_passes_a_non_json_fan_out_body_through_as_text() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "plain text payload", contentType: "text/plain"); + wire.QueueResponse(HttpStatusCode.OK, "plain text payload", contentType: "text/plain"); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddScript("'u1','u2' | Invoke-MgxRequest -Uri '/users/{id}' -Raw"); + var output = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + Assert.Equal(2, output.Count); + Assert.All(output, item => Assert.Equal("plain text payload", item.BaseObject)); + } + } + + /// + /// The relation fan-out reads a body the same way, and deserialized the raw bytes: a + /// byte-order mark lost the relation to a per-URL error and left the object carrying null. + /// + [Fact] + public void A_bom_prefixed_relation_body_is_attached() + { + var wire = new MockHttpHandler(); + var bom = new byte[] { 0xEF, 0xBB, 0xBF }; + var body = System.Text.Encoding.UTF8.GetBytes("""{"id":"m1","displayName":"BOM"}"""); + wire.QueueBytes(HttpStatusCode.OK, [.. bom, .. body], "application/json"); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddScript( + "[pscustomobject]@{ id = 'u1' } | Expand-MgxRelation -Uri '/users/{id}/manager' -As manager -Flatten"); + var output = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + var item = Assert.Single(output); + var manager = Assert.IsType( + item.Properties["manager"].Value); + Assert.Equal("BOM", manager["displayName"]); + } + } + + /// + /// A relation is not always an entity. Graph answers /$count with the number in the body + /// and text/plain on it, in both the bare and the charset-carrying form, and the count is + /// what the caller asked for - Expand-MgxRelation has no -Raw to receive it any other way. + /// + [Theory] + [InlineData("text/plain")] + [InlineData("text/plain; charset=utf-8")] + [InlineData("application/json")] + public void A_count_relation_attaches_the_count_whatever_type_it_declares(string contentType) + { + var wire = new MockHttpHandler(); + wire.QueueBytes(HttpStatusCode.OK, System.Text.Encoding.UTF8.GetBytes("5"), contentType); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddScript( + "[pscustomobject]@{ id = 'g1' } | Expand-MgxRelation -Uri '/groups/{id}/members/$count'" + + " -As memberCount -Flatten -ConsistencyLevel eventual"); + var output = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + var item = Assert.Single(output); + var count = Assert.IsType( + item.Properties["memberCount"].Value); + Assert.Equal(5L, count["Value"]); + } + } + + /// The ordinary relation, read as it always was. + [Fact] + public void A_json_relation_body_is_attached_as_the_entity() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, """{"id":"m1","displayName":"Ann"}"""); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddScript( + "[pscustomobject]@{ id = 'u1' } | Expand-MgxRelation -Uri '/users/{id}/manager' -As manager -Flatten"); + var output = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + var item = Assert.Single(output); + var manager = Assert.IsType( + item.Properties["manager"].Value); + Assert.Equal("m1", manager["id"]); + Assert.Equal("Ann", manager["displayName"]); + } + } + + /// + /// The other half of reading a body by what it carries: a proxy's HTML page is no relation + /// in any encoding, and stays a per-URL error naming what came back instead. + /// + [Fact] + public void An_html_relation_body_is_an_error_naming_the_content_type() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "proxy says no", contentType: "text/html"); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddScript( + "[pscustomobject]@{ id = 'u1' } | Expand-MgxRelation -Uri '/users/{id}/manager' -As manager -Flatten"); + var output = ps.Invoke(); + + var error = Assert.Single(ps.Streams.Error); + Assert.StartsWith("ExpandRelationError", error.FullyQualifiedErrorId, StringComparison.Ordinal); + Assert.Contains("text/html", error.Exception.Message, StringComparison.Ordinal); + var item = Assert.Single(output); + Assert.Null(item.Properties["manager"].Value); + } } [Fact] @@ -195,8 +318,7 @@ public void A_utf16_body_is_decoded_by_its_declared_charset() wire.QueueBytes(HttpStatusCode.OK, System.Text.Encoding.Unicode.GetBytes("plain utf-16 text"), "text/plain; charset=utf-16"); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { using var ps = CreateShell(); ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1").AddParameter("Raw", true); @@ -205,7 +327,58 @@ public void A_utf16_body_is_decoded_by_its_declared_charset() var item = Assert.Single(output); Assert.Equal("plain utf-16 text", item.BaseObject); } - finally { ResetTransport(); } + } + + /// + /// A charset .NET knows and refuses to construct. Encoding.GetEncoding answers + /// NotSupportedException for utf-7 rather than the ArgumentException every unknown name + /// raises, so the fallback to UTF-8 was one exception type short and a response header + /// ended the pipeline with a .NET deprecation message. + /// + [Theory] + [InlineData("application/json; charset=utf-7")] + [InlineData("application/json; charset=\"UTF-7\"")] + [InlineData("application/json; charset=windows-1252")] + public void A_charset_dotnet_will_not_construct_falls_back_to_utf8(string contentType) + { + var wire = new MockHttpHandler(); + wire.QueueBytes(HttpStatusCode.OK, + System.Text.Encoding.UTF8.GetBytes("{\"displayName\":\"charset\"}"), + contentType); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1"); + var output = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + var item = Assert.Single(output); + Assert.Equal("charset", ((System.Collections.Hashtable)item.BaseObject)["displayName"]); + } + } + + /// Same header on the write path, which reads the body the same way. + [Fact] + public void A_charset_dotnet_will_not_construct_does_not_end_a_write() + { + var wire = new MockHttpHandler(); + wire.QueueBytes(HttpStatusCode.OK, + System.Text.Encoding.UTF8.GetBytes("{\"displayName\":\"charset\"}"), + "application/json; charset=utf-7"); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users/u1") + .AddParameter("Method", "PATCH") + .AddParameter("Body", "{}") + .AddParameter("Confirm", false); + var output = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + var item = Assert.Single(output); + Assert.Equal("charset", ((System.Collections.Hashtable)item.BaseObject)["displayName"]); + } } [Fact] @@ -213,7 +386,7 @@ public void A_malformed_export_page_is_an_error_record_not_a_crash() { var wire = new MockHttpHandler(); wire.QueueResponse(HttpStatusCode.OK, "not a collection envelope"); - InjectTransport(wire); + using var transport = MgxTransportScope.Inject(wire); var outFile = Path.Combine(Path.GetTempPath(), $"mgx-export-{Guid.NewGuid():N}.jsonl"); try { @@ -229,7 +402,6 @@ public void A_malformed_export_page_is_an_error_record_not_a_crash() finally { File.Delete(outFile); - ResetTransport(); } } @@ -238,11 +410,8 @@ public void A_304_from_a_conditional_download_is_not_an_error() { var wire = new MockHttpHandler(); wire.QueueEmpty((HttpStatusCode)304); - InjectTransport(wire); - Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, true); - Base.GetField("s_cachedTotalTimeoutSeconds", Static)!.SetValue(null, - new ResilientGraphClientOptions().TotalTimeoutSeconds); - try + // Get-MgxContent refuses a transport mgx does not own, so this one claims ownership. + using (MgxTransportScope.Inject(wire, owned: true)) { using var ps = CreateShell(); ps.AddScript(""" @@ -253,7 +422,6 @@ public void A_304_from_a_conditional_download_is_not_an_error() Assert.Empty(output); Assert.Empty(ps.Streams.Error); } - finally { ResetTransport(); } } [Fact] @@ -261,8 +429,7 @@ public void A_write_answered_with_html_is_an_error_record() { var wire = new MockHttpHandler(); wire.QueueResponse(HttpStatusCode.OK, "gateway", contentType: "text/html"); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { using var ps = CreateShell(); ps.AddCommand("Invoke-MgxRequest") @@ -276,6 +443,5 @@ public void A_write_answered_with_html_is_an_error_record() var error = Assert.Single(ps.Streams.Error); Assert.StartsWith("NonJsonResponse", error.FullyQualifiedErrorId); } - finally { ResetTransport(); } } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/SkipLogicTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/SkipLogicTests.cs index 71c07b7..460e6b3 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/SkipLogicTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/SkipLogicTests.cs @@ -1,5 +1,4 @@ using System.Net; -using System.Reflection; using Mgx.Engine.Http; namespace Mgx.IntegrationTests; @@ -12,37 +11,9 @@ namespace Mgx.IntegrationTests; [Collection("Pipeline")] public class SkipLogicTests { - private static readonly Type Base = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); - private const BindingFlags Static = BindingFlags.NonPublic | BindingFlags.Static; private const string NotFoundBody = """{ "error": { "code": "Request_ResourceNotFound", "message": "missing" } }"""; - private static void InjectTransport(HttpMessageHandler wire) - { - Base.GetField("s_graphHttpClient", Static)!.SetValue(null, new HttpClient(wire)); - Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, - Mgx.Cmdlets.Base.MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); - Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); - Base.GetField("s_clientOptions", Static)!.SetValue(null, - new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); - } - - private static void ResetTransport() - { - // Restore every static InjectTransport touched - a later test in the collection - // that drives a cmdlet without injecting must not inherit this class's transport. - Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); - Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); - Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); - Base.GetField("s_cachedTotalTimeoutSeconds", Static)!.SetValue(null, 0); - Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); - Base.GetField("s_clientOptions", Static)!.SetValue(null, new ResilientGraphClientOptions()); - ResiliencePipelineFactory.Reset(); - } - private static (int Output, List Errors, List Warnings) Run(string script) { @@ -69,13 +40,11 @@ public void A_404_is_skipped_without_an_error_on_every_path(string script) { var wire = new MockHttpHandler(); wire.SetDefaultResponse(HttpStatusCode.NotFound, NotFoundBody); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { var (_, errors, _) = Run(script); Assert.Empty(errors); } - finally { ResetTransport(); } } [Theory] @@ -87,13 +56,11 @@ public void The_same_404_is_an_error_without_the_switch(string script) { var wire = new MockHttpHandler(); wire.SetDefaultResponse(HttpStatusCode.NotFound, NotFoundBody); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { var (_, errors, _) = Run(script); Assert.NotEmpty(errors); } - finally { ResetTransport(); } } [Fact] @@ -101,8 +68,7 @@ public void Skipped_items_are_counted_in_one_warning_summary() { var wire = new MockHttpHandler(); wire.SetDefaultResponse(HttpStatusCode.NotFound, NotFoundBody); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { var (output, errors, warnings) = Run( "'u1','u2','u3' | Invoke-MgxRequest -Uri '/users/{id}' -SkipNotFound"); @@ -110,6 +76,5 @@ public void Skipped_items_are_counted_in_one_warning_summary() Assert.Empty(errors); Assert.Contains(warnings, w => w.Contains("Skipped 3")); } - finally { ResetTransport(); } } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/StaleTempSweepOwnershipTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/StaleTempSweepOwnershipTests.cs new file mode 100644 index 0000000..998e448 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/StaleTempSweepOwnershipTests.cs @@ -0,0 +1,145 @@ +using System.Management.Automation; +using System.Net; +using System.Text; + +namespace Mgx.IntegrationTests; + +/// +/// The stale-temp sweep runs on every fresh export, and it searched with "{output}.*.tmp" - +/// a glob whose '*' spans dots. An export to "users" therefore swept the temp of an +/// interrupted export to "users.jsonl", whose checkpoint still named it. That checkpoint then +/// resolves to a missing file and the interrupted run re-enumerates from page one, with +/// nothing to say that another mgx run deleted what it had collected. +/// +[Collection("Pipeline")] +public class StaleTempSweepOwnershipTests +{ + private const string Page1 = """ + {"value":[{"id":"u1"},{"id":"u2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2"} + """; + private const string Page2 = """ + {"value":[{"id":"u3"}]} + """; + private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + + private sealed class ScriptedHandler : HttpMessageHandler + { + private readonly Queue<(HttpStatusCode Status, string Body)> _steps = new(); + private readonly object _lock = new(); + + public void Queue(HttpStatusCode status, string body) + { + lock (_lock) _steps.Enqueue((status, body)); + } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + (HttpStatusCode Status, string Body) step; + lock (_lock) + step = _steps.Count > 0 ? _steps.Dequeue() : (HttpStatusCode.InternalServerError, ServerError); + return Task.FromResult(new HttpResponseMessage(step.Status) + { + RequestMessage = request, + Content = new StringContent(step.Body, Encoding.UTF8, "application/json") + }); + } + } + + private static PowerShell Shell() + { + var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + return ps; + } + + private static void Export(string output, string? checkpoint, bool all, int top = 0) + { + using var ps = Shell(); + var cmd = ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", output); + if (checkpoint != null) cmd.AddParameter("CheckpointPath", checkpoint); + if (all) cmd.AddParameter("All"); + if (top > 0) cmd.AddParameter("Top", top); + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + } + + [Fact] + public void A_fresh_export_does_not_sweep_a_longer_named_exports_temp() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-sweep-owner-{Guid.NewGuid():N}")).FullName; + var theirs = Path.Combine(dir, "users.jsonl"); + var mine = Path.Combine(dir, "users"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // An export to "users.jsonl" collects page 1 into its temp and dies on page 2. + handler.Queue(HttpStatusCode.OK, Page1); + Export(theirs, checkpoint, all: true); + var temp = Assert.Single(Directory.GetFiles(dir, "users.jsonl.*.tmp")); + Assert.False(File.Exists(theirs)); + + // An unrelated export to "users" in the same directory. + handler.Queue(HttpStatusCode.OK, Page1); + Export(mine, checkpoint: null, all: false, top: 1); + Assert.True(File.Exists(temp), "another export's temp file was deleted by this export's sweep"); + + // The interrupted export resumes onto what it had already collected. + handler.Queue(HttpStatusCode.OK, Page2); + Export(theirs, checkpoint, all: true); + Assert.Equal( + ["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(theirs)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The sweep deletes files it did not create, on the strength of their names, so the name + /// has to be one a run could have given its own temp: the output's, a dot, 32 hex digits, + /// ".tmp". Reaching wider took "users.jsonl.backup.tmp" - somebody else's file, in a + /// directory mgx was only ever asked to write one output into. + /// + [Fact] + public void A_fresh_export_does_not_sweep_a_tmp_no_run_could_have_written() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-sweep-shape-{Guid.NewGuid():N}")).FullName; + var output = Path.Combine(dir, "users.jsonl"); + var backup = Path.Combine(dir, "users.jsonl.backup.tmp"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + var kept = new[] { "{\"id\":\"kept-1\"}", "{\"id\":\"kept-2\"}" }; + File.WriteAllLines(backup, kept); + + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + Export(output, checkpoint: null, all: true); + + Assert.Equal( + ["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.True(File.Exists(backup), "a file mgx never wrote was deleted by the sweep"); + Assert.Equal(kept, File.ReadAllLines(backup)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/StalledBodyTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/StalledBodyTests.cs new file mode 100644 index 0000000..789a7ba --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/StalledBodyTests.cs @@ -0,0 +1,137 @@ +using System.Management.Automation; +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// A server that sends response headers and then stops sending the body, without closing the +/// connection. Requests go out with HttpCompletionOption.ResponseHeadersRead, so neither +/// HttpClient.Timeout nor the pipeline's attempt timeout bounds the content read - the engine's +/// own BodyReadTimeout is the only thing that does. The collection path applied it; the single +/// request, the write and the entity fan-out read the body on the bare cmdlet token and waited +/// for a byte that never came, until Ctrl-C. +/// +[Collection("Pipeline")] +public class StalledBodyTests +{ + /// Sends one byte of body and then nothing, honoring only the read's own token. + private sealed class StallingContent : HttpContent + { + protected override async Task SerializeToStreamAsync(Stream stream, TransportContext? context) + => await SerializeToStreamAsync(stream, context, CancellationToken.None); + + protected override async Task SerializeToStreamAsync( + Stream stream, TransportContext? context, CancellationToken cancellationToken) + { + stream.WriteByte((byte)'{'); + await stream.FlushAsync(cancellationToken); + await Task.Delay(Timeout.Infinite, cancellationToken); + } + + protected override bool TryComputeLength(out long length) + { + length = 4096; + return true; + } + } + + private sealed class StallingHandler(HttpStatusCode status) : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var content = new StallingContent(); + content.Headers.TryAddWithoutValidation("Content-Type", "application/json"); + return Task.FromResult(new HttpResponseMessage(status) + { + RequestMessage = request, + Content = content + }); + } + } + + private static MgxTransportScope InjectStallingTransport( + HttpStatusCode status = HttpStatusCode.OK) => + MgxTransportScope.Inject(new StallingHandler(status), options: new ResilientGraphClientOptions + { + NoRateLimit = true, + MaxRetryAttempts = 1, + AttemptTimeoutSeconds = 2 + }); + + /// + /// Runs the cmdlet on a worker so a hang fails the test instead of wedging the suite. + /// + private static IReadOnlyList RunAndCollectErrors(Action build) + { + var work = Task.Run(() => + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + build(ps); + // A terminating error carries the record too, and a stall that ends the pipeline + // rather than reporting an error record is exactly what these tests are about. + try { ps.Invoke(); } + catch (RuntimeException ex) { return (IReadOnlyList)[ex.ErrorRecord]; } + return (IReadOnlyList)ps.Streams.Error.ToList(); + }); + + Assert.True(work.Wait(TimeSpan.FromSeconds(30)), + "the cmdlet was still running 30s after the server stalled the response body"); + return work.Result; + } + + [Theory] + [InlineData("GET")] + [InlineData("PATCH")] + public void A_stalled_body_ends_the_request_instead_of_hanging(string method) + { + using var transport = InjectStallingTransport(); + var errors = RunAndCollectErrors(ps => + { + var cmd = ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users/u1") + .AddParameter("Method", method); + if (method != "GET") cmd.AddParameter("Body", "{}").AddParameter("Confirm", false); + }); + + Assert.Contains(errors, e => + e.ToString().Contains(ResilientGraphClient.BodyReadTimedOutMessage, StringComparison.Ordinal)); + } + + [Fact] + public void A_stalled_body_ends_an_entity_fan_out_instead_of_hanging() + { + using var transport = InjectStallingTransport(); + var errors = RunAndCollectErrors(ps => + ps.AddScript("'u1','u2' | Invoke-MgxRequest -Uri '/users/{id}'")); + + Assert.Contains(errors, e => + e.ToString().Contains(ResilientGraphClient.BodyReadTimedOutMessage, StringComparison.Ordinal)); + } + + /// + /// The error body on the collection path. It was the one body read in the client without + /// the timeout wrapper, so a server that stalled while sending a 500 ended the pipeline + /// with a raw TaskCanceledException: outside the error contract, and past the filter the + /// cmdlet catches its transport failures with. + /// + [Fact] + public void A_stalled_error_body_on_the_all_path_is_reported_as_a_transport_failure() + { + using var transport = InjectStallingTransport(HttpStatusCode.InternalServerError); + var errors = RunAndCollectErrors(ps => + ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("All", true)); + + var error = Assert.Single(errors); + Assert.Equal(ResilientGraphClient.BodyReadTimedOutMessage, error.Exception.Message); + Assert.StartsWith("HttpError", error.FullyQualifiedErrorId, StringComparison.Ordinal); + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/TopWithAllTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/TopWithAllTests.cs index 53111d1..56603c9 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/TopWithAllTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/TopWithAllTests.cs @@ -1,6 +1,5 @@ using System.Management.Automation; using System.Net; -using System.Reflection; using System.Text; using Mgx.Cmdlets.Base; using Mgx.Engine.Http; @@ -45,37 +44,10 @@ protected override Task SendAsync( } } - private static void InjectMock(HttpMessageHandler handler) - { - ResiliencePipelineFactory.Reset(); - var t = typeof(MgxCmdletBase); - t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, new HttpClient(handler)); - t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - t.GetField("s_ownsHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, false); - t.GetField("s_graphEndpoint", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! - .SetValue(null, "https://graph.microsoft.com"); - t.GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! - .SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); - } - - private static void CleanupMock() - { - var t = typeof(MgxCmdletBase); - t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - ResiliencePipelineFactory.Reset(); - } - private static (int Count, int Requests) Enumerate(int top, bool all, int pageSize) { var handler = new PagedHandler(pageSize, maxPages: 40); - InjectMock(handler); - try + using (MgxTransportScope.Inject(handler)) { using var ps = PowerShell.Create(); ps.AddCommand("Import-Module") @@ -95,7 +67,6 @@ private static (int Count, int Requests) Enumerate(int top, bool all, int pageSi var output = ps.Invoke(); return (output.Count, handler.RequestCount); } - finally { CleanupMock(); } } [Fact] @@ -132,7 +103,7 @@ public void Export_stops_at_Top_rather_than_walking_the_whole_collection() // The sibling of the Invoke-MgxRequest case. Both cmdlets document -Top as a total // maximum, so they must not disagree about what -All -Top means. var handler = new PagedHandler(pageSize: 50, maxPages: 40); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); var dir = Directory.CreateDirectory( Path.Combine(Path.GetTempPath(), $"mgx-topall-{Guid.NewGuid():N}")).FullName; var output = Path.Combine(dir, "out.jsonl"); @@ -159,7 +130,6 @@ public void Export_stops_at_Top_rather_than_walking_the_whole_collection() } finally { - CleanupMock(); try { Directory.Delete(dir, true); } catch { } } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs index 6fc66fa..7fe1c21 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs @@ -1,5 +1,4 @@ using System.Net; -using System.Reflection; using Mgx.Engine.Http; namespace Mgx.IntegrationTests; @@ -13,35 +12,6 @@ namespace Mgx.IntegrationTests; [Collection("Pipeline")] public class UriEncodingTests { - private static readonly Type Base = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); - private const BindingFlags Static = BindingFlags.NonPublic | BindingFlags.Static; - - private static void InjectTransport(HttpMessageHandler wire) - { - Base.GetField("s_graphHttpClient", Static)!.SetValue(null, new HttpClient(wire)); - Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, - Mgx.Cmdlets.Base.MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); - Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); - Base.GetField("s_clientOptions", Static)!.SetValue(null, - new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); - } - - private static void ResetTransport() - { - // Restore every static InjectTransport touched - a later test in the collection - // that drives a cmdlet without injecting must not inherit this class's transport. - Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); - Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); - Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); - Base.GetField("s_cachedTotalTimeoutSeconds", Static)!.SetValue(null, 0); - Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); - Base.GetField("s_clientOptions", Static)!.SetValue(null, new ResilientGraphClientOptions()); - ResiliencePipelineFactory.Reset(); - } - private static string WireUriFor(MockHttpHandler wire, Action configure) { using var ps = System.Management.Automation.PowerShell.Create(); @@ -64,8 +34,7 @@ public void A_hash_in_a_path_reaches_the_wire_instead_of_becoming_a_fragment() { var wire = new MockHttpHandler(); wire.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") .AddParameter("Uri", "/me/drive/root:/reports/a#b.txt")); @@ -73,7 +42,6 @@ public void A_hash_in_a_path_reaches_the_wire_instead_of_becoming_a_fragment() Assert.Contains("/reports/a%23b.txt", uri); Assert.DoesNotContain("#", uri); } - finally { ResetTransport(); } } [Fact] @@ -81,8 +49,7 @@ public void A_raw_filter_is_escaped_once() { var wire = new MockHttpHandler(); wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") .AddParameter("Uri", "/users") @@ -90,7 +57,6 @@ public void A_raw_filter_is_escaped_once() Assert.Contains("$filter=displayName%20eq%20%27O%27%27Brien%20%26%20S%C3%B6hne%27", uri); } - finally { ResetTransport(); } } [Fact] @@ -98,8 +64,7 @@ public void A_pre_encoded_filter_is_not_encoded_twice() { var wire = new MockHttpHandler(); wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") .AddParameter("Uri", "/users") @@ -108,7 +73,6 @@ public void A_pre_encoded_filter_is_not_encoded_twice() Assert.Contains("$filter=displayName%20eq%20%27Bob%27", uri); Assert.DoesNotContain("%25", uri); } - finally { ResetTransport(); } } [Fact] @@ -116,8 +80,7 @@ public void A_literal_percent_that_is_not_a_triplet_is_escaped() { var wire = new MockHttpHandler(); wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") .AddParameter("Uri", "/users") @@ -125,7 +88,6 @@ public void A_literal_percent_that_is_not_a_triplet_is_escaped() Assert.Contains("50%25%20off", uri); } - finally { ResetTransport(); } } [Fact] @@ -135,14 +97,9 @@ public void A_piped_drive_item_id_is_escaped_into_the_content_path() // built from pipeline data, so the injection point escapes rather than trusts. var wire = new MockHttpHandler(); wire.QueueBytes(HttpStatusCode.OK, [1, 2, 3], "application/octet-stream"); - InjectTransport(wire); // Get-MgxContent refuses borrowed transports (the SDK's RedirectHandler would - // auto-follow the content 302); this injected client is mgx-owned. An owned client - // is rebuilt when the cached timeout disagrees with options, so keep them equal. - Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, true); - Base.GetField("s_cachedTotalTimeoutSeconds", Static)!.SetValue(null, - new ResilientGraphClientOptions().TotalTimeoutSeconds); - try + // auto-follow the content 302); this injected client is mgx-owned. + using (MgxTransportScope.Inject(wire, owned: true)) { using var ps = System.Management.Automation.PowerShell.Create(); ps.AddCommand("Import-Module") @@ -165,7 +122,6 @@ public void A_piped_drive_item_id_is_escaped_into_the_content_path() Assert.True(captured.Count > 0, $"no request sent; errors: {errText}"); Assert.Contains("/drives/b%21x_y/items/01ABC%21DEF%3AXYZ/content", captured[0].Uri); } - finally { ResetTransport(); } } [Fact] @@ -177,8 +133,7 @@ public void Every_hostile_filter_round_trips_through_encoding() { var wire = new MockHttpHandler(); wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") .AddParameter("Uri", "/users") @@ -192,7 +147,6 @@ public void Every_hostile_filter_round_trips_through_encoding() Assert.DoesNotContain('+', query); Assert.Equal(filter, System.Uri.UnescapeDataString(query)); } - finally { ResetTransport(); } } } @@ -201,8 +155,7 @@ public void A_top_already_in_the_uri_is_not_sent_twice() { var wire = new MockHttpHandler(); wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") .AddParameter("Uri", "/users?$top=5") @@ -211,7 +164,6 @@ public void A_top_already_in_the_uri_is_not_sent_twice() Assert.Single(System.Text.RegularExpressions.Regex.Matches(uri, "top=")); Assert.Contains("$top=5", uri); } - finally { ResetTransport(); } } [Fact] @@ -219,8 +171,7 @@ public void A_filter_already_in_the_uri_wins_over_the_parameter_with_a_warning() { var wire = new MockHttpHandler(); wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { using var ps = System.Management.Automation.PowerShell.Create(); ps.AddCommand("Import-Module") @@ -242,7 +193,6 @@ public void A_filter_already_in_the_uri_wins_over_the_parameter_with_a_warning() // Deferring must be loud: the caller's -Filter was not sent. Assert.Contains(ps.Streams.Warning, w => w.Message.Contains("-Filter")); } - finally { ResetTransport(); } } [Fact] @@ -250,8 +200,7 @@ public void An_encoded_option_name_in_the_uri_still_defers_the_parameter() { var wire = new MockHttpHandler(); wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") .AddParameter("Uri", "/users?%24top=10") @@ -260,7 +209,6 @@ public void An_encoded_option_name_in_the_uri_still_defers_the_parameter() // %24top IS $top to the server; a second $top would draw a 400. Assert.Single(System.Text.RegularExpressions.Regex.Matches(uri, "top=")); } - finally { ResetTransport(); } } [Fact] @@ -271,8 +219,7 @@ public void A_caller_count_in_the_uri_does_not_arm_the_count_degradation_retry() var wire = new MockHttpHandler(); wire.SetDefaultResponse(HttpStatusCode.BadRequest, """{ "error": { "code": "BadRequest", "message": "no" } }"""); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { using var ps = System.Management.Automation.PowerShell.Create(); ps.AddCommand("Import-Module") @@ -289,7 +236,6 @@ public void A_caller_count_in_the_uri_does_not_arm_the_count_degradation_retry() // $count was never mgx's to drop. Assert.Single(wire.CapturedRequests); } - finally { ResetTransport(); } } [Fact] @@ -302,8 +248,7 @@ public void Every_hostile_path_segment_survives_id_templating() { var wire = new MockHttpHandler(); wire.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { using var ps = System.Management.Automation.PowerShell.Create(); ps.AddCommand("Import-Module") @@ -320,7 +265,6 @@ public void Every_hostile_path_segment_survives_id_templating() Assert.Contains($"/items/{System.Uri.EscapeDataString(segment)}", sent.Uri); Assert.DoesNotContain('#', sent.Uri); } - finally { ResetTransport(); } } } diff --git a/tests/Mgx.IntegrationTests/Engine/Batch/BatchEdgeCaseTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/BatchEdgeCaseTests.cs index fd38931..293b5b8 100644 --- a/tests/Mgx.IntegrationTests/Engine/Batch/BatchEdgeCaseTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Batch/BatchEdgeCaseTests.cs @@ -124,7 +124,7 @@ public async Task Batch_PartialRetry_10Succeed_10Throttled_ThenAllSucceed() // ── R2-3: Mismatched response count ─────────────────────────────────────── [Fact] - public async Task Batch_MismatchedResponseCount_Throws() + public async Task Batch_MismatchedResponseCount_IsReportedAsAChunkFailure() { // Send 5 items but server returns only 3 responses var truncatedResponse = """ @@ -148,12 +148,16 @@ public async Task Batch_MismatchedResponseCount_Throws() .Select(i => new BatchOperation($"/users/user{i}", "GET")) .ToList(); - var ex = await Assert.ThrowsAsync( - () => batchClient.ExecuteBatchIndexedAsync(operations)); + var result = await batchClient.ExecuteBatchIndexedAsync(operations); + var ex = Assert.IsType(result.ChunkFailure); Assert.Contains("count mismatch", ex.Message); Assert.Contains("sent 5", ex.Message); Assert.Contains("received 3", ex.Message); + // The envelope cannot be matched to what was sent, so no item has an answer - but the + // requests went out, and every one of them is accounted for by position. + Assert.Equal(5, result.Results.Count); + Assert.DoesNotContain(result.Results, r => r.Response.Status == GraphBatchClient.NotSentStatus); } // ── R2-8: 0 items — empty batch ────────────────────────────────────────── diff --git a/tests/Mgx.IntegrationTests/Engine/Batch/BatchWriteTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/BatchWriteTests.cs index c2dd1b4..03e039f 100644 --- a/tests/Mgx.IntegrationTests/Engine/Batch/BatchWriteTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Batch/BatchWriteTests.cs @@ -645,7 +645,7 @@ public async Task BatchGet_RetryExhausted_ReturnsLastError() } [Fact] - public async Task BatchGet_ResponseCountMismatch_Throws() + public async Task BatchGet_ResponseCountMismatch_IsReportedAsAChunkFailure() { // Send 3 requests but mock returns only 2 response items var truncatedResponse = """ @@ -671,8 +671,9 @@ public async Task BatchGet_ResponseCountMismatch_Throws() new("/users/c"), }; - var ex = await Assert.ThrowsAsync( - () => batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None)); + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + var ex = Assert.IsType(result.ChunkFailure); Assert.Contains("count mismatch", ex.Message); Assert.Contains("sent 3", ex.Message); Assert.Contains("received 2", ex.Message); @@ -1270,7 +1271,7 @@ public async Task R2_3b_PartialRetry_SomeFailSomeSucceed() } [Fact] - public async Task R2_3c_MismatchedResponseIds_Throws() + public async Task R2_3c_MismatchedResponseIds_AreReportedAsAChunkFailure() { // Response IDs don't match request IDs — should throw InvalidOperationException var mismatchedResponse = """ @@ -1298,16 +1299,18 @@ public async Task R2_3c_MismatchedResponseIds_Throws() new BatchOperation("/users/2") }; - var ex = await Assert.ThrowsAsync( - () => batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None)); + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + var ex = Assert.IsType(result.ChunkFailure); Assert.Contains("missing result", ex.Message); } [Fact] - public async Task R2_3d_Phase2_TransportError_PropagatesException() + public async Task R2_3d_Phase2_TransportError_IsReportedWithTheResults() { - // Per-chunk retries exhaust on 503. Phase 2 batch-level retry hits - // a transport error (HttpRequestException). Should propagate. + // Per-chunk retries exhaust on 503. Phase 2 batch-level retry hits a transport error + // (HttpRequestException). The pass runs once every chunk has been sent, so the run + // already holds the outcome of the whole batch: the failure is reported alongside it. var handler = new MockHttpHandler(); // Per-chunk: 4 attempts all 503 for (int i = 0; i < 4; i++) @@ -1331,9 +1334,14 @@ public async Task R2_3d_Phase2_TransportError_PropagatesException() new BatchOperation("/users/2") }; - // The transport error during Phase 2 should propagate - await Assert.ThrowsAnyAsync( - () => batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None)); + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.NotNull(result.ChunkFailure); + // The 503s are what the server said about these two operations, and the retry pass + // failing did not change that. + Assert.Equal(2, result.Results.Count); + Assert.All(result.Results, r => Assert.Equal(503, r.Response.Status)); + Assert.Equal(2, result.Telemetry.Failed); } // ═══════════════════════════════════════════════════════════════ @@ -1700,7 +1708,7 @@ public async Task BatchMixed_GetAndPost_BothProcessed() } [Fact] - public async Task BatchResponseCountMismatch_ThrowsInvalidOperationException() + public async Task BatchResponseCountMismatch_IsReportedAsAChunkFailure() { var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, """ @@ -1722,9 +1730,9 @@ public async Task BatchResponseCountMismatch_ThrowsInvalidOperationException() new("/users/3") }; - var ex = await Assert.ThrowsAsync( - () => batchClient.ExecuteBatchIndexedAsync(operations)); + var result = await batchClient.ExecuteBatchIndexedAsync(operations); + var ex = Assert.IsType(result.ChunkFailure); Assert.Contains("response count mismatch", ex.Message, StringComparison.OrdinalIgnoreCase); } @@ -1744,7 +1752,7 @@ public async Task BatchEmptyOperations_ReturnsEmptyResult() } [Fact] - public async Task BatchEmptyResponsesFromGraph_ThrowsInvalidOperationException() + public async Task BatchEmptyResponsesFromGraph_IsReportedAsAChunkFailure() { var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, """{ "responses": [] }"""); @@ -1753,9 +1761,9 @@ public async Task BatchEmptyResponsesFromGraph_ThrowsInvalidOperationException() using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); var batchClient = new GraphBatchClient(client); - var ex = await Assert.ThrowsAsync( - () => batchClient.ExecuteBatchIndexedAsync(new[] { new BatchOperation("/users/1") })); + var result = await batchClient.ExecuteBatchIndexedAsync(new[] { new BatchOperation("/users/1") }); + var ex = Assert.IsType(result.ChunkFailure); Assert.Contains("empty or malformed", ex.Message, StringComparison.OrdinalIgnoreCase); } } diff --git a/tests/Mgx.IntegrationTests/Engine/Batch/BulkWriteTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/BulkWriteTests.cs index 3dec881..290143c 100644 --- a/tests/Mgx.IntegrationTests/Engine/Batch/BulkWriteTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Batch/BulkWriteTests.cs @@ -52,6 +52,39 @@ public async Task BulkWriteAsync_AllSucceed_ReturnsCorrectCounts() ResiliencePipelineFactory.Reset(); } + /// + /// A body that opens with a UTF-8 byte-order mark is a body. Deserialized from the bytes it + /// was refused for an invalid start character, and the write - a 201, applied on the server - + /// was reported as a status-0 failure of the request that carried it. + /// + [Fact] + public async Task BulkWriteAsync_BomPrefixedBody_CountsAsSucceededAndParses() + { + ResiliencePipelineFactory.Reset(); + var handler = new MockHttpHandler(); + byte[] bom = [0xEF, 0xBB, 0xBF]; + var body = System.Text.Encoding.UTF8.GetBytes(TestData.SingleUser); + handler.QueueBytes(HttpStatusCode.Created, [.. bom, .. body], "application/json"); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, NoRateLimitOptions); + var fanOut = new ConcurrentFanOut(client, maxConcurrency: 2); + + var ops = new List<(string id, string url)> + { + ("op1", "https://graph.microsoft.com/v1.0/users") + }; + + var result = await fanOut.BulkWriteAsync(HttpMethod.Post, ops, """{"displayName":"Test"}"""); + + Assert.Equal(1, result.Succeeded); + Assert.Empty(result.Errors); + var response = Assert.Single(result.Responses); + Assert.Equal("user1", response.Response.GetProperty("id").GetString()); + + ResiliencePipelineFactory.Reset(); + } + [Fact] public async Task BulkWriteAsync_EmptyOperations_ReturnsEmptyResult() { @@ -248,9 +281,11 @@ public async Task BulkWriteAsync_InvalidJsonBody_NoDoubleCount() Assert.Equal(1, result.Failed); Assert.Single(result.Errors); Assert.Equal("bad-json", result.Errors[0].Id); - // StatusCode 0 proves error came from deserialization-failure catch path, - // not from the HTTP-error path (which would carry the real status code) - Assert.Equal(0, result.Errors[0].StatusCode); + // What the body did is what tells the deserialization failure from the HTTP-error + // path. The status is the one the server answered with: the write reached it, and a + // 0 there says it never did. + Assert.Equal(201, result.Errors[0].StatusCode); + Assert.Contains("invalid start of a value", result.Errors[0].Message, StringComparison.Ordinal); ResiliencePipelineFactory.Reset(); } @@ -299,6 +334,72 @@ public async Task BulkWriteAsync_MixedErrors_PreservesStatusCodes() ResiliencePipelineFactory.Reset(); } + /// + /// A server that sends the headers of an error and then stops sending the body. The stall + /// was recorded as if it were the body, run through the Graph-error parser that does not + /// know it, and reached the caller as "HTTP 500" - which is the one thing it already knew. + /// + [Fact] + public async Task BulkWriteAsync_StalledErrorBody_KeepsTheStallInTheRecord() + { + ResiliencePipelineFactory.Reset(); + + using var httpClient = new HttpClient(new StallingErrorHandler()); + using var client = new ResilientGraphClient(httpClient, NoRateLimitOptions) + { + BodyReadTimeout = TimeSpan.FromSeconds(2) + }; + var fanOut = new ConcurrentFanOut(client, maxConcurrency: 1); + + var ops = new List<(string id, string url)> + { + ("op1", "https://graph.microsoft.com/v1.0/users/u1") + }; + + var result = await fanOut.BulkWriteAsync(HttpMethod.Patch, ops, """{"displayName":"Test"}"""); + + Assert.Equal(1, result.Failed); + var error = Assert.Single(result.Errors); + Assert.Equal(500, error.StatusCode); + Assert.Equal(ResilientGraphClient.BodyReadTimedOutMessage, error.Message); + + ResiliencePipelineFactory.Reset(); + } + + /// + /// The same stall after the server answered 201. The write is on the server; only the read + /// of what it created failed. Recorded as status 0 - the code for a write that never + /// reached an HTTP status - an applied POST read as a connection failure, which is the one + /// kind a caller may safely repeat, and repeating it creates the entity twice. + /// + [Fact] + public async Task BulkWriteAsync_StalledBodyAfterCreated_KeepsTheStatusTheServerAnswered() + { + ResiliencePipelineFactory.Reset(); + + using var httpClient = new HttpClient(new StallingErrorHandler(HttpStatusCode.Created)); + using var client = new ResilientGraphClient(httpClient, NoRateLimitOptions) + { + BodyReadTimeout = TimeSpan.FromSeconds(2) + }; + var fanOut = new ConcurrentFanOut(client, maxConcurrency: 1); + + var ops = new List<(string id, string url)> + { + ("a1", "https://graph.microsoft.com/v1.0/groups/g1/members/$ref") + }; + + var result = await fanOut.BulkWriteAsync(HttpMethod.Post, ops, + """{"@odata.id":"https://graph.microsoft.com/v1.0/users/a1"}"""); + + Assert.Equal(1, result.Failed); + var error = Assert.Single(result.Errors); + Assert.Equal(201, error.StatusCode); + Assert.Equal(ResilientGraphClient.BodyReadTimedOutMessage, error.Message); + + ResiliencePipelineFactory.Reset(); + } + // ── Per-item exception swallowing verification ───────────────────── /// @@ -417,3 +518,44 @@ protected override Task SendAsync( throw _exceptionFactory(); } } + +/// +/// Sends the headers of the given status and one byte of body, then nothing, honoring only +/// the read's own token - the shape of a response whose body stalls. The status defaults to +/// a 500; a write is answered 201 and stalls exactly the same way. +/// +public class StallingErrorHandler(HttpStatusCode status = HttpStatusCode.InternalServerError) + : HttpMessageHandler +{ + private sealed class StallingContent : HttpContent + { + protected override async Task SerializeToStreamAsync(Stream stream, TransportContext? context) + => await SerializeToStreamAsync(stream, context, CancellationToken.None); + + protected override async Task SerializeToStreamAsync( + Stream stream, TransportContext? context, CancellationToken cancellationToken) + { + stream.WriteByte((byte)'{'); + await stream.FlushAsync(cancellationToken); + await Task.Delay(Timeout.Infinite, cancellationToken); + } + + protected override bool TryComputeLength(out long length) + { + length = 4096; + return true; + } + } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var content = new StallingContent(); + content.Headers.TryAddWithoutValidation("Content-Type", "application/json"); + return Task.FromResult(new HttpResponseMessage(status) + { + RequestMessage = request, + Content = content + }); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Batch/ParallelChunkPartialResultTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/ParallelChunkPartialResultTests.cs new file mode 100644 index 0000000..57c920c --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Batch/ParallelChunkPartialResultTests.cs @@ -0,0 +1,854 @@ +using System.Net; +using Mgx.Engine.Http; +using Mgx.Engine.Models; + +namespace Mgx.IntegrationTests; + +/// +/// The partial-result contract of a chunked batch does not depend on how many chunks are in +/// flight at once. With -BatchChunkConcurrency above 1 the chunks run in parallel, and a chunk +/// whose own POST fails used to take the whole run down with it: Task.WhenAll rethrew and the +/// results of the chunks that had already been applied on the server were discarded with the +/// array holding them. +/// +[Collection("Pipeline")] +public class ParallelChunkPartialResultTests +{ + private const string Rejection = + "{\"error\":{\"code\":\"BadRequest\",\"message\":\"batch rejected\"}}"; + + private static string BatchResponse(int count, int status) => + "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, count).Select(i => + $"{{ \"id\": \"{i}\", \"status\": {status} }}")) + "] }"; + + /// + /// Rejects one chunk by its CONTENT, not by arrival order, so sequential and parallel runs + /// see the same wire behavior. + /// + /// Nothing is answered until the whole opening wave is on the wire, so no chunk of it is + /// stopped before it has sent: left to the scheduler the refusal can be recorded while a + /// chunk that holds a permit has not reached the handler yet, and that chunk then turns + /// back correctly at a wire count the test reads as the thing it exists to catch. + /// + /// + /// The rejected chunk is then answered first, so the permit it holds is released only after + /// the run has recorded the refusal. The rest of the wave is held until the run says it has + /// stopped sending, so that permit is the only one a queued chunk can be admitted on and + /// the refusal is recorded before the last chunk asks whether to send. A POST arriving from + /// outside the wave releases them as well: that is the thing the guard exists to prevent + /// and the thing the counts have to be able to see, so a run that sends one is counted + /// rather than held here behind it. + /// + /// + private sealed class RejectOneChunkHandler : HttpMessageHandler + { + private readonly string _marker; + private readonly int _wave; + private readonly TaskCompletionSource _inFlight = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _settled = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _posts; + private int _arrived; + private int _landing; + public int Posts => Volatile.Read(ref _posts); + + /// Body text carried by the one chunk to reject. + /// Chunks other than that one which hold a permit from the start. + /// Zero for a sequential run, which holds one chunk at a time and needs no gate. + public RejectOneChunkHandler(string marker, int wave) + { + _marker = marker; + _wave = wave; + if (wave == 0) _settled.TrySetResult(); + } + + /// Armed on the run: the refusal is recorded and nothing further may send. + public void RunStopped() => _settled.TrySetResult(); + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _posts); + if (Interlocked.Increment(ref _arrived) >= _wave + 1) _inFlight.TrySetResult(); + await Task.Yield(); + var body = await request.Content!.ReadAsStringAsync(cancellationToken); + await _inFlight.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + if (body.Contains(_marker, StringComparison.Ordinal)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent(Rejection, System.Text.Encoding.UTF8, "application/json") + }; + } + + if (Interlocked.Increment(ref _landing) > _wave) _settled.TrySetResult(); + await _settled.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(BatchResponse(20, 204), + System.Text.Encoding.UTF8, "application/json") + }; + } + } + + /// + /// Four chunks, so at concurrency 3 one of them has to wait for a permit and the guard that + /// decides whether a waiting chunk still goes out is actually consulted. At 60 operations + /// every chunk held a permit from the start and the guard was never reached. + /// + private static async Task<(BatchExecutionResult Result, int Posts)> Run(int concurrency, int wave) + { + ResiliencePipelineFactory.Reset(); + // "u21" is the first operation of the second chunk and appears in no other chunk. + var wire = new RejectOneChunkHandler("/users/u21\"", wave); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client, batchChunkConcurrency: concurrency) + { + SendingStopped = wire.RunStopped + }; + + var ops = Enumerable.Range(1, 80) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + return (result, wire.Posts); + } + + // Sequentially the run stops at the refusal: two POSTs, two chunks never sent. In parallel + // three chunks are on the wire and the fourth is still queued when the refusal is recorded, + // so it never goes out either - one chunk refused, one stopped, two applied. + [Theory] + [InlineData(1, 0, 2, 20, 40)] + [InlineData(3, 2, 3, 40, 20)] + public async Task A_failed_chunk_keeps_the_results_of_the_chunks_that_landed( + int concurrency, int wave, int expectedPosts, int expectedLanded, int expectedNotSent) + { + var (result, posts) = await Run(concurrency, wave); + + Assert.NotNull(result.ChunkFailure); + Assert.IsType(result.ChunkFailure); + + // Every operation is accounted for by position, so a caller can line the results up + // against its own input list. + Assert.Equal(80, result.Results.Count); + Assert.Equal(expectedPosts, posts); + + var landed = result.Results.Count(r => r.Response.Status == 204); + var refused = result.Results.Count(r => r.Response.Status == 400); + var notSent = result.Results.Count(r => r.Response.Status == GraphBatchClient.NotSentStatus); + + // Exact counts: an inequality here passes just as well when a chunk that should have + // been stopped went out anyway, which is the thing worth knowing. + Assert.Equal(expectedLanded, landed); + Assert.Equal(20, refused); + Assert.Equal(expectedNotSent, notSent); + Assert.Equal(notSent, result.NotSent.Count); + } + + /// + /// Answers nothing until as many chunks are on the wire as the run can hold at once. The + /// two failures then reach the run at the same time rather than one after the other, and a + /// chunk that was going to send is not stopped before it has - left to the scheduler a + /// chunk can still be waiting to start when the refusals land, and it is then correctly + /// reported as never sent, which is a different outcome from the one asked about here. + /// + /// Chunks the run holds at once, so all of them and no more. + private sealed class TwoChunksRefusedTogetherHandler(int inFlight) : HttpMessageHandler + { + private int _arrived; + private readonly TaskCompletionSource _all = new(TaskCreationOptions.RunContinuationsAsynchronously); + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + await Task.Yield(); + var body = await request.Content!.ReadAsStringAsync(cancellationToken); + var refusal = body.Contains("/users/u1\"", StringComparison.Ordinal) ? "chunk A" + : body.Contains("/users/u21\"", StringComparison.Ordinal) ? "chunk B" + : null; + if (Interlocked.Increment(ref _arrived) == inFlight) _all.TrySetResult(); + await _all.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + if (refusal == null) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(BatchResponse(20, 204), + System.Text.Encoding.UTF8, "application/json") + }; + } + + return new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent( + $"{{\"error\":{{\"code\":\"BadRequest\",\"message\":\"{refusal} rejected\"}}}}", + System.Text.Encoding.UTF8, "application/json") + }; + } + } + + [Fact] + public async Task A_second_chunk_failing_at_the_same_time_is_still_reported() + { + ResiliencePipelineFactory.Reset(); + using var httpClient = new HttpClient(new TwoChunksRefusedTogetherHandler(inFlight: 3)); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var messages = new List(); + var batchClient = new GraphBatchClient(client, batchChunkConcurrency: 3) + { + VerboseWriter = messages.Add + }; + + var ops = Enumerable.Range(1, 60) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + batchClient.DrainVerboseMessages(); + + Assert.NotNull(result.ChunkFailure); + // One of the two is the failure the caller is handed. The other lost a race it had no + // stake in, and used to disappear with no record that it had happened at all. + var reported = result.ChunkFailure!.Message.Contains("chunk A", StringComparison.Ordinal) + ? "chunk A" : "chunk B"; + var lost = reported == "chunk A" ? "chunk B" : "chunk A"; + Assert.Contains(messages, m => m.Contains(lost, StringComparison.Ordinal)); + + // Both refused chunks report their own items as attempted, not as never sent. + for (int i = 0; i < 40; i++) + Assert.Equal(400, result.Results[i].Response.Status); + for (int i = 40; i < 60; i++) + Assert.Equal(204, result.Results[i].Response.Status); + } + + /// + /// With chunks in flight together there is no "earlier" chunk to blame: the two that were + /// refused ran alongside each other, and the one that never went out was behind both. + /// + [Fact] + public void An_operation_that_was_never_sent_is_not_blamed_on_an_earlier_chunk() + { + using (MgxTransportScope.Inject(new TwoChunksRefusedTogetherHandler(inFlight: 2), + options: new ResilientGraphClientOptions + { + NoRateLimit = true, + MaxRetryAttempts = 1, + BatchChunkConcurrency = 2 + })) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", Enumerable.Range(1, 60).Select(i => $"/users/u{i}").ToArray()) + .AddParameter("Method", "PATCH"); + ps.Invoke(); + + // Two chunks hold both permits until both are refused, so the third can only reach + // the semaphore after a refusal has been recorded: it never goes out. + var notSent = ps.Streams.Error + .Where(e => e.FullyQualifiedErrorId.StartsWith("BatchItemNotSent", StringComparison.Ordinal)) + .ToList(); + Assert.Equal(20, notSent.Count); + Assert.All(notSent, e => Assert.DoesNotContain("earlier", e.Exception.Message, + StringComparison.OrdinalIgnoreCase)); + Assert.All(notSent, e => Assert.Contains("was not sent", e.Exception.Message, + StringComparison.Ordinal)); + } + } + + /// + /// One chunk's call fails with no HTTP response at all - the shape a stalled $batch body, + /// an open circuit or a dead connection arrives in. + /// + /// Nothing is answered until as many chunks are on the wire as the run can carry at once, so + /// the one that fails does so with the others already sent. Left to the scheduler a chunk can + /// still be waiting to start when the failure lands - it is then correctly stopped, which is + /// a different outcome, asked about by a different test. + /// + /// + private sealed class OneChunkThrowsHandler(int inFlight) : HttpMessageHandler + { + private readonly TaskCompletionSource _all = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _arrived; + private int _posts; + public int Posts => Volatile.Read(ref _posts); + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _posts); + if (Interlocked.Increment(ref _arrived) == inFlight) _all.TrySetResult(); + await Task.Yield(); + await _all.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + var body = await request.Content!.ReadAsStringAsync(cancellationToken); + if (body.Contains("/users/u21\"", StringComparison.Ordinal)) + throw new HttpRequestException("connection reset"); + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(BatchResponse(20, 204), + System.Text.Encoding.UTF8, "application/json") + }; + } + } + + /// + /// A chunk can fail without a status to fail with. Every one of those shapes used to travel + /// out of the task and take the results array with it, so the chunks the server had already + /// applied came back as nothing. + /// + [Theory] + [InlineData(1)] + [InlineData(3)] + public async Task A_chunk_that_fails_without_a_response_keeps_the_other_chunks(int concurrency) + { + ResiliencePipelineFactory.Reset(); + // Sixty operations are three chunks, so at concurrency 3 the run can hold all three at + // once and the handler answers none of them until it has all three. The failure is then + // reached with nothing left to stop, which is the case this asserts on. + var wire = new OneChunkThrowsHandler(Math.Min(concurrency, 3)); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client, batchChunkConcurrency: concurrency); + + var ops = Enumerable.Range(1, 60) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.IsType(result.ChunkFailure); + Assert.Equal(60, result.Results.Count); + + // The first chunk was applied on the server whichever way the second one died. + for (int i = 0; i < 20; i++) + Assert.Equal(204, result.Results[i].Response.Status); + + // The failed chunk's own items went out and may have been applied. Without a status of + // their own they read as attempted-and-unknown, never as unattempted. + for (int i = 20; i < 40; i++) + Assert.Equal(503, result.Results[i].Response.Status); + + // Sequentially the third chunk never goes out; in parallel it was already in flight. + Assert.Equal(concurrency == 1 ? 20 : 0, result.NotSent.Count); + } + + /// + /// Three chunks, all in flight at once. One is refused; one is throttled and would retry a + /// second later; one is applied. The throttled chunk's retry is the only POST that could + /// still go out after the refusal is recorded, so the wire count says whether anything did. + /// + /// The order the question needs is gated, not raced. Nothing is answered until all three + /// chunks are on the wire, so none of them is stopped before it has sent and the count of + /// three means three chunks that really went out. The refusal then goes back first, and the + /// throttle only once the run has recorded it - so the throttled chunk meets the stop at + /// its next attempt, instead of a one-second Retry-After having to outlast the refusal on a + /// loaded machine to reach the same place. + /// + /// + private sealed class RefuseOneThrottleOneHandler : HttpMessageHandler + { + private readonly TaskCompletionSource _allInFlight = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _stopped = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _arrived; + private int _posts; + public int Posts => Volatile.Read(ref _posts); + + /// Armed on the run: the refusal is recorded and nothing further may send. + public void RunStopped() => _stopped.TrySetResult(); + + private static string Throttled(int count) => + "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, count).Select(i => + $"{{ \"id\": \"{i}\", \"status\": 429, \"headers\": {{ \"Retry-After\": \"1\" }} }}")) + "] }"; + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _posts); + if (Interlocked.Increment(ref _arrived) == 3) _allInFlight.TrySetResult(); + await Task.Yield(); + var body = await request.Content!.ReadAsStringAsync(cancellationToken); + // Answered only once the caller has started the others, so all three are genuinely + // in flight - a handler that answers inline serializes them and the guard never has + // two chunks to arbitrate. + await _allInFlight.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + + if (body.Contains("/users/u21\"", StringComparison.Ordinal)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent(Rejection, System.Text.Encoding.UTF8, "application/json") + }; + } + if (body.Contains("/users/u1\"", StringComparison.Ordinal)) + { + await _stopped.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(Throttled(20), System.Text.Encoding.UTF8, "application/json") + }; + } + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(BatchResponse(20, 204), + System.Text.Encoding.UTF8, "application/json") + }; + } + } + + [Fact] + public async Task Nothing_further_is_posted_once_a_chunk_has_been_refused() + { + ResiliencePipelineFactory.Reset(); + var wire = new RefuseOneThrottleOneHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client, batchChunkConcurrency: 3) + { + SendingStopped = wire.RunStopped + }; + + var ops = Enumerable.Range(1, 60) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.NotNull(result.ChunkFailure); + // One POST per chunk and not one more: no per-chunk retry after the refusal, and no + // follow-up batch either. Each of those would be a write the caller is told nothing about. + Assert.Equal(3, wire.Posts); + Assert.Equal(0, result.Telemetry.BatchLevelRetries); + + // The throttled chunk stops where it is, holding what the server last said about it. + for (int i = 0; i < 20; i++) + Assert.Equal(429, result.Results[i].Response.Status); + for (int i = 20; i < 40; i++) + Assert.Equal(400, result.Results[i].Response.Status); + for (int i = 40; i < 60; i++) + Assert.Equal(204, result.Results[i].Response.Status); + Assert.Empty(result.NotSent); + } + + /// + /// Throttles one chunk for a minute, and refuses another only once the throttled chunk is + /// into the wait between its attempts - the run says when, rather than the refusal being + /// held back long enough that it usually is. The refusal then arrives with the backoff + /// running, which is the only state the question of whether the wait ends with it can be + /// asked from. Nothing is answered until all three chunks are on the wire, so none of them + /// is stopped before it has sent. + /// + private sealed class RefusesDuringARetryWaitHandler : HttpMessageHandler + { + private readonly TaskCompletionSource _inFlight = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _waiting = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _arrived; + private int _posts; + public int Posts => Volatile.Read(ref _posts); + + /// Armed on the run: the throttled chunk is in the wait between two attempts. + public void RetryWaitStarted() => _waiting.TrySetResult(); + + private static string Throttled(int count) => + "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, count).Select(i => + $"{{ \"id\": \"{i}\", \"status\": 429, \"headers\": {{ \"Retry-After\": \"60\" }} }}")) + "] }"; + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _posts); + if (Interlocked.Increment(ref _arrived) == 3) _inFlight.TrySetResult(); + await Task.Yield(); + var body = await request.Content!.ReadAsStringAsync(cancellationToken); + await _inFlight.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + if (body.Contains("/users/u1\"", StringComparison.Ordinal)) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(Throttled(20), + System.Text.Encoding.UTF8, "application/json") + }; + } + if (body.Contains("/users/u21\"", StringComparison.Ordinal)) + { + await _waiting.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + return new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent(Rejection, System.Text.Encoding.UTF8, "application/json") + }; + } + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(BatchResponse(20, 204), + System.Text.Encoding.UTF8, "application/json") + }; + } + } + + /// + /// The run stops sending the moment a chunk is refused, and a chunk already waiting out a + /// Retry-After has nothing left to wait for. Consulted only before the backoff started, the + /// stop left the call blocked for the rest of the delay - here a minute, and up to the + /// two-minute cap plus jitter against a server that asks for longer. + /// + [Fact] + public async Task A_refusal_ends_a_retry_wait_that_is_already_running() + { + ResiliencePipelineFactory.Reset(); + var wire = new RefusesDuringARetryWaitHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client, batchChunkConcurrency: 3) + { + RetryWaitStarted = wire.RetryWaitStarted + }; + + var ops = Enumerable.Range(1, 60) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + sw.Stop(); + + Assert.NotNull(result.ChunkFailure); + Assert.True(sw.Elapsed < TimeSpan.FromSeconds(15), + $"the call waited out {sw.Elapsed.TotalSeconds:F1}s of a Retry-After it had stopped honoring"); + // Nothing more went out: the wait ended because the run stopped, not because it expired. + Assert.Equal(3, wire.Posts); + for (int i = 0; i < 20; i++) + Assert.Equal(429, result.Results[i].Response.Status); + for (int i = 20; i < 40; i++) + Assert.Equal(400, result.Results[i].Response.Status); + for (int i = 40; i < 60; i++) + Assert.Equal(204, result.Results[i].Response.Status); + } + + /// + /// The chunk that was rejected went out on the wire, so its operations carry the refusal. + /// Reporting them as never sent would send a caller back to re-submit writes that may + /// already have been applied. + /// + [Fact] + public async Task The_rejected_chunks_operations_carry_the_refusal_not_a_not_sent_status() + { + var (result, _) = await Run(3, wave: 2); + + Assert.DoesNotContain(result.NotSent, op => op.Url == "/users/u21"); + for (int i = 20; i < 40; i++) + Assert.Equal(400, result.Results[i].Response.Status); + for (int i = 0; i < 20; i++) + Assert.Equal(204, result.Results[i].Response.Status); + } + + /// + /// The verbose summary is the run's own account of what it did. A retry counted when it was + /// decided on rather than when it went out is a request that never left: the refusal ends + /// the backoff, the chunk breaks at its next attempt, and the run reports twenty item + /// retries over a wire that carried one POST per chunk. + /// + [Fact] + public async Task Retries_a_refusal_stopped_are_not_counted_as_retries_that_went_out() + { + ResiliencePipelineFactory.Reset(); + var wire = new RefusesDuringARetryWaitHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client, batchChunkConcurrency: 3) + { + RetryWaitStarted = wire.RetryWaitStarted + }; + + var ops = Enumerable.Range(1, 60) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.NotNull(result.ChunkFailure); + // One POST per chunk and no fourth: the twenty the throttled chunk was going to resend + // never went out. + Assert.Equal(3, wire.Posts); + Assert.Equal(0, result.Telemetry.ItemRetries); + // The throttles and the wait are not in question - those twenty 429s arrived, and the + // chunk did spend time on the backoff before the refusal ended it. + Assert.Equal(20, result.Telemetry.ThrottleEncounters); + } + + /// + /// Throttles one chunk's ENVELOPE - a 429 on the $batch POST itself, with a Retry-After the + /// transport honors - and refuses a second chunk once that throttle has landed, while a + /// third is still on the wire. The throttled chunk is inside the transport's own retry, a + /// layer below the loop that reads the stop, and the wire count says whether its writes + /// went out again after the run had stopped sending. + /// + /// The third chunk is answered only once the run has recorded the refusal, so it is on the + /// wire at that moment by the run's own account of when the moment was. The caller holds + /// the throttled chunk's second attempt at the head of the attempt until the same point, + /// so that attempt meets the stop rather than racing a five-second Retry-After against the + /// round trip that carries the refusal. Nothing at all is answered until all three chunks + /// are on the wire, so none of them is stopped before it has sent. + /// + /// + private sealed class ThrottlesTheEnvelopeThenRefusesHandler : HttpMessageHandler + { + private readonly TaskCompletionSource _inFlight = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _throttled = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _stopped = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _arrived; + private int _posts; + private int _throttledChunkPosts; + public int Posts => Volatile.Read(ref _posts); + public int ThrottledChunkPosts => Volatile.Read(ref _throttledChunkPosts); + + /// Armed on the run: the refusal is recorded and nothing further may send. + public void RunStopped() => _stopped.TrySetResult(); + + /// Completes with that same point, for a caller ordering an attempt against it. + public Task Stopped => _stopped.Task; + + private static HttpResponseMessage Answered(HttpRequestMessage request) => + new(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(BatchResponse(20, 204), + System.Text.Encoding.UTF8, "application/json") + }; + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _posts); + if (Interlocked.Increment(ref _arrived) == 3) _inFlight.TrySetResult(); + await Task.Yield(); + var body = await request.Content!.ReadAsStringAsync(cancellationToken); + await _inFlight.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + + if (body.Contains("/users/u1\"", StringComparison.Ordinal)) + { + if (Interlocked.Increment(ref _throttledChunkPosts) == 1) + { + var throttle = new HttpResponseMessage(HttpStatusCode.TooManyRequests) + { + RequestMessage = request, + Content = new StringContent(Rejection, System.Text.Encoding.UTF8, "application/json") + }; + throttle.Headers.RetryAfter = + new System.Net.Http.Headers.RetryConditionHeaderValue(TimeSpan.FromSeconds(5)); + _throttled.TrySetResult(); + return throttle; + } + // A second POST of this chunk is the run sending after it stopped. Answered + // rather than refused, so the writes it applied show up in the result and the + // test fails on what happened rather than on how it was reported. + return Answered(request); + } + + if (body.Contains("/users/u21\"", StringComparison.Ordinal)) + { + await _throttled.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + return new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent(Rejection, System.Text.Encoding.UTF8, "application/json") + }; + } + + // Still on the wire when the refusal above is recorded: the server is about to + // answer all twenty of its writes, and that answer is the record of what it applied. + await _stopped.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + return Answered(request); + } + } + + /// + /// A chunk inside the transport's own throttle retry when a sibling is refused. The stop + /// reaches the decision to send again, and not the attempt on the wire: nothing further + /// goes out, a chunk that was mid-request keeps the answers the server gave it, and the + /// call comes back rather than waiting out a Retry-After the run has stopped honoring. + /// + [Fact] + public async Task A_chunk_inside_the_transports_retry_stops_with_the_run() + { + ResiliencePipelineFactory.Reset(); + var wire = new ThrottlesTheEnvelopeThenRefusesHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + // The second attempt of the throttled chunk is the POST in question. Held at the head of + // the attempt until the refusal has been recorded, so whether it goes out is decided by + // the stop and not by which of the two finished first. + client.AttemptEntryGate = async attempt => + { + if (attempt < 2) return; + await wire.Stopped.WaitAsync(TimeSpan.FromSeconds(30)); + }; + var batchClient = new GraphBatchClient(client, batchChunkConcurrency: 3) + { + SendingStopped = wire.RunStopped + }; + + var ops = Enumerable.Range(1, 60) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + sw.Stop(); + + Assert.NotNull(result.ChunkFailure); + + // Nothing further went out. The throttled chunk's second POST is the only one that + // could, and it carries twenty writes the caller would be told nothing about. + Assert.Equal(1, wire.ThrottledChunkPosts); + Assert.Equal(3, wire.Posts); + + // The chunk that was on the wire keeps what the server said about each of its items. + // Ending its request to stop the run would trade twenty known outcomes for twenty + // writes that may or may not have been applied. + for (int i = 40; i < 60; i++) + Assert.Equal(204, result.Results[i].Response.Status); + + // The throttled chunk carries the status its envelope was refused with, the same one it + // would carry had the transport run out of retries instead of being stopped. + for (int i = 0; i < 20; i++) + Assert.Equal(429, result.Results[i].Response.Status); + for (int i = 20; i < 40; i++) + Assert.Equal(400, result.Results[i].Response.Status); + Assert.Empty(result.NotSent); + + // The Retry-After was five seconds. Waiting it out holds the call open for a delay the + // run has already decided not to act on. + Assert.True(sw.Elapsed < TimeSpan.FromSeconds(3), + $"the call waited out {sw.Elapsed.TotalSeconds:F1}s of a Retry-After it had stopped honoring"); + } + + /// + /// Refuses whichever POST holds the one permit that was free, and answers every later one. + /// The refusal waits for the other chunks to be queued at the limiter - counted there, not + /// allowed time to get there - so a later POST is a chunk that waited out its permit and + /// sent anyway, not one that reached the queue after the refusal had already been recorded. + /// + private sealed class RefusesTheFirstPostHandler( + System.Threading.RateLimiting.TokenBucketRateLimiter limiter, int queued) : HttpMessageHandler + { + private int _posts; + public int Posts => Volatile.Read(ref _posts); + + private int Queued => (int)limiter.GetStatistics()!.CurrentQueuedCount; + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + if (Interlocked.Increment(ref _posts) == 1) + { + var waited = System.Diagnostics.Stopwatch.StartNew(); + while (Queued < queued) + { + if (waited.Elapsed > TimeSpan.FromSeconds(30)) + throw new TimeoutException( + $"only {Queued} of {queued} chunks reached the permit queue"); + await Task.Delay(5, cancellationToken); + } + return new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent(Rejection, System.Text.Encoding.UTF8, "application/json") + }; + } + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(BatchResponse(20, 204), + System.Text.Encoding.UTF8, "application/json") + }; + } + } + + /// + /// A rate limiter of one permit a second, which the tuning help recommends verbatim for + /// OneNote and Planner, and chunk parallelism above one. Two chunks are parked waiting for a + /// permit when the third is refused; the stop they were given has to reach the wait they are + /// in, or each POSTs its twenty writes as its permit comes up - seconds after the run + /// recorded that it had stopped sending, and with nothing that reports on them. + /// + [Fact] + public async Task Chunks_waiting_for_a_rate_limit_permit_do_not_post_after_a_refusal() + { + ResiliencePipelineFactory.Reset(); + // The limiter the run's requests queue at, held so the handler can count who is in that + // queue rather than guess how long they take to reach it. + var (pipeline, limiter) = ResiliencePipelineFactory.GetOrCreate(new ResilientGraphClientOptions + { + RateLimitBurst = 1, + RateLimitPerSecond = 1, + RateLimitQueueLimit = 100, + MaxRetryAttempts = 1 + }); + var wire = new RefusesTheFirstPostHandler(limiter!, queued: 2); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, pipeline, limiter); + var batchClient = new GraphBatchClient(client, batchChunkConcurrency: 3); + + var ops = Enumerable.Range(1, 60) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + sw.Stop(); + + Assert.NotNull(result.ChunkFailure); + // One permit was free at the start and the chunk that took it was refused. The next two + // permits fall due a second and two seconds later; neither may carry a POST. + Assert.Equal(1, wire.Posts); + Assert.True(sw.Elapsed < TimeSpan.FromSeconds(3), + $"the run held on for {sw.Elapsed.TotalSeconds:F1}s of permits it had stopped needing"); + + // Forty operations never reached the server, and that is what they are reported as. A + // status here instead would say the writes were attempted and may have been applied, + // which is the one thing that would stop a caller resubmitting them. + Assert.Equal(40, result.NotSent.Count); + var refused = result.Results.Count(r => r.Response.Status == 400); + var neverSent = result.Results.Count(r => r.Response.Status == GraphBatchClient.NotSentStatus); + Assert.Equal(20, refused); + Assert.Equal(40, neverSent); + Assert.DoesNotContain(result.Results, r => r.Response.Status == 503); + + // Retries counted in front of a POST that never went out are requests the summary + // claims left when nothing did. + Assert.Equal(0, result.Telemetry.ItemRetries); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs index 5e580e8..7ba6f16 100644 --- a/tests/Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs @@ -1,3 +1,5 @@ +using System.Management.Automation; +using System.Management.Automation.Host; using System.Net; using Mgx.Engine.Http; using Mgx.Engine.Models; @@ -70,6 +72,567 @@ public async Task A_failed_chunk_does_not_discard_the_chunks_that_already_succee $"only {known} of 30 operations came back; the first chunk's 20 results were discarded"); } + /// + /// Answers the chunk in part - some items applied, the rest throttled - and then refuses the + /// retry that carries the throttled ones. The items the first answer covered were applied on + /// the server whatever happens to the retry. + /// + private sealed class HalfLandsThenRefusesHandler : HttpMessageHandler + { + private int _posts; + public int Posts => Volatile.Read(ref _posts); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + if (Interlocked.Increment(ref _posts) == 1) + { + var body = "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, 20).Select(i => + $"{{ \"id\": \"{i}\", \"status\": {(i <= 15 ? 204 : 429)} }}")) + "] }"; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") + }); + } + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent( + "{\"error\":{\"code\":\"BadRequest\",\"message\":\"batch rejected\"}}", + System.Text.Encoding.UTF8, "application/json") + }); + } + } + + [Fact] + public async Task A_chunk_whose_retry_is_refused_keeps_the_items_the_server_answered() + { + ResiliencePipelineFactory.Reset(); + var wire = new HalfLandsThenRefusesHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client); + + var ops = Enumerable.Range(1, 20) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.NotNull(result.ChunkFailure); + // The server answered these 15 with 204: it applied them. A later attempt of the same + // chunk being refused does not unmake that. + for (int i = 0; i < 15; i++) + Assert.Equal(204, result.Results[i].Response.Status); + // The five the refusal carried were sent and may have been applied, so they read as a + // failure of the request - never as an operation that was not attempted. + for (int i = 15; i < 20; i++) + Assert.Equal(400, result.Results[i].Response.Status); + Assert.Empty(result.NotSent); + + // The chunk that failed is the chunk that met the throttling. Counted only off the + // chunks that returned normally, a run throttled into a refusal reported no throttling + // at all - and no retries, and no time spent waiting on either. + Assert.Equal(5, result.Telemetry.ThrottleEncounters); + Assert.Equal(5, result.Telemetry.ItemRetries); + Assert.True(result.Telemetry.TotalRetryDelayMs > 0, + "the retry delay this chunk waited out was not reported"); + } + + [Fact] + public void Items_a_refused_chunk_already_had_answers_for_stay_out_of_the_dead_letter_file() + { + var deadLetter = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + try + { + using (MgxTransportScope.Inject(new HalfLandsThenRefusesHandler())) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", Enumerable.Range(1, 20).Select(i => $"/users/u{i}").ToArray()) + .AddParameter("Method", "PATCH") + .AddParameter("DeadLetterPath", deadLetter); + ps.Invoke(); + } + + // The dead-letter file is documented as re-pipeable. A line for a write the server + // confirmed is an instruction to send it a second time. + var lines = File.Exists(deadLetter) ? File.ReadAllLines(deadLetter) : []; + Assert.Equal(5, lines.Length); + for (int i = 1; i <= 15; i++) + Assert.DoesNotContain(lines, l => l.Contains($"\"/users/u{i}\"", StringComparison.Ordinal)); + for (int i = 16; i <= 20; i++) + Assert.Contains(lines, l => l.Contains($"\"/users/u{i}\"", StringComparison.Ordinal)); + } + finally + { + if (File.Exists(deadLetter)) File.Delete(deadLetter); + } + } + + /// + /// Three chunks: one applied, one refused, one never sent. The counts have to add up to + /// something a reader can act on - "60 succeeded, 0 failed" over a run that sent one chunk + /// is worse than no summary at all. + /// + [Fact] + public async Task Operations_that_were_never_sent_count_as_neither_succeeded_nor_failed() + { + ResiliencePipelineFactory.Reset(); + using var httpClient = new HttpClient(new FirstChunkOnlyHandler()); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client); + + var ops = Enumerable.Range(1, 60) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.Equal(60, result.Telemetry.TotalRequests); + Assert.Equal(20, result.Telemetry.Succeeded); + Assert.Equal(20, result.Telemetry.Failed); + Assert.Equal(20, result.NotSent.Count); + } + + /// + /// Both runs stop, and stopping is not what tells them apart. In the first the refusal came + /// before the batch-level retry pass, which was then skipped for every candidate. In the + /// second the pass had already POSTed twenty items and applied fifteen of them when the + /// follow-up carrying the other five was refused. The line the caller sees turns on which + /// of those happened, not on the stop they share: told the pass never went out, nobody has + /// reason to check the tenant before creating fifteen users a second time. + /// + [Fact] + public void A_retry_pass_that_went_out_is_not_reported_as_one_that_was_withheld() + { + var withheld = Say(new FirstChunkOnlyHandler(), 60, "PATCH"); + var sent = Say(new RetryPassLandsThenIsRefusedHandler(), 20, "POST", + new System.Collections.Hashtable { ["displayName"] = "u" }); + + Assert.Contains("the retry pass was withheld", withheld.Warning, StringComparison.Ordinal); + // Nothing of the pass reached the wire, so the run credits itself with none of it. + Assert.DoesNotContain("Batch-level retries", withheld.Summary, StringComparison.Ordinal); + + Assert.DoesNotContain("withheld", sent.Warning, StringComparison.Ordinal); + Assert.Contains("The retry pass was sent and then refused", sent.Warning, StringComparison.Ordinal); + Assert.Contains(" Batch-level retries: 20.", sent.Summary, StringComparison.Ordinal); + + // Neither run took its items through every attempt they had, so neither may say it did. + Assert.DoesNotContain("after all retry attempts", withheld.Warning, StringComparison.Ordinal); + Assert.DoesNotContain("after all retry attempts", sent.Warning, StringComparison.Ordinal); + } + + /// Answers every chunk in full; five items of the first come back 404. + private sealed class AnswersEveryChunkHandler : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var body = "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, 20).Select(i => + $"{{ \"id\": \"{i}\", \"status\": {(i <= 15 ? 200 : 404)} }}")) + "] }"; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") + }); + } + } + + /// + /// Nothing stopped this run: every chunk was POSTed and answered, and the items that failed + /// had whatever attempts they qualified for. The warning says so, and says nothing about + /// operations left unsent, because there were none. + /// + [Fact] + public void A_run_that_was_not_stopped_still_reports_its_failures_as_retried() + { + using (MgxTransportScope.Inject(new AnswersEveryChunkHandler())) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", Enumerable.Range(1, 20).Select(i => $"/users/u{i}").ToArray()) + .AddParameter("Method", "GET"); + ps.Invoke(); + + var warning = Assert.Single(ps.Streams.Warning, + w => w.Message.Contains("batch items failed", StringComparison.Ordinal)).Message; + Assert.Contains("5 of 20 batch items failed.", warning, StringComparison.Ordinal); + Assert.Contains("after all retry attempts", warning, StringComparison.Ordinal); + Assert.DoesNotContain("were not sent", warning, StringComparison.Ordinal); + } + } + + /// + /// The dead-letter file takes the refused and the never-sent alike, and the line that says + /// what was written has to keep them apart: one may already have been applied on the server + /// and the other certainly was not, which is the whole of what a caller re-piping the file + /// is deciding about. + /// + [Fact] + public void The_dead_letter_line_does_not_call_an_unsent_operation_a_failed_one() + { + var deadLetter = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + try + { + using (MgxTransportScope.Inject(new FirstChunkOnlyHandler())) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", Enumerable.Range(1, 60).Select(i => $"/users/u{i}").ToArray()) + .AddParameter("Method", "PATCH") + .AddParameter("DeadLetterPath", deadLetter) + .AddParameter("Verbose", true); + ps.Invoke(); + + Assert.Equal(40, File.ReadAllLines(deadLetter).Length); + Assert.Contains(ps.Streams.Verbose, v => + v.Message.Contains("Wrote 20 failed and 20 not-sent items to dead-letter file", + StringComparison.Ordinal)); + } + } + finally + { + if (File.Exists(deadLetter)) File.Delete(deadLetter); + } + } + + /// + /// Answers both items of the only chunk and leaves the "status" member off the second - the + /// truncated-or-rewritten envelope the count check names, one member short instead of one + /// sub-response short. + /// + private sealed class OmitsAStatusHandler : HttpMessageHandler + { + private int _posts; + public int Posts => Volatile.Read(ref _posts); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _posts); + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent( + "{ \"responses\": [ { \"id\": \"1\", \"status\": 204, \"body\": null }, " + + "{ \"id\": \"2\", \"body\": null } ] }", + System.Text.Encoding.UTF8, "application/json") + }); + } + } + + /// + /// The POST went out and the server answered it. Whatever the envelope failed to say about + /// the second write, the one thing it cannot be is a write that never left - which is what a + /// status deserialized to 0 says, that being NotSentStatus itself. + /// + [Fact] + public async Task An_item_answered_without_a_status_is_not_handed_back_as_never_sent() + { + ResiliencePipelineFactory.Reset(); + var wire = new OmitsAStatusHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client); + + var ops = Enumerable.Range(1, 2) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.Equal(1, wire.Posts); + Assert.NotEqual(GraphBatchClient.NotSentStatus, result.Results[1].Response.Status); + Assert.True(result.Results[1].Response.Status >= 400, + $"a write the server answered came back as {result.Results[1].Response.Status}"); + Assert.Empty(result.NotSent); + + // The item the envelope did answer keeps that answer, as it does for every other shape + // of chunk failure. + Assert.Equal(204, result.Results[0].Response.Status); + Assert.Equal(1, result.Telemetry.Succeeded); + Assert.Equal(1, result.Telemetry.Failed); + + var failure = Assert.IsType(result.ChunkFailure); + Assert.Contains("without a status", failure.Message); + } + + /// Answers the first chunk in full, then an envelope that answers nobody. + private sealed class SecondChunkGarbledHandler : HttpMessageHandler + { + private int _posts; + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var body = Interlocked.Increment(ref _posts) == 1 + ? BatchResponse(1, 20, 200) + : "{ \"responses\": [ { \"id\": \"1\", \"status\": 200 } ] }"; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") + }); + } + } + + /// + /// An envelope that does not answer what was sent is a protocol violation, and it is + /// reported as one. It is not a reason to discard the chunk the server did answer. + /// + [Fact] + public async Task A_chunk_answered_with_a_garbled_envelope_keeps_the_chunk_before_it() + { + ResiliencePipelineFactory.Reset(); + using var httpClient = new HttpClient(new SecondChunkGarbledHandler()); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client); + + var ops = Enumerable.Range(1, 40) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + var failure = Assert.IsType(result.ChunkFailure); + Assert.Contains("count mismatch", failure.Message); + for (int i = 0; i < 20; i++) + Assert.Equal(200, result.Results[i].Response.Status); + for (int i = 20; i < 40; i++) + Assert.Equal(503, result.Results[i].Response.Status); + } + + /// + /// Answers the first chunk, throttles the second through every attempt it has, and then + /// fails the follow-up batch that carries the throttled items - the retry pass, which runs + /// once every chunk has been sent. + /// + private sealed class RefusesTheRetryPassHandler(bool garbled) : HttpMessageHandler + { + private int _posts; + + private static string Throttled(int count) => + "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, count).Select(i => + $"{{ \"id\": \"{i}\", \"status\": 429, \"headers\": {{ \"Retry-After\": \"1\" }} }}")) + "] }"; + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var post = Interlocked.Increment(ref _posts); + var body = await request.Content!.ReadAsStringAsync(cancellationToken); + if (body.Contains("/users/u1\"", StringComparison.Ordinal)) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(BatchResponse(1, 20, 204), + System.Text.Encoding.UTF8, "application/json") + }; + } + // The second chunk's own four attempts; the fifth POST of its items is the + // follow-up batch. + if (post <= 5) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(Throttled(20), + System.Text.Encoding.UTF8, "application/json") + }; + } + // Two shapes of failure: a refusal with a status, and an envelope that answers + // one of the twenty requests it was sent. + return garbled + ? new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent("{ \"responses\": [ { \"id\": \"1\", \"status\": 204 } ] }", + System.Text.Encoding.UTF8, "application/json") + } + : new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent( + "{\"error\":{\"code\":\"BadRequest\",\"message\":\"batch rejected\"}}", + System.Text.Encoding.UTF8, "application/json") + }; + } + } + + /// + /// The retry pass runs after every chunk has been sent, so by the time it fails the run + /// holds the outcome of the whole batch. Throwing hands back the failure and nothing else: + /// twenty writes the server confirmed, and twenty it throttled, are lost with the array. + /// + [Fact] + public async Task A_refused_retry_pass_keeps_what_the_run_already_knew() + { + ResiliencePipelineFactory.Reset(); + using var httpClient = new HttpClient(new RefusesTheRetryPassHandler(garbled: false)); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client); + + var ops = Enumerable.Range(1, 40) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.IsType(result.ChunkFailure); + Assert.Equal(40, result.Results.Count); + // The first chunk was applied. The retried items keep the last thing the server said + // about them, which is what they are: throttled, not retried, still to be re-submitted. + for (int i = 0; i < 20; i++) + Assert.Equal(204, result.Results[i].Response.Status); + for (int i = 20; i < 40; i++) + Assert.Equal(429, result.Results[i].Response.Status); + Assert.Equal(20, result.Telemetry.Succeeded); + Assert.Equal(20, result.Telemetry.Failed); + Assert.Empty(result.NotSent); + } + + /// + /// The other shape the retry pass fails in. A truncated envelope is an InvalidOperationException, + /// which the cmdlet does not catch, so the run's results left EndProcessing unreported: no + /// output objects, no error records, nothing on the pipeline but the exception. + /// + [Fact] + public void A_garbled_retry_pass_still_reports_the_run() + { + using (MgxTransportScope.Inject(new RefusesTheRetryPassHandler(garbled: true))) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", Enumerable.Range(1, 40).Select(i => $"/users/u{i}").ToArray()) + .AddParameter("Method", "PATCH"); + var output = ps.Invoke(); + + Assert.Equal(40, output.Count); + Assert.Contains(ps.Streams.Error, + e => e.FullyQualifiedErrorId.StartsWith("BatchChunkFailed", StringComparison.Ordinal)); + } + } + + /// + /// Answers the $batch POST with a status Graph never sends. A proxy or a captive portal + /// between the client and Graph does, and the chunk is refused either way. + /// + private sealed class NotModifiedHandler : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + => Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotModified) + { + RequestMessage = request, + Content = new StringContent("not modified", + System.Text.Encoding.UTF8, "text/html") + }); + } + + /// + /// A refusal below 400 is still a refusal. Recorded verbatim it put every item of the + /// chunk in the range a success is counted in, so twenty writes that never reached Graph + /// were reported as twenty that had. + /// + [Fact] + public async Task A_chunk_refused_with_a_3xx_gives_its_items_a_failing_status() + { + ResiliencePipelineFactory.Reset(); + using var httpClient = new HttpClient(new NotModifiedHandler()); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client); + + var ops = Enumerable.Range(1, 20) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.NotNull(result.ChunkFailure); + Assert.All(result.Results, r => Assert.True(r.Response.Status >= 400, + $"a refused write was recorded as status {r.Response.Status}")); + Assert.Equal(0, result.Telemetry.Succeeded); + Assert.Equal(20, result.Telemetry.Failed); + Assert.Empty(result.NotSent); + } + + /// + /// The dead-letter file keeps nothing under 400, so the same 3xx left the run with no + /// record of the twenty writes to re-submit. + /// + [Fact] + public void Items_refused_with_a_3xx_are_written_to_the_dead_letter_file() + { + var deadLetter = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + try + { + using (MgxTransportScope.Inject(new NotModifiedHandler())) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", Enumerable.Range(1, 20).Select(i => $"/users/u{i}").ToArray()) + .AddParameter("Method", "PATCH") + .AddParameter("DeadLetterPath", deadLetter); + ps.Invoke(); + } + + var lines = File.Exists(deadLetter) ? File.ReadAllLines(deadLetter) : []; + Assert.Equal(20, lines.Length); + } + finally + { + if (File.Exists(deadLetter)) File.Delete(deadLetter); + } + } + /// Counts requests; answers an empty batch envelope. private sealed class CountingHandler : HttpMessageHandler { @@ -97,23 +660,8 @@ protected override Task SendAsync( [Fact] public void WhatIf_does_not_send_a_read_only_batch() { - ResiliencePipelineFactory.Reset(); var wire = new CountingHandler(); - var t = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); - t.GetField("s_graphHttpClient", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)! - .SetValue(null, new HttpClient(wire)); - t.GetField("s_cachedAuthFingerprint", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)! - .SetValue(null, Mgx.Cmdlets.Base.MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - t.GetField("s_ownsHttpClient", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)! - .SetValue(null, false); - t.GetField("s_graphEndpoint", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.Public)! - .SetValue(null, "https://graph.microsoft.com"); - t.GetField("s_clientOptions", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.Public)! - .SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); - - try + using (MgxTransportScope.Inject(wire)) { using var ps = System.Management.Automation.PowerShell.Create(); ps.AddCommand("Import-Module") @@ -133,11 +681,503 @@ public void WhatIf_does_not_send_a_read_only_batch() Assert.Equal(0, wire.Requests); Assert.Empty(output); } + } + + /// + /// Throttles a whole chunk until its per-item retries are gone, so every item goes into the + /// batch-level retry pass. That pass's first attempt applies most of them and throttles the + /// rest; the follow-up carrying the rest is refused. The items the pass applied are writes + /// the server confirmed, and the only place they are recorded is the answer it gave. + /// + private sealed class RetryPassLandsThenIsRefusedHandler : HttpMessageHandler + { + private int _posts; + public int Posts => Volatile.Read(ref _posts); + + private static string Throttled(int count) => + "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, count).Select(i => + $"{{ \"id\": \"{i}\", \"status\": 429, \"headers\": {{ \"Retry-After\": \"1\" }} }}")) + "] }"; + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var post = Interlocked.Increment(ref _posts); + // Four posts exhaust the chunk's own retries, leaving all 20 items throttled and + // eligible for the batch-level pass. + if (post <= 4) + return Task.FromResult(Ok(Throttled(20))); + // The pass's own first attempt: fifteen creates applied, five still throttled. + if (post == 5) + { + var body = "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, 20).Select(i => + i <= 15 + ? $"{{ \"id\": \"{i}\", \"status\": 201, \"body\": {{ \"id\": \"u{i}\" }} }}" + : $"{{ \"id\": \"{i}\", \"status\": 429, \"headers\": {{ \"Retry-After\": \"1\" }} }}")) + + "] }"; + return Task.FromResult(Ok(body)); + } + // The follow-up carrying the five is refused outright. + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent( + "{\"error\":{\"code\":\"BadRequest\",\"message\":\"batch rejected\"}}", + System.Text.Encoding.UTF8, "application/json") + }); + + HttpResponseMessage Ok(string body) => new(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") + }; + } + } + + /// + /// The batch-level retry pass is a send like any other, and the server answers it. When a + /// later attempt of that pass is refused, the items the earlier attempt applied still have + /// to come back with the status the server gave them: reported with the throttle that sent + /// them into the pass, fifteen confirmed creates read as work still outstanding, and the + /// dead-letter file the caller replays creates them a second time. + /// + [Fact] + public async Task Items_the_retry_pass_applied_keep_that_answer_when_its_follow_up_is_refused() + { + ResiliencePipelineFactory.Reset(); + var wire = new RetryPassLandsThenIsRefusedHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client); + + var ops = Enumerable.Range(1, 20) + .Select(i => new BatchOperation("/users", "POST", + System.Text.Json.JsonSerializer.Deserialize( + $"{{\"displayName\":\"u{i}\"}}"))) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.Equal(6, wire.Posts); + Assert.NotNull(result.ChunkFailure); + // The fifteen the pass created. The server said 201; nothing after that unsays it. + for (int i = 0; i < 15; i++) + Assert.Equal(201, result.Results[i].Response.Status); + // The five the refused follow-up carried keep the last thing the server said about + // them - a throttle, which is a failure, and never a status below 400. + for (int i = 15; i < 20; i++) + Assert.Equal(429, result.Results[i].Response.Status); + Assert.Empty(result.NotSent); + Assert.Equal(15, result.Telemetry.Succeeded); + Assert.Equal(5, result.Telemetry.Failed); + } + + // ------------------------------------------------------------------------------------ + // What a run says about itself, transcribed per scenario. + // + // Four lines carry it: the dead-letter line, the chunk-failure target, the verbose summary + // and the warning - the last being all a caller running without -Verbose sees. They are + // read together by whoever is deciding what to re-submit, so each one being right on its + // own is not enough; they have to agree as a set. + // ------------------------------------------------------------------------------------ + + /// The four lines of one run, as the streams carried them. + private sealed record Prose( + string Summary, + string? Warning, + string? ChunkFailureTarget, + string? DeadLetterLine, + int DeadLetterLines, + IReadOnlyList Verbose); + + /// + /// Drives the cmdlet over and collects the four lines. Always with a + /// dead-letter path and -Verbose, so every scenario is asked the same question. + /// + private static Prose Say(HttpMessageHandler wire, int count, string method, object? body = null) + { + var deadLetter = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + try + { + using (MgxTransportScope.Inject(wire)) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + var cmd = ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", Enumerable.Range(1, count).Select(i => $"/users/u{i}").ToArray()) + .AddParameter("Method", method) + .AddParameter("DeadLetterPath", deadLetter) + .AddParameter("Verbose", true); + if (body != null) cmd.AddParameter("Body", body); + ps.Invoke(); + + var verbose = ps.Streams.Verbose.Select(v => v.Message).ToList(); + return new Prose( + Summary: Assert.Single(verbose, v => v.StartsWith("Batch: ", StringComparison.Ordinal)), + Warning: ps.Streams.Warning.Select(w => w.Message) + .FirstOrDefault(w => w.Contains("batch items failed", StringComparison.Ordinal)), + ChunkFailureTarget: ps.Streams.Error + .Where(e => e.FullyQualifiedErrorId.StartsWith("BatchChunkFailed", StringComparison.Ordinal)) + .Select(e => e.TargetObject as string) + .FirstOrDefault(), + DeadLetterLine: verbose.FirstOrDefault(v => v.StartsWith("Wrote ", StringComparison.Ordinal)), + DeadLetterLines: File.Exists(deadLetter) ? File.ReadAllLines(deadLetter).Length : 0, + Verbose: verbose); + } + } finally { - t.GetField("s_graphHttpClient", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthFingerprint", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!.SetValue(null, null); - ResiliencePipelineFactory.Reset(); + if (File.Exists(deadLetter)) File.Delete(deadLetter); } } + + /// Answers every chunk in full, every item applied. + private sealed class AppliesEveryItemHandler : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(BatchResponse(1, 20, 204), + System.Text.Encoding.UTF8, "application/json") + }); + } + + /// + /// Nothing failed, so three of the four lines have nothing to say and do not say it. The + /// warning in particular: a run that worked must not put one on the stream at all. + /// + [Fact] + public void A_clean_run_says_so_and_says_nothing_else() + { + var prose = Say(new AppliesEveryItemHandler(), 20, "PATCH"); + + Assert.Contains("Batch: 20 succeeded, 0 failed out of 20 requests", prose.Summary, StringComparison.Ordinal); + Assert.DoesNotContain("not sent", prose.Summary, StringComparison.Ordinal); + Assert.DoesNotContain("Batch-level retries", prose.Summary, StringComparison.Ordinal); + Assert.Null(prose.Warning); + Assert.Null(prose.ChunkFailureTarget); + Assert.Null(prose.DeadLetterLine); + Assert.Equal(0, prose.DeadLetterLines); + } + + /// + /// The refused chunk is the last one, so nothing was left unsent and no line mentions + /// anything that was. The retry pass genuinely never went out: the run stopped before it. + /// + [Fact] + public void A_chunk_refused_with_nothing_behind_it_reports_no_unsent_work() + { + var prose = Say(new FirstChunkOnlyHandler(), 25, "PATCH"); + + Assert.Contains("Batch: 20 succeeded, 5 failed out of 25 requests", prose.Summary, StringComparison.Ordinal); + Assert.DoesNotContain("not sent", prose.Summary, StringComparison.Ordinal); + Assert.DoesNotContain("Batch-level retries", prose.Summary, StringComparison.Ordinal); + Assert.Equal( + "5 of 25 batch items failed." + + " A chunk failed, so the run stopped sending and the retry pass was withheld." + + " Check $Error for details on each item.", + prose.Warning); + Assert.Equal("5 of 25 operations failed, 0 were not sent", prose.ChunkFailureTarget); + Assert.StartsWith("Wrote 5 failed items to dead-letter file: ", prose.DeadLetterLine, StringComparison.Ordinal); + Assert.Equal(5, prose.DeadLetterLines); + } + + /// + /// A chunk behind the refused one never went out, and all four lines have to keep it apart + /// from the writes that were refused: one of those may have been applied on the server and + /// the other certainly was not. + /// + [Fact] + public void Operations_never_sent_are_named_by_every_line_that_counts_them() + { + var prose = Say(new FirstChunkOnlyHandler(), 60, "PATCH"); + + Assert.Contains("Batch: 20 succeeded, 20 failed, 20 not sent out of 60 requests", prose.Summary, StringComparison.Ordinal); + Assert.Equal( + "20 of 60 batch items failed and 20 were not sent." + + " A chunk failed, so the run stopped sending and the retry pass was withheld." + + " Check $Error for details on each item.", + prose.Warning); + Assert.Equal("20 of 60 operations failed, 20 were not sent", prose.ChunkFailureTarget); + Assert.StartsWith("Wrote 20 failed and 20 not-sent items to dead-letter file: ", + prose.DeadLetterLine, StringComparison.Ordinal); + Assert.Equal(40, prose.DeadLetterLines); + } + + /// + /// Throttles the chunk through every attempt it has, then answers the batch-level pass in + /// full: fifteen applied, five not found. Nothing stopped the run. + /// + private sealed class ThrottlesThenAnswersThePassHandler : HttpMessageHandler + { + private int _posts; + public int Posts => Volatile.Read(ref _posts); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var post = Interlocked.Increment(ref _posts); + var body = post <= 4 + ? "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, 20).Select(i => + $"{{ \"id\": \"{i}\", \"status\": 429, \"headers\": {{ \"Retry-After\": \"1\" }} }}")) + "] }" + : "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, 20).Select(i => + $"{{ \"id\": \"{i}\", \"status\": {(i <= 15 ? 204 : 404)} }}")) + "] }"; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") + }); + } + } + + /// + /// The pass went out and the server answered it. Nothing stopped the run, so the five that + /// failed had every attempt they qualified for and the warning says so. + /// + [Fact] + public void A_retry_pass_that_ran_to_the_end_leaves_the_run_unstopped() + { + var wire = new ThrottlesThenAnswersThePassHandler(); + var prose = Say(wire, 20, "PATCH"); + + Assert.Equal(5, wire.Posts); + Assert.Contains("Batch: 15 succeeded, 5 failed out of 20 requests", prose.Summary, StringComparison.Ordinal); + Assert.Contains(" Batch-level retries: 20.", prose.Summary, StringComparison.Ordinal); + Assert.Equal( + "5 of 20 batch items failed." + + " They failed after all retry attempts." + + " Check $Error for details on each item.", + prose.Warning); + Assert.Null(prose.ChunkFailureTarget); + Assert.StartsWith("Wrote 5 failed items to dead-letter file: ", prose.DeadLetterLine, StringComparison.Ordinal); + Assert.Equal(5, prose.DeadLetterLines); + } + + /// + /// The pass carried twenty items, applied fifteen of them, and was refused on the follow-up + /// carrying the other five. Twenty went out on the wire, which is what the count reports. + /// + [Fact] + public void A_retry_pass_refused_after_it_applied_work_still_counts_what_it_sent() + { + var wire = new RetryPassLandsThenIsRefusedHandler(); + var prose = Say(wire, 20, "POST", new System.Collections.Hashtable { ["displayName"] = "u" }); + + Assert.Equal(6, wire.Posts); + Assert.Contains("Batch: 15 succeeded, 5 failed out of 20 requests", prose.Summary, StringComparison.Ordinal); + Assert.Contains(" Batch-level retries: 20.", prose.Summary, StringComparison.Ordinal); + Assert.Equal( + "5 of 20 batch items failed." + + " The retry pass was sent and then refused, so the run stopped sending." + + " Check $Error for details on each item.", + prose.Warning); + Assert.Equal("5 of 20 operations failed, 0 were not sent", prose.ChunkFailureTarget); + Assert.StartsWith("Wrote 5 failed items to dead-letter file: ", prose.DeadLetterLine, StringComparison.Ordinal); + Assert.Equal(5, prose.DeadLetterLines); + } + + /// + /// Throttles both chunks through every attempt they have, so all forty items become + /// candidates for the batch-level pass - two chunks of it - and refuses the first chunk + /// that pass sends. The loop breaks there, so the second chunk of the pass never goes out. + /// + private sealed class RefusesTheFirstChunkOfThePassHandler : HttpMessageHandler + { + private int _posts; + public int Posts => Volatile.Read(ref _posts); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + // Four attempts each for two chunks exhausts the per-chunk retries of both. + if (Interlocked.Increment(ref _posts) <= 8) + { + var body = "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, 20).Select(i => + $"{{ \"id\": \"{i}\", \"status\": 429, \"headers\": {{ \"Retry-After\": \"1\" }} }}")) + "] }"; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") + }); + } + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent( + "{\"error\":{\"code\":\"BadRequest\",\"message\":\"batch rejected\"}}", + System.Text.Encoding.UTF8, "application/json") + }); + } + } + + /// + /// Forty items were candidates for the pass and twenty of them were POSTed: the refusal of + /// the first chunk ends the pass, and the twenty behind it are never re-sent. Candidacy is + /// its own line and says forty; what went out is the count, and it is not the same number. + /// + [Fact] + public void A_retry_pass_refused_on_its_first_chunk_never_sends_the_second() + { + var wire = new RefusesTheFirstChunkOfThePassHandler(); + var prose = Say(wire, 40, "PATCH"); + + // Eight per-chunk attempts and one POST of the pass. The pass's second chunk is the + // POST that would make this nine, and it never happened. + Assert.Equal(9, wire.Posts); + Assert.Contains("Batch: 0 succeeded, 40 failed out of 40 requests", prose.Summary, StringComparison.Ordinal); + // Twenty, not forty: the second chunk of the pass is the POST that never happened. + Assert.Contains(" Batch-level retries: 20.", prose.Summary, StringComparison.Ordinal); + Assert.Equal( + "40 of 40 batch items failed." + + " The retry pass was sent and then refused, so the run stopped sending." + + " Check $Error for details on each item.", + prose.Warning); + Assert.Equal("40 of 40 operations failed, 0 were not sent", prose.ChunkFailureTarget); + Assert.StartsWith("Wrote 40 failed items to dead-letter file: ", prose.DeadLetterLine, StringComparison.Ordinal); + Assert.Equal(40, prose.DeadLetterLines); + // Candidacy is stated separately and is not in question: forty items did exhaust their + // per-chunk retries, whatever the pass then managed to send. + Assert.Contains(prose.Verbose, v => + v.Contains("Batch-level retry: 40 items exhausted per-chunk retries", StringComparison.Ordinal)); + } + + /// The lines -WhatIf writes go to the host, so a test that reads them needs one. + private sealed class GateHost : PSHost + { + private readonly Guid _id = Guid.NewGuid(); + public GateHostUI Recorder { get; } = new(); + public override string Name => "MgxBatchGateHost"; + public override Version Version => new(1, 0); + public override Guid InstanceId => _id; + public override PSHostUserInterface UI => Recorder; + public override System.Globalization.CultureInfo CurrentCulture => System.Globalization.CultureInfo.InvariantCulture; + public override System.Globalization.CultureInfo CurrentUICulture => System.Globalization.CultureInfo.InvariantCulture; + public override void EnterNestedPrompt() { } + public override void ExitNestedPrompt() { } + public override void NotifyBeginApplication() { } + public override void NotifyEndApplication() { } + public override void SetShouldExit(int exitCode) { } + } + + private sealed class GateHostUI : PSHostUserInterface + { + public List Lines { get; } = []; + public override PSHostRawUserInterface? RawUI => null; + public override void Write(string value) => Lines.Add(value); + public override void Write(ConsoleColor f, ConsoleColor b, string value) => Lines.Add(value); + public override void WriteLine(string value) => Lines.Add(value); + public override void WriteErrorLine(string value) => Lines.Add(value); + public override void WriteDebugLine(string value) => Lines.Add(value); + public override void WriteVerboseLine(string value) => Lines.Add(value); + public override void WriteWarningLine(string value) => Lines.Add(value); + public override void WriteProgress(long sourceId, ProgressRecord record) { } + public override string ReadLine() => string.Empty; + public override System.Security.SecureString ReadLineAsSecureString() => new(); + public override Dictionary Prompt( + string caption, string message, System.Collections.ObjectModel.Collection descriptions) => []; + public override int PromptForChoice( + string caption, string message, + System.Collections.ObjectModel.Collection choices, int defaultChoice) => defaultChoice; + public override PSCredential PromptForCredential( + string caption, string message, string userName, string targetName) => PSCredential.Empty; + public override PSCredential PromptForCredential( + string caption, string message, string userName, string targetName, + PSCredentialTypes allowedCredentialTypes, PSCredentialUIOptions options) => PSCredential.Empty; + } + + /// + /// Runs the batch under -WhatIf over per-item methods and returns the line the gate wrote. + /// The wire is armed and counted: a gate that lets the batch through is a different failure + /// from one that describes it wrongly, and the two have to be told apart. + /// + private static (string Line, int Requests) GateSays(params string[] methods) + { + var wire = new CountingHandler(); + using (MgxTransportScope.Inject(wire)) + { + var host = new GateHost(); + using var runspace = System.Management.Automation.Runspaces.RunspaceFactory.CreateRunspace(host); + runspace.Open(); + using var ps = System.Management.Automation.PowerShell.Create(); + ps.Runspace = runspace; + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", methods + .Select((m, i) => (object)new System.Collections.Hashtable + { + ["Url"] = $"/users/u{i + 1}", + ["Method"] = m + }) + .ToArray()) + .AddParameter("WhatIf", true); + ps.Invoke(); + + return (Assert.Single(host.Recorder.Lines, l => l.Contains("What if:", StringComparison.Ordinal)), + wire.Requests); + } + } + + /// + /// -WhatIf is the one surface whose whole purpose is stating what is about to happen, and + /// the target it states is the batch's own account of itself. The total is deliberately the + /// whole batch, reads included - they spend resource units and emit output - but a write + /// verb across that total names requests the batch does not hold: one DELETE among nineteen + /// reads announced twenty deletions, and five creates among fifteen reads announced twenty + /// creates, byte-identical to a batch of twenty that really was all creates. + /// + [Theory] + // A single write among reads: the verb is the alarming part, and it belonged to one item. + [InlineData(1, "DELETE", 19, "Send batch\" on target \"20 requests (1 DELETE) via $batch\"")] + [InlineData(5, "POST", 15, "Send batch\" on target \"20 requests (5 POST) via $batch\"")] + // No reads to distinguish: the whole batch is that one method, and the verb is the total. + [InlineData(20, "POST", 0, "Send batch\" on target \"POST 20 requests via $batch\"")] + [InlineData(0, "POST", 20, "Send batch\" on target \"GET 20 requests via $batch\"")] + public void The_gate_names_the_writes_a_batch_holds_not_the_batch_as_writes( + int writes, string writeMethod, int reads, string expected) + { + var methods = Enumerable.Repeat(writeMethod, writes) + .Concat(Enumerable.Repeat("GET", reads)) + .ToArray(); + + var (line, requests) = GateSays(methods); + + Assert.Contains(expected, line, StringComparison.Ordinal); + Assert.Equal(0, requests); + } + + /// + /// Several write methods alongside reads keep the shape they already had: the total, then + /// the writes by method. It states a total and a subset of it, and nothing in it is a + /// count of requests that are not there. + /// + [Fact] + public void The_gate_keeps_naming_each_write_method_of_a_mixed_batch() + { + var methods = Enumerable.Repeat("POST", 5) + .Concat(Enumerable.Repeat("PATCH", 3)) + .Concat(Enumerable.Repeat("GET", 12)) + .ToArray(); + + var (line, requests) = GateSays(methods); + + Assert.Contains("Send batch\" on target \"20 requests (5 POST, 3 PATCH) via $batch\"", + line, StringComparison.Ordinal); + Assert.Equal(0, requests); + } } diff --git a/tests/Mgx.IntegrationTests/Engine/Http/AdaptiveRequestPacerTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/AdaptiveRequestPacerTests.cs index 50da15d..6cc56a7 100644 --- a/tests/Mgx.IntegrationTests/Engine/Http/AdaptiveRequestPacerTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Http/AdaptiveRequestPacerTests.cs @@ -1,4 +1,5 @@ using System.Diagnostics; +using System.Globalization; using System.Net; using Mgx.Engine.Http; @@ -13,6 +14,7 @@ namespace Mgx.IntegrationTests.Engine; /// Stateful tests run inside PacerScope (which re-enables the gate the suite-wide /// ModuleInitializer turned off) and in the Pipeline collection so nothing runs concurrently /// with them against the process-static state. +/// (Corpus: M365DSC-7273, sustained throttling.) /// [Collection("Pipeline")] public class AdaptiveRequestPacerTests @@ -185,6 +187,34 @@ public void RecordResponse_captures_the_throttle_percentage_gauge() Assert.Contains("proximity 95%", state); } + [Fact] + public void DescribeState_renders_the_latency_ratio_with_an_invariant_separator() + { + // Get-MgxTelemetry's PacingState is parsed by the benchmark harness, so a comma-decimal + // locale must not turn "1.8x" into "1,8x". + using var scope = new PacerScope(); + + var original = CultureInfo.CurrentCulture; + try + { + CultureInfo.CurrentCulture = new CultureInfo("de-DE"); + + // The baseline is a slow EMA, so a second, slower sample leaves a fractional ratio: + // baseline 100ms -> 112ms against a last latency of 200ms. + AdaptiveRequestPacer.RecordLatency(WorkloadBucket.Directory, 100); + AdaptiveRequestPacer.RecordLatency(WorkloadBucket.Directory, 200); + + var state = AdaptiveRequestPacer.DescribeState(); + Assert.NotNull(state); + Assert.Contains("1.8x of 112ms baseline", state); + Assert.DoesNotContain(",8x", state); + } + finally + { + CultureInfo.CurrentCulture = original; + } + } + [Fact] public async Task Proximity_gauge_paces_requests_beyond_what_the_rate_cap_alone_would() { diff --git a/tests/Mgx.IntegrationTests/Engine/Http/GraphContentClientTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/GraphContentClientTests.cs index 55b1c4e..01ec286 100644 --- a/tests/Mgx.IntegrationTests/Engine/Http/GraphContentClientTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Http/GraphContentClientTests.cs @@ -2,7 +2,6 @@ using System.Management.Automation; using System.Net; using System.Net.Http.Headers; -using System.Reflection; using System.Text; using Mgx.Cmdlets.Base; using Mgx.Engine.Http; @@ -175,40 +174,12 @@ public async Task A_transport_that_auto_followed_the_redirect_is_refused() private const string TestTenantId = "test-tenant-00000000-0000-0000-0000-000000000000"; /// - /// Points MgxCmdletBase's cached transport at this class's Graph mock and declares whether - /// mgx owns it (false = the Graph SDK's client, which ships a RedirectHandler). Reflection - /// because the statics are private; the fingerprint has to match what the test shell's - /// stubbed Get-MgContext produces, or GetClient() sees a credential change and rebuilds. + /// Points MgxCmdletBase's transport at this class's Graph mock and declares whether mgx owns + /// it (false = the Graph SDK's client, which ships a RedirectHandler). The mock client is + /// this class's own, so the scope borrows it rather than disposing it at the end of a test. /// - private void InjectCmdletTransport(bool owned, HttpClient? client = null) - { - ResiliencePipelineFactory.Reset(); - var options = new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }; - SetCmdletStatic("s_graphHttpClient", client ?? _graphHttpClient); - SetCmdletStatic("s_cachedAuthFingerprint", - MgxCmdletBase.BuildAuthFingerprint(new { TenantId = TestTenantId }, null)); - SetCmdletStatic("s_cachedAuthContextRef", null); - SetCmdletStatic("s_ownsHttpClient", owned); - SetCmdletStatic("s_graphEndpoint", "https://graph.microsoft.com"); - SetCmdletStatic("s_clientOptions", options); - SetCmdletStatic("s_cachedTotalTimeoutSeconds", options.TotalTimeoutSeconds); - } - - private static void ResetCmdletTransport() - { - SetCmdletStatic("s_graphHttpClient", null); - SetCmdletStatic("s_cachedAuthFingerprint", null); - SetCmdletStatic("s_cachedAuthContextRef", null); - SetCmdletStatic("s_ownsHttpClient", false); - SetCmdletStatic("s_cachedTotalTimeoutSeconds", 0); - SetCmdletStatic("s_clientOptions", ResilientGraphClientOptions.Default); - ResiliencePipelineFactory.Reset(); - } - - private static void SetCmdletStatic(string field, object? value) => - typeof(MgxCmdletBase) - .GetField(field, BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Static)! - .SetValue(null, value); + private MgxTransportScope InjectCmdletTransport(bool owned, HttpClient? client = null) => + MgxTransportScope.Inject(client ?? _graphHttpClient, owned); private static PowerShell CreateContentShell() { @@ -241,37 +212,30 @@ public void GetMgxContent_refuses_to_fetch_over_a_transport_mgx_does_not_own() // validated - the exact leak the two-hop design exists to prevent. _graphHandler.SetDefaultResponse(HttpStatusCode.OK, "bytes-that-must-not-be-fetched"); - try + // 1. Borrowed SDK transport: refuse, and refuse before the request is sent. + using (InjectCmdletTransport(owned: false)) + using (var ps = CreateContentShell()) { - // 1. Borrowed SDK transport: refuse, and refuse before the request is sent. - InjectCmdletTransport(owned: false); - using (var ps = CreateContentShell()) - { - ps.AddCommand("Get-MgxContent").AddParameter("Uri", contentUri); - var thrown = Record.Exception(() => ps.Invoke()); + ps.AddCommand("Get-MgxContent").AddParameter("Uri", contentUri); + var thrown = Record.Exception(() => ps.Invoke()); - Assert.Contains(ErrorIds(ps, thrown), - id => id.Contains("ContentRequiresOwnedTransport", StringComparison.Ordinal)); - Assert.Equal(0, _graphHandler.RequestCount); - } - - // 2. The identical call on the mgx-owned transport has to go through, or step 1 - // proves nothing: a cmdlet that always refuses would pass it too. - InjectCmdletTransport(owned: true); - using (var ps = CreateContentShell()) - { - ps.AddCommand("Get-MgxContent").AddParameter("Uri", contentUri); - var results = ps.Invoke(); - - Assert.Empty(ps.Streams.Error); - var bytes = Assert.IsType(Assert.Single(results).BaseObject); - Assert.Equal("bytes-that-must-not-be-fetched", Encoding.UTF8.GetString(bytes)); - Assert.Equal(1, _graphHandler.RequestCount); - } + Assert.Contains(ErrorIds(ps, thrown), + id => id.Contains("ContentRequiresOwnedTransport", StringComparison.Ordinal)); + Assert.Equal(0, _graphHandler.RequestCount); } - finally + + // 2. The identical call on the mgx-owned transport has to go through, or step 1 + // proves nothing: a cmdlet that always refuses would pass it too. + using (InjectCmdletTransport(owned: true)) + using (var ps = CreateContentShell()) { - ResetCmdletTransport(); + ps.AddCommand("Get-MgxContent").AddParameter("Uri", contentUri); + var results = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + var bytes = Assert.IsType(Assert.Single(results).BaseObject); + Assert.Equal("bytes-that-must-not-be-fetched", Encoding.UTF8.GetString(bytes)); + Assert.Equal(1, _graphHandler.RequestCount); } } @@ -348,19 +312,14 @@ public override void Flush() { } { var handler = new SizedBodyHandler(declaredLength, actualBytes); using var http = new HttpClient(handler); - try + using (InjectCmdletTransport(owned: true, client: http)) { - InjectCmdletTransport(owned: true, client: http); using var ps = CreateContentShell(); ps.AddCommand("Get-MgxContent").AddParameter("Uri", "/me/drive/items/01ABC/content"); Collection output = []; var thrown = Record.Exception(() => output = ps.Invoke()); return (output, [.. ErrorIds(ps, thrown)], handler.BytesServed); } - finally - { - ResetCmdletTransport(); - } } [Fact] @@ -444,9 +403,8 @@ protected override Task SendAsync( { var handler = new RangeHonouringHandler(body); using var http = new HttpClient(handler); - try + using (InjectCmdletTransport(owned: true, client: http)) { - InjectCmdletTransport(owned: true, client: http); using var ps = CreateContentShell(); ps.AddCommand("Get-MgxContent").AddParameter("Uri", "/me/drive/items/01ABC/content"); foreach (var (name, value) in rangeParameters) @@ -457,10 +415,6 @@ protected override Task SendAsync( Assert.Empty(ps.Streams.Error); return (Assert.IsType(Assert.Single(output).BaseObject), handler.RangeHeader); } - finally - { - ResetCmdletTransport(); - } } [Fact] @@ -518,9 +472,8 @@ protected override Task SendAsync( { var handler = new RangeIgnoringHandler(body); using var http = new HttpClient(handler); - try + using (InjectCmdletTransport(owned: true, client: http)) { - InjectCmdletTransport(owned: true, client: http); using var ps = CreateContentShell(); ps.AddCommand("Get-MgxContent").AddParameter("Uri", "/me/drive/items/01ABC/content"); foreach (var (name, value) in rangeParameters) @@ -531,10 +484,6 @@ protected override Task SendAsync( Assert.Empty(ps.Streams.Error); return (Assert.IsType(Assert.Single(output).BaseObject), handler.RangeHeader); } - finally - { - ResetCmdletTransport(); - } } [Fact] @@ -562,9 +511,9 @@ public void GetMgxContent_honours_Offset_locally_when_the_server_ignores_the_ran var outFile = Path.Combine(Path.GetTempPath(), $"mgx-offset-{Guid.NewGuid():N}.bin"); var handler = new RangeIgnoringHandler(body); using var http = new HttpClient(handler); + using var transport = InjectCmdletTransport(owned: true, client: http); try { - InjectCmdletTransport(owned: true, client: http); using var ps = CreateContentShell(); ps.AddCommand("Get-MgxContent") .AddParameter("Uri", "/me/drive/items/01ABC/content") @@ -578,7 +527,6 @@ public void GetMgxContent_honours_Offset_locally_when_the_server_ignores_the_ran } finally { - ResetCmdletTransport(); if (File.Exists(outFile)) File.Delete(outFile); } } diff --git a/tests/Mgx.IntegrationTests/Engine/Http/HeaderCultureTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/HeaderCultureTests.cs new file mode 100644 index 0000000..bb2aa25 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/HeaderCultureTests.cs @@ -0,0 +1,92 @@ +using System.Globalization; +using System.Management.Automation; +using System.Net; +using System.Reflection; +using Mgx.Cmdlets.Base; + +namespace Mgx.IntegrationTests; + +/// +/// A -Headers value that is not a string is rendered by mgx before it goes on the wire, and +/// object.ToString() renders under the thread's culture: the same script sent "X-Ratio: 0,5" +/// on a de-DE machine and "X-Ratio: 0.5" on an en-US one. +/// +[Collection("Pipeline")] +public class HeaderCultureTests +{ + private static readonly MethodInfo Build = + typeof(MgxCmdletBase).GetMethod("BuildRequestHeaders", + BindingFlags.Static | BindingFlags.NonPublic)!; + + private static Dictionary Headers(System.Collections.Hashtable values) => + (Dictionary)Build.Invoke(null, [null, values])!; + + private static void InCulture(string name, Action body) + { + var previous = CultureInfo.CurrentCulture; + CultureInfo.CurrentCulture = CultureInfo.GetCultureInfo(name); + try { body(); } + finally { CultureInfo.CurrentCulture = previous; } + } + + [Theory] + [InlineData("de-DE")] + [InlineData("en-US")] + [InlineData("fr-FR")] + public void A_non_string_header_value_renders_the_same_in_every_culture(string culture) + { + InCulture(culture, () => + { + var headers = Headers(new System.Collections.Hashtable + { + ["X-Ratio"] = 0.5d, + ["X-Decimal"] = 1.7m, + ["X-Count"] = 100, + ["X-List"] = new object[] { 0.5d, 100 } + }); + + Assert.Equal("0.5", headers["X-Ratio"]); + Assert.Equal("1.7", headers["X-Decimal"]); + Assert.Equal("100", headers["X-Count"]); + Assert.Equal("0.5, 100", headers["X-List"]); + }); + } + + /// A -Headers key is typed object as well, and was rendered under the culture. + [Theory] + [InlineData("de-DE")] + [InlineData("en-US")] + [InlineData("fr-FR")] + public void A_non_string_header_name_renders_the_same_in_every_culture(string culture) + { + InCulture(culture, () => + { + var headers = Headers(new System.Collections.Hashtable { [1.5d] = "x" }); + + Assert.Equal("1.5", Assert.Single(headers.Keys)); + Assert.Equal("x", headers["1.5"]); + }); + } + + /// The value as sent, read back off the captured request. + [Fact] + public void The_value_on_the_wire_does_not_depend_on_the_operators_locale() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "{\"id\":\"u1\"}"); + using var transport = MgxTransportScope.Inject(wire); + + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript( + "[System.Threading.Thread]::CurrentThread.CurrentCulture = 'de-DE'; " + + "Invoke-MgxRequest -Uri '/users/u1' -Headers @{ 'X-Ratio' = 0.5 }"); + ps.Invoke(); + + var sent = Assert.Single(wire.CapturedRequests); + Assert.Equal("0.5", Assert.Single(sent.Headers["X-Ratio"])); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Http/RateLimiterLifetimeTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/RateLimiterLifetimeTests.cs new file mode 100644 index 0000000..c1dd757 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/RateLimiterLifetimeTests.cs @@ -0,0 +1,87 @@ +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// The rate limiter's lifetime against option changes. Every ResilientGraphClient and every +/// ResilientDelegatingHandler captures the limiter it was built with as a readonly field, and +/// the Enable-MgxResilience handler is not rebuilt when options change - so a limiter released +/// by GetOrCreate or Reset is still the one live traffic goes through. Disposing it on a timer +/// left those sessions throwing ObjectDisposedException minutes after a Set-MgxOption call. +/// +[Collection("Pipeline")] +public class RateLimiterLifetimeTests +{ + /// + /// Longer than the window the old timer waited, so a scheduled disposal has fired by the + /// time the held handler sends. + /// + private static readonly TimeSpan PastTheOldDisposalWindow = TimeSpan.FromSeconds(1.5); + + private static (ResilientDelegatingHandler Handler, System.Threading.RateLimiting.TokenBucketRateLimiter Limiter) + BuildHeldHandler(HttpMessageHandler inner) + { + // TotalTimeoutSeconds was what the old code used as the disposal delay; 1s keeps the + // wait short without changing what is being proven. + var options = new ResilientGraphClientOptions { TotalTimeoutSeconds = 1 }; + var (pipeline, rateLimiter) = ResiliencePipelineFactory.GetOrCreate(options); + Assert.NotNull(rateLimiter); + return (new ResilientDelegatingHandler(pipeline, rateLimiter) { InnerHandler = inner }, rateLimiter); + } + + [Fact] + public async Task A_held_handler_keeps_its_limiter_when_options_change() + { + ResiliencePipelineFactory.Reset(); + try + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + var (handler, limiter) = BuildHeldHandler(wire); + using var held = handler; + using var httpClient = new HttpClient(held); + + // Set-MgxOption builds a fresh options instance, so GetOrCreate rebuilds. The + // old code read the disposal delay off the INCOMING options, so the replacement + // has to carry the same short timeout or the wait below outlasts nothing. + ResiliencePipelineFactory.GetOrCreate( + new ResilientGraphClientOptions { RateLimitPerSecond = 10, TotalTimeoutSeconds = 1 }); + await Task.Delay(PastTheOldDisposalWindow); + + var response = await httpClient.GetAsync("https://graph.microsoft.com/v1.0/users/user1"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(1, wire.RequestCount); + // The lease path above already went through the limiter; reading its statistics + // fails on a disposed instance and says so more precisely than the send does. + Assert.NotNull(limiter.GetStatistics()); + } + finally { ResiliencePipelineFactory.Reset(); } + } + + [Fact] + public async Task A_held_handler_keeps_its_limiter_across_a_reset() + { + ResiliencePipelineFactory.Reset(); + try + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + var (handler, limiter) = BuildHeldHandler(wire); + using var held = handler; + using var httpClient = new HttpClient(held); + + // A tenant change drops the cached pipeline and limiter outright. + ResiliencePipelineFactory.Reset(); + await Task.Delay(PastTheOldDisposalWindow); + + var response = await httpClient.GetAsync("https://graph.microsoft.com/v1.0/users/user1"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(1, wire.RequestCount); + Assert.NotNull(limiter.GetStatistics()); + } + finally { ResiliencePipelineFactory.Reset(); } + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Http/SendStopTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/SendStopTests.cs new file mode 100644 index 0000000..0477799 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/SendStopTests.cs @@ -0,0 +1,233 @@ +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// A caller that has stopped sending - a batch chunk refused while its siblings are still going - +/// hands the transport a stop token. What that stop is allowed to reach is the whole contract: +/// the decision to send again, never the attempt already on the wire, whose answer is the only +/// record of what the server applied. Driven here against the transport directly, so the states +/// the guard arbitrates are named rather than inferred from a chunked batch. +/// +[Collection("Pipeline")] +public class SendStopTests +{ + private static readonly ResilientGraphClientOptions Options = new() + { + NoRateLimit = true, + MaxRetryAttempts = 2, + TotalTimeoutSeconds = 120, + AttemptTimeoutSeconds = 60 + }; + + /// + /// Holds the first POST open until the test says so, and answers it with a status the + /// pipeline would ordinarily retry. The stop is set while that POST is on the wire. + /// + private sealed class HoldsThePostOpenHandler : HttpMessageHandler + { + private readonly TaskCompletionSource _entered = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _release = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _posts; + + public Task Entered => _entered.Task; + public void Release() => _release.TrySetResult(); + public int Posts => Volatile.Read(ref _posts); + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _posts); + _entered.TrySetResult(); + await _release.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + return new HttpResponseMessage(HttpStatusCode.TooManyRequests) + { + RequestMessage = request, + Content = new StringContent("{}", System.Text.Encoding.UTF8, "application/json") + }; + } + } + + /// + /// The attempt is on the wire when the stop arrives. Ending it there would trade the answer + /// the server is about to give for a write that may or may not have been applied - the very + /// thing the caller stopped in order to keep. The stop reaches the next attempt instead, + /// which never goes out. + /// + [Fact] + public async Task A_stop_that_lands_on_an_attempt_in_flight_leaves_it_alone() + { + ResiliencePipelineFactory.Reset(); + var wire = new HoldsThePostOpenHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, Options); + using var stop = new CancellationTokenSource(); + + var call = client.PostAsync("https://graph.microsoft.com/v1.0/$batch", + new StringContent("{}"), CancellationToken.None, + paceGate: false, stopRetries: stop.Token); + + await wire.Entered.WaitAsync(TimeSpan.FromSeconds(30)); + stop.Cancel(); + wire.Release(); + + using var response = await call.WaitAsync(TimeSpan.FromSeconds(30)); + + // The answer to the attempt that was flying survives the stop, in full. + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + // A 429 on a POST is the one thing this pipeline retries, and it did not: the stop + // reached the decision to send again. + Assert.Equal(1, wire.Posts); + } + + /// + /// Answers the first POST with a 429 asking for a minute, so the request is unambiguously + /// inside its backoff when the stop arrives. + /// + private sealed class ThrottlesForAMinuteHandler : HttpMessageHandler + { + private readonly TaskCompletionSource _answered = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _posts; + + public Task Answered => _answered.Task; + public int Posts => Volatile.Read(ref _posts); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _posts); + var response = new HttpResponseMessage(HttpStatusCode.TooManyRequests) + { + RequestMessage = request, + Content = new StringContent("{}", System.Text.Encoding.UTF8, "application/json") + }; + response.Headers.RetryAfter = + new System.Net.Http.Headers.RetryConditionHeaderValue(TimeSpan.FromSeconds(60)); + _answered.TrySetResult(); + return Task.FromResult(response); + } + } + + /// + /// Between attempts there is nothing on the wire to protect, and a backoff the caller has + /// stopped honoring is time spent holding the run open for a request it will not send. The + /// wait ends with the stop, and what the server last said is what the request reports - a + /// stopped retry is not a request that got no answer. + /// + [Fact] + public async Task A_stop_between_attempts_ends_the_wait_and_keeps_the_last_status() + { + ResiliencePipelineFactory.Reset(); + var wire = new ThrottlesForAMinuteHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, Options); + using var stop = new CancellationTokenSource(); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + var call = client.PostAsync("https://graph.microsoft.com/v1.0/$batch", + new StringContent("{}"), CancellationToken.None, + paceGate: false, stopRetries: stop.Token); + + await wire.Answered.WaitAsync(TimeSpan.FromSeconds(30)); + // Long enough that the retry decision is behind us and the backoff is running: the + // question is whether a wait already under way ends, not whether one is refused. + await Task.Delay(500); + stop.Cancel(); + + var failure = await Assert.ThrowsAsync( + () => call.WaitAsync(TimeSpan.FromSeconds(30))); + sw.Stop(); + + Assert.Equal(ResilientGraphClient.RetriesStoppedMessage, failure.Message); + // The 429 is what the server last said. A stopped retry that reported no status at all + // would have the caller's items clamped to a generic failure instead. + Assert.Equal(HttpStatusCode.TooManyRequests, failure.StatusCode); + Assert.Equal(1, wire.Posts); + Assert.True(sw.Elapsed < TimeSpan.FromSeconds(30), + $"the call waited out {sw.Elapsed.TotalSeconds:F1}s of a Retry-After it had stopped honoring"); + } + + /// + /// Throttles the first POST with a Retry-After of zero so the retry follows at once, and + /// answers anything after it. Counts every request it is handed without consulting the + /// token: a transport has the request before a cancel reaches it, so a send counted here + /// went out whatever the token said a moment later. + /// + private sealed class CountsEverySendHandler : HttpMessageHandler + { + private int _sends; + + public int Sends => Volatile.Read(ref _sends); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var n = Interlocked.Increment(ref _sends); + var response = new HttpResponseMessage( + n == 1 ? HttpStatusCode.TooManyRequests : HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent("{}", System.Text.Encoding.UTF8, "application/json") + }; + if (n == 1) + response.Headers.RetryAfter = + new System.Net.Http.Headers.RetryConditionHeaderValue(TimeSpan.Zero); + return Task.FromResult(response); + } + } + + /// + /// The stop arrives after the next attempt has been decided on but before it claims the + /// request - the stretch in which the attempt is being built. Nothing is on the wire to + /// protect there, so the attempt ends where it stands. Reading the stop and sending anyway + /// would leave the caller told that no further attempt was sent over one the server has: + /// the stop sets the state and cancels the link as two steps, and a request handed to the + /// transport in between is already gone when the token lands. + /// + [Fact] + public async Task A_stop_that_lands_before_the_next_attempt_claims_it_keeps_it_off_the_wire() + { + ResiliencePipelineFactory.Reset(); + var wire = new CountsEverySendHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, Options); + using var stop = new CancellationTokenSource(); + + var reachedSecondAttempt = + new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + client.AttemptEntryGate = async attempt => + { + if (attempt < 2) return; + reachedSecondAttempt.TrySetResult(); + await release.Task; + }; + + var call = client.PostAsync("https://graph.microsoft.com/v1.0/$batch", + new StringContent("{}"), CancellationToken.None, + paceGate: false, stopRetries: stop.Token); + + await reachedSecondAttempt.Task.WaitAsync(TimeSpan.FromSeconds(30)); + // Cancel runs the registration on this thread, so both halves of the stop - the state + // it takes and the link it cancels - are behind us before the attempt is let go. The + // window is entered on purpose rather than waited for. + stop.Cancel(); + release.SetResult(); + + var outcome = await Record.ExceptionAsync(async () => + { + using var response = await call.WaitAsync(TimeSpan.FromSeconds(30)); + }); + + // The attempt the stop overtook never reached the transport. + Assert.Equal(1, wire.Sends); + var failure = Assert.IsType(outcome); + // ...so the sentence the caller is given about it is true. + Assert.Equal(ResilientGraphClient.RetriesStoppedMessage, failure.Message); + Assert.Equal(HttpStatusCode.TooManyRequests, failure.StatusCode); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Http/TransportSwapTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/TransportSwapTests.cs new file mode 100644 index 0000000..f60f0ef --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/TransportSwapTests.cs @@ -0,0 +1,82 @@ +using System.Net; +using System.Reflection; +using System.Text; +using Mgx.Cmdlets.Base; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// A cmdlet builds one ResilientGraphClient over the shared transport and pages through it for +/// the life of its enumeration. Anything that swaps the transport - Connect-MgGraph to another +/// identity, Set-MgxOption, Enable-MgxResilience, Remove-Module - must leave that client able +/// to fetch its next page. The replaced HttpClient used to be disposed TotalTimeoutSeconds +/// later, a per-request timeout used as a resource lifetime, so a long-running operation died +/// mid-stream with ObjectDisposedException. +/// +[Collection("Pipeline")] +public class TransportSwapTests +{ + private sealed class PageHandler : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) => + Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent("""{"value":[{"id":"u1"}]}""", + Encoding.UTF8, "application/json") + }); + } + + private static readonly BindingFlags Statics = BindingFlags.NonPublic | BindingFlags.Static; + + private static ResilientGraphClientOptions ShortTimeoutOptions() + { + var options = new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }; + // A legal value (the range is 1..3600), set so the old disposal timer would fire + // inside the test rather than five minutes after it. + typeof(ResilientGraphClientOptions) + .GetField("_totalTimeoutSeconds", BindingFlags.Instance | BindingFlags.NonPublic)! + .SetValue(options, 1); + return options; + } + + [Fact] + public async Task A_client_a_running_cmdlet_holds_survives_a_transport_reset() + { + ResiliencePipelineFactory.Reset(); + var t = typeof(MgxCmdletBase); + var options = ShortTimeoutOptions(); + var transport = new HttpClient(new PageHandler()); + t.GetField("s_graphHttpClient", Statics)!.SetValue(null, transport); + t.GetField("s_ownsHttpClient", Statics)!.SetValue(null, true); + t.GetField("s_clientOptions", Statics | BindingFlags.Public)!.SetValue(null, options); + + try + { + // What a cmdlet holds for the whole of its enumeration. + using var held = new ResilientGraphClient(transport, options); + + var page1 = await held.SendAsync(HttpMethod.Get, "https://graph.microsoft.com/v1.0/users"); + Assert.Equal(HttpStatusCode.OK, page1.StatusCode); + + // Another thread swaps the transport out from under it. + MgxCmdletBase.ResetHttpClient(); + await Task.Delay(TimeSpan.FromSeconds(2.5)); + + var page2 = await held.SendAsync(HttpMethod.Get, "https://graph.microsoft.com/v1.0/users?$skiptoken=P2"); + Assert.Equal(HttpStatusCode.OK, page2.StatusCode); + } + finally + { + t.GetField("s_graphHttpClient", Statics)!.SetValue(null, null); + t.GetField("s_ownsHttpClient", Statics)!.SetValue(null, false); + t.GetField("s_clientOptions", Statics | BindingFlags.Public)! + .SetValue(null, ResilientGraphClientOptions.Default); + t.GetField("s_cachedAuthFingerprint", Statics)!.SetValue(null, null); + t.GetField("s_cachedAuthContextRef", Statics)!.SetValue(null, null); + ResiliencePipelineFactory.Reset(); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/BoundarySizeTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/BoundarySizeTests.cs new file mode 100644 index 0000000..8624bcf --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/BoundarySizeTests.cs @@ -0,0 +1,89 @@ +using System.Net; +using System.Text; +using System.Text.Json; +using Mgx.Engine.Http; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// Collection sizes that sit on a page seam: exactly full, one over, and the 999/1000/1001 +/// band around Graph's largest page. An off-by-one in the follow-the-nextLink loop drops the +/// last page or the last item, and only these sizes expose it. +/// (Corpus: M365DSC-7274, silent truncation.) +/// +[Collection("Pipeline")] +public class BoundarySizeTests +{ + /// One page of items, with a nextLink unless it is last. + private static string Page(int firstIndex, int count, int pageNumber, bool isLast) + { + var json = new StringBuilder("""{"value":["""); + for (var i = 0; i < count; i++) + { + if (i > 0) json.Append(','); + json.Append($$"""{"id":"user{{firstIndex + i}}"}"""); + } + json.Append(']'); + if (!isLast) + { + json.Append(""","@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=page"""); + json.Append(pageNumber + 1).Append('"'); + } + json.Append('}'); + return json.ToString(); + } + + /// + /// is the shape Graph actually returns when the + /// collection divides evenly: the last full page still carries a nextLink, and following + /// it yields an empty page. Without it a full page is simply the end, which is a + /// single-request enumeration that crosses no seam at all. + /// + [Theory] + [InlineData(100, 100, false)] // exactly one full page, ended by its own absent nextLink + [InlineData(100, 100, true)] // the same size, ended by an empty page behind a nextLink + [InlineData(101, 100, false)] // one item spills into a second page + [InlineData(999, 999, false)] // exactly Graph's largest page + [InlineData(999, 999, true)] // and the same, with the empty page Graph appends + [InlineData(1000, 999, false)] // one item past it + [InlineData(1001, 999, false)] // two items past it + public async Task Every_item_comes_back_at_a_page_boundary_size(int total, int pageSize, bool trailingEmptyPage) + { + var handler = new MockHttpHandler(); + var expectedPages = 0; + for (var sent = 0; sent < total; sent += pageSize) + { + var count = Math.Min(pageSize, total - sent); + handler.QueueResponse(HttpStatusCode.OK, + Page(sent + 1, count, expectedPages + 1, isLast: sent + count == total && !trailingEmptyPage)); + expectedPages++; + } + if (trailingEmptyPage) + { + handler.QueueResponse(HttpStatusCode.OK, Page(total + 1, 0, expectedPages + 1, isLast: true)); + expectedPages++; + } + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var iterator = new PageIterator(client); + + var items = new List(); + await foreach (var item in iterator.StreamAllWithCountAsync( + $"https://graph.microsoft.com/v1.0/users?$top={pageSize}", maxItems: 0, onCount: null)) + { + items.Add(item); + } + + Assert.Equal(total, items.Count); + Assert.Equal(expectedPages, handler.RequestCount); + Assert.Equal("user1", items[0].GetProperty("id").GetString()); + Assert.Equal($"user{total}", items[^1].GetProperty("id").GetString()); + // MockHttpHandler answers in queue order regardless of URL, so the counts above hold + // even for an iterator that re-sent the initial URL. Read the URLs to pin the chain. + Assert.All(handler.Requests.Skip(1).Select((r, i) => (r, i)), + x => Assert.Equal($"https://graph.microsoft.com/v1.0/users?$skiptoken=page{x.i + 2}", + x.r.RequestUri!.ToString())); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/NamedTempPromotionTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/NamedTempPromotionTests.cs new file mode 100644 index 0000000..891ec81 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/NamedTempPromotionTests.cs @@ -0,0 +1,108 @@ +using System.Reflection; + +namespace Mgx.IntegrationTests.Engine.Pagination; + +/// +/// Promotion of the temp a checkpoint names, which is how every checkpoint this release writes +/// recovers: the file is taken by name and by the length recorded for it, so nothing has to be +/// guessed about which run's items are in it. What the name does not say is whether that run is +/// finished. Promotion replaces the output with those bytes and then unlinks the temp, so the +/// same claim adoption takes in OrphanAdoptionTests has to be taken here - a file nobody can be +/// found holding is an interrupted run's leftovers, and one that cannot be claimed belongs to a +/// run that is still writing it. +/// +public class NamedTempPromotionTests +{ + private static readonly MethodInfo Promote = + typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( + "TryPromoteNamedTemp", BindingFlags.Static | BindingFlags.NonPublic)!; + + private static readonly MethodInfo CanPromote = + typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( + "CanPromoteNamedTemp", BindingFlags.Static | BindingFlags.NonPublic)!; + + private static bool Invoke(string outputPath, string tempFileName, long dataLength) => + (bool)Promote.Invoke(null, [outputPath, tempFileName, dataLength])!; + + private static bool InvokeCan(string outputPath, string tempFileName, long dataLength) => + (bool)CanPromote.Invoke(null, [outputPath, tempFileName, dataLength])!; + + private static string NewDir() + { + var d = Path.Combine(Path.GetTempPath(), $"mgx-promote-{Guid.NewGuid():N}"); + Directory.CreateDirectory(d); + return d; + } + + /// + /// The state a crash leaves: the previous run's output, this run's items in the temp beside + /// it, and a checkpoint naming both. The temp replaces the output, trimmed to the length + /// recorded, and goes. + /// + [Fact] + public void Promotes_a_named_temp_nothing_holds() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + // The writer's position at the last flush, which is what a checkpoint records. + var counted = new FileInfo(temp).Length; + File.AppendAllText(temp, "{\"id\":\"after-the-flush\"}\n"); + + Assert.True(InvokeCan(output, Path.GetFileName(temp), counted)); + Assert.True(Invoke(output, Path.GetFileName(temp), counted)); + + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + Assert.Empty(Directory.GetFiles(dir, "*.tmp")); + } + finally { Directory.Delete(dir, true); } + } + + /// + /// The same three files, with the run that wrote the temp still writing it. Promotion + /// copied its rows into this run's output and unlinked the file underneath it, so that run + /// wrote into an unlinked inode and its own closing move failed against a name that was no + /// longer there. Being named by a checkpoint says which file holds the items; it says + /// nothing about whether anyone still has it open. + /// + [Fact] + public void Refuses_a_named_temp_a_live_run_still_holds() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var live = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + + // Opened the way an export opens its own temp: new StreamWriter(path, append). + using (var writer = new StreamWriter(live, append: false)) + { + writer.WriteLine("{\"id\":\"a\"}"); + writer.WriteLine("{\"id\":\"b\"}"); + writer.Flush(); + var counted = new FileInfo(live).Length; + Assert.True(counted > 0); + + Assert.False(InvokeCan(output, Path.GetFileName(live), counted)); + Assert.False(Invoke(output, Path.GetFileName(live), counted)); + + Assert.Equal(["{\"id\":\"old\"}"], File.ReadAllLines(output)); + Assert.False(File.Exists($"{output}.adopt")); + Assert.True(File.Exists(live), "a running export's temp was promoted out from under it"); + + // The run that owns it goes on writing into the file it still has. + writer.WriteLine("{\"id\":\"c\"}"); + } + + Assert.Equal( + ["{\"id\":\"a\"}", "{\"id\":\"b\"}", "{\"id\":\"c\"}"], + File.ReadAllLines(live)); + } + finally { Directory.Delete(dir, true); } + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/OpaqueLinkTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/OpaqueLinkTests.cs index e706108..22b12ca 100644 --- a/tests/Mgx.IntegrationTests/Engine/Pagination/OpaqueLinkTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/OpaqueLinkTests.cs @@ -6,7 +6,7 @@ namespace Mgx.IntegrationTests; /// A service-issued link is opaque: validation may refuse it, but a link that passes is /// followed byte-identically - no re-encoding, no normalization, no re-parsing. A repaired /// link is a different link, and Graph's skiptokens do not survive repair. -/// (GraphSDK-2488.) +/// (Corpus: GraphSDK-2488, opaque service links.) /// public class OpaqueLinkTests { diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/OrphanAdoptionTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/OrphanAdoptionTests.cs index 63a9ad2..af7b4f9 100644 --- a/tests/Mgx.IntegrationTests/Engine/Pagination/OrphanAdoptionTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/OrphanAdoptionTests.cs @@ -4,10 +4,10 @@ namespace Mgx.IntegrationTests.Engine.Pagination; /// /// Orphan-temp adoption after a JSONL run is interrupted, for checkpoints that predate the -/// recorded temp name and length. With only a line count and a glob to go on, adoption is -/// safe solely when no output exists; against an existing output it must refuse and leave the -/// caller to re-enumerate, since nothing ties the newest temp to the run the checkpoint is -/// actually about. +/// recorded temp name and length. With only a line count and the newest file a run could have +/// named - "{output}.{32-hex}.tmp" - to go on, adoption is safe solely when no output exists; +/// against an existing output it must refuse and leave the caller to re-enumerate, since +/// nothing ties that temp to the run the checkpoint is actually about. /// public class OrphanAdoptionTests { @@ -15,9 +15,16 @@ public class OrphanAdoptionTests typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( "TryAdoptOrphanedTemp", BindingFlags.Static | BindingFlags.NonPublic)!; + private static readonly MethodInfo CanAdopt = + typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( + "CanAdoptOrphanedTemp", BindingFlags.Static | BindingFlags.NonPublic)!; + private static bool Invoke(string outputPath, long itemCount) => (bool)Adopt.Invoke(null, [outputPath, itemCount])!; + private static bool InvokeCan(string outputPath, long itemCount) => + (bool)CanAdopt.Invoke(null, [outputPath, itemCount])!; + private static string NewDir() { var d = Path.Combine(Path.GetTempPath(), $"mgx-adopt-{Guid.NewGuid():N}"); @@ -32,7 +39,7 @@ public void Adopts_the_temp_when_no_output_exists_yet() try { var output = Path.Combine(dir, "out.jsonl"); - File.WriteAllLines($"{output}.abc.tmp", ["{\"id\":\"a\"}", "{\"id\":\"b\"}", "{\"id\":\"c\"}"]); + File.WriteAllLines($"{output}.{Guid.NewGuid():N}.tmp", ["{\"id\":\"a\"}", "{\"id\":\"b\"}", "{\"id\":\"c\"}"]); Assert.True(Invoke(output, 2)); @@ -54,7 +61,7 @@ public void Refuses_when_an_output_already_exists() File.WriteAllLines(output, ["{\"id\":\"run1-1\"}", "{\"id\":\"run1-2\"}"]); // Run 2 crashed: a temp is on disk, but nothing says it is run 2's rather than a // survivor of some unrelated enumeration against the same output name. - File.WriteAllLines($"{output}.def.tmp", ["{\"id\":\"run2-1\"}", "{\"id\":\"run2-2\"}", "{\"id\":\"unflushed\"}"]); + File.WriteAllLines($"{output}.{Guid.NewGuid():N}.tmp", ["{\"id\":\"run2-1\"}", "{\"id\":\"run2-2\"}", "{\"id\":\"unflushed\"}"]); Assert.False(Invoke(output, 2)); @@ -74,7 +81,7 @@ public void Refuses_when_the_temp_holds_less_than_the_checkpoint_promises() { var output = Path.Combine(dir, "out.jsonl"); File.WriteAllLines(output, ["{\"id\":\"kept\"}"]); - File.WriteAllLines($"{output}.ghi.tmp", ["{\"id\":\"only-one\"}"]); + File.WriteAllLines($"{output}.{Guid.NewGuid():N}.tmp", ["{\"id\":\"only-one\"}"]); Assert.False(Invoke(output, 5)); @@ -85,6 +92,131 @@ public void Refuses_when_the_temp_holds_less_than_the_checkpoint_promises() finally { Directory.Delete(dir, true); } } + /// + /// A count is a count of line STARTS unless something says otherwise, and ReadLine returns + /// an unterminated tail as a line like any other - so a temp cut inside the row the + /// checkpoint counts handed the fragment over and it was written into the output as an + /// item, with the rest of the enumeration appended behind it. mgx's own runs flush before + /// they count, so this is a temp truncated under one rather than one it left. + /// + [Fact] + public void Refuses_when_the_counted_row_was_cut_short() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText($"{output}.{Guid.NewGuid():N}.tmp", + "{\"id\":\"a\"}\n{\"id\":\"b\"}\n{\"id\":\"to"); + + Assert.False(Invoke(output, 3)); + + Assert.False(File.Exists(output)); + Assert.False(File.Exists($"{output}.adopt")); + } + finally { Directory.Delete(dir, true); } + } + + /// + /// The state an interrupted run does leave: rows written after the last flush are torn or + /// missing, and the checkpoint counts none of them. Refusing here would cost a + /// re-enumeration for nothing. + /// + [Fact] + public void Adopts_when_only_the_uncounted_row_was_cut_short() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText($"{output}.{Guid.NewGuid():N}.tmp", + "{\"id\":\"a\"}\n{\"id\":\"b\"}\n{\"id\":\"to"); + + Assert.True(Invoke(output, 2)); + + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + Assert.Empty(Directory.GetFiles(dir, "*.tmp")); + } + finally { Directory.Delete(dir, true); } + } + + /// + /// "Orphan" is an assumption about a file this run did not write, and a second export + /// against the same output holds one matching the same glob right now. Adoption copies what + /// it picks into the output and then unlinks it, so it took the live run's rows as its own + /// and left that run writing into an unlinked inode - the failure the stale-temp sweep asks + /// CanTakeExclusively about before it deletes anything. Adoption has to ask the same + /// question, and a file it cannot claim is not an orphan. + /// + [Fact] + public void Refuses_a_temp_a_live_run_still_holds() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + var live = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + + // Opened the way an export opens its own temp: new StreamWriter(path, append). + using (var writer = new StreamWriter(live, append: false)) + { + writer.WriteLine("{\"id\":\"other-1\"}"); + writer.WriteLine("{\"id\":\"other-2\"}"); + writer.Flush(); + + Assert.False(InvokeCan(output, 2)); + Assert.False(Invoke(output, 2)); + + Assert.False(File.Exists(output)); + Assert.False(File.Exists($"{output}.adopt")); + Assert.True(File.Exists(live), "a running export's temp was consumed as an orphan"); + + // The run that owns it goes on writing into the file it still has. + writer.WriteLine("{\"id\":\"other-3\"}"); + } + + Assert.Equal( + ["{\"id\":\"other-1\"}", "{\"id\":\"other-2\"}", "{\"id\":\"other-3\"}"], + File.ReadAllLines(live)); + } + finally { Directory.Delete(dir, true); } + } + + /// + /// The newest temp is the one a live run is writing into, which is exactly the file + /// adoption must not take - so passing over it cannot mean giving up. The dead run's temp + /// underneath is the orphan recovery exists for, and it is still the one to adopt. + /// + [Fact] + public void Adopts_the_orphan_underneath_a_temp_a_live_run_holds() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + + // A dead run's temp, aged so the live one below is unambiguously newer. + var orphan = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllLines(orphan, ["{\"id\":\"dead-1\"}", "{\"id\":\"dead-2\"}"]); + File.SetLastWriteTimeUtc(orphan, DateTime.UtcNow.AddMinutes(-5)); + + var live = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + using (var writer = new StreamWriter(live, append: false)) + { + writer.WriteLine("{\"id\":\"live-1\"}"); + writer.WriteLine("{\"id\":\"live-2\"}"); + writer.Flush(); + + Assert.True(Invoke(output, 2)); + + Assert.Equal(["{\"id\":\"dead-1\"}", "{\"id\":\"dead-2\"}"], File.ReadAllLines(output)); + Assert.False(File.Exists(orphan)); + Assert.True(File.Exists(live), "a running export's temp was consumed as an orphan"); + } + } + finally { Directory.Delete(dir, true); } + } + [Fact] public void Does_nothing_when_there_is_no_temp() { diff --git a/tests/Mgx.IntegrationTests/Fakes/FakeGraphSession.cs b/tests/Mgx.IntegrationTests/Fakes/FakeGraphSession.cs new file mode 100644 index 0000000..04473f3 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Fakes/FakeGraphSession.cs @@ -0,0 +1,42 @@ +namespace Microsoft.Graph.PowerShell.Authentication; + +/// +/// Stands in for the Graph SDK's GraphSession singleton. MgxCmdletBase.FindType resolves that +/// type by full name across the loaded assemblies, so a type of this name declared here is what +/// the injection paths find in a process where the SDK is absent. +/// +/// Everything mgx reads off the real session is duck-typed reflection - Instance, AuthContext, +/// GraphHttpClient, Environment - so matching the member names is the whole contract. +/// +/// Instance is null until a test arms it through Mgx.IntegrationTests.GraphSessionScope, and the +/// scope detaches it again on dispose. Nothing else in the suite sees a session. +/// +internal sealed class GraphSession +{ + private static GraphSession? s_instance; + + public static GraphSession? Instance => s_instance; + + public object? AuthContext { get; set; } + + public HttpClient? GraphHttpClient { get; set; } + + public object? Environment { get; set; } + + internal static GraphSession Attach() + { + var session = new GraphSession(); + s_instance = session; + return session; + } + + internal static void Detach() => s_instance = null; +} + +/// The members mgx reads off GraphSession.Environment, and nothing more. +internal sealed class FakeGraphEnvironment +{ + public string GraphEndpoint { get; set; } = "https://graph.microsoft.com"; + + public string AzureADEndpoint { get; set; } = "https://login.microsoftonline.com"; +} diff --git a/tests/Mgx.IntegrationTests/Fakes/GraphSessionScope.cs b/tests/Mgx.IntegrationTests/Fakes/GraphSessionScope.cs new file mode 100644 index 0000000..ad0e12d --- /dev/null +++ b/tests/Mgx.IntegrationTests/Fakes/GraphSessionScope.cs @@ -0,0 +1,56 @@ +using Mgx.Cmdlets.Base; +using Mgx.Cmdlets.Cmdlets.Configuration; +using Mgx.Engine.Http; +using Microsoft.Graph.PowerShell.Authentication; + +namespace Mgx.IntegrationTests; + +/// +/// Arms the fake GraphSession for the life of the scope and puts the process back the way it was +/// on dispose: no session instance, no injection state, no cached mgx client, a reset pipeline +/// factory. Without the restore a later test would find a session where the suite assumes none. +/// +internal sealed class GraphSessionScope : IDisposable +{ + internal GraphSession Session { get; } + + private readonly ResilientGraphClientOptions _previousOptions; + private readonly string _previousEndpoint; + + private GraphSessionScope(HttpClient? graphHttpClient, object? authContext) + { + // A scenario that drives the real GetClient sets both of these - the options because it + // chooses what the limiter under test does, the endpoint because GetClient reads it off + // the session it is about to arm. Neither belongs to the tests that run after it. + _previousOptions = MgxCmdletBase.s_clientOptions; + _previousEndpoint = MgxCmdletBase.s_graphEndpoint; + + MgxCmdletBase.ResetHttpClient(); + Session = GraphSession.Attach(); + Session.GraphHttpClient = graphHttpClient; + Session.AuthContext = authContext; + Session.Environment = new FakeGraphEnvironment(); + } + + internal static GraphSessionScope Arm(HttpClient? graphHttpClient = null, object? authContext = null) => + new(graphHttpClient, authContext); + + /// An AuthContext-shaped object. BuildAuthFingerprint reads these members by name. + internal static object AuthContextFor(string tenantId, string clientId) => new + { + TenantId = tenantId, + ClientId = clientId, + AuthType = "AppOnly", + Environment = "Global" + }; + + public void Dispose() + { + GraphSession.Detach(); + EnableMgxResilience.ReleaseInjection(); + MgxCmdletBase.s_clientOptions = _previousOptions; + MgxCmdletBase.s_graphEndpoint = _previousEndpoint; + MgxCmdletBase.ResetHttpClient(); + ResiliencePipelineFactory.Reset(); + } +} diff --git a/tests/Mgx.IntegrationTests/Fakes/MockHttpHandler.cs b/tests/Mgx.IntegrationTests/Fakes/MockHttpHandler.cs index 2f78599..36205f1 100644 --- a/tests/Mgx.IntegrationTests/Fakes/MockHttpHandler.cs +++ b/tests/Mgx.IntegrationTests/Fakes/MockHttpHandler.cs @@ -35,6 +35,7 @@ public class MockHttpHandler : HttpMessageHandler private readonly Queue _responses = new(); private readonly List _requests = []; private readonly List _captured = []; + private readonly List _served = []; private readonly object _lock = new(); private MockResponse? _defaultResponse; @@ -57,6 +58,15 @@ public List CapturedRequests get { lock (_lock) { return [.. _captured]; } } } + /// + /// The status codes handed back, in order. A test whose assertion is that nothing was + /// reported needs some way to know the wire reported something to begin with. + /// + public List ServedStatusCodes + { + get { lock (_lock) { return [.. _served]; } } + } + public void QueueResponse(HttpStatusCode statusCode, string? body = null, Dictionary? headers = null, string contentType = "application/json") { _responses.Enqueue(new MockResponse(statusCode, body, headers, null, contentType, null)); @@ -122,6 +132,7 @@ protected override async Task SendAsync(HttpRequestMessage _requests.Add(request); _captured.Add(captured); mock = _responses.Count > 0 ? _responses.Dequeue() : (_defaultResponse ?? new MockResponse(HttpStatusCode.OK, null, null, null, null, null)); + if (mock.Exception == null) _served.Add(mock.StatusCode); } if (mock.Exception != null) diff --git a/tests/Mgx.IntegrationTests/Mgx.IntegrationTests.csproj b/tests/Mgx.IntegrationTests/Mgx.IntegrationTests.csproj index 583b507..7ad159d 100644 --- a/tests/Mgx.IntegrationTests/Mgx.IntegrationTests.csproj +++ b/tests/Mgx.IntegrationTests/Mgx.IntegrationTests.csproj @@ -5,7 +5,8 @@ (LTS) and up. This project stays on net10.0 because Microsoft.PowerShell.SDK 7.6.x requires it, and referencing a net8.0 project from net10.0 is fine. What that buys is the shipped assembly exercised on the newest runtime; the older hosts are covered by - the `compat` matrix in CI, which imports the module under 7.4/7.5/7.6 for real. --> + the `compat` matrix in CI, which imports the module for real under the ends of the + supported range, 7.4 and 7.6. --> net10.0 Major enable diff --git a/tests/Mgx.IntegrationTests/Serialization/WireBodyTests.cs b/tests/Mgx.IntegrationTests/Serialization/WireBodyTests.cs index 43aeb6b..10814d4 100644 --- a/tests/Mgx.IntegrationTests/Serialization/WireBodyTests.cs +++ b/tests/Mgx.IntegrationTests/Serialization/WireBodyTests.cs @@ -1,5 +1,7 @@ +using System.Collections; +using System.Globalization; using System.Net; -using System.Reflection; +using System.Text.Json.Serialization; using Mgx.Engine.Http; namespace Mgx.IntegrationTests; @@ -12,42 +14,12 @@ namespace Mgx.IntegrationTests; [Collection("Pipeline")] public class WireBodyTests { - private static readonly Type Base = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); - private const BindingFlags Static = BindingFlags.NonPublic | BindingFlags.Static; - - private static void InjectTransport(HttpMessageHandler wire) - { - Base.GetField("s_graphHttpClient", Static)!.SetValue(null, new HttpClient(wire)); - Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, - Mgx.Cmdlets.Base.MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); - Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); - Base.GetField("s_clientOptions", Static)!.SetValue(null, - new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); - } - - private static void ResetTransport() - { - // Restore every static InjectTransport touched - a later test in the collection - // that drives a cmdlet without injecting must not inherit this class's transport. - Base.GetField("s_graphHttpClient", Static)!.SetValue(null, null); - Base.GetField("s_cachedAuthFingerprint", Static)!.SetValue(null, null); - Base.GetField("s_ownsHttpClient", Static)!.SetValue(null, false); - Base.GetField("s_cachedTotalTimeoutSeconds", Static)!.SetValue(null, 0); - Base.GetField("s_graphEndpoint", Static)!.SetValue(null, "https://graph.microsoft.com"); - Base.GetField("s_clientOptions", Static)!.SetValue(null, new ResilientGraphClientOptions()); - ResiliencePipelineFactory.Reset(); - } - /// Runs a POST built by the given script and returns the wire body text. private static string WireBodyFor(string bodyExpression) { var wire = new MockHttpHandler(); wire.SetDefaultResponse(HttpStatusCode.NoContent); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { using var ps = System.Management.Automation.PowerShell.Create(); ps.AddCommand("Import-Module") @@ -65,13 +37,127 @@ private static string WireBodyFor(string bodyExpression) Assert.True(captured.Count > 0, $"no request sent; errors: {errors}"); return captured[0].BodyText!; } - finally { ResetTransport(); } } [Fact] public void An_enum_serializes_as_its_camel_case_name() => Assert.Equal("""{"day":"monday"}""", WireBodyFor("@{ day = [System.DayOfWeek]::Monday }")); + [Fact] + public void A_flags_enum_combination_joins_its_names_without_a_space() + => Assert.Equal("""{"opt":"ignoreCase,multiline"}""", + WireBodyFor("@{ opt = [System.Text.RegularExpressions.RegexOptions]'IgnoreCase, Multiline' }")); + + [Fact] + public void A_single_flag_serializes_as_its_camel_case_name() + => Assert.Equal("""{"opt":"multiline"}""", + WireBodyFor("@{ opt = [System.Text.RegularExpressions.RegexOptions]::Multiline }")); + + /// Read and Fetch share a value; Enum.ToString and STJ pick different names for it. + [Flags] + private enum AliasedFlags { Read = 1, Fetch = 1, Write = 2 } + + private enum AliasedPlain { Read = 1, Fetch = 1, Write = 2 } + + [Flags] + private enum WireFlags { None = 0, Read = 1, Write = 2 } + + /// Read carries a wire name Enum.ToString cannot see. + [Flags] + private enum RenamedFlags + { + [JsonStringEnumMemberName("read-only")] Read = 1, + Write = 2, + } + + /// BC covers B and C, so 7 has a composite member and a bit left over. + [Flags] + private enum CompositeFlags + { + A = 1, + B = 2, + C = 4, + [JsonStringEnumMemberName("b-c")] BC = 6, + } + + private enum PlatformEnum { IOS, Android } + + private sealed class FlagsKeyedHolder + { + public Dictionary Scores { get; } = new() { [WireFlags.Read | WireFlags.Write] = 1 }; + } + + private sealed class RenamedKeyedHolder + { + public Dictionary Scores { get; } = new() { [RenamedFlags.Read | RenamedFlags.Write] = 1 }; + } + + private static string Serialize(object body) + => Mgx.Cmdlets.Cmdlets.InvokeMgxRequest.SerializeBody(body); + + [Fact] + public void An_aliased_flag_gets_the_same_name_as_the_same_enum_without_flags() + { + // Resolving a single name from Enum.ToString would silently rename it: the two + // members share a value, and Enum.ToString and STJ's name cache disagree on which + // one wins. Only the multi-name form belongs to the flags path. + var withFlags = Serialize(new Hashtable { ["v"] = AliasedFlags.Read }); + var withoutFlags = Serialize(new Hashtable { ["v"] = AliasedPlain.Read }); + Assert.Equal(withoutFlags, withFlags); + } + + [Fact] + public void A_flags_value_with_no_names_stays_a_number_under_any_culture() + { + var original = CultureInfo.CurrentCulture; + try + { + // sv-SE writes a negative sign as U+2212, so a number recognized by inspecting + // Enum.ToString would be quoted as a name instead and Graph would refuse it. + CultureInfo.CurrentCulture = new CultureInfo("sv-SE"); + Assert.Equal("""{"v":-3}""", Serialize(new Hashtable { ["v"] = (WireFlags)(-3) })); + } + finally { CultureInfo.CurrentCulture = original; } + } + + [Fact] + public void A_flags_enum_used_as_a_dictionary_key_joins_its_names_too() + => Assert.Equal("""{"thing":{"Scores":{"read,write":1}}}""", + Serialize(new Hashtable { ["thing"] = new FlagsKeyedHolder() })); + + [Fact] + public void An_aliased_flag_in_a_combination_gets_the_name_it_gets_alone() + // Enum.ToString calls this value Fetch and STJ calls it Read, so a combination read + // from Enum.ToString sent "fetch,write" where the member alone was sent "read". + => Assert.Equal("""{"v":"read,write"}""", + Serialize(new Hashtable { ["v"] = AliasedFlags.Read | AliasedFlags.Write })); + + [Fact] + public void A_renamed_flag_keeps_its_wire_name_in_a_combination() + => Assert.Equal("""{"v":"read-only,write"}""", + Serialize(new Hashtable { ["v"] = RenamedFlags.Read | RenamedFlags.Write })); + + [Fact] + public void A_renamed_combination_names_a_dictionary_key_the_same_way() + => Assert.Equal("""{"thing":{"Scores":{"read-only,write":1}}}""", + Serialize(new Hashtable { ["thing"] = new RenamedKeyedHolder() })); + + [Fact] + public void A_member_covering_several_bits_wins_over_the_bits() + // 7 is A|B|C, and BC is a member: components are taken largest first, so BC is chosen + // over B and C, and written after A because the names go out in ascending value order. + => Assert.Equal("""{"v":"a,b-c"}""", Serialize(new Hashtable { ["v"] = (CompositeFlags)7 })); + + [Fact] + public void A_composite_member_on_its_own_stays_a_single_name() + => Assert.Equal("""{"v":"b-c"}""", Serialize(new Hashtable { ["v"] = CompositeFlags.BC })); + + [Fact] + public void A_leading_acronym_is_lower_cased_whole() + // The -Body help documents this: Graph spells the same value "iOS", so a caller who + // needs that has to pass the string. Pinned so the help and the wire cannot drift. + => Assert.Equal("""{"v":"ios"}""", Serialize(new Hashtable { ["v"] = PlatformEnum.IOS })); + [Fact] public void A_byte_array_serializes_as_base64() => Assert.Equal("""{"key":"AQID"}""", WireBodyFor("@{ key = [byte[]](1,2,3) }")); @@ -121,8 +207,7 @@ private static (List Errors, int Reque { var wire = new MockHttpHandler(); wire.SetDefaultResponse(HttpStatusCode.NoContent); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { using var ps = System.Management.Automation.PowerShell.Create(); ps.AddCommand("Import-Module") @@ -136,7 +221,6 @@ private static (List Errors, int Reque ps.Invoke(); return (ps.Streams.Error.ToList(), wire.CapturedRequests.Count); } - finally { ResetTransport(); } } [Fact] @@ -202,8 +286,7 @@ public void A_refused_batch_body_fails_that_item_and_sends_the_rest() var wire = new MockHttpHandler(); wire.QueueResponse(HttpStatusCode.OK, """{ "responses": [ { "id": "1", "status": 204 } ] }"""); - InjectTransport(wire); - try + using (MgxTransportScope.Inject(wire)) { using var ps = System.Management.Automation.PowerShell.Create(); ps.AddCommand("Import-Module") @@ -226,6 +309,5 @@ public void A_refused_batch_body_fails_that_item_and_sends_the_rest() Assert.Contains("SecureString", error.Exception.Message); Assert.Single(wire.CapturedRequests); // the good item still went } - finally { ResetTransport(); } } } diff --git a/tests/Mgx.IntegrationTests/TestSetup/MgxTransportScope.cs b/tests/Mgx.IntegrationTests/TestSetup/MgxTransportScope.cs new file mode 100644 index 0000000..0b633b2 --- /dev/null +++ b/tests/Mgx.IntegrationTests/TestSetup/MgxTransportScope.cs @@ -0,0 +1,112 @@ +using Mgx.Cmdlets.Base; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// Points MgxCmdletBase at a mock transport for the life of the scope and puts every static it +/// touched back the way it found it, so a later test in the Pipeline collection that drives a +/// cmdlet without injecting cannot inherit this one's wire. +/// +/// The seam sits above the auth probe in GetClient, so a test shell needs no Graph session and no +/// matching fingerprint. The fields are internal and assigned directly rather than reflected: +/// renaming one is a compile error here instead of a NullReferenceException at run time. +/// +/// Every scope resets ResiliencePipelineFactory on entry and exit. Retry-count assertions read +/// the pipeline the factory hands out, so a pipeline built under another test's MaxRetryAttempts +/// would change what those counts mean. +/// +internal sealed class MgxTransportScope : IDisposable +{ + private readonly HttpClient? _createdClient; + private readonly HttpClient? _previousTransport; + private readonly bool _previousOwned; + private readonly ResilientGraphClientOptions _previousOptions; + private readonly string _previousEndpoint; + + private MgxTransportScope(HttpClient client, bool createdHere, bool owned, + ResilientGraphClientOptions options, string endpoint) + { + _createdClient = createdHere ? client : null; + _previousTransport = MgxCmdletBase.s_testTransport; + _previousOwned = MgxCmdletBase.s_testTransportOwned; + _previousOptions = MgxCmdletBase.s_clientOptions; + _previousEndpoint = MgxCmdletBase.s_graphEndpoint; + + ResiliencePipelineFactory.Reset(); + MgxCmdletBase.s_clientOptions = options; + MgxCmdletBase.s_graphEndpoint = endpoint; + Publish(client, owned); + } + + /// + /// Arms a transport: the transport itself, then the epoch that retires whatever preceded it. + /// GetClient takes the epoch, the transport, then the epoch again, and rereads on a change - + /// so the transport going first is what lets an unchanged epoch mean the two belong together. + /// Bumping first would leave a window in which the new epoch is live over the old transport, + /// which a reader cannot tell from a settled state. + /// + /// Entering and leaving both go through here, so the two directions cannot drift apart. + /// + /// + private void Publish(HttpClient? transport, bool owned) + { + MgxCmdletBase.s_testTransportOwned = owned; + MgxCmdletBase.s_testTransport = transport; + BetweenPublishWrites?.Invoke(); + Interlocked.Increment(ref MgxCmdletBase.s_testTransportEpoch); + } + + /// + /// Runs between the two writes above, which is the only point from which their order is + /// visible. Set by the one test that pins it; null for every other scope, so a test running + /// in another collection at the same time is unaffected. + /// + internal Action? BetweenPublishWrites { get; set; } + + /// The options a scope installs unless the caller passes its own. + internal static ResilientGraphClientOptions DefaultOptions => + new() { NoRateLimit = true, MaxRetryAttempts = 1 }; + + /// + /// Arms the seam with an HttpClient over . The scope owns that client + /// and disposes it on exit. + /// + internal static MgxTransportScope Inject( + HttpMessageHandler wire, + bool owned = false, + ResilientGraphClientOptions? options = null, + string endpoint = "https://graph.microsoft.com") => + new(new HttpClient(wire), createdHere: true, owned, options ?? DefaultOptions, endpoint); + + /// + /// Arms the seam with a caller-owned HttpClient, for a test that needs the same instance + /// across several scopes or disposes it itself. + /// + internal static MgxTransportScope Inject( + HttpClient client, + bool owned = false, + ResilientGraphClientOptions? options = null, + string endpoint = "https://graph.microsoft.com") => + new(client, createdHere: false, owned, options ?? DefaultOptions, endpoint); + + /// + /// Replaces the options the armed transport runs under, without rebuilding the transport. + /// Bumps the epoch, so a cmdlet instance holding a client from before the change rebuilds. + /// + internal static void SetOptions(ResilientGraphClientOptions options) + { + ResiliencePipelineFactory.Reset(); + MgxCmdletBase.s_clientOptions = options; + Interlocked.Increment(ref MgxCmdletBase.s_testTransportEpoch); + } + + public void Dispose() + { + Publish(_previousTransport, _previousOwned); + MgxCmdletBase.s_graphEndpoint = _previousEndpoint; + MgxCmdletBase.s_clientOptions = _previousOptions; + ResiliencePipelineFactory.Reset(); + _createdClient?.Dispose(); + } +} diff --git a/tests/Mgx.IntegrationTests/TestSetup/RegressionCorpusIndexTests.cs b/tests/Mgx.IntegrationTests/TestSetup/RegressionCorpusIndexTests.cs new file mode 100644 index 0000000..6cb35de --- /dev/null +++ b/tests/Mgx.IntegrationTests/TestSetup/RegressionCorpusIndexTests.cs @@ -0,0 +1,141 @@ +using System.Text.RegularExpressions; + +namespace Mgx.IntegrationTests; + +/// +/// Provenance is only useful if it can be trusted, and a doc comment naming an issue number is +/// unverifiable on its own. Every issue id written anywhere under tests/ has to have a row in +/// tests/CORPUS.md saying what it guarantees and where. +/// +/// The guard runs one way only. The reverse - every indexed id has a test - cannot hold while +/// the index carries rows that name the release bringing their test, which is the honest way to +/// record a gap. +/// +public class RegressionCorpusIndexTests +{ + private const string IndexPath = "tests/CORPUS.md"; + + /// + /// Matches an issue id and any ids abbreviated onto it: "M365DSC-5306/7175" is two ids, and + /// a scan that read it as one would let the second go unindexed. + /// + private static readonly Regex IssueId = new( + @"\b(M365DSC|GraphSDK)-(\d+)((?:/\d+)*)", RegexOptions.Compiled); + + private static IEnumerable IdsIn(string text) + { + foreach (Match match in IssueId.Matches(text)) + { + var prefix = match.Groups[1].Value; + yield return $"{prefix}-{match.Groups[2].Value}"; + foreach (var abbreviated in match.Groups[3].Value.Split('/', StringSplitOptions.RemoveEmptyEntries)) + yield return $"{prefix}-{abbreviated}"; + } + } + + /// Walk up from the test binaries until the repository-relative path exists. + private static string FindRepositoryPath(string relativePath) + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir != null) + { + var candidate = Path.Combine(dir.FullName, relativePath); + if (File.Exists(candidate) || Directory.Exists(candidate)) return candidate; + dir = dir.Parent; + } + + throw new DirectoryNotFoundException( + $"Could not locate '{relativePath}' above {AppContext.BaseDirectory}"); + } + + /// + /// Every source file under tests/, minus build output, the index itself, and this file - which + /// carries fabricated ids on purpose and would otherwise fail its own guard. + /// + private static IEnumerable CorpusSourceFiles() + { + var testsRoot = FindRepositoryPath("tests"); + var excluded = new[] + { + Path.GetFullPath(FindRepositoryPath(IndexPath)), + Path.GetFullPath(FindRepositoryPath( + Path.Combine("tests", "Mgx.IntegrationTests", "TestSetup", "RegressionCorpusIndexTests.cs"))) + }; + + return Directory.EnumerateFiles(testsRoot, "*", SearchOption.AllDirectories) + .Where(path => path.EndsWith(".cs", StringComparison.OrdinalIgnoreCase) + || path.EndsWith(".ps1", StringComparison.OrdinalIgnoreCase) + || path.EndsWith(".psm1", StringComparison.OrdinalIgnoreCase) + || path.EndsWith(".md", StringComparison.OrdinalIgnoreCase)) + .Where(path => !path.Contains($"{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}", + StringComparison.Ordinal) + && !path.Contains($"{Path.DirectorySeparatorChar}obj{Path.DirectorySeparatorChar}", + StringComparison.Ordinal)) + .Where(path => !excluded.Contains(Path.GetFullPath(path), StringComparer.Ordinal)); + } + + /// The ids a body of text claims, that the index does not carry. + private static IReadOnlyList Unindexed(string indexText, string claimingText) + { + var indexed = IdsIn(indexText).ToHashSet(StringComparer.Ordinal); + return [.. IdsIn(claimingText).Distinct(StringComparer.Ordinal).Where(id => !indexed.Contains(id))]; + } + + [Fact] + public void Every_issue_id_under_tests_has_a_row_in_the_index() + { + var indexText = File.ReadAllText(FindRepositoryPath(IndexPath)); + + var missing = new SortedDictionary>(StringComparer.Ordinal); + var claimed = new HashSet(StringComparer.Ordinal); + var scanned = 0; + foreach (var file in CorpusSourceFiles()) + { + scanned++; + var text = File.ReadAllText(file); + claimed.UnionWith(IdsIn(text)); + + foreach (var id in Unindexed(indexText, text)) + { + if (!missing.TryGetValue(id, out var files)) missing[id] = files = []; + files.Add(Path.GetFileName(file)); + } + } + + // A scan that reaches nothing reports nothing missing, so the guard has to fail on its + // own silence first. A repository root resolved above the wrong directory, an extension + // filter that stops matching, an exclusion that widens - each of those disarms the check + // below while leaving it green. The floors carry slack because files and ids are added + // over time; what they catch is a scan that has stopped running, not one that has moved. + Assert.True(scanned >= 50, + $"the scan reached {scanned} file(s) under tests/ - it is not reading the suite"); + Assert.True(claimed.Count >= 12, + $"the scan found {claimed.Count} issue id(s) under tests/ - it is not reading provenance"); + + Assert.True(missing.Count == 0, + "issue ids claimed under tests/ with no row in " + IndexPath + ": " + + string.Join("; ", missing.Select(e => $"{e.Key} ({string.Join(", ", e.Value)})"))); + } + + [Fact] + public void A_fabricated_id_in_a_test_file_fails_the_guard() + { + // What the guard above is worth: an id nobody has indexed has to be reported, including + // one abbreviated onto an id that IS indexed. + var indexText = File.ReadAllText(FindRepositoryPath(IndexPath)); + + Assert.Equal(["GraphSDK-9999"], + Unindexed(indexText, "/// (Corpus: GraphSDK-9999, an issue that does not exist.)")); + + Assert.Equal(["M365DSC-9999"], + Unindexed(indexText, "/// (Corpus: M365DSC-7273/9999, one real id and one invented.)")); + } + + [Fact] + public void The_index_is_where_the_suite_says_it_is() + { + // The file moved from TestSetup/ during 2.1.4. A stale path here would make both guards + // above throw rather than fail, which reads as a broken test rather than a broken index. + Assert.True(File.Exists(FindRepositoryPath(IndexPath))); + } +} diff --git a/tests/RELEASING.md b/tests/RELEASING.md deleted file mode 100644 index e6ddb5c..0000000 --- a/tests/RELEASING.md +++ /dev/null @@ -1,65 +0,0 @@ -# Releasing - -`main` carries one commit per release. There are no merge commits: a release is `dev` squashed -onto `main`, tagged, then published. - -Every command below runs from the repository root, not from this directory. - -## Before - -1. **Both suites, both platforms.** - ``` - dotnet test tests/Mgx.IntegrationTests/Mgx.IntegrationTests.csproj - pwsh -c 'Invoke-Pester -Path ./tests/Unit' - ``` - Run them on Windows too. A green run on one OS says nothing about file locking, path - comparison or the exceptions Windows raises where Unix raises different ones - each of those - has shipped a defect here. - -2. **The live suite, actually run.** - ``` - pwsh -c 'Invoke-Pester -Path ./tests/Live' - ``` - With credentials, against a real tenant. The mocked suites cannot see a request Graph - rejects: `Enable-MgxResilience` once shipped a client with no `BaseAddress`, breaking every - relative-URI call, with every test green. "Skipped" is not a pass - see `tests/Live/README.md`. - -3. **Install it the way a user does.** Build Release, then in a *fresh* shell with only the - Gallery dependencies present, import the staged `module/`, run `Test-ModuleManifest`, and - check `Get-Help` for every exported cmdlet. Stale generated help drops parameters silently. - -4. **Read the published numbers.** Every figure in `README.md` must be reproducible from - `tests/benchmarks` as it stands. Re-measure rather than reuse: the tenant's throttling - ceiling has moved by 40% between consecutive days. - -5. **Version and notes.** `ModuleVersion` in `module/mgx.psd1`, the manifest's `ReleaseNotes`, - and the `CHANGELOG.md` section all agree. - -## Squashing - -```bash -git checkout main && git pull origin main -git merge --squash dev -git commit -F - -git diff --stat dev # MUST be empty - if it prints anything, the squash lost something - -git tag -a vX.Y.Z -m "mgx X.Y.Z" -git push origin main && git push origin vX.Y.Z -``` - -The squash message is the public record of the release; the CHANGELOG carries the per-defect -detail. Describe what the release *is*, not the churn that produced it. - -Do not open it by restating what the version number already says. "Patch release", a count of -the fixes, "no feature or API changes", "the PowerShell 7.4 floor is unchanged" - a reader can -see all of that from the version and the entries beneath it. A feature, an API change or a -floor move is worth a sentence precisely because it is not the default; the absence of one is -not. Start with what changed. - -## After - -`Publish-Module` from the tagged `main`. - -`dev` and `main` share no history once squashed, so `dev` reads as permanently ahead. Retire it -and branch a fresh one from `main` for the next version rather than carrying the divergence. diff --git a/tests/Unit/Mgx.Behaviour.Tests.ps1 b/tests/Unit/Mgx.Behaviour.Tests.ps1 index 89db075..48cef8a 100644 --- a/tests/Unit/Mgx.Behaviour.Tests.ps1 +++ b/tests/Unit/Mgx.Behaviour.Tests.ps1 @@ -1513,10 +1513,11 @@ Describe 'Expand-MgxRelation Buffer Warning' { } } -Describe 'HttpClient Ownership Disposal Guard' { - # Tests the s_ownsHttpClient guard in MgxCmdletBase.ScheduleDelayedHttpClientDispose. - # When s_ownsHttpClient is false (SDK fallback path), ResetHttpClient must NOT dispose - # the HttpClient (it belongs to the SDK). When true, it must dispose after delay. +Describe 'HttpClient Reset Leaves A Held Client Alone' { + # A replaced transport is dropped, not closed. Every ResilientGraphClient built on it + # captured it in its constructor and keeps sending through it for as long as its + # enumeration runs, so closing it on a timer killed long-running cmdlets mid-stream; and a + # borrowed SDK client was never mgx's to close at all. ResetHttpClient must leave both. BeforeAll { # Ensure Mgx.Engine types are loaded (transitive dep of Mgx.Cmdlets) @@ -1554,7 +1555,6 @@ Describe 'HttpClient Ownership Disposal Guard' { $script:CmdletBaseType.GetField('s_clientOptions', $script:Flags).SetValue( $null, $script:ShortTimeoutOptions) - # ResetHttpClient -> ScheduleDelayedHttpClientDispose checks s_ownsHttpClient # ResetHttpClient is public static, so use 'Static,Public' (not $script:Flags which is NonPublic) $script:CmdletBaseType.GetMethod('ResetHttpClient', [System.Reflection.BindingFlags]'Static,Public').Invoke($null, $null) @@ -1566,7 +1566,7 @@ Describe 'HttpClient Ownership Disposal Guard' { $script:DisposedField.GetValue($httpClient) | Should -BeFalse } - It 'Should dispose HttpClient when s_ownsHttpClient is true' { + It 'Should NOT dispose an owned HttpClient a running cmdlet may still hold' { $httpClient = [System.Net.Http.HttpClient]::new() $script:CmdletBaseType.GetField('s_graphHttpClient', $script:Flags).SetValue($null, $httpClient) @@ -1577,11 +1577,11 @@ Describe 'HttpClient Ownership Disposal Guard' { $script:CmdletBaseType.GetMethod('ResetHttpClient', [System.Reflection.BindingFlags]'Static,Public').Invoke($null, $null) - # Wait longer than TotalTimeoutSeconds (1s) to let scheduled disposal fire + # Longer than the TotalTimeoutSeconds (1s) the disposal used to be scheduled on Start-Sleep -Seconds 3 - # HttpClient SHOULD be disposed (we own it) - $script:DisposedField.GetValue($httpClient) | Should -BeTrue + # Ownership says who may replace it, not how long anything still using it will live + $script:DisposedField.GetValue($httpClient) | Should -BeFalse } } diff --git a/tests/Unit/Mgx.ErrorContract.Tests.ps1 b/tests/Unit/Mgx.ErrorContract.Tests.ps1 index 7e4fcbe..4af4b16 100644 --- a/tests/Unit/Mgx.ErrorContract.Tests.ps1 +++ b/tests/Unit/Mgx.ErrorContract.Tests.ps1 @@ -3,8 +3,8 @@ <# Pins what about_Mgx_Errors documents, as far as it reaches without a transport: pre-flight failures terminate with stable ids, land in -ErrorVariable, and set $? - false. Per-item wire behavior (BatchItemError, -ErrorAction Stop mid-batch) is pinned - by the xUnit suite, which fakes HTTP. + false. Per-item wire behavior is pinned by the xUnit suite, which fakes HTTP - + ErrorActionMatrixTests drives all four preferences across the six failure contexts. #> BeforeAll { @@ -22,31 +22,67 @@ Describe 'about_Mgx_Errors' { } } +# Carried as text rather than as scriptblocks so each case can be driven twice: once inside a +# try, which is what proves it terminates, and once bare with -ErrorVariable appended, which is +# the only way to read $? and the collected record off the statement that terminated. +$script:PreFlightCases = @( + @{ Name = 'an absolute Uri on Invoke-MgxRequest' + Command = "Invoke-MgxRequest -Uri 'https://graph.microsoft.com/v1.0/users'" + ErrorId = 'AbsoluteUriNotAllowed' } + @{ Name = 'an absolute Uri on Export-MgxCollection' + Command = "Export-MgxCollection -Uri 'https://graph.microsoft.com/v1.0/users' -OutputFile ([IO.Path]::GetTempFileName())" + ErrorId = 'AbsoluteUriNotAllowed' } + @{ Name = 'an absolute Uri on Sync-MgxDelta' + Command = "Sync-MgxDelta -Uri 'https://graph.microsoft.com/v1.0/users/delta' -DeltaPath ([IO.Path]::GetTempFileName())" + ErrorId = 'AbsoluteUriNotAllowed' } + @{ Name = '-Search without ConsistencyLevel' + Command = "Invoke-MgxRequest -Uri '/users' -Search 'displayName:x'" + ErrorId = 'ConsistencyLevelRequired' } + @{ Name = 'a -Uri without {id} on Expand-MgxRelation' + Command = "[pscustomobject]@{ id = 'x' } | Expand-MgxRelation -Uri '/users/x/manager' -As manager" + ErrorId = 'MissingIdPlaceholder' } +) + Describe 'Pre-flight failures terminate' { - It ' terminates with ' -TestCases @( - @{ Name = 'an absolute Uri on Invoke-MgxRequest' - Script = { Invoke-MgxRequest -Uri 'https://graph.microsoft.com/v1.0/users' } - ErrorId = 'AbsoluteUriNotAllowed' } - @{ Name = 'an absolute Uri on Export-MgxCollection' - Script = { Export-MgxCollection -Uri 'https://graph.microsoft.com/v1.0/users' -OutputFile ([IO.Path]::GetTempFileName()) } - ErrorId = 'AbsoluteUriNotAllowed' } - @{ Name = 'an absolute Uri on Sync-MgxDelta' - Script = { Sync-MgxDelta -Uri 'https://graph.microsoft.com/v1.0/users/delta' -DeltaPath ([IO.Path]::GetTempFileName()) } - ErrorId = 'AbsoluteUriNotAllowed' } - @{ Name = '-Search without ConsistencyLevel' - Script = { Invoke-MgxRequest -Uri '/users' -Search 'displayName:x' } - ErrorId = 'ConsistencyLevelRequired' } - @{ Name = 'a -Uri without {id} on Expand-MgxRelation' - Script = { [pscustomobject]@{ id = 'x' } | Expand-MgxRelation -Uri '/users/x/manager' -As manager } - ErrorId = 'MissingIdPlaceholder' } - ) { + It ' terminates with ' -TestCases $script:PreFlightCases { # A terminating error is catchable; a non-terminating one is not. Catching it IS # the assertion that it terminates. $caught = $null - try { & $Script } catch { $caught = $_ } + try { & ([scriptblock]::Create($Command)) } catch { $caught = $_ } $caught | Should -Not -BeNullOrEmpty $caught.FullyQualifiedErrorId | Should -Match "^$ErrorId" } + + It ' sets $? false and lands in -ErrorVariable' -TestCases $script:PreFlightCases { + # ThrowTerminatingError ends the statement, not the script, so $? and -ErrorVariable are + # readable from the statement after it - which a catch would have hidden. Both reads have + # to happen in the session that ran the command, so the probe runs in a fresh runspace: + # under Pester the same error takes the whole test body with it, and $? outside the probe + # would report the wrapper rather than the cmdlet. + $probe = [powershell]::Create() + try { + $script = @' +Import-Module -Name '__MANIFEST__' -Force +$ev = $null +__COMMAND__ -ErrorVariable ev 2>$null +[pscustomobject]@{ Ok = $?; Collected = @($ev) } +'@ + $null = $probe.AddScript( + $script.Replace('__MANIFEST__', $script:Paths.ManifestPath).Replace('__COMMAND__', $Command)) + $result = $probe.Invoke()[0] + } + finally { $probe.Dispose() } + + $result.Ok | Should -BeFalse + $result.Collected.Count | Should -Be 1 + + # A terminating error reaches -ErrorVariable as the CmdletInvocationException carrying the + # record, not as the record itself. + $collected = $result.Collected[0] + $record = if ($collected -is [System.Management.Automation.ErrorRecord]) { $collected } + else { $collected.ErrorRecord } + $record.FullyQualifiedErrorId | Should -Match "^$ErrorId" + } } Describe 'about_Mgx_Errors vocabulary' { diff --git a/tests/Unit/Mgx.HelpFreshness.Tests.ps1 b/tests/Unit/Mgx.HelpFreshness.Tests.ps1 index c0eb0da..edc13d5 100644 --- a/tests/Unit/Mgx.HelpFreshness.Tests.ps1 +++ b/tests/Unit/Mgx.HelpFreshness.Tests.ps1 @@ -41,6 +41,7 @@ Describe 'Compiled help matches its markdown source' { @{ Text = 'ODataType'; Why = '2.0.0 replaced PSObjects with hashtables; @odata.type is a key now' } @{ Text = 'instead of PSObjects'; Why = '-Raw returns raw JSON instead of hashtables' } @{ Text = 'ephemeral and deleted on success'; Why = 'Sync-MgxDelta gained -CheckpointPath in 2.1.0' } + @{ Text = 'belongs to a chunk that was refused'; Why = 'after a chunk failure the batch-level retry pass is skipped for every candidate, including ones a cleanly answered chunk left failing' } ) $raw = Get-Content $script:Maml -Raw $hits = $stale | Where-Object { $raw -match [regex]::Escape($_.Text) } diff --git a/tests/benchmarks/14-pacing-cold-cost.ps1 b/tests/benchmarks/14-pacing-cold-cost.ps1 index 43e544a..9c5d0da 100644 --- a/tests/benchmarks/14-pacing-cold-cost.ps1 +++ b/tests/benchmarks/14-pacing-cold-cost.ps1 @@ -54,4 +54,6 @@ $null = $ids | Invoke-MgxRequest -Uri '/users/{id}?$select=id' -Concurrency $Con $sw.Stop() $t = Get-MgxTelemetry -"RESULT|{0}|{1}|{2}" -f $sw.Elapsed.TotalSeconds, $t.Requests, $t.AdaptivePacingWaitMs +# The parent arm parses this back with [double], which is invariant; format it the same way +# or a comma decimal separator reads as a thousands group and the wall clock is 10000x out. +"RESULT|{0}|{1}|{2}" -f $sw.Elapsed.TotalSeconds.ToString([cultureinfo]::InvariantCulture), $t.Requests, $t.AdaptivePacingWaitMs From be1740a25b7f3f12924254fb2ad173aaffb12386 Mon Sep 17 00:00:00 2001 From: gromedev <61883549+gromedev@users.noreply.github.com> Date: Mon, 14 Sep 2026 13:49:50 +0200 Subject: [PATCH 06/10] mgx 2.1.5 --- .github/workflows/build-and-test.yml | 22 +- .gitignore | 6 + CHANGELOG.md | 49 + Directory.Build.props | 2 +- build.ps1 | 188 +- module/en-US/Mgx.Cmdlets.dll-Help.xml | 135 +- module/en-US/about_Mgx_Tuning.help.txt | 2 +- module/help/Disable-MgxResilience.md | 2 +- module/help/Export-MgxCollection.md | 2 +- module/help/Invoke-MgxBatchRequest.md | 47 +- module/help/Invoke-MgxRequest.md | 10 +- module/help/Set-MgxOption.md | 35 +- module/help/Sync-MgxDelta.md | 6 +- module/mgx.psd1 | 22 +- src/Mgx.Cmdlets/AlcInitializer.cs | 42 +- src/Mgx.Cmdlets/Base/MgxCmdletBase.cs | 2261 ++++++++++++- .../Cmdlets/Batch/InvokeMgxBatchRequest.cs | 824 ++++- .../Configuration/DisableMgxResilience.cs | 41 +- .../Configuration/EnableMgxResilience.cs | 32 +- .../Cmdlets/Configuration/SetMgxOption.cs | 56 +- src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs | 687 +++- .../Cmdlets/Export/ExportMgxCollection.cs | 792 ++++- src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs | 81 +- src/Mgx.Cmdlets/Mgx.Cmdlets.csproj | 16 +- src/Mgx.Engine/Http/GraphBatchClient.cs | 16 +- src/Mgx.Engine/Http/GraphRequestTracer.cs | 73 +- src/Mgx.Engine/Http/ResilientGraphClient.cs | 8 +- .../Http/ResilientGraphClientOptions.cs | 11 +- src/Mgx.Engine/Http/SensitiveNames.cs | 105 + src/Mgx.Engine/Mgx.Engine.csproj | 2 +- src/Mgx.Engine/Pagination/DeltaState.cs | 77 +- src/Mgx.Engine/Pagination/HeldEntry.cs | 301 ++ .../Pagination/PaginationCheckpoint.cs | 13 +- src/Mgx.Engine/Pagination/ScratchName.cs | 502 +++ tests/CORPUS.md | 66 - tests/Ecosystem/Mgx.Ecosystem.Tests.ps1 | 2 +- tests/Live/Mgx.Live.Tests.ps1 | 24 + .../Base/TypeCacheInvalidationTests.cs | 39 + .../Cmdlets/DeltaCheckpointIntegrityTests.cs | 106 + .../Cmdlets/DeltaQueryTests.cs | 8 +- .../Cmdlets/DeltaRecoveryTests.cs | 33 +- .../Cmdlets/DeltaSharedCheckpointTests.cs | 2978 +++++++++++++++-- .../Cmdlets/ExpandRelationBufferTests.cs | 86 + .../Cmdlets/ExportCheckpointIntegrityTests.cs | 109 + .../ExportDegradedQueryCheckpointTests.cs | 206 +- .../Cmdlets/ExportSharedCheckpointTests.cs | 2458 ++++++++++++++ .../Cmdlets/ExportWhatIfCheckpointTests.cs | 627 ++++ .../NamedTempPromotionOwnershipTests.cs | 449 ++- .../Cmdlets/OrphanTempOwnershipTests.cs | 171 +- .../Cmdlets/RequestWhatIfCheckpointTests.cs | 574 ++++ .../Cmdlets/ResilienceWrapTests.cs | 305 ++ .../Cmdlets/SetOptionGuardTests.cs | 339 ++ .../Cmdlets/StalledBodyTests.cs | 21 - .../Engine/Batch/BatchWriteTests.cs | 1432 +++++++- .../Engine/Batch/BulkWriteTests.cs | 24 +- .../Engine/Batch/FaultInjectionBatchTests.cs | 306 ++ .../Engine/Batch/PartialBatchResultTests.cs | 105 + .../Engine/Errors/FaultInjectionCatalog.cs | 482 +++ .../Engine/Http/BodyRedactionTests.cs | 249 ++ .../Engine/Http/ConfigurableOptionsTests.cs | 34 +- .../Http/FaultInjectionDownloadTests.cs | 189 ++ .../Engine/Http/RetryTests.cs | 393 +++ .../Pagination/AtomicSaveHygieneTests.cs | 274 ++ .../Engine/Pagination/BoundarySizeTests.cs | 194 +- .../Pagination/CheckpointRetentionTests.cs | 13 +- .../Pagination/FaultInjectionFanOutTests.cs | 400 +++ .../FaultInjectionPaginationTests.cs | 534 +++ .../Engine/Pagination/HeldEntryTests.cs | 149 + .../Pagination/NamedTempPromotionTests.cs | 1691 +++++++++- .../Engine/Pagination/OrphanAdoptionTests.cs | 17 +- .../Engine/SdkVersionTests.cs | 22 +- .../Fakes/MockHttpHandler.cs | 96 +- .../Fakes/MockResponseRules.cs | 311 ++ .../Fakes/PagedCollection.cs | 147 + .../Fakes/RecordingHost.cs | 88 + .../Fakes/StallingContent.cs | 93 + .../Fakes/StubHttpMessageHandler.cs | 74 +- .../TestSetup/ModuleAssemblyClosureTests.cs | 360 ++ .../TestSetup/RegressionCorpusIndexTests.cs | 94 +- tests/TestHarness.psm1 | 17 +- .../Benchmarks/Mgx.BenchBaseline.Tests.ps1 | 882 +++++ .../Mgx.BenchClampSummary.Tests.ps1 | 145 + .../Benchmarks/Mgx.BenchLoadShape.Tests.ps1 | 175 + .../Unit/Benchmarks/Mgx.BenchResult.Tests.ps1 | 373 +++ .../Benchmarks/Mgx.BenchWatchdog.Tests.ps1 | 65 + tests/Unit/Benchmarks/Mgx.MockPlan.Tests.ps1 | 131 + tests/Unit/Mgx.Behaviour.Tests.ps1 | 398 +-- tests/Unit/Mgx.Harness.Tests.ps1 | 147 + tests/Unit/Mgx.HelpFreshness.Tests.ps1 | 536 ++- tests/benchmarks/06-fault-gauntlet.ps1 | 39 +- .../10-pacing-under-real-throttling.ps1 | 168 +- tests/benchmarks/13-resource-unit-rate.ps1 | 23 + tests/benchmarks/16-pathological-gauntlet.ps1 | 330 ++ .../17-pathological-environment.ps1 | 336 ++ tests/benchmarks/18-spo-latency-clamp.ps1 | 427 +++ tests/benchmarks/README.md | 28 +- tests/benchmarks/baseline.json | 227 ++ tests/benchmarks/common.ps1 | 1570 ++++++++- tests/benchmarks/mock-graph-server.ps1 | 480 ++- tests/benchmarks/run.ps1 | 64 +- .../tenant-resilience-comparison.ps1 | 102 - 101 files changed, 27714 insertions(+), 1789 deletions(-) create mode 100644 src/Mgx.Engine/Http/SensitiveNames.cs create mode 100644 src/Mgx.Engine/Pagination/HeldEntry.cs create mode 100644 src/Mgx.Engine/Pagination/ScratchName.cs delete mode 100644 tests/CORPUS.md create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/ExpandRelationBufferTests.cs create mode 100644 tests/Mgx.IntegrationTests/Cmdlets/RequestWhatIfCheckpointTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Batch/FaultInjectionBatchTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Errors/FaultInjectionCatalog.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Http/FaultInjectionDownloadTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Pagination/FaultInjectionFanOutTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Pagination/FaultInjectionPaginationTests.cs create mode 100644 tests/Mgx.IntegrationTests/Engine/Pagination/HeldEntryTests.cs create mode 100644 tests/Mgx.IntegrationTests/Fakes/MockResponseRules.cs create mode 100644 tests/Mgx.IntegrationTests/Fakes/PagedCollection.cs create mode 100644 tests/Mgx.IntegrationTests/Fakes/RecordingHost.cs create mode 100644 tests/Mgx.IntegrationTests/Fakes/StallingContent.cs create mode 100644 tests/Mgx.IntegrationTests/TestSetup/ModuleAssemblyClosureTests.cs create mode 100644 tests/Unit/Benchmarks/Mgx.BenchBaseline.Tests.ps1 create mode 100644 tests/Unit/Benchmarks/Mgx.BenchClampSummary.Tests.ps1 create mode 100644 tests/Unit/Benchmarks/Mgx.BenchLoadShape.Tests.ps1 create mode 100644 tests/Unit/Benchmarks/Mgx.BenchResult.Tests.ps1 create mode 100644 tests/Unit/Benchmarks/Mgx.BenchWatchdog.Tests.ps1 create mode 100644 tests/Unit/Benchmarks/Mgx.MockPlan.Tests.ps1 create mode 100644 tests/Unit/Mgx.Harness.Tests.ps1 create mode 100644 tests/benchmarks/16-pathological-gauntlet.ps1 create mode 100644 tests/benchmarks/17-pathological-environment.ps1 create mode 100644 tests/benchmarks/18-spo-latency-clamp.ps1 create mode 100644 tests/benchmarks/baseline.json delete mode 100644 tests/benchmarks/tenant-resilience-comparison.ps1 diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 130a3ad..6d3b1b8 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -58,7 +58,7 @@ jobs: - name: Assert xUnit actually ran shell: pwsh run: | - $floor = 2995 + $floor = 3380 $trx = Get-ChildItem TestResults -Filter *.trx | Select-Object -First 1 if (-not $trx) { throw "No .trx produced - the test run did not report results." } $c = ([xml](Get-Content $trx.FullName -Raw)).TestRun.ResultSummary.Counters @@ -82,13 +82,18 @@ jobs: Import-Module ./tests/TestHarness.psm1 $result = Invoke-TestHarness -IgnoreCodeCoverage "Pester executed $($result.TotalCount): $($result.PassedCount) passed, $($result.FailedCount) failed, $($result.SkippedCount) skipped" | Write-Host - if ($result.FailedCount -gt 0) { - throw "$($result.FailedCount) Pester test(s) failed." + # FailedCount alone misses a container that never ran - a parse error or a top-level + # throw fails the container with FailedCount still 0, so Result and + # FailedContainersCount are checked too, and both gates on this leg (and the compat + # leg's twin below) trip on the same condition. + if ($result.FailedCount -gt 0 -or $result.FailedContainersCount -gt 0 -or $result.Result -ne 'Passed') { + $failed = $result.FailedContainers | ForEach-Object { $_.Name } + throw "$($result.FailedCount) Pester test(s) failed, $($result.FailedContainersCount) container(s) failed: $($failed -join ', ')" } # Silence is not success here either. This floor keeps a small margin rather than # sitting on the measured total, because the number it has to hold on is the one the # compat leg below reports and that host is not this one. Raise both together. - $floor = 240 + $floor = 333 $ran = [int]$result.PassedCount + [int]$result.SkippedCount if ($ran -lt $floor) { throw "Only $ran Pester test(s) ran, expected at least $floor. The suite is not running." @@ -178,10 +183,15 @@ jobs: Import-Module ./tests/TestHarness.psm1 -Force $r = Invoke-TestHarness -IgnoreCodeCoverage "$($r.PassedCount) passed, $($r.FailedCount) failed, $($r.SkippedCount) skipped" | Write-Host - if ($r.FailedCount -gt 0) { throw "$($r.FailedCount) test(s) failed" } + # Same condition as the main leg's Pester gate: FailedCount alone misses a container + # that never ran at all. + if ($r.FailedCount -gt 0 -or $r.FailedContainersCount -gt 0 -or $r.Result -ne 'Passed') { + $failed = $r.FailedContainers | ForEach-Object { $_.Name } + throw "$($r.FailedCount) test(s) failed, $($r.FailedContainersCount) container(s) failed: $($failed -join ', ')" + } # The same margin the main leg carries, for the same reason: this is the host the # Pester total has to be true on, and nothing measures it outside this job. - $floor = 240 + $floor = 333 $ran = [int]$r.PassedCount + [int]$r.SkippedCount if ($ran -lt $floor) { throw "Only $ran test(s) ran on this host, expected at least $floor." diff --git a/.gitignore b/.gitignore index f8d8919..0d6023c 100644 --- a/.gitignore +++ b/.gitignore @@ -22,5 +22,11 @@ module/.build-hash tests/TestResults.xml tests/benchmarks/results/* +# results/ above stays local: it is the append-only per-run log, and one machine's numbers are +# not another's. The baseline beside it is the pinned snapshot a later run is read against, and +# it is committed, so it has to be re-included after *.json has had its say. Nothing here +# re-includes results/ itself. +!tests/benchmarks/baseline.json + # Generated by the CI compat job when run locally compat-surface-tests.ps1 diff --git a/CHANGELOG.md b/CHANGELOG.md index dfd44d7..13e632d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,54 @@ # Changelog +## 2.1.5 + +### Fixed + +- `Export-MgxCollection` and `Sync-MgxDelta` stop a second run resuming the same interrupted operation rather than interleaving both runs' rows in one output file. +- Resumed runs hold their output file open until the last write, so a concurrent run over the same command line cannot take the file in between. +- Concurrent promotions over one output file on macOS or Linux no longer both succeed; the second stops and the first run's rows survive. +- Recovering an interrupted run removes a link at the staging name rather than following it, no longer hangs on a pipe there, and stops on a directory naming it. +- Resume checkpoints whose temp file is held by another run or cannot be opened stop the run before anything is written rather than re-enumerating over the file. +- Output files the account cannot open, or a directory in their place, stop a resume naming the reason and keep the checkpoint rather than reporting the items missing. +- Failed copies during recovery stop the run with both files left as found instead of reporting the temp file missing and starting over. +- Resume checkpoints from before 2.1 refused by another run keep their temp file, and `Sync-MgxDelta` recovers that shape as `Export-MgxCollection` does. +- Resume stops report what proceeding would have done to which file, and unopenable files are described by the open's own reason rather than always as a permission. +- `Sync-MgxDelta` no longer refuses its own retry after an expired delta token as another sync's. +- `Export-MgxCollection -WhatIf` on an interrupted export reports the stop the run would reach rather than previewing an append, and previews nothing it would not do. +- `Invoke-MgxRequest` with several values piped into a `-Uri` without a placeholder asks the checkpoint question once, as documented. +- `Set-MgxOption` with no option parameter no longer previews or prompts for a change it does not make, and `-Reset` beside an option reports the option ignored. +- `Invoke-MgxBatchRequest` refuses an item whose body cannot be read for redaction and sends the rest rather than ending the run. +- Dead-letter lines no longer carry pre-authenticated URLs with their secrets, whether as the item URL, a bare body, or a value inside one. +- Relative pre-authenticated URLs in batch bodies are cut from the dead-letter file as absolute ones are. +- Unwritable dead-letter files are reported as errors before the batch is sent, and a batch that fails nothing leaves no empty file. +- Batch item errors and session telemetry survive runs ended by `-WarningAction Stop` or `-ErrorAction Stop`. +- Dead-letter lines carrying the redaction marker in any value are refused on re-pipe before the first chunk is sent, with the refusal naming what it found. +- `-DeadLetterPath` on a non-file-system provider or an empty path is refused rather than writing into the working directory. +- `Invoke-MgxBatchRequest -WhatIf` reports the count to be sent and the items it would refuse. +- Batch outcome warnings read as one sentence when bodies were withheld or the dead-letter file failed. +- Two batches sharing one dead-letter path no longer lose each other's lines when the clean one finishes first. +- `-Debug` traces and the dead-letter file redact the same names and URLs, with the remaining differences documented. +- `Enable-MgxResilience -WhatIf` previews without touching the session, and `Disable-MgxResilience -WhatIf` names the action it would take. +- Removing the module restores every `Set-MgxOption` setting, the endpoint, and the telemetry counters to their defaults, and a re-import reads the endpoint from the session. +- The circuit-breaker minimum `Set-MgxOption` accepts matches the minimum the breaker accepts. + +### Added + +- Added a committed benchmark baseline (`tests/benchmarks/baseline.json`) and a comparison run reporting each scenario against it. +- Added a programmable fault plan for the mock Graph server, a pathological gauntlet, and auth-replacement and module-interference benchmarks. +- Added a SharePoint drive latency measurement that refuses to summarize a run whose clocks disagree. +- Added fault injection at every point of an operation and pagination checks at page-size boundaries. +- Added a help-freshness test comparing the compiled help with its markdown source. +- Added a Pester CI gate that fails on a test file that could not run. + +### Changed + +- A general tidy of the tests tree. + +### Documentation + +- Updated help documentation with what `-WhatIf` skips, what the dead-letter file holds, and which routes hold the output open, and completed every SYNTAX block. + ## 2.1.4 # Changelog diff --git a/Directory.Build.props b/Directory.Build.props index b8d8598..680d97e 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,7 +1,7 @@ - 2.1.4 + 2.1.5 Thomas Maillo Grome Mgx (c) 2026 Thomas Maillo Grome. All rights reserved. diff --git a/build.ps1 b/build.ps1 index fdcf96a..dfee59c 100644 --- a/build.ps1 +++ b/build.ps1 @@ -1,23 +1,165 @@ # Build script for Mgx PowerShell module # Compiles both projects and stages output into module/ directory +[CmdletBinding()] param( [ValidateSet('Debug', 'Release')] - [string]$Configuration = 'Release' + [string]$Configuration = 'Release', + + # Where the module is staged. Defaults to this repo's own module/, so a plain + # ./build.ps1 behaves exactly as before. Pointing it elsewhere - a fork's rebranded + # build, a scratch install - never touches the repo's module/. + [string]$ModuleRoot = (Join-Path $PSScriptRoot 'module'), + + # The manifest/psm1/format-file base name. Only matters together with -ModuleRoot: + # the repository's own module/ is always mgx, so renaming without redirecting throws. + [string]$ModuleName = 'mgx' ) $ErrorActionPreference = 'Stop' -$ModuleRoot = Join-Path $PSScriptRoot 'module' + +# Trim trailing separators before comparing or joining: "-ModuleRoot ./module/" would otherwise +# compare unequal to the repo's own module/ (they differ by a trailing slash), take the copy +# path below, and die copying mgx.psd1 onto itself. Windows and macOS default volumes fold case, +# so two spellings of the same path there name the same directory; Linux volumes are +# case-sensitive, so only there does a differently-cased path mean a different directory. +$ModuleRoot = [System.IO.Path]::GetFullPath($ModuleRoot, $PSScriptRoot).TrimEnd( + [System.IO.Path]::DirectorySeparatorChar, [System.IO.Path]::AltDirectorySeparatorChar) +$RepoModuleRoot = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot 'module')).TrimEnd( + [System.IO.Path]::DirectorySeparatorChar, [System.IO.Path]::AltDirectorySeparatorChar) +$PathComparison = if ($IsLinux) { [StringComparison]::Ordinal } else { [StringComparison]::OrdinalIgnoreCase } + +# Spelling isn't identity: on macOS /tmp is a symlink to /private/tmp, so +# "-ModuleRoot /tmp/x/module" and this repo's own "/private/tmp/x/module" normalize to two +# different strings that are nonetheless the same directory - the path compare below then +# misses the collision, takes the "renamed copy" branch, and dies "Cannot overwrite the item +# ... with itself" copying mgx.psd1 onto itself. Device+inode name a directory the way the +# filesystem does, independent of which symlinked spelling reached it, so prefer that when +# both directories already exist and we're not on Windows (whose Get-Item exposes no +# UnixStat). Fall back to the path compare where either side doesn't exist yet (a fresh +# -ModuleRoot has nothing to stat) or UnixStat itself is unavailable (older PowerShell, an +# unusual provider). A -ModuleRoot that is itself a symlink to module/ needs the same care: +# Get-Item -LiteralPath on a link stats the link's own inode, not the directory it points to, +# so each side is resolved to its final link target before its UnixStat is read. +$IsRepoModuleRoot = $null +if (-not $IsWindows -and (Test-Path -LiteralPath $ModuleRoot) -and (Test-Path -LiteralPath $RepoModuleRoot)) { + $moduleRootItem = Get-Item -LiteralPath $ModuleRoot + if ($moduleRootItem.LinkType) { + $moduleRootTarget = $moduleRootItem.ResolveLinkTarget($true) + $moduleRootItem = if ($moduleRootTarget -and (Test-Path -LiteralPath $moduleRootTarget.FullName)) { Get-Item -LiteralPath $moduleRootTarget.FullName } else { $null } + } + $repoModuleRootItem = Get-Item -LiteralPath $RepoModuleRoot + if ($repoModuleRootItem.LinkType) { + $repoModuleRootTarget = $repoModuleRootItem.ResolveLinkTarget($true) + $repoModuleRootItem = if ($repoModuleRootTarget -and (Test-Path -LiteralPath $repoModuleRootTarget.FullName)) { Get-Item -LiteralPath $repoModuleRootTarget.FullName } else { $null } + } + $moduleRootStat = if ($moduleRootItem) { $moduleRootItem.UnixStat } else { $null } + $repoModuleRootStat = if ($repoModuleRootItem) { $repoModuleRootItem.UnixStat } else { $null } + if ($moduleRootStat -and $repoModuleRootStat) { + $IsRepoModuleRoot = ($moduleRootStat.DeviceId -eq $repoModuleRootStat.DeviceId) -and + ($moduleRootStat.Inode -eq $repoModuleRootStat.Inode) + } +} +if ($null -eq $IsRepoModuleRoot) { + $IsRepoModuleRoot = [string]::Equals($ModuleRoot, $RepoModuleRoot, $PathComparison) +} + +# A -ModuleRoot that is itself a link needs to be refused here, before anything is created +# or cleaned: a dangling link's target doesn't exist, and a link to a plain file isn't a +# directory either way. Left unchecked, the copy branch below dies partway through with +# "Could not find a part of the path" - accurate about the symptom, silent about the cause. +$moduleRootLinkItem = Get-Item -LiteralPath $ModuleRoot -ErrorAction SilentlyContinue +if ($moduleRootLinkItem -and $moduleRootLinkItem.LinkType) { + $moduleRootLinkTarget = $moduleRootLinkItem.ResolveLinkTarget($true) + $targetExists = $moduleRootLinkTarget -and (Test-Path -LiteralPath $moduleRootLinkTarget.FullName) + $targetDisplay = if ($moduleRootLinkTarget) { $moduleRootLinkTarget.FullName } else { $moduleRootLinkItem.Target } + $reason = if (-not $targetExists) { + 'does not exist' + } elseif (-not (Test-Path -LiteralPath $moduleRootLinkTarget.FullName -PathType Container)) { + 'is not a directory' + } + if ($reason) { + throw "-ModuleRoot '$ModuleRoot' is a link to '$targetDisplay', which $reason. " + + "Point -ModuleRoot at a directory, or create the target first." + } +} + +# An existing plain file - not a link, the case just above - reaches the same copy branch +# below and dies with the same raw "Could not find a part of the path" the link check exists +# to replace. +if (Test-Path -LiteralPath $ModuleRoot -PathType Leaf) { + throw "-ModuleRoot '$ModuleRoot' is a file, not a directory. Point -ModuleRoot at a directory." +} + +if ($ModuleName -ne 'mgx' -and $IsRepoModuleRoot) { + throw "-ModuleName only applies with -ModuleRoot: the repository's own module/ is always mgx. " + + "Pass -ModuleRoot to stage a renamed module." +} + +# $ModuleName becomes a file name two blocks down (Join-Path $ModuleRoot "$ModuleName.psd1"): +# an empty string stages a hidden .psd1 that "succeeds" and imports nothing, and a separator +# like 'a/b' makes Copy-Item treat it as a subdirectory nothing created, dying "Could not find +# a part of the path" with no mention of -ModuleName anywhere in the message. A name built out +# of the punctuation alone passes the pattern and is no better: '.' stages '..psd1', '..psm1' +# and '..Format.ps1xml' and imports 12 commands under a module whose name is a single dot, and +# '_' and '-' name modules nobody asked for the same way. One letter or digit somewhere in it +# is what makes the staged files something a caller can ask for by name. +if ($ModuleName -notmatch '^[A-Za-z0-9._-]+$' -or $ModuleName -notmatch '[A-Za-z0-9]') { + throw "-ModuleName '$ModuleName' is not a module base name. Give a plain name such as 'mgx' or 'fork'." +} + $DepsDir = Join-Path $ModuleRoot 'Dependencies' Write-Host "Building Mgx ($Configuration)..." -ForegroundColor Cyan +# A ModuleRoot outside the repo starts empty. Stage the manifest, psm1, format file and +# about_* help topics from the repo's own module/ under $ModuleName's file names, so the +# version gate below, the integrity check, and Test-ModuleManifest afterward all have +# something to read. Copied verbatim: what a renamed manifest's RootModule or +# FormatsToProcess says is the source's concern, and the default name keeps them +# consistent. New-ExternalHelp regenerates the MAML later; it does not write these. +if (-not $IsRepoModuleRoot) { + New-Item $ModuleRoot -ItemType Directory -Force | Out-Null + Copy-Item (Join-Path $RepoModuleRoot 'mgx.psd1') (Join-Path $ModuleRoot "$ModuleName.psd1") -Force + Copy-Item (Join-Path $RepoModuleRoot 'mgx.psm1') (Join-Path $ModuleRoot "$ModuleName.psm1") -Force + Copy-Item (Join-Path $RepoModuleRoot 'mgx.Format.ps1xml') (Join-Path $ModuleRoot "$ModuleName.Format.ps1xml") -Force + + if ($ModuleName -ne 'mgx') { + # The copy above is byte-for-byte, so the staged manifest still declares RootModule = + # 'mgx.psm1' and FormatsToProcess = @('mgx.Format.ps1xml') while the files sitting beside + # it are named $ModuleName.*. Import-Module resolves RootModule against a file that was + # never staged and fails "Cannot find path ... mgx.psm1" - far from build.ps1 and with no + # mention of -ModuleName. Rewrite just those two values by text substitution on the staged + # copy only: Update-ModuleManifest reformats the whole file (key order, comments, quoting) + # for what should be a two-value change. + $stagedManifestPath = Join-Path $ModuleRoot "$ModuleName.psd1" + $stagedManifestText = Get-Content -LiteralPath $stagedManifestPath -Raw + $rootModulePattern = "(?m)^(\s*RootModule\s*=\s*)'mgx\.psm1'" + $formatsPattern = "(?m)^(\s*FormatsToProcess\s*=\s*)@\('mgx\.Format\.ps1xml'\)" + if ($stagedManifestText -notmatch $rootModulePattern -or $stagedManifestText -notmatch $formatsPattern) { + throw "Module integrity check failed: $stagedManifestPath no longer has the RootModule = " + + "'mgx.psm1' / FormatsToProcess = @('mgx.Format.ps1xml') lines this rewrite expects, " + + "so it would still import as mgx." + } + $stagedManifestText = $stagedManifestText -replace $rootModulePattern, "`${1}'$ModuleName.psm1'" + $stagedManifestText = $stagedManifestText -replace $formatsPattern, "`${1}@('$ModuleName.Format.ps1xml')" + Set-Content -LiteralPath $stagedManifestPath -Value $stagedManifestText -NoNewline + } + + $repoHelpEnUs = Join-Path $RepoModuleRoot 'en-US' + if (Test-Path $repoHelpEnUs) { + $helpEnUs = Join-Path $ModuleRoot 'en-US' + New-Item $helpEnUs -ItemType Directory -Force | Out-Null + Get-ChildItem $repoHelpEnUs -Filter 'about_*.help.txt' | Copy-Item -Destination $helpEnUs -Force + } +} + # Version gate: MgxSdkVersion derives the SdkVersion header from the assembly version set in # Directory.Build.props, so the header can no longer drift from the code on its own (it said # 0.3.0 for three releases while the module shipped 1.0.x). The manifest ModuleVersion is # still a separate declaration, so the one remaining mismatch - props vs manifest - is # checked before the compile, failing in a second instead of a minute. -$manifestVersion = (Import-PowerShellDataFile (Join-Path $ModuleRoot 'mgx.psd1')).ModuleVersion +$manifestVersion = (Import-PowerShellDataFile (Join-Path $ModuleRoot "$ModuleName.psd1")).ModuleVersion $propsFile = Join-Path $PSScriptRoot 'Directory.Build.props' # Select-Object -First on a filtered list: .Project.PropertyGroup.Version returns an ARRAY the # moment a second top-level exists, and comparing an array to a string does not @@ -28,10 +170,12 @@ if (-not $propsVersion) { throw "Version gate failed: no found in $propsFile" } if ($propsVersion -ne $manifestVersion) { - throw "Version gate failed: Directory.Build.props is '$propsVersion' but mgx.psd1 " + + throw "Version gate failed: Directory.Build.props is '$propsVersion' but $ModuleName.psd1 " + "ModuleVersion is '$manifestVersion'. Update one to match the other." } -Write-Host "Version gate: mgx/$propsVersion matches manifest" -ForegroundColor DarkGray +# "mgx/" names the wire header (MgxSdkVersion) sent with every request; that is the SDK's own +# name and does not change with -ModuleName - only the manifest file being checked does. +Write-Host "Version gate: mgx/$propsVersion matches $ModuleName.psd1" -ForegroundColor DarkGray # Clean previous build artifacts if (Test-Path $DepsDir) { Remove-Item $DepsDir -Recurse -Force } @@ -42,7 +186,7 @@ Get-ChildItem $ModuleRoot -Filter '*.pdb' | Remove-Item -Force Get-ChildItem $ModuleRoot -Filter '*.deps.json' | Remove-Item -Force # Build the solution -dotnet build "$PSScriptRoot/Mgx.slnx" -c $Configuration --nologo +dotnet build "$PSScriptRoot/Mgx.slnx" -c $Configuration --nologo "-p:ModuleRoot=$ModuleRoot" if ($LASTEXITCODE -ne 0) { throw "Build failed with exit code $LASTEXITCODE" } # Copy Cmdlets + Engine DLLs (load in default ALC) @@ -63,7 +207,7 @@ if (Test-Path $depsJson) { Write-Warning "Mgx.Cmdlets.deps.json not found in build output - diagnostics may be limited" } -# Copy third-party dependencies into Dependencies/ (loaded via ALC Resolving handler on first use) +# Copy dependencies into Dependencies/ (loaded via ALC Resolving handler on first use) # Polly.Core and System.Threading.RateLimiting are not in the module root; # Mgx.Engine is NOT here - it is loaded via RequiredAssemblies in mgx.psd1 (see comment there). New-Item $DepsDir -ItemType Directory -Force | Out-Null @@ -71,9 +215,15 @@ New-Item $DepsDir -ItemType Directory -Force | Out-Null # Copy dependency DLLs that need ALC isolation (Polly, RateLimiting) # After D1+D2: replaced Microsoft.Extensions.Http.Resilience with direct Polly.Core 8.6.6 # This eliminated 28 transitive dependencies (51 total down to 23 packages) +# +# System.IO.Pipelines is here for a different reason: nothing in mgx compiles against it. It is +# in the closure of the System.Text.Json a PowerShell 7.5 or newer host supplies, and net8.0 - +# what src/ targets - does not include it in Microsoft.NETCore.App, so a host that does not +# supply it has nowhere to resolve it from unless the module carries it. See Mgx.Cmdlets.csproj. $DepsToIsolate = @( 'Polly.Core.dll' 'System.Threading.RateLimiting.dll' + 'System.IO.Pipelines.dll' ) foreach ($dep in $DepsToIsolate) { @@ -86,8 +236,8 @@ foreach ($dep in $DepsToIsolate) { } # Verify module output is in expected state -$RequiredRoot = @('Mgx.Cmdlets.dll', 'Mgx.Engine.dll', 'mgx.psd1', 'mgx.psm1') -$RequiredDeps = @('Polly.Core.dll', 'System.Threading.RateLimiting.dll') +$RequiredRoot = @('Mgx.Cmdlets.dll', 'Mgx.Engine.dll', "$ModuleName.psd1", "$ModuleName.psm1") +$RequiredDeps = @('Polly.Core.dll', 'System.Threading.RateLimiting.dll', 'System.IO.Pipelines.dll') foreach ($f in $RequiredRoot) { if (-not (Test-Path (Join-Path $ModuleRoot $f))) { @@ -104,6 +254,24 @@ if ($orphans) { throw "Module integrity check failed: Mgx assemblies found in Dependencies/ (should only be in root): $($orphans.Name -join ', ')" } +# The checks above only prove the required files exist - never that Import-Module can actually +# load them. A staged manifest whose RootModule or FormatsToProcess still names the wrong file +# (see the rewrite above) passes every Test-Path check here and only fails once a caller runs +# Import-Module for real, with a path error that says nothing about build.ps1 or -ModuleName. +# Import the staged manifest for real, in a fresh process, so it sees exactly what that caller +# would. The try/catch is load-bearing, not decoration: a bad RootModule or FormatsToProcess +# fails manifest validation through a WriteError that -ErrorAction Stop turns terminating only +# for that statement - pwsh's own exit code stays 0 and execution falls through to +# Get-Command unless the catch below turns it into an explicit exit 1. Emitting $_ as a plain +# expression (not Write-Error) keeps the caught message free of the ANSI codes PowerShell's +# default error formatting would otherwise embed in it. +$stagedPsd1 = Join-Path $ModuleRoot "$ModuleName.psd1" +$importResult = & pwsh -NoProfile -NonInteractive -c "try { Import-Module '$stagedPsd1' -ErrorAction Stop; (Get-Command -Module '$ModuleName').Count } catch { `$_.Exception.Message; exit 1 }" 2>&1 +if ($LASTEXITCODE -ne 0) { + throw "Module integrity check failed: Import-Module '$stagedPsd1' failed in a fresh shell: $importResult" +} +Write-Host "Imported ${ModuleName}: $importResult commands" -ForegroundColor DarkGray + # Compiled help is generated from module/help/*.md, and nothing regenerated it: the shipped # MAML drifted two releases behind its own source, so Get-Help documented output shapes removed # in 2.0.0, denied a parameter added in 2.1.0, and gave a range the cmdlet rejects. Regenerating @@ -129,6 +297,6 @@ if (Test-Path $helpSource) { Write-Host "`nBuild complete!" -ForegroundColor Green Write-Host "Module output: $ModuleRoot" -ForegroundColor Yellow Write-Host "`nTo use:" -ForegroundColor Cyan -Write-Host " Import-Module '$ModuleRoot/mgx.psd1'" -ForegroundColor White +Write-Host " Import-Module '$ModuleRoot/$ModuleName.psd1'" -ForegroundColor White Write-Host " Connect-MgGraph -Scopes 'User.Read.All'" -ForegroundColor White Write-Host " Invoke-MgxRequest /users -All" -ForegroundColor White diff --git a/module/en-US/Mgx.Cmdlets.dll-Help.xml b/module/en-US/Mgx.Cmdlets.dll-Help.xml index 725ab84..6c96a93 100644 --- a/module/en-US/Mgx.Cmdlets.dll-Help.xml +++ b/module/en-US/Mgx.Cmdlets.dll-Help.xml @@ -10,7 +10,7 @@ - Disable-MgxResilience removes the Polly resilience injection from the Microsoft.Graph SDK's HTTP transport, restoring the original SDK HttpClient that was saved by Enable-MgxResilience. + Disable-MgxResilience removes the Polly resilience injection from the Microsoft.Graph SDK's HTTP transport, restoring the original SDK HttpClient that was saved by Enable-MgxResilience. A client the SDK replaced after the injection went on - a Connect-MgGraph to another tenant builds one - is not this module's to restore and is left as it was found, with a warning saying so; the injection's own state comes off either way. After calling this cmdlet, SDK cmdlets revert to their default retry behavior. @@ -759,7 +759,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP CheckpointPath - Path to a JSON checkpoint file for resumable exports. Deleted on successful completion. + Path to a JSON checkpoint file for resumable exports. Deleted on successful completion. A run stops - with nothing written to the output, the checkpoint or a temp - when the temp file the checkpoint names, or the output it would append to or replace, is open in another export or cannot be opened for writing (a permission, a directory or a pipe at the path, a link that cannot be followed, a path too long for the file system, a socket, a directory above it this account cannot search), or when the copy it stages before promoting a temp fails, in which case both files are left as found: the rows it counts are in that file and this checkpoint is the only thing that counts them, and exporting anyway would write over both. This holds whether the run would append, promote a temp, or export from the beginning after its temp vanished. The copy a promotion stages is written beside the output under a name the run creates itself, and the name is cleared first: a link, a pipe, a socket, a copy left by an interrupted promotion or a copy this account cannot open standing there is deleted and the removal reported as a warning, while a directory or a copy another run holds stops the run instead. A -WhatIf pass removes nothing: where something stands at that name it says what the run would remove there and leaves it, and where the name is clear it makes the same create at no length before taking it back - so a preview stops where the run would and deletes nothing on the way. Wait for that export to finish, give this run its own -OutputFile and -CheckpointPath, or grant write access to that file - or to the directory holding it, where that is what the run names - and run again. A resumed or promoting run holds the output open for as long as it is appending, which is what stops a second run over the same command line; a run recovering a checkpoint older than 2.1 opens the output as it always has. On Linux and macOS a PowerShell reader of a held file fails with a sharing violation until the run ends, while tail and cat read it as before. String @@ -975,7 +975,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP CheckpointPath - Path to a JSON checkpoint file for resumable exports. Deleted on successful completion. + Path to a JSON checkpoint file for resumable exports. Deleted on successful completion. A run stops - with nothing written to the output, the checkpoint or a temp - when the temp file the checkpoint names, or the output it would append to or replace, is open in another export or cannot be opened for writing (a permission, a directory or a pipe at the path, a link that cannot be followed, a path too long for the file system, a socket, a directory above it this account cannot search), or when the copy it stages before promoting a temp fails, in which case both files are left as found: the rows it counts are in that file and this checkpoint is the only thing that counts them, and exporting anyway would write over both. This holds whether the run would append, promote a temp, or export from the beginning after its temp vanished. The copy a promotion stages is written beside the output under a name the run creates itself, and the name is cleared first: a link, a pipe, a socket, a copy left by an interrupted promotion or a copy this account cannot open standing there is deleted and the removal reported as a warning, while a directory or a copy another run holds stops the run instead. A -WhatIf pass removes nothing: where something stands at that name it says what the run would remove there and leaves it, and where the name is clear it makes the same create at no length before taking it back - so a preview stops where the run would and deletes nothing on the way. Wait for that export to finish, give this run its own -OutputFile and -CheckpointPath, or grant write access to that file - or to the directory holding it, where that is what the run names - and run again. A resumed or promoting run holds the output open for as long as it is appending, which is what stops a second run over the same command line; a run recovering a checkpoint older than 2.1 opens the output as it always has. On Linux and macOS a PowerShell reader of a held file fails with a sharing violation until the run ends, while tail and cat read it as before. String @@ -1974,7 +1974,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP Body Request body for all requests when piping string URLs. Ignored when pipeline input carries its own Body member. - Bodies follow the same serialization contract as Invoke-MgxRequest -Body. An item whose body serialization refuses a value (a SecureString, NaN) fails on its own, like an item whose body is not valid JSON; the rest of the batch is sent. + Bodies follow the same serialization contract as Invoke-MgxRequest -Body. An item whose body serialization refuses a value (a SecureString, NaN) fails on its own, like an item whose body is not valid JSON; the rest of the batch is sent. So does an item whose body the redaction check cannot read: a raw-string body carrying an unpaired surrogate escape - `{"displayName":"Jos\ud83d"}` - is a document the JSON parser accepts and the check cannot read a string out of, and the check is what decides whether a body may be sent, so that item is refused naming the reason and the rest of the batch is sent. Object @@ -1986,7 +1986,12 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP DeadLetterPath - Path to a JSONL file where the batch's failed and never-sent items are appended: anything the server answered with a status >= 400, the items of a chunk whose own POST was refused, which carry that refusal's status, and the items of chunks that were never POSTed, which carry status 0. Nothing the server confirmed is ever written, so the file holds only work that is still outstanding. Each line contains Url, Method, Body (with sensitive fields redacted), Status, Error, and Timestamp. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Read it to decide what to resubmit, and supply the sensitive fields again yourself. A refused item's request did reach the server, so it may have been applied without an answer coming back; resending one is a decision about duplicates rather than a free retry. + Path to a JSONL file where the batch's failed and never-sent items are appended: anything the server answered with a status >= 400, the items of a chunk whose own POST failed, which carry that failure's status, and the items of chunks that were never POSTed, which carry status 0. Nothing the server confirmed is ever written, so the file holds only work that is still outstanding. Each line holds Timestamp, Url, Method and Status, then Body where the request had one, with sensitive fields redacted, then Error where the item's own response carried an error message: a never-sent item has no response to read one from, and neither does an item of a refused chunk the server did not answer, so their lines end at Body. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Any field whose name carries password, secret, credential, key, token, assertion, passphrase or connectionstring is replaced by a redaction marker, and so is any field whose name carries downloadUrl, because a pre-authenticated URL fetches the file with no token of its own. A value that is a URL carrying a capability parameter - `sig`, `tempauth`, `guestaccesstoken`, `authkey`, `X-Amz-Signature` - is cut after that parameter and the rest replaced by the marker, the way a -Debug trace cuts it, so the host and the parameter name still say which URL failed; an ordinary link is left whole, because `@odata.nextLink` and a `webUrl` are half of what the line is read for. Where the URL itself names the secret - resetPassword, uploadSecret, addKey, synchronization/secrets - the whole Body is that marker, because those bodies hold the credential under a name that says nothing. A body the redaction cannot walk at all - one naming the same property twice is such a body - gets the marker whole for the same reason, and the run writes a warning naming the item whose body was withheld. What stands in the file is: + + ***REDACTED*** + + Read it to decide what to resubmit, and supply the sensitive fields again yourself. Re-piping the file - `Get-Content dead.jsonl | ConvertFrom-Json | Invoke-MgxBatchRequest` - resends the lines that carry no marker, and refuses the ones that do as an error record before the first chunk leaves: a Body that is the marker whole is not JSON, and a Body carrying it in a field parses but is no longer the body that was sent, so that one is refused naming the field. Any body carrying that marker text is refused the same way before the first chunk leaves, whether it came from the file or not. Under -ErrorAction Stop - which this cmdlet's description recommends - the pipeline then ends with nothing sent at all: re-pipe at the default error preference, or filter the marker lines out first. A refused item carries that status whether its POST reached the server and went unanswered or never left at all - an open circuit, a rate limiter with no permit for it - and the status does not separate the two. Read it as the first: the write may have been applied, so resending one is a decision about duplicates rather than a free retry. Only status 0 says nothing was sent. + The file is opened before the first request goes out and held open for the run, so a path this run cannot write - a directory standing at it, a parent directory that is not there, a file this account may not write - is refused with nothing sent at all, and the error says so: fix the path and run the same command again. It has to be a file system path with something in it. `Env:\X` resolves to a bare name relative to nothing and an empty path resolves to the working directory, so both are refused rather than written somewhere nobody named. A run with no failed and no never-sent items has no lines for the file, and one it created for them is removed again, so a batch that succeeded whole leaves nothing behind; a file that was already there holds an earlier run's outstanding work and is left as it was, and so is one a second run appending to the same path has put a line in. String @@ -2006,6 +2011,18 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP False + + ConsistencyLevel + + ConsistencyLevel header added to each individual batch item. Required when any batch item URL contains $search (Graph advanced query capabilities). Takes precedence over the same key in -Headers. Source: Advanced query capabilities on Microsoft Entra ID objects (https://learn.microsoft.com/en-us/graph/aad-advanced-queries) + + String + + String + + + None + Headers @@ -2058,7 +2075,8 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP WhatIf - Shows what would happen if the cmdlet runs. The cmdlet is not run. + Shows what would happen if the cmdlet runs. The cmdlet is not run. The gate covers reads as well as writes: an all-GET batch is described and not sent, because even a read batch spends resource units, can be throttled, and emits objects into the pipeline. Invoke-MgxRequest makes the other choice and sends its reads under -WhatIf. + The count it names is what would be sent. Bodies are validated and checked for the redaction marker before the gate, so an item either of those refuses is left out of that count and named beside it - `PATCH 1 request via $batch; 2 refused` - and each refusal is written as the error record it is, under -WhatIf as without it: a refusal is not an action the gate governs. SwitchParameter @@ -2097,7 +2115,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP Body Request body for all requests when piping string URLs. Ignored when pipeline input carries its own Body member. - Bodies follow the same serialization contract as Invoke-MgxRequest -Body. An item whose body serialization refuses a value (a SecureString, NaN) fails on its own, like an item whose body is not valid JSON; the rest of the batch is sent. + Bodies follow the same serialization contract as Invoke-MgxRequest -Body. An item whose body serialization refuses a value (a SecureString, NaN) fails on its own, like an item whose body is not valid JSON; the rest of the batch is sent. So does an item whose body the redaction check cannot read: a raw-string body carrying an unpaired surrogate escape - `{"displayName":"Jos\ud83d"}` - is a document the JSON parser accepts and the check cannot read a string out of, and the check is what decides whether a body may be sent, so that item is refused naming the reason and the rest of the batch is sent. Object @@ -2109,7 +2127,12 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP DeadLetterPath - Path to a JSONL file where the batch's failed and never-sent items are appended: anything the server answered with a status >= 400, the items of a chunk whose own POST was refused, which carry that refusal's status, and the items of chunks that were never POSTed, which carry status 0. Nothing the server confirmed is ever written, so the file holds only work that is still outstanding. Each line contains Url, Method, Body (with sensitive fields redacted), Status, Error, and Timestamp. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Read it to decide what to resubmit, and supply the sensitive fields again yourself. A refused item's request did reach the server, so it may have been applied without an answer coming back; resending one is a decision about duplicates rather than a free retry. + Path to a JSONL file where the batch's failed and never-sent items are appended: anything the server answered with a status >= 400, the items of a chunk whose own POST failed, which carry that failure's status, and the items of chunks that were never POSTed, which carry status 0. Nothing the server confirmed is ever written, so the file holds only work that is still outstanding. Each line holds Timestamp, Url, Method and Status, then Body where the request had one, with sensitive fields redacted, then Error where the item's own response carried an error message: a never-sent item has no response to read one from, and neither does an item of a refused chunk the server did not answer, so their lines end at Body. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Any field whose name carries password, secret, credential, key, token, assertion, passphrase or connectionstring is replaced by a redaction marker, and so is any field whose name carries downloadUrl, because a pre-authenticated URL fetches the file with no token of its own. A value that is a URL carrying a capability parameter - `sig`, `tempauth`, `guestaccesstoken`, `authkey`, `X-Amz-Signature` - is cut after that parameter and the rest replaced by the marker, the way a -Debug trace cuts it, so the host and the parameter name still say which URL failed; an ordinary link is left whole, because `@odata.nextLink` and a `webUrl` are half of what the line is read for. Where the URL itself names the secret - resetPassword, uploadSecret, addKey, synchronization/secrets - the whole Body is that marker, because those bodies hold the credential under a name that says nothing. A body the redaction cannot walk at all - one naming the same property twice is such a body - gets the marker whole for the same reason, and the run writes a warning naming the item whose body was withheld. What stands in the file is: + + ***REDACTED*** + + Read it to decide what to resubmit, and supply the sensitive fields again yourself. Re-piping the file - `Get-Content dead.jsonl | ConvertFrom-Json | Invoke-MgxBatchRequest` - resends the lines that carry no marker, and refuses the ones that do as an error record before the first chunk leaves: a Body that is the marker whole is not JSON, and a Body carrying it in a field parses but is no longer the body that was sent, so that one is refused naming the field. Any body carrying that marker text is refused the same way before the first chunk leaves, whether it came from the file or not. Under -ErrorAction Stop - which this cmdlet's description recommends - the pipeline then ends with nothing sent at all: re-pipe at the default error preference, or filter the marker lines out first. A refused item carries that status whether its POST reached the server and went unanswered or never left at all - an open circuit, a rate limiter with no permit for it - and the status does not separate the two. Read it as the first: the write may have been applied, so resending one is a decision about duplicates rather than a free retry. Only status 0 says nothing was sent. + The file is opened before the first request goes out and held open for the run, so a path this run cannot write - a directory standing at it, a parent directory that is not there, a file this account may not write - is refused with nothing sent at all, and the error says so: fix the path and run the same command again. It has to be a file system path with something in it. `Env:\X` resolves to a bare name relative to nothing and an empty path resolves to the working directory, so both are refused rather than written somewhere nobody named. A run with no failed and no never-sent items has no lines for the file, and one it created for them is removed again, so a batch that succeeded whole leaves nothing behind; a file that was already there holds an earlier run's outstanding work and is left as it was, and so is one a second run appending to the same path has put a line in. String @@ -2130,14 +2153,14 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP False - + ConsistencyLevel ConsistencyLevel header added to each individual batch item. Required when any batch item URL contains $search (Graph advanced query capabilities). Takes precedence over the same key in -Headers. Source: Advanced query capabilities on Microsoft Entra ID objects (https://learn.microsoft.com/en-us/graph/aad-advanced-queries) - + String - + String None @@ -2194,7 +2217,8 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP WhatIf - Shows what would happen if the cmdlet runs. The cmdlet is not run. + Shows what would happen if the cmdlet runs. The cmdlet is not run. The gate covers reads as well as writes: an all-GET batch is described and not sent, because even a read batch spends resource units, can be throttled, and emits objects into the pipeline. Invoke-MgxRequest makes the other choice and sends its reads under -WhatIf. + The count it names is what would be sent. Bodies are validated and checked for the redaction marker before the gate, so an item either of those refuses is left out of that count and named beside it - `PATCH 1 request via $batch; 2 refused` - and each refusal is written as the error record it is, under -WhatIf as without it: a refusal is not an action the gate governs. SwitchParameter @@ -2232,7 +2256,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP System.Collections.Hashtable - Per-request results with Url, Method, Status, and Body keys. Status says what became of the operation: an item the server answered keeps the status it answered with; an item whose chunk was POSTed and then refused carries the refusal's status (>= 400), because its request went out and may have been applied; only an item in a chunk that was never POSTed gets status 0, and only those carry NotSent - another chunk of the batch failed, not necessarily one before it, since chunks can run in parallel. Body here is the RESPONSE body - for a failure that is the error envelope, not the request - so piping results straight back resubmits the wrong thing. Use Url and Method to rebuild the requests you want to retry. + Per-request results with Url, Method, Status, and Body keys. Status says what became of the operation: an item the server answered keeps the status it answered with; an item whose chunk was refused carries the refusal's status (>= 400), which covers both a POST that went out unanswered and one an open circuit or a rate limiter stopped before it left, so read it as a write that may have been applied; only an item in a chunk that was never POSTed gets status 0, and only those carry NotSent - another chunk of the batch failed, not necessarily one before it, since chunks can run in parallel. Body here is the RESPONSE body - for a failure that is the error envelope, not the request - so piping results straight back resubmits the wrong thing. Use Url and Method to rebuild the requests you want to retry. @@ -2322,9 +2346,12 @@ $users | ForEach-Object { "/users/$($_.id)" } | Invoke-MgxBatchRequest -Method D ------ Example 8: Capture failures to a dead-letter file ------ 1..1000 | ForEach-Object { [PSCustomObject]@{ Url = "/users"; Method = "POST"; Body = @{ displayName = "User-$_"; mailNickname = "user$_"; userPrincipalName = "user$_@contoso.com"; passwordProfile = @{ password = "P@ss$(Get-Random -Minimum 10000)!" } } } -} | Invoke-MgxBatchRequest -DeadLetterPath ./failed-users.jsonl +} | Invoke-MgxBatchRequest -DeadLetterPath ./failed-users.jsonl + +# The redacted password field, as failed-users.jsonl holds it: +# "passwordProfile":"***REDACTED***" - Creates 1000 users via batch. Any failures (status >= 400) are appended to the JSONL dead-letter file with Url, Method, Body (passwords redacted), Status, and Error. Sensitive fields like passwordProfile are automatically replaced with *REDACTED *. + Creates 1000 users via batch. Anything the server answered with a status >= 400 is appended to the JSONL dead-letter file, and so is anything that was never sent, which carries status 0; each line holds Timestamp, Url, Method, Status and Body, then Error where the item's own response carried an error message - a never-sent item has no response to read one from, and neither does an item of a refused chunk the server did not answer, so those lines end at Body. The Body is redacted by field name, by field value and by URL: a field whose name carries password, secret, credential, key, token, assertion, passphrase or connectionstring is replaced by the marker shown above, which is what passwordProfile gets here; a value that is a pre-authenticated URL is cut after its capability parameter, the way a -Debug trace cuts it; and a request whose own path names the secret it carries - resetPassword, uploadSecret, addKey - has its whole Body replaced by that marker instead, as does an item whose body could not be read for redaction. @@ -2427,7 +2454,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } CheckpointPath - Path to a JSON checkpoint file for resumable pagination. On interruption (Ctrl+C), progress is saved. On re-run with the same checkpoint path, pagination resumes from where it left off. The checkpoint file is deleted on successful completion. + Path to a JSON checkpoint file for resumable pagination. Consulted only by a collection read carrying -All, -Top, -Skip, -Filter, -Search, -Sort or -CountVariable; any other request - a plain collection GET, one with only -PageSize, a single-entity GET, a fan-out over two or more piped IDs, a write - binds it and ignores it. A single piped ID is not a fan-out: it is sent as a direct request and, on a collection read, consults the checkpoint like any other. On interruption (Ctrl+C), progress is saved. On re-run with the same checkpoint path, pagination resumes from where it left off. The checkpoint file is deleted on successful completion. Those writes are gated on their own: -WhatIf reads the file and leaves it exactly as it was, and -Confirm asks about it once per run that consults it, including when several values are piped into a -Uri that carries no placeholder, which runs the request once per value. String @@ -2451,7 +2478,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Confirm - Prompts you for confirmation before running the cmdlet. + Prompts before a write and not before a read. POST, PATCH, PUT and DELETE are confirmed - a direct request one prompt at a time, a fan-out over piped IDs one prompt for all of its items - and a GET is sent without a prompt, the same division -WhatIf makes. With -CheckpointPath there is one prompt more, but only on a collection read carrying -All, -Top, -Skip, -Filter, -Search, -Sort or -CountVariable, asked once per run before any page is fetched, covering the saves and the deletes of the checkpoint file itself - including when several values are piped into a -Uri that carries no placeholder, which runs the request once per value; any other request - a plain collection GET, one with only -PageSize, a single-entity GET, a fan-out over two or more piped IDs, a write - binds the parameter and never reads it, so none of them adds a prompt. A single piped ID is not a fan-out: it is sent as a direct request and, on a collection read, prompts the same as any other. SwitchParameter @@ -2524,9 +2551,9 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Pipeline input for fan-out operations. Each value replaces the {id} placeholder in the URI template. Accepts string values or objects with an Id property. - String + Object - String + Object None @@ -2669,7 +2696,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } WhatIf - Shows what would happen if the cmdlet runs. The cmdlet is not run. + Describes the write instead of sending it. Only writes are gated: POST, PATCH, PUT and DELETE, single or fanned out over piped IDs, are described and not sent. Reads are sent - a GET under -WhatIf connects, pages, spends resource units, can be throttled, and emits its objects - which is the PowerShell convention and what makes a dry run under $WhatIfPreference useful, since the gated writes are then previewed against real data. The checkpoint at -CheckpointPath is read and then left alone: a preview resumes from the position it finds, so it sends the same reads the run would rather than paging from the start, and it neither saves over that file nor deletes it, so the resume position is exactly where it was. Invoke-MgxBatchRequest makes the other choice and gates reads too. SwitchParameter @@ -2732,7 +2759,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } CheckpointPath - Path to a JSON checkpoint file for resumable pagination. On interruption (Ctrl+C), progress is saved. On re-run with the same checkpoint path, pagination resumes from where it left off. The checkpoint file is deleted on successful completion. + Path to a JSON checkpoint file for resumable pagination. Consulted only by a collection read carrying -All, -Top, -Skip, -Filter, -Search, -Sort or -CountVariable; any other request - a plain collection GET, one with only -PageSize, a single-entity GET, a fan-out over two or more piped IDs, a write - binds it and ignores it. A single piped ID is not a fan-out: it is sent as a direct request and, on a collection read, consults the checkpoint like any other. On interruption (Ctrl+C), progress is saved. On re-run with the same checkpoint path, pagination resumes from where it left off. The checkpoint file is deleted on successful completion. Those writes are gated on their own: -WhatIf reads the file and leaves it exactly as it was, and -Confirm asks about it once per run that consults it, including when several values are piped into a -Uri that carries no placeholder, which runs the request once per value. String @@ -2756,7 +2783,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Confirm - Prompts you for confirmation before running the cmdlet. + Prompts before a write and not before a read. POST, PATCH, PUT and DELETE are confirmed - a direct request one prompt at a time, a fan-out over piped IDs one prompt for all of its items - and a GET is sent without a prompt, the same division -WhatIf makes. With -CheckpointPath there is one prompt more, but only on a collection read carrying -All, -Top, -Skip, -Filter, -Search, -Sort or -CountVariable, asked once per run before any page is fetched, covering the saves and the deletes of the checkpoint file itself - including when several values are piped into a -Uri that carries no placeholder, which runs the request once per value; any other request - a plain collection GET, one with only -PageSize, a single-entity GET, a fan-out over two or more piped IDs, a write - binds the parameter and never reads it, so none of them adds a prompt. A single piped ID is not a fan-out: it is sent as a direct request and, on a collection read, prompts the same as any other. SwitchParameter @@ -2830,9 +2857,9 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Pipeline input for fan-out operations. Each value replaces the {id} placeholder in the URI template. Accepts string values or objects with an Id property. - String + Object - String + Object None @@ -2984,7 +3011,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } WhatIf - Shows what would happen if the cmdlet runs. The cmdlet is not run. + Describes the write instead of sending it. Only writes are gated: POST, PATCH, PUT and DELETE, single or fanned out over piped IDs, are described and not sent. Reads are sent - a GET under -WhatIf connects, pages, spends resource units, can be throttled, and emits its objects - which is the PowerShell convention and what makes a dry run under $WhatIfPreference useful, since the gated writes are then previewed against real data. The checkpoint at -CheckpointPath is read and then left alone: a preview resumes from the position it finds, so it sends the same reads the run would rather than paging from the start, and it neither saves over that file nor deletes it, so the resume position is exactly where it was. Invoke-MgxBatchRequest makes the other choice and gates reads too. SwitchParameter @@ -3248,7 +3275,7 @@ Import-Csv ./users.csv | CircuitBreakerMinThroughput - Minimum number of requests in the sampling window before the circuit breaker evaluates failure ratio. Range: 1-1000. Default: 40. + Minimum number of requests in the sampling window before the circuit breaker evaluates failure ratio. Range: 2-1000. Default: 40. Two is the least the breaker will hold; a ratio over a single request is not a measurement. Int32 @@ -3374,6 +3401,28 @@ Import-Csv ./users.csv | 300 + + WhatIf + + Shows what would happen if the cmdlet runs. The cmdlet is not run. Every path that changes an option - -Reset and each individual parameter alike - sits behind this same check, so nothing is changed under -WhatIf. + + + SwitchParameter + + + False + + + Confirm + + Prompts you for confirmation before running the cmdlet. + + + SwitchParameter + + + False + @@ -3440,7 +3489,7 @@ Import-Csv ./users.csv | CircuitBreakerMinThroughput - Minimum number of requests in the sampling window before the circuit breaker evaluates failure ratio. Range: 1-1000. Default: 40. + Minimum number of requests in the sampling window before the circuit breaker evaluates failure ratio. Range: 2-1000. Default: 40. Two is the least the breaker will hold; a ratio over a single request is not a measurement. Int32 @@ -3569,6 +3618,30 @@ Import-Csv ./users.csv | 300 + + WhatIf + + Shows what would happen if the cmdlet runs. The cmdlet is not run. Every path that changes an option - -Reset and each individual parameter alike - sits behind this same check, so nothing is changed under -WhatIf. + + SwitchParameter + + SwitchParameter + + + False + + + Confirm + + Prompts you for confirmation before running the cmdlet. + + SwitchParameter + + SwitchParameter + + + False + @@ -3661,7 +3734,7 @@ Import-Csv ./users.csv | Delta tokens are saved to the file specified by -DeltaPath and persist across successful completions. -CheckpointPath is different: it is an ephemeral mid-run resume position, saved at page boundaries while an enumeration is in flight and deleted the moment it completes. Use both on long enumerations - a killed 3-hour drive sync resumes where it stopped instead of starting over. Drive delta is fully supported: `/me/drive/root/delta`, `/drives/{id}/root/delta`, `/users/{id}/drive/root/delta`, and `/groups/{id}/drive/root/delta` all work, including drive-specific `-Prefer` behaviors and the drive form of `-Latest`. Items that no longer match the query appear with an `@removed` property containing `{"reason": "changed"}` (item moved out of scope or soft-deleted) or `{"reason": "deleted"}` (permanently deleted). Filter these with `Where-Object { -not $_.'@removed' }`. The completion message counts them separately. - Delta tokens expire after approximately 7 days for directory objects (users, groups, applications). When a token expires, Graph returns HTTP 410 Gone. Sync-MgxDelta handles this automatically by deleting the stale token (and any resume checkpoint) and performing a full re-sync with a warning. + Delta tokens expire after approximately 7 days for directory objects (users, groups, applications). When a token expires, Graph returns HTTP 410 Gone. Sync-MgxDelta handles this automatically by deleting the stale token (and this run's own resume checkpoint) and performing a full re-sync with a warning. Resume semantics are at-least-once: in pipeline mode the page in flight at a crash is re-emitted in full on resume; in JSONL mode mid-page checkpoints keep the window to at most 500 items. Deduplicate on `id` downstream if exact-once matters. Source: Use delta query to track changes in Microsoft Graph data (https://learn.microsoft.com/en-us/graph/delta-query-overview) @@ -3699,7 +3772,8 @@ Import-Csv ./users.csv | CheckpointPath - Path for the ephemeral mid-run resume checkpoint. Saved at page boundaries (and every 500 items in JSONL mode) while an enumeration is in flight; deleted on successful completion. Any event that invalidates the enumeration - HTTP 410 Gone, -FullSync, or a -Property/-Filter/-Prefer change - deletes it too. Must differ from -DeltaPath and -OutputFile. + Path for the ephemeral mid-run resume checkpoint. Saved at page boundaries (and every 500 items in JSONL mode) while an enumeration is in flight; deleted on successful completion. Any event that invalidates the enumeration - HTTP 410 Gone, -FullSync, or a -Property/-Filter/-Prefer change - deletes it too. A completion and a 410 delete this run's own checkpoint; one they recognize as another sync's is left as it stands, for the sync that resumes from exactly it. Must differ from -DeltaPath and -OutputFile. + A run stops - with nothing written to the output, the checkpoint, a temp or the delta state - when the temp file the checkpoint names, or the output it would append to or replace, is open in another sync or cannot be opened for writing (a permission, a directory or a pipe at the path, a link that cannot be followed, a path too long for the file system, a socket, a directory above it this account cannot search), or when the copy it stages before promoting a temp fails, in which case both files are left as found: the changes it counts are in that file and this checkpoint is the only thing that counts them, and syncing anyway would write over both and advance the token past those changes. This holds whether the run would append, promote a temp, or re-enumerate from the last saved delta token after its temp vanished. The copy a promotion stages is written beside the output under a name the run creates itself, and the name is cleared first: a link, a pipe, a socket, a copy left by an interrupted promotion or a copy this account cannot open standing there is deleted and the removal reported as a warning, while a directory or a copy another run holds stops the run instead. Wait for that sync to finish, give this run its own -OutputFile and -CheckpointPath, or grant write access to that file - or to the directory holding it, where that is what the run names - and run again. A resumed or promoting run holds the output open for as long as it is appending, which is what stops a second run over the same command line; a run recovering a checkpoint older than 2.1 opens the output as it always has. On Linux and macOS a PowerShell reader of a held file fails with a sharing violation until the run ends, while tail and cat read it as before. String @@ -3844,7 +3918,8 @@ Import-Csv ./users.csv | CheckpointPath - Path for the ephemeral mid-run resume checkpoint. Saved at page boundaries (and every 500 items in JSONL mode) while an enumeration is in flight; deleted on successful completion. Any event that invalidates the enumeration - HTTP 410 Gone, -FullSync, or a -Property/-Filter/-Prefer change - deletes it too. Must differ from -DeltaPath and -OutputFile. + Path for the ephemeral mid-run resume checkpoint. Saved at page boundaries (and every 500 items in JSONL mode) while an enumeration is in flight; deleted on successful completion. Any event that invalidates the enumeration - HTTP 410 Gone, -FullSync, or a -Property/-Filter/-Prefer change - deletes it too. A completion and a 410 delete this run's own checkpoint; one they recognize as another sync's is left as it stands, for the sync that resumes from exactly it. Must differ from -DeltaPath and -OutputFile. + A run stops - with nothing written to the output, the checkpoint, a temp or the delta state - when the temp file the checkpoint names, or the output it would append to or replace, is open in another sync or cannot be opened for writing (a permission, a directory or a pipe at the path, a link that cannot be followed, a path too long for the file system, a socket, a directory above it this account cannot search), or when the copy it stages before promoting a temp fails, in which case both files are left as found: the changes it counts are in that file and this checkpoint is the only thing that counts them, and syncing anyway would write over both and advance the token past those changes. This holds whether the run would append, promote a temp, or re-enumerate from the last saved delta token after its temp vanished. The copy a promotion stages is written beside the output under a name the run creates itself, and the name is cleared first: a link, a pipe, a socket, a copy left by an interrupted promotion or a copy this account cannot open standing there is deleted and the removal reported as a warning, while a directory or a copy another run holds stops the run instead. Wait for that sync to finish, give this run its own -OutputFile and -CheckpointPath, or grant write access to that file - or to the directory holding it, where that is what the run names - and run again. A resumed or promoting run holds the output open for as long as it is appending, which is what stops a second run over the same command line; a run recovering a checkpoint older than 2.1 opens the output as it always has. On Linux and macOS a PowerShell reader of a held file fails with a sharing violation until the run ends, while tail and cat read it as before. String diff --git a/module/en-US/about_Mgx_Tuning.help.txt b/module/en-US/about_Mgx_Tuning.help.txt index 99e57f3..7143585 100644 --- a/module/en-US/about_Mgx_Tuning.help.txt +++ b/module/en-US/about_Mgx_Tuning.help.txt @@ -74,7 +74,7 @@ CIRCUIT BREAKER Setting Default Hard Max Purpose ------- ------- -------- ------- CircuitBreakerFailureRatio 10% 100% Failure ratio to trip - CircuitBreakerMinThroughput 40 1,000 Min requests before evaluating + CircuitBreakerMinThroughput 40 1,000 Requests before evaluating (2+) CircuitBreakerSamplingDurationSeconds 30s 300s Evaluation window CircuitBreakerDurationSeconds 15s 300s Open-state duration diff --git a/module/help/Disable-MgxResilience.md b/module/help/Disable-MgxResilience.md index 599fc49..56e337f 100644 --- a/module/help/Disable-MgxResilience.md +++ b/module/help/Disable-MgxResilience.md @@ -17,7 +17,7 @@ Disable-MgxResilience [-ProgressAction ] [-WhatIf] [-Confirm] ``` ## DESCRIPTION -Disable-MgxResilience removes the Polly resilience injection from the Microsoft.Graph SDK's HTTP transport, restoring the original SDK HttpClient that was saved by Enable-MgxResilience. +Disable-MgxResilience removes the Polly resilience injection from the Microsoft.Graph SDK's HTTP transport, restoring the original SDK HttpClient that was saved by Enable-MgxResilience. A client the SDK replaced after the injection went on - a Connect-MgGraph to another tenant builds one - is not this module's to restore and is left as it was found, with a warning saying so; the injection's own state comes off either way. After calling this cmdlet, SDK cmdlets revert to their default retry behavior. diff --git a/module/help/Export-MgxCollection.md b/module/help/Export-MgxCollection.md index 99e006d..b29a65c 100644 --- a/module/help/Export-MgxCollection.md +++ b/module/help/Export-MgxCollection.md @@ -86,7 +86,7 @@ Accept wildcard characters: False ``` ### -CheckpointPath -Path to a JSON checkpoint file for resumable exports. Deleted on successful completion. +Path to a JSON checkpoint file for resumable exports. Deleted on successful completion. A run stops - with nothing written to the output, the checkpoint or a temp - when the temp file the checkpoint names, or the output it would append to or replace, is open in another export or cannot be opened for writing (a permission, a directory or a pipe at the path, a link that cannot be followed, a path too long for the file system, a socket, a directory above it this account cannot search), or when the copy it stages before promoting a temp fails, in which case both files are left as found: the rows it counts are in that file and this checkpoint is the only thing that counts them, and exporting anyway would write over both. This holds whether the run would append, promote a temp, or export from the beginning after its temp vanished. The copy a promotion stages is written beside the output under a name the run creates itself, and the name is cleared first: a link, a pipe, a socket, a copy left by an interrupted promotion or a copy this account cannot open standing there is deleted and the removal reported as a warning, while a directory or a copy another run holds stops the run instead. A -WhatIf pass removes nothing: where something stands at that name it says what the run would remove there and leaves it, and where the name is clear it makes the same create at no length before taking it back - so a preview stops where the run would and deletes nothing on the way. Wait for that export to finish, give this run its own -OutputFile and -CheckpointPath, or grant write access to that file - or to the directory holding it, where that is what the run names - and run again. A resumed or promoting run holds the output open for as long as it is appending, which is what stops a second run over the same command line; a run recovering a checkpoint older than 2.1 opens the output as it always has. On Linux and macOS a PowerShell reader of a held file fails with a sharing violation until the run ends, while tail and cat read it as before. ```yaml Type: String diff --git a/module/help/Invoke-MgxBatchRequest.md b/module/help/Invoke-MgxBatchRequest.md index ba28885..9a5da17 100644 --- a/module/help/Invoke-MgxBatchRequest.md +++ b/module/help/Invoke-MgxBatchRequest.md @@ -14,8 +14,8 @@ Bundle multiple Graph API requests into /$batch calls. ``` Invoke-MgxBatchRequest [-Uri] [-Method ] [-Body ] [-ConsistencyLevel ] - [-Headers ] [-ThrottlePriority ] [-ApiVersion ] [-ProgressAction ] - [-WhatIf] [-Confirm] [] + [-Headers ] [-ThrottlePriority ] [-ApiVersion ] [-DeadLetterPath ] + [-ProgressAction ] [-WhatIf] [-Confirm] [] ``` ## DESCRIPTION @@ -110,9 +110,12 @@ Batches search queries that require the ConsistencyLevel header. 1..1000 | ForEach-Object { [PSCustomObject]@{ Url = "/users"; Method = "POST"; Body = @{ displayName = "User-$_"; mailNickname = "user$_"; userPrincipalName = "user$_@contoso.com"; passwordProfile = @{ password = "P@ss$(Get-Random -Minimum 10000)!" } } } } | Invoke-MgxBatchRequest -DeadLetterPath ./failed-users.jsonl + +# The redacted password field, as failed-users.jsonl holds it: +# "passwordProfile":"***REDACTED***" ``` -Creates 1000 users via batch. Any failures (status >= 400) are appended to the JSONL dead-letter file with Url, Method, Body (passwords redacted), Status, and Error. Sensitive fields like passwordProfile are automatically replaced with ***REDACTED***. +Creates 1000 users via batch. Anything the server answered with a status >= 400 is appended to the JSONL dead-letter file, and so is anything that was never sent, which carries status 0; each line holds Timestamp, Url, Method, Status and Body, then Error where the item's own response carried an error message - a never-sent item has no response to read one from, and neither does an item of a refused chunk the server did not answer, so those lines end at Body. The Body is redacted by field name, by field value and by URL: a field whose name carries password, secret, credential, key, token, assertion, passphrase or connectionstring is replaced by the marker shown above, which is what passwordProfile gets here; a value that is a pre-authenticated URL is cut after its capability parameter, the way a -Debug trace cuts it; and a request whose own path names the secret it carries - resetPassword, uploadSecret, addKey - has its whole Body replaced by that marker instead, as does an item whose body could not be read for redaction. ### Example 9: Pipeline usage at scale (recommended pattern) ```powershell @@ -147,7 +150,7 @@ Accept wildcard characters: False ### -Body Request body for all requests when piping string URLs. Ignored when pipeline input carries its own Body member. -Bodies follow the same serialization contract as Invoke-MgxRequest -Body. An item whose body serialization refuses a value (a SecureString, NaN) fails on its own, like an item whose body is not valid JSON; the rest of the batch is sent. +Bodies follow the same serialization contract as Invoke-MgxRequest -Body. An item whose body serialization refuses a value (a SecureString, NaN) fails on its own, like an item whose body is not valid JSON; the rest of the batch is sent. So does an item whose body the redaction check cannot read: a raw-string body carrying an unpaired surrogate escape - `{"displayName":"Jos\ud83d"}` - is a document the JSON parser accepts and the check cannot read a string out of, and the check is what decides whether a body may be sent, so that item is refused naming the reason and the rest of the batch is sent. ```yaml Type: Object @@ -162,7 +165,15 @@ Accept wildcard characters: False ``` ### -DeadLetterPath -Path to a JSONL file where the batch's failed and never-sent items are appended: anything the server answered with a status >= 400, the items of a chunk whose own POST was refused, which carry that refusal's status, and the items of chunks that were never POSTed, which carry status 0. Nothing the server confirmed is ever written, so the file holds only work that is still outstanding. Each line contains Url, Method, Body (with sensitive fields redacted), Status, Error, and Timestamp. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Read it to decide what to resubmit, and supply the sensitive fields again yourself. A refused item's request did reach the server, so it may have been applied without an answer coming back; resending one is a decision about duplicates rather than a free retry. +Path to a JSONL file where the batch's failed and never-sent items are appended: anything the server answered with a status >= 400, the items of a chunk whose own POST failed, which carry that failure's status, and the items of chunks that were never POSTed, which carry status 0. Nothing the server confirmed is ever written, so the file holds only work that is still outstanding. Each line holds Timestamp, Url, Method and Status, then Body where the request had one, with sensitive fields redacted, then Error where the item's own response carried an error message: a never-sent item has no response to read one from, and neither does an item of a refused chunk the server did not answer, so their lines end at Body. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Any field whose name carries password, secret, credential, key, token, assertion, passphrase or connectionstring is replaced by a redaction marker, and so is any field whose name carries downloadUrl, because a pre-authenticated URL fetches the file with no token of its own. A value that is a URL carrying a capability parameter - `sig`, `tempauth`, `guestaccesstoken`, `authkey`, `X-Amz-Signature` - is cut after that parameter and the rest replaced by the marker, the way a -Debug trace cuts it, so the host and the parameter name still say which URL failed; an ordinary link is left whole, because `@odata.nextLink` and a `webUrl` are half of what the line is read for. Where the URL itself names the secret - resetPassword, uploadSecret, addKey, synchronization/secrets - the whole Body is that marker, because those bodies hold the credential under a name that says nothing. A body the redaction cannot walk at all - one naming the same property twice is such a body - gets the marker whole for the same reason, and the run writes a warning naming the item whose body was withheld. What stands in the file is: + +``` +***REDACTED*** +``` + +Read it to decide what to resubmit, and supply the sensitive fields again yourself. Re-piping the file - `Get-Content dead.jsonl | ConvertFrom-Json | Invoke-MgxBatchRequest` - resends the lines that carry no marker, and refuses the ones that do as an error record before the first chunk leaves: a Body that is the marker whole is not JSON, and a Body carrying it in a field parses but is no longer the body that was sent, so that one is refused naming the field. Any body carrying that marker text is refused the same way before the first chunk leaves, whether it came from the file or not. Under -ErrorAction Stop - which this cmdlet's description recommends - the pipeline then ends with nothing sent at all: re-pipe at the default error preference, or filter the marker lines out first. A refused item carries that status whether its POST reached the server and went unanswered or never left at all - an open circuit, a rate limiter with no permit for it - and the status does not separate the two. Read it as the first: the write may have been applied, so resending one is a decision about duplicates rather than a free retry. Only status 0 says nothing was sent. + +The file is opened before the first request goes out and held open for the run, so a path this run cannot write - a directory standing at it, a parent directory that is not there, a file this account may not write - is refused with nothing sent at all, and the error says so: fix the path and run the same command again. It has to be a file system path with something in it. `Env:\X` resolves to a bare name relative to nothing and an empty path resolves to the working directory, so both are refused rather than written somewhere nobody named. A run with no failed and no never-sent items has no lines for the file, and one it created for them is removed again, so a batch that succeeded whole leaves nothing behind; a file that was already there holds an earlier run's outstanding work and is left as it was, and so is one a second run appending to the same path has put a line in. ```yaml Type: String @@ -194,11 +205,8 @@ Accept wildcard characters: False ### -ConsistencyLevel ConsistencyLevel header added to each individual batch item. Required when any batch item URL contains $search (Graph advanced query capabilities). Takes precedence over the same key in -Headers. Source: [Advanced query capabilities on Microsoft Entra ID objects](https://learn.microsoft.com/en-us/graph/aad-advanced-queries) -### -Headers -Custom headers applied to each individual batch item. Accepts a hashtable of key-value pairs. Merged with -ConsistencyLevel and -ThrottlePriority (dedicated parameters take precedence over matching keys in -Headers). - ```yaml -Type: Hashtable +Type: String Parameter Sets: (All) Aliases: @@ -209,16 +217,13 @@ Accept pipeline input: False Accept wildcard characters: False ``` -### -ThrottlePriority -Throttle priority hint for Graph API. Graph uses this to decide which requests to throttle first under pressure. Valid values: Low, Normal, High. - -Sets `x-ms-throttle-priority` header on each batch item. Use `Low` for background jobs that should yield to interactive workloads. +### -Headers +Custom headers applied to each individual batch item. Accepts a hashtable of key-value pairs. Merged with -ConsistencyLevel and -ThrottlePriority (dedicated parameters take precedence over matching keys in -Headers). ```yaml -Type: String +Type: Hashtable Parameter Sets: (All) Aliases: -Accepted values: Low, Normal, High Required: False Position: Named @@ -227,10 +232,16 @@ Accept pipeline input: False Accept wildcard characters: False ``` +### -ThrottlePriority +Throttle priority hint for Graph API. Graph uses this to decide which requests to throttle first under pressure. Valid values: Low, Normal, High. + +Sets `x-ms-throttle-priority` header on each batch item. Use `Low` for background jobs that should yield to interactive workloads. + ```yaml Type: String Parameter Sets: (All) Aliases: +Accepted values: Low, Normal, High Required: False Position: Named @@ -271,7 +282,9 @@ Accept wildcard characters: False ``` ### -WhatIf -Shows what would happen if the cmdlet runs. The cmdlet is not run. +Shows what would happen if the cmdlet runs. The cmdlet is not run. The gate covers reads as well as writes: an all-GET batch is described and not sent, because even a read batch spends resource units, can be throttled, and emits objects into the pipeline. Invoke-MgxRequest makes the other choice and sends its reads under -WhatIf. + +The count it names is what would be sent. Bodies are validated and checked for the redaction marker before the gate, so an item either of those refuses is left out of that count and named beside it - `PATCH 1 request via $batch; 2 refused` - and each refusal is written as the error record it is, under -WhatIf as without it: a refusal is not an action the gate governs. ```yaml Type: SwitchParameter @@ -311,7 +324,7 @@ String URLs, or hashtables or PSCustomObjects with Url, Method, and Body members ## OUTPUTS ### System.Collections.Hashtable -Per-request results with Url, Method, Status, and Body keys. Status says what became of the operation: an item the server answered keeps the status it answered with; an item whose chunk was POSTed and then refused carries the refusal's status (>= 400), because its request went out and may have been applied; only an item in a chunk that was never POSTed gets status 0, and only those carry NotSent - another chunk of the batch failed, not necessarily one before it, since chunks can run in parallel. Body here is the RESPONSE body - for a failure that is the error envelope, not the request - so piping results straight back resubmits the wrong thing. Use Url and Method to rebuild the requests you want to retry. +Per-request results with Url, Method, Status, and Body keys. Status says what became of the operation: an item the server answered keeps the status it answered with; an item whose chunk was refused carries the refusal's status (>= 400), which covers both a POST that went out unanswered and one an open circuit or a rate limiter stopped before it left, so read it as a write that may have been applied; only an item in a chunk that was never POSTed gets status 0, and only those carry NotSent - another chunk of the batch failed, not necessarily one before it, since chunks can run in parallel. Body here is the RESPONSE body - for a failure that is the error envelope, not the request - so piping results straight back resubmits the wrong thing. Use Url and Method to rebuild the requests you want to retry. ## NOTES Each batch item is retried individually on 429 (throttled) or 5xx errors (for idempotent methods). POST requests only retry on 429 because POST is non-idempotent - retrying a failed POST on 5xx could create duplicates if the server processed the request before the error. This matches the Kiota SDK retry behavior. Source: [Microsoft Graph error responses and resource types](https://learn.microsoft.com/en-us/graph/errors) diff --git a/module/help/Invoke-MgxRequest.md b/module/help/Invoke-MgxRequest.md index 37b75bd..546fa00 100644 --- a/module/help/Invoke-MgxRequest.md +++ b/module/help/Invoke-MgxRequest.md @@ -26,7 +26,7 @@ Invoke-MgxRequest [-Uri] [-Method ] [-Body ] [-Property Invoke-MgxRequest [-Uri] [-Method ] [-Body ] [-Property ] [-ExpandProperty ] [-ConsistencyLevel ] [-Headers ] [-ApiVersion ] [-Raw] [-Filter ] [-Sort ] [-Search ] [-Skip ] [-All] [-Top ] - [-PageSize ] [-CountVariable ] [-CheckpointPath ] [-NoPageSize] [-InputObject ] + [-PageSize ] [-CountVariable ] [-CheckpointPath ] [-NoPageSize] [-InputObject ] [-Concurrency ] [-SkipNotFound] [-SkipForbidden] [-ProgressAction ] [-WhatIf] [-Confirm] [] ``` @@ -198,7 +198,7 @@ Accept wildcard characters: False ``` ### -CheckpointPath -Path to a JSON checkpoint file for resumable pagination. On interruption (Ctrl+C), progress is saved. On re-run with the same checkpoint path, pagination resumes from where it left off. The checkpoint file is deleted on successful completion. +Path to a JSON checkpoint file for resumable pagination. Consulted only by a collection read carrying -All, -Top, -Skip, -Filter, -Search, -Sort or -CountVariable; any other request - a plain collection GET, one with only -PageSize, a single-entity GET, a fan-out over two or more piped IDs, a write - binds it and ignores it. A single piped ID is not a fan-out: it is sent as a direct request and, on a collection read, consults the checkpoint like any other. On interruption (Ctrl+C), progress is saved. On re-run with the same checkpoint path, pagination resumes from where it left off. The checkpoint file is deleted on successful completion. Those writes are gated on their own: -WhatIf reads the file and leaves it exactly as it was, and -Confirm asks about it once per run that consults it, including when several values are piped into a -Uri that carries no placeholder, which runs the request once per value. ```yaml Type: String @@ -228,7 +228,7 @@ Accept wildcard characters: False ``` ### -Confirm -Prompts you for confirmation before running the cmdlet. +Prompts before a write and not before a read. POST, PATCH, PUT and DELETE are confirmed - a direct request one prompt at a time, a fan-out over piped IDs one prompt for all of its items - and a GET is sent without a prompt, the same division -WhatIf makes. With -CheckpointPath there is one prompt more, but only on a collection read carrying -All, -Top, -Skip, -Filter, -Search, -Sort or -CountVariable, asked once per run before any page is fetched, covering the saves and the deletes of the checkpoint file itself - including when several values are piped into a -Uri that carries no placeholder, which runs the request once per value; any other request - a plain collection GET, one with only -PageSize, a single-entity GET, a fan-out over two or more piped IDs, a write - binds the parameter and never reads it, so none of them adds a prompt. A single piped ID is not a fan-out: it is sent as a direct request and, on a collection read, prompts the same as any other. ```yaml Type: SwitchParameter @@ -321,7 +321,7 @@ Accept wildcard characters: False Pipeline input for fan-out operations. Each value replaces the {id} placeholder in the URI template. Accepts string values or objects with an Id property. ```yaml -Type: String +Type: Object Parameter Sets: Pipeline Aliases: Id @@ -514,7 +514,7 @@ Accept wildcard characters: False ``` ### -WhatIf -Shows what would happen if the cmdlet runs. The cmdlet is not run. +Describes the write instead of sending it. Only writes are gated: POST, PATCH, PUT and DELETE, single or fanned out over piped IDs, are described and not sent. Reads are sent - a GET under -WhatIf connects, pages, spends resource units, can be throttled, and emits its objects - which is the PowerShell convention and what makes a dry run under $WhatIfPreference useful, since the gated writes are then previewed against real data. The checkpoint at -CheckpointPath is read and then left alone: a preview resumes from the position it finds, so it sends the same reads the run would rather than paging from the start, and it neither saves over that file nor deletes it, so the resume position is exactly where it was. Invoke-MgxBatchRequest makes the other choice and gates reads too. ```yaml Type: SwitchParameter diff --git a/module/help/Set-MgxOption.md b/module/help/Set-MgxOption.md index 6341a2d..d212769 100644 --- a/module/help/Set-MgxOption.md +++ b/module/help/Set-MgxOption.md @@ -17,7 +17,8 @@ Set-MgxOption [-RateLimitBurst ] [-RateLimitPerSecond ] [-NoRateLi [-RateLimitQueueLimit ] [-MaxRetryAfterSeconds ] [-MaxRetryAttempts ] [-TotalTimeoutSeconds ] [-AttemptTimeoutSeconds ] [-CircuitBreakerDurationSeconds ] [-CircuitBreakerFailureRatio ] [-CircuitBreakerMinThroughput ] - [-CircuitBreakerSamplingDurationSeconds ] [-BatchItemsPerSecond ] [-Reset] [] + [-CircuitBreakerSamplingDurationSeconds ] [-BatchChunkConcurrency ] + [-BatchItemsPerSecond ] [-Reset] [-WhatIf] [-Confirm] [] ``` ## DESCRIPTION @@ -133,7 +134,7 @@ Accept wildcard characters: False ``` ### -CircuitBreakerMinThroughput -Minimum number of requests in the sampling window before the circuit breaker evaluates failure ratio. Range: 1-1000. Default: 40. +Minimum number of requests in the sampling window before the circuit breaker evaluates failure ratio. Range: 2-1000. Default: 40. Two is the least the breaker will hold; a ratio over a single request is not a measurement. ```yaml Type: Int32 @@ -297,6 +298,36 @@ Accept pipeline input: False Accept wildcard characters: False ``` +### -WhatIf +Shows what would happen if the cmdlet runs. The cmdlet is not run. Every path that changes an option - -Reset and each individual parameter alike - sits behind this same check, so nothing is changed under -WhatIf. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: wi + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Confirm +Prompts you for confirmation before running the cmdlet. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: cf + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + ### CommonParameters This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see [about_CommonParameters](http://go.microsoft.com/fwlink/?LinkID=113216). diff --git a/module/help/Sync-MgxDelta.md b/module/help/Sync-MgxDelta.md index 4465403..4f9bf76 100644 --- a/module/help/Sync-MgxDelta.md +++ b/module/help/Sync-MgxDelta.md @@ -28,7 +28,7 @@ Drive delta is fully supported: `/me/drive/root/delta`, `/drives/{id}/root/delta Items that no longer match the query appear with an `@removed` property containing `{"reason": "changed"}` (item moved out of scope or soft-deleted) or `{"reason": "deleted"}` (permanently deleted). Filter these with `Where-Object { -not $_.'@removed' }`. The completion message counts them separately. -Delta tokens expire after approximately 7 days for directory objects (users, groups, applications). When a token expires, Graph returns HTTP 410 Gone. Sync-MgxDelta handles this automatically by deleting the stale token (and any resume checkpoint) and performing a full re-sync with a warning. +Delta tokens expire after approximately 7 days for directory objects (users, groups, applications). When a token expires, Graph returns HTTP 410 Gone. Sync-MgxDelta handles this automatically by deleting the stale token (and this run's own resume checkpoint) and performing a full re-sync with a warning. Resume semantics are at-least-once: in pipeline mode the page in flight at a crash is re-emitted in full on resume; in JSONL mode mid-page checkpoints keep the window to at most 500 items. Deduplicate on `id` downstream if exact-once matters. @@ -123,7 +123,9 @@ Accept wildcard characters: False ``` ### -CheckpointPath -Path for the ephemeral mid-run resume checkpoint. Saved at page boundaries (and every 500 items in JSONL mode) while an enumeration is in flight; deleted on successful completion. Any event that invalidates the enumeration - HTTP 410 Gone, -FullSync, or a -Property/-Filter/-Prefer change - deletes it too. Must differ from -DeltaPath and -OutputFile. +Path for the ephemeral mid-run resume checkpoint. Saved at page boundaries (and every 500 items in JSONL mode) while an enumeration is in flight; deleted on successful completion. Any event that invalidates the enumeration - HTTP 410 Gone, -FullSync, or a -Property/-Filter/-Prefer change - deletes it too. A completion and a 410 delete this run's own checkpoint; one they recognize as another sync's is left as it stands, for the sync that resumes from exactly it. Must differ from -DeltaPath and -OutputFile. + +A run stops - with nothing written to the output, the checkpoint, a temp or the delta state - when the temp file the checkpoint names, or the output it would append to or replace, is open in another sync or cannot be opened for writing (a permission, a directory or a pipe at the path, a link that cannot be followed, a path too long for the file system, a socket, a directory above it this account cannot search), or when the copy it stages before promoting a temp fails, in which case both files are left as found: the changes it counts are in that file and this checkpoint is the only thing that counts them, and syncing anyway would write over both and advance the token past those changes. This holds whether the run would append, promote a temp, or re-enumerate from the last saved delta token after its temp vanished. The copy a promotion stages is written beside the output under a name the run creates itself, and the name is cleared first: a link, a pipe, a socket, a copy left by an interrupted promotion or a copy this account cannot open standing there is deleted and the removal reported as a warning, while a directory or a copy another run holds stops the run instead. Wait for that sync to finish, give this run its own -OutputFile and -CheckpointPath, or grant write access to that file - or to the directory holding it, where that is what the run names - and run again. A resumed or promoting run holds the output open for as long as it is appending, which is what stops a second run over the same command line; a run recovering a checkpoint older than 2.1 opens the output as it always has. On Linux and macOS a PowerShell reader of a held file fails with a sharing violation until the run ends, while tail and cat read it as before. ```yaml Type: String diff --git a/module/mgx.psd1 b/module/mgx.psd1 index 1235670..9fee977 100644 --- a/module/mgx.psd1 +++ b/module/mgx.psd1 @@ -1,6 +1,6 @@ @{ RootModule = 'mgx.psm1' - ModuleVersion = '2.1.4' + ModuleVersion = '2.1.5' GUID = 'a3f7e8d2-5b4c-4a1e-9f6d-2c8b0e3a7d5f' Author = 'Thomas Maillo Grome' CompanyName = 'Mgx' @@ -52,19 +52,17 @@ LicenseUri = 'https://github.com/gromedev/mgx/blob/main/LICENSE' ProjectUri = 'https://github.com/gromedev/mgx' ReleaseNotes = @' -v2.1.4 +v2.1.5 Fixed -- Interrupted exports resume safely: unreadable, foreign, legacy, and case-differing checkpoints are handled without data loss or duplication. -- Batch items the server answered keep their results through chunk failures; the dead-letter file never contains an applied write. -- Batch summaries count only sent operations, and any chunk refusal reads as a failure. -- A stalled response or error body fails at the body-read timeout in the error contract's terms. -- A UTF-8 byte-order mark no longer discards fan-out, bulk-write, or expanded-relation results. -- Headers, telemetry ratios, and flags enum combinations render culture-invariantly with their attribute-supplied names. -- Removing the module restores the SDK's own client; re-enabling resilience never stacks wrappers. -- A resuming run never takes a file a live run is writing, and never resumes a checkpoint it gave up on. -- A caller's stop keeps the next retry off the wire; completed runs leave no abandoned partial files. +- Two runs resuming or recovering one interrupted export or sync no longer write into the same file: the second stops and names the file the first holds. +- Recovering an interrupted run never follows a link, waits on a pipe, or starts over on a copy that failed; every stop says what going on would have done and leaves both files as found. +- A checkpoint from before 2.1 keeps the temp file it stands for when another run refuses it, and Sync-MgxDelta recovers that shape as Export-MgxCollection does. +- Invoke-MgxBatchRequest refuses an item it cannot read instead of ending the run, cuts every pre-authenticated URL from the dead-letter file, checks the file before sending, and keeps item errors and telemetry under a Stop preference. +- Set-MgxOption previews and prompts only for a change it makes; Invoke-MgxRequest asks its checkpoint question once per run; -WhatIf previews on every cmdlet report what the run would do. +- Removing the module returns every option, the endpoint and the telemetry counters to their defaults. Added -- Ecosystem isolation matrix over Az.Accounts, PnP.PowerShell, and ExchangeOnlineManagement, with its own CI job. +- A committed benchmark baseline with a comparison run, a programmable fault plan for the mock Graph server, and pathological and interference gauntlets. +- Fault injection at every point of an operation, pagination checks at page-size boundaries, and a help-freshness test over the compiled help. See CHANGELOG.md for the full list. '@ } diff --git a/src/Mgx.Cmdlets/AlcInitializer.cs b/src/Mgx.Cmdlets/AlcInitializer.cs index 8760648..a53e9fe 100644 --- a/src/Mgx.Cmdlets/AlcInitializer.cs +++ b/src/Mgx.Cmdlets/AlcInitializer.cs @@ -1,6 +1,7 @@ using System.Management.Automation; using System.Reflection; using System.Runtime.Loader; +using Mgx.Engine.Http; namespace Mgx.Cmdlets; @@ -10,6 +11,12 @@ namespace Mgx.Cmdlets; /// msgraph-load-context) to avoid type identity conflicts. Only loads from /// the Dependencies folder for assemblies not found anywhere. /// Pattern adopted from Mge project's ALC coexistence investigation. +/// +/// Dependencies/ is not only mgx's own third-party packages. It also carries assemblies the +/// module never compiles against but a host may not supply - System.IO.Pipelines, which the +/// System.Text.Json of a newer PowerShell needs and net8.0's framework does not include. Those +/// arrive here by the same route: nothing has them loaded, so the fallback below is the only +/// answer, and an empty Dependencies/ is a FileNotFoundException at first use. /// public class AlcInitializer : IModuleAssemblyInitializer, IModuleAssemblyCleanup { @@ -17,9 +24,16 @@ public class AlcInitializer : IModuleAssemblyInitializer, IModuleAssemblyCleanup Path.GetDirectoryName(typeof(AlcInitializer).Assembly.Location)!, "Dependencies"); + // Whether ResolveDependency is subscribed. A process can import the module more than once, + // and += is not idempotent: the second subscription would outlive the -= a removal does and + // go on answering dependency loads out of this module's Dependencies folder for a session + // that no longer has the module. + private static int s_resolverHooked; + public void OnImport() { - AssemblyLoadContext.Default.Resolving += ResolveDependency; + if (Interlocked.Exchange(ref s_resolverHooked, 1) == 0) + AssemblyLoadContext.Default.Resolving += ResolveDependency; // Re-arms type-cache invalidation. The hook is attached from a static constructor that // has long since run by the time a second import happens, and removal detaches it, so @@ -95,7 +109,10 @@ public void OnRemove(PSModuleInfo module) System.Diagnostics.Debug.WriteLine($"[Mgx ALC] Cleanup on remove failed: {ex.Message}"); } - AssemblyLoadContext.Default.Resolving -= ResolveDependency; + // One -= per subscription, and the flag is what says there is one to take off: a removal + // that ran without an import behind it has nothing to detach. + if (Interlocked.Exchange(ref s_resolverHooked, 0) == 1) + AssemblyLoadContext.Default.Resolving -= ResolveDependency; // After the resolver: this only detaches an event handler, needs no dependency // resolution, and must not run before ResetHttpClient (which may trigger loads @@ -120,13 +137,32 @@ public void OnRemove(PSModuleInfo module) /// Disable-MgxResilience can still take it off - the restore here just makes that /// recovery unnecessary on the normal path. /// + /// + /// The Set-MgxOption surface goes with them, and so do the endpoint and the timeout a client + /// build derives, and the telemetry counters a session accumulates. They are the per-process + /// state a removal used to leave set: a fresh import started on the previous import's tuning, + /// with Get-MgxOption reporting it as current - a re-import that looks like a reset was not + /// one - the VersionedBaseUrl its cmdlets composed named the cloud the import before it had + /// connected to until its own first build read the session again, and Get-MgxTelemetry + /// answered for traffic the session had never sent. + /// /// internal static void ReleaseStaticState() { Base.MgxCmdletBase.ResetHttpClient(); + Base.MgxCmdletBase.SetClientOptions(ResilientGraphClientOptions.Default); + Base.MgxCmdletBase.ReleaseEndpointAndTimeout(); + + // Same shape again: the counters are one process's running total, so a removal that + // leaves them set has Get-MgxTelemetry crediting a fresh import with the requests, + // retries and waits of the import before it. Reset leaves the adaptive pacer's control + // state alone, which is right here too - that describes the tenant's current throttle + // regime rather than anything the departing import accumulated. + MgxTelemetryCollector.Current.Reset(); + Cmdlets.Configuration.EnableMgxResilience.ReleaseInjection(); - // Last: both calls above resolve types through this cache, so clearing it earlier would + // Last: the calls above resolve types through this cache, so clearing it earlier would // only refill it with the entries the removal exists to drop. Base.MgxCmdletBase.ClearTypeCache(); } diff --git a/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs b/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs index e56c9e2..6f4a6f9 100644 --- a/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs +++ b/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs @@ -10,6 +10,7 @@ using System.Text.RegularExpressions; using Mgx.Engine.Http; using Mgx.Engine.Models; +using Mgx.Engine.Pagination; using Polly.CircuitBreaker; namespace Mgx.Cmdlets.Base; @@ -56,7 +57,13 @@ public abstract class MgxCmdletBase : MgxCmdletCore // TotalTimeoutSeconds the cached client's HttpClient.Timeout was derived from. private static int s_cachedTotalTimeoutSeconds; - internal static volatile string s_graphEndpoint = "https://graph.microsoft.com"; + /// + /// The cloud a process answers for before any session has been read, and what a module + /// removal puts the endpoint back to. + /// + internal const string DefaultGraphEndpoint = "https://graph.microsoft.com"; + + internal static volatile string s_graphEndpoint = DefaultGraphEndpoint; internal static volatile ResilientGraphClientOptions s_clientOptions = ResilientGraphClientOptions.Default; // Test transport seam. The xUnit assembly arms these three through MgxTransportScope, and @@ -206,7 +213,7 @@ protected ResilientGraphClient GetClient() ? null : new WeakReference(identity.AuthContext); s_cachedTotalTimeoutSeconds = clientOptions.TotalTimeoutSeconds; - s_graphEndpoint = GetGraphEndpoint(WriteWarning, WriteVerbose) ?? "https://graph.microsoft.com"; + s_graphEndpoint = GetGraphEndpoint(WriteWarning, WriteVerbose) ?? DefaultGraphEndpoint; } httpClient = s_graphHttpClient!; } @@ -627,7 +634,7 @@ internal static void TryPreInitHttpClient(Action warn, Action ve ? null : new WeakReference(identity.AuthContext); s_cachedTotalTimeoutSeconds = options.TotalTimeoutSeconds; - s_graphEndpoint = GetGraphEndpoint(warn, verbose) ?? "https://graph.microsoft.com"; + s_graphEndpoint = GetGraphEndpoint(warn, verbose) ?? DefaultGraphEndpoint; } } @@ -653,7 +660,7 @@ private static bool ClientIsStale(AuthIdentity identity, ResilientGraphClientOpt // Use detected endpoint instead of hardcoded graph.microsoft.com // (supports sovereign clouds: GCC-High, DoD, China) - var endpoint = GetGraphEndpoint(WriteWarning, WriteVerbose) ?? "https://graph.microsoft.com"; + var endpoint = GetGraphEndpoint(WriteWarning, WriteVerbose) ?? DefaultGraphEndpoint; // Save AzureADEndpoint before probe (same issue as ForceInitializeAndGetClient: // Invoke-MgGraphRequest replaces GraphSession.Environment with a new object @@ -834,32 +841,680 @@ internal static void SetClientOptions(ResilientGraphClientOptions options) } /// - /// True when nothing else holds the file. FileShare.None is honoured between .NET processes - /// on both Windows and Unix, so a writer that has it open makes this fail rather than let a - /// sweep take a file out from under it. + /// Puts back the two values a client build derives rather than a caller sets: the endpoint + /// read off the session it built for, and the TotalTimeoutSeconds the transport was sized to. + /// Both are per-process, so a removal that leaves them set gives the next import the cloud + /// the one before it connected to - in the VersionedBaseUrl every cmdlet composes its request + /// URLs from - until that import's own first build derives one. + /// + /// Beside SetClientOptions in ReleaseStaticState rather than inside ResetHttpClient, for the + /// reason the options release is: a caller that saves and restores these around + /// ResetHttpClient, as the suite's session scopes do, would have its restore released + /// instead of the state. + /// + /// + internal static void ReleaseEndpointAndTimeout() + { + lock (s_initLock) + { + s_graphEndpoint = DefaultGraphEndpoint; + s_cachedTotalTimeoutSeconds = 0; + } + } + + /// + /// Why an exclusive claim on a file did not succeed, where it did not. Asked of a temp a + /// checkpoint names, and of the output a checkpoint records its items into: both are files + /// a second run over the same command line can be writing right now, and neither is one + /// this run may take from under it. + /// + private enum FileClaim + { + /// Nothing else holds it, and this account can open it read-write. + Taken, + + /// + /// Something has it open. FileShare.None is honored between .NET processes on both + /// Windows and Unix, so a writer that holds the file makes the claim fail rather than + /// let a sweep take it out from under them. + /// + Held, + + /// + /// It is not a file this run can write at the offset a checkpoint records: mode bits, a + /// denying ACL, a read-only filesystem, a directory standing where the file should be, a + /// symlink loop or an over-long path, or a FIFO that opens and then has no length and no + /// offset to write at. Every failure but the sharing one, in other words, and no second + /// run is implied by any of them - which is the whole reason this is told apart from the + /// case above. + /// + Unopenable, + + /// + /// There is no such file, or no such directory to hold it. Nothing is wrong and no + /// second run is implied: it is the case a caller answers by starting over, and it is + /// told apart from the two above because on a file that is + /// not there raises the same IOException a sharing violation does - so read as one, a + /// missing file reported a second run over a file neither run has. + /// + Absent, + } + + /// + /// Ask for the file, and say what happened. The failures are not one answer: a file another + /// run holds is a file nothing is wrong with, a file this account cannot open read-write has + /// no other run behind it, and a file that is not there is neither - so a caller that + /// collapses them tells the reader to go looking for a run that does not exist. + /// + /// The claim is not finished at the open on Unix. .NET backs the share mode there with a + /// non-blocking flock taken AFTER open(2), so the handle binds to whatever inode the name + /// resolved to and the exclusion over it is granted several syscalls later - and a second + /// run that renamed its own file onto the name in between leaves this one holding an inode + /// with no name on it, with nothing about the handle to say so. So the name is asked again + /// where says it moved, and a name that keeps moving + /// is : another run is mutating it right now, which is what + /// every caller here reads Held as. Windows answers this question at the open itself and + /// keeps the answer it had. + /// /// - private static bool CanTakeExclusively(string path) + /// + /// What a second opener may still do. for a claim taken and let + /// go in the same breath; the output hold asks for + /// instead, since it keeps the handle for the whole run. + /// + /// + /// The open file where the answer is , and where a claim that + /// was granted could not then be verified - answered with + /// the file still open, since a handle this never hands over is one nothing releases + /// deterministically: only the finalizer does, whenever the GC gets to it. Null for every + /// other answer. The caller owns it wherever it comes with one. + /// + /// + /// Why the file could not be claimed, where the answer is + /// - the open's own refusal, or the failure of the question that finishes a claim the open + /// was granted; null for every other answer. A caller building a stop names this instead of + /// guessing at a cause, and reads to know + /// whether granting write access is advice this run can honestly give. + /// + /// + /// for a claim on a file that has to be there already, which is + /// every caller that reads as an answer. + /// for a claim whose point is to exist and be refused + /// even where the file does not: what it makes is reported in , + /// and it answers Absent for the four names no create can be made through - a link + /// pointing at nothing, which is followed rather than replaced; a parent directory that is + /// not there, which is nothing to make a file in; a path whose parent component is a file + /// rather than a directory, which is nothing to make one in either and reaches the same + /// answer by ENOTDIR; and a name whose entry has gone by the time it is opened, where the + /// attempts left to ask again in have run out or something is standing at the name again by + /// the time the answer is read. + /// + /// + /// Whether the handle is over a file this open made, because nothing stood at the name. True + /// wherever the attempt made the file, including a claim that could not be verified - which + /// is what arms the finally's take-back - and false on every attempt after one the + /// verification found had moved: a claim asked again over a name that has moved is a claim + /// on whatever stands there now, which is not this run's to take away. + /// + /// The name is read a syscall in front of the open, so what this answers is the state the + /// open was asked for in. A file a second mgx run put there in between is one that run's own + /// claim refuses, and no claim of this one's comes away holding it; a file that nothing + /// holds - a shell's touch, an older mgx, any writer that takes no claim - is opened rather + /// than made and reported here as made all the same. What that costs is the one thing this + /// is read for: the empty output a promotion takes back where it stops is, in that window, a + /// file the run found rather than one it made. + /// + /// + /// + /// Whether the attempt this answer came from asked the open to make the file, because + /// nothing stood at the name and no link stood there either. says + /// a file was made; this says one was asked for, which is what tells an open refused over a + /// file that is there from a create the directory would not take. Decided per attempt like + /// the create itself, so it describes the look this answer came out of and no earlier one. + /// + private static FileClaim OpenExclusively(string path, FileShare share, out FileStream? stream, + out UnopenableReason? reason, FileMode mode = FileMode.Open) => + OpenExclusively(path, share, out stream, out reason, mode, out _, out _); + + /// + private static FileClaim OpenExclusively(string path, FileShare share, out FileStream? stream, + out UnopenableReason? reason, FileMode mode, out bool created, out bool askedToMakeIt) { + for (var attempt = 1; ; attempt++) + { + // A file is made here only where nothing stands at the name and no link does either: + // a create follows a link out of the directory and puts this run's file wherever it + // points, and a name something already stands at is one this claim opens rather than + // makes. Decided per attempt, since a claim is asked again precisely when the name + // has moved - and a second attempt that created a file over what a first attempt + // found would be holding an entry it did not make. + var make = mode == FileMode.OpenOrCreate && NothingStandsAt(path); + var claim = AskForTheFile(path, share, make ? FileMode.OpenOrCreate : FileMode.Open, + out stream, out reason); + created = make && claim == FileClaim.Taken; + askedToMakeIt = make; + + // A file that stood at the name a line ago and does not now: opened as one that has + // to be there, it answers Absent, and a caller that asked for OpenOrCreate is one + // that reads a name with nothing at it as a file it makes. So the name is asked + // again, where it is free to be made now. A link pointing at nothing is not that - + // it stands at the name, a create through it goes elsewhere, and Absent is what its + // caller already reads. + if (claim == FileClaim.Absent && mode == FileMode.OpenOrCreate + && attempt < HeldEntry.ClaimAttempts && NothingStandsAt(path)) + { + continue; + } + + if (claim != FileClaim.Taken) return claim; + + bool standing; + try + { + standing = HeldEntry.StillStandsAt(stream!, path); + } + catch (Exception ex) + { + // The exclusion is granted and the question that finishes it cannot be + // answered: a parent that stopped answering for the name, a share that went, a + // call that failed for a reason this cannot read. A claim that cannot be + // verified is not a claim, so it is refused - and refused as an answer rather + // than as a throw, with the handle and what the attempt made handed back + // exactly as a Taken answer hands them over. Every caller releases what it was + // given on its own path, and already reads Unopenable as a file it does not + // take; one of them releases in a finally, and takes the empty output it made + // back under that same claim. Emptying the out parameters here disarms that + // finally, which then holds nothing and unlinks nothing - and a file with no + // rows in it that no run holds is left standing at the caller's -OutputFile. + // + // The reason is the failure's own sentence, which names the call and the errno + // and says the claim on the file could not be finished. It is not a permission + // on the file: the search bit that went is the parent's, and a stale handle or + // a filesystem error is nobody's grant to give, so advice to open up the file + // would send the reader at the wrong thing. + reason = new UnopenableReason(FirstLine(ex.Message).TrimEnd('.'), + IsPermission: false); + return FileClaim.Unopenable; + } + + if (standing) return claim; + + // Granted over a file that has left the name. Dropped rather than reported, and the + // name asked again from the open: the run that moved it either holds what stands + // there now, which the next claim is refused by, or has let go, which the next claim + // takes honestly. + stream!.Dispose(); + stream = null; + created = false; + if (attempt == HeldEntry.ClaimAttempts) return FileClaim.Held; + } + } + + /// + /// Nothing stands at the name, and no link stands there either - the one state a claim may + /// make its own file in. Read off the directory entry rather than through an open, which is + /// how a dangling link is told from an absent name at all: both answer that nothing is + /// there, and only one of them takes a create somewhere else. + /// + private static bool NothingStandsAt(string path) => + !Path.Exists(path) && new FileInfo(path).LinkTarget == null; + + /// + /// The open itself, and which of the four answers it gave. Every caller reaches it through + /// , which is where a claim is finished. + /// + private static FileClaim AskForTheFile(string path, FileShare share, FileMode mode, + out FileStream? stream, out UnopenableReason? reason) + { + stream = null; + reason = null; try { - using var _ = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.None); - return true; + stream = new FileStream(path, mode, FileAccess.ReadWrite, share); + if (!stream.CanSeek) + { + // Something opened, and it is not a file with a length and an offset in it. A + // FIFO standing at the output path opens read-write without blocking, and every + // question asked of the handle afterwards - how long is it, cut it back to the + // recorded length, append at the end - is one a stream that cannot seek answers + // with a NotSupportedException, which is neither of the two failures a caller + // here knows how to report and went past both guards with the handle still open. + // So it is decided at the open: nothing this run can resume into, and let go + // before a caller is told to wait for it. Nothing here throws, so the reason is + // this method's own words rather than an exception's - a FIFO is not a + // permission, and granting write access to one would not give it a length. + stream.Dispose(); + stream = null; + reason = new UnopenableReason("the file has no offset to write at", + IsPermission: false); + return FileClaim.Unopenable; + } + return FileClaim.Taken; } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + catch (FileNotFoundException) + { + return FileClaim.Absent; + } + catch (DirectoryNotFoundException) + { + // The same answer one level up: ENOENT on a path component. Both are IOExceptions, + // and both have to be caught above the sharing case below rather than reasoned about + // from an HRESULT afterwards. + return FileClaim.Absent; + } + catch (UnauthorizedAccessException ex) + { + reason = ReasonForUnopenable(path, ex); + return FileClaim.Unopenable; + } + catch (IOException ex) + { + // The sharing violation is named, and every other way an open can fail is a file + // this run cannot open. Decided the other way round - anything not on a list of + // access errnos is another run - a symlink loop and an over-long path at the output + // path both stopped the run with "Another export is still writing", sending the + // reader to wait for a second export over a path no run of any kind could open. + if (IsSharingViolation(ex)) return FileClaim.Held; + reason = ReasonForUnopenable(path, ex); + return FileClaim.Unopenable; + } + } + + /// + /// Why an open of failed, refined once for every site that has to + /// say it. The claim in is one; the create a promotion stages + /// its copy with is the other, and it built its own words out of the raw exception - so one + /// disk state was two sentences, and a directory at the staging name read "Access to the + /// path '...' is denied" in the applying pass and "a directory stands at the path" in the + /// preview of the same run. + /// + /// carries the runtime's own first line with its + /// sentence-ending period taken off, since every caller quotes it inside a sentence it + /// punctuates itself. The two causes told apart from the rest are the two with a different + /// way out than granting write access on the file: a directory standing at the path, which + /// no grant makes writable at an offset, and a parent directory this account cannot search, + /// where the grant belongs to the directory and not to the file the sentence names. + /// + /// + /// A directory is asked of the filesystem rather than of the errno, since it reaches an + /// access failure as EISDIR or as EACCES depending on the mode on it and both say the same + /// thing about the path - and, from a create, as neither: a create refuses an existing name + /// with "already exists" whatever stands there, so a directory that appears at the staging + /// name between the sweep and the create is asked about on that route too, and the two + /// instructions call one disk state by one name. An unsearchable parent is read off the + /// entry the open failed on, + /// by : a parent that is not there reaches + /// above and never gets here, so what is left to + /// tell apart is a directory this account may not look inside from a file inside one it may. + /// + /// + private static UnopenableReason ReasonForUnopenable(string path, Exception ex, + bool fromCreate = false) + { + // Only an access failure can be one of the two named causes. Every other way an open + // fails - a symlink loop, an over-long path, a socket, a full volume - is the runtime's + // words and nothing this run adds to them. + if (ex is not UnauthorizedAccessException) + { + // Except the one a create has of its own: it refuses a name that already exists + // with "The file '...' already exists", whatever kind of entry stands there, so a + // directory appearing at the staging name between the sweep and the create read as + // a file in the way while the sweep, one instruction earlier, called the same disk + // state a directory. Asked of the filesystem, the way the access failure below asks + // it, so one entry reads one way wherever the run met it. + if (fromCreate && Directory.Exists(path)) + return new UnopenableReason("a directory stands at the path", IsPermission: false, + IsDirectory: true); + return new UnopenableReason(FirstLine(ex.Message).TrimEnd('.'), IsPermission: false); + } + + if (Directory.Exists(path)) + return new UnopenableReason("a directory stands at the path", IsPermission: false, + IsDirectory: true); + + var parent = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(parent) && Directory.Exists(parent) + && TheParentRefusesToBeSearched(path)) + return new UnopenableReason($"the directory '{parent}' cannot be searched", + IsPermission: true, IsUnsearchableParent: true); + + return new UnopenableReason(FirstLine(ex.Message).TrimEnd('.'), IsPermission: true); + } + + /// + /// Whether the entry at cannot even be asked about, which is what a + /// directory missing its search bit refuses - and the one thing that separates a file this + /// account may not write from a file it cannot reach at all. Asked of the entry rather than + /// of the directory, because a directory answers for itself either way: mode 000 or not, a + /// stat of the directory needs the search bit on ITS parent and not on itself, so + /// is true of a directory nothing can be looked up + /// inside. What the missing bit refuses is every name under it, present or absent alike. + /// + /// Told from a name that is simply not there, which is what a create is about to make: a + /// searchable directory answers an absent name with ENOENT, and only an unsearchable one + /// answers a permission. Listing is not the question either - a directory with the search + /// bit and not the read bit refuses to be listed and resolves every name in it - so nothing + /// here enumerates. + /// + /// + private static bool TheParentRefusesToBeSearched(string path) + { + try { File.GetAttributes(path); } + catch (UnauthorizedAccessException) { return true; } + catch (Exception ex) when (ex is IOException or ArgumentException + or NotSupportedException) { } + return false; + } + + /// + /// The exception's own explanation, first line only. A caller quotes the runtime's words for + /// why an open failed, and quoting more than the first line risks folding a second sentence, + /// or a path repeated on its own line, into what reads as one clause. + /// + private static string FirstLine(string message) + { + var newline = message.IndexOfAny(['\r', '\n']); + return newline < 0 ? message : message[..newline]; + } + + /// + /// + /// Ask for the file exclusively and let it go again, which is all a caller wants that only + /// has to report on it. The reason a failed claim is unopenable is not asked for here: every + /// caller of this overload already has its own words for the one failure it distinguishes + /// from a hold, or does not report a cause at all. + /// + private static FileClaim ClaimExclusively(string path, out UnopenableReason? reason) + { + var claim = OpenExclusively(path, FileShare.None, out var stream, out reason); + stream?.Dispose(); + return claim; + } + + /// + /// Whether an IOException from opening a file is the sharing violation the share mode exists + /// to produce, rather than one of the other ways an open fails. The engine reads it for the + /// scratch names it sweeps, and one implementation is what keeps the claim here and that + /// sweep from disagreeing about which errno means a second run. + /// + /// + private static bool IsSharingViolation(IOException ex) => + ScratchName.IsSharingViolation(ex); + + /// + /// True when nothing else holds the file and this account can open it read-write. The sweep + /// and the adoption search ask only this much: a file they cannot claim is a file they pass + /// over, whichever of the two reasons it is. + /// + private static bool CanTakeExclusively(string path) => + ClaimExclusively(path, out _) == FileClaim.Taken; + + /// + /// What each directory answered when it was asked whether it keeps two spellings of a name + /// apart. Keyed by full path: a volume does not change its mind, so one answer per directory + /// per process is all of it, and two callers racing for the same key cost a second look and + /// nothing else. An answer read off the directory's own entries is kept here like a probed + /// one; the platform rule a caller falls back to where it may not write is not, so the next + /// caller that may write still asks the volume. + /// + private static readonly ConcurrentDictionary s_caseSensitiveDirectories = + new(StringComparer.Ordinal); + + /// The answer where the directory cannot be asked: the rule the platform applies by default. + protected static bool PlatformCaseRule => OperatingSystem.IsLinux(); + + /// + /// Whether the filesystem under keeps "Users.jsonl" and + /// "users.jsonl" apart. Asked of the directory rather than of the operating system, because + /// the operating system does not answer for it: an APFS volume can be created + /// case-sensitive, a Windows directory can be flagged case-sensitive on its own, and where + /// the guess went the wrong way the temp helpers matched names the filesystem does not - a + /// sweep reaching another export's live temp, and two distinct outputs comparing as one file. + /// + /// It is answered without touching the directory wherever it can be: an entry already there, + /// looked for under its own spelling inverted, settles it - present under the other spelling + /// and the volume folds case, absent and it keeps the two apart. Only a directory with no + /// such entry needs a file written, and says whether this caller + /// may write one. Everything reached above a ShouldProcess gate passes false: -WhatIf + /// reports what a run would do, and creating and removing a file to find out moves the + /// caller's directory's write time. A caller that may not write, in a directory that cannot + /// answer for itself, gets the platform rule - which is what every release before the probe + /// used everywhere - and that answer is not remembered, so the run below still asks the + /// volume. The two therefore differ only in a directory holding nothing, on a volume whose + /// rule is not its platform's default. + /// + /// A directory this run cannot write to answers with the platform rule as well: the probe is + /// a better answer where it can be taken, not a precondition, and a run that cannot write + /// its own output directory has its own error to raise rather than this one. + /// + internal static bool DirectoryNamesAreCaseSensitive(string? directory, bool mayWrite) + { + if (string.IsNullOrEmpty(directory)) return PlatformCaseRule; + string key; + try { key = Path.GetFullPath(directory); } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException + or PathTooLongException or IOException) + { + return PlatformCaseRule; + } + + if (s_caseSensitiveDirectories.TryGetValue(key, out var settled)) return settled; + + // Before this module writes a probe here, take back the one an earlier run left. Only + // on the write path: a caller that may not write deletes nothing either. + if (mayWrite) ReclaimCaseProbeLitter(key); + + if (ReadOnlyCaseAnswer(key) is { } observed) + return s_caseSensitiveDirectories.GetOrAdd(key, observed); + + if (!mayWrite) return PlatformCaseRule; + + return s_caseSensitiveDirectories.GetOrAdd(key, ProbeCaseSensitivity); + } + + /// + /// Whether answering this directory's case rule would still take a write. False once the + /// answer is known, and false where an entry already in the directory gives it - which this + /// keeps, since having asked, it costs nothing more. True only for a directory that exists + /// and has nothing in it to read the answer off, which is the one place a preview and the + /// run it describes can disagree, and the only place worth a line saying so. A directory + /// that is not there is answered no: neither side of a gate can ask one, so they agree. + /// + protected static bool DirectoryCaseRuleNeedsProbe(string? directory) + { + if (string.IsNullOrEmpty(directory)) return false; + string key; + try { key = Path.GetFullPath(directory); } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException + or PathTooLongException or IOException) + { + return false; + } + if (s_caseSensitiveDirectories.ContainsKey(key)) return false; + if (!Directory.Exists(key)) return false; + if (ReadOnlyCaseAnswer(key) is not { } observed) return true; + s_caseSensitiveDirectories.GetOrAdd(key, observed); + return false; + } + + /// + /// The answer read off the directory's own entries, or null where they do not give one. An + /// entry that is there under one spelling is there under the inverted spelling too exactly + /// when the volume folds case, so asking for the inverted name settles it with nothing + /// written. + /// + /// Three kinds of candidate are passed over rather than answered from. A name with no ASCII + /// letter inverts to itself, and a name carrying anything outside ASCII inverts through case + /// mappings the filesystem need not share - a sharp s and a dotted capital I are not their + /// own round trip - so a folding volume could be read as a splitting one. A directory that + /// already holds BOTH spellings says nothing either: that is what a case-sensitive volume + /// looks like, not a fold. Neither does an entry unlinked between the listing and the + /// lookup, which is a file that is simply gone rather than a name the volume refused, so the + /// entry is looked for again before its absence is read as an answer. The probe files below + /// are skipped for the same reason: one of them may be another run's, about to vanish. + /// + private static bool? ReadOnlyCaseAnswer(string directory) + { + try + { + var names = Directory.GetFileSystemEntries(directory) + .Select(Path.GetFileName) + .Where(n => !string.IsNullOrEmpty(n)) + .Select(n => n!) + .ToList(); + var present = new HashSet(names, StringComparer.Ordinal); + foreach (var name in names) + { + if (IsCaseProbeName(name)) continue; + if (InvertCase(name) is not { } inverted) continue; + if (present.Contains(inverted)) continue; + var other = Path.Combine(directory, inverted); + if (File.Exists(other) || Directory.Exists(other)) return false; + var mine = Path.Combine(directory, name); + if (File.Exists(mine) || Directory.Exists(mine)) return true; + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { } + return null; + } + + /// + /// The name with every ASCII letter's case turned over, or null where that is not a + /// different name this filesystem has to have an opinion about: a name with no ASCII letter + /// in it, or one carrying anything outside ASCII. + /// + private static string? InvertCase(string name) + { + var chars = name.ToCharArray(); + var sawLetter = false; + for (var i = 0; i < chars.Length; i++) + { + var c = chars[i]; + if (c > 0x7F) return null; + if (c is >= 'a' and <= 'z') { chars[i] = (char)(c - 32); sawLetter = true; } + else if (c is >= 'A' and <= 'Z') { chars[i] = (char)(c + 32); sawLetter = true; } + } + return sawLetter ? new string(chars) : null; + } + + /// + /// True for a name the probe below gives its own file: ".mgx-case-", 32 lowercase hex + /// digits, and nothing else. Nothing but this writes one, which is what makes deleting a + /// leftover safe - and holding the exact shape is what keeps that promise about a directory + /// the caller also keeps their own ".mgx-case-notes" in. + /// + private static bool IsCaseProbeName(string candidate) + { + const string prefix = ".mgx-case-"; + if (candidate.Length != prefix.Length + 32) return false; + if (!candidate.StartsWith(prefix, StringComparison.Ordinal)) return false; + for (var i = prefix.Length; i < candidate.Length; i++) { + if (candidate[i] is (>= '0' and <= '9') or (>= 'a' and <= 'f')) continue; return false; } + return true; + } + + /// + /// Delete the probe files an earlier run left in this directory. DeleteOnClose is the + /// kernel's promise on Windows and is kept however the process ends; on Unix the unlink + /// happens where the stream is disposed, so a process killed outright leaves its probe + /// behind, and nothing on disk refers to it again. One is reclaimed the way the stale-temp + /// sweep reclaims a temp - claimed exclusively first - because a probe another run is taking + /// right now is open, and unlinking it under them would answer their question with the + /// wrong half of it. + /// + private static void ReclaimCaseProbeLitter(string directory) + { + try + { + foreach (var entry in Directory.EnumerateFiles(directory, ".mgx-case-*").ToList()) + { + if (!IsCaseProbeName(Path.GetFileName(entry))) continue; + if (!CanTakeExclusively(entry)) continue; + try { File.Delete(entry); } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { } + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { } } /// - /// How two file names are compared here. Directory.EnumerateFiles matches its pattern the - /// way the filesystem does - so on NTFS and APFS "Users.jsonl" answers to "users.jsonl" and - /// the glob returns it - and a predicate filtering those results with an ordinal comparison + /// One probe file, created under a name of its own so nothing else can be reading it, and + /// deleted by hand as well as by the handle. The name is not one a run gives its own temp, + /// so neither the sweep nor adoption can reach it even in the moment it exists. + /// + /// DeleteOnClose is not the same promise on both platforms. Windows unlinks at the last + /// handle close, which the kernel performs however the process ends; Unix unlinks where the + /// stream is disposed, so a process killed outright leaves the file where it is. That is + /// what the reclaim above collects, and it runs before this writes anything. + /// + private static bool ProbeCaseSensitivity(string directory) + { + var name = $".mgx-case-{Guid.NewGuid():N}"; + var probe = Path.Combine(directory, name); + try + { + using (new FileStream(probe, FileMode.CreateNew, FileAccess.Write, FileShare.None, + bufferSize: 1, FileOptions.DeleteOnClose)) + { + // It exists under the name it was created with. Whether it also exists under + // the other casing is the whole question. + return !File.Exists(Path.Combine(directory, name.ToUpperInvariant())); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return PlatformCaseRule; + } + finally + { + try { File.Delete(probe); } catch { } + } + } + + /// + /// Test seam: pins the answer for one directory, so the case-sensitive decision path runs on + /// a host whose own filesystem is not. Null forgets it and the next question probes again. + /// Internal, assigned directly rather than reflected; every shipping path leaves it alone. + /// + internal static void SetDirectoryCaseSensitivity(string directory, bool? caseSensitive) + { + var key = Path.GetFullPath(directory); + if (caseSensitive is { } value) s_caseSensitiveDirectories[key] = value; + else s_caseSensitiveDirectories.TryRemove(key, out _); + } + + /// + /// How two file names in one directory are compared. Directory.EnumerateFiles matches its + /// pattern the way the filesystem does - so where "Users.jsonl" answers to "users.jsonl" the + /// glob returns it - and a predicate filtering those results with an ordinal comparison /// disagreed with the enumeration that produced them: an export whose -OutputFile differed - /// only in case abandoned its own temp and re-enumerated in silence. + /// only in case abandoned its own temp and re-enumerated in silence. Both sides are keyed + /// from the one answer above, so they cannot disagree again. + /// + private static StringComparison FileNameComparisonIn(string? directory, bool mayWrite) => + DirectoryNamesAreCaseSensitive(directory, mayWrite) + ? StringComparison.Ordinal + : StringComparison.OrdinalIgnoreCase; + + /// + /// How the "{output}.*.tmp" glob is matched in one directory. The two-argument + /// EnumerateFiles takes its casing from the platform, which is the guess the comparison + /// above no longer makes; the rest of these are that overload's own defaults - Win32 + /// globbing, nothing skipped by attribute, an inaccessible entry raised rather than + /// swallowed - spelled out so that naming the casing does not quietly change them too. /// - private static readonly StringComparison s_fileNameComparison = - OperatingSystem.IsLinux() ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; + private static EnumerationOptions TempGlobOptions(string directory, bool mayWrite) => new() + { + MatchCasing = DirectoryNamesAreCaseSensitive(directory, mayWrite) + ? MatchCasing.CaseSensitive + : MatchCasing.CaseInsensitive, + MatchType = MatchType.Win32, + AttributesToSkip = 0, + IgnoreInaccessible = false, + }; /// /// Whether two paths name the same file, normalized and compared the way the filesystem @@ -869,13 +1524,17 @@ private static bool CanTakeExclusively(string path) /// a checkpoint is untrusted input once it is on disk, and an unusable name is not this /// run's output. /// - protected static bool SamePath(string? left, string? right) + protected static bool SamePath(string? left, string? right, bool mayWrite) { if (left == null || right == null) return false; try { - return string.Equals(Path.GetFullPath(left), Path.GetFullPath(right), - s_fileNameComparison); + var recorded = Path.GetFullPath(left); + var mine = Path.GetFullPath(right); + // Keyed on the second path, which is this run's own output at every call site, and + // so names the directory whose rule decides whether the other spelling is that file. + return string.Equals(recorded, mine, + FileNameComparisonIn(Path.GetDirectoryName(mine), mayWrite)); } catch (Exception ex) when (ex is ArgumentException or NotSupportedException or PathTooLongException or IOException) @@ -925,6 +1584,20 @@ protected static bool SameResourceIdentity(string? recorded, string? current) c < 0 ? string.Empty : current[c..], StringComparison.Ordinal); } + /// + /// Whether the checkpoint records anything for to weigh: + /// a path to compare, or a length to hold a file against. A checkpoint recording neither has + /// nothing there to be weighed, and asking anyway answers no for want of a measurement - so + /// the shape every release before 2.1 wrote was read as another run's, its temp swept by the + /// sweep a refusal holds off over the temp it names, and the position counting those items + /// left standing to refuse the next -Latest. That shape is decided on the evidence it does + /// have - a temp carrying this output's own name, beside an output that is not there - by + /// the reconcile, which refuses it wherever an output exists. Both cmdlets ask this one + /// question in front of the predicate, so one checkpoint reads one way in either. + /// + protected static bool RecordsAFileToWeigh(PaginationCheckpoint checkpoint) => + checkpoint.OutputFile != null || checkpoint.DataLength != null; + /// /// Whether the file a checkpoint counted its items into is one this run collects into. A /// checkpoint that records the output answers directly. One written before that field @@ -937,11 +1610,13 @@ protected static bool SameResourceIdentity(string? recorded, string? current) /// recorded length below zero is not a measurement. /// protected static bool RecordedOutputMatches( - string? recordedOutput, string? recordedTemp, long? dataLength, string outputPath) + string? recordedOutput, string? recordedTemp, long? dataLength, string outputPath, + bool mayWrite) { - if (recordedOutput != null) return SamePath(recordedOutput, outputPath); + if (recordedOutput != null) return SamePath(recordedOutput, outputPath, mayWrite); if (dataLength is not { } length || length < 0) return false; - if (recordedTemp != null) return ResolveNamedTemp(outputPath, recordedTemp, length) != null; + if (recordedTemp != null) + return ResolveNamedTemp(outputPath, recordedTemp, length, mayWrite) != null; try { var info = new FileInfo(outputPath); @@ -954,45 +1629,94 @@ protected static bool RecordedOutputMatches( } /// - /// Remove leftover "{outputPath}.{guid}.tmp" files. Called only when no resume is pending, - /// where every such file is an orphan by definition - except one a live run still holds, - /// which CanTakeExclusively keeps out of reach. "Such a file" is decided by name and not by - /// the glob alone: the glob reaches a longer output's temps too, and those describe work - /// this run knows nothing about. The name has to be one a run gives its own temp, 32 hex - /// digits and all: anything looser is a promise that no file mgx did not write is deleted, - /// made about a directory the caller also keeps their own "users.jsonl.backup.tmp" in. + /// Remove leftover "{outputPath}.{guid}.tmp" files, and the copy a promotion stages at + /// "{outputPath}.adopt". Called only when no resume is pending, where every such file is an + /// orphan by definition - except one a live run still holds, which the claim keeps out of + /// reach. "Such a file" is decided by name and not by the glob alone: the glob reaches a + /// longer output's temps too, and those describe work this run knows nothing about. The name + /// has to be one a run gives its own temp, 32 hex digits and all: anything looser is a + /// promise that no file mgx did not write is deleted, made about a directory the caller also + /// keeps their own "users.jsonl.backup.tmp" in. The staging name is exact and reached by + /// name, since no glob of this shape covers it. /// protected void DeleteStaleTemps(string outputPath) { + // "Orphan" is an assumption about a file this run did not create, and a second export + // running against the same output right now owns a file matching the same glob. Windows + // refuses to delete a file someone holds open, so it declined by accident; Unix does + // not, and the other run went on writing into an unlinked inode and lost everything it + // had fetched. Ask for the file exclusively first - if that fails, someone is using it + // and it is not an orphan. + // + // And which of the two failures it was. A file another run holds is a file nothing is + // wrong with; a file this account cannot open read-write has no second run behind it at + // all, and one sentence for both sent the caller looking for a run that does not exist. + // + // Which of the second kind it was is the open's own answer and not this line's to guess: + // "the open was refused on permissions, not on sharing" was said of a directory, of a + // pipe with no length to write at and of a path too long for the volume, and a caller + // who went and granted write access to the file found nothing had changed. + void SweepIfUnheld(string stale, string deleted) + { + var claim = ClaimExclusively(stale, out var reason); + // A file that has gone between the directory listing and the claim is a file this + // sweep has nothing left to do about, and nothing to say either. + if (claim == FileClaim.Absent) return; + if (claim != FileClaim.Taken) + { + WriteVerbose(claim == FileClaim.Held + ? $"Left '{Path.GetFileName(stale)}' alone: another run is writing to it." + : $"Left '{Path.GetFileName(stale)}' alone: this run cannot open it for " + + $"writing - {reason?.Reason ?? "the open was refused"}."); + return; + } + try + { + File.Delete(stale); + WriteVerbose($"{deleted}: {Path.GetFileName(stale)}"); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + WriteWarning($"Could not delete orphaned temp file '{stale}': {ex.Message}. Delete it manually."); + } + } + try { var dir = Path.GetDirectoryName(outputPath); if (string.IsNullOrEmpty(dir) || !Directory.Exists(dir)) return; var outputName = Path.GetFileName(outputPath); - foreach (var stale in Directory.EnumerateFiles(dir, outputName + ".*.tmp") - .Where(p => IsRunTempName(outputName, Path.GetFileName(p))) + foreach (var stale in Directory.EnumerateFiles(dir, outputName + ".*.tmp", + TempGlobOptions(dir, mayWrite: true)) + .Where(p => IsRunTempName(dir, outputName, Path.GetFileName(p), + mayWrite: true)) .ToList()) { - // "Orphan" is an assumption about a file this run did not create, and a second - // export running against the same output right now owns a file matching the same - // glob. Windows refuses to delete a file someone holds open, so it declined by - // accident; Unix does not, and the other run went on writing into an unlinked - // inode and lost everything it had fetched. Ask for the file exclusively first - - // if that fails, someone is using it and it is not an orphan. - if (!CanTakeExclusively(stale)) - { - WriteVerbose($"Left '{Path.GetFileName(stale)}' alone: another run is writing to it."); - continue; - } - try - { - File.Delete(stale); - WriteVerbose($"Deleted an orphaned temp file from an earlier interrupted run: {Path.GetFileName(stale)}"); - } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) - { - WriteWarning($"Could not delete orphaned temp file '{stale}': {ex.Message}. Delete it manually."); - } + SweepIfUnheld(stale, + "Deleted an orphaned temp file from an earlier interrupted run"); + } + + // And the copy a promotion stages beside the output, which the glob above cannot + // reach: no run leaves a file under that name on purpose - it is renamed away where + // the promotion lands and removed where it stops - so one standing here is a run + // killed between those two, holding a partial copy of somebody's rows that nothing + // on disk refers to. + // + // Through the promotion's own sweep of that name, and not the claim-then-delete a + // temp takes. The claim answers for a regular file and for nothing else: a link at + // the name was followed to whatever it pointed at, a pipe and a socket came back + // unopenable and were left standing - reported as a file this account could not + // open - and the next promotion then met exactly the entry that sweep exists to + // clear. One primitive decides the name wherever a run reaches it. + var adoptPath = StagingPathFor(outputPath); + if (SweepStagingName(adoptPath, apply: true, out var adoptFailure, out var adoptSwept)) + { + if (adoptSwept is { } removed) WriteVerbose(removed.Sentence); + } + else + { + WriteVerbose($"Left '{Path.GetFileName(adoptPath)}' alone: " + + $"{adoptFailure!.Value.Reason}."); } } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) @@ -1010,14 +1734,15 @@ protected void DeleteStaleTemps(string outputPath) /// the checkpoint promised, since a shorter one means the items it counted are not all /// there. /// - private static string? ResolveNamedTemp(string outputPath, string tempFileName, long dataLength) + private static string? ResolveNamedTemp(string outputPath, string tempFileName, long dataLength, + bool mayWrite) { if (dataLength <= 0) return null; var dir = Path.GetDirectoryName(outputPath); if (string.IsNullOrEmpty(dir) || !Directory.Exists(dir)) return null; if (!string.Equals(tempFileName, Path.GetFileName(tempFileName), StringComparison.Ordinal)) return null; - if (!IsRunTempName(Path.GetFileName(outputPath), tempFileName)) return null; + if (!IsRunTempName(dir, Path.GetFileName(outputPath), tempFileName, mayWrite)) return null; var tempPath = Path.Combine(dir, tempFileName); if (string.Equals(Path.GetFullPath(tempPath), Path.GetFullPath(outputPath), StringComparison.OrdinalIgnoreCase)) @@ -1029,15 +1754,19 @@ protected void DeleteStaleTemps(string outputPath) /// /// True when is a name a fresh run gives its temp: - /// the output's own name, a dot, 32 lowercase hex digits (Guid "N"), and ".tmp". + /// the output's own name, a dot, 32 lowercase hex digits (Guid "N"), and ".tmp". Both names + /// live in , which is what decides whether a spelling that + /// differs only in case is the same name. /// - protected static bool IsRunTempName(string outputFileName, string candidate) + protected static bool IsRunTempName(string directory, string outputFileName, string candidate, + bool mayWrite) { + var comparison = FileNameComparisonIn(directory, mayWrite); var prefix = outputFileName + "."; const string suffix = ".tmp"; if (candidate.Length != prefix.Length + 32 + suffix.Length) return false; - if (!candidate.StartsWith(prefix, s_fileNameComparison)) return false; - if (!candidate.EndsWith(suffix, s_fileNameComparison)) return false; + if (!candidate.StartsWith(prefix, comparison)) return false; + if (!candidate.EndsWith(suffix, comparison)) return false; for (var i = prefix.Length; i < prefix.Length + 32; i++) { if (candidate[i] is (>= '0' and <= '9') or (>= 'a' and <= 'f')) continue; @@ -1060,24 +1789,72 @@ private static bool EndsWithNewline(string path) return fs.ReadByte() == '\n'; } + /// What promoting the temp a checkpoint names answered. + protected enum TempPromotion + { + /// + /// Nothing was promoted: the temp is absent, shorter than the checkpoint promised, or + /// still held by the run writing it. The items it counted are in no file this run can + /// reach, and the caller must not resume past them. + /// + NotPromoted, + + /// + /// The items are in the output now, and the output is this run's until the writing ends. + /// + Taken, + + /// + /// Another run has the output open. Nothing was promoted into it - the claim is asked + /// for before the move - so the temp, and the checkpoint naming it, are as they were + /// found. + /// + OutputHeld, + + /// + /// The output is not a file this run can write at the recorded offset, and nothing was + /// promoted into it either. + /// + OutputUnopenable, + + /// + /// The copy a promotion stages beside the output could not be written, or could not be + /// moved onto the output once it was: a directory or another run's handle at the staging + /// name, a full disk, a directory that turned read-only between the claim and the copy. + /// Nothing was replaced and nothing was removed - the temp holds every item the + /// checkpoint counts, and the checkpoint still names it - so this is not the temp that + /// went missing, which is the one answer a caller may delete a checkpoint over. + /// + StagingFailed, + } + /// /// Replace the output with the first bytes of a named temp, - /// then remove the temp. A fresh run that finishes moves its temp over whatever the output - /// held, so recovering an unfinished one has to reach that same file; appending instead - /// would leave the previous run's rows - already consumed - in front of this one's. Unlike - /// the glob-and-newest form, this takes exactly the file the checkpoint recorded, so a - /// leftover from an unrelated run cannot be merged in. Bytes rather than lines: the length - /// was taken from the writer's own position, so it cannot disagree with itself about line - /// endings or a torn final line. Returns false when the temp is absent, shorter than the - /// checkpoint promised, or still held by the run writing it, which means the caller must - /// not resume past the items it counted. - /// - protected static bool TryPromoteNamedTemp(string outputPath, string tempFileName, long dataLength) + /// take the output for the rest of the run, and then remove the temp. A fresh run that + /// finishes moves its temp over whatever the output held, so recovering an unfinished one + /// has to reach that same file; appending instead would leave the previous run's rows - + /// already consumed - in front of this one's. Unlike the glob-and-newest form, this takes + /// exactly the file the checkpoint recorded, so a leftover from an unrelated run cannot be + /// merged in. Bytes rather than lines: the length was taken from the writer's own position, + /// so it cannot disagree with itself about line endings or a torn final line. + /// + /// What follows a promotion is an append onto the file it landed in, which is the same thing + /// the resumed route does and takes the same hold. The output is asked for BEFORE the move + /// that replaces it, since a rename asks nothing of the lock on either side; + /// is what each platform's lock lets that be. + /// + /// + protected TempPromotion TryPromoteNamedTemp(string outputPath, string tempFileName, + long dataLength, out UnopenableReason? reason, out StagingFailure? staging, + out StagingRemoval? swept) { + reason = null; + staging = null; + swept = null; try { - var tempPath = ResolveNamedTemp(outputPath, tempFileName, dataLength); - if (tempPath == null) return false; + var tempPath = ResolveNamedTemp(outputPath, tempFileName, dataLength, mayWrite: true); + if (tempPath == null) return TempPromotion.NotPromoted; // The claim adoption and the stale-temp sweep take, for the reason they take it: // this copies the file's rows into the output and then unlinks it, so a run that is @@ -1088,124 +1865,1159 @@ protected static bool TryPromoteNamedTemp(string outputPath, string tempFileName // running right now, in which case this is its live output. Taken before anything // reads the file, so both platforms decide it here rather than Windows failing the // unlink afterwards with the rows already copied. - if (!CanTakeExclusively(tempPath)) return false; - - // Staged like the other forms, so the destination is replaced in one Move rather - // than truncated and refilled in place. - var adoptPath = outputPath + ".adopt"; - using (var writer = new FileStream(adoptPath, FileMode.Create, FileAccess.Write)) + // + // And held from here through the move, rather than taken and let go before the copy. + // Two runs over one command line reach this line together, and a claim released + // before the copy is one the second run passes as well: both staged the same bytes, + // one of them lost the staged file's own claim, and the run that lost it read the + // temp as missing, deleted the position counting its items, and exported fresh over + // the output the other was holding and appending to. One claim over the whole + // promotion refuses the second run for as long as the first is promoting - and by + // the time it is let go the temp is gone and the output is this run's, which is what + // refuses it from there on. + FileStream? claimedTemp = null; + FileStream? standing = null; + FileStream? writer = null; + var adoptPath = StagingPathFor(outputPath); + var staged = false; + var createdOutput = false; + var landed = TempPromotion.NotPromoted; + try { - using var temp = new FileStream(tempPath, FileMode.Open, FileAccess.Read); - var buffer = new byte[81920]; - long remaining = dataLength; - while (remaining > 0) + // The staging name, cleared before either claim below is taken, and the reason + // nothing is ever opened through that name with a mode that follows it or waits + // on it. First, because the sweep asks for what stands at the name exclusively + // and a hard link there is an ALIAS for an inode this run is about to hold: run + // it under those claims and a link to the temp, or to the output, answered its + // own claim Held - the run stopped saying another run had the staged copy open, + // over a name nothing but itself was holding, while the preview holding neither + // deleted the link and answered that the promotion would land. Nothing of this + // promotion's is open here, so an alias for either file answers Taken and is + // unlinked as the alias it is, leaving what it pointed at alone. + // + // A failure here is its own answer, and not the temp's. Read as "the temp is + // missing or incomplete" - which is what the catch around this whole method used + // to make of a directory at the staging name, another run's handle on it or a + // full disk - the caller said the items were in no file at all, deleted the + // position counting them and swept the temp that was holding every one of them. + if (!SweepStagingName(adoptPath, apply: true, out staging, out swept)) + return landed = TempPromotion.StagingFailed; + + if (OpenExclusively(tempPath, FileShare.None, out claimedTemp, out _) + != FileClaim.Taken) + return TempPromotion.NotPromoted; + + // The output the move will replace, asked for before a byte of it is staged. + // Held or unopenable is the whole answer on either platform, and it is reached + // with the directory beside the output untouched: staging first and asking after + // wrote the whole temp out under a name of this run's - 172 ms and a moved + // directory mtime for a 256 MiB temp - to be taken straight back again. + // + // On Unix the handle is kept from here through the move, which is what closes + // the interval a claim released in front of the rename left open: rename(2) + // honors no lock on either side of it, so a second run that took the output in + // between had its rows replaced under it, in an unnamed inode, with no warning + // on any stream. Windows cannot keep it - it refuses to rename over a + // destination another handle has open - so there the claim is let go where it + // is taken and asked for again past the move. + // + // An output that is not there yet is created here in order to be held, and + // removed again by the finally on every answer but Taken: a promotion that + // stops leaves the directory as it found it, and one that lands replaced the + // file with the staged copy in the rename itself. + landed = ClaimStandingOutput(outputPath, out standing, out reason, + out createdOutput); + if (landed != TempPromotion.Taken) return landed; + + // Staged like the other forms, so the destination is replaced in one Move rather + // than truncated and refilled in place. Under the name the sweep above cleared, + // and created by an open that refuses to use anything already standing there: + // what appears at the name in the instants between the two is this run's staging + // failure and never its copy's destination. + try + { + // CreateNew, so the name is this run's own new entry or the open fails: + // nothing at it is followed, truncated or waited on. ReadWrite, because this + // handle is the hold the run keeps past the rename. FileShare.None from the + // instant the entry exists, so the copy is never a file a second promotion + // can claim, and the discard below can only ever unlink a file this handle + // holds. + writer = ScratchName.CreateNew(adoptPath); + staged = true; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { - var read = temp.Read(buffer, 0, (int)Math.Min(buffer.Length, remaining)); - if (read <= 0) break; - writer.Write(buffer, 0, read); - remaining -= read; + // Something appeared at the name in the instants since the sweep, or the + // directory beside the output refuses a new file in it at all - a mode this + // account may not write, a volume with no room. Neither is a temp that has + // gone missing. + staging = StagingFailureAt(adoptPath, ex, fromCreate: true); + return landed = TempPromotion.StagingFailed; } - if (remaining > 0) + try { - writer.Dispose(); - File.Delete(adoptPath); - return false; + var buffer = new byte[81920]; + long remaining = dataLength; + while (remaining > 0) + { + var read = claimedTemp!.Read(buffer, 0, + (int)Math.Min(buffer.Length, remaining)); + if (read <= 0) break; + writer.Write(buffer, 0, read); + remaining -= read; + } + writer.Flush(); + // Shorter than the length recorded for it, which is the one thing the + // resolve above measured and cannot have measured wrong twice: the temp has + // lost bytes since. Those items are not all in a file this run can reach, + // which is the caller's own route for a temp that is missing or short - the + // only route left to it, now that a staging failure answers for itself. + if (remaining > 0) return landed = TempPromotion.NotPromoted; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + staging = StagingFailureAt(adoptPath, ex); + return landed = TempPromotion.StagingFailed; } + + landed = MoveOntoTheOutputAndKeepIt(writer, adoptPath, outputPath, dataLength, + out reason, out staging); + // The staged copy's handle is the run's hold on the output past a move that went + // through, and on Windows it was closed before the move. Either way it is no + // longer this frame's to close. + if (landed == TempPromotion.Taken) writer = null; + if (landed != TempPromotion.Taken) return landed; + + // Unlinked after the output was taken, and only where it was taken. A promotion + // that does not land leaves the temp, and the checkpoint still naming it, exactly + // as they were found: the position and the file it counts are the pair they were, + // and the run that comes back promotes the same bytes to the same length over the + // same output, which is the one answer here that costs nothing. + // + // The unlink is the finally below, under the claim this run has held the temp by + // since before it read a byte of it - on Windows past that handle instead, since + // a file this process has open cannot be unlinked there without + // FILE_SHARE_DELETE. Nothing is left open for a second run to pass by then: the + // output is this run's, and that is the claim that stops it. + return TempPromotion.Taken; + } + finally + { + // The empty output the standing claim made where there was none. It exists only + // so that an absent output can be refused to a second run, and a promotion that + // did not land has nothing to leave at the path: the run before it left nothing + // there, and a run that comes back to this checkpoint reaches the same branch. + // Taken back under the claim that made it, in the one order this platform takes + // a name back in - a name goes only while the run unlinking it holds what + // stands there, and a file that is no longer this claim's is one it leaves. + // + // Every other answer lets the handle go and unlinks nothing: past the move what + // it locks is an inode with no name left on it, and the file at the path is the + // one the adopt handle holds; on a stop it is let go with nothing replaced. + if (createdOutput && landed != TempPromotion.Taken) + ScratchName.Discard(outputPath, standing); + else standing?.Dispose(); + + // The temp, taken back under the claim it was read through, and only where the + // output was taken: a promotion that did not land leaves it and the checkpoint + // naming it exactly as it found them, which is what lets the next run promote + // the same bytes to the same length over the same output. + if (landed == TempPromotion.Taken) ScratchName.Discard(tempPath, claimedTemp); + else claimedTemp?.Dispose(); + + // Every answer but Taken leaves the disk as it was found, so the copy staged + // beside the output goes with the promotion that did not land: no glob of a + // later run reaches that name, and no run puts it there to keep. Guarded on + // this run having created it, which is now the only way the name can hold a + // file at all - the create refuses anything already there, and the handle it + // comes away with is held until the move - so what this unlinks is always the + // entry this run made and never a staging file a second promotion has open. + if (landed != TempPromotion.Taken && staged) DiscardStagedFile(adoptPath, writer); + else writer?.Dispose(); } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return TempPromotion.NotPromoted; + } + } + + /// + /// The name a promotion stages its copy under: the output's own name with a suffix, in the + /// output's own directory, so the move that puts it at the output path is a rename within one + /// filesystem. No run leaves a file under this name on purpose - it is renamed away where the + /// promotion lands and removed where it stops - so one standing there is a run killed between + /// the two, and takes it with the temps. + /// + protected static string StagingPathFor(string outputPath) => outputPath + ".adopt"; + + /// + /// Clear the staging name, or say why the promotion stops there: + /// decides the entry and this puts its answer into the words a promotion says it in. The + /// kinds and the order they are decided in are that primitive's - the same one the + /// checkpoint and the delta state clear their own scratch names with - and the reason + /// nothing is ever opened through this name with a mode that follows it or waits on it: a + /// promotion that opened it Create/Write staged its copy into whatever stood there, taking + /// a symlink's target away with it and renaming the link onto the output path, and a FIFO + /// with no reader blocked the open forever with this run holding the temp and the output + /// and no way for StopProcessing to reach it. + /// + /// Run first in both passes - before the claim on the temp and the claim on the standing + /// output, and so before anything of this promotion's is open. That place in the order is + /// what lets the sweep's own claim answer at all: run under the promotion's claims, it + /// asked for an entry that may be a hard link to one of the two inodes those claims hold, + /// and a second open file description over a locked inode is refused even from this process + /// - so an alias for the temp, or for the output, read as a staged copy another run had + /// open. + /// + /// + /// + /// Whether the name is cleared or only read. The applying pass takes what stands there off + /// the name; the preview leaves it and reports what a run would take, because a preview + /// removes nothing - its deletions were as real as a run's, so a caller asking what would + /// be touched had a file of theirs deleted to be told that nothing would be. + /// + /// Why the promotion stops here, where it does; null where the name is + /// clear. One wording for one disk state on either pass: a run and a preview of it refuse a + /// directory, or another run's copy, in the same words. + /// What was taken off the name, or - on the preview - what stands there + /// and would be; null where nothing did. The sentence is the pass's own and the kind is the + /// same phrase in both, so a stop that names it and a warning that reports it agree. + private static bool SweepStagingName(string adoptPath, bool apply, + out StagingFailure? failure, out StagingRemoval? found) + { + failure = null; + found = null; + var name = Path.GetFileName(adoptPath); + var entry = apply + ? ScratchName.Sweep(adoptPath, out var stood) + : ScratchName.Inspect(adoptPath, out stood); + switch (entry) + { + case ScratchEntry.Clear: + return true; + case ScratchEntry.Directory: + failure = new StagingFailure(adoptPath, "a directory stands at the path"); + return false; + case ScratchEntry.Held: + failure = new StagingFailure(adoptPath, "another run has the staged copy open"); + return false; + case ScratchEntry.Unreadable: + // The entry could not be read, or could not be unlinked. Refined the way every + // other failed open here is refined, so one disk state is one sentence wherever + // the run met it. + failure = StagingFailureAt(adoptPath, stood!); + return false; + default: + var kind = StagingEntryKind(entry); + found = new StagingRemoval( + apply ? SweptFromStagingName(name, kind) : WouldSweepStagingName(name, kind), + kind); + return true; + } + } + + /// + /// What stood at the staging name, in the phrase every sentence about it uses: the warning a + /// run writes when it took the entry off the name, the -WhatIf line naming what a run would + /// take off it, and the stop that has to say the name was cleared before the staging failed. + /// One phrase per kind, because the same entry read three ways in three sentences is three + /// disk states as far as the reader can tell. + /// + private static string StagingEntryKind(ScratchEntry entry) => entry switch + { + ScratchEntry.Link => "a link, and not what it pointed at", + ScratchEntry.Leftover => "a copy left by an interrupted promotion", + // A regular file, and the one thing that is known about it is that this account could + // not open it. Read into the sentence below, a 0444 or a mode-000 file was reported as a + // pipe or a socket - a kind of entry it is not, and one the caller cannot have put there + // by the means they actually used. + ScratchEntry.Unopenable => "a copy this account cannot open", + _ => "an entry no copy could be staged in, such as a pipe or a socket", + }; + + /// + /// What a preview says about an entry at the staging name: what the run would take off it + /// and what it would put there instead. Said as a WARNING beside the gate's own line, since + /// the gate names one of two actions on -OutputFile and this is neither - and the create + /// test is skipped for that name, because nothing standing at it can be tested without + /// removing it and this pass removes nothing. + /// + private static string WouldSweepStagingName(string name, string kind) => + $"The run would remove what stands at the staging name '{name}' ({kind}) and stage its " + + "copy there."; + + /// + /// What a sweep of the staging name took off it, or what a preview found standing there: the + /// sentence the pass reports and the phrase another sentence names it by. + /// + protected readonly record struct StagingRemoval(string Sentence, string Kind); + + /// + /// What came off the staging name, in the one sentence both cmdlets say it in. Said as a + /// WARNING, since written to the verbose stream it reached nobody who had not asked for + /// verbose - and said once the run's own outcome for the recovery is known, because a + /// warning is a terminating error under -WarningAction Stop: written where the deletion + /// happens, it ended the run with the entry gone, the temp still holding the items and + /// nothing recovered. The verbose line stays at the deletion as the record of when it + /// happened, so -WarningAction SilentlyContinue does not hide it entirely. + /// + private static string SweptFromStagingName(string name, string kind) => + $"Removed what stood at the staging name '{name}': {kind}."; + + /// + /// Why a promotion could not put its copy at the output path, and where it was writing it. + /// The reason is 's, which is the same refinement the claim + /// on a file carries out of a failed open: a caller quotes it inside a sentence it + /// punctuates itself, and the two passes over one disk state have to answer in one wording. + /// + protected readonly record struct StagingFailure(string Path, string Reason); + + /// + /// Whether the open that failed was the create the copy is staged + /// with, whose refusal of an existing name is the runtime's "already exists" for every kind + /// of entry alike - so the refinement has to ask the filesystem what the entry is. + private static StagingFailure StagingFailureAt(string adoptPath, Exception ex, + bool fromCreate = false) => + new(adoptPath, ReasonForUnopenable(adoptPath, ex, fromCreate).Reason); + + /// + /// Put the staged file at the output path and come away holding what is now there, or replace + /// nothing and say why. A promotion is followed by an append onto the file it landed in, so + /// that file has to be this run's from the move onward - and the output standing at the path + /// has to be nobody else's before the move, because a rename asks nothing of the lock on + /// either side of it. + /// + /// On Unix .NET backs with an advisory flock, which belongs to + /// the open file description, and rename(2) moves a directory entry: so the lock the + /// staged copy was created under is still held over the same inode once that inode is the + /// output. The hold therefore travels with the move, and it is the handle the copy was + /// written through - the one the create came away with, never let go and never asked for a + /// second time. Measured on this host: a FileShare.None open of the moved-to path, from this + /// process and from another, is refused while that handle is open, which is also why the + /// output is not asked for again afterwards - flock refuses a second open file description + /// from this very process. + /// + /// + /// The claim on the output the move replaces is 's, taken by + /// the caller before it staged anything and still standing here. Asking for it and letting it + /// go in front of the rename, which is what this did, left the interval between the two + /// instructions open: rename(2) does not honor the lock on the destination either, so a + /// second run that took the output in that interval had it replaced under it - its rows in an + /// unlinked inode, and no warning on any stream. Kept across the move instead, the claim + /// refuses that run for as long as the promotion lasts, and past the move the handle this run + /// has open IS the output. That holds on both branches of the claim: an output that is not + /// there is created by it in order to be held, so a promotion over an absent output refuses + /// a second one the same way a promotion over an existing output does. + /// + /// + /// Windows will not rename over a destination another handle has open, nor rename a source + /// this process has open, so neither hold can travel with the move there. The order this had + /// stands: claim the standing output and let it go, stage, close the copy, move, take the + /// output on the far side. What that leaves is a window two statements wide rather than a + /// page fetch wide, and an output already replaced where the claim on the far side fails. + /// + /// + private TempPromotion MoveOntoTheOutputAndKeepIt(FileStream staged, string adoptPath, + string outputPath, long dataLength, out UnopenableReason? reason, + out StagingFailure? staging) + { + reason = null; + staging = null; + if (OperatingSystem.IsWindows()) + { + // Closed before the move, which is the order that platform allows: it will not + // rename a source this process has open. What that leaves is a name unheld for the + // width of the two instructions, where every other platform holds it from the + // create through the rename. + staged.Dispose(); + if (!TryMoveStagedFile(adoptPath, outputPath, out staging)) + return TempPromotion.StagingFailed; + return TakeOutputForCheckpoint(outputPath, dataLength, out reason) switch + { + CheckpointOutput.Taken => TempPromotion.Taken, + CheckpointOutput.HeldByAnotherRun => TempPromotion.OutputHeld, + CheckpointOutput.CannotBeOpened => TempPromotion.OutputUnopenable, + // The file this run wrote a moment ago is gone, or shorter than it wrote it. + // Something else has replaced it, so those rows are in no file again and the + // caller's own route for a promotion that did not land is the answer. + _ => TempPromotion.NotPromoted, + }; + } + + // The handle the copy was written through, still open, so the lock travels with the + // inode through the rename. Asking for the name again here - which is what this did - + // let go of it between the write and the claim: a second promotion took the copy in + // that interval and had it unlinked under its handle by the run that wrote it. There is + // no interval now, and no second claim to be refused: the name has been this run's + // since the create made it. + if (!TryMoveStagedFile(adoptPath, outputPath, out staging)) + return TempPromotion.StagingFailed; + + // The handle names the output now, and the run keeps it until its writing ends. Its + // length is the length just written, which is what the caller repoints the checkpoint + // at; nothing reads the path it was opened under, which is a name no longer on disk. + ReleaseOutputHold(); + _outputHold = new OutputHold(staged); + return TempPromotion.Taken; + } + + /// + /// The rename itself, and whether it went through. Everything up to it leaves the disk as it + /// was found, so a move that fails has replaced nothing either - and saying so is the answer, + /// where the catch around the whole promotion would have called it a temp that had gone. + /// + private static bool TryMoveStagedFile(string adoptPath, string outputPath, + out StagingFailure? staging) + { + staging = null; + try + { File.Move(adoptPath, outputPath, overwrite: true); - try { File.Delete(tempPath); } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { } return true; } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { + staging = StagingFailureAt(adoptPath, ex); return false; } } /// - /// Cut the output back to the length a checkpoint recorded for it, dropping anything the - /// interrupted run wrote after its last save. Those items are re-fetched, so dropping them - /// is what keeps a resume from duplicating them. Returns false when the output is shorter - /// than recorded, which means it is no longer the file the checkpoint describes. + /// Claim the output a promotion is about to replace and, on Unix, come away holding it. A + /// rename asks nothing of the lock on either side of it, so the interval between letting this + /// claim go and the move is one a second run can take the output in - and it is measured in + /// instructions only until something widens it. The handle is therefore kept from before the + /// staging through the rename, and let go on the far side, where what it locks is an inode + /// with no name left on it. + /// + /// Windows keeps nothing: it refuses to rename over a destination another handle has open, so + /// a claim held here would fail the move itself. There the claim is let go where it is taken + /// and the output is asked for again past the move, which is the order that platform's lock + /// allows. + /// + /// + /// The file's length is no part of the question, which is what tells this from + /// , whose open this otherwise is. No caller of + /// this one can read anything from a length. A promotion replaces the output whatever length + /// it has. And a checkpoint naming a temp that has gone says its items were in that temp, so + /// an output nothing holds is evidence of nothing at all: two rows of one shape are two rows + /// of another shape's length, so a previous run's file measures exactly what a promotion + /// would have left about as often as a promoted one does. + /// + /// + /// FileMode.OpenOrCreate, so the claim is taken on the one branch it used to have nothing to + /// hold. An output that is not there is the state a fresh run's promotion is the ordinary end + /// of, and FileMode.Open answered that Absent with no handle: nothing was held through the + /// staging or the rename, so two runs with different temps over one output path both got + /// past here, both renamed a copy onto that path, and the first one's rows ended in an + /// unlinked inode with its own handle appending into nothing on any disk. The claim makes + /// the file in order to hold it, which refuses the second run exactly as an existing output + /// would - and comes back, so a promotion that does not land can + /// take away again the empty file it made. + /// + /// + /// + /// Whether the file at the path is this claim's own, made by the open because nothing stood + /// there. The claim answers it, since OpenOrCreate never reports an absent file and the + /// claim is the one that knows which of its attempts came away with the handle: a claim + /// asked again over a name that moved is a claim on a file it did not make, and a file a + /// second mgx run put there between the question and the open is one that run's own claim + /// refuses. What is not told apart is a file nothing holds that appeared in that window - a + /// shell's touch, an older mgx, any writer that takes no claim - which this claim opens, + /// reports here as its own, and takes back again where the promotion stops. + /// + /// The claim, still open, for the caller to let go past the move; + /// null on Windows and where the open itself was refused. Open where the claim was granted + /// and could not be verified, for the finally to take back what the attempt made. Where + /// is true this is a handle on an empty file of this run's + /// making. + private static TempPromotion ClaimStandingOutput(string outputPath, out FileStream? standing, + out UnopenableReason? reason, out bool created) + { + if (OperatingSystem.IsWindows()) + { + // Nothing is made here, because nothing is kept: this platform lets the claim go + // where it takes it, so a file created to hold would be released in the same breath + // and refuse no second run - an entry at the caller's output path for no gain. The + // window past the move is that platform's, as it was. + created = false; + var released = OpenExclusively(outputPath, OutputHoldShare, out standing, out reason); + standing?.Dispose(); + standing = null; + return PromotionForStandingClaim(released); + } + + // OpenOrCreate, and the claim makes the file only where nothing stands at the name and + // no link stands there either: a link is followed by a create, so a dangling one would + // put this run's empty file wherever it points - outside the output's own directory - + // and the move that follows replaces the link itself, which leaves that file behind. + // That state is read off the directory entry, without opening it, and read again by + // every attempt the claim makes. + var claim = OpenExclusively(outputPath, OutputHoldShare, out standing, out reason, + FileMode.OpenOrCreate, out created, out var askedToMakeIt); + + // The output is not there and could not be made, so there is nothing held and nothing + // made to take back - and nothing about a second run to report either. What refused it + // is the directory beside it, and what a run does about a directory it cannot put a file + // in is stop on the staging create, in that same directory: the one question both passes + // ask, and the one wording they both answer in. So the answer is left to it rather than + // named here in words the preview has no way to reach. + // + // Read off the attempt that gave the answer, and not off a second look from here. The + // claim decides per attempt whether it is making the file: a name a first look found a + // file at is a create the directory refused, once that file leaves the name in between, + // and a name a create was refused at can have a file standing at it a line later. A + // look from here reads each of those as the other. + // + // And on the create having been refused, which is what leaves nothing held: a create the + // claim was granted and then could not verify is Unopenable off the same attempt, with + // the file made and this run holding it. Left to the staging create, that one would go on + // to rename a copy onto an output no run here can say is still the one it claimed. + if (claim == FileClaim.Unopenable && askedToMakeIt && standing is null) + { + reason = null; + return TempPromotion.Taken; + } + + return PromotionForStandingClaim(claim); + } + + /// + /// + /// The same question, let go again with nothing taken, nothing cut and nothing made, for the + /// callers that only have to report on the file. FileMode.Open here and not the claim's + /// OpenOrCreate: a pass that holds nothing has no second run to refuse, and an output that is + /// not there is one the promotion creates - which is the answer this gives it, and it gives + /// it without leaving a file of its own at the caller's output path. Whether that directory + /// will take a new file at all is a different question, and the staging create asks it in + /// the same directory on both passes. /// - protected static bool TryTrimOutputToCheckpoint(string outputPath, long dataLength) + private static TempPromotion StandingOutputRefusal(string outputPath, + out UnopenableReason? reason) + { + var claim = OpenExclusively(outputPath, OutputHoldShare, out var standing, out reason); + standing?.Dispose(); + return PromotionForStandingClaim(claim); + } + + /// + /// What a claim on the output a promotion would replace means for the promotion. Absent is + /// not among the answers, and for two different reasons: the claim creates the file rather + /// than report it missing, and the report-only form reads a missing output as one the + /// promotion would make. + /// + private static TempPromotion PromotionForStandingClaim(FileClaim claim) => claim switch + { + FileClaim.Held => TempPromotion.OutputHeld, + FileClaim.Unopenable => TempPromotion.OutputUnopenable, + _ => TempPromotion.Taken, + }; + + /// + /// The same answer in the words the two cmdlets build a stop out of. + protected static ClaimRefusal RefusalForStandingOutput(string outputPath, + out UnopenableReason? reason) => + StandingOutputRefusal(outputPath, out reason) switch + { + TempPromotion.OutputHeld => ClaimRefusal.AnotherRunHasIt, + TempPromotion.OutputUnopenable => ClaimRefusal.CannotBeOpened, + _ => ClaimRefusal.None, + }; + + /// + /// Undo the staging. A promotion that stops before its move replaced nothing, and the file it + /// had written beside the output goes with it: no glob of a later run reaches that name, and + /// no run put it there to keep. + /// + /// Unlinked under the handle that holds it, which is what closes the interval between the + /// two: released first, the name was decided and unheld for as long as the discard took, and + /// a second run that swept it in that interval created its own file at the name and had it + /// unlinked from under the handle it was holding it by. Windows keeps dispose-then-delete, + /// since a file this process has open cannot be unlinked there without FILE_SHARE_DELETE. + /// + /// + /// This run's own handle on the entry, disposed here; null where it has + /// none. + private static void DiscardStagedFile(string adoptPath, FileStream? held) => + ScratchName.Discard(adoptPath, held); + + /// + /// Point the checkpoint at the output this run now holds, and save it there. The items are in + /// that file, so a second interruption must not promote the same temp again - and the file it + /// would promote is already gone. + /// + /// The length is the held stream's, read off the handle this run has open rather than stat'd + /// off the name a second time. They are the same number where nothing has moved; where + /// something has, the handle is the file this run wrote and the name is whatever stands there + /// now. + /// + /// + protected void RepointCheckpointAtHeldOutput(PaginationCheckpoint checkpoint, + string checkpointPath, string outputPath, OutputHold hold) { try { - // A checkpoint is untrusted input once it is on disk, and SetLength rejects a - // negative length with an ArgumentOutOfRangeException the catch below does not - // cover - so a hand-edited length escaped as a terminating error naming neither the - // checkpoint nor the file, and left itself on disk to fail the same way next run. - // ResolveNamedTemp already guards its own length; this is the same guard. - if (dataLength < 0) return false; - if (!File.Exists(outputPath)) return false; - var actual = new FileInfo(outputPath).Length; - if (actual < dataLength) return false; - if (actual == dataLength) return true; - using var fs = new FileStream(outputPath, FileMode.Open, FileAccess.Write); - fs.SetLength(dataLength); - return true; + checkpoint.TempFile = null; + checkpoint.OutputFile = Path.GetFullPath(outputPath); + checkpoint.DataLength = hold.Length; + checkpoint.Save(checkpointPath); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { - return false; + WriteWarning($"Checkpoint save failed after recovery: {ex.Message}"); + } + } + + /// + /// What a second opener of the output may still do while a resumed run holds it, which is + /// not the same question on the two platforms because the lock is not the same lock. + /// + /// .NET on Unix backs FileShare with an advisory flock: + /// takes LOCK_EX, every other mode takes LOCK_SH, and LOCK_SH does not refuse a second + /// writer - measured on this host, where a second FileStream opened Append/Write/Read + /// against a LOCK_SH holder succeeds. LOCK_EX is therefore the only mode that makes a second + /// run's claim fail, and it refuses every .NET opener including a reader; a reader that does + /// not take a lock at all, such as tail or cat, reads the file as before. + /// + /// + /// Windows needs no such trade. The handle is opened for writing, so a second writer is + /// refused by its access rather than by its sharing, and is + /// what lets a reader keep looking at the file for the length of the run. + /// + /// + private static FileShare OutputHoldShare => + OperatingSystem.IsWindows() ? FileShare.Read : FileShare.None; + + /// + /// The output of a resumed run, open, cut back to the length its checkpoint records, and + /// this run's until the writing ends. The claim on that file and the handle the writing goes + /// through are one handle: taken and let go a page-fetch apart, they left a window in which + /// a second run over the same command line passed the same claim, cut the same file back and + /// appended to it - nine lines where seven belonged, and in a sync a delta token advanced + /// past the difference. + /// + protected sealed class OutputHold : IDisposable + { + private readonly FileStream _stream; + + internal OutputHold(FileStream stream) => _stream = stream; + + /// The file's length, read off the open handle rather than the directory entry. + internal long Length => _stream.Length; + + /// + /// Cut the file back to the length the checkpoint recorded, dropping whatever the + /// interrupted run wrote after its last save. Those items are re-fetched, so dropping + /// them is what keeps a resume from writing them twice. + /// + internal void TrimTo(long dataLength) => _stream.SetLength(dataLength); + + /// + /// A writer onto the end of the same handle. Never a second open of the output: on Unix + /// the hold's own LOCK_EX refuses it - flock belongs to the open file description, so + /// even this process is refused - and on Windows the hold's write access does. + /// + internal StreamWriter AppendingWriter() + { + _stream.Position = _stream.Length; + // What StreamWriter(path, append) would have used: UTF-8 with no byte-order mark, + // throwing on unpaired surrogates, over a 1 KiB buffer. leaveOpen keeps the hold + // standing when the writer closes - the run has a checkpoint to delete and a + // summary to write before the file is anyone else's. + return new StreamWriter(_stream, new UTF8Encoding(false, true), 1024, leaveOpen: true); } + + public void Dispose() => _stream.Dispose(); + } + + /// + /// The hold this run has on the output its own checkpoint records items into, from the + /// reconcile that took it until the writing ends. Null on every other run: a fresh export + /// collects into a temp of its own and promotes it, and a run with nothing to resume has no + /// output to keep. + /// + private OutputHold? _outputHold; + + /// The hold, where this run has one. + protected OutputHold? HeldOutput => _outputHold; + + /// + /// Let the output go. Called where the writing ends, on every way out of the attempt loop, + /// and again from by way of + /// - so a run that ends between the reconcile and the loop, on a + /// missing Graph connection or a pipeline stop, releases it too. Idempotent. + /// + protected void ReleaseOutputHold() + { + var hold = Interlocked.Exchange(ref _outputHold, null); + hold?.Dispose(); } /// - /// Whether TryPromoteNamedTemp would find a temp to promote, without promoting it. The - /// ShouldProcess gate has to name the action a real run would take before the run is - /// allowed to take any, and -WhatIf has to leave every staged file exactly as it found it. - /// The claim is asked for here too, and released again: a temp another run holds is one a - /// real run would refuse, so answering from the file's existence alone reported a recovery - /// that would not have happened. + /// Releases the output hold when the enclosing scope ends, whichever way it ends: the + /// summary object, a reported error, a cancellation, or an exception on its way out of the + /// attempt loop. Declared with using above the loop rather than written as a finally + /// around it, which is the same guarantee without moving five hundred lines sideways. /// - protected static bool CanPromoteNamedTemp(string outputPath, string tempFileName, long dataLength) + protected readonly struct OutputHoldScope(MgxCmdletBase cmdlet) : IDisposable { + public void Dispose() => cmdlet.ReleaseOutputHold(); + } + + /// + protected OutputHoldScope ReleasingOutputWhenWritingEnds() => new(this); + + /// + /// What this run may do with the output its own checkpoint records its items into: take it, + /// stop, or start over. One open answers both halves of the question - whether the file is + /// this run's to write and whether it is the file the checkpoint describes - because they + /// are one file and one handle. Asking them separately, with a claim let go in between, + /// answered a directory entry twice and a file's contents never: File.Exists is false + /// for a file in a directory this account cannot search, so a whole unreadable output was + /// reported as no longer holding the items it held and the position counting them deleted. + /// + protected enum CheckpointOutput + { + /// + /// It is the file the checkpoint describes, it now ends exactly at the recorded length, + /// and this run holds it for the rest of the writing. + /// + Taken, + + /// Another run has it open. + HeldByAnotherRun, + + /// + /// It is not a file this run can write at the recorded offset - mode bits, a denying + /// ACL, a read-only filesystem, a directory standing at the path, a symlink loop, or a + /// FIFO with no offset to write at. Nothing here knows whether the items are in it, so + /// the caller stops rather than report a file it never read as a file that has lost its + /// contents. + /// + CannotBeOpened, + + /// + /// It is not the file the checkpoint describes: absent, shorter than the recorded + /// length, or recorded under a length that is not one. The items counted are in no file, + /// which is what the callers delete the checkpoint over and enumerate again from the + /// beginning. + /// + NotTheRecordedFile, + } + + /// + /// Take the output for the rest of this run and cut it back to the length its checkpoint + /// records. The handle is kept in where the answer is + /// , and released by + /// where the run stops appending to it. + /// + protected CheckpointOutput TakeOutputForCheckpoint(string outputPath, long dataLength, + out UnopenableReason? reason) + { + // A checkpoint is untrusted input once it is on disk, and SetLength rejects a negative + // length with an ArgumentOutOfRangeException the catch below does not cover - so a + // hand-edited length escaped as a terminating error naming neither the checkpoint nor + // the file, and left itself on disk to fail the same way next run. ResolveNamedTemp + // already guards its own length; this is the same guard, and it is above the open + // because a length that is not one describes no file worth opening. + reason = null; + if (dataLength < 0) return CheckpointOutput.NotTheRecordedFile; + + var claim = OpenExclusively(outputPath, OutputHoldShare, out var stream, out reason); + if (claim != FileClaim.Taken) + { + // An answer that is not Taken can still carry a handle: a claim granted over the + // file and then refused because the question that finishes it could not be answered + // comes back Unopenable with the file open, so that the caller lets go of it the way + // it lets go of one it kept. Nothing is kept here, so it goes here. + stream?.Dispose(); + return claim switch + { + FileClaim.Held => CheckpointOutput.HeldByAnotherRun, + FileClaim.Unopenable => CheckpointOutput.CannotBeOpened, + _ => CheckpointOutput.NotTheRecordedFile, + }; + } + + var hold = new OutputHold(stream!); try { - var tempPath = ResolveNamedTemp(outputPath, tempFileName, dataLength); - return tempPath != null && CanTakeExclusively(tempPath); + // Off the handle, not the directory entry: the file this run has is the file it + // measures, and no second stat can disagree with it. + if (hold.Length < dataLength) + { + hold.Dispose(); + return CheckpointOutput.NotTheRecordedFile; + } + if (hold.Length > dataLength) hold.TrimTo(dataLength); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // The cut failed on a handle this run holds, so nothing else is writing the file and + // the account can open it. Whatever the filesystem is refusing, this run has not + // read the file's contents and may not say the items are gone from it. The open + // itself did not fail, so OpenExclusively left no reason above - this is the one + // Unopenable a caught exception names rather than a fixed phrase or ex is + // UnauthorizedAccessException, since a permission that blocks only a truncate and + // not the open that preceded it is not itself the more common shape. + hold.Dispose(); + reason = new UnopenableReason(FirstLine(ex.Message).TrimEnd('.'), + ex is UnauthorizedAccessException); + return CheckpointOutput.CannotBeOpened; + } + + ReleaseOutputHold(); + _outputHold = hold; + return CheckpointOutput.Taken; + } + + /// + /// The same answer without taking anything: the open is made and let go, and nothing is cut. + /// The export's ShouldProcess gate has to name the action a real run would take before the + /// run is allowed to take any, and -WhatIf has to leave every file exactly as it found it - + /// including its length. + /// + protected static CheckpointOutput WouldTakeOutputForCheckpoint(string outputPath, + long dataLength, out UnopenableReason? reason) + { + reason = null; + if (dataLength < 0) return CheckpointOutput.NotTheRecordedFile; + + var claim = OpenExclusively(outputPath, OutputHoldShare, out var stream, out reason); + using (stream) + { + return claim switch + { + FileClaim.Held => CheckpointOutput.HeldByAnotherRun, + FileClaim.Unopenable => CheckpointOutput.CannotBeOpened, + FileClaim.Absent => CheckpointOutput.NotTheRecordedFile, + _ => stream!.Length >= dataLength + ? CheckpointOutput.Taken + : CheckpointOutput.NotTheRecordedFile, + }; } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { return false; } } /// - /// Whether the temp a checkpoint names is one another run has open. It is the single reason - /// a promotion fails that says nothing is wrong with the files: every row the checkpoint - /// counted is in that temp, and the enumeration that counted them is still collecting into - /// it. The other failures - no temp, a temp shorter than the recorded length - mean those - /// items are in no file at all, which is what the callers delete the checkpoint over and - /// re-enumerate. A checkpoint whose temp is being written is the position a running export - /// resumes from, so deleting it, or reporting its items as not on disk, describes a run - /// that has not failed and takes away what it would have come back to. + /// The refusal half of , for the two callers that have to say + /// which file a stop is about and why. /// - protected static bool NamedTempIsHeld(string outputPath, string tempFileName, long dataLength) + protected static ClaimRefusal RefusalFrom(CheckpointOutput taken) => taken switch { + CheckpointOutput.HeldByAnotherRun => ClaimRefusal.AnotherRunHasIt, + CheckpointOutput.CannotBeOpened => ClaimRefusal.CannotBeOpened, + _ => ClaimRefusal.None, + }; + + /// + /// What TryPromoteNamedTemp would answer, without promoting anything. The ShouldProcess gate + /// has to name the action a real run would take before the run is allowed to take any, and + /// -WhatIf has to leave every staged file exactly as it found it. The claim on the temp is + /// asked for here too, and released again: a temp another run holds is one a real run would + /// refuse, so answering from the file's existence alone reported a recovery that would not + /// have happened. Which of the two ways that claim failed is not reported - a temp another + /// run holds and a temp this account cannot open are both temps a real run does not promote, + /// and there is one answer to give the gate either way. The caller that has to tell them + /// apart in a sentence asks . + /// + /// The output the promotion would land in is asked about as well, the way the applying pass + /// asks and with nothing taken and nothing cut. A promotion over an output another run holds + /// stops the run, which is neither of the two actions the gate names, and a preview that + /// never asked reported an append over a file the run would not have reached. + /// + /// + /// The staging name is the one place a real run is not read-only, and the preview reads it + /// anyway: asking only whether something already stands there previewed a recovery over an + /// output directory this account cannot write in at all. Where the name is clear, the + /// preview makes the same create a real run makes, at no length and removed again under the + /// handle that made it, so the two passes fail on the same open for the same reason. Where + /// something already stands at the name, the preview leaves it alone: sweeping it would + /// make deletions as real as a run's, so a caller asking what would be touched would have + /// had a link, a pipe or a leftover copy of theirs deleted just to be told that nothing + /// would be. What stands there comes back instead, for the caller to warn about. What it + /// leaves behind is what it found: the two files the checkpoint stands for are untouched, + /// and the staging name holds what it always held. + /// + /// + protected static TempPromotion CanPromoteNamedTemp(string outputPath, string tempFileName, + long dataLength, out UnopenableReason? reason, out StagingFailure? staging, + out StagingRemoval? swept) + { + reason = null; + staging = null; + swept = null; try { - var tempPath = ResolveNamedTemp(outputPath, tempFileName, dataLength); - return tempPath != null && !CanTakeExclusively(tempPath); + var tempPath = ResolveNamedTemp(outputPath, tempFileName, dataLength, mayWrite: false); + if (tempPath == null) return TempPromotion.NotPromoted; + + // The name the copy is staged under, read in the same place in the order a real run + // sweeps it: first, before either file the checkpoint stands for is asked about, so + // a hard link at the name is an alias nothing of this pass's holds. The gate names + // the action a run would take, and a run that would stop on its own staging was + // previewed as a recovery: a directory there, or another run's copy, is refused in + // the run and refused here in the same words. + // + // Read and not swept. A preview that cleared the name made deletions as real as a + // run's - a caller asking what would be touched had a link, a pipe or a leftover + // copy of theirs deleted in order to be told that nothing would be, and on -WhatIf + // over a checkpoint the run would then have stopped on. What stands there comes back + // instead, for the caller to say what the run would take off the name. + var adoptPath = StagingPathFor(outputPath); + if (!SweepStagingName(adoptPath, apply: false, out staging, out swept)) + return TempPromotion.StagingFailed; + + if (!CanTakeExclusively(tempPath)) return TempPromotion.NotPromoted; + + var standing = StandingOutputRefusal(outputPath, out reason); + if (standing != TempPromotion.Taken) return standing; + + // And the create itself, at no length and taken back again, where the name is clear. + // Claiming the name was not the question a run answers there: the create is what a + // directory this account may not write in refuses, and asking only whether something + // already stands at the name previewed a recovery over an output the run could never + // have staged a copy beside. Made with the mode the run makes it with, so the two + // passes fail on the same open for the same reason. + // + // Skipped where something stands at the name: the run would remove that entry and + // create its own, and there is no way to make the same create without taking the + // entry off the name first. The caller reports what the run would remove instead of + // this pass removing it. + if (swept == null) + { + FileStream probe; + try + { + probe = ScratchName.CreateNew(adoptPath); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + staging = StagingFailureAt(adoptPath, ex, fromCreate: true); + return TempPromotion.StagingFailed; + } + + // The entry this pass made is the only thing in the directory that was not there + // before it, and a preview leaves the directory as it found it. Unlinked under + // the handle that made it, so no second run can take the name in between. + DiscardStagedFile(adoptPath, probe); + } + + return TempPromotion.Taken; } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { return false; } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return TempPromotion.NotPromoted; + } + } + + /// + /// Why one of the two files a checkpoint stands for - the temp it names, or the output it + /// records its items into - is not this run's to take, where it is not. + /// + protected enum ClaimRefusal + { + /// + /// Nothing refuses it: there is no such file of this checkpoint's, or nothing stands + /// between this run and it - which, for a temp whose promotion has already failed, + /// means the items it counted are not all in a file this run can reach. + /// + None, + + /// Another run has it open. + AnotherRunHasIt, + + /// This account cannot open it read-write. + CannotBeOpened, + } + + /// Which of the two files a checkpoint stands for a stop is about. + protected enum CheckpointFile + { + /// + /// The temp it names, holding the items it counts while the output still holds an + /// earlier run's. + /// + NamedTemp, + + /// + /// The output it records its items into, which a run that has already resumed writes + /// straight to - so its checkpoint names no temp at all. + /// + Output, } /// - /// Whether TryTrimOutputToCheckpoint would find the output the checkpoint describes, - /// without cutting it. + /// How this run reached a checkpoint's output: appending straight into it, promoting a + /// named temp over it, or exporting fresh after the named temp it counted on had already + /// vanished. A stop over the output reads differently on each - where the checkpoint's + /// items are, and what going on would have done to the file - so a caller building the + /// sentence needs to be told which one it is on rather than assuming the appending shape, + /// which used to be the only one that reached a temp-based checkpoint's own claim on the + /// output at all. /// - protected static bool CanTrimOutputToCheckpoint(string outputPath, long dataLength) + protected enum CheckpointStopRoute + { + /// + /// The checkpoint names no temp: a run that had already resumed was appending straight + /// into the output, so the items it counts are in that file, and going on would cut it + /// back to the recorded length under the run still writing past it. + /// + Appending, + + /// + /// The checkpoint names a temp that is still there: the items it counts are in that + /// temp, and going on would promote them, replacing the output outright rather than + /// appending to it. + /// + Promoting, + + /// + /// The checkpoint names a temp that is gone: the items it counted are recoverable from + /// no file this run can reach, and going on would enumerate the collection again from + /// the beginning and replace the output with that result. + /// + FreshAfterVanishedTemp, + } + + /// + /// Why a claim answered , carried out of + /// with the verdict so a caller building a stop can name the + /// open's own reason instead of guessing at one. Built by + /// , which is also what a promotion's staging failure is + /// refined by, so one disk state reads the same whichever site reached it. + /// carries no sentence-ending period of its own - taken off where the + /// runtime's words are read - since a caller quotes it inside a sentence it punctuates + /// itself. is read off which failure produced + /// , not out of its text: the text is the runtime's own words, in + /// whatever language this host reports errors in, and is not this run's to parse for a + /// cause it already knows structurally. + /// + /// and are the two causes worth + /// telling from a permission on the file, because they are the two with a different way out. + /// A directory standing at the output path opens as an access failure on both platforms - + /// EISDIR and a denying ACL arrive in the same catch - and no grant makes a directory + /// writable at an offset: what the caller does about it is remove it or point -OutputFile + /// somewhere else. A file whose parent cannot be searched is an access failure naming the + /// file, and the grant that opens it belongs to the directory - so the advice that named the + /// file sent the caller to change a mode that was never the one refusing them. Both are read + /// off the filesystem rather than the errno, since which failure an open produces depends on + /// the mode standing in the way. + /// + /// + protected readonly record struct UnopenableReason(string Reason, bool IsPermission, + bool IsDirectory = false, bool IsUnsearchableParent = false); + + /// + /// Which of the two files a claim refused, and why, or + /// where neither did. The file and the reason decide what the run can honestly say: the + /// file settles what going on would do to it, and the reason settles whether there is a + /// second run to wait for or a mode to change. and + /// refine that for an output refusal - which of the three ways this + /// run reached the output, and, where the claim failed on a mode rather than a hold, the + /// open's own explanation - and are not read for a temp refusal, which has never had more + /// than one route to it. + /// + protected readonly record struct CheckpointFileRefusal(CheckpointFile File, + ClaimRefusal Refusal, CheckpointStopRoute? Route = null, UnopenableReason? Reason = null) + { + /// Nothing refused either file, and the run goes on. + public static CheckpointFileRefusal None => new(CheckpointFile.NamedTemp, ClaimRefusal.None); + + /// Whether a file was refused at all. + public bool Refuses => Refusal != ClaimRefusal.None; + + /// Whether the reason was another run holding it, rather than the mode on it. + public bool Held => Refusal == ClaimRefusal.AnotherRunHasIt; + } + + /// + /// Why the temp a checkpoint names could not be claimed. Two of the answers leave both + /// files exactly where they are, and they are not the same thing to tell the caller. + /// + /// Another run having it open is the single reason a promotion fails that says nothing is + /// wrong with the files: every row the checkpoint counted is in that temp, and the + /// enumeration that counted them is still collecting into it - so the checkpoint is the + /// position a running export resumes from, and deleting it, or reporting its items as not + /// on disk, describes a run that has not failed and takes away what it would have come + /// back to. + /// + /// A temp this account cannot open read-write reaches the same refusal and has no second + /// run behind it at all. Said in the other's words, it sent the reader looking for an + /// export that was never there rather than at the file's owner and mode. + /// + /// The remaining failures - no temp, a temp shorter than the recorded length - mean those + /// items are in no file, which is what the callers delete the checkpoint over and + /// re-enumerate. + /// + protected static ClaimRefusal RefusalForNamedTemp( + string outputPath, string tempFileName, long dataLength, bool mayWrite) { try { - if (dataLength < 0) return false; - var info = new FileInfo(outputPath); - return info.Exists && info.Length >= dataLength; + var tempPath = ResolveNamedTemp(outputPath, tempFileName, dataLength, mayWrite); + if (tempPath == null) return ClaimRefusal.None; + return ClaimExclusively(tempPath, out _) switch + { + FileClaim.Held => ClaimRefusal.AnotherRunHasIt, + FileClaim.Unopenable => ClaimRefusal.CannotBeOpened, + _ => ClaimRefusal.None, + }; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return ClaimRefusal.None; } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { return false; } } + /// + /// Why the checkpoint at -CheckpointPath is not this run's to delete. Every refusal that + /// leaves the file standing records this, and the doors that would otherwise delete it - + /// the completion path, and the sync's 410 - read it and nothing else. A run's own saved + /// position, and a checkpoint refused because the request was rebuilt without a query + /// option the endpoint would not answer, record none: those the doors take with the run. + /// A checkpoint one of whose files - the temp it names, or the output it appends to - could + /// not be claimed reaches no door, because the run that found that out did not go on. + /// + protected enum SparedCheckpoint + { + /// + /// The ownership verdict: the file records another output, or an enumeration nothing + /// beside this run's output corroborates. Whatever wrote it resumes from exactly that. + /// + NotThisRuns, + } + + /// + /// What a door says when it leaves the checkpoint where it was. One reading reaches a door + /// at all: a run that cannot claim a file its own checkpoint stands for stops where it finds + /// out, so no door of a run that got as far as finishing is answering for that file. + /// + protected static string SparedCheckpointVerbose(string occasion) => + $"Left the resume checkpoint alone ({occasion}): it records an enumeration this " + + "run refused as not its own and never wrote over."; + + /// + /// The sentence a stop ends with, and the one a -WhatIf pass over the same condition ends + /// with instead. Both cmdlets refuse a checkpoint whose temp, or whose output, is held or + /// unopenable by ending the run there, and the difference between doing that and reporting + /// that it would be done is a tense. + /// + protected const string RunStopsHere = "This run stops here; nothing was written."; + + /// + protected const string RunWouldStopHere = "This run would stop here; nothing would be written."; + + /// + /// The same for a staging failure, where "nothing was written" is not what the run has to + /// say: the copy it was writing beside the output is gone again, and what the reader needs + /// to hear is that neither of the two files the checkpoint stands for moved. + /// + protected const string NothingWasChanged = "Nothing was changed;"; + + /// + protected const string NothingWouldBeChanged = "Nothing would be changed;"; + /// /// The newest "{output}.{32-hex}.tmp" beside the output that nothing else holds open, or /// null when there is none. Adoption copies the file it picks into the output and then @@ -1220,13 +3032,13 @@ protected static bool CanTrimOutputToCheckpoint(string outputPath, long dataLeng /// for. Taking the claim before anything reads the file also keeps Windows declining here, /// where a live writer already conflicts with the reads below rather than with this. /// - private static FileInfo? NewestUnheldTemp(string outputPath) + private static FileInfo? NewestUnheldTemp(string outputPath, bool mayWrite) { var dir = Path.GetDirectoryName(outputPath); if (string.IsNullOrEmpty(dir) || !Directory.Exists(dir)) return null; var outputName = Path.GetFileName(outputPath); - return Directory.EnumerateFiles(dir, outputName + ".*.tmp") - .Where(p => IsRunTempName(outputName, Path.GetFileName(p))) + return Directory.EnumerateFiles(dir, outputName + ".*.tmp", TempGlobOptions(dir, mayWrite)) + .Where(p => IsRunTempName(dir, outputName, Path.GetFileName(p), mayWrite)) .Select(p => new FileInfo(p)) .OrderByDescending(f => f.LastWriteTimeUtc) .FirstOrDefault(f => CanTakeExclusively(f.FullName)); @@ -1235,22 +3047,79 @@ protected static bool CanTrimOutputToCheckpoint(string outputPath, long dataLeng /// /// Whether TryAdoptOrphanedTemp would find a temp holding the counted items, without /// merging it into anything. + /// + /// The staging name is the one place a real run is not read-only, and the preview reads it + /// anyway: asking only whether something already stands there previewed a recovery where a + /// run would stop instead, since an output directory this account may read and not write + /// refuses the create and nothing standing at the name says so. Where the name is clear, + /// the preview makes the same create a real run makes, at no length and removed again under + /// the handle that made it, so the two passes fail on the same open for the same reason. + /// Where something already stands at the name, the preview leaves it alone and reports what + /// stands there instead, for the caller to warn about. What it leaves behind is what it + /// found: the temp is untouched, the output is still not there, and the staging name holds + /// what it always held. + /// /// - protected static bool CanAdoptOrphanedTemp(string outputPath, long itemCount) + /// The file adoption would take the items from, as soon as it is + /// picked; null where there is none. What a caller names in a staging stop, since a + /// checkpoint on this route records no temp of its own. + /// Why the adoption stops at the staging name, where it does. Its own + /// answer and not a temp that has gone: the items are all in the file + /// names, and a caller that read this as a missing temp deleted + /// the position counting them. + /// What stands at the staging name and what a real run would remove, + /// for the caller to warn about; null where nothing stands there. + protected static bool CanAdoptOrphanedTemp(string outputPath, long itemCount, + out string? tempName, out StagingFailure? staging, out StagingRemoval? swept) { + tempName = null; + staging = null; + swept = null; try { if (itemCount <= 0) return false; if (File.Exists(outputPath)) return false; - var temp = NewestUnheldTemp(outputPath); + var temp = NewestUnheldTemp(outputPath, mayWrite: false); if (temp == null) return false; + tempName = temp.Name; - // Whole rows only, as below. + // Whole rows only, as below. Read and let go before the name beside the output is + // touched, so nothing of this pass's is open over the sweep. var endsWhole = EndsWithNewline(temp.FullName); long counted = 0; - using var reader = new StreamReader(temp.FullName); - while (counted < itemCount && reader.ReadLine() != null) counted++; - return counted >= itemCount && (endsWhole || reader.ReadLine() != null); + using (var reader = new StreamReader(temp.FullName)) + { + while (counted < itemCount && reader.ReadLine() != null) counted++; + if (counted < itemCount || !(endsWhole || reader.ReadLine() != null)) return false; + } + + // And the name the copy is staged under, read where the run below clears it, so the + // two passes answer one disk state in the same words - and read only, because a + // preview removes nothing. The create test follows it where the name is clear; where + // something stands there the run would remove that entry first, and nothing at the + // name can be tested without doing so. + var adoptPath = StagingPathFor(outputPath); + if (!SweepStagingName(adoptPath, apply: false, out staging, out swept)) return false; + if (swept == null) + { + FileStream probe; + try + { + probe = ScratchName.CreateNew(adoptPath); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + staging = StagingFailureAt(adoptPath, ex, fromCreate: true); + return false; + } + + // The entry this pass made is the only thing in the directory that was not there + // before it, and a preview leaves the directory as it found it. Unlinked under + // the handle that made it, so no second run can take the name in between. + DiscardStagedFile(adoptPath, probe); + } + + return true; } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { return false; } } @@ -1273,16 +3142,24 @@ protected static bool CanAdoptOrphanedTemp(string outputPath, long itemCount) /// cannot leave that state - every checkpoint site flushes the writer before recording the /// count, so the rows a checkpoint counts are always terminated - but a temp truncated /// under it by something outside the run can. Returns false when nothing usable exists, - /// leaving the caller to the stale-checkpoint path, which re-enumerates and loses nothing. + /// leaving the caller to the stale-checkpoint path, which re-enumerates and loses nothing - + /// except where came back with it, which is the one false that + /// route must not be taken for. /// - protected static bool TryAdoptOrphanedTemp(string outputPath, long itemCount) + /// + protected static bool TryAdoptOrphanedTemp(string outputPath, long itemCount, + out string? tempName, out StagingFailure? staging, out StagingRemoval? swept) { + tempName = null; + staging = null; + swept = null; try { if (itemCount <= 0) return false; if (File.Exists(outputPath)) return false; - var temp = NewestUnheldTemp(outputPath); + var temp = NewestUnheldTemp(outputPath, mayWrite: true); if (temp == null) return false; + tempName = temp.Name; // A file that ends mid-row can only end mid-row in its LAST one, so the counted // rows are whole unless the count reaches that far - which is the case where @@ -1292,27 +3169,64 @@ protected static bool TryAdoptOrphanedTemp(string outputPath, long itemCount) // Staged so the output appears in one Move. A merge that dies halfway leaves only // the staging file behind, and the caller keeps its checkpoint - the safe direction. + // + // Under the same claim on the name a promotion stages by, and for the same reasons: + // the copy went into whatever stood at the name, so a symlink there sent it into the + // link's target and the move then put the link at the output path. The name is + // cleared first and created by an open that refuses to use anything already there, + // and the handle is held from the create through the rename. long copied = 0; - var adoptPath = outputPath + ".adopt"; - using (var writer = new StreamWriter(adoptPath, append: false)) + var adoptPath = StagingPathFor(outputPath); + if (!SweepStagingName(adoptPath, apply: true, out staging, out swept)) return false; + FileStream? staged = null; + try { - using var reader = new StreamReader(temp.FullName); - string? line; - while (copied < itemCount && (line = reader.ReadLine()) != null) + try { - writer.WriteLine(line); - copied++; + staged = ScratchName.CreateNew(adoptPath); } - countedRowIsWhole = endsWhole || reader.ReadLine() != null; - } - if (copied < itemCount || !countedRowIsWhole) - { - // Temp holds less than the checkpoint promises, or ends inside the last row it - // promises - unusable either way. - File.Delete(adoptPath); - return false; + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // Something appeared at the name in the instants since the sweep, or the + // directory beside the output refuses a new file in it at all. Neither is a + // temp that has gone missing, and the caller's route for one of those + // deletes the position counting items this temp holds every one of. + staging = StagingFailureAt(adoptPath, ex, fromCreate: true); + return false; + } + // What StreamWriter(path, append) wrote through: UTF-8 with no byte-order mark, + // throwing on unpaired surrogates. + using (var writer = new StreamWriter(staged, new UTF8Encoding(false, true), 1024, + leaveOpen: true)) + { + using var reader = new StreamReader(temp.FullName); + string? line; + while (copied < itemCount && (line = reader.ReadLine()) != null) + { + writer.WriteLine(line); + copied++; + } + countedRowIsWhole = endsWhole || reader.ReadLine() != null; + } + if (copied < itemCount || !countedRowIsWhole) + { + // Temp holds less than the checkpoint promises, or ends inside the last row + // it promises - unusable either way. The copy goes with the adoption that + // did not land, unlinked under the handle that holds the name. + DiscardStagedFile(adoptPath, staged); + staged = null; + return false; + } + // Windows will not rename a source this process has open, so there the handle + // goes before the move; everywhere else it holds the name across it. + if (OperatingSystem.IsWindows()) + { + staged.Dispose(); + staged = null; + } + File.Move(adoptPath, outputPath, overwrite: true); } - File.Move(adoptPath, outputPath, overwrite: true); + finally { staged?.Dispose(); } temp.Delete(); return true; } @@ -1751,12 +3665,19 @@ internal static Encoding ResolveEncoding(string? charset) } /// - /// Release the Graph client. Invoked by under - /// the Interlocked guard, so this runs exactly once even when StopProcessing - /// (pipeline-stopping thread) races EndProcessing (pipeline thread). + /// Release the Graph client, and the output a resumed run holds. Invoked by + /// under the Interlocked guard, so this runs exactly once + /// even when StopProcessing (pipeline-stopping thread) races EndProcessing (pipeline thread). + /// + /// The hold is released where the writing ends, which is inside the attempt loop's own + /// finally - this is the backstop for every way out that never reaches it: a run that stops + /// between the reconcile and the loop because there is no Graph connection, and a pipeline + /// stopped from outside. + /// /// protected override void DisposeCore() { _client?.Dispose(); + ReleaseOutputHold(); } } diff --git a/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs b/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs index 80fd976..9ebd131 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs @@ -6,6 +6,7 @@ using Mgx.Cmdlets.Base; using Mgx.Engine.Http; using Mgx.Engine.Models; +using Mgx.Engine.Pagination; using Polly.CircuitBreaker; namespace Mgx.Cmdlets.Cmdlets.Batch; @@ -89,12 +90,42 @@ public class InvokeMgxBatchRequest : MgxCmdletBase public string ApiVersion { get; set; } = "v1.0"; /// - /// Path to a JSONL file where failed batch items (status >= 400) are appended. - /// Each line contains Url, Method, Body (original request), Status, and Error. - /// The file can be re-piped to Invoke-MgxBatchRequest for retry: + /// Path to a JSONL file where the batch's failed and never-sent items are appended: + /// anything the server answered with a status >= 400, the items of a chunk whose own POST + /// failed, which carry that failure's status, and the items of chunks that were never + /// POSTed, which carry status 0. Nothing the server confirmed is written, so the file holds + /// only work that is still outstanding. + /// + /// Each line holds Timestamp, Url, Method, Status, Body - when the request had one, with + /// credential-named fields and pre-authenticated URLs redacted as a -Debug trace redacts + /// them, or the redaction marker whole when the URL names the secret or the body could not + /// be read for redaction, which is warned about by item - and Error, in that order. Error is + /// there only when the item's own response body carried an error code or message: a + /// never-sent item at status 0 has no response at all, and neither does an item of a refused + /// chunk the server did not answer, so those lines end at Body. + /// + /// It is a record of what to retry rather than a request to replay: a redacted Body is no + /// longer the body that was sent. Re-piping it for retry: /// Get-Content dead.jsonl | ConvertFrom-Json | Invoke-MgxBatchRequest + /// A line carrying the redaction marker does not go back out that way: it is refused as an + /// error record. A Body that is the marker whole is not JSON, so the record names no field; + /// one carrying the marker in a field parses, and that record names the field - or names + /// the key, since a pre-authenticated URL used as a field name is cut the same way a value + /// is. Under -ErrorAction Stop, the first such refusal ends the pipeline before any chunk of + /// the batch is sent. Rebuild those requests from their Url and Method, supplying the + /// credential again. + /// + /// A status >= 400 does not say the write was not applied - the POST may have reached the + /// server and gone unanswered - so resending one is a decision about duplicates rather than + /// a free retry. Only status 0 says nothing was sent; a body the envelope could not + /// serialize surfaces as a chunk failure at 503 like a refused POST. + /// + /// A file system path, and one with something in it: an empty string resolves to the + /// working directory, and a path on another provider - Env:\X - resolves to a bare name + /// this run would make a file of wherever it happened to be standing. /// [Parameter] + [ValidateNotNullOrEmpty] public string? DeadLetterPath { get; set; } private string VersionedBaseUrl => $"{s_graphEndpoint}/{ApiVersion}"; @@ -114,6 +145,19 @@ protected override void ProcessRecord() protected override void EndProcessing() { GraphBatchClient? batchClient = null; + + // The dead-letter file, held open for the run. Declared out here so the finally can + // close it on a run that never reached its own write, and so the three things that + // decide whether an empty file is left behind - was it this run that created it, did + // this run put a line in it, and what did the file hold when this run's own handle last + // looked - are readable from there too. The stream beside the writer is that handle: + // the writer buffers characters, the stream is what can be asked the file's length. + StreamWriter? deadLetterWriter = null; + FileStream? deadLetterStream = null; + string? deadLetterFileThisRunCreated = null; + var deadLetterLines = 0; + long? deadLetterLengthAtClose = null; + try { if (_collected.Count == 0) @@ -122,10 +166,25 @@ protected override void EndProcessing() return; } - // Resolve dead-letter path early (before network calls) - string? resolvedDeadLetterPath = DeadLetterPath != null - ? GetUnresolvedProviderPathFromPSPath(DeadLetterPath) - : null; + // Resolved before any network call, and told to be a file system path. Every other + // provider resolves to something this run cannot append to, and resolves it + // silently: "Env:\X" comes back as "X", a name relative to nothing, so the run made + // a file called X in whatever directory it was standing in and reported the dead + // letters written - to a file nobody named, under a name taken from a variable. + string? resolvedDeadLetterPath = null; + if (DeadLetterPath != null) + { + resolvedDeadLetterPath = SessionState.Path.GetUnresolvedProviderPathFromPSPath( + DeadLetterPath, out var provider, out _); + if (provider.ImplementingType.FullName != FileSystemProviderType) + { + ThrowTerminatingError(new ErrorRecord( + new ArgumentException( + $"-DeadLetterPath '{DeadLetterPath}' is not a file system path."), + "DeadLetterPathNotFileSystem", ErrorCategory.InvalidArgument, DeadLetterPath)); + return; + } + } // Validate: $search in any URL requires ConsistencyLevel var hasSearch = _collected.Any(c => @@ -140,28 +199,116 @@ protected override void EndProcessing() return; } - // -WhatIf is documented as "the cmdlet is not run", without qualification, so the - // gate covers reads as well. A read-only batch changes nothing on the server, but it - // spends resource units, can be throttled, and emits objects into the pipeline - - // none of which is "not run", and none of which the caller asked for. - var writeOps = _collected.Where(c => + // Convert to BatchOperation list. An item whose body is not valid JSON fails on + // its own (non-terminating error) instead of aborting the whole batch; `submitted` + // keeps result indices aligned with the operations actually sent. + var operations = new List(_collected.Count); + var submitted = new List(_collected.Count); + foreach (var input in _collected) + { + JsonElement? body = null; + if (input.Body != null) + { + try + { + var json = InvokeMgxRequest.SerializeBody(input.Body); + body = JsonSerializer.Deserialize(json); + } + catch (Exception ex) when (ex is JsonException or ArgumentException) + { + // ArgumentException: a value serialization refuses (SecureString, NaN), + // named by property path. JsonException: a string body that is not JSON. + WriteError(new ErrorRecord( + new ArgumentException( + $"Body for {input.Method} {input.Url} is not valid JSON: {ex.Message}", ex), + "InvalidBatchItemBody", ErrorCategory.InvalidArgument, input.Url)); + continue; + } + + // A dead-letter line piped back in. The marker stands where something the + // file must not carry used to be, so this is not the body that was sent, + // and sending it writes the literal marker to the tenant - under the field + // name that says what it replaced, or, when the file cut a capability URL + // used as a field name, AS the field name. A whole-body marker is refused + // already, because the marker alone is not JSON; a marker in a field or in + // a key parses, and the recipe in -DeadLetterPath's help ends at the same + // place whichever it is. + // + // The walk reads every string of the body, and a string the JSON parser + // accepted is not always one it will hand over: an unpaired surrogate + // escape, "Jos\ud83d", parses as a string value and throws when read. That + // is one item's body and nothing more, so it is refused like a body that is + // not JSON at all - the guard is the only thing that decides whether this + // body may be sent, and a body it cannot read is one it cannot clear. + string? markerField; + try + { + markerField = FindRedactionMarker(body!.Value); + } + catch (Exception ex) when (ex is InvalidOperationException or JsonException) + { + WriteError(new ErrorRecord( + new ArgumentException( + $"Body for {input.Method} {input.Url} could not be read for the " + + $"redaction check: {Reason(ex.Message)}. Send it as a hashtable " + + "or a string this run can read.", ex), + "InvalidBatchItemBody", ErrorCategory.InvalidArgument, input.Url)); + continue; + } + + if (markerField != null) + { + WriteError(new ErrorRecord( + new ArgumentException( + $"Body for {input.Method} {input.Url} carries the redaction marker " + + $"'{RedactionMarker}' at '{markerField}'. A dead-letter line is not the " + + "body that was sent: rebuild the request from its Url and Method, " + + "supplying the credential again. A body of your own must not carry " + + "that text."), + "InvalidBatchItemBody", ErrorCategory.InvalidArgument, input.Url)); + continue; + } + } + + operations.Add(new BatchOperation(NormalizeToRelativeUrl(input.Url), input.Method, body)); + submitted.Add(input); + } + + if (operations.Count == 0) + return; + + // The gate covers reads as well as writes. A read-only batch changes nothing on the + // server, but it spends resource units, can be throttled, and emits objects into + // the pipeline, none of which a caller who typed -WhatIf asked for. The -WhatIf + // help says so, and says Invoke-MgxRequest makes the other choice. + // + // Over the items that will be sent, not the items that were piped in. The body + // validation and the marker guard have run by now, and a target counting what + // they refused named requests the run was never going to make, at the one + // surface whose whole purpose is saying what is about to happen. The refusals + // are named as their own count beside it: a caller who typed -WhatIf to find + // out what would happen is owed both halves of the answer, and the error + // records for them are already in the stream, since a refusal is not something + // the gate governs. + var writeOps = submitted.Where(c => !string.Equals(c.Method, "GET", StringComparison.OrdinalIgnoreCase)).ToList(); + var refused = _collected.Count - submitted.Count; string target; if (writeOps.Count == 0) { - target = $"GET {_collected.Count} requests via $batch"; + target = $"GET {Requests(submitted.Count)} via $batch"; } - // Only when the batch is that method and nothing else. The count is the whole - // batch by the reasoning above, and a write verb welded onto it described requests - // that were not in it: one DELETE among nineteen GETs read "DELETE 20 requests via - // $batch" at the one surface whose purpose is saying what is about to happen. With - // reads present the writes are named as their own count, the way a batch of several - // write methods already names them. - else if (writeOps.Count == _collected.Count + // Only when the batch is that method and nothing else. The count is everything + // that will be sent, reads included, by the reasoning above, and a write verb + // welded onto it described requests that were not in it: one DELETE among nineteen + // GETs read "DELETE 20 requests via $batch". With reads present the writes are + // named as their own count, the way a batch of several write methods already names + // them. + else if (writeOps.Count == submitted.Count && writeOps.All(o => string.Equals(o.Method, writeOps[0].Method, StringComparison.OrdinalIgnoreCase))) { - target = $"{writeOps[0].Method} {_collected.Count} requests via $batch"; + target = $"{writeOps[0].Method} {Requests(submitted.Count)} via $batch"; } else { @@ -169,9 +316,12 @@ protected override void EndProcessing() .GroupBy(o => o.Method.ToUpperInvariant()) .OrderByDescending(g => g.Count()) .Select(g => $"{g.Count()} {g.Key}"); - target = $"{_collected.Count} requests ({string.Join(", ", breakdown)}) via $batch"; + target = $"{Requests(submitted.Count)} ({string.Join(", ", breakdown)}) via $batch"; } + if (refused > 0) + target += $"; {refused} refused"; + if (!ShouldProcess(target, "Send batch")) return; @@ -191,46 +341,58 @@ protected override void EndProcessing() ItemHeaders = itemHeaders }; - // Convert to BatchOperation list. An item whose body is not valid JSON fails on - // its own (non-terminating error) instead of aborting the whole batch; `submitted` - // keeps result indices aligned with the operations actually sent. - var operations = new List(_collected.Count); - var submitted = new List(_collected.Count); - foreach (var input in _collected) + + // Opened before the first POST, and the handle kept for the run. Opened where the + // lines are written - at the end - an unusable path was discovered with the batch + // already applied: the caller had made every one of those changes and lost the + // record of which to retry, and the only thing that could have made that + // recoverable was learning it before anything went out. A directory standing at the + // path, a parent directory that is not there, a file this account may not write: + // all of them answer here, with nothing sent. + // + // Append, because the parameter's help promises a file a second run adds to rather + // than replaces, and shared the way each platform has to share it for that promise + // to hold: see DeadLetterHoldShare. + if (resolvedDeadLetterPath != null) { - JsonElement? body = null; - if (input.Body != null) + var existed = File.Exists(resolvedDeadLetterPath); + try { - try - { - var json = InvokeMgxRequest.SerializeBody(input.Body); - body = JsonSerializer.Deserialize(json); - } - catch (Exception ex) when (ex is JsonException or ArgumentException) - { - // ArgumentException: a value serialization refuses (SecureString, NaN), - // named by property path. JsonException: a string body that is not JSON. - WriteError(new ErrorRecord( - new ArgumentException( - $"Body for {input.Method} {input.Url} is not valid JSON: {ex.Message}", ex), - "InvalidBatchItemBody", ErrorCategory.InvalidArgument, input.Url)); - continue; - } + deadLetterStream = new FileStream( + resolvedDeadLetterPath, FileMode.Append, FileAccess.Write, + DeadLetterHoldShare); + deadLetterWriter = new StreamWriter(deadLetterStream); } - - operations.Add(new BatchOperation(NormalizeToRelativeUrl(input.Url), input.Method, body)); - submitted.Add(input); + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + WriteError(new ErrorRecord( + new IOException( + $"Failed to write dead-letter file '{resolvedDeadLetterPath}': " + + $"{Reason(ex.Message)} before the batch was sent; nothing was sent.", ex), + "DeadLetterWriteFailed", ErrorCategory.WriteError, resolvedDeadLetterPath)); + return; + } + if (!existed) + deadLetterFileThisRunCreated = resolvedDeadLetterPath; } - if (operations.Count == 0) - return; - var batchResult = batchClient.ExecuteBatchIndexedAsync(operations, CancellationToken) .GetAwaiter().GetResult(); var results = batchResult.Results; var telemetry = batchResult.Telemetry; + // Before anything else this run does with the batch. What the counters hold is + // what the wire did, and it did it whatever becomes of the run afterwards - so the + // session view has to be told here rather than at the end of the reporting, past + // the dead-letter write, the chunk failure and one error record per failed item. + // Every one of those can end the run under -ErrorAction Stop, and a throttled, + // retried batch that ended on one contributed nothing at all to Get-MgxTelemetry: + // the 429s it met and the seconds it spent in Retry-After sleeps were the very + // numbers a caller reads that view to find. Nothing between the call above and + // this can throw, so the record needs no finally to be sure of running. + RecordSessionTelemetry(telemetry); + // Every line this run prints about itself reads these: the dead-letter line, the // chunk-failure target, the verbose summary and the warning. Counting for // themselves, they drifted - the warning said the batch-level retry pass had been @@ -264,53 +426,122 @@ protected override void EndProcessing() WriteObject(result); } - // Write failed items to dead-letter file (append mode) - if (resolvedDeadLetterPath != null) + // Write failed items to the file opened before the batch went out. Two things about + // this write are collected rather than reported immediately. The write failing + // outright is an error record, written first of the errors below - the path was + // usable when the run opened it, so a failure here is the file going out from under + // an applied batch, which is the case a caller most needs the error stream for. A + // body the redactor could not read is a warning per item, written last of everything + // - WriteWarning under -WarningAction Stop throws, and a warning about one line of + // the file must not cost the caller the item errors or the telemetry. + var withheldBodies = new List(); + Exception? deadLetterWriteFailure = null; + if (deadLetterWriter != null) { - var wrote = false; try { - using var writer = new StreamWriter(resolvedDeadLetterPath, append: true); for (int i = 0; i < results.Count; i++) { var (_, item) = results[i]; if (item.Status < 400 && item.Status != GraphBatchClient.NotSentStatus) continue; var input = submitted[i]; + // The item's own URL, under the value rule the body's strings are read + // by. A pre-authenticated URL is a request a caller can make as well as + // a value a body can carry - a PUT to an upload session's uploadUrl, a + // GET of a @microsoft.graph.downloadUrl - and the URL written here is + // the one the caller gave, so the line carried the live signature while + // the same URL inside a body beside it was cut. var deadLetter = new JsonObject { ["Timestamp"] = DateTime.UtcNow.ToString("o"), - ["Url"] = input.Url, + ["Url"] = SensitiveNames.RedactCapabilityUrl(input.Url, RedactionMarker) + ?? input.Url, ["Method"] = input.Method, ["Status"] = item.Status, }; if (input.Body != null) { - var bodyJson = InvokeMgxRequest.SerializeBody(input.Body); - var bodyNode = JsonNode.Parse(bodyJson); - RedactSensitiveFields(bodyNode); - deadLetter["Body"] = bodyNode; + // A URL that names a secret takes the whole Body: on resetPassword or + // uploadSecret the credential is the field called `k` or `value`, and + // no name rule reaches it. Url, Method, Status and Error still say + // what failed, which is what the file is read for. + if (UrlNamesSecret(input.Url)) + { + deadLetter["Body"] = RedactionMarker; + } + else + { + // A body the redactor cannot walk takes the marker too. The walk + // is the only thing that decides which of a body's values may be + // written, so a body it cannot read is a body with nothing + // cleared for the file - the same position a URL naming the + // secret puts it in. A duplicate property name is how a body + // gets there: JsonNode.Parse accepts the document and building + // the JsonObject's dictionary to enumerate it throws, and Graph + // itself takes such a body, so the run must not end on one. + try + { + var bodyJson = InvokeMgxRequest.SerializeBody(input.Body); + var bodyNode = JsonNode.Parse(bodyJson); + deadLetter["Body"] = RedactSensitiveFields(bodyNode); + } + catch (Exception ex) + { + deadLetter["Body"] = RedactionMarker; + withheldBodies.Add( + $"Body for {input.Method} {input.Url} was withheld from the " + + $"dead-letter file: it could not be read for redaction ({ex.Message}). " + + "The line carries the redaction marker in its place."); + } + } } var errorMsg = TryExtractBatchErrorMessage(item); if (errorMsg != null) deadLetter["Error"] = errorMsg; - writer.WriteLine(deadLetter.ToJsonString(s_deadLetterJsonOptions)); + deadLetterWriter.WriteLine(deadLetter.ToJsonString(s_deadLetterJsonOptions)); + deadLetterLines++; } - wrote = true; } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { - WriteWarning($"Failed to write dead-letter file '{resolvedDeadLetterPath}': {ex.Message}"); + // Collected, not written - see the comment above the declaration. The + // exception itself, not just its message: the error record below keeps it + // as the inner exception, so $Error carries what actually refused. + deadLetterWriteFailure = ex; + } + + // Closed here rather than in the finally, so a failure flushing the last line + // is this run's to report and the file is on disk before the lines below say + // what is in it - and before an empty one is removed. The length is read off + // the handle first, while it still answers. + deadLetterLengthAtClose = DeadLetterLength(deadLetterStream); + deadLetterWriter.Dispose(); + deadLetterWriter = null; + deadLetterStream = null; + + // A file this run created and never put a line in says a batch had failures + // when it had none, and a caller watching that path for one would find it after + // every clean run. A file that was already there is an earlier run's record and + // is not this run's to remove; nor is one this run could not finish writing; + // nor is one that is no longer empty, whoever filled it - the share mode admits + // a second run appending to the same path, so "this run created it and wrote + // nothing" says nothing about what stands in it now. + if (deadLetterFileThisRunCreated != null && deadLetterLines == 0 + && deadLetterWriteFailure == null && deadLetterLengthAtClose == 0) + { + TryRemoveEmptyDeadLetterFile(deadLetterFileThisRunCreated); + deadLetterFileThisRunCreated = null; } // The file takes both, and they are not the same thing to a caller deciding what // to re-pipe: a refused write may already have been applied and one that was // never sent certainly was not. Only after the write finished - a line stating // what is in a file the run could not finish writing states it of nothing. - if (wrote && summary.Failed + summary.NotSent > 0) + if (deadLetterWriteFailure == null && summary.Failed + summary.NotSent > 0) { var written = summary.NotSent > 0 ? $"{summary.Failed} failed and {summary.NotSent} not-sent items" @@ -319,6 +550,31 @@ protected override void EndProcessing() } } + // The file the caller asked for was opened and then could not be written. An error + // record rather than a warning, because a warning is the one thing a caller can be + // running without: at -WarningAction SilentlyContinue, or Ignore, a run that lost + // the whole record of what to retry said nothing about it at all. + // + // First of the errors this run writes, ahead of the chunk failure and the item + // errors. Under -ErrorAction Stop the run ends here, and that is the intent - the + // item errors it ends before are the very lines the file was going to hold, so a + // caller told about them and not about the missing file would go looking for a + // file that is not there. The withheld-body warnings and the fold in the outcome + // warning still run for a caller who is not stopping. + // + // A run with nothing to write cannot get here: the loop above only touches the + // handle for a failed or not-sent item, so a failure means there were lines and + // some of them are gone. An unusable path is answered before the batch instead, + // where nothing has been applied and there is nothing to be told to retry. + if (deadLetterWriteFailure != null) + { + WriteError(new ErrorRecord( + new IOException( + $"Failed to write dead-letter file '{resolvedDeadLetterPath}': " + + deadLetterWriteFailure.Message, deadLetterWriteFailure), + "DeadLetterWriteFailed", ErrorCategory.WriteError, resolvedDeadLetterPath)); + } + // A chunk's POST failed while other chunks were being applied. Their results are // above; this says why the rest never went, and NotSent names them. After the // dead-letter write, like the item errors below, so -ErrorAction Stop cannot cut @@ -361,7 +617,14 @@ protected override void EndProcessing() } } - WriteBatchTelemetry(telemetry, summary); + WriteBatchTelemetry(telemetry, summary, withheldBodies.Count, deadLetterWriteFailure?.Message); + + // Last of everything this run owes for the batch: WriteWarning throws under + // -WarningAction Stop, and a warning that ends the run here must not be able to + // cost the caller the item errors above or the telemetry WriteBatchTelemetry just + // wrote - both already ran by the time one of these can throw. + foreach (var withheld in withheldBodies) + WriteWarning(withheld); } catch (Exception ex) when (ex is GraphServiceException or BrokenCircuitException or HttpRequestException) { @@ -378,6 +641,25 @@ protected override void EndProcessing() } finally { + // Only reached with the handle still open where the run did not get to its own + // write - a batch that threw, a cancellation, a stop. Guarded, because a throw from + // a finally would replace the failure the run is already carrying. + if (deadLetterWriter != null) + { + deadLetterLengthAtClose = DeadLetterLength(deadLetterStream); + try { deadLetterWriter.Dispose(); } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { } + } + + // Same reasoning as the removal after the write, for the same file on the paths + // that never reach it: an empty file this run made is a record of nothing. Under + // the same three conditions, the length included - a run that came through the + // write and left the file standing because something had been appended to it does + // not get a second, unconditional go at it here. + if (deadLetterFileThisRunCreated != null && deadLetterLines == 0 + && deadLetterLengthAtClose == 0) + TryRemoveEmptyDeadLetterFile(deadLetterFileThisRunCreated); + // Drain verbose messages even on exception so retry/throttle history is visible DrainClientMessages(); batchClient?.DrainVerboseMessages(); @@ -385,6 +667,81 @@ protected override void EndProcessing() } } + /// + /// What the dead-letter file holds, as this run's own handle reports it - so a line a second + /// run appended beside this one's is counted. Null where the handle cannot say, which is not + /// the same answer as zero: the removal below turns on the length, and a length nothing + /// measured leaves the file where it is. + /// + private static long? DeadLetterLength(FileStream? stream) + { + try { return stream?.Length; } + catch (Exception ex) when (ex is IOException or NotSupportedException + or ObjectDisposedException) + { + return null; + } + } + + /// + /// Remove a dead-letter file this run created and put nothing in - while it is still empty + /// and no second run has it open. Silent either way: an empty file holds nothing a caller + /// has to be told about, and a path that will not give it up is a path the next run will + /// simply append to. + /// + /// The claim is what makes the removal honest. The file is opened for append under + /// , which admits a second run appending to the same path - + /// the shape a caller gets from running batches in parallel over one -DeadLetterPath - and + /// unlink honors no lock, so "this run created it and wrote no lines" is not on its own a + /// statement about what stands in the file by the end. FileShare.None asks for the entry + /// exclusively: a sibling still holding it for append refuses this open and the file is + /// left, and under a claim that succeeds the length is read once more, so a sibling that + /// wrote and closed between the two readings is not deleted either. Only then is the path + /// unlinked, under the claim, in the order uses - so no + /// instant exists in which this run has decided on the name and let go of it. + /// + /// + /// + /// What a second opener of the dead-letter file may do while this run holds it, which is not + /// the same question on the two platforms because the lock is not the same lock. The + /// -DeadLetterPath help promises a file a second run adds to rather than replaces, and two + /// batches over one path is what running them in parallel gives, so a sibling appending to + /// it has to get in - and the removal below has to be refused for as long as one is there. + /// + /// .NET on Unix backs FileShare with an advisory flock: FileShare.None takes LOCK_EX and + /// every other mode takes LOCK_SH, and LOCK_SH refuses no second writer. FileShare.Read is + /// therefore what admits the sibling while still refusing the LOCK_EX the removal's claim + /// asks for - FileShare.ReadWrite would take no lock at all, and the claim would succeed + /// over a sibling's open file and delete it on a length read before that sibling had + /// written. + /// + /// + /// Windows checks the sharing both ways round: a second opener's access has to be one this + /// handle's share mode admits, and this handle's access has to be one the second opener's + /// share mode admits. FileShare.Read admits a reader and no appender at all, so a second + /// batch over one -DeadLetterPath could not open the file and wrote none of its dead letters + /// - the shape the help promises, refused on the platform. FileShare.ReadWrite lets the + /// sibling in, and the removal's claim is refused all the same: it asks FileShare.None, + /// which admits none of the writing this handle is already doing. + /// + /// + private static FileShare DeadLetterHoldShare => + OperatingSystem.IsWindows() ? FileShare.ReadWrite : FileShare.Read; + + private static void TryRemoveEmptyDeadLetterFile(string path) + { + FileStream? claim = null; + try + { + claim = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.None); + if (claim.Length != 0) return; + ScratchName.Discard(path, claim); + claim = null; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { } + finally { claim?.Dispose(); } + } + /// /// Parse pipeline input into a BatchInput. Supports: /// - String: use as URL with shared -Method/-Body parameters @@ -453,47 +810,273 @@ private string NormalizeToRelativeUrl(string url) } /// - /// Redact sensitive fields (passwordProfile, credentials, secrets) from a JSON body - /// before writing to the dead-letter file. Modifies the node in-place. + /// "1 request" or "n requests". The gate names a count that can be one - what a batch will + /// send is what the validation and the guard left of it - and "1 requests" is a sentence + /// about a number rather than about the batch. /// - internal static void RedactSensitiveFields(JsonNode? node) + private static string Requests(int count) => count == 1 ? "1 request" : $"{count} requests"; + + /// What stands in the dead-letter file for a value it must not carry. + internal const string RedactionMarker = "***REDACTED***"; + + /// + /// The one provider -DeadLetterPath may name. Matched on the implementing type rather than + /// on the provider's name, which a module is free to reuse. + /// + private const string FileSystemProviderType = "Microsoft.PowerShell.Commands.FileSystemProvider"; + + /// + /// Where a body carries the redaction marker, as a string value or inside a property name, + /// or null when it carries none. Every depth, and an array's elements as well as an + /// object's properties: the writer puts the marker wherever the walk found a credential, + /// and a keyCredentials array is where it usually finds one. The path is what the caller + /// needs - it says what the file took, and so what has to be supplied again. + /// + /// A name and a value are read the same way, as substrings, because the writer leaves the + /// marker inside text as often as alone. A pre-authenticated URL is cut at its capability + /// parameter and what stands in the file is the URL up to that parameter with the marker + /// after it - whether the URL was a field's value, an element of an array, or the field + /// NAME itself, which is what the cut on a key leaves. Reading a value for equality saw + /// only the cuts that took a whole value, so a line holding a cut uploadUri, + /// azureStorageUri or @microsoft.graph.downloadUrl read as a body that could be sent, and + /// sending it puts the literal marker where the signature belongs. + /// + /// A caller's own text carrying the marker is refused with the rest. The file's cut form is + /// a prefix plus the marker, so nothing in the text separates the two, and refusing a body + /// no file wrote costs the caller a rename; sending one the file did write costs a request + /// the tenant cannot honor and a credential the caller still thinks it supplied. + /// + internal static string? FindRedactionMarker(JsonElement element, string path = "") { - if (node is JsonArray rootArr) + switch (element.ValueKind) { - foreach (var item in rootArr) - if (item is JsonObject arrObj) - RedactSensitiveFields(arrObj); - return; + case JsonValueKind.String: + return element.GetString()!.Contains(RedactionMarker, StringComparison.Ordinal) + ? (path.Length == 0 ? "the whole body" : path) + : null; + case JsonValueKind.Object: + foreach (var property in element.EnumerateObject()) + { + var here = path.Length == 0 ? property.Name : $"{path}.{property.Name}"; + + // Before the value, so that a cut key whose own value says nothing is + // still reported as the key: what the cut leaves is the URL up to the + // capability parameter with the marker after it, and a cut that collided + // carries a #2, #3... suffix past that. Either way the name itself is the + // path returned, so the record names the key as written. + if (property.Name.Contains(RedactionMarker, StringComparison.Ordinal)) + return here; + + var found = FindRedactionMarker(property.Value, here); + if (found != null) return found; + } + return null; + case JsonValueKind.Array: + var index = 0; + foreach (var item in element.EnumerateArray()) + { + var found = FindRedactionMarker(item, $"{path}[{index++}]"); + if (found != null) return found; + } + return null; + default: + return null; } - if (node is not JsonObject obj) return; - foreach (var key in obj.Select(p => p.Key).ToArray()) + } + + /// + /// True when a field name or URL path segment carries one of the credential fragments. The + /// fragments are SensitiveNames', and so are the two value rules beside them: a property + /// whose NAME carries one of the fragments, a property whose name carries "downloadurl", and + /// a string value that is a URL carrying one of the capability parameters. That class carries + /// the reasoning for each. + /// + /// This file and a -Debug trace agree on the value rule for the form a body writes a URL in, + /// since a URL is text either way, and on the two name rules only where the value beside the + /// named key is itself a quoted string - a trace regex for a name rule matches a key and its value together, and + /// cannot match past a value that is not a string. This file does not stop there: the value + /// beside a matched key is cleared whatever it holds, so a number, a bool, an array or an + /// object beside a password or a downloadUrl field is missing from the dead-letter line and + /// still stands whole in -Debug output. + /// + /// Two things only the file has, because a trace has nothing to apply them to. The URL rule + /// below reads the request's own path and takes the whole Body; the marker also stands for a + /// body the walk could not read. The tracer sees a batch envelope as text - one item's url + /// is a string beside its body, not a request whose body the tracer knows - so an + /// uploadSecret body still reaches -Debug output whole, and nothing here changes that. + /// + private static bool IsSensitiveName(string name) => SensitiveNames.Matches(name); + + /// + /// True when the request's own path names the secret its body carries - resetPassword, + /// changePassword, uploadSecret, synchronization/secrets, addKey, vppTokens. Those bodies + /// hold the credential under a field name that says nothing about it, `k` or `value`, which + /// no name rule can reach; the caller of this decides on the whole Body instead. + /// + /// The path and nothing else. For an http or https URL that is what the parser gives after + /// the authority, so the host is out of it: a tenant that reaches Graph through a host + /// named keyvault.example.com is describing the machine, not the body, and losing every + /// body under it costs the file the half it is read for. Everything else is cut at the + /// first ? or #, a query string carrying filters rather than bodies. + /// + /// The scheme is checked because a parse alone does not say the string was absolute. On + /// Unix a path is a legal file URI, so "/users/u1?$select=keyCredentials" parses as + /// absolute with the query folded into an escaped AbsolutePath - the segment then reads as + /// keyCredentials and the whole body goes, on a $select. On Windows the same string does + /// not parse, and one relative URL would have been judged two ways on two hosts. + /// + /// Every segment is read decoded, because Graph reads the path decoded and the two forms + /// are one request: .../microsoft.graph.reset%50assword is a password reset. The parser + /// unescapes an unreserved character itself, so only the relative form - the one the batch + /// envelope actually carries - could hide a name behind an escape. + /// + internal static bool UrlNamesSecret(string? url) + { + if (string.IsNullOrEmpty(url)) return false; + + string path; + // System.Uri, not the cmdlet's -Uri parameter, which shadows the type name here. + if (System.Uri.TryCreate(url, UriKind.Absolute, out var absolute) + && (absolute.Scheme == System.Uri.UriSchemeHttp || absolute.Scheme == System.Uri.UriSchemeHttps)) + { + path = absolute.AbsolutePath; + } + else { - if (key.Equals("passwordProfile", StringComparison.OrdinalIgnoreCase) - || key.Equals("password", StringComparison.OrdinalIgnoreCase) - || key.Equals("secretText", StringComparison.OrdinalIgnoreCase) - || key.Equals("keyCredentials", StringComparison.OrdinalIgnoreCase) - || key.Equals("passwordCredentials", StringComparison.OrdinalIgnoreCase) - || key.Equals("clientSecret", StringComparison.OrdinalIgnoreCase) - || key.Equals("appPassword", StringComparison.OrdinalIgnoreCase) - || key.Equals("clientAssertion", StringComparison.OrdinalIgnoreCase)) + path = url; + var cut = path.IndexOfAny(['?', '#']); + if (cut >= 0) path = path[..cut]; + } + + foreach (var segment in path.Split('/', StringSplitOptions.RemoveEmptyEntries)) + { + if (IsSensitiveName(DecodeSegment(segment))) return true; + } + return false; + } + + /// + /// A path segment as Graph reads it. An unpaired or invalid escape is not a segment anyone + /// meant, so the raw text stands rather than the call failing on it. + /// + private static string DecodeSegment(string segment) + { + if (!segment.Contains('%')) return segment; + try { return System.Uri.UnescapeDataString(segment); } + catch (System.UriFormatException) { return segment; } + } + + /// + /// Redact every field whose name carries a credential token from a JSON body before writing + /// it to the dead-letter file. Modifies the node in-place and returns the body to write, + /// which is the same node except where a rule took the root itself: a node has no way to + /// replace itself in a parent it does not have. + /// + /// Every node, at every depth: an object's properties, an array's elements whatever they + /// are, and the root. A body is whatever the caller built, so descending only into the + /// shapes one was expected to have leaves the rest unread - a password inside an array that + /// is itself inside an array, {"settings":[[{"password":"..."}]]}, reached the file whole, + /// and a body that is one bare string, a pre-authenticated URL and nothing else, has no + /// property and no element to descend into at all and reached it whole too. + /// + /// A name usually decides, and takes the whole value with it. The other thing read rather + /// than matched by name is a pre-authenticated URL, a credential with no name to give it + /// away: an Intune azureStorageUri and a @microsoft.graph.downloadUrl fetch the bytes on the + /// strength of the URL alone. That read applies to a property's value and, separately, to + /// its name - a capability URL used as a key is what a -Debug trace already redacts, since + /// it matches any quoted string in the text, key or value alike. Either way the cut keeps + /// everything up to the capability parameter, so the line still says which host and which + /// kind of URL failed - and where the key was the URL, the value beside it goes whole, the + /// way a name that says credential takes its value. + /// + internal static JsonNode? RedactSensitiveFields(JsonNode? node) + { + // Before the branches below, because neither of them reaches a root that is one value. + // The rule is the value rule, the one the trace and the file agree on outright: a body + // that is a bare capability URL is the same credential in the same text whether it + // stands under a property name or alone. + if (RedactedUrlValue(node) is { } rootValue) + return rootValue; + + if (node is JsonArray arr) + { + for (var i = 0; i < arr.Count; i++) { - obj[key] = "***REDACTED***"; + if (RedactedUrlValue(arr[i]) is { } value) + arr[i] = value; + else + RedactSensitiveFields(arr[i]); } - else if (obj[key] is JsonObject child) + return node; + } + if (node is not JsonObject obj) return node; + foreach (var key in obj.Select(p => p.Key).ToArray()) + { + // The capability-URL cut on the key runs before the fragment check below, not + // after: "guestaccesstoken", "authkey" and a passwordreset path all contain a + // TokenList fragment ("token", "key", "password") as plain text, so testing the + // fragment first took that branch and left the key - secret included - untouched. + // JsonObject has no rename and no positional insert, so the cut removes the + // property and re-adds it under the cut name, which moves it to the end: re-adding + // is the only way to change a key. Two keys that cut to the same string - + // "?sig=AAA" and "?sig=BBB" both cut to "?sig=***REDACTED***" - would silently + // collapse to one entry through that re-add, the second overwriting the first, so a + // colliding cut name is suffixed #2, #3... instead. + // + // The value goes with the key, whatever it holds. A pre-authenticated URL used as a + // field name is a credential, so whatever a body files under one is the thing that + // URL was minted for. Deciding that on a fragment in the key's text let the secret + // choose: "?sig=abc" carries none and kept its value, "?sig=abcsecretdef" carries + // "secret" inside the signature and cleared it, and the two keys name the same kind + // of URL. The whole-value rules meet here - a name that says credential takes its + // value, and so does a name that IS one. + if (SensitiveNames.RedactCapabilityUrl(key, RedactionMarker) is { } redactedKey) { - RedactSensitiveFields(child); + obj.Remove(key); + obj[UniqueKey(obj, redactedKey)] = RedactionMarker; + continue; } - else if (obj[key] is JsonArray arr) + + if (IsSensitiveName(key) || SensitiveNames.NamesDownloadUrl(key)) { - foreach (var item in arr) - { - if (item is JsonObject arrObj) - RedactSensitiveFields(arrObj); - } + obj[key] = RedactionMarker; + continue; } + + if (RedactedUrlValue(obj[key]) is { } redactedValue) + obj[key] = redactedValue; + else + RedactSensitiveFields(obj[key]); + } + return node; + } + + /// + /// , or that name suffixed #2, #3... until one names no property + /// already has. The order met: reflects every + /// key this walk has already assigned, so the first collision on a given cut string gets + /// #2 and the next collision on THAT gets #3. + /// + private static string UniqueKey(JsonObject obj, string candidate) + { + if (!obj.ContainsKey(candidate)) return candidate; + for (var n = 2; ; n++) + { + var next = $"{candidate}#{n}"; + if (!obj.ContainsKey(next)) return next; } } + /// + /// The value rule of the two the trace and the file share: a string that is a URL carrying a + /// capability parameter, cut at that parameter. Null for every other node, which is what + /// leaves the rest of a body readable. + /// + private static string? RedactedUrlValue(JsonNode? node) => + node is JsonValue leaf && leaf.TryGetValue(out var text) + ? SensitiveNames.RedactCapabilityUrl(text, RedactionMarker) + : null; + private static string? TryExtractBatchErrorMessage(GraphBatchResponseItem item) { if (!item.Body.HasValue) return null; @@ -565,7 +1148,12 @@ private enum RetryPass Withheld, } - private void WriteBatchTelemetry(BatchTelemetry telemetry, BatchRunSummary summary) + /// + /// Fold what the batch met on the wire into the session view Get-MgxTelemetry reads. Called + /// as soon as the batch returns, ahead of everything this run writes about it: none of + /// those numbers depends on how the run ends, and all of them are lost if it ends early. + /// + private static void RecordSessionTelemetry(BatchTelemetry telemetry) { // Propagate per-item 429 counts to session telemetry if (telemetry.ThrottleEncounters > 0) @@ -575,7 +1163,11 @@ private void WriteBatchTelemetry(BatchTelemetry telemetry, BatchRunSummary summa // batch retry waits (they previously existed only in per-call BatchTelemetry) if (telemetry.TotalRetryDelayMs > 0) MgxTelemetryCollector.Current.RecordBatchRetryDelay(telemetry.TotalRetryDelayMs); + } + private void WriteBatchTelemetry( + BatchTelemetry telemetry, BatchRunSummary summary, int withheldBodyCount, string? deadLetterWriteFailure) + { // Always emit verbose summary with timing breakdown var elapsedSec = telemetry.TotalElapsedMs / 1000.0; var throughput = telemetry.TotalElapsedMs > 0 ? summary.Total / elapsedSec : 0; @@ -596,9 +1188,30 @@ private void WriteBatchTelemetry(BatchTelemetry telemetry, BatchRunSummary summa // nor deny it the ones it sent. if (summary.Failed > 0 || summary.NotSent > 0) { + // One sentence, so the base is built without its period and the period is added + // once at the end. The clauses below are semicolon-joined continuations of it, and + // a base that had already closed with a period left them reading as ".; 1 body was + // withheld from the dead-letter file" with nothing closing the sentence after them. var outcome = summary.NotSent > 0 - ? $"{summary.Failed} of {summary.Total} batch items failed and {summary.NotSent} were not sent." - : $"{summary.Failed} of {summary.Total} batch items failed."; + ? $"{summary.Failed} of {summary.Total} batch items failed and {summary.NotSent} were not sent" + : $"{summary.Failed} of {summary.Total} batch items failed"; + + // Folded in here, ahead of the retry-pass sentence, rather than left to a dedicated + // warning of its own after WriteBatchTelemetry returns: this outcome warning is the + // first WriteWarning call a failed batch makes, so it is the one a run under + // -WarningAction Stop is most likely to reach and stop on, and item failures are + // exactly when a body can go unread or the dead-letter file can fail to open. A + // caller stopped right here still learns both instead of learning neither. + if (withheldBodyCount > 0) + { + outcome += withheldBodyCount == 1 + ? "; 1 body was withheld from the dead-letter file" + : $"; {withheldBodyCount} bodies were withheld from the dead-letter file"; + } + if (deadLetterWriteFailure != null) + outcome += $"; the dead-letter file could not be written: {Reason(deadLetterWriteFailure)}"; + outcome += "."; + // "after all retry attempts" holds only of a run that ran them, and "withheld" only // of a pass that never left. Between them is the pass that went out, was answered, // and was refused on a later chunk: the items it carried may have been applied, and @@ -615,5 +1228,26 @@ private void WriteBatchTelemetry(BatchTelemetry telemetry, BatchRunSummary summa } } + /// + /// A failure's reason as a sentence quoting it carries it: its first line, and one trailing + /// period off that line. The quoting sentence closes with a period of its own, and a + /// platform message ends in one - "...is denied.", "...missing low surrogate." - so keeping + /// both put "..denied.." mid-line. Only one is trimmed: a reason that genuinely ends in an + /// ellipsis keeps what is left of it. + /// + private static string Reason(string text) + { + var line = FirstLine(text); + return line.EndsWith('.') ? line[..^1] : line; + } + + /// The first line of a possibly multi-line exception message, so a reason folded + /// into the one-line outcome warning stays one line. + private static string FirstLine(string text) + { + var end = text.IndexOfAny(['\r', '\n']); + return end < 0 ? text : text[..end]; + } + internal sealed record BatchInput(string Url, string Method, object? Body); } diff --git a/src/Mgx.Cmdlets/Cmdlets/Configuration/DisableMgxResilience.cs b/src/Mgx.Cmdlets/Cmdlets/Configuration/DisableMgxResilience.cs index e78777f..047c838 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Configuration/DisableMgxResilience.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Configuration/DisableMgxResilience.cs @@ -6,7 +6,9 @@ namespace Mgx.Cmdlets.Cmdlets.Configuration; /// /// Removes the Polly resilience injection from the Microsoft.Graph SDK's HTTP transport. -/// Restores the original SDK HttpClient that was saved by Enable-MgxResilience. +/// Restores the original SDK HttpClient that was saved by Enable-MgxResilience, when the +/// session is still holding the wrapper this module installed; a client the SDK has replaced +/// since the injection is left as it was found. /// [Cmdlet(VerbsLifecycle.Disable, "MgxResilience", SupportsShouldProcess = true)] public class DisableMgxResilience : PSCmdlet @@ -80,16 +82,39 @@ protected override void ProcessRecord() return; } - if (!ShouldProcess("Microsoft.Graph SDK HttpClient", - "Restore original SDK HttpClient (remove Polly resilience)")) + // Read before the gate, so what a preview reports is decided from state this cmdlet + // has only looked at. Whether the client on the session is one of ours is the same + // question the teardown asks through TryRestoreGenuineSdkClient, asked the same way: + // the bridge-target map, which spans import cycles, rather than the statics a removal + // drops. + var currentClient = clientProp.GetValue(instance) as HttpClient; + var sessionHoldsOurWrapper = EnableMgxResilience.IsInjectedWrapper(currentClient); + + // And it decides the action the gate names, because the two branches below do + // different things. Only one of them restores anything: the other releases the + // injection's own state and leaves the client the session is holding exactly where + // it found it. Naming a restore for both described a run that, in that case, was + // never going to happen - and the name is all a caller running -WhatIf gets. + var action = sessionHoldsOurWrapper + ? "Restore original SDK HttpClient (remove Polly resilience)" + : "Release Mgx resilience state and leave the SDK HttpClient as found"; + + if (!ShouldProcess("Microsoft.Graph SDK HttpClient", action)) return; - // Verify the current client is actually ours before restoring - var currentClient = clientProp.GetValue(instance) as HttpClient; - if (currentClient != null && !ReferenceEquals(currentClient, EnableMgxResilience.ResilientSdkClient)) + if (!sessionHoldsOurWrapper) { - WriteWarning("The current GraphHttpClient is not the one injected by Enable-MgxResilience. " + - "Another module or Connect-MgGraph may have replaced it. Restoring original anyway."); + // The SDK built a new client after the injection went on - a Connect-MgGraph to + // another tenant does that - so OriginalSdkClient is the previous identity's + // client and installing it would put those credentials back under every SDK + // cmdlet. The injection's own state still has to go, and ReleaseInjection is the + // unwind module removal runs: it restores nothing here, because the client the + // session is holding is not a wrapper of ours to take off. + EnableMgxResilience.ReleaseInjection(); + WriteWarning("The Microsoft.Graph SDK client was replaced after Enable-MgxResilience " + + "ran (by Connect-MgGraph or another module), so it was not this module's to " + + "restore and has been left as it was found. Mgx resilience is now off."); + return; } clientProp.SetValue(instance, originalClient); diff --git a/src/Mgx.Cmdlets/Cmdlets/Configuration/EnableMgxResilience.cs b/src/Mgx.Cmdlets/Cmdlets/Configuration/EnableMgxResilience.cs index 43e28d4..bf71b5b 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Configuration/EnableMgxResilience.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Configuration/EnableMgxResilience.cs @@ -158,6 +158,26 @@ protected override void ProcessRecord() var clientProp = instance.GetType().GetProperty("GraphHttpClient"); var currentClient = clientProp?.GetValue(instance) as HttpClient; + // Already enabled and our client is still the one installed: nothing to do. Reading + // the session is not a change, so this answer is the same with -WhatIf as without. + // The null test earns its place now that the session's client is only forced below + // the gate: a session holding nothing would otherwise compare equal to an injection + // whose build failed, and report itself active. + if (IsEnabled && currentClient != null && ReferenceEquals(currentClient, ResilientSdkClient)) + { + WriteVerbose("MgxResilience is already active."); + return; + } + + // Everything past this line changes state, so all of it is under the gate. The + // pre-initialization resets the pipeline factory and the SDK probe sends a real + // request and leaves a client on the session; dropping the wrapper we installed + // leaves Disable-MgxResilience with no record of what it is taking off. What -WhatIf + // reports is decided above it, from state this cmdlet has only read. + if (!ShouldProcess("Microsoft.Graph SDK HttpClient", + "Replace with Polly resilience pipeline (retry, circuit breaker, rate limiting)")) + return; + // Pre-initialize Mgx's own HTTP client before the SDK probe runs. // The probe calls Invoke-MgGraphRequest which changes Azure Identity internal // state and breaks GetAuthenticationProviderAsync for subsequent callers. @@ -180,14 +200,8 @@ protected override void ProcessRecord() return; } - // If already enabled, check if our client is still active if (IsEnabled) { - if (ReferenceEquals(currentClient, ResilientSdkClient)) - { - WriteVerbose("MgxResilience is already active."); - return; - } // Our client was replaced (e.g., by Connect-MgGraph or Set-MgRequestContext). WriteVerbose("MgxResilience was reset by SDK. Re-injecting resilience..."); // Not disposed: HttpClient.Dispose cancels its pending-request token source and @@ -202,10 +216,6 @@ protected override void ProcessRecord() ResiliencePipelineFactory.Reset(); } - if (!ShouldProcess("Microsoft.Graph SDK HttpClient", - "Replace with Polly resilience pipeline (retry, circuit breaker, rate limiting)")) - return; - // Wrap the genuine client, never a wrapper. The session can still be holding one from // an earlier import - its statics went with that import - and a second layer over it // is what OriginalSdkClient would then be restored to by Disable-MgxResilience. @@ -239,7 +249,7 @@ protected override void ProcessRecord() private HttpClient? ForceInitializeAndGetClient(object instance, PropertyInfo? clientProp) { // Use the Graph endpoint from the session (sovereign cloud support) - var endpoint = MgxCmdletBase.GetGraphEndpoint(WriteWarning, WriteVerbose) ?? "https://graph.microsoft.com"; + var endpoint = MgxCmdletBase.GetGraphEndpoint(WriteWarning, WriteVerbose) ?? MgxCmdletBase.DefaultGraphEndpoint; // Save AzureADEndpoint before probe. Invoke-MgGraphRequest replaces // GraphSession.Environment with a new object that has an empty AzureADEndpoint, diff --git a/src/Mgx.Cmdlets/Cmdlets/Configuration/SetMgxOption.cs b/src/Mgx.Cmdlets/Cmdlets/Configuration/SetMgxOption.cs index caa9ed9..b7a86e8 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Configuration/SetMgxOption.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Configuration/SetMgxOption.cs @@ -56,7 +56,7 @@ public class SetMgxOption : PSCmdlet public double CircuitBreakerFailureRatio { get; set; } [Parameter] - [ValidateRange(1, 1000)] + [ValidateRange(2, 1000)] public int CircuitBreakerMinThroughput { get; set; } [Parameter] @@ -89,9 +89,47 @@ protected override void ProcessRecord() { var bound = MyInvocation.BoundParameters; + // What the target names is what this run would set, which is not everything bound to it: + // PowerShell binds the common parameters too, so `-Verbose -WhatIf` previewed a change to + // "MgxOptions (Verbose, WhatIf)" - two names that set no option - for a run the guard + // below then reports as unchanged. Filtered through the set that guard counts with, in + // the order they were bound, dropped entirely when it leaves nothing to name, and never + // includes -Reset itself: what it names beside a reset is what the reset ignores, not the + // switch that caused it. + var optionNames = bound.Keys.Cast() + .Where(k => !CommonParameterNames.Contains(k) && k != nameof(Reset)) + .ToArray(); + + // No -Reset and no option: nothing to do (avoids unnecessary pipeline rebuild, which + // would destroy circuit breaker failure history). Ahead of ShouldProcess on purpose - the + // guard used to sit after the gate, so a bare `Set-MgxOption` previewed "Set" on + // "MgxOptions" and prompted for it under -Confirm before landing here and reporting the + // run it had just asked about as unchanged. + // + // Counted without the common parameters, which BoundParameters includes: -Verbose alone + // made this look like a real change, so `Set-MgxOption -Verbose` rebuilt the pipeline and + // discarded exactly the history this guard was written to keep. The same list the target + // below names, so what a preview says and what a run does cannot come apart. + if (!Reset.IsPresent && optionNames.Length == 0) + { + WriteVerbose("No parameters specified. Options unchanged."); + return; + } + + // -Reset restores every default; an option bound beside it does not survive that. Silently + // dropping it would read as the option having taken effect over the defaults it did not + // survive, so the run says what it is ignoring before it asks whether to proceed. + if (Reset.IsPresent && optionNames.Length > 0) + { + WriteWarning("-Reset resets every option; " + + $"{string.Join(", ", optionNames.Select(n => $"-{n}"))} given beside it are ignored."); + } + var target = Reset.IsPresent - ? "MgxOptions (reset to defaults)" - : $"MgxOptions ({string.Join(", ", bound.Keys)})"; + ? optionNames.Length == 0 + ? "MgxOptions (reset to defaults)" + : $"MgxOptions (reset to defaults; ignoring {string.Join(", ", optionNames)})" + : $"MgxOptions ({string.Join(", ", optionNames)})"; if (!ShouldProcess(target, "Set")) return; @@ -104,18 +142,6 @@ protected override void ProcessRecord() return; } - // No parameters passed: nothing to do (avoids unnecessary pipeline rebuild - // which would destroy circuit breaker failure history). - // - // Counted without the common parameters, which BoundParameters includes: -Verbose alone - // made this look like a real change, so `Set-MgxOption -Verbose` rebuilt the pipeline and - // discarded exactly the history this guard was written to keep. - if (!bound.Keys.Cast().Any(k => !CommonParameterNames.Contains(k))) - { - WriteVerbose("No parameters specified. Options unchanged."); - return; - } - // Start from current options, only override values the user actually passed var current = MgxCmdletBase.s_clientOptions; diff --git a/src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs b/src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs index 7371303..66ff48f 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs @@ -126,7 +126,10 @@ protected override void BeginProcessing() "DeltaPathOutputFileCollision", ErrorCategory.InvalidArgument, null)); return; } - DeltaState.ValidateWriteAccess(resolvedOutputPath); + // Named for what it is. Both probes are the same call, and one noun for both told a + // caller whose -OutputFile directory was not writable that they could not write to + // the delta state path - naming a file at -DeltaPath that had just passed its own. + DeltaState.ValidateWriteAccess(resolvedOutputPath, "output path"); } // The checkpoint must not collide with either state file: sharing a path would @@ -233,6 +236,17 @@ protected override void ProcessRecord() // A corrupt state means the previous position is unknown, which is exactly when // baselining from now would hide the most: everything since the last good sync. honourLatest = false; + // And the resume checkpoint goes with the position it counted into, the way it does + // at every other door out of the delta state. What it records is the delta-link + // spelling this run no longer builds, so leaving it standing bought a refusal of the + // run's own earlier position: a warning naming a second sync that does not exist, + // and a stale-temp sweep held off, over a temp that outlives the run by one because + // nothing counts it any more. + DeleteCheckpoint(resolvedCheckpointPath, "delta state corrupt"); + // Nothing downstream may still believe it is there. The -Latest guard below reads + // this to decide which of two sentences to write, and one of them tells the caller + // to delete a checkpoint that has just gone. + hasResumableCheckpoint = false; } if (existingState != null) @@ -284,6 +298,7 @@ protected override void ProcessRecord() if (!DeltaState.Delete(resolvedDeltaPath)) WriteVerbose($"Could not delete old delta state at '{DeltaPath}' (file may be locked). It will be overwritten."); DeleteCheckpoint(resolvedCheckpointPath, "property selection changed"); + hasResumableCheckpoint = false; // the -Latest warning below must not name it existingState = null; honourLatest = false; // a discarded state is not a fresh run } @@ -302,6 +317,7 @@ protected override void ProcessRecord() if (!DeltaState.Delete(resolvedDeltaPath)) WriteVerbose($"Could not delete old delta state at '{DeltaPath}' (file may be locked). It will be overwritten."); DeleteCheckpoint(resolvedCheckpointPath, "Prefer headers changed"); + hasResumableCheckpoint = false; // the -Latest warning below must not name it existingState = null; honourLatest = false; // a discarded state is not a fresh run } @@ -318,6 +334,7 @@ protected override void ProcessRecord() if (!DeltaState.Delete(resolvedDeltaPath)) WriteVerbose($"Could not delete old delta state at '{DeltaPath}' (file may be locked). It will be overwritten."); DeleteCheckpoint(resolvedCheckpointPath, "filter changed"); + hasResumableCheckpoint = false; // the -Latest warning below must not name it existingState = null; honourLatest = false; // a discarded state is not a fresh run } @@ -483,6 +500,14 @@ private void DeleteCheckpoint(string? checkpointPath, string reason) /// as the files beside it corroborated it, so losing that corroboration - a temp since /// removed, an output since replaced - is one sync and no second run anywhere, and it is /// the shape every release before this one wrote. + /// + /// Which is why the corroboration is asked for only where there is something to corroborate: + /// is the export's own guard, and without it + /// a checkpoint recording neither an output nor a length was refused for want of a + /// measurement - the whole pre-2.1 file-mode shape, recovered by the export and refused + /// here, with -Latest suppressed by the file it left standing for good. That shape reaches + /// the resource comparison below and, where the enumeration is this run's, the recovery the + /// export makes of it. /// private CheckpointOwnership OwnershipOf(PaginationCheckpoint checkpoint, string? outputPath, string requestUrl) @@ -495,8 +520,9 @@ private CheckpointOwnership OwnershipOf(PaginationCheckpoint checkpoint, string? return CheckpointOwnership.AnotherSyncs; } } - else if (!RecordedOutputMatches(checkpoint.OutputFile, checkpoint.TempFile, - checkpoint.DataLength, outputPath)) + else if (RecordsAFileToWeigh(checkpoint) + && !RecordedOutputMatches(checkpoint.OutputFile, checkpoint.TempFile, + checkpoint.DataLength, outputPath, mayWrite: true)) { return checkpoint.OutputFile != null ? CheckpointOwnership.AnotherSyncs @@ -546,9 +572,11 @@ private enum CheckpointOwnership private static bool RefusedTempIsOnDisk(string? tempFile, string outputPath) { if (tempFile == null || tempFile != Path.GetFileName(tempFile)) return false; - if (!IsRunTempName(Path.GetFileName(outputPath), tempFile)) return false; var dir = Path.GetDirectoryName(outputPath); - return !string.IsNullOrEmpty(dir) && File.Exists(Path.Combine(dir, tempFile)); + if (string.IsNullOrEmpty(dir)) return false; + if (!IsRunTempName(dir, Path.GetFileName(outputPath), tempFile, mayWrite: true)) + return false; + return File.Exists(Path.Combine(dir, tempFile)); } /// @@ -609,10 +637,221 @@ private static string UncorroboratedCheckpointWarning( }; /// - /// What a reconcile leaves for the caller to do. Two of the three say do not resume, and - /// they are not the same answer: one is a checkpoint a live sync is still using, which is - /// left standing along with the temp it names, and the other is a checkpoint describing - /// items that are in no file, which is deleted wherever this run is able to delete it. + /// How the run ends when a file its own checkpoint stands for is held open by another sync, or + /// cannot be opened read-write by this account. Two files reach this, and the reading is one: + /// the changes the checkpoint counts are in that file and in no other, the checkpoint is the + /// only thing on disk that counts them, and this sync has no enumeration to make that leaves + /// the pair intact - it would save its own position over the checkpoint at the first page + /// boundary and move the delta token past changes held in that file alone. So it says what it + /// found, what it did not do, and the two ways out, and every file is exactly where it was. + /// + /// Which file decides what going on would cost, and for an output refusal + /// decides it further: a checkpoint naming a temp + /// that is still there holds the changes in that temp, and going on promotes them, replacing + /// the output outright; one naming a temp that has vanished holds them nowhere this run can + /// reach, and going on re-enumerates from the last saved delta token and replaces the output + /// with the result; and one naming no temp at all was written by a run appending straight + /// into the output, so the changes are in the output itself and going on cuts it back under + /// its writer - whose next write then lands past the hole that offset leaves. + /// + /// Which of the two readings decides only the cause, the advice, and the id and category + /// under them. A held file has a second sync behind it that will either finish, and delete + /// this checkpoint itself, or die and leave it for its own re-run; an unopenable one has no + /// second sync anywhere, only a file whose own open failed for a reason named in + /// - a permission this account can be granted, or a + /// symlink loop, an over-long path or a pipe that granting one does nothing for. + /// + private static string CheckpointFileStopMessage(CheckpointFileRefusal refusal, + string checkpointPath, string outputPath, PaginationCheckpoint checkpoint) + { + // What going on would cost past a held temp, which is the output beside it - named only + // where there is one. A checkpoint naming a temp is reached with nothing at -OutputFile + // at all, and a run that said it would replace that file sent the caller to look at + // something no sync had written yet. + var goingOn = File.Exists(outputPath) + ? $"Going on would replace '{outputPath}', save this run's position over the " + + "checkpoint that counts those items, and advance the delta token past them." + : "Going on would save this run's position over the checkpoint that counts those " + + "items, and advance the delta token past them."; + + // The output is unopenable rather than held. The three routes agree past this point - + // the code beyond this check falls through to the same place a file that never held the + // recorded items reaches, which deletes the checkpoint, replaces the output and advances + // the token at the completion that follows - so only the first clause, naming where the + // items are now, and the recovery word in the advice change per route. + string UnopenableOutputSentence() + { + var reason = refusal.Reason!.Value; + var openFailed = $"'{outputPath}' could not be opened for writing: {reason.Reason}."; + var recovers = refusal.Route switch + { + CheckpointStopRoute.Promoting => " to recover them", + CheckpointStopRoute.FreshAfterVanishedTemp => "", + _ => " to resume from it", + }; + // Four ways out, because the causes do not share one. A permission on the file is + // granted; a permission on the directory holding it is granted there, and the file + // this sentence names is one the caller cannot reach to change or remove; a + // directory at the path is removed or pointed away from, and no grant would have + // made it writable at an offset; and a loop, an over-long path, a socket or a pipe + // is none of those - what this run can honestly say about them is what failed. + var advice = reason.IsDirectory + ? $"Remove it (or point -OutputFile elsewhere) and run again{recovers}, or " + + "remove the checkpoint to sync afresh." + : reason.IsUnsearchableParent + ? $"Grant access to that directory and run again{recovers}, or remove the " + + "checkpoint to sync afresh." + : reason.IsPermission + ? $"Grant write access to that file and run again{recovers}, or remove " + + "it and the checkpoint to sync afresh." + : $"Fix that and run again{recovers}, or remove it and the checkpoint to " + + "sync afresh."; + + return refusal.Route switch + { + CheckpointStopRoute.Promoting => + $"{openFailed} The resume checkpoint at '{checkpointPath}' records " + + $"{checkpoint.ItemsCollected} items into the temp '{checkpoint.TempFile}'; " + + $"this run cannot promote them into '{outputPath}'. Going on would delete " + + "the checkpoint that counts them, replace that file and advance the delta " + + $"token past them. {RunStopsHere} {advice}", + + CheckpointStopRoute.FreshAfterVanishedTemp => + $"{openFailed} The resume checkpoint at '{checkpointPath}' recorded " + + $"{checkpoint.ItemsCollected} items into a temp that is gone; this run " + + $"cannot write a fresh copy into '{outputPath}' either. Going on would " + + "delete the checkpoint that counts them, replace that file and advance the " + + $"delta token past them. {RunStopsHere} {advice}", + + _ => + $"{openFailed} The resume checkpoint at '{checkpointPath}' records " + + $"{checkpoint.ItemsCollected} items into it, so this run can neither " + + "resume into it nor say whether those items are still there. Going on " + + "would delete the checkpoint that counts them, replace that file and " + + $"advance the delta token past them. {RunStopsHere} {advice}", + }; + } + + return (refusal.File, refusal.Held) switch + { + (CheckpointFile.NamedTemp, true) => + "Another sync is still writing the temp file the resume checkpoint at " + + $"'{checkpointPath}' names, so the {checkpoint.ItemsCollected} items it records " + + $"are that run's. {goingOn} {RunStopsHere} Wait for that sync to finish, or " + + "give this one its own -OutputFile and -CheckpointPath.", + + (CheckpointFile.NamedTemp, false) => + $"The temp file '{checkpoint.TempFile}' that the resume checkpoint at " + + $"'{checkpointPath}' names cannot be opened for writing by this account - the " + + "open was refused on permissions, not on sharing - so the " + + $"{checkpoint.ItemsCollected} items it records cannot be recovered here. " + + $"{goingOn} {RunStopsHere} Grant write access to that file and run again to " + + "recover them, or remove it and the checkpoint to sync afresh.", + + (CheckpointFile.Output, true) => refusal.Route switch + { + CheckpointStopRoute.Promoting => + $"Another sync is still writing '{outputPath}'. The resume checkpoint at " + + $"'{checkpointPath}' records {checkpoint.ItemsCollected} items into the " + + $"temp '{checkpoint.TempFile}'; going on would replace '{outputPath}' with " + + "them under the run that holds it, save this run's position over the " + + "checkpoint that counts those items, and advance the delta token past " + + $"them. {RunStopsHere} Wait for that sync to finish, or give this one its " + + "own -OutputFile and -CheckpointPath.", + + CheckpointStopRoute.FreshAfterVanishedTemp => + $"Another sync is still writing '{outputPath}'. The resume checkpoint at " + + $"'{checkpointPath}' recorded {checkpoint.ItemsCollected} items into a " + + "temp that is gone; going on would re-enumerate from the last saved delta " + + $"token and replace '{outputPath}' under the run that holds it, save this " + + "run's position over the checkpoint that counts those items, and advance " + + $"the delta token past them. {RunStopsHere} Wait for that sync to finish, " + + "or give this one its own -OutputFile and -CheckpointPath.", + + _ => + $"Another sync is still writing '{outputPath}', which the resume checkpoint " + + $"at '{checkpointPath}' records {checkpoint.ItemsCollected} items into. " + + "Going on would cut that file back under it, save this run's position over " + + "the checkpoint that counts those items, and advance the delta token past " + + $"them. {RunStopsHere} Wait for that sync to finish, or give this one its " + + "own -OutputFile and -CheckpointPath.", + }, + + _ => UnopenableOutputSentence(), + }; + } + + /// + private static ErrorRecord CheckpointFileStop(CheckpointFileRefusal refusal, + string checkpointPath, string outputPath, PaginationCheckpoint checkpoint) => + new(new InvalidOperationException(CheckpointFileStopMessage(refusal, checkpointPath, + outputPath, checkpoint)), + (refusal.File, refusal.Held) switch + { + (CheckpointFile.NamedTemp, true) => "CheckpointTempHeld", + (CheckpointFile.NamedTemp, false) => "CheckpointTempUnopenable", + (CheckpointFile.Output, true) => "CheckpointOutputHeld", + _ => "CheckpointOutputUnopenable", + }, + // ResourceBusy for a file another run has, which is what a caller waits out. Past + // that the category follows the open's own reason rather than assuming a + // permission: PermissionDenied is a mode this account can be granted, and a symlink + // loop, an over-long path, a socket, a FIFO or a directory at the path is none - + // InvalidOperation says so, and a refusal that carries no reason at all is the + // temp's, whose own sentence names permissions. + refusal.Held + ? ErrorCategory.ResourceBusy + : refusal.Reason is { IsPermission: false } + ? ErrorCategory.InvalidOperation + : ErrorCategory.PermissionDenied, + checkpointPath); + + /// + /// How the run ends when the copy a promotion stages beside the output could not be written, + /// or could not be moved onto the output once it was. Nothing here is a file that has gone + /// missing, and reading it as one was what deleted the checkpoint and moved the delta token + /// past changes held in the temp: every item it counts is in that temp, whole and claimable, + /// and the run that comes back stages the same bytes over the same output. So the sentence + /// says where those changes are, what failed and where, and that both files are as they were + /// found - and the run stops rather than sync past a position it would then save over. + /// + /// The temp those changes are in. The checkpoint's own record of it + /// where it has one, and the file adoption picked where the checkpoint predates that field: + /// either way it is the file the reader has to keep, and a sentence that read it off the + /// checkpoint named nothing at all on the adoption route. + /// What the sweep took off the staging name on the way here, where + /// it took anything: the run deleted an entry and then stopped, and a reader who is told only + /// that the staging failed goes looking for a file that is no longer there. Named in the + /// stop rather than warned about separately, because a warning written before a terminating + /// error is what -WarningAction Stop ends the run on instead of this sentence. + private static string CheckpointStagingStopMessage(StagingFailure staging, + string checkpointPath, string outputPath, PaginationCheckpoint checkpoint, + string tempName, string? removedKind) => + $"The {checkpoint.ItemsCollected} items the resume checkpoint at '{checkpointPath}' " + + $"records are in '{tempName}', whole, but staging them into '{outputPath}' " + + $"failed at '{staging.Path}': {staging.Reason}" + + (removedKind != null + ? $", after removing what stood at the staging name: {removedKind}" + : "") + + $". {NothingWasChanged} fix that and run " + + "again to recover them, or remove the temp and the checkpoint to re-enumerate from the " + + "last saved delta token."; + + /// + private static ErrorRecord CheckpointStagingStop(StagingFailure staging, string checkpointPath, + string outputPath, PaginationCheckpoint checkpoint, string tempName, + string? removedKind) => + new(new InvalidOperationException(CheckpointStagingStopMessage(staging, checkpointPath, + outputPath, checkpoint, tempName, removedKind)), + "CheckpointStagingFailed", ErrorCategory.WriteError, checkpointPath); + + /// + /// What a reconcile leaves for the caller to do. Three of the four say do not resume, and + /// they are not the same answer: one is a checkpoint a live sync is still using, on which + /// the run ends with both files where they were; another is a promotion whose copy could + /// not be written or moved, on which the run also ends with both files where they were; + /// and the last is a checkpoint describing items that are in no file, which is deleted + /// wherever this run is able to delete it. /// private enum CheckpointRecovery { @@ -623,12 +862,26 @@ private enum CheckpointRecovery Resumable, /// - /// Another sync is writing the temp this checkpoint names. Nothing here is this run's - /// to recover or to remove: the checkpoint stays, its temp is spared this run's sweep, - /// and this run enumerates from the delta state. + /// Another sync is writing one of the files this checkpoint stands for - the temp it + /// names, or the output it appends to - or this account cannot open it read-write. + /// Nothing here is this run's to recover or to remove, and there is no enumeration it + /// can make that does not destroy what those files hold, so the caller ends the run on + /// it. Which file, and which of the two ways the claim failed, comes back beside this, + /// because between them they decide what the caller can honestly tell the reader to do + /// about it. /// Refused, + /// + /// The copy the promotion stages beside the output could not be written, or could not be + /// moved onto the output once it was. The temp holds every change the checkpoint counts + /// and the checkpoint still names it, so nothing here is recoverable by re-enumerating + /// and nothing is this run's to remove: the caller ends the run on it, naming the write + /// that failed and where. Told from Discarded because that one deletes the position + /// counting those changes and moves the token past them. + /// + StagingFailed, + /// /// The items the checkpoint counts are in no file this run can reach, so there is /// nothing to resume from. The delete that goes with it is attempted and may fail; the @@ -661,12 +914,17 @@ private enum CheckpointRecovery /// When the counted items turn out to be in no file, the delta link has not moved, so /// re-enumerating costs time and loses nothing while resuming past them loses them for good. /// - /// A named temp open in another run is none of those: the files are exactly what the - /// checkpoint says they are, and the sync they belong to has not been interrupted. Its - /// items are that run's, and the checkpoint is the position it comes back to, so both are - /// left where they are - and the caller still must not resume, since a position whose items - /// are in a file this run cannot have leaves them out of the output while the delta token - /// advances past them. + /// A file of the checkpoint's that another run has open is none of those: the files are + /// exactly what the checkpoint says they are, and the sync they belong to has not been + /// interrupted. Its items are that run's, and the checkpoint is the position it comes back + /// to, so neither is this run's to touch - and there is nothing left for this run to do + /// either. Resuming would leave those items out of the output while the delta token advanced + /// past them; re-enumerating would replace the output, save this run's position over the + /// checkpoint that counts them, and move the token anyway. Refused comes back, and the caller + /// ends the run there. The named temp is one such file; the output a checkpoint naming no + /// temp records its items into is the other, and there the run that holds it is one that has + /// already resumed - so a second run over the same command line finds nothing in the + /// checkpoint to refuse it by and has to be stopped by the claim instead. /// /// Whether the checkpoint may be resumed from is the answer, and not the file left on disk. /// A bail-out that deletes the checkpoint it has just given up on can fail to delete it - a @@ -676,20 +934,48 @@ private enum CheckpointRecovery /// re-enumerate past, and advanced the delta token over items that are in no file at all. /// Discarded binds either way; the file that outlives it is given up on again next run. /// - private CheckpointRecovery ReconcileCheckpointWithFiles(string checkpointPath, string outputPath, PaginationCheckpoint checkpoint) + private CheckpointRecovery ReconcileCheckpointWithFiles(string checkpointPath, + string outputPath, PaginationCheckpoint checkpoint, out CheckpointFileRefusal refused, + out StagingFailure? staging, out string? stagingTemp, out string? stagingRemoved) { + refused = CheckpointFileRefusal.None; + staging = null; + stagingTemp = null; + stagingRemoved = null; if (checkpoint.DataLength is not { } dataLength) { if (!File.Exists(outputPath)) { - if (TryAdoptOrphanedTemp(outputPath, checkpoint.ItemsCollected)) + var adopted = TryAdoptOrphanedTemp(outputPath, checkpoint.ItemsCollected, + out stagingTemp, out staging, out var adoptSwept); + + // What the adoption took off its own staging name before it created one there. + // The name is nobody's to keep - a link, a pipe or a copy left by an interrupted + // promotion standing at it is gone - so the run says which it was. Recorded here + // and warned about once this run's own outcome for the recovery is known: a + // warning is a terminating error under -WarningAction Stop, so written here it + // ended the run with the entry deleted, the changes still in the temp and + // nothing recovered. + stagingRemoved = adoptSwept?.Kind; + if (adoptSwept is { } adoptRemoval) WriteVerbose(adoptRemoval.Sentence); + + if (adopted) { WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted sync's temp file. Resuming from checkpoint."); + if (adoptSwept is { } removed) WriteWarning(removed.Sentence); return CheckpointRecovery.Resumable; } + // The copy could not be staged beside the output. Every change the checkpoint + // counts is in the temp this names, whole, so the delete below is not the answer: + // it would take the position counting them away and move the token past every + // one. What came off the name goes into the stop the caller writes, since a + // warning in front of a terminating error is what -WarningAction Stop ends on. + if (staging != null) return CheckpointRecovery.StagingFailed; + WriteWarning("Checkpoint found but output file is missing. Deleting stale checkpoint and starting fresh."); PaginationCheckpoint.Delete(checkpointPath); + if (adoptSwept is { } removedAnyway) WriteWarning(removedAnyway.Sentence); return CheckpointRecovery.Discarded; } @@ -702,19 +988,58 @@ private CheckpointRecovery ReconcileCheckpointWithFiles(string checkpointPath, s if (checkpoint.TempFile != null) { - if (TryPromoteNamedTemp(outputPath, checkpoint.TempFile, dataLength)) + stagingTemp = checkpoint.TempFile; + var promotion = TryPromoteNamedTemp(outputPath, checkpoint.TempFile, dataLength, + out var promotionReason, out staging, out var swept); + + // What the promotion took off its own staging name before it created one there. The + // name is nobody's to keep - a link, a pipe or a copy left by an interrupted + // promotion standing at it is gone - so the run says which it was. Recorded here and + // warned about once this run's own outcome for the recovery is known: a warning is a + // terminating error under -WarningAction Stop, so written here it ended the run with + // the entry deleted, the changes still in the temp and nothing recovered. + stagingRemoved = swept?.Kind; + if (swept is { } promotionRemoval) WriteVerbose(promotionRemoval.Sentence); + + // The copy could not be staged, or could not be moved once it was. Both files are + // where they were found and every change the checkpoint counts is still in the temp + // it names, so this is not the route below for a temp whose changes are in no file: + // that one deletes the position counting them, re-enumerates over the output and + // moves the token past them, and reached from here it did so with the temp holding + // every one. + if (promotion == TempPromotion.StagingFailed) return CheckpointRecovery.StagingFailed; + + // The output the promotion would land in is another run's, or is not a file this + // account can write at the recorded offset. The same stop the branch below reaches, + // about the same file for the same reason: the run that has the output is appending + // to it, and this one would be replacing it under them and moving the token past + // what it wrote. + // + // Asked for before the move, so nothing was replaced - and the temp, with the + // checkpoint still naming it, is exactly as it was found, since the temp is unlinked + // only where the output was kept. So the position and the file it counts are the + // pair they were, and the run that comes back promotes the same changes to the same + // length over the same output. + if (promotion is TempPromotion.OutputHeld or TempPromotion.OutputUnopenable) { + refused = new CheckpointFileRefusal(CheckpointFile.Output, + promotion == TempPromotion.OutputHeld + ? ClaimRefusal.AnotherRunHasIt + : ClaimRefusal.CannotBeOpened, + CheckpointStopRoute.Promoting, promotionReason); + return CheckpointRecovery.Refused; + } + + if (promotion == TempPromotion.Taken) + { + // Those changes are the output now, and the output is this run's - the promotion + // kept it through the move. Repoint the checkpoint at it immediately, so a second + // interruption cannot promote the same temp a second time - and before the + // warnings below, which under -WarningAction Stop are where the run ends: the + // position on disk has to be the one that resumes from what is now in the output. + RepointCheckpointAtHeldOutput(checkpoint, checkpointPath, outputPath, HeldOutput!); WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted sync's temp file. Resuming from checkpoint."); - // Those items are the output now. Repoint the checkpoint at it immediately, - // so a second interruption cannot promote the same temp a second time. - checkpoint.TempFile = null; - checkpoint.OutputFile = Path.GetFullPath(outputPath); - checkpoint.DataLength = new FileInfo(outputPath).Length; - try { checkpoint.Save(checkpointPath); } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) - { - WriteWarning($"Checkpoint save failed after recovery: {ex.Message}"); - } + if (swept is { } removed) WriteWarning(removed.Sentence); return CheckpointRecovery.Resumable; } @@ -723,26 +1048,71 @@ private CheckpointRecovery ReconcileCheckpointWithFiles(string checkpointPath, s // right now. Its items are that run's, the checkpoint is the position it comes back // to, and this run has nothing here to recover: deleting the checkpoint below, or // resuming from it, would be the same mistake as unlinking the temp, one file over. - if (NamedTempIsHeld(outputPath, checkpoint.TempFile, dataLength)) - { - WriteWarning( - "Another sync is still writing the temp file this resume checkpoint names, so the " - + $"{checkpoint.ItemsCollected} items it records are that run's and are not recovered here. " - + "Both files are left as they are and this run re-enumerates from the last saved delta " - + "token; no changes are lost. Two syncs writing one -OutputFile replace each other's " - + "result; give each its own."); - return CheckpointRecovery.Refused; - } + // + // Or the claim failed because this account cannot open the file read-write, which + // leaves both files in the same place and has no second sync behind it. The refusal + // is the same; only what it can honestly say about the cause differs. + // + // Refused says nothing here, because the sentence a caller acts on is about the run + // and not about the reconcile: this sync cannot go on without replacing the output + // beside these two files, saving its own position over the checkpoint that counts + // their rows, and moving the delta token past them. It ends instead, and the site + // that ends it is the one that can say so before anything has happened. + refused = new CheckpointFileRefusal(CheckpointFile.NamedTemp, + RefusalForNamedTemp(outputPath, checkpoint.TempFile, dataLength, mayWrite: true)); + if (refused.Refuses) return CheckpointRecovery.Refused; + + // The temp is not there, or is shorter than the length recorded for it - and the + // first of those is what a promotion leaves for the instant between its unlink and + // the save that repoints the position at the output. A second run over the same + // command line, released with the first, read that instant and said those changes + // were not on disk, deleted the position counting them, re-enumerated over the + // output the promoter was holding, and moved the delta token past both runs' work. + // So the output is asked about before any of that, and a promoter holding it ends + // this run where the branch below ends it. + // + // Only the refusals are read. An output nothing holds is not evidence that a + // promotion landed in it, whatever it measures: this checkpoint records its changes + // into the temp, and a previous sync's file standing at the path measures exactly + // what a promotion would have left about as often as a promoted one does - two rows + // of one shape being two rows of another shape's length. So that case takes the route + // it always took, which replaces the whole file rather than cutting it back to an + // offset counted in a different one. + refused = new CheckpointFileRefusal(CheckpointFile.Output, + RefusalForStandingOutput(outputPath, out var vanishedTempReason), + CheckpointStopRoute.FreshAfterVanishedTemp, vanishedTempReason); + if (refused.Refuses) return CheckpointRecovery.Refused; WriteWarning( $"The interrupted sync's temp file is missing or incomplete, so the {checkpoint.ItemsCollected} items it " + "recorded are not on disk. Re-enumerating from the last saved delta token; no changes are lost."); PaginationCheckpoint.Delete(checkpointPath); + // And what the sweep took off the staging name, after what this run did about the + // checkpoint. + if (swept is { } removedAnyway) WriteWarning(removedAnyway.Sentence); return CheckpointRecovery.Discarded; } - if (TryTrimOutputToCheckpoint(outputPath, dataLength)) return CheckpointRecovery.Resumable; - + // No temp is named, so the run that wrote this checkpoint was appending straight into the + // output and its items are in that file. The claim above, one file over: a sync that has + // resumed once holds -OutputFile open with those changes in it, and leaves a checkpoint + // with no temp for a second run over the same command line to refuse it by - so that run + // judged the checkpoint its own, cut the live file back to the recorded offset, appended, + // and moved the token past changes the output no longer held. + // + // So the claim is not let go. The handle it opens is the handle this run writes through: + // the cut is a SetLength on it, the writer further down appends onto it, and it is + // released where the writing ends. Let go here, a page-fetch before the writer took an + // open of its own, it left a window a second run walked straight through. + var taken = TakeOutputForCheckpoint(outputPath, dataLength, out var takenReason); + if (taken == CheckpointOutput.Taken) return CheckpointRecovery.Resumable; + + refused = new CheckpointFileRefusal(CheckpointFile.Output, RefusalFrom(taken), + CheckpointStopRoute.Appending, takenReason); + if (refused.Refuses) return CheckpointRecovery.Refused; + + // Not the file the checkpoint describes - absent, or shorter than the length recorded - + // so the changes it counts are in no file and there is nothing to append to. WriteWarning( $"'{outputPath}' no longer holds the {checkpoint.ItemsCollected} items the resume checkpoint records. " + "Re-enumerating from the last saved delta token; no changes are lost."); @@ -761,18 +1131,37 @@ private void ExecuteDeltaSync( string? filter, Stopwatch sw) { + // What the reconcile below takes, on a resumed run, is the output itself, and it is this + // run's until the writing ends. Released on every way out of the loop, and again from + // Dispose, which is what a pipeline stopped from outside reaches. + using var writingEnds = ReleasingOutputWhenWritingEnds(); + bool isFullResync = false; - // Whether this run has already decided the checkpoint on disk is another sync's, and - // the temp that checkpoint points at. Both outlive the attempt that refused, because - // the 410 door below sends the loop round again: an attempt that has forgotten the - // refusal deletes the position it was told to leave alone and sweeps away the temp - // holding its items, seconds after warning that neither would be touched. Ownership is - // still re-decided on every load - the checkpoint is a file, and the sync that owns it - // is still going - but only ever this way round, and never granted back: the retry - // rebuilds the request URL, so a checkpoint refused against the delta link can compare - // equal to the full re-sync one, and that is not evidence of anything. + // Whether this run has given up on the checkpoint on disk, and the temp that checkpoint + // points at. Both outlive the attempt that refused, because the 410 door below sends the + // loop round again: an attempt that has forgotten the refusal resumes from the position + // it was told to leave alone and sweeps away the temp holding its items, seconds after + // warning that neither would be touched. A refusal is still decided again on every load + // - the checkpoint is a file, and the sync that owns it is still going - but only ever + // this way round, and never granted back: the retry rebuilds the request URL, so a + // checkpoint refused against the delta link can compare equal to the full re-sync one, + // and that is not evidence of anything. var refusedCheckpoint = false; + // The temp that checkpoint named, or null - which is both "nothing refused" and "what + // was refused names no temp". sparedCheckpoint below tells those apart, and the sweep + // reads the pair: a checkpoint written before the name was recorded stands for a temp + // beside this output without saying which one, so its refusal holds the sweep off over + // all of them. string? refusedTemp = null; + // And which reading refused it, which is what the two doors that would otherwise delete + // the file - the completion path and the 410 - decide on. One reading a sync can carry + // this far: the ownership verdict's, where whatever wrote the file resumes from exactly + // that, so deleting it makes the warning that has just promised both files false. A + // checkpoint whose temp, or whose output, this run could not claim reaches no door, + // because the run that found that out ended there. The cost of sparing is that the file + // suppresses the next -Latest and holds this output's stale-temp sweep off while it + // stands. + SparedCheckpoint? sparedCheckpoint = null; // The temp an attempt of this run kept when it died, because a checkpoint counting its // items was on disk. Nothing else records it: the sweep at the top of the next attempt // is the only thing that reclaims a temp, and a refusal holds that off over every temp @@ -783,8 +1172,8 @@ private void ExecuteDeltaSync( // Whether an attempt of this run has written a checkpoint of its own over // -CheckpointPath. The refusal above leaves the file where it is and then the very next // page boundary saves over the same path, so what the refusal was protecting is gone - // and what is there is this run's - which is the one thing the 410 door has to know - // before it spares the file on the refusal's account. Run-scoped, unlike the + // and what is there is this run's - which is the other thing the two doors have to know + // before they spare the file on the refusal's account. Run-scoped, unlike the // per-attempt flag of the same reading further down: that one answers which attempt // saved the checkpoint now on disk, and the keep it feeds needs exactly that. var tookOverCheckpoint = false; @@ -826,8 +1215,10 @@ private void ExecuteDeltaSync( is { } refusal) { // What the refusal leaves has to survive the sweep in this same run, - // and the 410 retry that runs it a second time. + // the 410 retry that runs it a second time, and the delete either door + // ends with. refusedCheckpoint = true; + sparedCheckpoint = SparedCheckpoint.NotThisRuns; refusedTemp = orphanCp.TempFile; WriteWarning(refusal); } @@ -863,20 +1254,57 @@ private void ExecuteDeltaSync( if (orphanCp.NextLink != null) { recovery = ReconcileCheckpointWithFiles( - checkpointPath, outputPath, orphanCp); - - // A temp another sync still holds is refused the way an - // ownership refusal is, and for the same two reasons: the - // reload below would otherwise resume from a position whose - // items are in a file this run cannot have - leaving them out - // of the output while the delta token advances past them - and - // the sweep in this same run would take that temp the moment - // its holder is gone, with the checkpoint naming it still on - // disk. + checkpointPath, outputPath, orphanCp, + out var refused, out var staging, out var stagingTemp, + out var stagingRemoved); + + // The write that would have put the checkpoint's changes at the + // output path failed, and both files are where they were found. + // Read as a temp that had gone - which is what the catch around + // the promotion made of it - this run said those changes were on + // no disk, deleted the position counting them, re-enumerated over + // the output and moved the delta token past every one of them, + // with the temp holding them all the while. + if (recovery == CheckpointRecovery.StagingFailed) + { + ThrowTerminatingError(CheckpointStagingStop( + staging!.Value, checkpointPath, outputPath, orphanCp, + stagingTemp!, stagingRemoved)); + return; + } + + // A file of this checkpoint's that this run could not claim ends + // the run, here, before it has enumerated a page, written a byte + // of output, saved a checkpoint, swept a temp or touched the + // delta state. Every other answer this sync could give away the + // two files. + // + // The rows the checkpoint counts are in that file and in no + // other, and the checkpoint is the only thing on disk that + // counts them: a holder still running deletes it itself on the + // way out, a holder that dies keeps its temp for exactly as + // long as a checkpoint counting it is there, and where the + // claim failed on permissions instead it is the position write + // access has to be granted for. Going on takes that away by + // going on: the sweep at the top of this attempt reclaims the + // temp the moment its holder is gone, the first page boundary + // saves this run's own position over the same path, the + // completion door then deletes what it finds there as its own, + // and a page carrying a deltaLink moves the token past changes + // held in that file alone. Resuming is worse again - the items + // are in a file this run cannot have, so they are left out of + // the output while the token advances past them - and where the + // file is the output itself, resuming is a cut taken out from + // under a live writer, whose next write lands past the hole. + // + // Which leaves nothing this sync can do with the -OutputFile + // and -CheckpointPath it was given, and saying so is the whole + // of what it does. if (recovery == CheckpointRecovery.Refused) { - refusedCheckpoint = true; - refusedTemp = orphanCp.TempFile; + ThrowTerminatingError(CheckpointFileStop( + refused, checkpointPath, outputPath, orphanCp)); + return; } } else if (!File.Exists(outputPath)) @@ -923,6 +1351,7 @@ private void ExecuteDeltaSync( // Decided again on this load rather than carried over: the // checkpoint is a file, and the sync that owns it is still going. refusedCheckpoint = true; + sparedCheckpoint = SparedCheckpoint.NotThisRuns; refusedTemp = checkpoint.TempFile; WriteWarning(reloadedRefusal); } @@ -1031,7 +1460,17 @@ void SaveBoundaryCheckpoint(PageCompletedInfo info) } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { - WriteWarning($"Checkpoint save failed (page boundary): {ex.Message}"); + // Buffered rather than written here. This runs from the page-boundary + // callback, on whichever thread the iterator resumed on, and a page that + // yields nothing resumes on the thread pool - so a WriteWarning from + // here throws PSInvalidOperationException, and the disk problem it was + // reporting ends the sync instead of being reported. A delta enumeration + // meets that shape most: the empty-page limit is raised to 1000 for it + // precisely because delta endpoints return runs of empty pages behind + // nextLinks. The drains write it from the pipeline thread, on the same + // channel the client's own warnings take. + client.EnqueueWarning( + $"Checkpoint save failed (page boundary): {ex.Message}"); } } @@ -1065,17 +1504,48 @@ void SaveBoundaryCheckpoint(PageCompletedInfo info) $"Left the temp files beside '{outputPath}' alone: '{refusedTemp}' " + "holds the items of the checkpoint this run refused."); } + else if (sparedCheckpoint is SparedCheckpoint.NotThisRuns + && refusedTemp == null) + { + // The refused checkpoint names no temp, which is not the same as + // there being none: a checkpoint written before the name was + // recorded stands for a temp beside this output all the same, and + // nothing here says which of them it is. Keyed on the temp name + // alone, the refusal spared the checkpoint and the sweep took the + // one file it was pointing at, so the sync it belongs to came back + // to a position over nothing - seconds after a warning promising + // both files would be left as they are. The whole sweep goes with + // the refusal instead, and the orphans it would have taken are left + // to the next run over this output that has not refused anything. + WriteVerbose( + $"Left the temp files beside '{outputPath}' alone: one of them " + + "may hold the items of the checkpoint this run refused."); + } else { DeleteStaleTemps(outputPath); } } + // The output is let go the moment this attempt stops appending to it. Every + // way appendOutput is withdrawn - a checkpoint given up on after the + // reconcile, a nextLink that fails validation, a 410 sending the loop round + // for a full re-sync - ends with this run replacing the output from a temp + // instead, and a handle of its own on the destination is what that move + // fails against. + if (!appendOutput) ReleaseOutputHold(); + var writePath = appendOutput ? outputPath : $"{outputPath}.{Guid.NewGuid():N}.tmp"; // A resumed run appends to the output itself, so there is no temp to name. checkpointTempFile = appendOutput ? null : Path.GetFileName(writePath); try { - using (var writer = new StreamWriter(writePath, appendOutput)) + // The reconcile's own handle on the output, where this run resumed into + // it. Never a second open of that file while the hold stands: on Unix + // the hold's LOCK_EX refuses one from this process as readily as from + // another, and on Windows its write access does. + using (var writer = appendOutput && HeldOutput is { } held + ? held.AppendingWriter() + : new StreamWriter(writePath, appendOutput)) { var enumerable = iterator.StreamAllWithCountAsync( requestUrl, @@ -1272,7 +1742,36 @@ void SaveBoundaryCheckpoint(PageCompletedInfo info) // Success: the checkpoint's job is done - delete it BEFORE saving delta // state, so a crash between the two leaves a fresh incremental (correct) // rather than a resumable position into a completed enumeration (wrong). - DeleteCheckpoint(checkpointPath, "sync completed"); + // + // "Its job" is this run's job, and the file on disk is only this run's once no + // refusal left it standing and a save of its own has landed on the path. A run + // that refused the checkpoint and then finished without ever saving one - + // everything it had fitting in a single page, or a 410 arriving before the + // first boundary - was deleting the file it had warned it would leave alone, + // orphaning the temp it had just spared the sweep and sending the sync that + // needs both back to page one. + // + // One reading spares it: the ownership verdict's, where whatever wrote the + // file resumes from exactly it. It costs the next -Latest its baseline and + // holds this output's stale-temp sweep off, and that is cheaper than deleting + // a position another run is coming back to. A checkpoint whose temp, or whose + // output, this run could not claim never reaches here at all - that run stopped + // at the reconcile, having written nothing - so this door is only ever answering + // for a run that had an enumeration of its own to finish. + // + // The 410 door reads the same thing for the same reason. savedOwnCheckpoint + // answers only for the attempt that is finishing here and is not asked: every + // site that sets it sets the run-scoped flag beside it, which also covers the + // attempt a query the endpoint refused sent round again with a boundary + // checkpoint of this run's already over the path. + if (sparedCheckpoint is SparedCheckpoint.NotThisRuns && !tookOverCheckpoint) + { + WriteVerbose(SparedCheckpointVerbose("sync completed")); + } + else + { + DeleteCheckpoint(checkpointPath, "sync completed"); + } // And with it goes the last thing that could have named a temp an earlier // attempt kept - the attempt that finished need never have saved a checkpoint of @@ -1333,27 +1832,37 @@ void SaveBoundaryCheckpoint(PageCompletedInfo info) DrainClientMessages(); if (!DeltaState.Delete(deltaPath)) WriteVerbose("Could not delete expired delta state (file may be locked). It will be overwritten."); - // The refusal has to still hold over the file that is actually there. A - // checkpoint refused as another sync's is spared here because that sync resumes - // from exactly it - but the first page boundary of this run saves over the same - // path, and from then on the refused file is gone and the position on disk is - // this run's own. Sparing that one keeps a position into the enumeration the 410 - // has just declared dead: the retry re-enumerates in full, so the resource it - // records matches nothing this command line builds again, and every later run - // refuses it in turn - warning about a sync that was never there and holding off - // its stale-temp sweep on that account. It goes the way any other position into - // a dead enumeration goes. - if (refusedCheckpoint && !tookOverCheckpoint) + // The output this run took at the reconcile goes with the position it was taken + // for, which is the position the 410 has just declared dead. The retry starts + // over: it collects into a temp of its own and promotes that at the end, so it + // has no use for a handle opened at an offset counted in the enumeration behind + // it - and carried into the retry, that handle is one the retry's own reconcile + // asks about and is refused by. Measured: where the checkpoint delete below + // fails and the retry reads the same position again, the claim answered held and + // the run ended telling the caller to wait for a second sync that was this one. + ReleaseOutputHold(); + // The refusal has to still hold over the file that is actually there. A refused + // checkpoint is spared here because some run comes back to exactly it - but the + // first page boundary of this run saves over the same path, and from then on + // the refused file is gone and the position on disk is this run's own. Sparing + // that one keeps a position into the enumeration the 410 has just declared + // dead: the retry re-enumerates in full, so the resource it records matches + // nothing this command line builds again, and every later run refuses it in + // turn - warning about a sync that was never there and holding off its + // stale-temp sweep on that account. It goes the way any other position into a + // dead enumeration goes. + // + // A checkpoint whose temp, or whose output, this run could not claim does not + // reach this door either: a run that finds that out at the reconcile has not + // sent a request yet, so it cannot be the run a 410 answered. + if (sparedCheckpoint is SparedCheckpoint.NotThisRuns && !tookOverCheckpoint) { - // The token that expired is this run's; the checkpoint is not. It was - // refused moments ago as another sync's, and that sync resumes from exactly - // it - so an expired token of this one's is no reason to delete it, and - // deleting it took the other sync's position away right after the refusal - // said it would be left where it was. + // The token that expired is this run's; what the refusal left standing is + // not. It was left there moments ago for a sync that resumes from exactly + // it, and an expired token of this one's is no reason to take it away right + // after the warning said both files would be left where they were. WriteVerbose( - "Left the resume checkpoint alone (delta token expired (410 Gone)): it " - + "records an enumeration this run refused, which this run's token says " - + "nothing about."); + SparedCheckpointVerbose("delta token expired (410 Gone)")); } else { diff --git a/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs b/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs index 96bde4f..a60c248 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs @@ -168,8 +168,29 @@ protected override void ProcessRecord() // the remainder of the enumeration was appended onto a previous export's file. Every // form goes to the same comparison, and it settles which URL this run resumes before // anything is promoted, trimmed or deleted. + // + // And decided without writing anything, like everything else above the gate: which + // spellings of a file name the output's directory keeps apart is a question this run + // would otherwise answer by creating a file there, and -WhatIf reports what a run would + // do without moving the caller's directory's write time to do it. An entry already in + // the directory answers it for free; only a directory holding nothing cannot, and there + // the platform's own rule stands for this side of the gate while the run below asks the + // volume itself. Said out loud, because that is the one case where the two can differ. if (cpPath != null && File.Exists(cpPath) - && AttemptFormOf(PaginationCheckpoint.Load(cpPath), outputPath, countAutoAdded) is { } form) + && DirectoryCaseRuleNeedsProbe(Path.GetDirectoryName(outputPath))) + { + WriteVerbose( + $"Nothing in '{Path.GetDirectoryName(outputPath)}' says whether it keeps two " + + "spellings of a name apart, and nothing above the ShouldProcess gate writes a " + + "file to ask it, so the checkpoint's file names are compared here under the " + + $"{(PlatformCaseRule ? "case-sensitive" : "case-insensitive")} rule this " + + "platform applies by default. A run that goes past the gate asks the directory " + + "itself, so the two differ only on a volume whose rule is not that default."); + } + + if (cpPath != null && File.Exists(cpPath) + && AttemptFormOf(PaginationCheckpoint.Load(cpPath), outputPath, countAutoAdded, + mayWrite: false) is { } form) { includeAutoCount = form.IncludeCount; suppressTop = form.SuppressTop; @@ -183,7 +204,14 @@ protected override void ProcessRecord() // question that should need a Graph connection. var wouldAppend = ReconcileCheckpoint(cpPath, outputPath, includeAutoCount, suppressTop, apply: false); if (!ShouldProcess(outputPath, wouldAppend ? "Append JSONL data" : "Export JSONL data")) + { + // A preview that ends here is the only one that has to say this. The pass below + // reaches the same condition and ends the run on it, so a caller who is not + // previewing reads it as the error it is. + if (_wouldRemoveAtStagingName is { } wouldRemove) WriteWarning(wouldRemove); + if (_wouldStopOnCheckpointFile is { } wouldStop) WriteWarning(wouldStop); return; + } // Whether this run appends to the output rather than exporting into a fresh temp, // decided once, against every form of the URL the attempts below can build, and only @@ -193,9 +221,24 @@ protected override void ProcessRecord() // refusal has to outlast the attempt that made it. var appendToOutput = ReconcileCheckpoint(cpPath, outputPath, includeAutoCount, suppressTop, apply: true); + // What the reconcile just took, on a resumed run, is the output itself, and it is this + // run's until the writing ends. Released here on every way out of the loop below, + // including the one that never reaches it - GetClient on a session that is not connected + // - and again from Dispose, which is what a pipeline stopped from outside reaches. + using var writingEnds = ReleasingOutputWhenWritingEnds(); + // Init client after the gate (populates s_graphEndpoint for sovereign clouds) var client = GetClient(); + // Whether an attempt of this run has written a checkpoint of its own over + // -CheckpointPath. A refusal leaves the file where it is and the very next page boundary + // saves over the same path, so what the refusal was protecting is gone and what is there + // is this run's - which is what the completion path has to know before it spares the + // file on the refusal's account. Run-scoped, unlike the per-attempt flag of the same + // reading inside the loop: that one answers which attempt saved the checkpoint now on + // disk, and the temp it decides to keep needs exactly that. + var tookOverCheckpoint = false; + for (int attempt = 0; attempt < 3; attempt++) { try @@ -219,7 +262,8 @@ protected override void ProcessRecord() var checkpoint = PaginationCheckpoint.Load(cpPath); if (checkpoint != null) { - if (!DescribesThisExport(checkpoint, outputPath, includeAutoCount, suppressTop)) + if (!DescribesThisExport(checkpoint, outputPath, includeAutoCount, + suppressTop, mayWrite: true)) { // Ownership is re-decided on every load, not carried over from the // check above: the checkpoint is a file, the run that owns it is @@ -237,13 +281,48 @@ protected override void ProcessRecord() // second time. Withdrawn in silence, the recovery message was the // last thing on any stream, and a run that had quietly started over // was indistinguishable from one that resumed. - _refusedCheckpointTemp = checkpoint.TempFile; + // + // Two readings reach here and only one of them is about whose the + // file is, which is where they part company: the run does the same + // thing either way - drops the resume and exports fresh - and what + // it leaves on disk afterwards is the opposite thing. Every form of + // the request this run can build goes to the same comparison the + // adoption above the loop makes, so a checkpoint that matches one + // of them is this export's own, saved under the form an endpoint + // that refused $top or the auto-added $count has since sent the + // loop round without. The verdict is about the ones that match none. + var ownership = + AttemptFormOf(checkpoint, outputPath, countAutoAdded, + mayWrite: true) != null + ? CheckpointOwnership.Mine + : OwnershipOf(checkpoint, outputPath, includeAutoCount, + suppressTop, mayWrite: true); appendToOutput = false; - WriteWarning( - $"The resume checkpoint at '{cpPath}' no longer describes this export - the " - + "request was rebuilt without a query option the endpoint refused, or another " - + $"run replaced the file - so the resume is dropped and '{outputPath}' is " - + "exported from the beginning; nothing is lost."); + if (ownership == CheckpointOwnership.Mine) + { + // This export's own position, into the enumeration this run is + // about to make again and replace the output with. Nothing is + // held back on its account: the completion path deletes it as + // it deletes any checkpoint of this run's, and a temp it still + // names is this run's own for the sweep below to reclaim. Left + // standing instead, it is a position nothing comes back to, and + // the next run over the same command line reads it as its own - + // the finished output cut back to the byte count of an + // enumeration that has since been replaced, and the remainder + // appended from a nextLink counting items it no longer holds. + WriteWarning(RebuiltRequestWarning(cpPath, outputPath)); + } + else + { + // Another export's, or one nothing beside this output + // corroborates. The export it belongs to resumes from exactly + // that, so for the rest of the run the sweep below leaves this + // checkpoint's temp where it is, and the completion path leaves + // the checkpoint itself. + _sparedCheckpoint = SparedCheckpoint.NotThisRuns; + _refusedCheckpointTemp = checkpoint.TempFile; + WriteWarning(ReplacedCheckpointWarning(cpPath, outputPath)); + } } else if (checkpoint.NextLink == null) { @@ -305,7 +384,8 @@ protected override void ProcessRecord() // back to a position pointing at nothing and re-enumerated from the first // page - the cost the refusal was written to avoid. The sweep is all or // nothing over this output's temps, so a run that has just refused leaves - // them to the next run that has not. + // them to the next run that has not. A run that could not claim the temp + // its checkpoint names never gets here to decide: it stops above the client. // // And once more on a retry, where the checkpoint standing over the temp is // this run's own. An endpoint that refuses the auto-added $count, or $top, @@ -321,6 +401,24 @@ protected override void ProcessRecord() $"Left the temp files beside '{outputPath}' alone: '{_refusedCheckpointTemp}' " + "holds the items of the checkpoint this run refused."); } + else if (_sparedCheckpoint is SparedCheckpoint.NotThisRuns + && _refusedCheckpointTemp == null) + { + // The refused checkpoint names no temp, which is not the same as there + // being none: a checkpoint written before the name was recorded stands + // for a temp beside this output all the same, and there is nothing here + // that says which of them it is. Keyed on the temp name alone, the + // refusal spared the checkpoint and the sweep took the one file it was + // pointing at - so the export it belongs to came back to a position over + // nothing and re-enumerated from the first page, the exact cost the + // refusal is written to avoid, under a warning that had just promised + // the files would be left as they are. So the whole sweep goes with the + // refusal, and the orphans it would have taken are left to the next run + // over this output that has not refused anything. + WriteVerbose( + $"Left the temp files beside '{outputPath}' alone: one of them may " + + "hold the items of the checkpoint this run refused."); + } else if (_keptTempPath != null && File.Exists(_keptTempPath)) { WriteVerbose( @@ -333,6 +431,14 @@ protected override void ProcessRecord() DeleteStaleTemps(outputPath); } } + // The output is let go the moment this attempt stops appending to it. Every way + // appendToOutput is withdrawn - the checkpoint or the output gone between + // attempts, an ownership verdict landing on a reload, a 410 sending the loop + // round for a full pass - ends with this run replacing the output from a temp + // instead, and a handle of its own on the destination is what that move fails + // against. An attempt that still appends keeps the hold it was given. + if (!appendToOutput) ReleaseOutputHold(); + var writePath = appendToOutput ? outputPath : $"{outputPath}.{Guid.NewGuid():N}.tmp"; // What the checkpoint sites below should say about WHERE the counted items are. // A resumed run appends to the output itself, so it has no temp to name. @@ -376,7 +482,14 @@ void ReleaseKeptTemp() try { - using (var writer = new StreamWriter(writePath, appendToOutput)) + // The reconcile's own handle on the output, where this run resumed into it. + // Never a second open of that file while the hold stands: on Unix the hold's + // LOCK_EX refuses one from this process as readily as from another, and on + // Windows its write access does. The pre-length adoption route appends + // without a hold and opens its own, as it always has. + using (var writer = appendToOutput && HeldOutput is { } held + ? held.AppendingWriter() + : new StreamWriter(writePath, appendToOutput)) { var iterator = new PageIterator(client); @@ -406,11 +519,22 @@ void ReleaseKeptTemp() DataLength = checkpointDataLength }.Save(cpPath); savedOwnCheckpoint = true; + tookOverCheckpoint = true; ReleaseKeptTemp(); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { - WriteWarning($"Checkpoint save failed (page boundary): {ex.Message}"); + // Buffered rather than written here. This callback runs + // on whichever thread the iterator resumed on, and a + // page that yields nothing resumes on the thread pool - + // so a WriteWarning from here throws + // PSInvalidOperationException, and the disk problem it + // was reporting ends the export instead of being + // reported. The drains write it from the pipeline + // thread, on the same channel the client's own warnings + // take. + client.EnqueueWarning( + $"Checkpoint save failed (page boundary): {ex.Message}"); } } if (info.NextPageUrl != null) @@ -452,6 +576,7 @@ void ReleaseKeptTemp() DataLength = checkpointDataLength }.Save(cpPath); savedOwnCheckpoint = true; + tookOverCheckpoint = true; ReleaseKeptTemp(); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) @@ -487,7 +612,37 @@ void ReleaseKeptTemp() // The items are in the output now, so the position describing them is // spent. A crash in the gap costs a re-enumeration: the checkpoint names a // temp that is no longer there, which recovery reads as unusable. - if (cpPath != null) PaginationCheckpoint.Delete(cpPath); + // + // Spent for this export, and the file on disk is only this export's once no + // refusal left it standing and a save of its own has landed on the path. A + // run that refused what it found and then finished without ever saving one, + // everything it had fitting in a single page, deleted the file it had warned + // it would leave alone, orphaning the temp it had just spared the sweep and + // sending the export that needs both back to page one. + // + // One of the three readings spares it: the ownership verdict's, where + // whatever wrote the file resumes from exactly it. Not a checkpoint refused + // for a request the endpoint made this run rebuild - that one records a + // position into an enumeration this run has just replaced and nothing more, + // and left standing beside the finished output it is what the next run over + // the same command line reads as its own and promotes a superseded temp + // over a completed file from. And not a checkpoint whose temp, or whose + // output, this run could not claim, which reaches no door at all: that run + // stopped at the reconcile, above the client, having written nothing. + // + // tookOverCheckpoint is the flag, and savedOwnCheckpoint has nothing left + // to add: it answers for the attempt finishing here, and every site that + // sets it sets the run-scoped one beside it - which also covers the attempt + // a query the endpoint refused sent round again with a boundary checkpoint + // of this run's already over the path. + if (_sparedCheckpoint is SparedCheckpoint.NotThisRuns && !tookOverCheckpoint) + { + WriteVerbose(SparedCheckpointVerbose("export completed")); + } + else if (cpPath != null) + { + PaginationCheckpoint.Delete(cpPath); + } // And with it goes the last thing that could have named a temp an earlier // attempt kept - the attempt that finished need never have saved a @@ -705,6 +860,32 @@ private static string ForeignCheckpointWarning(string checkpointPath, string out + $"left as it is and '{outputPath}' is exported from the beginning. Two exports sharing " + "one -CheckpointPath overwrite each other's resume position; give each its own."; + /// + /// What a caller can act on when the request was rebuilt under this export's own feet: the + /// endpoint refused a query option part-way through, the loop went round without it, and + /// the position on disk counts a URL this run has stopped building. It is this export's + /// own, and the resume it was announced for is being taken back, so the sentence has to say + /// both - the collection is enumerated from the first page again, and the file goes the way + /// any of this run's own positions go once the output is complete. + /// + private static string RebuiltRequestWarning(string checkpointPath, string outputPath) => + $"The resume checkpoint at '{checkpointPath}' records this export under a form of the " + + "request the endpoint has since refused, so the resume is dropped and " + + $"'{outputPath}' is exported from the beginning; the checkpoint is this run's own and " + + "goes with the completed export."; + + /// + /// And what they can act on when a resume already announced turns out to be another run's + /// position: the file at -CheckpointPath was replaced between the reconcile's read and this + /// one, so what is there now describes an output or an enumeration that is not this one. + /// Both files are left where they are, because whatever wrote them resumes from exactly + /// that. + /// + private static string ReplacedCheckpointWarning(string checkpointPath, string outputPath) => + $"The resume checkpoint at '{checkpointPath}' no longer describes this export - another " + + $"run replaced the file - so the resume is dropped and '{outputPath}' is exported from " + + "the beginning; the checkpoint and the temp it names are left as they are."; + /// /// What a caller can act on when a checkpoint that records no output file is refused: it /// was written before the output was recorded, nothing beside this one's output stands for @@ -719,16 +900,224 @@ private static string UncorroboratedCheckpointWarning(string checkpointPath, str + $"'{outputPath}' is exported from the beginning; nothing is lost."; /// - /// What a caller can act on when the temp a checkpoint names is open in another run. Not - /// phrased as a fault: nothing has gone wrong with either file, two exports are writing one - /// -OutputFile, and the only thing the caller has to decide is which of them they meant to - /// keep - the one that finishes last is the one whose rows the output ends up with. + /// How the run ends when a file its own checkpoint stands for is open in another export, or + /// cannot be opened read-write by this account. Two files reach this, and the reading is one: + /// the rows the checkpoint counts are in that file and in no other, the checkpoint is the + /// only thing on disk that counts them, and this export has no pass to make that leaves the + /// pair intact - it saves its own position over the checkpoint at the first page boundary, + /// after which the completion door deletes what it finds there as its own. So it says what + /// it found, what it did not do, and the two ways out, and every file is exactly where it was. + /// + /// Which file decides what going on would cost, and for an output refusal + /// decides it further: a checkpoint naming a temp + /// that is still there holds the rows in that temp, and going on promotes them, replacing + /// the output outright; one naming a temp that has vanished holds them nowhere this run can + /// reach, and going on enumerates the collection again and replaces the output with the + /// result; and one naming no temp at all was written by a run appending straight into the + /// output, so the rows are in the output itself and going on cuts it back under its writer - + /// whose next write then lands past the hole that offset leaves. All three reach the same + /// claim on the same file, because a rename and a cut are both closed to a run that does not + /// hold it. + /// + /// Not phrased as a fault in the held case: nothing has gone wrong with either file, two + /// exports were pointed at one -OutputFile, and the caller has only to decide which of them + /// they meant. The unopenable case has no second export to go and find at all, and the + /// sentence written for the other one pointed the reader at a run that was never there + /// instead of at the open's own reason - which is named now, in , rather than assumed to be a permission: a symlink loop and an over-long path + /// reach it too, and granting write access fixes neither. + /// + /// The closing sentence is the caller's: a run that is stopping says so, and a -WhatIf pass + /// over the same two files says what would happen instead. + /// + private static string CheckpointFileStopMessage(CheckpointFileRefusal refusal, + string checkpointPath, string outputPath, PaginationCheckpoint checkpoint, string closing) + { + // What going on would cost past a held temp, which is the output beside it - named only + // where there is one. A checkpoint naming a temp is reached with nothing at -OutputFile + // at all, and a run that said it would replace that file sent the caller to look at + // something no export had written yet. + var goingOn = File.Exists(outputPath) + ? $"Going on would replace '{outputPath}' and save this run's position over the " + + "checkpoint that counts those items." + : "Going on would save this run's position over the checkpoint that counts those " + + "items."; + + // The output is unopenable rather than held. The three routes agree past this point - + // the code beyond this check falls through to the same place a file that never held the + // recorded items reaches, which deletes the checkpoint and replaces the output at the + // completion that follows - so only the first clause, naming where the items are now, + // and the recovery word in the advice change per route. + string UnopenableOutputSentence() + { + var reason = refusal.Reason!.Value; + var openFailed = $"'{outputPath}' could not be opened for writing: {reason.Reason}."; + var recovers = refusal.Route switch + { + CheckpointStopRoute.Promoting => " to recover them", + CheckpointStopRoute.FreshAfterVanishedTemp => "", + _ => " to resume from it", + }; + // Four ways out, because the causes do not share one. A permission on the file is + // granted; a permission on the directory holding it is granted there, and the file + // this sentence names is one the caller cannot reach to change or remove; a + // directory at the path is removed or pointed away from, and no grant would have + // made it writable at an offset; and a loop, an over-long path, a socket or a pipe + // is none of those - what this run can honestly say about them is what failed. + var advice = reason.IsDirectory + ? $"Remove it (or point -OutputFile elsewhere) and run again{recovers}, or " + + "remove the checkpoint to export afresh." + : reason.IsUnsearchableParent + ? $"Grant access to that directory and run again{recovers}, or remove the " + + "checkpoint to export afresh." + : reason.IsPermission + ? $"Grant write access to that file and run again{recovers}, or remove " + + "it and the checkpoint to export afresh." + : $"Fix that and run again{recovers}, or remove it and the checkpoint to " + + "export afresh."; + + return refusal.Route switch + { + CheckpointStopRoute.Promoting => + $"{openFailed} The resume checkpoint at '{checkpointPath}' records " + + $"{checkpoint.ItemsCollected} items into the temp '{checkpoint.TempFile}'; " + + $"this run cannot promote them into '{outputPath}'. Going on would delete " + + $"the checkpoint that counts them and replace that file. {closing} {advice}", + + CheckpointStopRoute.FreshAfterVanishedTemp => + $"{openFailed} The resume checkpoint at '{checkpointPath}' recorded " + + $"{checkpoint.ItemsCollected} items into a temp that is gone; this run " + + $"cannot export a fresh copy into '{outputPath}' either. Going on would " + + $"delete the checkpoint that counts them and replace that file. {closing} " + + $"{advice}", + + _ => + $"{openFailed} The resume checkpoint at '{checkpointPath}' records " + + $"{checkpoint.ItemsCollected} items into it, so this run can neither " + + "resume into it nor say whether those items are still there. Going on " + + "would delete the checkpoint that counts them and replace that file. " + + $"{closing} {advice}", + }; + } + + return (refusal.File, refusal.Held) switch + { + (CheckpointFile.NamedTemp, true) => + "Another export is still writing the temp file the resume checkpoint at " + + $"'{checkpointPath}' names, so the {checkpoint.ItemsCollected} items it records " + + $"are that run's. {goingOn} {closing} Wait for that export to finish, or give " + + "this one its own -OutputFile and -CheckpointPath.", + + (CheckpointFile.NamedTemp, false) => + $"The temp file '{checkpoint.TempFile}' that the resume checkpoint at " + + $"'{checkpointPath}' names cannot be opened for writing by this account - the " + + "open was refused on permissions, not on sharing - so the " + + $"{checkpoint.ItemsCollected} items it records cannot be recovered here. " + + $"{goingOn} {closing} Grant write access to that file and run again to recover " + + "them, or remove it and the checkpoint to export afresh.", + + (CheckpointFile.Output, true) => refusal.Route switch + { + CheckpointStopRoute.Promoting => + $"Another export is still writing '{outputPath}'. The resume checkpoint at " + + $"'{checkpointPath}' records {checkpoint.ItemsCollected} items into the " + + $"temp '{checkpoint.TempFile}'; going on would replace '{outputPath}' with " + + "them under the run that holds it, and save this run's position over the " + + $"checkpoint that counts those items. {closing} Wait for that export to " + + "finish, or give this one its own -OutputFile and -CheckpointPath.", + + CheckpointStopRoute.FreshAfterVanishedTemp => + $"Another export is still writing '{outputPath}'. The resume checkpoint at " + + $"'{checkpointPath}' recorded {checkpoint.ItemsCollected} items into a " + + "temp that is gone; going on would export from the beginning and replace " + + $"'{outputPath}' under the run that holds it, and save this run's position " + + $"over the checkpoint that counts those items. {closing} Wait for that " + + "export to finish, or give this one its own -OutputFile and " + + "-CheckpointPath.", + + _ => + $"Another export is still writing '{outputPath}', which the resume " + + $"checkpoint at '{checkpointPath}' records {checkpoint.ItemsCollected} " + + "items into. Going on would cut that file back under it and save this " + + "run's position over the checkpoint that counts those items. " + + $"{closing} Wait for that export to finish, or give this one its own " + + "-OutputFile and -CheckpointPath.", + }, + + _ => UnopenableOutputSentence(), + }; + } + + /// + /// The record the run ends on, which is that sentence in the present tense under an id and a + /// category naming which file the claim failed on and which of the two ways it failed. /// - private static string HeldTempWarning(string checkpointPath, string outputPath, long items) => - $"Another export is still writing the temp file the resume checkpoint at '{checkpointPath}' " - + $"names, so the {items} items it records are that run's and are not recovered here. Both " - + $"files are left as they are and '{outputPath}' is exported from the beginning. Two " - + "exports writing one -OutputFile replace each other's result; give each its own."; + private static ErrorRecord CheckpointFileStop(CheckpointFileRefusal refusal, + string checkpointPath, string outputPath, PaginationCheckpoint checkpoint) => + new(new InvalidOperationException(CheckpointFileStopMessage(refusal, checkpointPath, + outputPath, checkpoint, RunStopsHere)), + (refusal.File, refusal.Held) switch + { + (CheckpointFile.NamedTemp, true) => "CheckpointTempHeld", + (CheckpointFile.NamedTemp, false) => "CheckpointTempUnopenable", + (CheckpointFile.Output, true) => "CheckpointOutputHeld", + _ => "CheckpointOutputUnopenable", + }, + // ResourceBusy for a file another run has, which is what a caller waits out. Past + // that the category follows the open's own reason rather than assuming a + // permission: PermissionDenied is a mode this account can be granted, and a symlink + // loop, an over-long path, a socket, a FIFO or a directory at the path is none - + // InvalidOperation says so, and a refusal that carries no reason at all is the + // temp's, whose own sentence names permissions. + refusal.Held + ? ErrorCategory.ResourceBusy + : refusal.Reason is { IsPermission: false } + ? ErrorCategory.InvalidOperation + : ErrorCategory.PermissionDenied, + checkpointPath); + + /// + /// How the run ends when the copy a promotion stages beside the output could not be written, + /// or could not be moved onto the output once it was. Nothing here is a file that has gone + /// missing, and reading it as one was what deleted the checkpoint: every item it counts is in + /// the temp it names, whole and claimable, and the run that comes back stages the same bytes + /// over the same output. So the sentence says where those items are, what failed and where, + /// and that both files are as they were found - and the run stops rather than export past a + /// position it would then save over. + /// + /// The temp those items are in. The checkpoint's own record of it + /// where it has one, and the file adoption picked where the checkpoint predates that field: + /// either way it is the file the reader has to keep, and a sentence that read it off the + /// checkpoint named nothing at all on the adoption route. + /// What the sweep took off the staging name on the way here, where + /// it took anything: the run deleted an entry and then stopped, and a reader who is told only + /// that the staging failed goes looking for a file that is no longer there. Named in the + /// stop rather than warned about separately, because a warning written before a terminating + /// error is what -WarningAction Stop ends the run on instead of this sentence. + private static string CheckpointStagingStopMessage(StagingFailure staging, + string checkpointPath, string outputPath, PaginationCheckpoint checkpoint, string tempName, + string? removedKind, string closing) => + $"The {checkpoint.ItemsCollected} items the resume checkpoint at '{checkpointPath}' " + + $"records are in '{tempName}', whole, but staging them into '{outputPath}' " + + $"failed at '{staging.Path}': {staging.Reason}" + + (removedKind != null + ? $", after removing what stood at the staging name: {removedKind}" + : "") + + $". {closing} fix that and run again to " + + "recover them, or remove the temp and the checkpoint to export afresh."; + + /// + /// + /// The record the run ends on. A write that did not happen, under the checkpoint whose items + /// it was for: not a claim refused on either of the two files, which is what the ids and + /// categories beside this one are about. + /// + private static ErrorRecord CheckpointStagingStop(StagingFailure staging, string checkpointPath, + string outputPath, PaginationCheckpoint checkpoint, string tempName, + string? removedKind) => + new(new InvalidOperationException(CheckpointStagingStopMessage(staging, checkpointPath, + outputPath, checkpoint, tempName, removedKind, NothingWasChanged)), + "CheckpointStagingFailed", ErrorCategory.WriteError, checkpointPath); /// /// The form of this export's URL a checkpoint on disk was written under - whether the @@ -741,7 +1130,7 @@ private static string HeldTempWarning(string checkpointPath, string outputPath, /// this run would have built first. /// private (bool IncludeCount, bool SuppressTop)? AttemptFormOf( - PaginationCheckpoint? checkpoint, string outputPath, bool countAutoAdded) + PaginationCheckpoint? checkpoint, string outputPath, bool countAutoAdded, bool mayWrite) { if (checkpoint == null) return null; bool[] counts = countAutoAdded ? [true, false] : [false]; @@ -750,7 +1139,7 @@ private static string HeldTempWarning(string checkpointPath, string outputPath, { foreach (var suppressTop in tops) { - if (DescribesThisExport(checkpoint, outputPath, includeCount, suppressTop)) + if (DescribesThisExport(checkpoint, outputPath, includeCount, suppressTop, mayWrite)) return (includeCount, suppressTop); } } @@ -772,16 +1161,64 @@ private static string HeldTempWarning(string checkpointPath, string outputPath, /// promoting and trimming instead of before it, which is not a comparison that can refuse. /// private bool DescribesThisExport(PaginationCheckpoint checkpoint, string outputPath, - bool includeCount, bool suppressTop) - => OwnershipOf(checkpoint, outputPath, includeCount, suppressTop) == CheckpointOwnership.Mine; + bool includeCount, bool suppressTop, bool mayWrite) + => OwnershipOf(checkpoint, outputPath, includeCount, suppressTop, mayWrite) + == CheckpointOwnership.Mine; + + /// + /// Which reading refused the checkpoint this run found at -CheckpointPath, or null where + /// nothing refused it. Set wherever a refusal that leaves the file standing is reported and + /// never cleared - an attempt rebuilds the request URL, so a checkpoint refused under one + /// form of it can compare equal under another, and that is not evidence of anything. + /// + /// Read on the way out of a run that completed, where the delete would otherwise take a + /// position this run had warned it would leave alone. One of the three refusals records a + /// reading here: the ownership verdict, where whatever wrote the file resumes from exactly + /// it. A checkpoint refused because the endpoint made this run rebuild the request records + /// none - it is a position into an enumeration this run replaces and nothing else, so it + /// goes with the completed export and the temp it names is this run's own to sweep. And a + /// checkpoint whose temp, or whose output, this run could not claim never reaches a door: + /// that run stops at the reconcile, with nothing enumerated and nothing written. + /// + /// Not something can answer either: a checkpoint + /// written by a run that was appending to its output names no temp at all, and refusing + /// that one counts the same. + /// + private SparedCheckpoint? _sparedCheckpoint; /// /// The temp file a checkpoint this run refused still points at, or null. Refusing is the /// whole response: the checkpoint is left where it is because the export that wrote it /// resumes from exactly that, and the items it counted are in the temp it names. + /// + /// Null is two readings, and tells them apart: nothing was + /// refused, or what was refused records no temp name. The second stands for a temp all the + /// same - a checkpoint written before the name was recorded says only that the items are + /// somewhere beside this output - so the sweep reads the pair rather than this alone. + /// /// private string? _refusedCheckpointTemp; + /// + /// What a run would stop on - a temp it could not claim, or the output it would append to - + /// worked out by the preview pass above the ShouldProcess gate, or null where nothing would + /// stop it. The gate names one of two actions on -OutputFile and "nothing at all" is + /// neither, so the sentence goes out beside it - and only on the -WhatIf side, where a real + /// run has already ended on the same condition by the time the gate is past. + /// + private string? _wouldStopOnCheckpointFile; + + /// + /// What a run would take off the staging name before it staged its copy there, worked out by + /// the preview pass and reported beside the gate; null where the name is clear. The pass + /// used to clear the name itself, which made its deletions as real as a run's: a caller who + /// ran -WhatIf to find out what would be touched had the link, the pipe or the leftover copy + /// at that name deleted in order to be told that nothing would be. It is named now and left + /// where it is, and the create test that follows a clear name is skipped for it - nothing + /// standing at the name can be tested without taking it off. + /// + private string? _wouldRemoveAtStagingName; + /// /// The temp a failed attempt of this run left on disk for a checkpoint of its own, or null. /// The attempt loop comes back round on a query the endpoint refused, and the sweep it @@ -812,9 +1249,11 @@ private bool DescribesThisExport(PaginationCheckpoint checkpoint, string outputP private static bool RefusedTempIsOnDisk(string? tempFile, string outputPath) { if (tempFile == null || tempFile != Path.GetFileName(tempFile)) return false; - if (!IsRunTempName(Path.GetFileName(outputPath), tempFile)) return false; var dir = Path.GetDirectoryName(outputPath); - return !string.IsNullOrEmpty(dir) && File.Exists(Path.Combine(dir, tempFile)); + if (string.IsNullOrEmpty(dir)) return false; + if (!IsRunTempName(dir, Path.GetFileName(outputPath), tempFile, mayWrite: true)) + return false; + return File.Exists(Path.Combine(dir, tempFile)); } /// Whose the checkpoint on disk is, and where it is not this export's, why not. @@ -845,11 +1284,11 @@ private enum CheckpointOwnership /// and is refused with the enumerations that do not match. /// private CheckpointOwnership OwnershipOf(PaginationCheckpoint checkpoint, string outputPath, - bool includeCount, bool suppressTop) + bool includeCount, bool suppressTop, bool mayWrite) { - if ((checkpoint.OutputFile != null || checkpoint.DataLength != null) + if (RecordsAFileToWeigh(checkpoint) && !RecordedOutputMatches(checkpoint.OutputFile, checkpoint.TempFile, - checkpoint.DataLength, outputPath)) + checkpoint.DataLength, outputPath, mayWrite)) { return checkpoint.OutputFile != null ? CheckpointOwnership.AnotherExports @@ -867,7 +1306,9 @@ private CheckpointOwnership OwnershipOf(PaginationCheckpoint checkpoint, string /// whether the run that follows appends to the output rather than exporting into a fresh /// temp. With false nothing is written, deleted or warned about: /// the ShouldProcess gate has to name the action before the run is allowed to take any, and - /// -WhatIf has to leave every file it found exactly as it found it. + /// -WhatIf has to leave every file it found exactly as it found it - and leave the directory + /// holding them as it found it too, which is why the flag also decides whether the case rule + /// of that directory may be probed or has to be assumed. /// /// A checkpoint records which file its items were written to and how many bytes of that /// file they occupy, which makes three cases decidable instead of guessed. @@ -914,12 +1355,14 @@ private bool ReconcileCheckpoint(string? checkpointPath, string outputPath, return false; } - var ownership = OwnershipOf(checkpoint, outputPath, includeCount, suppressTop); + var ownership = OwnershipOf(checkpoint, outputPath, includeCount, suppressTop, apply); if (ownership != CheckpointOwnership.Mine) { if (apply) { - // What the refusal leaves has to survive the sweep in this same run. + // What the refusal leaves has to survive this same run - the sweep a few + // lines on, and the delete the completion path ends with. + _sparedCheckpoint = SparedCheckpoint.NotThisRuns; _refusedCheckpointTemp = checkpoint.TempFile; WriteWarning(ownership == CheckpointOwnership.AnotherExports ? ForeignCheckpointWarning(checkpointPath, outputPath) @@ -947,12 +1390,60 @@ private bool ReconcileCheckpoint(string? checkpointPath, string outputPath, // output it could be merged into - exactly the case this path used to be limited to. if (!File.Exists(outputPath)) { - if (apply - ? TryAdoptOrphanedTemp(outputPath, checkpoint.ItemsCollected) - : CanAdoptOrphanedTemp(outputPath, checkpoint.ItemsCollected)) + string? adoptedTemp; + StagingFailure? adoptStaging; + StagingRemoval? adoptSwept; + var adopted = apply + ? TryAdoptOrphanedTemp(outputPath, checkpoint.ItemsCollected, + out adoptedTemp, out adoptStaging, out adoptSwept) + : CanAdoptOrphanedTemp(outputPath, checkpoint.ItemsCollected, + out adoptedTemp, out adoptStaging, out adoptSwept); + + // What the adoption took off its own staging name before it created one there, + // or - on the preview, which takes nothing off it - what a run would. The + // applying pass records the deletion here, where it happened, and warns about it + // below once this run's own outcome for the recovery is known: a warning is a + // terminating error under -WarningAction Stop, so written here it ended the run + // with the entry deleted, the items still in the temp and nothing recovered. + if (adoptSwept is { } adoptRemoval) + { + if (apply) WriteVerbose(adoptRemoval.Sentence); + else _wouldRemoveAtStagingName = adoptRemoval.Sentence; + } + + // The copy could not be staged beside the output. Every item the checkpoint + // counts is in the temp this names, whole, so this is not the route below for a + // checkpoint whose output has gone: that one deletes the position counting them + // and exports afresh, and reached from here it did so with the temp holding + // every one. + if (adoptStaging is { } adoptFailure) + { + if (apply) + { + ThrowTerminatingError(CheckpointStagingStop(adoptFailure, checkpointPath, + outputPath, checkpoint, adoptedTemp!, adoptSwept?.Kind)); + return false; + } + + // The preview pass reaches the same name and may not throw. What a real run + // would do there is stop, which is not one of the two actions the gate names. + _wouldStopOnCheckpointFile = CheckpointStagingStopMessage(adoptFailure, + checkpointPath, outputPath, checkpoint, adoptedTemp!, null, + NothingWouldBeChanged); + return false; + } + + if (adopted) { if (apply) + { WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted export's temp file. Resuming from checkpoint."); + // And what the sweep took off the staging name, now that what this run + // did about the checkpoint is on the stream in front of it. Under + // -WarningAction Stop the run ends here, with the items recovered into + // the output and the position they resume from still on disk. + if (adoptSwept is { } removed) WriteWarning(removed.Sentence); + } return true; } @@ -960,6 +1451,7 @@ private bool ReconcileCheckpoint(string? checkpointPath, string outputPath, { WriteWarning("Checkpoint found but output file is missing. Deleting stale checkpoint and starting fresh."); PaginationCheckpoint.Delete(checkpointPath); + if (adoptSwept is { } removedAnyway) WriteWarning(removedAnyway.Sentence); } return false; } @@ -984,24 +1476,101 @@ private bool ReconcileCheckpoint(string? checkpointPath, string outputPath, if (checkpoint.TempFile != null) { - if (apply - ? TryPromoteNamedTemp(outputPath, checkpoint.TempFile, dataLength) - : CanPromoteNamedTemp(outputPath, checkpoint.TempFile, dataLength)) + TempPromotion promotion; + UnopenableReason? promotionReason; + StagingFailure? promotionStaging = null; + StagingRemoval? promotionSwept; + if (apply) + promotion = TryPromoteNamedTemp(outputPath, checkpoint.TempFile, dataLength, + out promotionReason, out promotionStaging, out promotionSwept); + else + promotion = CanPromoteNamedTemp(outputPath, checkpoint.TempFile, dataLength, + out promotionReason, out promotionStaging, out promotionSwept); + + // What the promotion took off its own staging name before it created one there, or - + // on the preview, which takes nothing off it - what a run would. The applying pass + // records the deletion here, where it happened, and warns about it below once this + // run's own outcome for the recovery is known: a warning is a terminating error + // under -WarningAction Stop, so written here it ended the run with the entry + // deleted, the items still in the temp and nothing recovered. + if (promotionSwept is { } promotionRemoval) + { + if (apply) WriteVerbose(promotionRemoval.Sentence); + else _wouldRemoveAtStagingName = promotionRemoval.Sentence; + } + + // The copy could not be staged, or could not be moved once it was. Both files are + // where they were found and every item the checkpoint counts is still in the temp it + // names, so this is not the route below for a temp whose items are in no file: that + // one deletes the position counting them and exports afresh over the output, and + // reached from here it did so with the temp holding all of them. + if (promotion == TempPromotion.StagingFailed) { if (apply) { - WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted export's temp file. Resuming from checkpoint."); - // Those items are the output now. Repoint the checkpoint at it before + ThrowTerminatingError(CheckpointStagingStop(promotionStaging!.Value, + checkpointPath, outputPath, checkpoint, checkpoint.TempFile, + promotionSwept?.Kind)); + return false; + } + + // The preview pass reaches the same name and may not throw. What a real run + // would do there is stop, which is not one of the two actions the gate names. + _wouldStopOnCheckpointFile = CheckpointStagingStopMessage(promotionStaging!.Value, + checkpointPath, outputPath, checkpoint, checkpoint.TempFile, null, + NothingWouldBeChanged); + return false; + } + + // The output the promotion would land in is another run's, or is not a file this + // account can write at the recorded offset. The same stop the branch below reaches, + // about the same file for the same reason: the run that has the output is appending + // to it, and this one would be replacing it under them. + // + // Asked for before the move, so nothing was replaced - and the temp, with the + // checkpoint still naming it, is exactly as it was found, since the temp is unlinked + // only where the output was kept. So the position and the file it counts are the + // pair they were, and the run that comes back promotes the same rows to the same + // length over the same output. + if (promotion is TempPromotion.OutputHeld or TempPromotion.OutputUnopenable) + { + var promotedOutputRefusal = new CheckpointFileRefusal(CheckpointFile.Output, + promotion == TempPromotion.OutputHeld + ? ClaimRefusal.AnotherRunHasIt + : ClaimRefusal.CannotBeOpened, + CheckpointStopRoute.Promoting, promotionReason); + if (apply) + { + ThrowTerminatingError(CheckpointFileStop(promotedOutputRefusal, checkpointPath, + outputPath, checkpoint)); + return false; + } + + // The preview pass asks the same question of the same file and may not throw: + // it reports what a run would do and does none of it. What it would do is stop, + // which is not one of the two actions the ShouldProcess gate names, so the gate + // writes this sentence beside the one it does name. + _wouldStopOnCheckpointFile = CheckpointFileStopMessage(promotedOutputRefusal, + checkpointPath, outputPath, checkpoint, RunWouldStopHere); + return false; + } + + if (promotion == TempPromotion.Taken) + { + if (apply) + { + // Those items are the output now, and the output is this run's - the + // promotion kept it through the move. Repoint the checkpoint at it before // anything else can fail, so a second interruption cannot promote the same - // temp again. - checkpoint.TempFile = null; - checkpoint.OutputFile = Path.GetFullPath(outputPath); - checkpoint.DataLength = new FileInfo(outputPath).Length; - try { checkpoint.Save(checkpointPath); } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) - { - WriteWarning($"Checkpoint save failed after recovery: {ex.Message}"); - } + // temp again - and before the warning below, which under -WarningAction Stop + // is where the run ends: the position on disk has to be the one that + // resumes from what is now in the output. + RepointCheckpointAtHeldOutput(checkpoint, checkpointPath, outputPath, + HeldOutput!); + WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted export's temp file. Resuming from checkpoint."); + // And what the sweep took off the staging name, now that what this run did + // about the checkpoint is on the stream in front of it. + if (promotionSwept is { } removed) WriteWarning(removed.Sentence); } return true; } @@ -1012,16 +1581,79 @@ private bool ReconcileCheckpoint(string? checkpointPath, string outputPath, // the position it comes back to, and this run exports from the beginning into a temp // of its own. Deleting the checkpoint below would be the same mistake as unlinking // the temp, one file over. - if (NamedTempIsHeld(outputPath, checkpoint.TempFile, dataLength)) + // + // Or the claim failed because this account cannot open the file read-write, which + // leaves both files in the same place and has no second export behind it. The + // refusal is the same; only what it can honestly say about the cause differs. + var namedTempRefusal = new CheckpointFileRefusal(CheckpointFile.NamedTemp, + RefusalForNamedTemp(outputPath, checkpoint.TempFile, dataLength, apply)); + if (namedTempRefusal.Refuses) + { + // The run ends here, before the client is asked for, before the sweep a few + // lines on, before a temp of this run's own exists and before anything is + // written to -OutputFile or -CheckpointPath. + // + // Leaving the two files alone and exporting anyway was not enough, because + // exporting IS what takes them: the first page boundary saves this run's own + // position over the same path, the completion door then deletes what it finds + // there as its own, and the holder's temp is an orphan the next sweep takes + // with the rows still in it. The property the caller wants - this run proceeds + // and the other run's work survives - is not available at any price, since + // proceeding means replacing the output the checkpoint records and saving over + // the checkpoint itself. So this run does not proceed. + // + // The rows it is protecting are in that temp and in no other file, and the + // checkpoint is the only thing on disk that counts them: a holder still running + // deletes it itself on the way out, a holder that dies keeps its temp for + // exactly as long as a checkpoint counting it is there, and where the claim + // failed on permissions instead it is the position write access has to be + // granted for. + if (apply) + { + ThrowTerminatingError(CheckpointFileStop(namedTempRefusal, checkpointPath, + outputPath, checkpoint)); + return false; + } + + // The preview pass reaches the same two files and may not throw: -WhatIf reports + // what a run would do and does none of it. What it would do is stop, which is + // not one of the two actions the ShouldProcess gate below names, so the gate + // writes this sentence beside the one it does name. + _wouldStopOnCheckpointFile = CheckpointFileStopMessage(namedTempRefusal, + checkpointPath, outputPath, checkpoint, RunWouldStopHere); + return false; + } + + // The temp is not there, or is shorter than the length recorded for it - and the + // first of those is what a promotion leaves for the instant between its unlink and + // the save that repoints the position at the output. A second run over the same + // command line, released with the first, read that instant and said those items were + // not on disk, deleted the position counting them, exported from the beginning and + // moved its own temp over the output at the end - which was the promoting run's + // output, held and being appended to. So the output is asked about before any of + // that, and a promoter holding it ends this run where the branch below ends it. + // + // Only the refusals are read. An output nothing holds is not evidence that a + // promotion landed in it, whatever it measures: this checkpoint records its items + // into the temp, and a previous export's file standing at the path measures exactly + // what a promotion would have left about as often as a promoted one does - two rows + // of one shape being two rows of another shape's length. So that case takes the route + // it always took, which replaces the whole file rather than cutting it back to an + // offset counted in a different one. + var vanishedTempRefusal = new CheckpointFileRefusal(CheckpointFile.Output, + RefusalForStandingOutput(outputPath, out var vanishedTempReason), + CheckpointStopRoute.FreshAfterVanishedTemp, vanishedTempReason); + if (vanishedTempRefusal.Refuses) { if (apply) { - // What the refusal leaves has to survive the sweep in this same run: the - // holder can be gone by the time the sweep runs, and the checkpoint left - // standing over that temp is what makes it recoverable rather than stale. - _refusedCheckpointTemp = checkpoint.TempFile; - WriteWarning(HeldTempWarning(checkpointPath, outputPath, checkpoint.ItemsCollected)); + ThrowTerminatingError(CheckpointFileStop(vanishedTempRefusal, checkpointPath, + outputPath, checkpoint)); + return false; } + + _wouldStopOnCheckpointFile = CheckpointFileStopMessage(vanishedTempRefusal, + checkpointPath, outputPath, checkpoint, RunWouldStopHere); return false; } @@ -1031,15 +1663,54 @@ private bool ReconcileCheckpoint(string? checkpointPath, string outputPath, $"The interrupted export's temp file is missing or incomplete, so the {checkpoint.ItemsCollected} items it " + "recorded are not on disk. Exporting again from the beginning; nothing is lost."); PaginationCheckpoint.Delete(checkpointPath); + // And what the sweep took off the staging name, after what this run did about + // the checkpoint. + if (promotionSwept is { } removedAnyway) WriteWarning(removedAnyway.Sentence); } return false; } - if (apply - ? TryTrimOutputToCheckpoint(outputPath, dataLength) - : CanTrimOutputToCheckpoint(outputPath, dataLength)) - return true; + // The checkpoint names no temp, so the run that wrote it was appending straight into the + // output: the items it counts are in that file, and cutting it back to the recorded + // length is what stops the ones written after the last save from being written twice. + // + // Which is the same claim on the output that the branch above makes on a temp, and for + // the same reason. A run that has resumed once holds -OutputFile open with the rows in + // it, and its checkpoint names no temp for a second run over the same command line to + // refuse it by - so that run judged the checkpoint its own, cut the live file back to + // its own offset and appended, and the holder's next write landed past the hole. + // + // So the claim is not let go. The handle it opens is the handle this run writes through: + // the cut below is a SetLength on it, the writer downstream appends onto it, and it is + // released where the writing ends. Asked for and released here, a page-fetch before the + // writer took an open of its own, it left a window a second run walked straight through + // - both runs past the same claim, both cutting, both appending. + CheckpointOutput taken; + UnopenableReason? takenReason; + if (apply) taken = TakeOutputForCheckpoint(outputPath, dataLength, out takenReason); + else taken = WouldTakeOutputForCheckpoint(outputPath, dataLength, out takenReason); + if (taken == CheckpointOutput.Taken) return true; + + var outputRefusal = new CheckpointFileRefusal(CheckpointFile.Output, RefusalFrom(taken), + CheckpointStopRoute.Appending, takenReason); + if (outputRefusal.Refuses) + { + // Where the temp branch ends the run, before the client, the sweep and the first + // page - and on the preview pass, where it says the same thing in the future tense. + if (apply) + { + ThrowTerminatingError(CheckpointFileStop(outputRefusal, checkpointPath, + outputPath, checkpoint)); + return false; + } + _wouldStopOnCheckpointFile = CheckpointFileStopMessage(outputRefusal, checkpointPath, + outputPath, checkpoint, RunWouldStopHere); + return false; + } + + // Not the file the checkpoint describes - absent, or shorter than the length recorded - + // so the items it counts are in no file and there is nothing to append to. if (apply) { WriteWarning( @@ -1047,6 +1718,7 @@ private bool ReconcileCheckpoint(string? checkpointPath, string outputPath, + "Exporting again from the beginning; nothing is lost."); PaginationCheckpoint.Delete(checkpointPath); } + return false; } } diff --git a/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs b/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs index 44d3622..1098509 100644 --- a/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs +++ b/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs @@ -126,6 +126,18 @@ public class InvokeMgxRequest : MgxCmdletBase private readonly List _pipelineIds = []; private bool _isFanOut; + /// + /// The checkpoint gate's answer for this invocation, asked once and reused. -CheckpointPath, + /// -Confirm and -WhatIf are all bound once for the whole run, but a -Uri without a {id} + /// placeholder is not a fan-out: ProcessRecord runs the direct branch below once per piped + /// item, and each of those calls reaches ExecuteList's checkpoint gate on its own. Without + /// this, three piped values asked ShouldProcess three times over the same file - three + /// prompts, and three "What if" lines under -WhatIf, for one run. Set on whichever item + /// reaches the gate first; every later item on that route reuses the answer instead of + /// asking again, a refusal included. + /// + private bool? _mayWriteCheckpoint; + /// /// Full base URL including API version (e.g., "https://graph.microsoft.com/v1.0"). /// @@ -141,6 +153,10 @@ public class InvokeMgxRequest : MgxCmdletBase protected override void BeginProcessing() { + // A fresh cmdlet instance answers this fresh each run; reset explicitly rather than + // relying on that, so the field's own comment is the one place its lifetime is stated. + _mayWriteCheckpoint = null; + // Reject absolute URLs (relative paths only); concatenation onto the versioned // base URL would otherwise silently produce /v1.0/https:/... on the wire. if (Uri.TrimStart().StartsWith("https://", StringComparison.OrdinalIgnoreCase) || @@ -309,6 +325,33 @@ private void ExecuteList(string relativeUri, string? sourceId) ? GetUnresolvedProviderPathFromPSPath(CheckpointPath) : null; + // -CheckpointPath is the one file this read path writes, and it is the caller's own + // resume position: every page boundary saves the next link over it, completion deletes + // it, and a checkpoint describing another enumeration is deleted where it is found. + // Reads go out under -WhatIf, by the convention above ExecuteWrite, but a preview that + // overwrites or removes the position a later run resumes from has changed the thing it + // claimed only to describe. So the file gets a gate of its own and the reads keep the + // convention: a refusal skips the saves and the deletes, and the enumeration goes on to + // page and emit. + // + // One decision, taken here rather than at each site. The page-boundary callback runs on + // whichever thread the iterator resumes on - a page that yields nothing resumes on the + // thread pool - and a cmdlet API call from there throws. -Confirm therefore asks once + // per run about one file, which is the question it has to answer anyway - and "once per + // run" has to hold across ProcessRecord calls too, not just within this one: a -Uri with + // no {id} placeholder is not a fan-out, so several piped values still reach ExecuteList + // once each through the direct branch in ProcessRecord, each with its own attempt loop + // and its own call here. _mayWriteCheckpoint (reset once in BeginProcessing) is what + // makes the second and later calls consult the first item's answer instead of asking + // ShouldProcess again - a refusal on the first item refuses the rest. + // + // Reading the checkpoint is not gated: resuming is what the run being previewed would + // do, and a preview that ignored it would page from the start and describe a different + // set of requests. The branches that refuse a checkpoint leave resume null and enumerate + // from the beginning either way - the delete is all the gate withholds. + var mayWriteCheckpoint = cpPath == null + || (_mayWriteCheckpoint ??= ShouldProcess(cpPath, "Save resume checkpoint")); + // If checkpoint was saved during a previous retry (URL without $count=true), // match the checkpoint's URL to avoid mismatch on resume if (countAutoAdded && cpPath != null) @@ -326,7 +369,8 @@ private void ExecuteList(string relativeUri, string? sourceId) var url = BuildCollectionUrl(relativeUri, includeCount: !string.IsNullOrEmpty(CountVariable) || includeAutoCount, noPageSize: suppressTop); - var iterator = new PageIterator(GetClient()); + var client = GetClient(); + var iterator = new PageIterator(client); // -All says how far to page, -Top says how much to return, and they are not the // same question: -All used to zero the cap, so asking for a bounded slice of a // large collection walked all of it. Worse, -Top also sets the page size, so the @@ -345,7 +389,7 @@ private void ExecuteList(string relativeUri, string? sourceId) if (checkpoint.NextLink == null) { // Completion marker: previous run finished - PaginationCheckpoint.Delete(cpPath); + if (mayWriteCheckpoint) PaginationCheckpoint.Delete(cpPath); } else if (string.Equals(checkpoint.Resource, url, StringComparison.Ordinal)) { @@ -361,12 +405,12 @@ private void ExecuteList(string relativeUri, string? sourceId) } else { - PaginationCheckpoint.Delete(cpPath); + if (mayWriteCheckpoint) PaginationCheckpoint.Delete(cpPath); } } else { - PaginationCheckpoint.Delete(cpPath); + if (mayWriteCheckpoint) PaginationCheckpoint.Delete(cpPath); } } } @@ -385,7 +429,7 @@ private void ExecuteList(string relativeUri, string? sourceId) onPageComplete: info => { // Save page-boundary checkpoint - if (cpPath != null && info.NextPageUrl != null) + if (mayWriteCheckpoint && cpPath != null && info.NextPageUrl != null) { try { @@ -398,7 +442,14 @@ private void ExecuteList(string relativeUri, string? sourceId) } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { - WriteWarning($"Checkpoint save failed: {ex.Message}"); + // Buffered rather than written here. This callback runs on + // whichever thread the iterator resumed on, and a page that + // yields nothing resumes on the thread pool - so a WriteWarning + // from here throws PSInvalidOperationException, and the disk + // problem it was reporting ends the enumeration instead of + // being reported. The drains below write it from the pipeline + // thread, on the same channel the client's own warnings take. + client.EnqueueWarning($"Checkpoint save failed: {ex.Message}"); } } }, @@ -429,7 +480,7 @@ private void ExecuteList(string relativeUri, string? sourceId) WriteCountDiscrepancyWarning(relativeUri, reportedODataCount.Value, itemCount, Filter); // Delete checkpoint on successful completion - if (cpPath != null) PaginationCheckpoint.Delete(cpPath); + if (mayWriteCheckpoint && cpPath != null) PaginationCheckpoint.Delete(cpPath); return; // Success, exit the retry loop } catch (PipelineStoppedException) @@ -471,6 +522,16 @@ private void ExecuteList(string relativeUri, string? sourceId) WriteGraphError(ex, relativeUri, ApiVersion); return; } + finally + { + // Every way out of the attempt above - the success path, a retry via continue, + // or any catch - drains here once, from the pipeline thread. The last catch + // already drains inside WriteGraphError before it writes its error record, so + // that exit reaches this a second time; TryDequeue found the queue already + // empty from the first drain, so the second call writes nothing and nothing + // doubles. + DrainClientMessages(); + } } } @@ -513,6 +574,12 @@ private void ExecuteGet(string relativeUri, string? sourceId) private void ExecuteWrite(HttpMethod method, string relativeUri, string? sourceId) { + // The gate sits on the write and on the bulk write below, and on nothing else: reads go + // out under -WhatIf, per the PowerShell convention, so that a dry run under + // $WhatIfPreference has real data for the gated writes to be previewed against. The help + // says so, because "the cmdlet is not run" would not be true of a GET that connects, + // pages, spends resource units and emits objects. Invoke-MgxBatchRequest gates its reads + // too, for the reason stated above its own gate; the two contracts are documented apart. if (!ShouldProcess(relativeUri, method.Method)) return; diff --git a/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj b/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj index 110d517..6e1d11f 100644 --- a/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj +++ b/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj @@ -17,6 +17,17 @@ + + @@ -36,7 +47,7 @@ ./build.ps1 -Configuration Debug, which copies explicitly and does not rely on this. --> - $(MSBuildThisFileDirectory)../../module + $(MSBuildThisFileDirectory)../../module $(ModuleRoot)/Dependencies @@ -49,9 +60,10 @@ - + + diff --git a/src/Mgx.Engine/Http/GraphBatchClient.cs b/src/Mgx.Engine/Http/GraphBatchClient.cs index c73d6a5..eed8d38 100644 --- a/src/Mgx.Engine/Http/GraphBatchClient.cs +++ b/src/Mgx.Engine/Http/GraphBatchClient.cs @@ -154,8 +154,14 @@ public async Task> ExecuteBatchAsync( /// /// Status recorded for the items of a chunk whose own POST failed without a status of its - /// own - a stalled body, an open circuit. The request went out, so those writes may have - /// been applied; NotSentStatus would claim the opposite. + /// own. Two refusals land on it and it cannot separate them: a POST that went out and whose + /// answer never completed - a stalled response body, a transport failure mid-flight - and a + /// POST that never left, refused by an open circuit or by a rate limiter with no permit for + /// it. It is read as the first, because that is the reading a caller can act on safely: the + /// writes may have been applied, so resending them is a decision about duplicates. + /// NotSentStatus is kept for the refusals nothing went out for and the code knows it - a + /// chunk the run's stop reached before its first attempt, and the chunks never POSTed at + /// all - where claiming the opposite would be the more expensive error. /// private const int RefusedWithoutStatus = 503; @@ -210,9 +216,9 @@ public ChunkSendFailedException( /// /// Writes a refused chunk's own slots: an item the server answered keeps that answer, the - /// rest carry the refusal. The POST went out for all of them, so none of them may be - /// reported as never sent - the caller has to be able to tell a write that may have landed - /// from one that certainly did not. + /// rest take the refusal status, covering both a POST that went out unanswered and one an + /// open circuit or a rate limiter stopped before it left - read as the first, so none of + /// them may be reported as never sent. /// private static void FillRefusedChunk( (BatchOperation Operation, GraphBatchResponseItem Response)?[] results, diff --git a/src/Mgx.Engine/Http/GraphRequestTracer.cs b/src/Mgx.Engine/Http/GraphRequestTracer.cs index 386095d..2538c7e 100644 --- a/src/Mgx.Engine/Http/GraphRequestTracer.cs +++ b/src/Mgx.Engine/Http/GraphRequestTracer.cs @@ -24,27 +24,74 @@ internal static partial class GraphRequestTracer private static readonly string[] ResponseHeaderPrefixes = ["RateLimit-", "x-ms-throttle-"]; - [GeneratedRegex("\"([^\"]*(?:password|secret|credential|token|key)[^\"]*)\"\\s*:\\s*\"[^\"]*\"", - RegexOptions.IgnoreCase)] - private static partial Regex SensitiveJsonValue(); + /// + /// Properties whose NAME says credential. The fragments come from SensitiveNames, the same + /// list the dead-letter writer reads, so a clientAssertion or a connectionString is missing + /// from a trace and from the file alike. Not source-generated, because the pattern is that + /// list: written out here it would be a second copy of it, free to drift. + /// + /// Compiled, because the other two patterns are source-generated and this one runs over the + /// same bodies: interpreted, it is the slowest thing in a -Debug trace of a large response. + private static readonly Regex SensitiveJsonValue = new( + $"\"([^\"]*(?:{SensitiveNames.RegexAlternation})[^\"]*)\"\\s*:\\s*\"[^\"]*\"", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled); /// /// A pre-authenticated URL is a credential whose property name says nothing about it: - /// "@microsoft.graph.downloadUrl" contains none of password/secret/credential/token/key, so + /// "@microsoft.graph.downloadUrl" carries none of the SensitiveNames fragments, so /// SensitiveJsonValue leaves it whole and the tempauth JWT reaches -Debug output. Match on /// the VALUE instead - a URL carrying one of the capability parameters Graph, SharePoint and - /// the download hosts actually use. Deliberately not "every URL": @odata.nextLink is a URL - /// too, and losing it from a trace would remove the thing paging bugs are diagnosed with. + /// the download hosts actually use. Those parameters are SensitiveNames', the definition the + /// dead-letter writer reads too, so a SAS URL missing from a trace is missing from the file; + /// that class says why the list is not simply every URL. /// - [GeneratedRegex("\"(https?://[^\"]*?(?:tempauth|guestaccesstoken|authkey|X-Amz-Signature|(? + /// A capability URL used as a property NAME, which takes the value beside it with it. A + /// pre-authenticated URL as a field name is itself a credential, so whatever a body files + /// under one is the thing that URL was minted for, and the dead-letter writer clears it. + /// CapabilityUrlValue() below finds such a key too - a quoted string is a quoted string, + /// key or value - but it stops at that string's closing quote and leaves the value standing. + /// The colon is what says the string was a key: a JSON value is followed by a comma or a + /// closing brace or bracket, never by one. + /// + /// That trailing `"[^"]*"` only reaches the value when it is itself a quoted string. The + /// dead-letter writer clears the value beside a cut key whatever it holds, so a number, a + /// bool, an array or an object beside a capability-URL key is missing from the dead-letter + /// line and still stands whole in -Debug output - this pattern has nothing to match there. + /// + [GeneratedRegex($"\"(https?://[^\"]*?(?:{SensitiveNames.CapabilityUrlParameters})=)[^\"]*\"\\s*:\\s*\"[^\"]*\"", + RegexOptions.IgnoreCase)] + private static partial Regex CapabilityUrlProperty(); + + /// + /// The value rule again on a URL that has lost its scheme and host. A $batch envelope + /// carries each item's url relative - Invoke-MgxBatchRequest normalizes it before the POST, + /// which is what Graph's batch format takes - so a pre-authenticated URL a caller passed as + /// an item URL reaches a trace as "/x?tempauth=..." and the two patterns above, anchored on + /// the scheme, have nothing in it to match. The leading slash stands in for the scheme: it + /// says the quoted string is a path rather than prose that happens to carry a parameter. + /// + /// A value only, with no property counterpart. The dead-letter writer reads a body's keys + /// and values through SensitiveNames.RedactCapabilityUrl, which reads the same two forms, so + /// a relative path is cut on both sides: in the file, where a value written whole is a live + /// credential on disk, and here. Where the two are read side by side is the item URL, which + /// is absolute in the file where a caller gave it absolute, relative on the wire, and cut + /// either way. + /// + [GeneratedRegex($"\"(/[^\"]*?(?:{SensitiveNames.CapabilityUrlParameters})=)[^\"]*\"", + RegexOptions.IgnoreCase)] + private static partial Regex RelativeCapabilityUrlValue(); + /// /// Property names that carry a pre-authenticated URL even when the value does not expose a - /// recognisable parameter - the short-form download hosts put the capability in the path. + /// recognizable parameter - the short-form download hosts put the capability in the path. + /// The fragment is SensitiveNames', which the writer reads one property name at a time. /// - [GeneratedRegex("\"([^\"]*downloadurl[^\"]*)\"\\s*:\\s*\"[^\"]*\"", + [GeneratedRegex($"\"([^\"]*{SensitiveNames.DownloadUrlFragment}[^\"]*)\"\\s*:\\s*\"[^\"]*\"", RegexOptions.IgnoreCase)] private static partial Regex DownloadUrlProperty(); @@ -134,12 +181,18 @@ private static string RedactHeaderValue(string name, string value) /// Redact credential-looking JSON properties and pre-authenticated URLs, then truncate. private static string Sanitize(string body) { - var redacted = SensitiveJsonValue().Replace(body, "\"$1\": \"\""); + var redacted = SensitiveJsonValue.Replace(body, "\"$1\": \"\""); // Property-name match first: covers a downloadUrl whose capability sits in the path. redacted = DownloadUrlProperty().Replace(redacted, "\"$1\": \"\""); + // A capability URL used as a key, and the value beside it, before the pass below cuts + // such a key on its own and leaves that value standing. + redacted = CapabilityUrlProperty().Replace(redacted, "\"$1\": \"\""); // Then any remaining URL value carrying a capability parameter, keeping the parameter // name visible so a trace still shows WHICH kind of URL was redacted. redacted = CapabilityUrlValue().Replace(redacted, "\"$1\""); + // And the same value with its scheme and host gone, which is the form a $batch item's + // own url has by the time it is on the wire. + redacted = RelativeCapabilityUrlValue().Replace(redacted, "\"$1\""); return redacted.Length <= MaxBodyChars ? redacted : redacted[..MaxBodyChars] + $"... [truncated, {redacted.Length - MaxBodyChars} more chars]"; diff --git a/src/Mgx.Engine/Http/ResilientGraphClient.cs b/src/Mgx.Engine/Http/ResilientGraphClient.cs index c2682c1..6e5ef23 100644 --- a/src/Mgx.Engine/Http/ResilientGraphClient.cs +++ b/src/Mgx.Engine/Http/ResilientGraphClient.cs @@ -132,9 +132,11 @@ public void DrainVerboseMessages() VerboseWriter(msg); } - // For engine components that already report through this client's buffered channel - // (PageIterator runs on the enumeration thread, where a cmdlet cannot WriteWarning). - internal void EnqueueWarning(string message) => _pendingWarnings.Enqueue(message); + // For anything that reports from the enumeration thread, where a cmdlet cannot + // WriteWarning: PageIterator itself, and the page-boundary callbacks the iterator invokes + // on whichever thread it resumed on. Public for the latter - they live in the cmdlets + // assembly - and paired with DrainWarningMessages, which is where the message is written. + public void EnqueueWarning(string message) => _pendingWarnings.Enqueue(message); // Same threading contract as DrainVerboseMessages. public void DrainWarningMessages() diff --git a/src/Mgx.Engine/Http/ResilientGraphClientOptions.cs b/src/Mgx.Engine/Http/ResilientGraphClientOptions.cs index e3d42b0..4fe71a0 100644 --- a/src/Mgx.Engine/Http/ResilientGraphClientOptions.cs +++ b/src/Mgx.Engine/Http/ResilientGraphClientOptions.cs @@ -126,13 +126,18 @@ public double CircuitBreakerFailureRatio : throw new ArgumentOutOfRangeException(nameof(CircuitBreakerFailureRatio), value, "Must be between 0.01 and 1.0."); } - /// Minimum requests before circuit breaker evaluates. Range: 1-1,000. Default: 40. + /// + /// Minimum requests before circuit breaker evaluates. Range: 2-1,000. Default: 40. + /// The floor is 2 because that is the least Polly's breaker will hold: a lower value is + /// refused when the pipeline is built, one call after the option was set and in Polly's + /// words, so it is refused here instead. + /// public int CircuitBreakerMinThroughput { get => _circuitBreakerMinThroughput; - init => _circuitBreakerMinThroughput = value is > 0 and <= 1000 + init => _circuitBreakerMinThroughput = value is >= 2 and <= 1000 ? value - : throw new ArgumentOutOfRangeException(nameof(CircuitBreakerMinThroughput), value, "Must be between 1 and 1,000."); + : throw new ArgumentOutOfRangeException(nameof(CircuitBreakerMinThroughput), value, "Must be between 2 and 1,000."); } /// Circuit breaker sampling window in seconds. Range: 5-300. Default: 30. diff --git a/src/Mgx.Engine/Http/SensitiveNames.cs b/src/Mgx.Engine/Http/SensitiveNames.cs new file mode 100644 index 0000000..b637539 --- /dev/null +++ b/src/Mgx.Engine/Http/SensitiveNames.cs @@ -0,0 +1,105 @@ +using System.Text.RegularExpressions; + +namespace Mgx.Engine.Http; + +/// +/// The name fragments that mark a value as a credential, matched case-insensitively as +/// substrings. Graph spells its secrets differently on nearly every resource - newPassword on +/// an administrative reset, preSharedKey and sharedSecret on Wi-Fi and VPN profiles, +/// uploadSecret on a trust framework key set - so a list of exact names is behind the surface +/// the day it is written. Substrings over-redact, and that is the cheaper error: a redacted +/// diagnostic field costs a line of context, an unredacted PSK is a credential sitting in a +/// file. +/// +/// One definition, two readers - and three rules, not one. The fragments are the first. A +/// property whose name carries "downloadurl" and a string value that is a URL carrying one of +/// the capability parameters are the other two, and they are here for the same reason: they say +/// credential without saying password, and each reader needs the same answer from them. +/// GraphRequestTracer applies all three to -Debug output as text; Invoke-MgxBatchRequest applies +/// all three to a dead-letter body as parsed JSON, and the fragments again to a request path. +/// +/// The value rule agrees on the form it is written in, since a URL is text either way, and on +/// both forms it comes in: a $batch item's own url is on the wire with its scheme and host +/// stripped, and a caller who was handed a relative URL by Graph passes it on as one, so a path +/// carrying a capability parameter is read as such by GraphRequestTracer and by +/// RedactCapabilityUrl below - which the file reads a body's keys and values through - alike. +/// The two name rules agree only where the value beside the named key is itself a quoted string +/// - all a trace regex can match. +/// Invoke-MgxBatchRequest clears that value whatever it holds, so a number, a bool, an array or +/// an object beside a password or a downloadUrl field is missing from the file and still stands +/// whole in a trace. +/// +public static class SensitiveNames +{ + private static readonly string[] TokenList = + [ + "password", "secret", "credential", "key", "token", "assertion", "passphrase", + "connectionstring" + ]; + + /// The fragments, lowercase. Compare case-insensitively. + public static IReadOnlyList Tokens => TokenList; + + /// True when a field name or URL path segment carries one of the fragments. + public static bool Matches(string name) + { + foreach (var token in TokenList) + { + if (name.Contains(token, StringComparison.OrdinalIgnoreCase)) + return true; + } + return false; + } + + /// + /// The fragments as a regex alternation, for a caller matching names inside text rather + /// than one name at a time. + /// + internal static string RegexAlternation { get; } = + string.Join('|', TokenList.Select(Regex.Escape)); + + /// + /// The name fragment that says a property's value is a pre-authenticated URL even when the + /// value exposes no parameter to recognize it by - the short-form download hosts put the + /// capability in the path, so "@microsoft.graph.downloadUrl" and its older spelling + /// "@content.downloadUrl" are credentials under a name no fragment above reaches. + /// + internal const string DownloadUrlFragment = "downloadurl"; + + /// + /// The query parameters that carry a pre-authenticated URL's capability, as a regex + /// alternation rather than a list of names because `sig` needs a lookbehind to be read as a + /// parameter of its own. Deliberately not every URL: @odata.nextLink is a URL too, and + /// losing it would remove the thing paging bugs are diagnosed with. + /// + internal const string CapabilityUrlParameters = + "tempauth|guestaccesstoken|authkey|X-Amz-Signature|(?True when a property name says its value is a pre-authenticated URL. + public static bool NamesDownloadUrl(string name) => + name.Contains(DownloadUrlFragment, StringComparison.OrdinalIgnoreCase); + + /// + /// The capability-parameter rule on one unescaped value, for the reader holding parsed JSON + /// rather than text. Returns the value cut after the first capability parameter's `=` with + /// in place of the rest - the shape the tracer's replacement + /// makes, which keeps the parameter name readable so it is still clear which kind of URL + /// went - or null when the value carries no capability parameter and may be kept whole. + /// + /// The pattern differs from the tracer's only in what bounds the value: a quote ends a + /// string in the JSON text the tracer reads, and bounds nothing in a value that has already + /// been unescaped. The parameters both of them test are the definition above, and so are the + /// two forms a URL arrives in - with its scheme, or with a leading slash standing in for one, + /// which is what says the string is a path rather than prose that happens to carry a + /// parameter. + /// + public static string? RedactCapabilityUrl(string value, string marker) + { + var match = CapabilityUrl.Match(value); + return match.Success ? match.Groups[1].Value + marker : null; + } + + private static readonly Regex CapabilityUrl = new( + $"^((?:https?://|/).*?(?:{CapabilityUrlParameters})=)", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled); +} diff --git a/src/Mgx.Engine/Mgx.Engine.csproj b/src/Mgx.Engine/Mgx.Engine.csproj index b161959..d7318b9 100644 --- a/src/Mgx.Engine/Mgx.Engine.csproj +++ b/src/Mgx.Engine/Mgx.Engine.csproj @@ -8,7 +8,7 @@ + into a build failure --> $(WarningsAsErrors);CA1416 diff --git a/src/Mgx.Engine/Pagination/DeltaState.cs b/src/Mgx.Engine/Pagination/DeltaState.cs index 69d1a41..0ca4f25 100644 --- a/src/Mgx.Engine/Pagination/DeltaState.cs +++ b/src/Mgx.Engine/Pagination/DeltaState.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using System.Text; using System.Text.Json; using System.Text.Json.Serialization; @@ -104,7 +105,17 @@ public void Save(string path) var tmpPath = normalizedPath + ".tmp"; try { - File.WriteAllText(tmpPath, json); + // Cleared and then created, rather than written into whatever stands at the + // name: an open that creates or truncates follows a symlink there and fills or + // truncates the link's target with this state, and a FIFO with no reader blocks + // the write forever - past a cancellation, since nothing on the pipeline thread + // can reach a blocked open. + using (var scratch = ScratchName.Create(tmpPath, "the delta state's staging file")) + using (var writer = new StreamWriter(scratch, new UTF8Encoding(false, true), + 1024, leaveOpen: true)) + { + writer.Write(json); + } File.Move(tmpPath, normalizedPath, overwrite: true); } catch @@ -145,10 +156,16 @@ public static bool Delete(string path) } /// - /// Validates write access to the delta file path. Call in BeginProcessing - /// to fail fast before any HTTP calls. + /// Validates write access to a path the run will write to. Call in BeginProcessing to fail + /// fast before any HTTP calls. /// - public static void ValidateWriteAccess(string path) + /// The file whose directory has to be writable. + /// + /// What that file is, for the failure to name. The sync probes two of them, and one noun for + /// both reported an -OutputFile directory it could not write to as the delta state path - + /// sending the caller to a file that had just passed the same probe. + /// + public static void ValidateWriteAccess(string path, string what = "delta state path") { // NOTE: a directory passes the probe below, because the probe writes ".probe" // NEXT to the target rather than to it, so the fail-fast check succeeds and the real @@ -161,16 +178,62 @@ public static void ValidateWriteAccess(string path) if (dir != null && !Directory.Exists(dir)) Directory.CreateDirectory(dir); + // The probe name is cleared before it is created, and created by an open that refuses + // to use anything already standing there. Written with Create/Write, the probe followed + // a symlink at that name and truncated the link's target to nothing - a file outside + // this directory, taken away by a check whose whole purpose is to touch nothing - and a + // FIFO left there blocked the open in BeginProcessing, before the run had anything a + // cancellation could unwind. var probe = path + ".probe"; + var swept = ScratchName.Sweep(probe, out var stood); + + // A file another run holds at that name is that run's own probe, taken out a moment ago + // in this very directory - which is the question this one is asking. Two runs over one + // -DeltaPath, or one -OutputFile, reach here together as a matter of course, and + // refusing either of them for the other's sake would end a run over nothing that is + // wrong with the path. The two refusals left say something about the path itself: a + // directory at the name, or an entry the directory will not let go of. + if (swept == ScratchEntry.Held) return; + if (swept.Refused()) + { + throw new InvalidOperationException( + $"Cannot write to {what} '{path}': {swept.WhatStood(probe, stood)}.", stood); + } + + FileStream created; try { - File.WriteAllText(probe, ""); - File.Delete(probe); + created = ScratchName.CreateNew(probe); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { + // Another run reached the name first, which on this name is its probe: a create + // refuses a name that already exists, and a create that meets the claim the other + // run is holding that name by is refused as a sharing violation. Either way it is + // the answer the sweep above gives a held probe, for the reason it gives it. + // + // Read off the refusal as well as off the name, because the entry that refuses this + // create is one the run that made it is about to take away: asked of the name alone, + // the question arrives after the answer has gone. Two syncs released together over + // one -DeltaPath had the second create refused by the first one's probe, found the + // name clear an instruction later, and ended in BeginProcessing naming a path + // nothing was wrong with. + // + // A directory is not that. It is the path's own refusal, it is not this run's to + // remove, and it is still standing to be asked about - which is asked of the + // filesystem, since a create meets one as an access failure on Windows and as a + // name that already exists on Unix. Neither is a create the directory refused + // outright, which is the failure this whole probe exists to reach. + var reachedFirst = File.Exists(probe) + || (ex is IOException refused + && (ScratchName.IsAlreadyExists(refused) + || ScratchName.IsSharingViolation(refused))); + if (reachedFirst && !Directory.Exists(probe)) return; throw new InvalidOperationException( - $"Cannot write to delta state path '{path}': {ex.Message}", ex); + $"Cannot write to {what} '{path}': {ex.Message}", ex); } + + // Unlinked under the handle that holds it, so the name is never decided and unheld. + ScratchName.Discard(probe, created); } } diff --git a/src/Mgx.Engine/Pagination/HeldEntry.cs b/src/Mgx.Engine/Pagination/HeldEntry.cs new file mode 100644 index 0000000..e13bc62 --- /dev/null +++ b/src/Mgx.Engine/Pagination/HeldEntry.cs @@ -0,0 +1,301 @@ +using System.Runtime.InteropServices; + +namespace Mgx.Engine.Pagination; + +/// +/// Whether the file a run holds is still the file standing at the name it asked for. A claim +/// taken on a name is finished here, and nowhere else. +/// +/// .NET backs on Unix with a non-blocking flock taken AFTER +/// open(2), so the two halves of a claim are not one instruction: the handle binds to +/// whatever inode the name resolved to at the open, and the exclusion over it is granted several +/// syscalls later. A run descheduled between them comes back holding an inode a second run +/// renamed its own file over in the interval - rename(2) honors no lock on either side - +/// and the lock is then granted over a file with no name left on it. Nothing about that handle +/// says so: it reads its own length, it takes appends, and the claim it stands for was over a +/// name that is now somebody else's. Two runs promoting over one output both answered Taken that +/// way, and the second renamed its copy over the first one's rows. +/// +/// +/// So the entry now at the name has to be the inode the handle holds, and that inode has to +/// still have a name. Both are asked of the platform after the lock is granted, and a caller +/// that cannot answer yes drops the handle and asks the name again rather than acting on it. +/// +/// +/// What that reaches is the writers that take a claim: a promotion's hold on the output it +/// renames onto and on the staging copy it renames from, the sweep of a scratch name, the hold +/// a resumed run keeps on the output it appends to, and the files a promotion takes back when it +/// is done - the temp it spent, which goes only where the promotion landed, and the staged copy +/// it wrote and the empty output it made in order to hold one, which go only where it stopped. +/// Between those, no run renames onto or unlinks a name whose current inode another run holds. +/// +/// +/// A writer that takes no claim is refused by none of it. This is a rule between claims, and +/// rename and unlink honor no lock in the first place: a plain export's final move onto its +/// output, the atomic saves of the checkpoint and the delta state - which let the staging claim +/// go before the rename that spends it - an older mgx, another tool, a shell. Each of those +/// replaces a name while a run holds what stands at it, and the run holding it goes on writing +/// into an inode that has left the directory. What the rule holds apart is two claims; it does +/// not hold the name still. +/// +/// +/// Windows needs none of it. The open there IS the exclusion, and the kernel refuses a rename +/// onto a name another handle has open and a rename of one too, so the question cannot come up +/// and every caller keeps the answer it had. +/// +/// +public static class HeldEntry +{ + /// + /// How many times a claim is asked again when the name moved under it before the caller is + /// told the name is held. Each attempt is a fresh open, so a re-ask over a name a second run + /// has taken meets that run's own lock and answers Held on its own; what the attempts are + /// for is the other outcome, a name whose entry was replaced by a run that has since let go, + /// where the next open is the one that succeeds. Three, because a run has to be descheduled + /// between an open and a lock to lose one at all, and nothing waits between them: a name + /// that moves under three consecutive claims is a name being mutated right now, which is + /// what a hold means to every caller here. + /// + public const int ClaimAttempts = 3; + + /// + /// Test seam: run in the instant between a claim being granted and the check that the name + /// still resolves to it, with the path that was claimed. That instant is the whole defect + /// and no test can hit it by racing for it, so it is fired here instead. Null on every + /// shipping path; internal, assigned directly rather than reflected, and reset by the test + /// that set it. + /// + internal static Action? BeforeTheClaimIsVerified; + + /// + /// Whether is the entry standing at : the + /// same inode on the same device, with a name still on it. False says the name moved under + /// the handle - unlinked, renamed away, or replaced by another run's file - and a caller + /// that acts on the name anyway acts on a file that is not the one it claimed. + /// + /// The fd is read off , and reading it hands the file whatever that + /// stream still has buffered to write. So a claim is verified with the buffer written out, + /// and a caller that reaches this with rows waiting in one pays for that write here and + /// meets its failure here rather than where it writes. One route does: a promotion whose + /// copy into the staged file fails part way through returns without reaching the flush that + /// follows the loop, and the discard in its finally verifies that writer - so the rows still + /// in the buffer are written out here, and a failure of that write is met here, in a discard + /// whose caller has already answered. The discard swallows that failure; the dispose that + /// follows it re-attempts the same write and lets the failure out, to the promotion's own + /// catch. Every other caller reaches this with nothing waiting, or with the buffer flushed an + /// instruction earlier. + /// + /// + /// + /// The question could not be answered: an entry point that is not there on this host, or a + /// failure from the call that is not the name resolving to nothing. Raised naming the call + /// and the errno rather than answered "still there", because a claim that cannot be verified + /// is not a claim, and every caller here reads a raised IOException as a file it has to + /// leave alone. + /// + public static bool StillStandsAt(FileStream held, string path) + { + BeforeTheClaimIsVerified?.Invoke(path); + + // The open is the exclusion on Windows and the kernel keeps the name still under it. + if (OperatingSystem.IsWindows()) return true; + + // The handle first, so its link count is read before the name is looked at: an entry + // that answers for the name afterwards is one this handle held while it still had a + // name, and a rename landing between the two readings takes this inode's last name with + // it and is read here. + var mine = OfTheHandle(held, path); + if (mine.Links == 0) return false; + + var there = AtTheName(path); + return there is { } entry && entry.Device == mine.Device && entry.Inode == mine.Inode; + } + + /// What a stat answered: the file's identity, and how many names it has. + private readonly record struct Identity(long Device, ulong Inode, ulong Links); + + private static Identity OfTheHandle(FileStream held, string path) + { + var handle = held.SafeFileHandle; + var counted = false; + try + { + // The fd, held for as long as the call takes: a handle disposed on another thread in + // the middle of it would otherwise be a stat of whatever fd the runtime handed that + // number out to next. + handle.DangerousAddRef(ref counted); + var fd = (int)handle.DangerousGetHandle(); + var buffer = new byte[StatBufferBytes]; + int answer; + try + { + answer = OperatingSystem.IsMacOS() + ? (MacPlainNamesAreIno64 ? MacFstat(fd, buffer) : MacFstatIno64(fd, buffer)) + : LinuxStatxOfFd(fd, string.Empty, AtEmptyPath, StatxWanted, buffer); + } + catch (EntryPointNotFoundException ex) { throw NotOnThisHost(FdCall, path, ex); } + + if (answer != 0) throw Failed(FdCall, path, Marshal.GetLastPInvokeError()); + return OperatingSystem.IsMacOS() ? MacIdentity(buffer) : LinuxIdentity(buffer, path); + } + finally + { + if (counted) handle.DangerousRelease(); + } + } + + /// + /// What stands at the name, or null where the name resolves to nothing. Symlinks are + /// followed, because the open that took the claim followed them too: the question is whether + /// this handle is what that same name reaches now. + /// + private static Identity? AtTheName(string path) + { + var buffer = new byte[StatBufferBytes]; + int answer; + try + { + answer = OperatingSystem.IsMacOS() + ? (MacPlainNamesAreIno64 ? MacStat(path, buffer) : MacStatIno64(path, buffer)) + : LinuxStatx(AtFdCwd, path, 0, StatxWanted, buffer); + } + catch (EntryPointNotFoundException ex) { throw NotOnThisHost(NameCall, path, ex); } + + if (answer == 0) + { + return OperatingSystem.IsMacOS() ? MacIdentity(buffer) : LinuxIdentity(buffer, path); + } + + var errno = Marshal.GetLastPInvokeError(); + if (NameResolvesToNothing(errno)) return null; + throw Failed(NameCall, path, errno); + } + + /// + /// The call each half is asked with, for the sentence a failure is raised in. + /// + private static string FdCall => OperatingSystem.IsMacOS() ? "fstat(2)" : "statx(2)"; + + /// + private static string NameCall => OperatingSystem.IsMacOS() ? "stat(2)" : "statx(2)"; + + /// + /// ENOENT and ENOTDIR, the two failures that say the name reaches no entry rather than that + /// the question failed - a name that has gone, and a name whose parent is no longer a + /// directory. Both numbers are the same on macOS and on Linux. Every other failure is + /// raised: an answer this cannot read is not an answer that the file is still there. + /// + private static bool NameResolvesToNothing(int errno) => errno is 2 or 20; + + /// + /// A libc without the symbol is a host this cannot verify a claim on. Raised rather than + /// answered "still there", which is the silence this whole file exists to remove. + /// + private static IOException NotOnThisHost(string named, string path, Exception ex) => + new($"{named} is not available on this host, so the claim on '{path}' cannot be " + + "verified.", ex); + + /// + private static IOException Failed(string named, string path, int errno) => + new($"{named} on '{path}' failed with errno {errno}, so the claim on it cannot be " + + "verified."); + + // --- macOS ------------------------------------------------------------------------------- + // + // struct stat with 64-bit inodes, which is the only shape either supported architecture + // reaches: st_dev int32 at 0, st_mode uint16 at 4, st_nlink uint16 at 6, st_ino uint64 at 8. + // The entry point differs by architecture and the struct does not. arm64 has never had the + // 32-bit-inode form, so the plain names ARE the 64-bit ones there; x86_64 carries both, and + // the plain names there are the old shape - a different struct, whose st_ino is 32 bits at + // offset 4 - so the "$INODE64" symbols are the ones asked for. This is what + // does to every stat call at compile time (__DARWIN_INODE64, suffixed on x86_64 and empty + // on arm64); nothing here can be compiled through that header, so the choice is made once, + // at runtime, off the architecture the process is actually running as - which is x64 under + // Rosetta, where the x64 libSystem is the one loaded. + + private static bool MacPlainNamesAreIno64 => + RuntimeInformation.ProcessArchitecture != Architecture.X64; + + private static Identity MacIdentity(byte[] buffer) => new( + BitConverter.ToInt32(buffer, 0), + BitConverter.ToUInt64(buffer, 8), + BitConverter.ToUInt16(buffer, 6)); + + [DllImport(Libc, EntryPoint = "fstat", SetLastError = true)] + private static extern int MacFstat(int fd, byte[] buffer); + + [DllImport(Libc, EntryPoint = "stat", SetLastError = true)] + private static extern int MacStat([MarshalAs(UnmanagedType.LPUTF8Str)] string path, + byte[] buffer); + + [DllImport(Libc, EntryPoint = "fstat$INODE64", SetLastError = true)] + private static extern int MacFstatIno64(int fd, byte[] buffer); + + [DllImport(Libc, EntryPoint = "stat$INODE64", SetLastError = true)] + private static extern int MacStatIno64([MarshalAs(UnmanagedType.LPUTF8Str)] string path, + byte[] buffer); + + // --- Linux ------------------------------------------------------------------------------- + // + // statx, and not stat: glibc exported no plain "stat" symbol before 2.33 - the header turned + // it into __xstat with a struct version argument - and struct stat's own layout is per + // architecture, so a fixed set of offsets would be a different struct on x86_64, arm64 and + // arm. struct statx is one shape everywhere and its offsets are the kernel's ABI: + // stx_nlink uint32 at 16, stx_ino uint64 at 32, and stx_dev_major / stx_dev_minor uint32 at + // 136 / 140, past four 16-byte timestamps and the two rdev halves at 128 / 132. The wrapper + // has been in glibc since 2.28 and in musl since 1.2, which is under every distribution + // .NET 8 supports; a host without it raises rather than answering. + + private const int AtFdCwd = -100; + private const int AtEmptyPath = 0x1000; + + /// STATX_NLINK (0x4) and STATX_INO (0x100), the two fields read out of it. + private const uint StatxWanted = 0x4 | 0x100; + + private static Identity LinuxIdentity(byte[] buffer, string path) + { + // What the kernel actually filled in. Asking for a field is not being given it - a + // filesystem that cannot answer clears its bit - and reading a field that was never + // written would compare this run's claim against a zero. + var answered = BitConverter.ToUInt32(buffer, 0); + if ((answered & StatxWanted) != StatxWanted) + { + throw new IOException( + $"statx(2) on '{path}' answered without the inode or the link count, so the " + + "claim on it cannot be verified."); + } + var major = BitConverter.ToUInt32(buffer, 136); + var minor = BitConverter.ToUInt32(buffer, 140); + return new Identity( + ((long)major << 32) | minor, + BitConverter.ToUInt64(buffer, 32), + BitConverter.ToUInt32(buffer, 16)); + } + + [DllImport(Libc, EntryPoint = "statx", SetLastError = true)] + private static extern int LinuxStatx(int dirFd, + [MarshalAs(UnmanagedType.LPUTF8Str)] string path, int flags, uint mask, byte[] buffer); + + /// + /// + /// The same call over an open file: the fd in the directory slot, an empty path, and + /// AT_EMPTY_PATH to say the fd is the file rather than the directory to resolve in. + /// + [DllImport(Libc, EntryPoint = "statx", SetLastError = true)] + private static extern int LinuxStatxOfFd(int fd, + [MarshalAs(UnmanagedType.LPUTF8Str)] string path, int flags, uint mask, byte[] buffer); + + /// + /// libSystem on macOS and the C library on Linux, under the name .NET resolves on both. + /// + private const string Libc = "libc"; + + /// + /// Room for either struct: 144 bytes for macOS's stat with 64-bit inodes, and 256 for + /// Linux's statx, which is 256 bytes by the kernel's ABI - the spare fields at its tail are + /// part of that size, and a field a later kernel adds is one of those spares being given a + /// name. One size for both, since the fields read out of each are at the offsets above and + /// nothing here walks off the end of what the call wrote. + /// + private const int StatBufferBytes = 256; +} diff --git a/src/Mgx.Engine/Pagination/PaginationCheckpoint.cs b/src/Mgx.Engine/Pagination/PaginationCheckpoint.cs index 4e364df..53653e3 100644 --- a/src/Mgx.Engine/Pagination/PaginationCheckpoint.cs +++ b/src/Mgx.Engine/Pagination/PaginationCheckpoint.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using System.Text; using System.Text.Json; using System.Text.Json.Serialization; @@ -110,7 +111,17 @@ public void Save(string path) var tmpPath = normalizedPath + ".tmp"; try { - File.WriteAllText(tmpPath, json); + // Cleared and then created, rather than written into whatever stands at the + // name: an open that creates or truncates follows a symlink there and fills or + // truncates the link's target with this checkpoint, and a FIFO with no reader + // blocks the write forever - past a cancellation, since nothing on the pipeline + // thread can reach a blocked open. + using (var scratch = ScratchName.Create(tmpPath, "the checkpoint's staging file")) + using (var writer = new StreamWriter(scratch, new UTF8Encoding(false, true), + 1024, leaveOpen: true)) + { + writer.Write(json); + } File.Move(tmpPath, normalizedPath, overwrite: true); } catch diff --git a/src/Mgx.Engine/Pagination/ScratchName.cs b/src/Mgx.Engine/Pagination/ScratchName.cs new file mode 100644 index 0000000..f31e6b9 --- /dev/null +++ b/src/Mgx.Engine/Pagination/ScratchName.cs @@ -0,0 +1,502 @@ +namespace Mgx.Engine.Pagination; + +/// +/// What stands at a scratch name. The same reading either way round: what a sweep took off the +/// name, or what an inspection found standing there and left. +/// +public enum ScratchEntry +{ + /// Nothing stands at the name. + Clear, + + /// A link. Swept, the link goes and what it pointed at is left alone. + Link, + + /// A regular file nothing holds - a leftover from a run that was killed. + Leftover, + + /// + /// A regular file this account cannot open read-write. Swept, since the name is nobody's to + /// keep, but nothing here could look inside it to say more than that. + /// + Unopenable, + + /// + /// Something that is not a file with a length and an offset in it - a pipe, a socket, a + /// device. Swept: nothing can be written at the name while it stands there. + /// + NotAFile, + + /// + /// A directory. Not this caller's to delete, and it could hold anything. + /// + Directory, + + /// A file another run has open. + Held, + + /// + /// The entry could not be read, or could not be unlinked. The exception says why. + /// + Unreadable, +} + +/// +/// The scratch names a run makes beside a file it is writing - the write probe, the atomic +/// saves' staging file, the copy a promotion stages - cleared before anything is opened +/// through them. +/// +/// No run leaves a file under one of those names on purpose: each is created, written and +/// renamed away or removed inside one operation, so an entry standing at one is a run that was +/// killed between two of those steps, or something a caller put there. Opening the name to +/// find out is what has to stop: an open that creates or truncates follows a symlink and +/// truncates the link's target, a FIFO with no reader blocks the open forever with no way for +/// a cancellation to reach it, and a create that refuses an existing name cannot say what kind +/// of entry refused it. So the entry is decided off the directory rather than through a +/// handle, and the kinds divide by what a run can honestly do about them: a link goes - the +/// link and never its target - and so does anything at the name that is not a regular file, +/// and a regular file no run holds. A directory and a file another run has open stop the +/// caller with the disk as it was found. +/// +/// +/// It lives here rather than beside the promotion that first needed it because the engine +/// cannot see the cmdlet base: the checkpoint and the delta state write scratch names of their +/// own, and one primitive is what keeps a pipe at a checkpoint's staging name from wedging a +/// run that a pipe at an output's staging name is refused at. +/// +/// +public static class ScratchName +{ + /// + /// Clear , or say why the caller stops there. Nothing of the + /// caller's may be open over the name when this runs: the claim below asks for the entry + /// exclusively, and a hard link at the name is an alias for whatever inode it points at - + /// so run under a claim on that inode, the sweep reads its own caller's lock as a second + /// run holding the entry. + /// + /// + /// The exception behind , for a caller that words + /// its own refusal from it; null for every other answer. + /// + public static ScratchEntry Sweep(string path, out Exception? failure) => + Decide(path, remove: true, out failure); + + /// + /// What stands at , with the name left exactly as it was found. The + /// same reading gives, so a pass that reports what a run would take off + /// the name reports the kind that run would name - and a preview that swept in order to find + /// out made deletions of its own, which is the one thing it is asked not to do. + /// + /// + public static ScratchEntry Inspect(string path, out Exception? failure) => + Decide(path, remove: false, out failure); + + private static ScratchEntry Decide(string path, bool remove, out Exception? failure) + { + // Asked again where the name moved under the claim below, and answered Held where it + // keeps moving: a name being mutated right now is a name this caller leaves alone, which + // is what every caller here already does with a name a second run holds. Each attempt is + // a fresh open, so the run that took the name answers the next one with its own lock. + for (var attempt = 1; ; attempt++) + { + if (DecideOnce(path, remove, out failure) is { } entry) return entry; + if (attempt == HeldEntry.ClaimAttempts) return ScratchEntry.Held; + } + } + + /// + /// One attempt at the name. Null where the claim below was granted over an entry that had + /// left the name - the one answer that is neither what stands there nor a refusal, and the + /// one asking again can change. + /// + private static ScratchEntry? DecideOnce(string path, bool remove, out Exception? failure) + { + failure = null; + try + { + // Read off the directory entry rather than through an open, which is what lets a + // dangling link and a symlink loop be answered at all: both are links and neither + // opens. File.Delete unlinks the link and leaves what it pointed at alone. + if (new FileInfo(path).LinkTarget != null) + { + if (remove) File.Delete(path); + return ScratchEntry.Link; + } + + if (Directory.Exists(path)) return ScratchEntry.Directory; + + FileStream? claim = null; + try + { + bool seekable; + try + { + // FileAccess.ReadWrite, because it is the one access a FIFO answers without + // blocking: a read-only or write-only open of a pipe with nothing on the + // other end waits for a peer forever, and a run parked there holds + // everything it had claimed with no way for a cancellation to reach it. + // FileMode.Open, so nothing is created and nothing is truncated. And + // FileShare.None, so what comes back a leftover is an entry no second run + // is using. + claim = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, + FileShare.None); + // Something opened, and it is not a file with a length and an offset in it: + // a FIFO opens read-write without blocking and answers every question after + // that with a NotSupportedException. Told apart here rather than at the + // first write, where a caller has already committed to the name. + seekable = claim.CanSeek; + } + catch (FileNotFoundException) { return ScratchEntry.Clear; } + catch (DirectoryNotFoundException) + { + // The same answer one level up: ENOENT on a path component. + return ScratchEntry.Clear; + } + catch (UnauthorizedAccessException) + { + // A file this account may not open. The name is still nobody's to keep, and + // the unlink asks the directory rather than the file. + if (remove) Unlink(path); + return ScratchEntry.Unopenable; + } + catch (IOException ex) + { + // The sharing violation is the one failure with a second run behind it. + // Everything else - a socket, a symlink loop, an over-long path - is an + // entry nothing can be written at. + if (IsSharingViolation(ex)) return ScratchEntry.Held; + if (remove) Unlink(path); + return ScratchEntry.NotAFile; + } + + // And the claim is not finished at the open. .NET takes the lock behind + // FileShare.None after open(2) on Unix, so this handle can have come away over + // an inode a second run renamed its own file over in between - the lock granted + // on a file with no name left on it, and the unlink below then taking that run's + // file off the name. What is held has to be what stands at the name. + if (!HeldEntry.StillStandsAt(claim, path)) return null; + + var entry = seekable ? ScratchEntry.Leftover : ScratchEntry.NotAFile; + if (!remove) return entry; + + // Held across the unlink rather than let go in front of it, so there is no + // instant in which the name is decided and unheld: a second run sweeping the + // same name in that interval created its own file at it and had it deleted from + // under the handle it was holding it by. The entry unlinked is the entry + // claimed - the line above is what makes that true - and unlink(2) honors no + // lock, so holding it costs the delete nothing. + // + // Windows keeps the order it had: a file this process has open cannot be + // unlinked there without FILE_SHARE_DELETE, so the handle goes in front of the + // delete and the interval is that platform's. + if (!OperatingSystem.IsWindows()) + { + File.Delete(path); + return entry; + } + + claim.Dispose(); + claim = null; + + // And on the far side of that interval the delete can meet a second run instead + // of the entry this sweep read: the name was let go, so a run sweeping it in + // that instant created its own file at it and is holding it, and the delete + // arrives at that run's claim. What stands there is not a leftover this sweep + // can answer for - it is the answer the claim above gives a name another run + // has, which every caller here already leaves the name alone on. Read as an + // entry nothing could be made of, it stopped runs over a name that was simply + // in use. + var taken = DeleteWaitingOutADeleteInFlight(path); + if (taken == null) return entry; + failure = taken; + return ScratchEntry.Held; + } + finally { claim?.Dispose(); } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + failure = ex; + return ScratchEntry.Unreadable; + } + } + + /// + /// Whether the entry stops the caller rather than being one a sweep clears. + /// + public static bool Refused(this ScratchEntry entry) => + entry is ScratchEntry.Directory or ScratchEntry.Held or ScratchEntry.Unreadable; + + /// Whether a sweep takes the entry off the name. + public static bool Removable(this ScratchEntry entry) => + entry is ScratchEntry.Link or ScratchEntry.Leftover or ScratchEntry.Unopenable + or ScratchEntry.NotAFile; + + /// + /// What stood at the name, for a caller that stops on it to name inside its own sentence. + /// Punctuated by the caller. + /// + public static string WhatStood(this ScratchEntry entry, string path, Exception? failure) + { + var name = Path.GetFileName(path); + return entry switch + { + ScratchEntry.Directory => $"a directory stands at '{name}'", + ScratchEntry.Held => $"another run has '{name}' open", + _ => $"'{name}' could not be cleared: " + + FirstLine(failure?.Message ?? "the reason is not known").TrimEnd('.'), + }; + } + + /// + /// The name, cleared and then made by this caller: the handle that comes back is over an + /// entry this run created, since the create refuses to use anything already standing there. + /// + /// + /// The name could not be cleared - a directory at it, another run holding it - or the + /// create was refused. A caller with a sentence of its own to say asks + /// and creates the file itself. + /// + public static FileStream Create(string path, string what) + { + var swept = Sweep(path, out var failure); + if (swept.Refused()) + { + throw new IOException( + $"Cannot write {what} '{path}': {swept.WhatStood(path, failure)}.", failure); + } + return CreateNew(path); + } + + /// + /// How many times a scratch name is asked again when Windows answers that this account may + /// not touch it, and how long each wait is. A delete of a name that has been started and has + /// not finished leaves the entry in the directory with every open of it refused, and both a + /// create and an unlink meet that as ERROR_ACCESS_DENIED - the same code a directory this + /// account cannot write answers with, so the two cannot be told apart from the exception. + /// + /// What tells them apart is time. The pending state ends when the last handle on the entry + /// goes, which is the next few milliseconds; a directory this account cannot write is still + /// refusing an hour later. So the name is asked again a few times over a few tens of + /// milliseconds, and only a refusal that outlasts that is the path's own - the one this + /// probe exists to reach, and it costs that path 40 ms it was going to stop on anyway. + /// + /// + /// Measured on Windows Server 2022, four runs creating and unlinking one name together: over + /// 40 runs of that shape, 154 creates and 64 unlinks were refused this way and not one of + /// them was anything about the directory. + /// + /// + /// Unix has a delete in flight of its own, and the same wait answers it. A create there is + /// two steps - the entry is made, and the lock over it is taken after - and a second run + /// sweeping the same name in between opens the entry this one just made, is granted the lock + /// first, reads a file nothing holds and takes it off the name. What comes back to the + /// create is the refused lock, over an entry that is on its way out of the directory; the + /// "already exists" a step later is the same instant with that unlink not yet landed. Both + /// are waited out, and only after a refused lock - a name a second run simply has its own + /// copy standing at is answered at once, as it always was. + /// + /// + private const int DeleteInFlightAttempts = 6; + + /// + private const int DeleteInFlightWaitMs = 8; + + /// + /// The create every scratch name is made by: FileMode.CreateNew refuses to use anything + /// already standing at the name, so the handle that comes back is over an entry this run + /// made and no other - which is what lets a caller unlink it afterwards without asking whose + /// it is. FileShare.None from the instant the entry exists, for the same reason. + /// + /// + public static FileStream CreateNew(string path) + { + var takenInTheGap = false; + for (var attempt = 1; ; attempt++) + { + try + { + return new FileStream(path, FileMode.CreateNew, FileAccess.ReadWrite, + FileShare.None); + } + catch (UnauthorizedAccessException) when (OperatingSystem.IsWindows() + && attempt < DeleteInFlightAttempts + && !Directory.Exists(path)) + { + Thread.Sleep(DeleteInFlightWaitMs); + } + catch (IOException ex) when (!OperatingSystem.IsWindows() + && attempt < DeleteInFlightAttempts + && (IsSharingViolation(ex) + || (takenInTheGap && IsAlreadyExists(ex)))) + { + // The entry this create made, claimed by a second run in the instant before the + // lock over it was taken - and read by that run as the one thing it could not + // be, a file nothing holds. It is being unlinked as this waits, and the name is + // this run's again on the other side of that. + takenInTheGap = true; + Thread.Sleep(DeleteInFlightWaitMs); + } + } + } + + /// + /// The unlink a Windows sweep takes the name back with, which answers for itself: a name + /// that cannot be cleared is one the caller stops on, so unlike this + /// does not swallow the failure. Two refusals are not that. A delete another run has already + /// started on the name is the name going, which is all the sweep wanted, and it is waited + /// out. A second run holding its own file at the name is that run's to answer for, and it is + /// reported as the hold it is. + /// + /// + /// Null where the name is clear. The sharing violation where a second run has taken the name + /// since this sweep let go of it. Any other refusal is raised, for the caller to report as + /// an entry it could not read. + /// + /// + private static IOException? DeleteWaitingOutADeleteInFlight(string path) + { + for (var attempt = 1; ; attempt++) + { + try + { + File.Delete(path); + return null; + } + catch (IOException ex) when (IsSharingViolation(ex)) + { + return ex; + } + catch (UnauthorizedAccessException) when (attempt < DeleteInFlightAttempts + && !Directory.Exists(path)) + { + Thread.Sleep(DeleteInFlightWaitMs); + } + } + } + + /// + /// Take the name back, under the handle that holds it. On Unix the entry is unlinked while + /// this caller still holds it, so the name is never decided and unheld: let go in front of + /// the delete, it is a name a second run can sweep, create its own file at and have deleted + /// from under it. Windows refuses to unlink a file this process has open without + /// FILE_SHARE_DELETE, so there the handle goes first and the interval is that platform's. + /// + /// Holding the entry is not on its own knowing that the name still reaches it - the lock + /// behind FileShare.None is granted after the open that resolved the name, and a handle can + /// come away over a file another run has since renamed its own copy over. So the name is + /// taken back only while is what stands at it, and left exactly as + /// it is otherwise: what is there belongs to the run that put it there. A discard with no + /// handle has nothing standing in this caller's name and removes nothing. + /// + /// + /// And a claim that cannot be verified is not a claim: where the question above fails rather + /// than answers - a host without the call, a parent directory that has stopped answering for + /// the name, a filesystem that fails it for a reason this cannot read - nothing is removed + /// and the name is left for the next sweep, whatever this caller is holding. So the temp a + /// promotion spends, the copy it staged and the write probe are taken back under the claim + /// where the claim can be finished, and left standing where it cannot. + /// + /// + /// Best effort, both ways round: a name that cannot be unlinked is a leftover the next + /// sweep takes, and a caller that has already answered must not fail on it. + /// + /// + public static void Discard(string path, FileStream? held) + { + if (!OperatingSystem.IsWindows()) + { + if (held != null && StillHolds(held, path)) Unlink(path); + held?.Dispose(); + return; + } + held?.Dispose(); + Unlink(path); + } + + /// + /// + /// Whether the handle is still what stands at the name, for a caller that must not fail. A + /// question that cannot be answered is answered no: the entry is left, which costs a leftover + /// the next sweep takes - where removing a file this run turns out not to hold costs the run + /// that does hold it its rows. + /// + private static bool StillHolds(FileStream held, string path) + { + try { return HeldEntry.StillStandsAt(held, path); } + catch (IOException) { return false; } + } + + private static void Unlink(string path) + { + try { File.Delete(path); } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { } + } + + /// + /// Whether an IOException from opening a file is the sharing violation the share mode exists + /// to produce, rather than one of the other ways an open fails. Named positively, because + /// the sharing case is the only one that means nothing is wrong with the file and another + /// run has it, and the only one a caller answers by waiting: a failure nothing here + /// recognizes is reported as a file this run cannot open, which is true of it whatever the + /// cause, instead of as a second run the caller can go and look for and not find. + /// + /// Windows carries the win32 code in the low half of the HRESULT. Unix puts the raw errno + /// there and nothing beside it, so the two forms cannot be read for each other: .NET backs + /// on Unix with a non-blocking flock, and a refused LOCK_EX + /// arrives as EWOULDBLOCK - 35 on macOS, measured for a second opener both inside the + /// process and outside it, and 11 on Linux, where the name EAGAIN shares that value. The + /// subclasses .NET maps by hand carry a win32-shaped HRESULT even on Unix - an over-long + /// path is a PathTooLongException with 0x800700CE in it - and are one more failure this + /// cannot mistake for a second run. + /// + /// + /// Each of those numbers is read on the platform that uses it and nowhere else, because the + /// two are a pair either way round: 11 is EDEADLK on macOS and 35 is EDEADLK on Linux. Taken + /// as a union, whichever platform this ran on read a deadlock the kernel had just refused to + /// enter as another run holding the file, and told the caller to wait for it. + /// + /// + public static bool IsSharingViolation(IOException ex) => ex.HResult switch + { + unchecked((int)0x80070020) => true, // ERROR_SHARING_VIOLATION + unchecked((int)0x80070021) => true, // ERROR_LOCK_VIOLATION + 35 => OperatingSystem.IsMacOS(), // EWOULDBLOCK (EDEADLK on Linux) + 11 => OperatingSystem.IsLinux(), // EAGAIN (EDEADLK on macOS) + _ => false, + }; + + /// + /// Whether an IOException from a create is the runtime's refusal of a name that already + /// exists, rather than one of the other ways a create fails. Read off the exception and not + /// off the name, which is the one thing a second look cannot answer: a create refused by + /// another run's scratch file is refused by an entry that run is about to take away, so by + /// the time the name can be asked about it is clear again - and a caller that decided from + /// that answer called a directory it could write in a directory it could not. + /// + /// What WEARS the name is a different question, and this cannot answer it: a file, a + /// directory and a link all refuse a create alike. A caller that has to tell them apart asks + /// the filesystem, which is honest about the entries that are still there to be asked about. + /// + /// + /// Windows carries the win32 code in the low half of the HRESULT and Unix puts the raw errno + /// there, the same pair reads. EEXIST is 17 on both Unixes + /// this runs on, so unlike the sharing errnos there is nothing here to tell macOS and Linux + /// apart; the platform is asked anyway, so that a win32 HRESULT can never be read as an + /// errno on the one host where the two are shaped alike. + /// + /// + public static bool IsAlreadyExists(IOException ex) => ex.HResult switch + { + unchecked((int)0x80070050) => true, // ERROR_FILE_EXISTS + unchecked((int)0x800700B7) => true, // ERROR_ALREADY_EXISTS + 17 => !OperatingSystem.IsWindows(), // EEXIST + _ => false, + }; + + /// The runtime's own first line of a message a caller quotes inside a sentence. + private static string FirstLine(string message) + { + var end = message.IndexOfAny(['\r', '\n']); + return end < 0 ? message : message[..end]; + } +} diff --git a/tests/CORPUS.md b/tests/CORPUS.md deleted file mode 100644 index 64f0474..0000000 --- a/tests/CORPUS.md +++ /dev/null @@ -1,66 +0,0 @@ -# Regression corpus - -Every entry below started as a real failure reported against Microsoft365DSC or the Graph -PowerShell SDK. The issue numbers are provenance, never scope: mgx does not reproduce the -reported case, it generalizes the class of failure and pins the guarantee the tests hold. -`tests/Mgx.IntegrationTests/TestSetup/HostileInputs.cs` carries the same provenance on the input -groups themselves; this file is the index over the whole suite. - -**Class** is the `MgxErrorClass` the failure surfaces as, from the classifier's own vocabulary in -`src/Mgx.Engine/Errors/MgxErrorClass.cs`. `n/a` is not a gap in the index: several of these -failures never reach the classifier at all, because nothing errors - a truncated enumeration, a -body the service accepts and misreads, a module that fails to load before a request exists. - -**Test type** is `wire` for xUnit over the mock transport, `unit` for xUnit with no transport, -`pester` for `tests/Unit`, and `live` for `tests/Live`, which needs a tenant. - -Test paths are relative to `tests/`. Where no test exists the row names the release that brings -one rather than pointing at something adjacent: an index that overstates coverage is worse than a -gap. - -## Covered - -| Issue | Source | Class | Guarantee | Test type | Subsystem | Tests | -| --- | --- | --- | --- | --- | --- | --- | -| M365DSC-7273 | Microsoft365DSC | Throttle | Sustained throttling is absorbed: Retry-After is honored, the rate halves on 429, and it climbs back afterwards. A 429 that outlives its retries reaches the caller under every `-ErrorAction`. | wire, live | pacing, retry | `Mgx.IntegrationTests/Engine/Http/RetryTests.cs`, `Mgx.IntegrationTests/Engine/Http/AdaptivePacingTests.cs`, `Mgx.IntegrationTests/Engine/Http/AdaptiveRequestPacerTests.cs`, `Mgx.IntegrationTests/Engine/Http/RetryAfterDateFormTests.cs`, `Mgx.IntegrationTests/Engine/Http/BatchPacingStateTests.cs`, `Mgx.IntegrationTests/Cmdlets/ErrorActionMatrixTests.cs` (throttling cells), `Live/Mgx.Throttle.Tests.ps1` | -| M365DSC-7274 | Microsoft365DSC | n/a | A large collection enumerates complete - through every nextLink, across page seams, and after a resume. | wire | pagination | `Mgx.IntegrationTests/Engine/Pagination/PaginationTests.cs`, `Mgx.IntegrationTests/Engine/Pagination/BoundarySizeTests.cs`, `Mgx.IntegrationTests/Engine/Pagination/CheckpointTests.cs`, `Mgx.IntegrationTests/Engine/Pagination/FanOutTests.cs`, `Mgx.IntegrationTests/Cmdlets/TopWithAllTests.cs`, `Mgx.IntegrationTests/Cmdlets/ExportCheckpointIntegrityTests.cs` | -| GraphSDK-3654 | Graph PowerShell SDK | InvalidRequest | A PowerShell wrapper never changes the wire JSON: `Serialize(x)` is byte-identical to `Serialize(PSObject.AsPSObject(x))`, at any nesting depth. | wire, unit | serialization | `Mgx.IntegrationTests/Serialization/PSWrapperEquivalenceTests.cs`, `Mgx.IntegrationTests/Serialization/WireBodyTests.cs` | -| GraphSDK-2942, GraphSDK-2709 | Graph PowerShell SDK | n/a | Hostile filter values are escaped exactly once, and the wire value decodes back to what the caller wrote. | wire | URI | `Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs`, `Mgx.IntegrationTests/TestSetup/HostileInputs.cs` (`FilterValues`) | -| GraphSDK-2488 | Graph PowerShell SDK | NotFound | A service-issued link is followed byte for byte - no re-encoding, no normalization - including across a checkpoint save and load. Validation may refuse a link, but never repairs one. | wire, unit | pagination | `Mgx.IntegrationTests/Engine/Pagination/OpaqueLinkTests.cs`, `Mgx.IntegrationTests/Engine/Pagination/SsrfValidationTests.cs`, `Mgx.IntegrationTests/TestSetup/HostileInputs.cs` (`OpaqueLinks`) | -| GraphSDK-1947 | Graph PowerShell SDK | NotFound | A `#` in a path stays in the path instead of being read as a URI fragment. | wire | URI | `Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs`, `Mgx.IntegrationTests/TestSetup/HostileInputs.cs` (`FilterValues`, `PathSegments`) | -| GraphSDK-2328 | Graph PowerShell SDK | InvalidRequest | Caller headers reach the wire as given: content headers land on the content collection rather than being dropped, and the merge is case-insensitive like HTTP header names are. | wire, unit | headers | `Mgx.IntegrationTests/Engine/Http/HeaderFidelityTests.cs`, `Mgx.IntegrationTests/Cmdlets/HeaderWireTests.cs` | -| GraphSDK-1425, GraphSDK-2088 | Graph PowerShell SDK | Permanent | A response that is not the JSON entity the caller expected - empty, HTML from a proxy, truncated - surfaces as verbose output or an error record, never an unhandled exception. | wire, unit | transport | `Mgx.IntegrationTests/Cmdlets/ResponseFidelityTests.cs`, `Mgx.IntegrationTests/Engine/ErrorBodyShapeTests.cs` | -| GraphSDK-3361 | Graph PowerShell SDK | InvalidRequest | Scalars and arrays serialize as themselves - a single-element array stays an array, a scalar does not become one. | wire, unit | serialization | `Mgx.IntegrationTests/Serialization/WireBodyTests.cs`, `Mgx.IntegrationTests/Serialization/PSWrapperEquivalenceTests.cs` | -| M365DSC-5306, M365DSC-7175 | Microsoft365DSC | InvalidRequest | A body that cannot be serialized is refused with a message naming the property path, and inside a batch it fails that item while the rest are sent. | wire | serialization, batch | `Mgx.IntegrationTests/Serialization/WireBodyTests.cs` | -| M365DSC-5354 | Microsoft365DSC | NotFound | Path segments carrying spaces, apostrophes, percent signs and non-ASCII survive into the request path unaltered. | wire | URI | `Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs`, `Mgx.IntegrationTests/TestSetup/HostileInputs.cs` (`PathSegments`) | -| M365DSC-7198 | Microsoft365DSC | NotFound | A failed batch item is an error the pipeline can see: it lands in `$Error`, counts for `-ErrorVariable`, and stops the pipeline under `-ErrorAction Stop`, with or without a dead-letter file. Results from chunks already applied are not lost when a later chunk fails. | wire | batch | `Mgx.IntegrationTests/Cmdlets/BatchErrorSurfacingTests.cs`, `Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs`, `Mgx.IntegrationTests/Cmdlets/ErrorActionMatrixTests.cs` (batch cells) | - -## Partly covered - -| Issue | Source | Class | Guarantee | Test type | Subsystem | Covered now / still planned | -| --- | --- | --- | --- | --- | --- | --- | -| M365DSC-4426 | Microsoft365DSC | Authentication | A 401 is classified as authentication rather than retried as transient, and a changed auth context re-keys the cached client instead of reusing the old token. | wire, unit | auth | Covered: classification and the fingerprint, in `Mgx.IntegrationTests/Engine/Errors/ErrorClassifierTests.cs`, `Mgx.IntegrationTests/Engine/Errors/ErrorPolicyParityTests.cs`, `Mgx.IntegrationTests/Cmdlets/Base/AuthFingerprintTests.cs`, `Mgx.IntegrationTests/Cmdlets/ResilienceInjectionScenarioTests.cs`. Planned: the mid-run expiry-and-reconnect drill - 2.1.5 fault injection. | -| GraphSDK-2148 | Graph PowerShell SDK | Permanent | mgx keeps working whatever else the session has loaded and in whatever order: no assembly-binding failure, no accidental SDK dependency. | wire | module load | Covered: the in-process half - injection, re-injection and teardown against a stand-in GraphSession, in `Mgx.IntegrationTests/Cmdlets/ResilienceInjectionScenarioTests.cs`. Covered: the load-order permutations, one child pwsh per permutation against installed Az.Accounts, PnP.PowerShell and ExchangeOnlineManagement, in `tests/Ecosystem/Mgx.Ecosystem.Tests.ps1`. | - -## Planned - -| Issue | Source | Class | Guarantee | Test type | Subsystem | Brought by | -| --- | --- | --- | --- | --- | --- | --- | -| M365DSC-7010 | Microsoft365DSC | Consistency | A read that immediately follows a write tolerates Graph's replication delay instead of reporting the object missing. | wire | consistency | 2.3.0. `MgxErrorClass.Consistency` is declared and mapped; nothing produces it yet. | - -## Unmapped - -| Issue | Source | Why it is here | -| --- | --- | --- | -| GraphSDK-2644 | Graph PowerShell SDK | No test claims this id and no failure class is recorded for it. The source issue needs reading before a row can say what it guarantees. Left in the index so the gap stays visible rather than disappearing with the row. | - -## Adding an entry - -A new provenance id gets a row here and a `(Corpus: , .)` line in the summary of -the test class that holds it, so the mapping reads the same from either end. A row whose -guarantee has no test says which release is meant to bring it. - -The mapping is guarded one way only, by -`tests/Mgx.IntegrationTests/TestSetup/RegressionCorpusIndexTests.cs`: every issue id written -anywhere under `tests/` has to appear here. The reverse cannot be guarded, because a row that -names the release bringing its test is exactly a row with no file to point at. diff --git a/tests/Ecosystem/Mgx.Ecosystem.Tests.ps1 b/tests/Ecosystem/Mgx.Ecosystem.Tests.ps1 index 9254326..755feb2 100644 --- a/tests/Ecosystem/Mgx.Ecosystem.Tests.ps1 +++ b/tests/Ecosystem/Mgx.Ecosystem.Tests.ps1 @@ -5,7 +5,7 @@ assemblies the neighboring modules already loaded, and teardown leaves nothing armed. A real Graph call under load order stays with tests/Live and the benchmarks. - Provenance: GraphSDK-2148 (module isolation) - see tests/CORPUS.md. Az.Accounts ships its + Provenance: GraphSDK-2148 (module isolation) - see tmp/CORPUS.md (private). Az.Accounts ships its own System.Text.Json; the interesting failure is a TypeLoadException the first time an engine type JITs, which is why the option round-trip and telemetry calls are the probe. diff --git a/tests/Live/Mgx.Live.Tests.ps1 b/tests/Live/Mgx.Live.Tests.ps1 index 7004ed0..629dd30 100644 --- a/tests/Live/Mgx.Live.Tests.ps1 +++ b/tests/Live/Mgx.Live.Tests.ps1 @@ -54,6 +54,30 @@ Describe 'Invoke-MgxRequest' { (Invoke-MgxRequest /users -All -Top $script:Slice -Property id | Measure-Object).Count | Should -Be $script:Slice } + It 'reaches the beta endpoint under -ApiVersion beta' { + (Invoke-MgxRequest /users -ApiVersion beta -Top $script:SmallSlice -Property id | Measure-Object).Count | + Should -Be $script:SmallSlice + } + It 'stamps each entity of an {id} fan-out with the id it came from' { + # The only thing that says which piped id produced which object once the fan-out has + # interleaved them. Nothing else in the suite asserts the property exists. + $ids = @(Invoke-MgxRequest /users -Top $script:SmallSlice -Property id | ForEach-Object { $_.id }) + $seen = @($ids | Invoke-MgxRequest '/users/{id}' -Property id | ForEach-Object { $_['_MgxSourceId'] }) + Compare-Object ($ids | Sort-Object) ($seen | Sort-Object) | Should -BeNullOrEmpty + } + It 'refuses an {id} Uri with nothing piped to it' { + { Invoke-MgxRequest '/users/{id}' } | Should -Throw '*pipeline*' + } + It 'returns the user the SDK returns for the same id' { + # Read against the SDK rather than against itself: same tenant, same id, same three + # fields. A mock can agree with whatever the test author wrote; the directory cannot. + $id = (Invoke-MgxRequest '/users?$top=1' -Property id -WarningAction SilentlyContinue).id + $mgx = Invoke-MgxRequest "/users/$id" -Property id,displayName,userPrincipalName + $mg = Get-MgUser -UserId $id -Property Id,DisplayName,UserPrincipalName + $mgx.id | Should -Be $mg.Id + $mgx.displayName | Should -Be $mg.DisplayName + $mgx.userPrincipalName | Should -Be $mg.UserPrincipalName + } } Describe 'Invoke-MgxBatchRequest' { diff --git a/tests/Mgx.IntegrationTests/Cmdlets/Base/TypeCacheInvalidationTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/Base/TypeCacheInvalidationTests.cs index 9b40d21..66101e0 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/Base/TypeCacheInvalidationTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/Base/TypeCacheInvalidationTests.cs @@ -98,6 +98,30 @@ public void A_second_import_leaves_one_subscription_for_a_removal_to_detach() } } + /// + /// The dependency resolver is the other subscription an import makes, and += is no more + /// idempotent there. A second one survives the -= a removal does and goes on answering + /// dependency loads out of a Dependencies folder belonging to a module the session removed. + /// + [Fact] + public void A_second_import_leaves_one_dependency_resolver_for_a_removal_to_detach() + { + try + { + new AlcInitializer().OnImport(); + new AlcInitializer().OnImport(); + Assert.Equal(1, ResolverSubscriptions()); + + // The removal itself, module argument and all - OnRemove never reads it. + new AlcInitializer().OnRemove(null!); + Assert.Equal(0, ResolverSubscriptions()); + } + finally + { + new AlcInitializer().OnImport(); + } + } + /// Whether anything this test resolved, or wrote itself, is still cached. private static bool AnyStillCached() => Resolved.Any(MgxCmdletBase.s_typeCache.ContainsKey); @@ -114,6 +138,21 @@ private static int InvalidationSubscriptions() return subscribers?.GetInvocationList().Count(d => d.Method.DeclaringType == typeof(MgxCmdletBase)) ?? 0; } + /// + /// How many times mgx's dependency resolver is subscribed to the default load context. + /// AssemblyLoadContext keeps its Resolving subscribers in a private instance field, and that + /// invocation list is the only place the count shows: a second subscription is silent until a + /// removal detaches one and the other stays. + /// + private static int ResolverSubscriptions() + { + var field = typeof(AssemblyLoadContext).GetField( + "_resolving", BindingFlags.Instance | BindingFlags.NonPublic); + Assert.True(field != null, "AssemblyLoadContext no longer keeps its Resolving subscribers in _resolving"); + var subscribers = (Delegate?)field!.GetValue(AssemblyLoadContext.Default); + return subscribers?.GetInvocationList().Count(d => d.Method.DeclaringType == typeof(AlcInitializer)) ?? 0; + } + /// /// Loads an assembly, which is what invalidation keys off. A dynamic one raises the event /// exactly as a file-backed load does, and leaves nothing behind that the rest of the suite diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DeltaCheckpointIntegrityTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DeltaCheckpointIntegrityTests.cs index 4cc8c96..dac7056 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/DeltaCheckpointIntegrityTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/DeltaCheckpointIntegrityTests.cs @@ -31,6 +31,16 @@ public class DeltaCheckpointIntegrityTests """; private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + private const string EmptyChangesPage = """ + {"value":[],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B3"} + """; + private const string ChangesPage3Linked = """ + {"value":[{"id":"b4"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B4"} + """; + private const string EmptyFinalPage = """ + {"value":[],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D2"} + """; + private static PowerShell Shell() { var ps = PowerShell.Create(); @@ -44,6 +54,38 @@ private static PowerShell Shell() return ps; } + /// + /// The shape a delta enumeration is full of: pages behind nextLinks with nothing in them, + /// including the last one before the deltaLink. PageIterator raises its empty-page limit to + /// 1000 for exactly this, so the sync walks them rather than giving up. Every answer takes + /// a real asynchronous hop - a socket read always yields and Task.FromResult never does, + /// and the thread the iterator resumes on is what these pages are here to settle. + /// + private sealed class EmptyMiddlePageHandler : HttpMessageHandler + { + private readonly object _lock = new(); + public List Urls { get; } = []; + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + lock (_lock) { Urls.Add(url); } + + await Task.Yield(); + + var body = url.Contains("skiptoken=B4", StringComparison.Ordinal) ? EmptyFinalPage + : url.Contains("skiptoken=B3", StringComparison.Ordinal) ? ChangesPage3Linked + : url.Contains("skiptoken=B2", StringComparison.Ordinal) ? EmptyChangesPage + : ChangesPage1; + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") + }; + } + } + private sealed record Env(string Dir, string DeltaPath, string CheckpointPath, string OutputPath); private static Env NewEnv(string checkpointName = "run.checkpoint") @@ -92,6 +134,22 @@ private static void SyncToPipeline(Env env) catch (CmdletInvocationException) { } } + /// + /// A sync whose warnings and error records the caller reads. Nothing is swallowed here: a + /// run that dies off the pipeline thread has to fail the test that says it does not. + /// + private static (List Warnings, int Errors) SyncReporting(Env env) + { + using var ps = Shell(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", env.DeltaPath) + .AddParameter("CheckpointPath", env.CheckpointPath) + .AddParameter("OutputFile", env.OutputPath); + ps.Invoke(); + return ([.. ps.Streams.Warning.Select(w => w.Message)], ps.Streams.Error.Count); + } + /// A checkpoint torn between the write and the rename: no position in it at all. private const string TornCheckpoint = """{"resource":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D1","nextLi"""; @@ -662,4 +720,52 @@ public void A_failed_promotion_keeps_the_checkpoint_and_the_temp_holding_the_syn } finally { try { Directory.Delete(env.Dir, true); } catch { } } } + + /// + /// A checkpoint the sync cannot write, over pages that are empty behind nextLinks. The + /// page-boundary callback runs on whichever thread the iterator resumed on, and a page that + /// writes nothing resumes on the thread pool - so warning about the disk from there ended + /// the sync at that page: the items behind it were never fetched, and the delta token never + /// advanced, leaving the caller to re-enumerate from the old one. The message is buffered + /// on the client's warning channel and written by a drain on the pipeline thread: one per + /// failed save, and the enumeration runs to the deltaLink. + /// + /// The last save fails at the boundary before the empty final page, so no item follows it + /// and the per-item drain never comes round again. The drain on the way out is what carries + /// that one. + /// + /// + [Fact] + public void A_failed_boundary_save_warns_and_the_sync_goes_on() + { + var env = NewEnv(); + try + { + // Every save into this path fails: Save stages its bytes under this name first, and + // no account writes a file over a directory. The directory holding it is not sealed + // instead, the way a read-only disk would seal it, because the sync writes its + // output and its delta state there too and neither write is what is under test. + Directory.CreateDirectory(env.CheckpointPath + ".tmp"); + + var handler = new EmptyMiddlePageHandler(); + using var transport = MgxTransportScope.Inject(handler); + var (warnings, errors) = SyncReporting(env); + + Assert.Equal(0, errors); + + // Each of the three boundaries carrying a nextLink tries to save, and each fails. + Assert.Equal(3, warnings.Count( + w => w.StartsWith("Checkpoint save failed (page boundary)", StringComparison.Ordinal))); + + // The empty page sits behind the second request, so a sync that died reporting its + // failed save never asks for the rest - where the items and the token both are. + Assert.Equal(4, handler.Urls.Count); + Assert.Contains("skiptoken=B4", handler.Urls[3]); + Assert.Equal( + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b4\"}"], + Ids(env.OutputPath)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(env.DeltaPath)!.DeltaLink); + } + finally { try { Directory.Delete(env.Dir, true); } catch { } } + } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DeltaQueryTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DeltaQueryTests.cs index a3225f6..5d31c39 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/DeltaQueryTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/DeltaQueryTests.cs @@ -1020,9 +1020,13 @@ public void Cmdlet_OutputFile_TempFileCleanedUpOnError() // Output file should NOT exist (error before atomic rename) Assert.False(File.Exists(outputPath), "Output file should not exist after error"); - // No .tmp files should remain + // No .tmp of this run's should remain. Named for the output this test generated, + // not for the prefix every process running the suite shares: the temps a sync + // leaves are "{output}.{guid}.tmp", so that name picks out this invocation's and + // nothing else's, and a sibling run's in-flight temp in the shared directory is + // another test's business. var dir = Path.GetDirectoryName(outputPath)!; - var tmpFiles = Directory.GetFiles(dir, "*.tmp").Where(f => f.Contains("cmdlet-tempclean")).ToArray(); + var tmpFiles = Directory.GetFiles(dir, Path.GetFileName(outputPath) + ".*.tmp"); Assert.Empty(tmpFiles); // Should have errors Assert.True(ps.HadErrors, "Should have errors from 500 response"); diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DeltaRecoveryTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DeltaRecoveryTests.cs index b8d65d8..9437d1a 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/DeltaRecoveryTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/DeltaRecoveryTests.cs @@ -325,11 +325,12 @@ public void Latest_is_ignored_while_a_resume_checkpoint_exists() } /// - /// A resume opens the output itself for append, and a denying ACL or read-only bit there - /// raises UnauthorizedAccessException - which does not derive from IOException, so a - /// handler catching only the latter turns an ordinary permission failure into an unhandled - /// error. It must surface the same way an unwritable path does everywhere else: as an - /// AccessDenied error record. + /// A resume appends into the output itself, so a denying ACL or read-only bit there is this + /// run's problem before any request is made: the claim on the file the checkpoint counts its + /// items into fails on permissions, and the run ends on an error record naming that file and + /// the two ways out of it. What it may never be is an unhandled failure - + /// UnauthorizedAccessException does not derive from IOException, and a handler catching only + /// the latter let an ordinary permission problem out of the cmdlet as one. /// [Fact] public void A_denied_output_ends_the_resume_as_an_error_record_not_an_unhandled_failure() @@ -372,11 +373,23 @@ public void A_denied_output_ends_the_resume_as_an_error_record_not_an_unhandled_ .AddParameter("DeltaPath", deltaPath) .AddParameter("CheckpointPath", checkpointPath) .AddParameter("OutputFile", outputPath); - var escaped = Record.Exception(() => ps.Invoke()); - - Assert.Null(escaped); - Assert.Contains(ps.Streams.Error, - e => e.FullyQualifiedErrorId.StartsWith("AccessDenied", StringComparison.Ordinal)); + // Only the wrapper a terminating error arrives in is caught here. Any other type + // escaping this call is the unhandled failure this is about, and fails the test. + List errors = []; + try { ps.Invoke(); } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.PermissionDenied, stop.CategoryInfo.Category); + Assert.Contains($"'{outputPath}' could not be opened for writing:", + stop.Exception.Message); + Assert.Contains( + "Grant write access to that file and run again to resume from it, or remove it " + + "and the checkpoint to sync afresh.", + stop.Exception.Message); } finally { diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DeltaSharedCheckpointTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DeltaSharedCheckpointTests.cs index 0e8d2fe..c7de015 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/DeltaSharedCheckpointTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/DeltaSharedCheckpointTests.cs @@ -1,3 +1,4 @@ +using System.Reflection; using System.Management.Automation; using System.Net; using System.Text; @@ -21,6 +22,16 @@ public class DeltaSharedCheckpointTests private const string ChangesPage2 = """ {"value":[{"id":"b3"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D2"} """; + // One page and no delta token, so a run that reaches it finishes without saving any state - + // and the run after it builds exactly the request URL this one did. + private const string ChangesPageNoToken = """ + {"value":[{"id":"b9"}]} + """; + // What "$deltatoken=latest" answers: nothing to return now, and a token to return from next + // time. + private const string LatestBaseline = """ + {"value":[],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D9"} + """; private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; private const string GroupsPage = """ @@ -62,6 +73,113 @@ private static void UsersState(string deltaPath) => ApiVersion = "v1.0", }.Save(deltaPath); + /// + /// Runs a sync on this thread and parks it inside the window the hold exists to close: past + /// the reconcile, which has claimed the output and cut it back, and short of the writer that + /// used to open a handle of its own a page fetch later. The verbose the resume announces + /// itself with is written from exactly there, and a subscriber to that stream blocks the + /// pipeline where it is raised. + /// + private static (string[] Warnings, ErrorRecord[] Errors) SyncParkedAtResume( + string deltaPath, string checkpointPath, string outputPath, + ManualResetEventSlim arrived, ManualResetEventSlim released) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + ps.Streams.Verbose.DataAdding += (_, e) => + { + if (e.ItemAdded is VerboseRecord v + && v.Message.Contains("Resuming delta enumeration", StringComparison.Ordinal) + && !arrived.IsSet) + { + arrived.Set(); + released.Wait(); + } + }; + + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("OutputFile", outputPath) + .AddParameter("Verbose"); + List errors = []; + try { ps.Invoke(); } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + return ([.. ps.Streams.Warning.Select(w => w.Message)], [.. errors]); + } + + /// + /// A sync built and waiting to be let go: the runspace is up and the module imported before + /// is set, so what releases is two runs + /// arriving at the reconcile together rather than two runspaces starting up. + /// + private static (string[] Warnings, ErrorRecord[] Errors) SyncReleasedWith(string deltaPath, + string checkpointPath, string outputPath, + ManualResetEventSlim ready, ManualResetEventSlim go) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("OutputFile", outputPath); + + ready.Set(); + go.Wait(); + List errors = []; + try { ps.Invoke(); } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + return ([.. ps.Streams.Warning.Select(w => w.Message)], [.. errors]); + } + + /// + /// Answers by URL rather than from a queue, and holds the first request that arrives until it + /// is let go. Two runs racing over one command line each get the page they ask for however + /// they interleave, and the one that reaches the wire - which is the one that got past the + /// reconcile - keeps the output for as long as the test needs it to. + /// + private sealed class GatedChangesByUrlHandler : HttpMessageHandler + { + private int _served; + + /// How many requests this handler has answered or is holding. + internal int Served => Volatile.Read(ref _served); + + /// Set the moment the first request arrives. + internal ManualResetEventSlim Arrived { get; } = new(false); + + /// Set by the test to let the parked request answer. + internal ManualResetEventSlim Released { get; } = new(false); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + if (Interlocked.Increment(ref _served) == 1) + { + Arrived.Set(); + Released.Wait(cancellationToken); + } + var body = request.RequestUri!.ToString() + .Contains("skiptoken=B2", StringComparison.Ordinal) ? ChangesPage2 : ChangesPage1; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } + private static void Sync(string deltaPath, string checkpointPath, string outputPath, string uri = "/users/delta") { @@ -80,8 +198,98 @@ private static void Sync(string deltaPath, string checkpointPath, string outputP catch (CmdletInvocationException) { } } + /// + /// The same sync with the error stream captured too. A run that will not touch the two files + /// the checkpoint stands for ends on a terminating error, and the id and the sentence are + /// the whole of what the caller has to act on. Both places PowerShell can put that record + /// are read, since which one it uses depends on how the command was added. + /// + private static (string[] Warnings, ErrorRecord[] Errors) SyncRun(string deltaPath, + string checkpointPath, string outputPath, bool latest = false) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("OutputFile", outputPath); + if (latest) ps.AddParameter("Latest"); + List errors = []; + try { ps.Invoke(); } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + return ([.. ps.Streams.Warning.Select(w => w.Message)], [.. errors]); + } + + /// + /// Everything a run that stops may not have moved: both state files, the output if there is + /// one, the temp the checkpoint names, and the directory's own entry list - which is what + /// says no temp of the stopped run's, and no probe of its own, outlived it. + /// + private sealed class DiskState + { + public byte[] Checkpoint = []; + public byte[] Temp = []; + public byte[]? Output; + public byte[]? Delta; + public string[] Entries = []; + } + + private static DiskState ReadDisk(string dir, string checkpointPath, string temp, + string outputPath, string deltaPath) => new() + { + Checkpoint = File.ReadAllBytes(checkpointPath), + Temp = ReadShared(temp), + Output = File.Exists(outputPath) ? ReadShared(outputPath) : null, + Delta = File.Exists(deltaPath) ? File.ReadAllBytes(deltaPath) : null, + Entries = [.. Directory.GetFiles(dir).Select(f => Path.GetFileName(f)) + .OrderBy(n => n, StringComparer.Ordinal)], + }; + + /// + /// The same, for the shape a resumed sync leaves: its changes are in the output itself and + /// the checkpoint names no temp, so there is no third file to read. + /// + private static DiskState ReadDisk(string dir, string checkpointPath, string outputPath, + string deltaPath) => new() + { + Checkpoint = File.ReadAllBytes(checkpointPath), + Output = ReadShared(outputPath), + Delta = File.Exists(deltaPath) ? File.ReadAllBytes(deltaPath) : null, + Entries = [.. Directory.GetFiles(dir).Select(f => Path.GetFileName(f)) + .OrderBy(n => n, StringComparer.Ordinal)], + }; + + private static void AssertUnmoved(DiskState before, DiskState after) + { + Assert.Equal(before.Entries, after.Entries); + Assert.Equal(before.Checkpoint, after.Checkpoint); + Assert.Equal(before.Temp, after.Temp); + Assert.Equal(before.Output, after.Output); + Assert.Equal(before.Delta, after.Delta); + } + + /// + /// Reads a file a live run still holds open, offering the sharing that holder asked for. + /// Windows checks the sharing both ways round - what the reader asks of the file, and what + /// it will let a handle already on it go on doing - so File.ReadAllBytes, which asks for + /// FileShare.Read and nothing more, is refused by a run holding the file to write it. + /// + private static byte[] ReadShared(string path) + { + using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + using var ms = new MemoryStream(); + fs.CopyTo(ms); + return ms.ToArray(); + } + private static string[] SyncWarnings(string deltaPath, string checkpointPath, string outputPath, - string uri = "/users/delta") + string uri = "/users/delta", bool latest = false, + string[]? property = null, string[]? prefer = null, string? filter = null) { using var ps = PowerShell.Create(); ps.AddCommand("Import-Module") @@ -93,6 +301,10 @@ private static string[] SyncWarnings(string deltaPath, string checkpointPath, st .AddParameter("DeltaPath", deltaPath) .AddParameter("CheckpointPath", checkpointPath) .AddParameter("OutputFile", outputPath); + if (latest) ps.AddParameter("Latest"); + if (property != null) ps.AddParameter("Property", property); + if (prefer != null) ps.AddParameter("Prefer", prefer); + if (filter != null) ps.AddParameter("Filter", filter); try { ps.Invoke(); } catch (CmdletInvocationException) { } return [.. ps.Streams.Warning.Select(w => w.Message)]; @@ -409,53 +621,54 @@ public void A_refused_checkpoints_temp_survives_the_run_that_refused_it() } /// - /// How far "spare the temps a refused checkpoint names" reaches. The sweep only ever - /// deletes names a run gives its own temp - the output's name, a dot, 32 hex digits, - /// ".tmp" - so a refused checkpoint naming "out.jsonl.{guid}.tmp" beside an output called - /// "out" names a file this sweep could not touch either way. Skipping on it bought that - /// file nothing and cost this output its own orphans, which the pre-length adoption path - /// then picks up on a line count alone. + /// The refusal's promise where the run that made it goes on to finish. A sync that refused + /// the checkpoint it found and then completed without ever saving one of its own - one page + /// was all it had - deleted the file it had warned a moment earlier it would leave alone, + /// so the temp it had spared the sweep was orphaned and the sync that owns both came back + /// to a position that was no longer there and re-enumerated from the beginning. /// [Fact] - public void A_refused_temp_this_sweep_could_never_reach_does_not_spare_this_outputs_orphans() + public void A_completed_sync_that_saved_no_checkpoint_leaves_the_refused_one_standing() { var dir = Directory.CreateDirectory( - Path.Combine(Path.GetTempPath(), $"mgx-delta-sweep-{Guid.NewGuid():N}")).FullName; + Path.Combine(Path.GetTempPath(), $"mgx-delta-donerefused-{Guid.NewGuid():N}")).FullName; var checkpointPath = Path.Combine(dir, "run.checkpoint"); var deltaA = Path.Combine(dir, "stateA.json"); var deltaB = Path.Combine(dir, "stateB.json"); - var mine = Path.Combine(dir, "out"); - var theirs = Path.Combine(dir, "out.jsonl"); + var output = Path.Combine(dir, "out.jsonl"); var handler = new MockHttpHandler(); handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); using var transport = MgxTransportScope.Inject(handler); try { - // The sync into "out" collects page one into its temp and dies on page two. + // Sync A dies on page 2: its two items are in its temp, and the checkpoint names it. handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); - Sync(deltaA, checkpointPath, mine); - var orphan = Assert.Single(Directory.GetFiles(dir, "out.*.tmp")); + Sync(deltaA, checkpointPath, output); - // The sync next door takes the shared checkpoint over: it refuses what it finds, - // saves its own position at its first page boundary, and dies on page two. The - // first sync's temp is now an orphan - nothing on disk describes it. - handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); - Sync(deltaB, checkpointPath, theirs); - var theirTemp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + var cp = PaginationCheckpoint.Load(checkpointPath)!; + var temp = Path.Combine(dir, cp.TempFile!); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(temp)); + var before = File.ReadAllBytes(checkpointPath); - // "out" again. It refuses a checkpoint naming a temp beside a different output, - // and that name must not stand in the way of its own sweep. - handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); - handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); - Sync(deltaA, checkpointPath, mine); + // Sync B: another collection over the same -CheckpointPath and -OutputFile. It + // refuses what it finds and then finishes in one page, so no boundary of its own + // is ever reached and nothing of this run's is written over that path. + handler.QueueResponse(HttpStatusCode.OK, GroupsPage); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.Equal(["{\"id\":\"g1\"}"], File.ReadAllLines(output)); + Assert.True(File.Exists(checkpointPath), + "the run that refused the checkpoint deleted it on the way out"); + Assert.Equal(before, File.ReadAllBytes(checkpointPath)); + Assert.True(File.Exists(temp), + "the refused checkpoint's items outlived the checkpoint that counted them"); + // A, back where it left off, rather than at page one. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + Sync(deltaA, checkpointPath, output); Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], - File.ReadAllLines(mine)); - Assert.False(File.Exists(orphan), - "a temp name the sweep could never reach suppressed this output's sweep"); - Assert.True(File.Exists(theirTemp), - "the refused checkpoint's own staging file was deleted"); + File.ReadAllLines(output)); } finally { @@ -464,21 +677,20 @@ public void A_refused_temp_this_sweep_could_never_reach_does_not_spare_this_outp } /// - /// What each refusal says. A checkpoint recording another sync's output is a second sync - /// over one -CheckpointPath, and naming that is what the caller acts on. One recording no - /// output at all is not: it was believed while the files beside it corroborated it, and a - /// single sync whose temp has since gone, or whose output has been replaced, reaches the - /// same refusal. Every release before this one recorded no output, so that is the ordinary - /// upgrade path - and it was told there was a second sync somewhere and to give each its - /// own -CheckpointPath, which fixes nothing it has. + /// And where the refusal has already been voided. This run reached a page boundary, which + /// saved its own position over the same path - so the file the success delete reaches is + /// this run's own, describing an enumeration that has just finished, and leaving it costs + /// the next -Latest run the baseline it asks for. Taking the path over is what a second + /// sync sharing it gets; the delete is not what makes that so. /// [Fact] - public void A_refusal_reports_the_cause_it_can_show() + public void A_completed_sync_that_wrote_over_the_refused_position_still_deletes_it() { var dir = Directory.CreateDirectory( - Path.Combine(Path.GetTempPath(), $"mgx-delta-refusal-{Guid.NewGuid():N}")).FullName; + Path.Combine(Path.GetTempPath(), $"mgx-delta-donetookover-{Guid.NewGuid():N}")).FullName; var checkpointPath = Path.Combine(dir, "run.checkpoint"); - var deltaPath = Path.Combine(dir, "state.json"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); var output = Path.Combine(dir, "out.jsonl"); var handler = new MockHttpHandler(); @@ -486,40 +698,21 @@ public void A_refusal_reports_the_cause_it_can_show() using var transport = MgxTransportScope.Inject(handler); try { - File.WriteAllLines(output, OldSync); - - // The shape an interrupted fresh sync left before outputs were recorded, whose temp - // is no longer beside it. - new PaginationCheckpoint - { - Resource = "https://graph.microsoft.com/v1.0/users/delta?$top=999", - NextLink = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2", - ItemsCollected = 2, - PageItemsAlreadyWritten = 0, - TempFile = $"out.jsonl.{Guid.NewGuid():N}.tmp", - OutputFile = null, - DataLength = 24, - }.Save(checkpointPath); - - var uncorroborated = SyncWarnings(deltaPath, checkpointPath, output); - Assert.Contains(uncorroborated, w => w.Contains("no longer corroborate")); - Assert.DoesNotContain(uncorroborated, - w => w.Contains("records an enumeration this run cannot resume from")); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; - // The same position, recording an output this run is not collecting into. - new PaginationCheckpoint - { - Resource = "https://graph.microsoft.com/v1.0/users/delta?$top=999", - NextLink = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2", - ItemsCollected = 2, - PageItemsAlreadyWritten = 0, - TempFile = null, - OutputFile = Path.Combine(dir, "elsewhere.jsonl"), - DataLength = 24, - }.Save(checkpointPath); + // Two pages, so the first boundary saves this run's position over the refused one. + handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); + handler.QueueResponse(HttpStatusCode.OK, GroupsResync); + Sync(deltaB, checkpointPath, output, "/groups/delta"); - Assert.Contains(SyncWarnings(deltaPath, checkpointPath, output), - w => w.Contains("records an enumeration this run cannot resume from")); + Assert.Equal(["{\"id\":\"g1\"}", "{\"id\":\"g2\"}", "{\"id\":\"g9\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpointPath), + "a completed sync left its own position into a finished enumeration behind"); + Assert.True(File.Exists(Path.Combine(dir, theirs)), + "the refused checkpoint's staging file was swept by the run that refused it"); } finally { @@ -528,19 +721,21 @@ public void A_refusal_reports_the_cause_it_can_show() } /// - /// And the cause it cannot show. A run interrupted partway through an incremental sync - /// leaves a checkpoint recording the delta URL it was enumerating. Lose the state under it - - /// corrupt, deleted by hand, a -DeltaPath moved - and the same command line comes back - /// building a full-sync URL, which is not that one, so it refuses its own earlier position. - /// The refusal is right; the reading was not. It named a different sync, over a - /// -CheckpointPath no second sync has ever touched, and told the caller to give each of them - /// its own - which is the arrangement they were already running. + /// The refusal that is not an ownership verdict at all, and what a sync can do about it: + /// nothing. A checkpoint refused because the temp it names could not be claimed counts rows + /// that are in that temp and in no other file, and it is the only thing on disk that counts + /// them. Every way of going on takes them: the sweep at the top of the attempt reclaims the + /// temp the moment its holder is gone, the first page boundary saves this run's position + /// over the checkpoint, the completion door then deletes what it finds there as its own, and + /// a page carrying a deltaLink moves the token past changes held in that temp alone. So the + /// run ends where it finds out, with one page's worth of work in front of it and none of it + /// done. /// [Fact] - public void A_refusal_does_not_name_a_second_sync_it_cannot_see() + public void A_held_temp_stops_the_sync_before_anything_is_written() { var dir = Directory.CreateDirectory( - Path.Combine(Path.GetTempPath(), $"mgx-delta-selfrefusal-{Guid.NewGuid():N}")).FullName; + Path.Combine(Path.GetTempPath(), $"mgx-delta-heldtemp-{Guid.NewGuid():N}")).FullName; var checkpointPath = Path.Combine(dir, "run.checkpoint"); var deltaPath = Path.Combine(dir, "state.json"); var output = Path.Combine(dir, "out.jsonl"); @@ -550,25 +745,53 @@ public void A_refusal_does_not_name_a_second_sync_it_cannot_see() using var transport = MgxTransportScope.Inject(handler); try { - // One sync, one -CheckpointPath: interrupted on page 2 of its incremental pass, so - // what it leaves records the delta URL it was reading. - UsersState(deltaPath); + // A sync dies on page 2: its two items are in its temp, and the checkpoint names + // both files. No token was saved, so every run below builds the same request URL + // and the checkpoint is this command line's own by every test there is. handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); Sync(deltaPath, checkpointPath, output); - Assert.Contains("$deltatoken=D0", PaginationCheckpoint.Load(checkpointPath)!.Resource); - // The state goes, and the same command line enumerates in full. - File.Delete(deltaPath); - handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); - handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); - var refusal = Assert.Single( - SyncWarnings(deltaPath, checkpointPath, output), - w => w.Contains(checkpointPath)); + var temp = Path.Combine(dir, PaginationCheckpoint.Load(checkpointPath)!.TempFile!); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(temp)); - Assert.Contains("records an enumeration this run cannot resume from", refusal); - Assert.Contains("either another sync's, sharing this -CheckpointPath", refusal); - Assert.Contains("this sync's own from a pass it no longer makes", refusal); - Assert.DoesNotContain("belongs to a different sync", refusal); + // The next run finds that temp held open - by another process, not necessarily + // another sync - with one page of changes waiting for it on the wire. + handler.QueueResponse(HttpStatusCode.OK, ChangesPageNoToken); + string[] warnings; + ErrorRecord[] errors; + DiskState before, after; + int requestsBefore; + using (new FileStream(temp, FileMode.Open, FileAccess.ReadWrite, FileShare.Read)) + { + before = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + requestsBefore = handler.RequestCount; + (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + after = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + } + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointTempHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Contains("Another sync is still writing the temp file", + stop.Exception.Message); + Assert.Contains("the 2 items it records are that run's", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Wait for that sync to finish, or give this one its own -OutputFile and " + + "-CheckpointPath.", + stop.Exception.Message); + + // Nothing that promises to leave the files alone and then goes on to replace them. + Assert.DoesNotContain(warnings, w => w.Contains("re-enumerates")); + Assert.DoesNotContain(warnings, w => w.Contains("Both files are left as they are")); + + // And nothing ran: no request, no output, no token, and both files exactly as the + // holder left them - down to the directory's own entry list, so no temp of this + // run's and no probe survives it either. + Assert.Equal(requestsBefore, handler.RequestCount); + AssertUnmoved(before, after); + Assert.False(File.Exists(output), "the stopped run wrote an output"); + Assert.Null(DeltaState.Load(deltaPath)); } finally { @@ -577,21 +800,18 @@ public void A_refusal_does_not_name_a_second_sync_it_cannot_see() } /// - /// A 410 says the token this run holds is dead. It says nothing about a checkpoint this - /// same run has already decided belongs to another sync, and deleting that one took the - /// other sync's position away seconds after warning that it would be left alone - while the - /// retry, running the stale-temp sweep a second time with the refusal forgotten, took the - /// temp holding its items too. Both halves of the other sync's progress, on the one path - /// that promised to touch neither. + /// The same disk with two pages on the wire instead of one, which is the shape that made + /// sparing the file pointless: a refusing run that reaches one page boundary saves its own + /// position over the very path it has just promised to leave, and the door it walks out of + /// deletes that as its own. A run that stops has no page boundary to reach. /// [Fact] - public void An_expired_token_does_not_delete_the_checkpoint_this_run_refused() + public void A_held_temp_stops_the_sync_that_would_have_paged_twice() { var dir = Directory.CreateDirectory( - Path.Combine(Path.GetTempPath(), $"mgx-delta-410refused-{Guid.NewGuid():N}")).FullName; + Path.Combine(Path.GetTempPath(), $"mgx-delta-heldtemp2p-{Guid.NewGuid():N}")).FullName; var checkpointPath = Path.Combine(dir, "run.checkpoint"); - var deltaA = Path.Combine(dir, "stateA.json"); - var deltaB = Path.Combine(dir, "stateB.json"); + var deltaPath = Path.Combine(dir, "state.json"); var output = Path.Combine(dir, "out.jsonl"); var handler = new MockHttpHandler(); @@ -599,34 +819,32 @@ public void An_expired_token_does_not_delete_the_checkpoint_this_run_refused() using var transport = MgxTransportScope.Inject(handler); try { - // Sync A dies on page 2: its two items are in its temp, and the checkpoint names - // both files. handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); - Sync(deltaA, checkpointPath, output); + Sync(deltaPath, checkpointPath, output); - var cp = PaginationCheckpoint.Load(checkpointPath)!; - var temp = Path.Combine(dir, cp.TempFile!); + var temp = Path.Combine(dir, PaginationCheckpoint.Load(checkpointPath)!.TempFile!); Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(temp)); - var before = File.ReadAllBytes(checkpointPath); - - // Sync B: another collection over the same -CheckpointPath and -OutputFile, holding - // a token Graph answers 410 for. It refuses A's checkpoint, re-syncs in full, and - // dies before a page boundary of its own. - StaleState(deltaB); - handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); - Sync(deltaB, checkpointPath, output, "/groups/delta"); - Assert.True(File.Exists(checkpointPath), - "the expired token deleted a checkpoint this run had refused as another sync's"); - Assert.Equal(before, File.ReadAllBytes(checkpointPath)); - Assert.True(File.Exists(temp), - "the retry swept away the temp the refusal a moment earlier had spared"); - - // A, back where it left off, rather than at the delta token. - handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); - Sync(deltaA, checkpointPath, output); - Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], - File.ReadAllLines(output)); + // Two pages: a first with a continuation, so the run would save a boundary + // checkpoint of its own, and a second that completes it. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + handler.QueueResponse(HttpStatusCode.OK, ChangesPageNoToken); + ErrorRecord[] errors; + DiskState before, after; + int requestsBefore; + using (new FileStream(temp, FileMode.Open, FileAccess.ReadWrite, FileShare.Read)) + { + before = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + requestsBefore = handler.RequestCount; + (_, errors) = SyncRun(deltaPath, checkpointPath, output); + after = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + } + + Assert.StartsWith("CheckpointTempHeld", Assert.Single(errors).FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(requestsBefore, handler.RequestCount); + AssertUnmoved(before, after); + Assert.Equal(2, PaginationCheckpoint.Load(checkpointPath)!.ItemsCollected); } finally { @@ -635,18 +853,19 @@ public void An_expired_token_does_not_delete_the_checkpoint_this_run_refused() } /// - /// And the run that refused still gets its own sync done: the 410 retry enumerates in full, - /// collects into the output and saves the token it was issued, with the other sync's temp - /// still where the refusal left it. + /// The shape the delta token turns into a permanent loss. A page carrying a deltaLink is a + /// completed enumeration, so a run that reaches one saves a token over the position the + /// held checkpoint records - and every later run of the same command line then builds a + /// request that checkpoint no longer matches and refuses it as another sync's, for good. A + /// run that stops saves no token, because it fetches no page. /// [Fact] - public void A_full_resync_after_a_refusal_still_completes_its_own_sync() + public void A_stopped_sync_saves_no_token_from_the_page_it_never_fetched() { var dir = Directory.CreateDirectory( - Path.Combine(Path.GetTempPath(), $"mgx-delta-410done-{Guid.NewGuid():N}")).FullName; + Path.Combine(Path.GetTempPath(), $"mgx-delta-heldtemptoken-{Guid.NewGuid():N}")).FullName; var checkpointPath = Path.Combine(dir, "run.checkpoint"); - var deltaA = Path.Combine(dir, "stateA.json"); - var deltaB = Path.Combine(dir, "stateB.json"); + var deltaPath = Path.Combine(dir, "state.json"); var output = Path.Combine(dir, "out.jsonl"); var handler = new MockHttpHandler(); @@ -655,18 +874,34 @@ public void A_full_resync_after_a_refusal_still_completes_its_own_sync() try { handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); - Sync(deltaA, checkpointPath, output); + Sync(deltaPath, checkpointPath, output); var temp = Path.Combine(dir, PaginationCheckpoint.Load(checkpointPath)!.TempFile!); - StaleState(deltaB); - handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); - handler.QueueResponse(HttpStatusCode.OK, GroupsPage); - Sync(deltaB, checkpointPath, output, "/groups/delta"); + // One page, and it finishes the enumeration: had the run taken it, the token would + // have moved past the changes in the temp beside it. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + ErrorRecord[] errors; + DiskState before, after; + using (new FileStream(temp, FileMode.Open, FileAccess.ReadWrite, FileShare.Read)) + { + before = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + (_, errors) = SyncRun(deltaPath, checkpointPath, output); + after = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + } - Assert.Equal(["{\"id\":\"g1\"}"], File.ReadAllLines(output)); - Assert.Contains("$deltatoken=G1", DeltaState.Load(deltaB)!.DeltaLink); - Assert.True(File.Exists(temp), - "the retry swept away the temp the refusal a moment earlier had spared"); + Assert.StartsWith("CheckpointTempHeld", Assert.Single(errors).FullyQualifiedErrorId, + StringComparison.Ordinal); + AssertUnmoved(before, after); + Assert.False(File.Exists(deltaPath), "a token was saved for a page nothing fetched"); + + // Which leaves the pair reachable. The holder dies, and the next run of the same + // command line is the one it was kept for. + var recovered = SyncWarnings(deltaPath, checkpointPath, output); + + Assert.Contains(recovered, w => w.Contains("Recovered 2 items")); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); } finally { @@ -675,23 +910,19 @@ public void A_full_resync_after_a_refusal_still_completes_its_own_sync() } /// - /// What the run that refused leaves beside its own output. The attempt a 410 sends round the - /// loop again keeps its temp - a checkpoint counting those items is on disk - and the retry - /// spares it, because the sweep is held off over every temp beside this output for as long - /// as the refusal stands. What ends the holding off was never written: the checkpoint is - /// replaced by the retry's, or deleted by the run that completed, without a word about the - /// file it used to name. So a sync that promoted its output and reported success left a - /// partial copy of the changes it had already collected beside the finished file, for some - /// later sync's sweep to reclaim. + /// The whole cost of getting that wrong, on one disk. Sync A holds its temp with two rows + /// in it; sync B over the same command line stops; A then dies, leaving the two files B + /// left alone. A's re-run is what recovers those rows - it promotes the temp over the + /// output and appends the rest of the enumeration - and it can only do that while the + /// checkpoint counting them is there. /// [Fact] - public void A_full_resync_after_a_refusal_leaves_no_temp_of_its_own() + public void The_rows_a_held_temp_holds_come_back_after_the_sync_that_stopped() { var dir = Directory.CreateDirectory( - Path.Combine(Path.GetTempPath(), $"mgx-delta-410kept-{Guid.NewGuid():N}")).FullName; + Path.Combine(Path.GetTempPath(), $"mgx-delta-heldtempback-{Guid.NewGuid():N}")).FullName; var checkpointPath = Path.Combine(dir, "run.checkpoint"); - var deltaA = Path.Combine(dir, "stateA.json"); - var deltaB = Path.Combine(dir, "stateB.json"); + var deltaPath = Path.Combine(dir, "state.json"); var output = Path.Combine(dir, "out.jsonl"); var handler = new MockHttpHandler(); @@ -699,27 +930,34 @@ public void A_full_resync_after_a_refusal_leaves_no_temp_of_its_own() using var transport = MgxTransportScope.Inject(handler); try { - // Sync A dies on page 2: the checkpoint names the temp holding its two items. + // A collects page one into its temp and is still going. handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); - Sync(deltaA, checkpointPath, output); - var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; - - // Sync B refuses that checkpoint, collects a page of its own into a temp, and loses - // its token to a 410 on the request after it. The retry re-syncs in full. - StaleState(deltaB); - handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); - handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); - handler.QueueResponse(HttpStatusCode.OK, GroupsResync); - Sync(deltaB, checkpointPath, output, "/groups/delta"); + Sync(deltaPath, checkpointPath, output); + var temp = Path.Combine(dir, PaginationCheckpoint.Load(checkpointPath)!.TempFile!); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(temp)); - Assert.Equal(["{\"id\":\"g9\"}"], File.ReadAllLines(output)); - Assert.Contains("$deltatoken=G1", DeltaState.Load(deltaB)!.DeltaLink); + // B stops. Nothing is queued for it, and it asks for nothing. + ErrorRecord[] errors; + using (new FileStream(temp, FileMode.Open, FileAccess.ReadWrite, FileShare.Read)) + { + (_, errors) = SyncRun(deltaPath, checkpointPath, output); + Assert.StartsWith("CheckpointTempHeld", + Assert.Single(errors).FullyQualifiedErrorId, StringComparison.Ordinal); + } + + // A dies, leaving exactly the two files B left alone. + Assert.True(File.Exists(checkpointPath), "B took A's position with it"); + Assert.True(File.Exists(temp), "B took A's temp with it"); + + // A's re-run. It promotes the two rows and resumes from the page its own + // checkpoint recorded, so the row below is appended to them rather than replacing + // them - which is what tells a recovery from a fresh enumeration here. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var warnings = SyncWarnings(deltaPath, checkpointPath, output); - // The one temp beside the output is the refused checkpoint's, which this run had no - // business touching either way. - var left = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); - Assert.Equal(theirs, Path.GetFileName(left)); - Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(left)); + Assert.Contains(warnings, w => w.Contains("Recovered 2 items")); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); } finally { @@ -728,20 +966,19 @@ public void A_full_resync_after_a_refusal_leaves_no_temp_of_its_own() } /// - /// The same compound with the 410 on the first request of all, so the attempt that dies has - /// saved no checkpoint and its temp is empty. The keep asked only whether a checkpoint was - /// on disk, and the one there is the refused sync's: it counts nothing this run wrote and - /// names a temp of its own, so an empty file was kept on a foreign file's account and left - /// beside the output the retry went on to finish. + /// What -Latest is told. A from-now baseline over these two files is a run like any other: + /// the checkpoint is a position an enumeration is still in progress at, so the baseline is + /// refused first, and then the temp it names stops the enumeration that was going to happen + /// instead. Nothing is written either way, and the token the caller asked for is not saved + /// over the changes the temp holds. /// [Fact] - public void An_attempt_that_saved_no_checkpoint_keeps_no_temp_on_a_refused_ones_account() + public void A_held_temp_stops_a_Latest_baseline_too() { var dir = Directory.CreateDirectory( - Path.Combine(Path.GetTempPath(), $"mgx-delta-410empty-{Guid.NewGuid():N}")).FullName; + Path.Combine(Path.GetTempPath(), $"mgx-delta-heldtemplatest-{Guid.NewGuid():N}")).FullName; var checkpointPath = Path.Combine(dir, "run.checkpoint"); - var deltaA = Path.Combine(dir, "stateA.json"); - var deltaB = Path.Combine(dir, "stateB.json"); + var deltaPath = Path.Combine(dir, "state.json"); var output = Path.Combine(dir, "out.jsonl"); var handler = new MockHttpHandler(); @@ -750,17 +987,26 @@ public void An_attempt_that_saved_no_checkpoint_keeps_no_temp_on_a_refused_ones_ try { handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); - Sync(deltaA, checkpointPath, output); - var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; - - StaleState(deltaB); - handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); - handler.QueueResponse(HttpStatusCode.OK, GroupsResync); - Sync(deltaB, checkpointPath, output, "/groups/delta"); + Sync(deltaPath, checkpointPath, output); + var temp = Path.Combine(dir, PaginationCheckpoint.Load(checkpointPath)!.TempFile!); - Assert.Equal(["{\"id\":\"g9\"}"], File.ReadAllLines(output)); - var left = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); - Assert.Equal(theirs, Path.GetFileName(left)); + handler.QueueResponse(HttpStatusCode.OK, LatestBaseline); + string[] warnings; + ErrorRecord[] errors; + DiskState before, after; + using (new FileStream(temp, FileMode.Open, FileAccess.ReadWrite, FileShare.Read)) + { + before = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + (warnings, errors) = SyncRun(deltaPath, checkpointPath, output, latest: true); + after = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + } + + var refused = Assert.Single(warnings, w => w.Contains("-Latest ignored")); + Assert.Contains("a resume checkpoint exists", refused); + Assert.StartsWith("CheckpointTempHeld", Assert.Single(errors).FullyQualifiedErrorId, + StringComparison.Ordinal); + AssertUnmoved(before, after); + Assert.False(File.Exists(deltaPath), "a from-now token was saved over the temp's rows"); } finally { @@ -769,19 +1015,21 @@ public void An_attempt_that_saved_no_checkpoint_keeps_no_temp_on_a_refused_ones_ } /// - /// And the temp goes at the moment it stops being anything, not at the end of the run. The - /// retry saves its own first page-boundary checkpoint, which names its own temp - so the - /// one the attempt before it left is from then on counted by nothing - and then dies. What - /// is beside the output is the temp the checkpoint names, and the refused sync's. + /// The other half of the same refusal, where there is no second sync at all: a temp this + /// account cannot open read-write. The advice - grant write access, run again - is the only + /// thing that recovers those rows, so the run giving it must still leave the position they + /// are counted by. It stops rather than enumerate over it. This follows the advice and + /// measures what comes back. /// [Fact] - public void A_retry_deletes_the_temp_its_own_checkpoint_has_stopped_naming() + public void An_unopenable_temp_stops_the_sync_and_says_how_to_recover_it() { + if (OperatingSystem.IsWindows()) return; + var dir = Directory.CreateDirectory( - Path.Combine(Path.GetTempPath(), $"mgx-delta-410release-{Guid.NewGuid():N}")).FullName; + Path.Combine(Path.GetTempPath(), $"mgx-delta-unopenabletemp-{Guid.NewGuid():N}")).FullName; var checkpointPath = Path.Combine(dir, "run.checkpoint"); - var deltaA = Path.Combine(dir, "stateA.json"); - var deltaB = Path.Combine(dir, "stateB.json"); + var deltaPath = Path.Combine(dir, "state.json"); var output = Path.Combine(dir, "out.jsonl"); var handler = new MockHttpHandler(); @@ -790,49 +1038,70 @@ public void A_retry_deletes_the_temp_its_own_checkpoint_has_stopped_naming() try { handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); - Sync(deltaA, checkpointPath, output); - var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + Sync(deltaPath, checkpointPath, output); + var temp = Path.Combine(dir, PaginationCheckpoint.Load(checkpointPath)!.TempFile!); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(temp)); - // The retry reaches a page boundary of its own and dies on the page after it. - StaleState(deltaB); - handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); - handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); - handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); - Sync(deltaB, checkpointPath, output, "/groups/delta"); + // 0444: the owner cannot open it read-write either, which is the claim failing on + // permissions rather than on another handle. + File.SetUnixFileMode(temp, UnixFileMode.UserRead); + + // Nothing queued: the run under test asks for nothing, and the page below belongs + // to the run that follows its advice. + var before = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + var requestsBefore = handler.RequestCount; + var (_, errors) = SyncRun(deltaPath, checkpointPath, output); + var after = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointTempUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Contains("cannot be opened for writing by this account", stop.Exception.Message); + Assert.Contains("not on sharing", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Grant write access to that file and run again to recover them, or remove it " + + "and the checkpoint to sync afresh.", + stop.Exception.Message); + Assert.Equal(requestsBefore, handler.RequestCount); + AssertUnmoved(before, after); + + // Follow it. + File.SetUnixFileMode(temp, UnixFileMode.UserRead | UnixFileMode.UserWrite); - var cp = PaginationCheckpoint.Load(checkpointPath); - Assert.NotNull(cp); - Assert.NotNull(cp!.TempFile); - Assert.NotEqual(theirs, cp.TempFile); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var recovered = SyncWarnings(deltaPath, checkpointPath, output); - var temps = Directory.GetFiles(dir, "out.jsonl.*.tmp") - .Select(Path.GetFileName).Order().ToArray(); - Assert.Equal(new[] { theirs, cp.TempFile }.Order().ToArray(), temps); - Assert.Equal(["{\"id\":\"g1\"}", "{\"id\":\"g2\"}"], - File.ReadAllLines(Path.Combine(dir, cp.TempFile!))); + Assert.Contains(recovered, w => w.Contains("Recovered 2 items")); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); } finally { - try { Directory.Delete(dir, true); } catch { } + try + { + foreach (var f in Directory.EnumerateFiles(dir)) + File.SetUnixFileMode(f, UnixFileMode.UserRead | UnixFileMode.UserWrite); + Directory.Delete(dir, true); + } + catch { } } } /// - /// The other end of it: the retry dies before a page boundary of its own. The temp the - /// attempt before it filled is what an interrupted run leaves and stays where it is, while - /// the retry's own, which nothing counts, is the one that goes. The position that counted - /// the first goes with the token it belongs to - the 410 door deletes it, because the - /// attempt that took the path over wrote it, and it enumerates what Graph has just declared - /// gone. + /// And the 410 door, which used to have a reading of its own to make here. It no longer + /// sees this case at all: a run that stops at the reconcile has not sent a request, so an + /// expired token is not something it can have been told. The door's own sentence said the + /// checkpoint was left alone and the same run then deleted it, which is the second way the + /// spared pair went. /// [Fact] - public void A_retry_that_dies_leaves_the_temp_its_own_attempt_filled() + public void A_stopped_sync_never_reaches_the_410_door() { var dir = Directory.CreateDirectory( - Path.Combine(Path.GetTempPath(), $"mgx-delta-410dead-{Guid.NewGuid():N}")).FullName; + Path.Combine(Path.GetTempPath(), $"mgx-delta-heldtemp410-{Guid.NewGuid():N}")).FullName; var checkpointPath = Path.Combine(dir, "run.checkpoint"); - var deltaA = Path.Combine(dir, "stateA.json"); - var deltaB = Path.Combine(dir, "stateB.json"); + var deltaPath = Path.Combine(dir, "state.json"); var output = Path.Combine(dir, "out.jsonl"); var handler = new MockHttpHandler(); @@ -840,24 +1109,32 @@ public void A_retry_that_dies_leaves_the_temp_its_own_attempt_filled() using var transport = MgxTransportScope.Inject(handler); try { + // No delta token is ever saved, so every run below builds the same request URL and + // the checkpoint this one leaves is that command line's own by every test there is. handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); - Sync(deltaA, checkpointPath, output); - var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + Sync(deltaPath, checkpointPath, output); + var temp = Path.Combine(dir, PaginationCheckpoint.Load(checkpointPath)!.TempFile!); - StaleState(deltaB); - handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); + // An expired token and the full re-sync behind it, both waiting on the wire. handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); - Sync(deltaB, checkpointPath, output, "/groups/delta"); - - Assert.False(File.Exists(checkpointPath), - "the expired token spared a position of this run's own into the dead enumeration"); - - var temps = Directory.GetFiles(dir, "out.jsonl.*.tmp") - .Select(Path.GetFileName).Order().ToArray(); - Assert.Equal(2, temps.Length); - var mine = temps.Single(t => t != theirs); - Assert.Equal(["{\"id\":\"g1\"}", "{\"id\":\"g2\"}"], - File.ReadAllLines(Path.Combine(dir, mine!))); + handler.QueueResponse(HttpStatusCode.OK, ChangesPageNoToken); + string[] warnings; + ErrorRecord[] errors; + DiskState before, after; + int requestsBefore; + using (new FileStream(temp, FileMode.Open, FileAccess.ReadWrite, FileShare.Read)) + { + before = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + requestsBefore = handler.RequestCount; + (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + after = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + } + + Assert.StartsWith("CheckpointTempHeld", Assert.Single(errors).FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.DoesNotContain(warnings, w => w.Contains("410 Gone")); + Assert.Equal(requestsBefore, handler.RequestCount); + AssertUnmoved(before, after); } finally { @@ -866,60 +1143,53 @@ public void A_retry_that_dies_leaves_the_temp_its_own_attempt_filled() } /// - /// And what the run after it finds. The refusal is decided against the file that was on - /// disk when the run started, and the very next page boundary saves this run's own position - /// over that path - so by the time the 410 door reads the refusal, the file it spares is - /// this run's, recording the enumeration Graph has just declared gone. The retry then dies - /// before saving a position of its own, and the door has already deleted the delta state, - /// so the next invocation of the same command line started with nothing to resume from and - /// found that checkpoint waiting. It refused it - correctly; it enumerates something this - /// command line can no longer build - and warned about a sync that was never there, while - /// holding off its stale-temp sweep for a second run over files nothing counted any more. + /// How far "spare the temps a refused checkpoint names" reaches. The sweep only ever + /// deletes names a run gives its own temp - the output's name, a dot, 32 hex digits, + /// ".tmp" - so a refused checkpoint naming "out.jsonl.{guid}.tmp" beside an output called + /// "out" names a file this sweep could not touch either way. Skipping on it bought that + /// file nothing and cost this output its own orphans, which the pre-length adoption path + /// then picks up on a line count alone. /// [Fact] - public void A_refused_position_this_run_wrote_over_goes_with_the_expired_token() + public void A_refused_temp_this_sweep_could_never_reach_does_not_spare_this_outputs_orphans() { var dir = Directory.CreateDirectory( - Path.Combine(Path.GetTempPath(), $"mgx-delta-410tookover-{Guid.NewGuid():N}")).FullName; + Path.Combine(Path.GetTempPath(), $"mgx-delta-sweep-{Guid.NewGuid():N}")).FullName; var checkpointPath = Path.Combine(dir, "run.checkpoint"); var deltaA = Path.Combine(dir, "stateA.json"); var deltaB = Path.Combine(dir, "stateB.json"); - var output = Path.Combine(dir, "out.jsonl"); + var mine = Path.Combine(dir, "out"); + var theirs = Path.Combine(dir, "out.jsonl"); var handler = new MockHttpHandler(); handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); using var transport = MgxTransportScope.Inject(handler); try { - // Sync A dies on page 2: the checkpoint names the temp holding its two items. + // The sync into "out" collects page one into its temp and dies on page two. handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); - Sync(deltaA, checkpointPath, output); - var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; - - // Sync B refuses that checkpoint, reaches a page boundary of its own - which saves - // over the same path - loses its token to a 410, and dies again in the re-sync - // before it can record a position of the enumeration it is now making. - StaleState(deltaB); - handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); - handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); - Sync(deltaB, checkpointPath, output, "/groups/delta"); + Sync(deltaA, checkpointPath, mine); + var orphan = Assert.Single(Directory.GetFiles(dir, "out.*.tmp")); - Assert.False(File.Exists(checkpointPath), - "the 410 door spared a position this run had written over the refused one"); - Assert.True(File.Exists(Path.Combine(dir, theirs)), - "the retry swept away the temp the refusal a moment earlier had spared"); - Assert.False(File.Exists(deltaB), "the expired delta state outlived the 410"); + // The sync next door takes the shared checkpoint over: it refuses what it finds, + // saves its own position at its first page boundary, and dies on page two. The + // first sync's temp is now an orphan - nothing on disk describes it. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaB, checkpointPath, theirs); + var theirTemp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); - // The next invocation of B's command line, with nothing of its own left to resume - // from: no checkpoint to refuse, so no warning about a sync that was never there, - // and the sweep it no longer holds off reclaims what the interrupted run left. - handler.QueueResponse(HttpStatusCode.OK, GroupsResync); - var warnings = SyncWarnings(deltaB, checkpointPath, output, "/groups/delta"); + // "out" again. It refuses a checkpoint naming a temp beside a different output, + // and that name must not stand in the way of its own sweep. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + Sync(deltaA, checkpointPath, mine); - Assert.DoesNotContain(warnings, w => w.Contains("resume checkpoint")); - Assert.Equal(["{\"id\":\"g9\"}"], File.ReadAllLines(output)); - Assert.Contains("$deltatoken=G1", DeltaState.Load(deltaB)!.DeltaLink); - Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(mine)); + Assert.False(File.Exists(orphan), + "a temp name the sweep could never reach suppressed this output's sweep"); + Assert.True(File.Exists(theirTemp), + "the refused checkpoint's own staging file was deleted"); } finally { @@ -928,20 +1198,21 @@ public void A_refused_position_this_run_wrote_over_goes_with_the_expired_token() } /// - /// The other branch of the same door, which is the refusal's promise where it still means - /// something: the 410 arrives on the first request of all, so no attempt of this run has - /// saved anything over the path and what is there is still the position that was refused. - /// It is left exactly as it was found, byte for byte, and it is still the other sync's - /// enumeration rather than this one's. + /// What each refusal says. A checkpoint recording another sync's output is a second sync + /// over one -CheckpointPath, and naming that is what the caller acts on. One recording no + /// output at all is not: it was believed while the files beside it corroborated it, and a + /// single sync whose temp has since gone, or whose output has been replaced, reaches the + /// same refusal. Every release before this one recorded no output, so that is the ordinary + /// upgrade path - and it was told there was a second sync somewhere and to give each its + /// own -CheckpointPath, which fixes nothing it has. /// [Fact] - public void An_expired_token_spares_the_refused_position_no_attempt_wrote_over() + public void A_refusal_reports_the_cause_it_can_show() { var dir = Directory.CreateDirectory( - Path.Combine(Path.GetTempPath(), $"mgx-delta-410intact-{Guid.NewGuid():N}")).FullName; + Path.Combine(Path.GetTempPath(), $"mgx-delta-refusal-{Guid.NewGuid():N}")).FullName; var checkpointPath = Path.Combine(dir, "run.checkpoint"); - var deltaA = Path.Combine(dir, "stateA.json"); - var deltaB = Path.Combine(dir, "stateB.json"); + var deltaPath = Path.Combine(dir, "state.json"); var output = Path.Combine(dir, "out.jsonl"); var handler = new MockHttpHandler(); @@ -949,23 +1220,186 @@ public void An_expired_token_spares_the_refused_position_no_attempt_wrote_over() using var transport = MgxTransportScope.Inject(handler); try { - handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); - Sync(deltaA, checkpointPath, output); - var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; - var before = File.ReadAllBytes(checkpointPath); + File.WriteAllLines(output, OldSync); - // The same compound as above, differing only in reaching no page boundary before - // the 410 - so the door decides over a path this run has not written to. - StaleState(deltaB); - handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); - Sync(deltaB, checkpointPath, output, "/groups/delta"); + // The shape an interrupted fresh sync left before outputs were recorded, whose temp + // is no longer beside it. + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users/delta?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = $"out.jsonl.{Guid.NewGuid():N}.tmp", + OutputFile = null, + DataLength = 24, + }.Save(checkpointPath); - Assert.Equal(before, File.ReadAllBytes(checkpointPath)); - var cp = PaginationCheckpoint.Load(checkpointPath); - Assert.NotNull(cp); - Assert.Contains("/users/delta", cp!.Resource); - Assert.Equal(theirs, cp.TempFile); - Assert.True(File.Exists(Path.Combine(dir, theirs)), + var uncorroborated = SyncWarnings(deltaPath, checkpointPath, output); + Assert.Contains(uncorroborated, w => w.Contains("no longer corroborate")); + Assert.DoesNotContain(uncorroborated, + w => w.Contains("records an enumeration this run cannot resume from")); + + // The same position, recording an output this run is not collecting into. + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users/delta?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = Path.Combine(dir, "elsewhere.jsonl"), + DataLength = 24, + }.Save(checkpointPath); + + Assert.Contains(SyncWarnings(deltaPath, checkpointPath, output), + w => w.Contains("records an enumeration this run cannot resume from")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And the cause it cannot show. A run interrupted partway through an incremental sync + /// leaves a checkpoint recording the delta URL it was enumerating. Lose the state under it - + /// corrupt, deleted by hand, a -DeltaPath moved - and the same command line comes back + /// building a full-sync URL, which is not that one, so it refuses its own earlier position. + /// The refusal is right; the reading was not. It named a different sync, over a + /// -CheckpointPath no second sync has ever touched, and told the caller to give each of them + /// its own - which is the arrangement they were already running. + /// + [Fact] + public void A_refusal_does_not_name_a_second_sync_it_cannot_see() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-selfrefusal-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // One sync, one -CheckpointPath: interrupted on page 2 of its incremental pass, so + // what it leaves records the delta URL it was reading. + UsersState(deltaPath); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + Assert.Contains("$deltatoken=D0", PaginationCheckpoint.Load(checkpointPath)!.Resource); + + // The state goes, and the same command line enumerates in full. + File.Delete(deltaPath); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var refusal = Assert.Single( + SyncWarnings(deltaPath, checkpointPath, output), + w => w.Contains(checkpointPath)); + + Assert.Contains("records an enumeration this run cannot resume from", refusal); + Assert.Contains("either another sync's, sharing this -CheckpointPath", refusal); + Assert.Contains("this sync's own from a pass it no longer makes", refusal); + Assert.DoesNotContain("belongs to a different sync", refusal); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A 410 says the token this run holds is dead. It says nothing about a checkpoint this + /// same run has already decided belongs to another sync, and deleting that one took the + /// other sync's position away seconds after warning that it would be left alone - while the + /// retry, running the stale-temp sweep a second time with the refusal forgotten, took the + /// temp holding its items too. Both halves of the other sync's progress, on the one path + /// that promised to touch neither. + /// + [Fact] + public void An_expired_token_does_not_delete_the_checkpoint_this_run_refused() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410refused-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // Sync A dies on page 2: its two items are in its temp, and the checkpoint names + // both files. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + + var cp = PaginationCheckpoint.Load(checkpointPath)!; + var temp = Path.Combine(dir, cp.TempFile!); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(temp)); + var before = File.ReadAllBytes(checkpointPath); + + // Sync B: another collection over the same -CheckpointPath and -OutputFile, holding + // a token Graph answers 410 for. It refuses A's checkpoint, re-syncs in full, and + // dies before a page boundary of its own. + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.True(File.Exists(checkpointPath), + "the expired token deleted a checkpoint this run had refused as another sync's"); + Assert.Equal(before, File.ReadAllBytes(checkpointPath)); + Assert.True(File.Exists(temp), + "the retry swept away the temp the refusal a moment earlier had spared"); + + // A, back where it left off, rather than at the delta token. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + Sync(deltaA, checkpointPath, output); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And the run that refused still gets its own sync done: the 410 retry enumerates in full, + /// collects into the output and saves the token it was issued, with the other sync's temp + /// still where the refusal left it. + /// + [Fact] + public void A_full_resync_after_a_refusal_still_completes_its_own_sync() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410done-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var temp = Path.Combine(dir, PaginationCheckpoint.Load(checkpointPath)!.TempFile!); + + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + handler.QueueResponse(HttpStatusCode.OK, GroupsPage); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.Equal(["{\"id\":\"g1\"}"], File.ReadAllLines(output)); + Assert.Contains("$deltatoken=G1", DeltaState.Load(deltaB)!.DeltaLink); + Assert.True(File.Exists(temp), "the retry swept away the temp the refusal a moment earlier had spared"); } finally @@ -975,22 +1409,1928 @@ public void An_expired_token_spares_the_refused_position_no_attempt_wrote_over() } /// - /// Answers by URL rather than in order, so what the sync asks for first is what decides - /// which items it receives. + /// What the run that refused leaves beside its own output. The attempt a 410 sends round the + /// loop again keeps its temp - a checkpoint counting those items is on disk - and the retry + /// spares it, because the sweep is held off over every temp beside this output for as long + /// as the refusal stands. What ends the holding off was never written: the checkpoint is + /// replaced by the retry's, or deleted by the run that completed, without a word about the + /// file it used to name. So a sync that promoted its output and reported success left a + /// partial copy of the changes it had already collected beside the finished file, for some + /// later sync's sweep to reclaim. + /// + [Fact] + public void A_full_resync_after_a_refusal_leaves_no_temp_of_its_own() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410kept-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // Sync A dies on page 2: the checkpoint names the temp holding its two items. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + + // Sync B refuses that checkpoint, collects a page of its own into a temp, and loses + // its token to a 410 on the request after it. The retry re-syncs in full. + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + handler.QueueResponse(HttpStatusCode.OK, GroupsResync); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.Equal(["{\"id\":\"g9\"}"], File.ReadAllLines(output)); + Assert.Contains("$deltatoken=G1", DeltaState.Load(deltaB)!.DeltaLink); + + // The one temp beside the output is the refused checkpoint's, which this run had no + // business touching either way. + var left = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(theirs, Path.GetFileName(left)); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(left)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same compound with the 410 on the first request of all, so the attempt that dies has + /// saved no checkpoint and its temp is empty. The keep asked only whether a checkpoint was + /// on disk, and the one there is the refused sync's: it counts nothing this run wrote and + /// names a temp of its own, so an empty file was kept on a foreign file's account and left + /// beside the output the retry went on to finish. + /// + [Fact] + public void An_attempt_that_saved_no_checkpoint_keeps_no_temp_on_a_refused_ones_account() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410empty-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + handler.QueueResponse(HttpStatusCode.OK, GroupsResync); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.Equal(["{\"id\":\"g9\"}"], File.ReadAllLines(output)); + var left = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(theirs, Path.GetFileName(left)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And the temp goes at the moment it stops being anything, not at the end of the run. The + /// retry saves its own first page-boundary checkpoint, which names its own temp - so the + /// one the attempt before it left is from then on counted by nothing - and then dies. What + /// is beside the output is the temp the checkpoint names, and the refused sync's. + /// + [Fact] + public void A_retry_deletes_the_temp_its_own_checkpoint_has_stopped_naming() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410release-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + + // The retry reaches a page boundary of its own and dies on the page after it. + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + var cp = PaginationCheckpoint.Load(checkpointPath); + Assert.NotNull(cp); + Assert.NotNull(cp!.TempFile); + Assert.NotEqual(theirs, cp.TempFile); + + var temps = Directory.GetFiles(dir, "out.jsonl.*.tmp") + .Select(Path.GetFileName).Order().ToArray(); + Assert.Equal(new[] { theirs, cp.TempFile }.Order().ToArray(), temps); + Assert.Equal(["{\"id\":\"g1\"}", "{\"id\":\"g2\"}"], + File.ReadAllLines(Path.Combine(dir, cp.TempFile!))); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other end of it: the retry dies before a page boundary of its own. The temp the + /// attempt before it filled is what an interrupted run leaves and stays where it is, while + /// the retry's own, which nothing counts, is the one that goes. The position that counted + /// the first goes with the token it belongs to - the 410 door deletes it, because the + /// attempt that took the path over wrote it, and it enumerates what Graph has just declared + /// gone. + /// + [Fact] + public void A_retry_that_dies_leaves_the_temp_its_own_attempt_filled() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410dead-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.False(File.Exists(checkpointPath), + "the expired token spared a position of this run's own into the dead enumeration"); + + var temps = Directory.GetFiles(dir, "out.jsonl.*.tmp") + .Select(Path.GetFileName).Order().ToArray(); + Assert.Equal(2, temps.Length); + var mine = temps.Single(t => t != theirs); + Assert.Equal(["{\"id\":\"g1\"}", "{\"id\":\"g2\"}"], + File.ReadAllLines(Path.Combine(dir, mine!))); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And what the run after it finds. The refusal is decided against the file that was on + /// disk when the run started, and the very next page boundary saves this run's own position + /// over that path - so by the time the 410 door reads the refusal, the file it spares is + /// this run's, recording the enumeration Graph has just declared gone. The retry then dies + /// before saving a position of its own, and the door has already deleted the delta state, + /// so the next invocation of the same command line started with nothing to resume from and + /// found that checkpoint waiting. It refused it - correctly; it enumerates something this + /// command line can no longer build - and warned about a sync that was never there, while + /// holding off its stale-temp sweep for a second run over files nothing counted any more. + /// + [Fact] + public void A_refused_position_this_run_wrote_over_goes_with_the_expired_token() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410tookover-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // Sync A dies on page 2: the checkpoint names the temp holding its two items. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + + // Sync B refuses that checkpoint, reaches a page boundary of its own - which saves + // over the same path - loses its token to a 410, and dies again in the re-sync + // before it can record a position of the enumeration it is now making. + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.False(File.Exists(checkpointPath), + "the 410 door spared a position this run had written over the refused one"); + Assert.True(File.Exists(Path.Combine(dir, theirs)), + "the retry swept away the temp the refusal a moment earlier had spared"); + Assert.False(File.Exists(deltaB), "the expired delta state outlived the 410"); + + // The next invocation of B's command line, with nothing of its own left to resume + // from: no checkpoint to refuse, so no warning about a sync that was never there, + // and the sweep it no longer holds off reclaims what the interrupted run left. + handler.QueueResponse(HttpStatusCode.OK, GroupsResync); + var warnings = SyncWarnings(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.DoesNotContain(warnings, w => w.Contains("resume checkpoint")); + Assert.Equal(["{\"id\":\"g9\"}"], File.ReadAllLines(output)); + Assert.Contains("$deltatoken=G1", DeltaState.Load(deltaB)!.DeltaLink); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other branch of the same door, which is the refusal's promise where it still means + /// something: the 410 arrives on the first request of all, so no attempt of this run has + /// saved anything over the path and what is there is still the position that was refused. + /// It is left exactly as it was found, byte for byte, and it is still the other sync's + /// enumeration rather than this one's. + /// + [Fact] + public void An_expired_token_spares_the_refused_position_no_attempt_wrote_over() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410intact-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + var before = File.ReadAllBytes(checkpointPath); + + // The same compound as above, differing only in reaching no page boundary before + // the 410 - so the door decides over a path this run has not written to. + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.Equal(before, File.ReadAllBytes(checkpointPath)); + var cp = PaginationCheckpoint.Load(checkpointPath); + Assert.NotNull(cp); + Assert.Contains("/users/delta", cp!.Resource); + Assert.Equal(theirs, cp.TempFile); + Assert.True(File.Exists(Path.Combine(dir, theirs)), + "the retry swept away the temp the refusal a moment earlier had spared"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A state file that cannot be read is a door out of the delta position like -FullSync and + /// the query-shape changes, and every one of those takes the resume checkpoint with the + /// position it discards. This one did not, and what it left describes the delta-link + /// spelling the fresh sync no longer builds - so the run refused its own earlier position, + /// warned about a second sync that was never there, and held its stale-temp sweep off over + /// a temp nothing counted any more. + /// + [Fact] + public void A_corrupt_delta_state_takes_the_checkpoint_of_the_position_it_abandons() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-corrupt-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // A sync resuming from a token, interrupted on page two: the checkpoint records the + // delta link it was enumerating, and its temp holds the two items it collected. + UsersState(deltaPath); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + var temp = Path.Combine(dir, PaginationCheckpoint.Load(checkpointPath)!.TempFile!); + Assert.True(File.Exists(temp)); + + // And the state file is unreadable by the time the next run starts. + File.WriteAllText(deltaPath, "{ this is not a delta state"); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var warnings = SyncWarnings(deltaPath, checkpointPath, output); + + Assert.Contains(warnings, w => w.Contains("is corrupt. Starting full sync.")); + Assert.DoesNotContain(warnings, + w => w.Contains("records an enumeration this run cannot resume from")); + Assert.False(File.Exists(checkpointPath), + "the abandoned position outlived the state it described"); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(["{\"id\":\"b3\"}"], File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And what the caller is told on the way past. -Latest is suppressed by a resume checkpoint + /// merely existing, and the sentence saying so tells the caller to delete that file - which + /// this door has just done. The advice has to describe the disk as it is. + /// + [Fact] + public void A_corrupt_state_does_not_send_the_caller_after_a_checkpoint_it_deleted() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-corruptlatest-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + UsersState(deltaPath); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + Assert.True(File.Exists(checkpointPath)); + + File.WriteAllText(deltaPath, "{ this is not a delta state"); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var warnings = SyncWarnings(deltaPath, checkpointPath, output, latest: true); + + Assert.Contains(warnings, w => w.Contains("the previous delta state was discarded")); + Assert.DoesNotContain(warnings, w => w.Contains("a resume checkpoint exists")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same sentence, at the three doors a changed query shape opens. A $select, a Prefer + /// header or a filter that differs from the state's discards that state and the resume + /// checkpoint counted into it - and the -Latest guard downstream still read a checkpoint as + /// standing, so the advice it wrote told the caller to go and delete a file the door had + /// just deleted, and said nothing about the state that had actually been discarded. + /// + [Theory] + [InlineData("select")] + [InlineData("prefer")] + [InlineData("filter")] + public void A_query_shape_door_does_not_send_the_caller_after_a_checkpoint_it_deleted(string door) + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-shapelatest-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // A sync resuming from a token, interrupted on page two: the checkpoint it leaves + // records the delta link it was enumerating. + UsersState(deltaPath); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + Assert.True(File.Exists(checkpointPath)); + + // The next run asks for a different shape of the same query, with -Latest on. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var warnings = SyncWarnings(deltaPath, checkpointPath, output, latest: true, + property: door == "select" ? ["id", "displayName"] : null, + prefer: door == "prefer" ? ["return=minimal"] : null, + filter: door == "filter" ? "startswith(displayName,'a')" : null); + + Assert.Contains(warnings, w => w.Contains(door switch + { + "select" => "Property selection changed since last sync", + "prefer" => "Prefer headers changed since last sync", + _ => "Filter changed since last sync", + })); + Assert.Contains(warnings, w => w.Contains("the previous delta state was discarded")); + Assert.DoesNotContain(warnings, w => w.Contains("a resume checkpoint exists")); + Assert.False(File.Exists(checkpointPath), + "the position counted into the discarded state outlived it"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// Answers by URL rather than in order, so what the sync asks for first is what decides + /// which items it receives. + /// + + /// + /// The disk a resumed sync leaves at a page boundary. One run dies with its changes in a + /// temp; the next promotes that temp over the output, appends the page it resumed onto, + /// saves its position against the output itself - there is no temp left to name - and dies + /// too. That last position is the one shape a shared -CheckpointPath offers no temp to + /// refuse a second run by. No page carried a deltaLink, so the token is still the one the + /// state file held when this started. + /// + private static void ResumedAndAppending(MockHttpHandler handler, string deltaPath, + string checkpointPath, string output) + { + UsersState(deltaPath); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + } + + /// + /// The refusal one file over from the temp claim, and the one it could not reach. A sync that + /// has resumed once writes straight into -OutputFile and saves a position naming no temp at + /// all, so a second run over the same command line found nothing in the checkpoint to refuse + /// it by: it judged the file its own, cut it back to the recorded offset under a live writer, + /// appended, and moved the delta token past changes the output no longer held. The holder's + /// next write landed past the hole the cut left. There is no pass this run can make that + /// leaves any of that standing, so it makes none. + /// + [Fact] + public void A_held_output_stops_the_sync_before_anything_is_written() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-heldout-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + ResumedAndAppending(handler, deltaPath, checkpointPath, output); + + // The premise, off the file: this position counts its changes into the output and + // names no temp, which is the shape the claim on the temp has nothing to say about. + var cp = PaginationCheckpoint.Load(checkpointPath)!; + Assert.Null(cp.TempFile); + Assert.Equal(Path.GetFullPath(output), cp.OutputFile); + Assert.Equal(new FileInfo(output).Length, cp.DataLength); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + + // Sync A is not dead. It holds the output the way a resumed run does - appending, + // sharing reads - and has written one more change since its last save, so the file + // is longer than the checkpoint records and cutting it back leaves a hole under it. + // And the page waiting on the wire carries a deltaLink, so going on would move the + // token as well. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + DiskState before, after; + int requestsBefore; + string[] warnings; + ErrorRecord[] errors; + using (var live = new StreamWriter( + new FileStream(output, FileMode.Append, FileAccess.Write, FileShare.Read))) + { + live.WriteLine("{\"id\":\"b9-live\"}"); + live.Flush(); + Assert.True(new FileInfo(output).Length > cp.DataLength, + "the holder wrote nothing past the recorded length, so no cut would show"); + + before = ReadDisk(dir, checkpointPath, output, deltaPath); + requestsBefore = handler.RequestCount; + (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.ResourceBusy, stop.CategoryInfo.Category); + Assert.Contains($"Another sync is still writing '{output}'", + stop.Exception.Message); + Assert.Contains(checkpointPath, stop.Exception.Message); + Assert.Contains($"records {cp.ItemsCollected} items into", stop.Exception.Message); + Assert.Contains("would cut that file back under it", stop.Exception.Message); + Assert.Contains("advance the delta token past them", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Wait for that sync to finish, or give this one its own -OutputFile and " + + "-CheckpointPath.", + stop.Exception.Message); + + // And not the sentence for a file whose contents this run never looked at. + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + Assert.DoesNotContain(warnings, w => w.Contains("Re-enumerating")); + + // Nothing ran: no request, and the output, the position, the delta state and the + // directory's entry list exactly as the holder left them. Read after the record + // and not before it, so a run that took the checkpoint with it fails on the + // answer it did not give. + after = ReadDisk(dir, checkpointPath, output, deltaPath); + Assert.Equal(requestsBefore, handler.RequestCount); + AssertUnmoved(before, after); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + + // A goes on, and the change it writes next lands where its last one ended. + live.WriteLine("{\"id\":\"b10-live\"}"); + live.Flush(); + } + + // Ordinal, both of them: the culture-aware comparisons treat a NUL as ignorable, + // which is the one byte this is looking for. + var published = File.ReadAllText(output); + Assert.False(published.Contains('\0'), + "the cut left a hole for the holder's next write to land past"); + Assert.EndsWith("{\"id\":\"b9-live\"}" + Environment.NewLine + + "{\"id\":\"b10-live\"}" + Environment.NewLine, published, + StringComparison.Ordinal); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other half of the same claim, where there is no second sync at all: the output the + /// checkpoint counts its changes into cannot be opened for writing by this account. It was + /// reported as a file that no longer holds those changes - a reading of the file's contents, + /// given about a file this run had not been able to open - and the position counting them was + /// deleted on the strength of it, with the token then advancing past them. The advice is the + /// only thing that recovers them, so the run giving it leaves all of that standing. This + /// follows the advice and measures what comes back. + /// + [Fact] + public void An_unopenable_output_stops_the_sync_and_says_how_to_recover_it() + { + if (OperatingSystem.IsWindows()) return; + + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-unopenout-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + ResumedAndAppending(handler, deltaPath, checkpointPath, output); + var cp = PaginationCheckpoint.Load(checkpointPath)!; + Assert.Null(cp.TempFile); + var recorded = cp.DataLength!.Value; + + // The changes the checkpoint counts are there, and one written after its last save + // with them - so a run that can open the file cuts back to the recorded length. + File.AppendAllText(output, "{\"id\":\"b9-past-the-save\"}" + Environment.NewLine); + Assert.True(new FileInfo(output).Length > recorded); + + // 0444: the owner cannot open it read-write either, which is the claim failing on + // permissions rather than on another handle. + File.SetUnixFileMode(output, UnixFileMode.UserRead); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var before = ReadDisk(dir, checkpointPath, output, deltaPath); + var requestsBefore = handler.RequestCount; + var (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.PermissionDenied, stop.CategoryInfo.Category); + Assert.Contains($"'{output}' could not be opened for writing:", stop.Exception.Message); + Assert.Contains("is denied", stop.Exception.Message); + // The reason's own trailing period is trimmed before it is quoted, so the sentence + // this joins into is punctuated once, not twice. + Assert.DoesNotContain(".. ", stop.Exception.Message); + Assert.Contains("is denied. The resume checkpoint", stop.Exception.Message); + Assert.Contains($"The resume checkpoint at '{checkpointPath}' records", + stop.Exception.Message); + Assert.Contains("advance the delta token past them", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Grant write access to that file and run again to resume from it, or remove it " + + "and the checkpoint to sync afresh.", + stop.Exception.Message); + + // Not said about a file whose contents this run could not read, and not acted on. + // The disk is read after the record and not before it, so a run that deleted the + // position fails on the answer it did not give. + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + Assert.Equal(requestsBefore, handler.RequestCount); + AssertUnmoved(before, ReadDisk(dir, checkpointPath, output, deltaPath)); + Assert.True(File.Exists(checkpointPath), + "the position counting the changes the advice recovers was deleted"); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + + // Follow it. The run that comes back cuts the output to the recorded length and + // appends the page it resumes onto, so the change written past the last save goes, + // nothing is written twice, and the token moves only now. + File.SetUnixFileMode(output, UnixFileMode.UserRead | UnixFileMode.UserWrite); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var resumed = SyncWarnings(deltaPath, checkpointPath, output); + + Assert.DoesNotContain(resumed, w => w.Contains("no longer holds")); + Assert.Equal( + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b1\"}", "{\"id\":\"b2\"}", + "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try + { + foreach (var f in Directory.EnumerateFiles(dir)) + File.SetUnixFileMode(f, UnixFileMode.UserRead | UnixFileMode.UserWrite); + Directory.Delete(dir, true); + } + catch { } + } + } + + /// + /// The answer the claim must not swallow. An output shorter than the length the checkpoint + /// records is not the file that checkpoint describes - the changes it counts are in no file + /// at all - and nothing holds it and nothing refuses to open it. That still takes today's + /// route: say so, delete the position, and re-enumerate from the last saved token. + /// + [Fact] + public void A_stale_length_output_still_reenumerates_from_the_token() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-staleout-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + ResumedAndAppending(handler, deltaPath, checkpointPath, output); + var cp = PaginationCheckpoint.Load(checkpointPath)!; + Assert.Null(cp.TempFile); + + // The output is replaced by one that does not hold those changes. + File.WriteAllText(output, "{\"id\":\"short\"}" + Environment.NewLine); + Assert.True(new FileInfo(output).Length < cp.DataLength); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + Assert.Empty(errors); + Assert.Contains(warnings, + w => w.Contains($"'{output}' no longer holds the {cp.ItemsCollected} items")); + Assert.False(File.Exists(checkpointPath), "the stale position was left on disk"); + Assert.Equal(["{\"id\":\"b3\"}"], File.ReadAllLines(output)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + private sealed class ByUrlHandler : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var body = request.RequestUri!.ToString().Contains("skiptoken=B2", StringComparison.Ordinal) + ? ChangesPage2 + : ChangesPage1; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } + + /// + /// The disk a resumed sync leaves at a page boundary, with every change in the output + /// exactly once. One run dies with its changes in a temp; the next promotes that temp over + /// the output, saves its position against the output itself - there is no temp left to name + /// - and dies at its first request. No page carried a deltaLink, so the token is still the + /// one the state file held when this started. + /// + private static void PromotedAndAppending(MockHttpHandler handler, string deltaPath, + string checkpointPath, string output) + { + UsersState(deltaPath); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + Sync(deltaPath, checkpointPath, output); + } + + /// + /// Two syncs over one command line, both past the stat the reconcile makes. The claim on the + /// output was asked for and let go inside the reconcile and the writer opened its own handle + /// a page fetch later, so for that window neither run held the file: both passed the claim, + /// both cut it back to the recorded offset, both appended, and the token moved past changes + /// the file no longer held. The claim is the handle now, so the run that takes it keeps it + /// across that window, and the second run meets a file the first one has. + /// + /// A is parked in the window itself. The verbose announcing the resume is written after the + /// reconcile - the claim taken, the change past the last save cut away - and before the + /// writer is opened, which is exactly where the two runs used to overlap. + /// + [Fact] + public void Two_syncs_released_together_leave_one_with_the_output_and_the_token() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-interleave-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + // A real second thread, not a scheduled task: the two runs have to be able to be inside + // the cmdlet at the same time, and joining one is not a blocking wait on work the test + // framework might have scheduled behind it. + (string[] Warnings, ErrorRecord[] Errors) aResult = default; + Exception? aFailed = null; + Thread? a = null; + using var arrived = new ManualResetEventSlim(false); + using var released = new ManualResetEventSlim(false); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + PromotedAndAppending(handler, deltaPath, checkpointPath, output); + + var cp = PaginationCheckpoint.Load(checkpointPath)!; + Assert.Null(cp.TempFile); + Assert.Equal(Path.GetFullPath(output), cp.OutputFile); + var recorded = cp.DataLength!.Value; + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(output)); + Assert.Equal(new FileInfo(output).Length, recorded); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + + // A change written after the last save, so the cut A makes is visible from outside. + File.AppendAllText(output, "{\"id\":\"b9-past-the-save\"}" + Environment.NewLine); + Assert.True(new FileInfo(output).Length > recorded); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var requestsBefore = handler.RequestCount; + + a = new Thread(() => + { + try + { + aResult = SyncParkedAtResume(deltaPath, checkpointPath, output, + arrived, released); + } + catch (Exception ex) { aFailed = ex; } + }); + a.Start(); + Assert.True(arrived.Wait(TimeSpan.FromSeconds(60)), + "sync A never reached the window between its reconcile and its writer"); + + // Past the reconcile: the change written after the last save has been cut away. And + // short of the writer: no page has been asked for, so nothing has been appended. + Assert.Equal(recorded, new FileInfo(output).Length); + Assert.Equal(requestsBefore, handler.RequestCount); + + var (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.ResourceBusy, stop.CategoryInfo.Category); + Assert.Contains($"Another sync is still writing '{output}'", stop.Exception.Message); + Assert.Contains($"records {cp.ItemsCollected} items into", stop.Exception.Message); + Assert.Contains("advance the delta token past them", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + + // B sent no request and took nothing with it: the position A resumes from is still + // on disk, the file is what A cut it back to, and the token has not moved. + Assert.Equal(requestsBefore, handler.RequestCount); + Assert.True(File.Exists(checkpointPath), "B deleted the position A is resuming from"); + Assert.Equal(recorded, new FileInfo(output).Length); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + + released.Set(); + Assert.True(a.Join(TimeSpan.FromSeconds(60)), "sync A never finished"); + Assert.Null(aFailed); + Assert.Empty(aResult.Errors); + + // One run had the file. Every change once, in order, the position spent, and the + // token advanced exactly once - to the link the page A fetched carried. + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpointPath), + "the completed sync left its position behind"); + var state = DeltaState.Load(deltaPath)!; + Assert.Contains("$deltatoken=D2", state.DeltaLink); + Assert.Equal(3, state.ItemCount); + } + finally + { + released.Set(); + a?.Join(TimeSpan.FromSeconds(30)); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A directory standing where the output should be. It is not a file, so File.Exists said no + /// and the run took the route it takes for an output that is not there at all - warning that + /// the file no longer held the changes, deleting the position, and re-enumerating from a + /// token that is about to advance past them. Opening a directory read-write is refused on + /// both platforms, and that is an answer. The write probe above this passes: it writes + /// beside the path, not to it. + /// + [Fact] + public void A_directory_standing_at_the_output_path_stops_the_sync() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-outdir-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + PromotedAndAppending(handler, deltaPath, checkpointPath, output); + var cp = PaginationCheckpoint.Load(checkpointPath)!; + Assert.Null(cp.TempFile); + var position = File.ReadAllBytes(checkpointPath); + var state = File.ReadAllBytes(deltaPath); + + File.Delete(output); + Directory.CreateDirectory(output); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var requestsBefore = handler.RequestCount; + var (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + // A directory is not a permission, whatever errno the open came back with: no grant + // makes one writable at an offset, and the way out is to take it off the path. + Assert.Equal(ErrorCategory.InvalidOperation, stop.CategoryInfo.Category); + Assert.Contains($"'{output}' could not be opened for writing: a directory stands at " + + "the path.", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Remove it (or point -OutputFile elsewhere) and run again to resume from it, or " + + "remove the checkpoint to sync afresh.", + stop.Exception.Message); + Assert.DoesNotContain("Grant write access", stop.Exception.Message); + + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + Assert.DoesNotContain(warnings, w => w.Contains("Re-enumerating")); + Assert.Equal(requestsBefore, handler.RequestCount); + Assert.True(File.Exists(checkpointPath), "the position was deleted over a directory"); + Assert.Equal(position, File.ReadAllBytes(checkpointPath)); + Assert.Equal(state, File.ReadAllBytes(deltaPath)); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + Assert.True(Directory.Exists(output)); + Assert.Empty(Directory.GetFileSystemEntries(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The two write probes the sync makes before it fetches anything, and what each of them + /// calls the file it could not write. Both are the same call, and one noun for both told a + /// caller whose -OutputFile directory was not writable that they could not write to the + /// delta state path - naming a file that had just passed its own probe. + /// + [Fact] + public void The_write_probe_names_the_file_it_could_not_write() + { + if (OperatingSystem.IsWindows()) return; + + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-probe-{Guid.NewGuid():N}")).FullName; + var outDir = Directory.CreateDirectory(Path.Combine(dir, "out")).FullName; + var stateDir = Directory.CreateDirectory(Path.Combine(dir, "state")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // r-x: the directory can be searched and not written, which is what the probe + // fails on. Where the write below succeeds the account is root, a mode says + // nothing, and there is nothing here to measure. + File.SetUnixFileMode(outDir, UnixFileMode.UserRead | UnixFileMode.UserExecute); + var output = Path.Combine(outDir, "out.jsonl"); + var writable = true; + try { File.WriteAllText(output, ""); } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + writable = false; + } + if (writable) return; + + var deltaPath = Path.Combine(dir, "state.json"); + var (_, errors) = SyncRun(deltaPath, checkpointPath, output); + var stop = Assert.Single(errors); + Assert.Contains($"Cannot write to output path '{output}'", stop.Exception.Message); + Assert.DoesNotContain("delta state path", stop.Exception.Message); + + // And the probe on -DeltaPath keeps its own noun. + File.SetUnixFileMode(stateDir, UnixFileMode.UserRead | UnixFileMode.UserExecute); + var unwritableState = Path.Combine(stateDir, "state.json"); + var (_, stateErrors) = SyncRun(unwritableState, checkpointPath, + Path.Combine(dir, "fine.jsonl")); + var stateStop = Assert.Single(stateErrors); + Assert.Contains($"Cannot write to delta state path '{unwritableState}'", + stateStop.Exception.Message); + } + finally + { + try + { + foreach (var d in new[] { outDir, stateDir }) + File.SetUnixFileMode(d, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + Directory.Delete(dir, true); + } + catch { } + } + } + + /// + /// The disk an interrupted fresh sync leaves: its changes are in a temp beside the output, + /// the checkpoint names that temp, and there is no output at all yet. The run after it + /// promotes the temp over the output and appends onto what it promoted. + /// + private static void InterruptedIntoATemp(MockHttpHandler handler, string deltaPath, + string checkpointPath, string output) + { + UsersState(deltaPath); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + } + + /// + /// The other route that ends in an append, one file over. A sync whose checkpoint names a + /// temp promotes it over the output and then appends onto it - and used to hold nothing + /// between the move that put the changes there and the writer's own open a page fetch later. + /// The checkpoint it repoints in that gap names no temp, which is the shape a second run + /// over the same command line reads as an output it may cut back, append to, and move the + /// token past. So the promotion takes the file it promoted into. + /// + [Fact] + public void A_promoting_sync_holds_the_output_it_promoted_into() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-promotehold-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + (string[] Warnings, ErrorRecord[] Errors) aResult = default; + Exception? aFailed = null; + Thread? a = null; + using var arrived = new ManualResetEventSlim(false); + using var released = new ManualResetEventSlim(false); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + InterruptedIntoATemp(handler, deltaPath, checkpointPath, output); + + // The premise: the changes are in a temp, the checkpoint names it, and nothing is at + // -OutputFile for anyone to be holding. + var before = PaginationCheckpoint.Load(checkpointPath)!; + Assert.NotNull(before.TempFile); + Assert.False(File.Exists(output)); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var requestsBefore = handler.RequestCount; + + a = new Thread(() => + { + try + { + aResult = SyncParkedAtResume(deltaPath, checkpointPath, output, + arrived, released); + } + catch (Exception ex) { aFailed = ex; } + }); + a.Start(); + Assert.True(arrived.Wait(TimeSpan.FromSeconds(60)), + "sync A never reached the window between its promotion and its writer"); + + // Past the promotion: the changes are in the output, the temp is gone, and the + // position on disk names no temp - which is exactly what a second run reads as an + // output it may take. And short of the writer: no page has been asked for. + // + // Measured off the directory entry, since the run holds the file and a reader of it + // is refused until the run ends. What it holds is read at the end, when it does not. + var promotedBytes = ("{\"id\":\"b1\"}" + Environment.NewLine + + "{\"id\":\"b2\"}" + Environment.NewLine).Length; + Assert.Equal(promotedBytes, new FileInfo(output).Length); + var repointed = PaginationCheckpoint.Load(checkpointPath)!; + Assert.Null(repointed.TempFile); + Assert.Equal(promotedBytes, repointed.DataLength); + Assert.Equal(["out.jsonl", "run.checkpoint", "state.json"], + Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal)); + Assert.Equal(requestsBefore, handler.RequestCount); + + var (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.ResourceBusy, stop.CategoryInfo.Category); + Assert.Contains($"Another sync is still writing '{output}'", stop.Exception.Message); + Assert.Contains("advance the delta token past them", stop.Exception.Message); + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + + // B sent no request, took nothing with it, and moved no token. + Assert.Equal(requestsBefore, handler.RequestCount); + Assert.True(File.Exists(checkpointPath), "B deleted the position A is resuming from"); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + + released.Set(); + Assert.True(a.Join(TimeSpan.FromSeconds(60)), "sync A never finished"); + Assert.Null(aFailed); + Assert.Empty(aResult.Errors); + + // One run had the file. Every change once, in order, the position spent, and the + // token advanced exactly once. + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpointPath), + "the completed sync left its position behind"); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + released.Set(); + a?.Join(TimeSpan.FromSeconds(30)); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The promotion route's own claim on the file it replaces. A sync whose checkpoint names a + /// temp puts those changes over the output and appends onto what it put there, and it asked + /// for the file only after the move - which a rename asks nothing about, on either side of + /// it. So the move went through over an output another run was holding and appending to, that + /// run's next write landed in an inode with no name left on it, and this one would have gone + /// on to move the delta token past changes the file at the path no longer held. + /// + [Fact] + public void A_promotion_over_a_held_output_stops_the_sync_and_replaces_nothing() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-promoteheld-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + InterruptedIntoATemp(handler, deltaPath, checkpointPath, output); + var cp = PaginationCheckpoint.Load(checkpointPath)!; + var temp = Path.Combine(dir, cp.TempFile!); + Assert.True(File.Exists(temp), "the interrupted run left no temp to promote"); + Assert.False(File.Exists(output)); + + // A previous sync's changes at -OutputFile, and the run that has them: appending, + // sharing reads, the way a resumed run holds its own output. + File.WriteAllLines(output, OldSync); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + using (var live = new StreamWriter( + new FileStream(output, FileMode.Append, FileAccess.Write, FileShare.Read)) + { AutoFlush = true }) + { + var before = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + var requestsBefore = handler.RequestCount; + var (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.ResourceBusy, stop.CategoryInfo.Category); + Assert.Contains($"Another sync is still writing '{output}'", + stop.Exception.Message); + // The temp, not the output, is where these items are counted on the promotion + // route, and going on replaces the output rather than cutting it back. + Assert.Contains($"The resume checkpoint at '{checkpointPath}' records " + + $"{cp.ItemsCollected} items into the temp '{cp.TempFile}'", + stop.Exception.Message); + Assert.Contains($"going on would replace '{output}' with them under the run " + + "that holds it", stop.Exception.Message); + Assert.DoesNotContain("cut that file back", stop.Exception.Message); + Assert.Contains("advance the delta token past them", stop.Exception.Message); + + // No request, no recovery, no token: the temp still holds the changes the + // checkpoint counts, the checkpoint still names it, and the staged file the move + // would have come from was taken back. + Assert.Equal(requestsBefore, handler.RequestCount); + Assert.DoesNotContain(warnings, w => w.Contains("Recovered")); + AssertUnmoved(before, ReadDisk(dir, checkpointPath, temp, output, deltaPath)); + Assert.False(File.Exists($"{output}.adopt")); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + + // And the same inode, not a new file wearing the name: the holder's next write + // is read back through the path. + live.WriteLine("{\"id\":\"old-0000004\"}"); + } + + Assert.Equal([.. OldSync, "{\"id\":\"old-0000004\"}"], File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The window the second run walked through, one cmdlet over. A checkpoint naming a temp that + /// is not on disk was read as changes in no file at all: the sync said so, deleted the + /// position counting them, re-enumerated from the last saved token and moved its own temp over + /// the output - which was the promoting run's output, held and being appended to - and then + /// advanced the token past both runs' work. The output is asked about before any of that now. + /// + [Fact] + public void A_vanished_temp_over_a_held_output_stops_the_sync() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-vanished-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + InterruptedIntoATemp(handler, deltaPath, checkpointPath, output); + + // The instant a promoting run leaves between its temp's unlink and the save that + // repoints the position: the changes are in the output at the length recorded for + // them, the temp is gone, and the checkpoint still names it. + var cp = PaginationCheckpoint.Load(checkpointPath)!; + var temp = Path.Combine(dir, cp.TempFile!); + var recorded = cp.DataLength!.Value; + File.WriteAllBytes(output, File.ReadAllBytes(temp)[..(int)recorded]); + File.Delete(temp); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + using var live = new StreamWriter( + new FileStream(output, FileMode.Append, FileAccess.Write, FileShare.Read)); + + var before = ReadDisk(dir, checkpointPath, output, deltaPath); + var requestsBefore = handler.RequestCount; + var (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Contains($"Another sync is still writing '{output}'", stop.Exception.Message); + // The temp this checkpoint named is gone, not held - so the items are recorded + // (past tense) into a temp the sentence does not go on to name, and going on means + // re-enumerating fresh rather than cutting the output back. + Assert.Contains($"The resume checkpoint at '{checkpointPath}' recorded " + + $"{cp.ItemsCollected} items into a temp that is gone", stop.Exception.Message); + Assert.Contains("going on would re-enumerate from the last saved delta token and " + + $"replace '{output}' under the run that holds it", stop.Exception.Message); + Assert.DoesNotContain("cut that file back", stop.Exception.Message); + Assert.Contains("advance the delta token past them", stop.Exception.Message); + Assert.DoesNotContain(warnings, w => w.Contains("missing or incomplete")); + Assert.Equal(requestsBefore, handler.RequestCount); + AssertUnmoved(before, ReadDisk(dir, checkpointPath, output, deltaPath)); + Assert.True(File.Exists(checkpointPath), + "the position the promoting run is resuming from was deleted"); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// Two syncs released together on the promotion route. Both completed: whichever lost the + /// promotion found the temp gone, warned that the changes it counted were not on disk, deleted + /// the position counting them, re-enumerated over the output the other one was holding, and + /// moved the delta token past work that was no longer in any file. One of them ends the run + /// instead now, and the token moves once. /// - private sealed class ByUrlHandler : HttpMessageHandler + [Fact] + public void Two_syncs_released_together_on_the_promotion_route_advance_the_token_once() { - protected override Task SendAsync( - HttpRequestMessage request, CancellationToken cancellationToken) + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-together-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + var gate = new GatedChangesByUrlHandler(); + var results = new (string[] Warnings, ErrorRecord[] Errors)[2]; + var failures = new Exception?[2]; + var threads = new Thread[2]; + var ready = new[] { new ManualResetEventSlim(false), new ManualResetEventSlim(false) }; + using var go = new ManualResetEventSlim(false); + try { - var body = request.RequestUri!.ToString().Contains("skiptoken=B2", StringComparison.Ordinal) - ? ChangesPage2 - : ChangesPage1; - return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + var setup = new MockHttpHandler(); + setup.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using (MgxTransportScope.Inject(setup)) { - RequestMessage = request, - Content = new StringContent(body, Encoding.UTF8, "application/json") - }); + InterruptedIntoATemp(setup, deltaPath, checkpointPath, output); + } + Assert.NotNull(PaginationCheckpoint.Load(checkpointPath)!.TempFile); + Assert.False(File.Exists(output)); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + + using var transport = MgxTransportScope.Inject(gate); + for (var i = 0; i < threads.Length; i++) + { + var slot = i; + threads[slot] = new Thread(() => + { + try + { + results[slot] = SyncReleasedWith(deltaPath, checkpointPath, output, + ready[slot], go); + } + catch (Exception ex) { failures[slot] = ex; } + }); + threads[slot].Start(); + } + + foreach (var r in ready) + Assert.True(r.Wait(TimeSpan.FromSeconds(60)), "a sync never came up"); + go.Set(); + + // One of them got past the reconcile, so it holds the output and is parked on its + // first request. It stays there until the other has finished, which is what the run + // that stops does with nothing on the wire at all. + Assert.True(gate.Arrived.Wait(TimeSpan.FromSeconds(60)), + "neither sync reached the wire"); + Assert.True( + SpinWait.SpinUntil(() => threads.Any(t => !t.IsAlive), TimeSpan.FromSeconds(60)), + "both syncs were still running with one of them holding the output"); + + gate.Released.Set(); + foreach (var t in threads) + Assert.True(t.Join(TimeSpan.FromSeconds(60)), "a sync never finished"); + Assert.All(failures, Assert.Null); + + // Exactly one run stopped, on one of the files this checkpoint stands for, and it + // neither asked for a page nor said the changes it was protecting were lost. + // + // Which file it stopped on is the race's to decide, and both answers are the same + // run: the stop that loses the temp, or the output the promoter is holding, and the + // stop that reaches the staging name first and finds the promoter's copy standing at + // it. The second is the narrow one - it needs this run to arrive between the other's + // create and its rename - and it is not one platform's: the staging name is swept + // before either claim on both. So the sentence asserted is the one that belongs to + // whichever route the run took, and what is asserted of the run itself is the same + // either way, below: one winner, every change once, and nothing of this one's left + // on the disk. + var stop = Assert.Single(results.SelectMany(r => r.Errors)); + Assert.StartsWith("Checkpoint", stop.FullyQualifiedErrorId, StringComparison.Ordinal); + if (stop.FullyQualifiedErrorId.StartsWith("CheckpointStagingFailed", + StringComparison.Ordinal)) + { + // The promoter's own staged copy, held from the create through the rename. Named + // as that and not as an entry nothing could be made of, and the sentence says + // the changes are still whole in the temp the checkpoint names. + Assert.Contains("another run has the staged copy open", stop.Exception.Message); + Assert.Contains("whole, but staging them into", stop.Exception.Message); + Assert.Contains("Nothing was changed;", stop.Exception.Message); + } + else + { + Assert.Contains("This run stops here; nothing was written.", + stop.Exception.Message); + } + Assert.DoesNotContain(results.SelectMany(r => r.Warnings), + w => w.Contains("missing or incomplete")); + + // One run had the file: every change once, in order, the position spent by the run + // that finished, and the token moved exactly once. + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpointPath)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Empty(Directory.GetFiles(dir, "*.adopt")); + } + finally + { + gate.Released.Set(); + foreach (var t in threads) t?.Join(TimeSpan.FromSeconds(30)); + foreach (var r in ready) r.Dispose(); + gate.Arrived.Dispose(); + gate.Released.Dispose(); + gate.Dispose(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The write a promotion makes before it replaces anything, refused. A directory standing + /// where the staged copy goes is one of the ways it fails, and the answer used to come out of + /// the catch around the whole promotion as "nothing was promoted" - which the caller reads as + /// the temp having gone: it said the changes it counted were not on disk, deleted the position + /// counting them, re-enumerated over the output and moved the delta token past every one of + /// them. The staging says so for itself now, and both files and the token are where they were. + /// + [Fact] + public void A_directory_at_the_staging_name_stops_the_sync_and_changes_nothing() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-staging-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + InterruptedIntoATemp(handler, deltaPath, checkpointPath, output); + var cp = PaginationCheckpoint.Load(checkpointPath)!; + var temp = Path.Combine(dir, cp.TempFile!); + Assert.True(File.Exists(temp), "the interrupted run left no temp to promote"); + + var adopt = $"{output}.adopt"; + Directory.CreateDirectory(adopt); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var before = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + var requestsBefore = handler.RequestCount; + var (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointStagingFailed", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.WriteError, stop.CategoryInfo.Category); + Assert.Contains($"The {cp.ItemsCollected} items the resume checkpoint at " + + $"'{checkpointPath}' records are in '{cp.TempFile}', whole, but staging them " + + $"into '{output}' failed at '{adopt}': ", stop.Exception.Message); + Assert.Contains("Nothing was changed; fix that and run again to recover them, or " + + "remove the temp and the checkpoint to re-enumerate from the last saved delta " + + "token.", stop.Exception.Message); + + // The runtime's own words, punctuated once: the reason is quoted inside a sentence + // that ends it, and a message already ending in a period put two there. + Assert.DoesNotContain(".. Nothing was changed;", stop.Exception.Message); + + Assert.DoesNotContain(warnings, w => w.Contains("missing or incomplete")); + Assert.DoesNotContain(warnings, w => w.Contains("Recovered")); + Assert.Equal(requestsBefore, handler.RequestCount); + AssertUnmoved(before, ReadDisk(dir, checkpointPath, temp, output, deltaPath)); + Assert.True(Directory.Exists(adopt), "the run removed the caller's directory"); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same write refused the other way: another run holding the staging name, which is what + /// two syncs over one command line reach a page apart. Nothing this run may take, and nothing + /// it may report as a temp that has gone. + /// + [Fact] + public void A_held_staging_name_stops_the_sync_and_leaves_the_holders_file() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-staging-held-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + InterruptedIntoATemp(handler, deltaPath, checkpointPath, output); + var cp = PaginationCheckpoint.Load(checkpointPath)!; + var temp = Path.Combine(dir, cp.TempFile!); + var adopt = $"{output}.adopt"; + + using var holder = new FileStream(adopt, FileMode.Create, FileAccess.ReadWrite, + FileShare.None); + holder.Write("{\"id\":\"theirs\"}\n"u8); + holder.Flush(); + var holderBytes = holder.Length; + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var before = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + var requestsBefore = handler.RequestCount; + var (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointStagingFailed", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.WriteError, stop.CategoryInfo.Category); + Assert.Contains($"staging them into '{output}' failed at '{adopt}': ", + stop.Exception.Message); + Assert.Contains("Nothing was changed;", stop.Exception.Message); + Assert.DoesNotContain(warnings, w => w.Contains("missing or incomplete")); + Assert.Equal(requestsBefore, handler.RequestCount); + + AssertUnmoved(before, ReadDisk(dir, checkpointPath, temp, output, deltaPath)); + Assert.Equal(holderBytes, holder.Length); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A pipe at the staging name, which is one of the kinds the sweep removes. What came off + /// the name went to the verbose stream, so the deletion reached nobody who had not asked for + /// verbose - and a sync that removes an entry beside the caller's output has to say it did. + /// It is a warning now, in the one sentence both cmdlets say it in. + /// + [Fact] + public void A_pipe_at_the_staging_name_is_swept_and_the_sync_warns_what_it_removed() + { + if (OperatingSystem.IsWindows()) return; + + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-staging-pipe-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + var adopt = $"{output}.adopt"; + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + InterruptedIntoATemp(handler, deltaPath, checkpointPath, output); + Mkfifo(adopt); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + + // On its own thread with a watchdog: an open of a pipe with no reader never comes + // back, and a test that called the sync inline would hang the suite. + (string[] Warnings, ErrorRecord[] Errors) result = ([], []); + var running = new Thread(() => result = SyncRun(deltaPath, checkpointPath, output)) + { IsBackground = true }; + running.Start(); + Assert.True(running.Join(TimeSpan.FromSeconds(30)), + "the sync never returned: the staging open is waiting for a reader on the pipe"); + + Assert.Empty(result.Errors); + Assert.Contains(result.Warnings, w => w.Contains( + $"Removed what stood at the staging name '{Path.GetFileName(adopt)}': " + + "an entry no copy could be staged in, such as a pipe or a socket.")); + Assert.Contains(result.Warnings, w => w.Contains("Recovered 2 items")); + + // The pipe is gone, the promotion landed in a regular file at the output path, and + // the resumed page is behind the promoted changes. + Assert.False(File.Exists(adopt)); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { File.Delete(adopt); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + private static void Mkfifo(string path) + { + using var mkfifo = System.Diagnostics.Process.Start("/usr/bin/mkfifo", [path]); + Assert.NotNull(mkfifo); + mkfifo.WaitForExit(); + Assert.Equal(0, mkfifo.ExitCode); + } + + /// + /// The two output paths a grant does nothing for. A symlink loop and a FIFO both refuse the + /// open, and both used to end under ErrorCategory.PermissionDenied - the category a caller + /// filtering on it reads as a mode to change, over paths where there is no mode to change + /// and the sentence beside it already said as much. The open's own reason decides the + /// category now, and a 0444 output above still ends under PermissionDenied. + /// + [Fact] + public void An_output_no_grant_would_open_stops_the_sync_under_the_right_category() + { + if (OperatingSystem.IsWindows()) return; + + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-outcategory-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + PromotedAndAppending(handler, deltaPath, checkpointPath, output); + Assert.Null(PaginationCheckpoint.Load(checkpointPath)!.TempFile); + var position = File.ReadAllBytes(checkpointPath); + + // A link that points at itself: the open comes back ELOOP, which is neither a hold + // nor a mode. + File.Delete(output); + using (var ln = System.Diagnostics.Process.Start("/bin/ln", ["-s", output, output])) + { + ln!.WaitForExit(); + Assert.Equal(0, ln.ExitCode); + } + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var requestsBefore = handler.RequestCount; + var loop = Assert.Single(SyncRun(deltaPath, checkpointPath, output).Errors); + + Assert.StartsWith("CheckpointOutputUnopenable", loop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.InvalidOperation, loop.CategoryInfo.Category); + Assert.Contains("Too many levels of symbolic links", loop.Exception.Message); + // No trailing period on this reason, so trimming one off it is a no-op: the single + // period the sentence joins on is the one it always had. + Assert.Contains($"Too many levels of symbolic links : '{output}'. The resume " + + "checkpoint", loop.Exception.Message); + Assert.DoesNotContain("Grant write access", loop.Exception.Message); + + // A pipe: it opens, and then has no length and no offset to write at. + File.Delete(output); + using (var mkfifo = System.Diagnostics.Process.Start("/usr/bin/mkfifo", [output])) + { + mkfifo!.WaitForExit(); + Assert.Equal(0, mkfifo.ExitCode); + } + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var fifo = Assert.Single(SyncRun(deltaPath, checkpointPath, output).Errors); + + Assert.StartsWith("CheckpointOutputUnopenable", fifo.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.InvalidOperation, fifo.CategoryInfo.Category); + Assert.Contains($"'{output}' could not be opened for writing: the file has no offset " + + "to write at.", fifo.Exception.Message); + Assert.DoesNotContain("Grant write access", fifo.Exception.Message); + + // Neither run asked for a page, and neither took the position counting the changes + // the advice tells the caller to come back for. + Assert.Equal(requestsBefore, handler.RequestCount); + Assert.Equal(position, File.ReadAllBytes(checkpointPath)); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { File.Delete(output); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The 410 door sends the attempt loop round again with whatever the attempt before it was + /// holding, and a run that resumed from a checkpoint holds -OutputFile. The door deletes the + /// checkpoint, so the retry normally finds nothing to reconcile - but that delete can fail, + /// on a checkpoint directory this account may read and not unlink from, and then the retry + /// reads the same position again with the output still this run's. The claim answered held, + /// about this run's own handle, and the run ended telling the caller to wait for a second + /// sync and to give it its own -OutputFile: there was no second sync, and both were already + /// this one's. The hold goes with the position it was taken for, which is the position the + /// 410 has just declared dead. + /// + [Fact] + public void An_expired_token_does_not_leave_the_retry_refused_by_this_runs_own_hold() + { + if (OperatingSystem.IsWindows()) return; + + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410hold-{Guid.NewGuid():N}")).FullName; + var cpDir = Directory.CreateDirectory(Path.Combine(dir, "cp")).FullName; + var checkpointPath = Path.Combine(cpDir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // An interrupted first pass, resumed: its changes are in the output, the checkpoint + // records that file and no temp, and there is no delta state yet - a token comes + // with the deltaLink at the end. So the URL this run builds is the URL it rebuilds + // for the retry, and one position is this run's on both attempts. + File.WriteAllText(output, "{\"id\":\"a1\"}\n{\"id\":\"a2\"}\n"); + var recorded = new FileInfo(output).Length; + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users/delta?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = output, + DataLength = recorded, + }.Save(checkpointPath); + + // The delete the 410 door makes, refused: 0555 on the checkpoint's own directory. + using (var chmod = System.Diagnostics.Process.Start("/bin/chmod", ["555", cpDir])) + { + chmod!.WaitForExit(); + Assert.Equal(0, chmod.ExitCode); + } + + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + // The premise: the door could not take the position, so the retry read it again. + Assert.Contains(warnings, w => w.Contains("Could not delete resume checkpoint")); + Assert.Contains(warnings, w => w.Contains("Delta token expired (HTTP 410 Gone)")); + + // And the retry was not refused by the handle the attempt before it left standing. + Assert.DoesNotContain(errors, + e => e.FullyQualifiedErrorId.StartsWith("CheckpointOutputHeld", + StringComparison.Ordinal)); + Assert.DoesNotContain(errors, + e => e.Exception.Message.Contains("Another sync is still writing")); + Assert.Empty(errors); + } + finally + { + using (var chmod = System.Diagnostics.Process.Start("/bin/chmod", ["755", cpDir])) + chmod!.WaitForExit(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The sync's twin of the export's unsearchable-parent advice. The open comes back as an + /// access failure naming the file, and the sentence read it as a mode on the file and told + /// the caller to grant write access to it - a file they cannot stat, let alone chmod, + /// through a parent that refuses to be looked into. The grant is on the directory, and the + /// sentence says which directory. + /// + /// Built from the refusal rather than run into: this sync asks whether it can write beside + /// -OutputFile before it reconciles anything, and a directory that refuses to be searched + /// refuses that probe first - so the sentence is reached here the way the run would reach + /// it, and the words it produces are what this pins. + /// + [Fact] + public void The_unsearchable_parent_stop_says_where_the_grant_belongs() + { + var baseType = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); + var reasonType = baseType.GetNestedType("UnopenableReason", + BindingFlags.NonPublic | BindingFlags.Public)!; + var refusalType = baseType.GetNestedType("CheckpointFileRefusal", + BindingFlags.NonPublic | BindingFlags.Public)!; + var fileType = baseType.GetNestedType("CheckpointFile", + BindingFlags.NonPublic | BindingFlags.Public)!; + var claimType = baseType.GetNestedType("ClaimRefusal", + BindingFlags.NonPublic | BindingFlags.Public)!; + var routeType = baseType.GetNestedType("CheckpointStopRoute", + BindingFlags.NonPublic | BindingFlags.Public)!; + var message = typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).GetMethod( + "CheckpointFileStopMessage", BindingFlags.Static | BindingFlags.NonPublic)!; + + const string vault = "/somewhere/vault"; + const string output = $"{vault}/out.jsonl"; + var reason = Activator.CreateInstance(reasonType, + [$"the directory '{vault}' cannot be searched", true, false, true]); + var refusal = Activator.CreateInstance(refusalType, + [ + Enum.Parse(fileType, "Output"), + Enum.Parse(claimType, "CannotBeOpened"), + Enum.Parse(routeType, "Appending"), + reason, + ]); + var checkpoint = new PaginationCheckpoint { ItemsCollected = 7, DataLength = 42 }; + + var sentence = (string)message.Invoke(null, + [refusal, "/somewhere/run.checkpoint", output, checkpoint])!; + + Assert.Contains($"'{output}' could not be opened for writing: the directory " + + $"'{vault}' cannot be searched.", sentence); + Assert.Contains( + "Grant access to that directory and run again to resume from it, or remove the " + + "checkpoint to sync afresh.", + sentence); + Assert.DoesNotContain("Grant write access to that file", sentence); + } + + /// + /// Both streams of a sync, for the scenarios whose evidence is on the verbose one: whether + /// the run swept the temps beside its output is not something a warning says. + /// + private static (string[] Warnings, string[] Verbose) SyncStreams(string deltaPath, + string checkpointPath, string outputPath, string uri = "/users/delta", + bool latest = false) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", uri) + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("OutputFile", outputPath) + .AddParameter("Verbose"); + if (latest) ps.AddParameter("Latest"); + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + return ([.. ps.Streams.Warning.Select(w => w.Message)], + [.. ps.Streams.Verbose.Select(v => v.Message)]); + } + + /// + /// The shape every release before 2.1 wrote: the position, the count, and nothing about + /// which files the changes are in. Written by rewriting a real checkpoint, so the resource + /// and the nextLink are the ones this run built rather than ones a test guessed at. + /// + private static void StripToPre21Shape(string checkpointPath) + { + var cp = PaginationCheckpoint.Load(checkpointPath)!; + Assert.NotNull(cp.TempFile); + Assert.NotNull(cp.DataLength); + cp.TempFile = null; + cp.OutputFile = null; + cp.DataLength = null; + cp.Save(checkpointPath); + } + + /// + /// A refusal leaves the checkpoint standing because the sync that wrote it resumes from + /// exactly that, and the changes it counted are in a temp beside this output. A checkpoint + /// written before the temp's name was recorded names none, so keyed on that name the sweep + /// in the same run found nothing to spare and deleted the file the spared position was + /// pointing at - seconds after a warning promising both files would be left as they are. + /// The refusal takes the whole sweep with it instead. + /// + [Fact] + public void A_refusal_of_a_checkpoint_naming_no_temp_holds_the_whole_sweep_off() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-sweepoff-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var groupsState = Path.Combine(dir, "groups-state.json"); + var usersState = Path.Combine(dir, "users-state.json"); + var output = Path.Combine(dir, "out.jsonl"); + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // A groups sync killed after its first page: two changes in a temp beside this + // output, no output yet, and a position counting them. + handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); + Sync(groupsState, checkpointPath, output, "/groups/delta"); + var orphan = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + var counted = File.ReadAllBytes(orphan); + StripToPre21Shape(checkpointPath); + Assert.False(File.Exists(output)); + + // A users sync over the same -CheckpointPath and -OutputFile. It refuses that + // position, and dies on its own first request without an output or a temp of its + // own to leave behind. + var (warnings, verbose) = SyncStreams(usersState, checkpointPath, output); + + Assert.Contains(warnings, w => w.Contains( + "records an enumeration this run cannot resume from", StringComparison.Ordinal)); + Assert.Contains(verbose, v => v.Contains( + $"Left the temp files beside '{output}' alone: one of them may hold the items " + + "of the checkpoint this run refused.", StringComparison.Ordinal)); + Assert.True(File.Exists(orphan), + "the sweep took the temp the refused checkpoint stands for"); + Assert.Equal(counted, File.ReadAllBytes(orphan)); + Assert.True(File.Exists(checkpointPath), "the refused position was deleted"); + Assert.False(File.Exists(output), "the run that stopped left an output behind"); + + // And the sync that owns both comes back to find them where it left them. + handler.QueueResponse(HttpStatusCode.OK, GroupsResync); + var (recovered, _) = SyncStreams(groupsState, checkpointPath, output, "/groups/delta"); + + Assert.Contains(recovered, w => w.Contains( + "Recovered 2 items from an interrupted sync's temp file.", + StringComparison.Ordinal)); + Assert.Equal(["{\"id\":\"g1\"}", "{\"id\":\"g2\"}", "{\"id\":\"g9\"}"], + File.ReadAllLines(output)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The corroboration a checkpoint recording no output is believed on is asked for only + /// where there is something to corroborate. Applied to a checkpoint recording neither an + /// output nor a length, the predicate answered no for want of a measurement - so the whole + /// pre-2.1 file-mode shape was refused here while the export recovered it, this cmdlet's + /// own adoption branch was reachable from no shape any release ever wrote, and the position + /// left standing refused every -Latest that followed for as long as it was there. + /// + [Fact] + public void A_checkpoint_recording_neither_field_recovers_and_latest_follows_it() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-neither-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // A first sync, with no state to resume from, killed after its first page: two + // changes in a temp, no output, and no delta token yet. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + var orphan = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + StripToPre21Shape(checkpointPath); + Assert.False(File.Exists(output)); + Assert.False(File.Exists(deltaPath)); + + // The same command line comes back to its own position. The changes it counts are + // in the temp beside the output it names, which is the evidence that shape is + // decided on - and the page that finishes it carries no token, so nothing but the + // checkpoint can have a say in the -Latest below. + handler.QueueResponse(HttpStatusCode.OK, ChangesPageNoToken); + var (warnings, _) = SyncStreams(deltaPath, checkpointPath, output); + + Assert.Contains(warnings, w => w.Contains( + "Recovered 2 items from an interrupted sync's temp file.", + StringComparison.Ordinal)); + Assert.DoesNotContain(warnings, w => w.Contains("no longer corroborate it", + StringComparison.Ordinal)); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b9\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(orphan), "the adopted temp outlived the recovery"); + Assert.False(File.Exists(checkpointPath), + "the completed sync left the position it recovered from behind"); + + // And -Latest is answerable again: the file that would have refused it for good + // went with the sync that finished. + handler.QueueResponse(HttpStatusCode.OK, LatestBaseline); + var (latest, _) = SyncStreams(deltaPath, checkpointPath, output, latest: true); + + Assert.DoesNotContain(latest, w => w.Contains("-Latest ignored", + StringComparison.Ordinal)); + Assert.Contains("$deltatoken=D9", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { Directory.Delete(dir, true); } catch { } } } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ExpandRelationBufferTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ExpandRelationBufferTests.cs new file mode 100644 index 0000000..8336e8d --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/ExpandRelationBufferTests.cs @@ -0,0 +1,86 @@ +using System.Collections; +using System.Management.Automation; +using System.Net; + +namespace Mgx.IntegrationTests; + +/// +/// Expand-MgxRelation holds every piped object until the pipeline ends, because the fan-out +/// cannot start before it knows all the ids. That is a whole collection resident in memory, +/// and the 50,000th arrival is where the cmdlet says so - while the caller can still stop it +/// and put a -Filter or a -Top upstream. +/// +/// The count is the buffer's, not the fan-out's, so the objects below all carry one id: the +/// dedup leaves a single request to answer and what the test measures is the buffering. +/// +[Collection("Pipeline")] +public class ExpandRelationBufferTests +{ + private const int WarnAt = 50_000; + + private static PowerShell Shell() + { + var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + return ps; + } + + /// + /// objects down the pipe, and everything the run had to say. + /// Built here rather than generated in script: the shape under test is what the cmdlet + /// does with the objects, not what it costs PowerShell to make them. + /// + private static (int Output, List Warnings, List Errors, int Requests) + Expand(int count) + { + var input = new Hashtable[count]; + for (int i = 0; i < count; i++) + input[i] = new Hashtable(StringComparer.OrdinalIgnoreCase) { ["id"] = "u1" }; + + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); + using (MgxTransportScope.Inject(wire)) + { + using var ps = Shell(); + ps.Runspace.SessionStateProxy.SetVariable("objects", input); + ps.AddScript("$objects | Expand-MgxRelation -Uri '/users/{id}/manager' -As manager"); + var output = ps.Invoke(); + + return (output.Count, ps.Streams.Warning.Select(w => w.Message).ToList(), + ps.Streams.Error.ToList(), wire.RequestCount); + } + } + + [Fact] + public void The_fifty_thousandth_buffered_object_warns_what_is_being_held() + { + var (output, warnings, errors, requests) = Expand(WarnAt); + + Assert.Empty(errors); + + // One id, so one request: nothing here waited on a fan-out. + Assert.Equal(1, requests); + Assert.Equal(WarnAt, output); + + var warned = Assert.Single(warnings); + Assert.Contains("Buffered 50,000+ objects in memory", warned, StringComparison.Ordinal); + Assert.Contains("-Filter", warned, StringComparison.Ordinal); + Assert.Contains("-Top", warned, StringComparison.Ordinal); + } + + [Fact] + public void One_object_short_of_the_count_is_not_warned_about() + { + var (output, warnings, errors, _) = Expand(WarnAt - 1); + + Assert.Empty(errors); + Assert.Equal(WarnAt - 1, output); + Assert.Empty(warnings); + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ExportCheckpointIntegrityTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ExportCheckpointIntegrityTests.cs index cf6967a..18e1b33 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/ExportCheckpointIntegrityTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/ExportCheckpointIntegrityTests.cs @@ -24,6 +24,13 @@ public class ExportCheckpointIntegrityTests """; private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + private const string EmptyPage = """ + {"value":[],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P3"} + """; + private const string LastPage = """ + {"value":[{"id":"u4"},{"id":"u5"}]} + """; + private sealed class ScriptedHandler : HttpMessageHandler { private readonly Queue<(HttpStatusCode Status, string Body)> _steps = new(); @@ -52,6 +59,36 @@ protected override Task SendAsync( } } + /// + /// Three pages with the middle one empty behind a nextLink, which is how a page boundary is + /// reached with no item written since the fetch. Every answer takes a real asynchronous hop: + /// a socket read always yields and Task.FromResult never does, and the thread the iterator + /// resumes on is what these pages are here to settle. + /// + private sealed class EmptyMiddlePageHandler : HttpMessageHandler + { + private readonly object _lock = new(); + public List Urls { get; } = []; + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + lock (_lock) { Urls.Add(url); } + + await Task.Yield(); + + var body = url.Contains("skiptoken=P3", StringComparison.Ordinal) ? LastPage + : url.Contains("skiptoken=P2", StringComparison.Ordinal) ? EmptyPage + : Page1; + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }; + } + } + private static long? Export(string outputPath, string? checkpointPath, bool all, int top = 0, string uri = "/users") { @@ -80,6 +117,32 @@ protected override Task SendAsync( return null; } + /// + /// An export whose warnings and error records the caller reads, rather than only the item + /// count Export hands back. Nothing is swallowed here: a run that dies off the pipeline + /// thread has to fail the test that says it does not. + /// + private static (List Warnings, int Errors) ExportReporting( + string outputPath, string checkpointPath) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("All"); + ps.Invoke(); + return ([.. ps.Streams.Warning.Select(w => w.Message)], ps.Streams.Error.Count); + } + private static string[] Lines(string path) => File.Exists(path) ? File.ReadAllLines(path) : []; private static string NewDir() => @@ -753,4 +816,50 @@ public void A_checkpoint_written_under_another_spelling_of_the_endpoint_still_re try { Directory.Delete(dir, true); } catch { } } } + + /// + /// A checkpoint the export cannot write, over pages whose middle one is empty behind a + /// nextLink. The page-boundary callback runs on whichever thread the iterator resumed on, + /// and a page that writes nothing resumes on the thread pool - so warning about the disk + /// from there ended the export at that page, taking the pages behind it with it. The + /// message is buffered on the client's warning channel and written by a drain on the + /// pipeline thread: one per failed save, and every page still reaches the output. + /// + [Fact] + public void A_failed_boundary_save_warns_and_the_export_goes_on() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + // Every save into this path fails: Save stages its bytes under this name first, and + // no account writes a file over a directory. The directory holding it is not sealed + // instead, the way a read-only disk would seal it, because the export writes its + // output there too and that write is not what is under test. + Directory.CreateDirectory(checkpoint + ".tmp"); + + var handler = new EmptyMiddlePageHandler(); + using var transport = MgxTransportScope.Inject(handler); + var (warnings, errors) = ExportReporting(output, checkpoint); + + Assert.Equal(0, errors); + + // Both boundaries carrying a nextLink try to save, and both fail. + Assert.Equal(2, warnings.Count( + w => w.StartsWith("Checkpoint save failed (page boundary)", StringComparison.Ordinal))); + + // The empty page sits behind the second request, so an export that died reporting + // its failed save never asks for the third - and the output ends two items short. + Assert.Equal(3, handler.Urls.Count); + Assert.Contains("skiptoken=P3", handler.Urls[2]); + Assert.Equal( + ["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u4\"}", "{\"id\":\"u5\"}"], + Lines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ExportDegradedQueryCheckpointTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ExportDegradedQueryCheckpointTests.cs index 77c96bf..2980d9f 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/ExportDegradedQueryCheckpointTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/ExportDegradedQueryCheckpointTests.cs @@ -190,7 +190,7 @@ protected override Task SendAsync( } private static (long? Count, List Warnings) ExportReporting( - string outputPath, string checkpointPath, string filter) + string outputPath, string checkpointPath, string? filter) { using var ps = PowerShell.Create(); ps.AddCommand("Import-Module") @@ -198,12 +198,12 @@ private static (long? Count, List Warnings) ExportReporting( ps.Invoke(); ps.Commands.Clear(); - ps.AddCommand("Export-MgxCollection") - .AddParameter("Uri", "/users") - .AddParameter("OutputFile", outputPath) - .AddParameter("CheckpointPath", checkpointPath) - .AddParameter("Filter", filter) - .AddParameter("All"); + var cmd = ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("All"); + if (filter != null) cmd.AddParameter("Filter", filter); long? count = null; try { @@ -453,7 +453,8 @@ public void An_export_says_so_when_a_rebuilt_request_costs_it_the_resume_it_anno var (reported, warnings) = ExportReporting(output, checkpoint, filter); Assert.Contains(warnings, w => w.Contains("Resuming from checkpoint")); - Assert.Contains(warnings, w => w.Contains("no longer describes this export")); + Assert.Contains(warnings, + w => w.Contains("under a form of the request the endpoint has since refused")); // And the run really did start over, which is what the warning is about. Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], @@ -708,4 +709,193 @@ public void A_retry_that_dies_before_a_boundary_leaves_only_the_temp_its_checkpo try { Directory.Delete(dir, true); } catch { } } } + // Ids of unequal width, so a byte count taken from one enumeration and applied to the file + // another left lands inside a line rather than on a boundary. + // + // The collection as the first run below reads it, paged two items and then one. An export + // is a snapshot of a collection that goes on changing, and the runs here are minutes apart: + // what the skip token names is a position in THIS enumeration, and the byte count beside it + // describes the bytes THIS enumeration wrote. + private const string GrowingPage1 = """ + {"value":[{"id":"u1"},{"id":"uu22"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2"} + """; + private const string GrowingPage2 = """ + {"value":[{"id":"uuu333"}]} + """; + + // The same collection with one item created at the front, which is what the runs after the + // first enumerate - paged two and two, under a skip token of their own. + private const string GrownPage1 = """ + {"value":[{"id":"a0"},{"id":"u1"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=Q2"} + """; + private const string GrownPage2 = """ + {"value":[{"id":"uu22"},{"id":"uuu333"}]} + """; + + // And the whole of it in a single page: how the endpoint answers the form of the request it + // does not refuse, so the attempt that finishes reaches no page boundary and saves nothing. + private const string GrownWholeCollection = """ + {"value":[{"id":"a0"},{"id":"u1"},{"id":"uu22"},{"id":"uuu333"}]} + """; + + private static readonly string[] GrownCollection = + ["{\"id\":\"a0\"}", "{\"id\":\"u1\"}", "{\"id\":\"uu22\"}", "{\"id\":\"uuu333\"}"]; + + private static HttpResponseMessage Json(HttpStatusCode status, string body) => + new(status) { Content = new StringContent(body, Encoding.UTF8, "application/json") }; + + /// What the resumed-link entry arms is called, + /// so a run can assert by name that does not + /// carry it - it is the one entry that ever answers the refusal these tests inject. + private const string FaultName = "fault-resumed-link"; + + /// + /// An endpoint that pages the form of the request carrying + /// and answers the form without it whole, over a collection that grows once the first run + /// is done with it. The refusal lands on the continuation, which is the shape that leaves a + /// page-boundary checkpoint on disk before the attempt loop goes round again; + /// says which of the three invocations is talking to it. + /// + private static void ProgramGrowingEndpoint( + MockHttpHandler handler, string refusedOption, string refusal, Func run) + { + handler + // The position the first run's enumeration handed out. It dies on it; the run that + // resumes is refused the option its URL carries; the run after that is answered + // from the snapshot the first run was reading. + .When(r => r.Uri.Contains("$skiptoken=P2", StringComparison.Ordinal), FaultName) + .Respond(_ => run() switch + { + 1 => Json(HttpStatusCode.InternalServerError, ServerError), + 2 => Json(HttpStatusCode.BadRequest, refusal), + _ => Json(HttpStatusCode.OK, GrowingPage2) + }) + // The position the enumeration over the grown collection hands out. + .When(r => r.Uri.Contains("$skiptoken=Q2", StringComparison.Ordinal)) + .Respond(_ => Json(HttpStatusCode.OK, GrownPage2)) + // Page one of the form the endpoint pages, against the collection as it stands. + .When(r => r.Uri.Contains(refusedOption, StringComparison.Ordinal)) + .Respond(_ => Json(HttpStatusCode.OK, run() == 1 ? GrowingPage1 : GrownPage1)) + // And the form it does not refuse. + .When(_ => true) + .Respond(_ => Json(HttpStatusCode.OK, GrownWholeCollection)); + } + + /// + /// What a rebuilt request leaves at -CheckpointPath, and what the run after it does with + /// that. The endpoint refuses $top on the continuation, so the attempt loop goes round with + /// a position of this run's already on disk, and the attempt that follows compares it + /// against a URL the run has stopped building and refuses it. That refusal says nothing + /// about whose the file is: it is this export's own position, into an enumeration this run + /// goes on to make again and replace the output with. Kept beside the finished output, it + /// is a position nothing comes back to, and the next run over the same command line reads + /// it as its own - the completed file cut back to the byte count of an enumeration two runs + /// old, and the rest appended from a skip token that counts items it no longer holds. + /// + [Fact] + public void A_checkpoint_refused_for_a_rebuilt_request_goes_with_the_completed_export() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var run = 1; + var handler = new MockHttpHandler(); + ProgramGrowingEndpoint(handler, "$top=", TopUnsupported, () => run); + using var transport = MgxTransportScope.Inject(handler); + + // Run one: page one collected into a temp under the automatic $top, and the + // continuation dies. The checkpoint records that form of the URL and names the temp. + var firstResult = Export(output, checkpoint); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + Assert.Null(firstResult); + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.Contains("$top=", cp!.Resource); + Assert.NotNull(cp.TempFile); + + // Run two: the temp is promoted and the resume announced, then the continuation + // comes back Request_UnsupportedQuery, the URL is rebuilt without $top, and the + // form the endpoint answers hands over the whole collection in one page - so the + // attempt that finishes never reaches a boundary and saves no checkpoint of its own. + run = 2; + var (secondCount, secondWarnings) = ExportReporting(output, checkpoint, null); + + Assert.Contains(secondWarnings, w => w.Contains("Resuming from checkpoint")); + Assert.Equal(GrownCollection, File.ReadAllLines(output)); + Assert.Equal(4, secondCount); + + // Taken here and weighed below, because what the file costs is what run three does. + var checkpointSurvivedRunTwo = File.Exists(checkpoint); + + // Run three, the same command line, with $top answered again. + run = 3; + var (thirdCount, thirdWarnings) = ExportReporting(output, checkpoint, null); + + Assert.Equal(GrownCollection, File.ReadAllLines(output)); + Assert.Equal(4, thirdCount); + Assert.DoesNotContain(thirdWarnings, + w => w.Contains("resum", StringComparison.OrdinalIgnoreCase)); + Assert.False(checkpointSurvivedRunTwo, + "a completed export left a position into the enumeration it had just replaced"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same on the other option: a -Filter run auto-adds $count, the continuation comes + /// back a bare 400, and the URL is rebuilt without it. The checkpoint the attempt before + /// saved is this export's own under the form that carried $count, and the run that refuses + /// it goes on to enumerate the collection again and replace the output. + /// + [Fact] + public void A_checkpoint_refused_for_a_dropped_count_goes_with_the_completed_export() + { + const string filter = "startsWith(displayName,'a')"; + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var run = 1; + var handler = new MockHttpHandler(); + ProgramGrowingEndpoint(handler, "$count=true", CountUnsupported, () => run); + using var transport = MgxTransportScope.Inject(handler); + + var firstResult = Export(output, checkpoint, filter); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + Assert.Null(firstResult); + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.Contains("$count=true", cp!.Resource); + Assert.NotNull(cp.TempFile); + + run = 2; + var (secondCount, secondWarnings) = ExportReporting(output, checkpoint, filter); + + Assert.Contains(secondWarnings, w => w.Contains("Resuming from checkpoint")); + Assert.Equal(GrownCollection, File.ReadAllLines(output)); + Assert.Equal(4, secondCount); + + var checkpointSurvivedRunTwo = File.Exists(checkpoint); + + run = 3; + var (thirdCount, thirdWarnings) = ExportReporting(output, checkpoint, filter); + + Assert.Equal(GrownCollection, File.ReadAllLines(output)); + Assert.Equal(4, thirdCount); + Assert.DoesNotContain(thirdWarnings, + w => w.Contains("resum", StringComparison.OrdinalIgnoreCase)); + Assert.False(checkpointSurvivedRunTwo, + "a completed export left a position into the enumeration it had just replaced"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ExportSharedCheckpointTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ExportSharedCheckpointTests.cs index 24b729d..acd11b3 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/ExportSharedCheckpointTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/ExportSharedCheckpointTests.cs @@ -21,6 +21,14 @@ public class ExportSharedCheckpointTests private const string UsersPage2 = """ {"value":[{"id":"u3"}]} """; + // The same page with a continuation behind it, so a resumed run writes and flushes a page + // before the request after it can be held open. + private const string UsersPage2Continued = """ + {"value":[{"id":"u3"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P3"} + """; + private const string UsersPage3 = """ + {"value":[{"id":"u4"}]} + """; private const string GroupsPage1 = """ {"value":[{"id":"g1"},{"id":"g2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/groups?$skiptoken=P2"} """; @@ -28,6 +36,9 @@ public class ExportSharedCheckpointTests {"value":[{"id":"g3"}]} """; private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + private const string TopUnsupported = """ + {"error":{"code":"Request_UnsupportedQuery","message":"$top is not supported on this endpoint"}} + """; private static readonly string[] OldExport = ["{\"id\":\"old-0000001\"}", "{\"id\":\"old-0000002\"}", "{\"id\":\"old-0000003\"}"]; @@ -36,6 +47,10 @@ private sealed class ScriptedHandler : HttpMessageHandler { private readonly Queue<(HttpStatusCode Status, string Body)> _steps = new(); private readonly object _lock = new(); + private int _served; + + /// How many requests this handler has answered. A run that stops adds none. + public int Served { get { lock (_lock) return _served; } } public void Queue(HttpStatusCode status, string body) { @@ -47,7 +62,83 @@ protected override Task SendAsync( { (HttpStatusCode Status, string Body) step; lock (_lock) + { + _served++; step = _steps.Count > 0 ? _steps.Dequeue() : (HttpStatusCode.InternalServerError, ServerError); + } + return Task.FromResult(new HttpResponseMessage(step.Status) + { + RequestMessage = request, + Content = new StringContent(step.Body, Encoding.UTF8, "application/json") + }); + } + } + + /// + /// Refuses the automatic $top on the page after the first: page one answers with the option + /// on it, the nextLink that follows comes back Request_UnsupportedQuery, and the retry + /// without it finds the whole collection in a single page. An attempt loop that goes round + /// with a page-boundary checkpoint already on disk, and finishes without saving one. + /// + private sealed class TopRefusedAfterPageOneHandler : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + var (status, body) = url.Contains("skiptoken=P2", StringComparison.Ordinal) + ? (HttpStatusCode.BadRequest, TopUnsupported) + : url.Contains("$top=", StringComparison.Ordinal) + ? (HttpStatusCode.OK, UsersPage1) + : (HttpStatusCode.OK, UsersPage2); + return Task.FromResult(new HttpResponseMessage(status) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } + + /// + /// A scripted handler that holds one request open until it is let go: the run parked there + /// has written and flushed everything the pages before it carried, and is appending still. + /// + private sealed class GatedHandler : HttpMessageHandler + { + private readonly Queue<(HttpStatusCode Status, string Body)> _steps = new(); + private readonly int _gateOn; + private readonly object _lock = new(); + private int _served; + + /// Which request, counting from one, is held at the gate. + internal GatedHandler(int gateOn) => _gateOn = gateOn; + + internal void Queue(HttpStatusCode status, string body) + { + lock (_lock) _steps.Enqueue((status, body)); + } + + /// Set the moment the request being gated arrives. + internal ManualResetEventSlim Arrived { get; } = new(false); + + /// Set by the test to let the parked request answer. + internal ManualResetEventSlim Released { get; } = new(false); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + int n; + (HttpStatusCode Status, string Body) step; + lock (_lock) + { + n = ++_served; + step = _steps.Count > 0 ? _steps.Dequeue() : (HttpStatusCode.OK, UsersPage2); + } + if (n == _gateOn) + { + Arrived.Set(); + Released.Wait(cancellationToken); + } return Task.FromResult(new HttpResponseMessage(step.Status) { RequestMessage = request, @@ -56,6 +147,114 @@ protected override Task SendAsync( } } + /// + /// Runs an export on this thread and parks it inside the window the hold exists to close: + /// past the reconcile, which has claimed the output and cut it back, and short of the writer + /// that used to open a handle of its own a page fetch later. The verbose the resume announces + /// itself with is written from exactly there, and a subscriber to that stream blocks the + /// pipeline where it is raised. + /// + private static (string[] Warnings, ErrorRecord[] Errors) ExportParkedAtResume( + string uri, string outputPath, string checkpointPath, + ManualResetEventSlim arrived, ManualResetEventSlim released) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + ps.Streams.Verbose.DataAdding += (_, e) => + { + if (e.ItemAdded is VerboseRecord v + && v.Message.Contains("Resuming from checkpoint", StringComparison.Ordinal) + && !arrived.IsSet) + { + arrived.Set(); + released.Wait(); + } + }; + + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", uri) + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("All") + .AddParameter("Verbose"); + List errors = []; + try { ps.Invoke(); } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + return ([.. ps.Streams.Warning.Select(w => w.Message)], [.. errors]); + } + + /// + /// An export built and waiting to be let go: the runspace is up and the module imported + /// before is set, so what releases is two runs + /// arriving at the reconcile together rather than two runspaces starting up. + /// + private static (string[] Warnings, ErrorRecord[] Errors) ExportReleasedWith( + string uri, string outputPath, string checkpointPath, + ManualResetEventSlim ready, ManualResetEventSlim go) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", uri) + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("All"); + + ready.Set(); + go.Wait(); + List errors = []; + try { ps.Invoke(); } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + return ([.. ps.Streams.Warning.Select(w => w.Message)], [.. errors]); + } + + /// + /// Answers by URL rather than from a queue, and holds the first request that arrives until it + /// is let go. Two runs racing over one command line each get the page they ask for however + /// they interleave, and the one that reaches the wire - which is the one that got past the + /// reconcile - keeps the output for as long as the test needs it to. + /// + private sealed class GatedUsersByUrlHandler : HttpMessageHandler + { + private int _served; + + /// How many requests this handler has answered or is holding. + internal int Served => Volatile.Read(ref _served); + + /// Set the moment the first request arrives. + internal ManualResetEventSlim Arrived { get; } = new(false); + + /// Set by the test to let the parked request answer. + internal ManualResetEventSlim Released { get; } = new(false); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + if (Interlocked.Increment(ref _served) == 1) + { + Arrived.Set(); + Released.Wait(cancellationToken); + } + var body = request.RequestUri!.ToString() + .Contains("skiptoken=P2", StringComparison.Ordinal) ? UsersPage2 : UsersPage1; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } + private static long? Export(string uri, string outputPath, string checkpointPath) { using var ps = PowerShell.Create(); @@ -111,10 +310,106 @@ private static string[] ExportWarnings(string uri, string outputPath, string che return [.. ps.Streams.Warning.Select(w => w.Message)]; } + /// + /// The same export with the error stream captured too. A run that will not touch the two + /// files the checkpoint stands for ends on a terminating error, and the id and the sentence + /// are the whole of what the caller has to act on. Both places PowerShell can put that + /// record are read, since which one it uses depends on how the command was added. + /// + private static (string[] Warnings, ErrorRecord[] Errors) ExportRun(string uri, + string outputPath, string checkpointPath) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", uri) + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("All"); + List errors = []; + try { ps.Invoke(); } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + return ([.. ps.Streams.Warning.Select(w => w.Message)], [.. errors]); + } + + /// + /// Everything a run that stops may not have moved: the checkpoint, the temp it names, the + /// output if there is one, and the directory's own entry list - which is what says no temp + /// of the stopped run's outlived it. + /// + private sealed class DiskState + { + public byte[] Checkpoint = []; + public byte[] Temp = []; + public byte[]? Output; + public string[] Entries = []; + } + + private static DiskState ReadDisk(string dir, string checkpointPath, string temp, + string outputPath) => new() + { + Checkpoint = File.ReadAllBytes(checkpointPath), + Temp = ReadShared(temp), + Output = File.Exists(outputPath) ? ReadShared(outputPath) : null, + Entries = [.. Directory.GetFiles(dir).Select(f => Path.GetFileName(f)) + .OrderBy(n => n, StringComparer.Ordinal)], + }; + + /// + /// The same, for the shape a resumed export leaves: its items are in the output itself and + /// the checkpoint names no temp, so there is no third file to read. + /// + private static DiskState ReadDisk(string dir, string checkpointPath, string outputPath) => new() + { + Checkpoint = File.ReadAllBytes(checkpointPath), + Output = ReadShared(outputPath), + Entries = [.. Directory.GetFiles(dir).Select(f => Path.GetFileName(f)) + .OrderBy(n => n, StringComparer.Ordinal)], + }; + + private static void AssertUnmoved(DiskState before, DiskState after) + { + Assert.Equal(before.Entries, after.Entries); + Assert.Equal(before.Checkpoint, after.Checkpoint); + Assert.Equal(before.Temp, after.Temp); + Assert.Equal(before.Output, after.Output); + } + + /// + /// Reads a file a live run still holds open, offering the sharing that holder asked for. + /// Windows checks the sharing both ways round - what the reader asks of the file, and what + /// it will let a handle already on it go on doing - so File.ReadAllBytes, which asks for + /// FileShare.Read and nothing more, is refused by a run holding the file to write it. + /// + private static byte[] ReadShared(string path) + { + using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + using var ms = new MemoryStream(); + fs.CopyTo(ms); + return ms.ToArray(); + } + private static string NewDir() => Directory.CreateDirectory( Path.Combine(Path.GetTempPath(), $"mgx-export-shared-{Guid.NewGuid():N}")).FullName; + /// + /// A FIFO at , which is one of the things a caller can leave standing + /// where -OutputFile points. .NET has no call for it, so the platform's own is used. + /// + private static void Mkfifo(string path) + { + using var mkfifo = System.Diagnostics.Process.Start("/usr/bin/mkfifo", [path]); + Assert.NotNull(mkfifo); + mkfifo.WaitForExit(); + Assert.Equal(0, mkfifo.ExitCode); + } + /// /// What each refusal says. A checkpoint recording another export's output is a second /// export over one -CheckpointPath, and naming that is what the caller acts on. One @@ -539,4 +834,2167 @@ public void A_refused_temp_this_sweep_could_never_reach_does_not_spare_this_outp try { Directory.Delete(dir, true); } catch { } } } + /// + /// What a refusing run leaves behind when it succeeds. The completion path deletes the + /// checkpoint on the way out - the position it describes is spent - and after a refusal the + /// file on disk is not this run's to spend: an export whose whole collection fits in one + /// page never writes one of its own, so the file deleted there was the other export's, the + /// temp the sweep had just been told to spare belonged to nothing, and that export came + /// back to page one. + /// + [Fact] + public void A_refused_checkpoint_survives_a_refusing_export_that_completes() + { + var dir = NewDir(); + var outA = Path.Combine(dir, "outA.jsonl"); + var outB = Path.Combine(dir, "outB.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // Export A dies on page 2: its two items are in its temp, the output was never + // promoted, and the checkpoint names both. + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", outA, checkpoint); + + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.NotNull(cp.TempFile); + var temp = Path.Combine(dir, cp.TempFile); + var before = File.ReadAllBytes(checkpoint); + + // Export B: its own output over A's checkpoint path, and one page of its own + // collection is all there is - so it refuses A's position as another export's, + // saves none of its own, and reaches the completion path. + handler.Queue(HttpStatusCode.OK, GroupsPage2); + var refusal = ExportWarnings("/groups", outB, checkpoint); + Assert.Contains(refusal, w => w.Contains("belongs to a different export")); + Assert.Equal(["{\"id\":\"g3\"}"], File.ReadAllLines(outB)); + + Assert.True(File.Exists(checkpoint), "the other export's resume position was deleted"); + Assert.Equal(before, File.ReadAllBytes(checkpoint)); + Assert.True(File.Exists(temp), + "the refused checkpoint's items were left with nothing describing them"); + + // A, back where it left off, rather than at page 1. + handler.Queue(HttpStatusCode.OK, UsersPage2); + Assert.Equal(3, Export("/users", outA, checkpoint)); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(outA)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other half: a refusal is not a standing exemption from the delete. A run that + /// refuses what it finds and then saves its own position at its first page boundary has + /// taken the path over, and what the completion path deletes is that. Left standing, it is + /// a position into an enumeration that has finished, naming a temp released a line later - + /// which every later export over that path refuses in turn, warning about a second export + /// that is not there. + /// + [Fact] + public void A_refusing_export_that_saved_its_own_checkpoint_deletes_it_on_completion() + { + var dir = NewDir(); + var outA = Path.Combine(dir, "outA.jsonl"); + var outB = Path.Combine(dir, "outB.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", outA, checkpoint); + Assert.True(File.Exists(checkpoint)); + + // Export B: two pages, so it refuses A's position and then writes its own over the + // same path at the boundary between them. + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + Assert.Equal(3, Export("/users", outB, checkpoint)); + + Assert.False(File.Exists(checkpoint), + "a position into a finished enumeration was left on disk"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And over a retry, where the attempt that saved the checkpoint is not the attempt that + /// finishes. An endpoint refusing the automatic $top sends the loop round again with a + /// boundary checkpoint of this run's already on disk, and the retry - everything it asks + /// for arriving in one page - saves none. Read per attempt, "this run never wrote over the + /// file it refused" is true of the retry and false of the run, and the position left behind + /// is the run's own. + /// + [Fact] + public void A_checkpoint_this_run_saved_before_a_retry_is_deleted_when_the_retry_completes() + { + var dir = NewDir(); + var outA = Path.Combine(dir, "outA.jsonl"); + var outB = Path.Combine(dir, "outB.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + // Export A dies on page 2 and leaves its position behind. + var handler = new ScriptedHandler(); + using (MgxTransportScope.Inject(handler)) + { + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", outA, checkpoint); + Assert.True(File.Exists(checkpoint)); + } + + // Export B refuses it, saves its own at its first page boundary, and is sent round + // the attempt loop by the refusal of $top on the page after. + using (MgxTransportScope.Inject(new TopRefusedAfterPageOneHandler())) + { + Assert.Equal(1, Export("/users", outB, checkpoint)); + } + + Assert.False(File.Exists(checkpoint), + "the position this run saved before its retry was left on disk"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And where a refusal is not about ownership at all, and where refusing is not enough. A + /// checkpoint whose named temp another run still holds counts rows that are in that temp and + /// in no other file, and it is the only thing on disk that counts them - so the holder + /// deletes it itself if it finishes, and keeps its temp for exactly as long as this file is + /// there if it does not. Exporting anyway is what takes it: the first page boundary saves + /// this run's own position over the same path, the completion door deletes that as its own, + /// and the holder's temp is left an orphan for the next sweep. There is no pass this run can + /// make that leaves the pair standing, so it makes none. + /// + [Fact] + public void A_held_temp_stops_the_export_before_anything_is_written() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // A run of this export dies on page 2: the two items it collected are in its temp, + // and the checkpoint beside it records this output and this enumeration. + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", output, checkpoint); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + + // It is not dead. It still holds that temp, the way a run between pages does, with + // the counted rows in it - and the rest of the collection is on the wire, one page + // of it, which is all the run below would have needed. + var counted = File.ReadAllText(temp); + using var live = new StreamWriter(temp, append: false); + live.Write(counted); + live.Flush(); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var before = ReadDisk(dir, checkpoint, temp, output); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + var after = ReadDisk(dir, checkpoint, temp, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointTempHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Contains("Another export is still writing the temp file", + stop.Exception.Message); + Assert.Contains(checkpoint, stop.Exception.Message); + Assert.Contains("the 2 items it records are that run's", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Wait for that export to finish, or give this one its own -OutputFile and " + + "-CheckpointPath.", + stop.Exception.Message); + + // Nothing that promises to leave both files and then replaces them. + Assert.DoesNotContain(warnings, w => w.Contains("exported from the beginning")); + Assert.DoesNotContain(warnings, w => w.Contains("Both files are left as they are")); + + // And nothing happened: no request, no output, and both files exactly as the holder + // left them - down to the directory's entry list, so no temp of this run's outlived + // it either. + Assert.Equal(servedBefore, handler.Served); + AssertUnmoved(before, after); + Assert.False(File.Exists(output), "the stopped run published an output"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same disk with two pages on the wire instead of one, which is the shape that made + /// leaving the file alone worth nothing: a run that reaches one page boundary saves its own + /// position over the very path it has just promised to leave, and walks out of a door that + /// deletes that as this run's own. A run that stops has no page boundary to reach. + /// + [Fact] + public void A_held_temp_stops_the_export_that_would_have_paged_twice() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", output, checkpoint); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"], File.ReadAllLines(temp)); + + var counted = File.ReadAllText(temp); + using var live = new StreamWriter(temp, append: false); + live.Write(counted); + live.Flush(); + + // A first page with a continuation, so the run would save a boundary checkpoint of + // its own, and a second that finishes the collection. + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + var before = ReadDisk(dir, checkpoint, temp, output); + var servedBefore = handler.Served; + var (_, errors) = ExportRun("/users", output, checkpoint); + var after = ReadDisk(dir, checkpoint, temp, output); + + Assert.StartsWith("CheckpointTempHeld", Assert.Single(errors).FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(servedBefore, handler.Served); + AssertUnmoved(before, after); + Assert.Equal(2, PaginationCheckpoint.Load(checkpoint)!.ItemsCollected); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The whole cost of getting that wrong, on one disk. Export A holds its temp with two rows + /// in it; export B over the same command line stops; A then dies, leaving the two files B + /// left alone. A's re-run is what recovers those rows - it promotes the temp over the output + /// and appends the rest of the enumeration - and it can only do that while the checkpoint + /// counting them is there. + /// + [Fact] + public void The_rows_a_held_temp_holds_come_back_after_the_export_that_stopped() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // A collects page one into its temp and is still going. + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", output, checkpoint); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"], File.ReadAllLines(temp)); + + var counted = File.ReadAllText(temp); + using (var live = new StreamWriter(temp, append: false)) + { + live.Write(counted); + live.Flush(); + + // B stops. Nothing is queued for it, and it asks for nothing. + var (_, errors) = ExportRun("/users", output, checkpoint); + Assert.StartsWith("CheckpointTempHeld", + Assert.Single(errors).FullyQualifiedErrorId, StringComparison.Ordinal); + } + + // A dies, leaving exactly the two files B left alone. + Assert.True(File.Exists(checkpoint), "B took A's position with it"); + Assert.True(File.Exists(temp), "B took A's temp with it"); + + // A's re-run. It promotes the two rows and resumes from the page its own checkpoint + // recorded, so the request below is the nextLink and not page one - which is what + // tells a recovery from a fresh export here. + handler.Queue(HttpStatusCode.OK, UsersPage2); + var warnings = ExportWarnings("/users", output, checkpoint); + + Assert.Contains(warnings, w => w.Contains("Recovered 2 items")); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other half of the same refusal, where there is no second export at all: a temp this + /// account cannot open read-write. The advice - grant write access, run again - is the only + /// thing that recovers those rows, so the run giving it must still leave the position they + /// are counted by. It stops rather than export over it. This follows the advice and measures + /// what comes back. + /// + [Fact] + public void An_unopenable_temp_stops_the_export_and_says_how_to_recover_it() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", output, checkpoint); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"], File.ReadAllLines(temp)); + + // 0444: the owner cannot open it read-write either, which is the claim failing on + // permissions rather than on another handle. + File.SetUnixFileMode(temp, UnixFileMode.UserRead); + + var before = ReadDisk(dir, checkpoint, temp, output); + var servedBefore = handler.Served; + var (_, errors) = ExportRun("/users", output, checkpoint); + var after = ReadDisk(dir, checkpoint, temp, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointTempUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Contains(Path.GetFileName(temp), stop.Exception.Message); + Assert.Contains("cannot be opened for writing by this account", stop.Exception.Message); + Assert.Contains("not on sharing", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Grant write access to that file and run again to recover them, or remove it " + + "and the checkpoint to export afresh.", + stop.Exception.Message); + Assert.Equal(servedBefore, handler.Served); + AssertUnmoved(before, after); + Assert.False(File.Exists(output), "the stopped run published an output"); + + // Follow it. + File.SetUnixFileMode(temp, UnixFileMode.UserRead | UnixFileMode.UserWrite); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var warnings = ExportWarnings("/users", output, checkpoint); + + Assert.Contains(warnings, w => w.Contains("Recovered 2 items")); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + try + { + foreach (var f in Directory.EnumerateFiles(dir)) + File.SetUnixFileMode(f, UnixFileMode.UserRead | UnixFileMode.UserWrite); + Directory.Delete(dir, true); + } + catch { } + } + } + /// + /// The disk a resumed export leaves at a page boundary. One run dies with its items in a + /// temp; the next promotes that temp over the output, appends the page it resumed onto, + /// saves its position against the output itself - there is no temp left to name - and dies + /// too. That last position is the one shape a shared -CheckpointPath offers no temp to + /// refuse a second run by. + /// + private static void ResumedAndAppending(ScriptedHandler handler, string output, + string checkpoint) + { + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", output, checkpoint); + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", output, checkpoint); + } + + /// + /// The refusal one file over from the temp claim, and the one it could not reach. A run that + /// has resumed once writes straight into -OutputFile and saves a position naming no temp at + /// all, so a second run over the same command line found nothing in the checkpoint to refuse + /// it by: it judged the file its own, cut it back to the recorded offset under a live writer + /// and appended, and the holder's next write landed past the hole that left. There is no + /// pass this run can make that leaves the pair standing either - the rows the checkpoint + /// counts are in the output now - so it makes none. + /// + [Fact] + public void A_held_output_stops_the_export_before_anything_is_written() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + ResumedAndAppending(handler, output, checkpoint); + + // The premise, off the file: this position counts its items into the output and + // names no temp, which is the shape the claim on the temp has nothing to say about. + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + Assert.Equal(Path.GetFullPath(output), cp.OutputFile); + Assert.Equal(new FileInfo(output).Length, cp.DataLength); + + // Export A is not dead. It holds the output the way a resumed run does - appending, + // sharing reads - and has written one more row since its last save, so the file is + // longer than the checkpoint records and cutting it back leaves a hole under it. + handler.Queue(HttpStatusCode.OK, UsersPage2); + DiskState before, after; + int servedBefore; + string[] warnings; + ErrorRecord[] errors; + using (var live = new StreamWriter( + new FileStream(output, FileMode.Append, FileAccess.Write, FileShare.Read))) + { + live.WriteLine("{\"id\":\"u9-live\"}"); + live.Flush(); + Assert.True(new FileInfo(output).Length > cp.DataLength, + "the holder wrote nothing past the recorded length, so no cut would show"); + + before = ReadDisk(dir, checkpoint, output); + servedBefore = handler.Served; + (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.ResourceBusy, stop.CategoryInfo.Category); + Assert.Contains($"Another export is still writing '{output}'", + stop.Exception.Message); + Assert.Contains(checkpoint, stop.Exception.Message); + Assert.Contains($"records {cp.ItemsCollected} items into", stop.Exception.Message); + Assert.Contains("would cut that file back under it", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Wait for that export to finish, or give this one its own -OutputFile and " + + "-CheckpointPath.", + stop.Exception.Message); + + // And not the sentence for a file whose contents this run never looked at. + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + Assert.DoesNotContain(warnings, w => w.Contains("exported from the beginning")); + + // Nothing ran: no request, and the two files and the directory's entry list + // exactly as the holder left them. Read after the record and not before it, so + // a run that took the checkpoint with it fails on the answer it did not give. + after = ReadDisk(dir, checkpoint, output); + Assert.Equal(servedBefore, handler.Served); + AssertUnmoved(before, after); + + // A goes on, and the row it writes next lands where its last one ended. + live.WriteLine("{\"id\":\"u10-live\"}"); + live.Flush(); + } + + // Ordinal, both of them: the culture-aware comparisons treat a NUL as ignorable, + // which is the one byte this is looking for. + var published = File.ReadAllText(output); + Assert.False(published.Contains('\0'), + "the cut left a hole for the holder's next write to land past"); + Assert.EndsWith("{\"id\":\"u9-live\"}" + Environment.NewLine + + "{\"id\":\"u10-live\"}" + Environment.NewLine, published, + StringComparison.Ordinal); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other half of the same claim, where there is no second export at all: the output the + /// checkpoint counts its items into cannot be opened for writing by this account. It was + /// reported as a file that no longer holds those items - a reading of the file's contents, + /// given about a file this run had not been able to open - and the position counting them + /// was deleted on the strength of it. The advice is the only thing that recovers them, so + /// the run giving it leaves both files standing. This follows the advice and measures what + /// comes back. + /// + [Fact] + public void An_unopenable_output_stops_the_export_and_says_how_to_recover_it() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + ResumedAndAppending(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + var recorded = cp.DataLength!.Value; + + // The rows the checkpoint counts are there, and a row written after its last save + // with them - so a run that can open the file cuts back to the recorded length. + File.AppendAllText(output, "{\"id\":\"u9-past-the-save\"}" + Environment.NewLine); + Assert.True(new FileInfo(output).Length > recorded); + + // 0444: the owner cannot open it read-write either, which is the claim failing on + // permissions rather than on another handle. + File.SetUnixFileMode(output, UnixFileMode.UserRead); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var before = ReadDisk(dir, checkpoint, output); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.PermissionDenied, stop.CategoryInfo.Category); + Assert.Contains($"'{output}' could not be opened for writing:", stop.Exception.Message); + Assert.Contains("is denied", stop.Exception.Message); + // The reason's own trailing period is trimmed before it is quoted, so the sentence + // this joins into is punctuated once, not twice. + Assert.DoesNotContain(".. ", stop.Exception.Message); + Assert.Contains("is denied. The resume checkpoint", stop.Exception.Message); + Assert.Contains($"The resume checkpoint at '{checkpoint}' records", + stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Grant write access to that file and run again to resume from it, or remove it " + + "and the checkpoint to export afresh.", + stop.Exception.Message); + + // Not said about a file whose contents this run could not read, and not acted on. + // The disk is read after the record and not before it, so a run that deleted the + // position fails on the answer it did not give. + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + Assert.Equal(servedBefore, handler.Served); + AssertUnmoved(before, ReadDisk(dir, checkpoint, output)); + Assert.True(File.Exists(checkpoint), + "the position counting the items the advice recovers was deleted"); + + // Follow it. The run that comes back cuts the output to the recorded length and + // appends the page it resumes onto, so the row written past the last save goes and + // nothing is written twice. + File.SetUnixFileMode(output, UnixFileMode.UserRead | UnixFileMode.UserWrite); + handler.Queue(HttpStatusCode.OK, UsersPage2); + var resumed = ExportWarnings("/users", output, checkpoint); + + Assert.DoesNotContain(resumed, w => w.Contains("no longer holds")); + Assert.Equal( + ["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u1\"}", "{\"id\":\"u2\"}", + "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + try + { + foreach (var f in Directory.EnumerateFiles(dir)) + File.SetUnixFileMode(f, UnixFileMode.UserRead | UnixFileMode.UserWrite); + Directory.Delete(dir, true); + } + catch { } + } + } + + /// + /// The answer the claim must not swallow. An output shorter than the length the checkpoint + /// records is not the file that checkpoint describes - the items it counts are in no file at + /// all - and nothing holds it and nothing refuses to open it. That still takes today's + /// route: say so, delete the position, and export from the beginning. + /// + [Fact] + public void A_stale_length_output_still_exports_from_the_beginning() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + ResumedAndAppending(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + + // The output is replaced by one that does not hold those items. + File.WriteAllText(output, "{\"id\":\"short\"}" + Environment.NewLine); + Assert.True(new FileInfo(output).Length < cp.DataLength); + + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + Assert.Empty(errors); + Assert.Contains(warnings, + w => w.Contains($"'{output}' no longer holds the {cp.ItemsCollected} items")); + Assert.False(File.Exists(checkpoint), "the stale position was left on disk"); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The disk a resumed export leaves at a page boundary, with every item in the output + /// exactly once. One run dies with its items in a temp; the next promotes that temp over the + /// output, saves its position against the output itself - there is no temp left to name - + /// and dies at its first request. That last position is the one shape a shared + /// -CheckpointPath offers no temp to refuse a second run by. + /// + private static void PromotedAndAppending(ScriptedHandler handler, string output, + string checkpoint) + { + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", output, checkpoint); + Export("/users", output, checkpoint); + } + + /// + /// Two exports over one command line, both past the stat the reconcile makes. The claim on + /// the output was asked for and let go inside the reconcile and the writer opened its own + /// handle a page fetch later, so for that window neither run held the file: both passed the + /// claim, both cut it back to the recorded offset, both appended, and the file came back + /// holding more lines than belonged in it with the checkpoint deleted over the lot. The + /// claim is the handle now, so the run that takes it keeps it across that window, and the + /// second run meets a file the first one has. + /// + /// A is parked in the window itself. The verbose announcing the resume is written after the + /// reconcile - the claim taken, the row past the last save cut away - and before the writer + /// is opened, which is exactly where the two runs used to overlap. + /// + [Fact] + public void Two_exports_released_together_leave_one_with_the_output() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + // A real second thread, not a scheduled task: the two runs have to be able to be inside + // the cmdlet at the same time, and joining one is not a blocking wait on work the test + // framework might have scheduled behind it. + (string[] Warnings, ErrorRecord[] Errors) aResult = default; + Exception? aFailed = null; + Thread? a = null; + using var arrived = new ManualResetEventSlim(false); + using var released = new ManualResetEventSlim(false); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + PromotedAndAppending(handler, output, checkpoint); + + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + Assert.Equal(Path.GetFullPath(output), cp.OutputFile); + var recorded = cp.DataLength!.Value; + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"], File.ReadAllLines(output)); + Assert.Equal(new FileInfo(output).Length, recorded); + + // A row written after the last save, so the cut A makes is visible from outside. + File.AppendAllText(output, "{\"id\":\"u9-past-the-save\"}" + Environment.NewLine); + Assert.True(new FileInfo(output).Length > recorded); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var servedBefore = handler.Served; + + a = new Thread(() => + { + try + { + aResult = ExportParkedAtResume("/users", output, checkpoint, + arrived, released); + } + catch (Exception ex) { aFailed = ex; } + }); + a.Start(); + Assert.True(arrived.Wait(TimeSpan.FromSeconds(60)), + "export A never reached the window between its reconcile and its writer"); + + // Past the reconcile: the row written after the last save has been cut away. And + // short of the writer: no page has been asked for, so nothing has been appended. + Assert.Equal(recorded, new FileInfo(output).Length); + Assert.Equal(servedBefore, handler.Served); + + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.ResourceBusy, stop.CategoryInfo.Category); + Assert.Contains($"Another export is still writing '{output}'", stop.Exception.Message); + Assert.Contains($"records {cp.ItemsCollected} items into", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + + // B sent no request and took nothing with it: the position A resumes from is still + // on disk, and the file is still exactly what A cut it back to. + Assert.Equal(servedBefore, handler.Served); + Assert.True(File.Exists(checkpoint), "B deleted the position A is resuming from"); + Assert.Equal(recorded, new FileInfo(output).Length); + + released.Set(); + Assert.True(a.Join(TimeSpan.FromSeconds(60)), "export A never finished"); + Assert.Null(aFailed); + Assert.Empty(aResult.Errors); + + // One run had the file. Every item once, in order, and the position spent. + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpoint), "the completed export left its position behind"); + } + finally + { + released.Set(); + a?.Join(TimeSpan.FromSeconds(30)); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// What a caller reading the output while a resumed export appends to it gets. The hold is + /// FileShare.None on Unix because that is the only mode .NET backs with LOCK_EX, and LOCK_EX + /// is the only lock that refuses a second writer there - LOCK_SH, which every other mode + /// takes and which the writer's own open used to be, does not. The cost is that it refuses + /// every .NET opener including a reader, and the help says so; tail and cat take no lock at + /// all and read the file as they always did. Windows needs no such trade: the handle's own + /// write access refuses a second writer, so FileShare.Read can let a reader keep looking. + /// + [Fact] + public void A_resumed_export_holds_the_output_against_a_writer_and_tail_still_reads_it() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + (string[] Warnings, ErrorRecord[] Errors) aResult = default; + Exception? aFailed = null; + Thread? a = null; + var gate = new GatedHandler(gateOn: 2); + try + { + { + // The setup runs on its own wire, so the gate below counts only A's requests. + var handler = new ScriptedHandler(); + using var scoped = MgxTransportScope.Inject(handler); + PromotedAndAppending(handler, output, checkpoint); + Assert.Null(PaginationCheckpoint.Load(checkpoint)!.TempFile); + } + + // A page with a continuation, then the page held at the gate. So by the time A is + // parked it has written and flushed a row of its own and is appending still. + gate.Queue(HttpStatusCode.OK, UsersPage2Continued); + gate.Queue(HttpStatusCode.OK, UsersPage3); + using var transport = MgxTransportScope.Inject(gate); + + a = new Thread(() => + { + try { aResult = ExportRun("/users", output, checkpoint); } + catch (Exception ex) { aFailed = ex; } + }); + a.Start(); + Assert.True(gate.Arrived.Wait(TimeSpan.FromSeconds(60)), + "export A never reached its second request"); + + // The property the hold exists for, on both platforms: no second writer. + var refusedWriter = Assert.ThrowsAny(() => + new FileStream(output, FileMode.Append, FileAccess.Write, FileShare.Read)); + Assert.Contains("being used by another process", refusedWriter.Message); + + if (OperatingSystem.IsWindows()) + { + using var reader = new FileStream(output, FileMode.Open, FileAccess.Read, + FileShare.ReadWrite); + Assert.True(reader.Length > 0); + } + else + { + var refusedReader = Assert.ThrowsAny(() => + new FileStream(output, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)); + Assert.Contains("being used by another process", refusedReader.Message); + + // And what the help promises instead. tail takes no lock, so the rows the run + // has written so far come back exactly as they are on disk - the two it resumed + // over and the one it has appended since. + using var tail = System.Diagnostics.Process.Start( + new System.Diagnostics.ProcessStartInfo + { + FileName = "tail", + ArgumentList = { "-c", "4096", output }, + RedirectStandardOutput = true, + RedirectStandardError = true, + }); + Assert.NotNull(tail); + var read = tail!.StandardOutput.ReadToEnd(); + Assert.True(tail.WaitForExit(30_000), "tail did not finish"); + Assert.Equal(0, tail.ExitCode); + Assert.Equal("{\"id\":\"u1\"}" + Environment.NewLine + + "{\"id\":\"u2\"}" + Environment.NewLine + + "{\"id\":\"u3\"}" + Environment.NewLine, read); + } + + gate.Released.Set(); + Assert.True(a.Join(TimeSpan.FromSeconds(60)), "export A never finished"); + Assert.Null(aFailed); + Assert.Empty(aResult.Errors); + + // And the file is everyone's again the moment the writing ends. + using (var after = new FileStream(output, FileMode.Open, FileAccess.ReadWrite, + FileShare.None)) + { + Assert.True(after.Length > 0); + } + Assert.Equal( + ["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}", "{\"id\":\"u4\"}"], + File.ReadAllLines(output)); + } + finally + { + gate.Released.Set(); + a?.Join(TimeSpan.FromSeconds(30)); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// An output in a directory this account cannot search. File.Exists answers false for it - + /// the same answer it gives for a file that is not there - so the run reported that the + /// output no longer held the items the checkpoint counts, and deleted the position counting + /// them, over a whole file it had never opened. The open is what decides now, and being + /// refused on permissions is not the same answer as finding nothing. + /// + /// And which permission. The refusal is an access failure naming the file, so the stop + /// quoted the runtime's words for it and told the caller to grant write access to that file + /// - a file they cannot stat, let alone chmod, through a parent that refuses to be looked + /// into. The grant that opens it is on the directory, and the run says which directory. + /// Told from a mode on the file, which reaches the same catch, by whether the entry can be + /// asked about at all: a directory missing its search bit refuses every name under it, and + /// one that has the bit answers an absent name with ENOENT. Not by whether the directory + /// itself is there - a stat of it needs the search bit on ITS parent, so a mode-000 + /// directory answers Directory.Exists true. This follows the advice and measures what comes + /// back. + /// + [Fact] + public void An_output_whose_directory_cannot_be_searched_stops_the_export() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var outDir = Directory.CreateDirectory(Path.Combine(dir, "out")).FullName; + var cpDir = Directory.CreateDirectory(Path.Combine(dir, "cp")).FullName; + var output = Path.Combine(outDir, "out.jsonl"); + var checkpoint = Path.Combine(cpDir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + PromotedAndAppending(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + var contents = File.ReadAllBytes(output); + var position = File.ReadAllBytes(checkpoint); + + // 000: the directory cannot be searched, so nothing in it can be reached by name. + // That the stat now says false is the premise - it is the answer this used to be + // decided on. Where it still says true the account is root, a mode says nothing, and + // there is nothing here to measure. + File.SetUnixFileMode(outDir, 0); + if (File.Exists(output)) return; + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.PermissionDenied, stop.CategoryInfo.Category); + Assert.Contains($"'{output}' could not be opened for writing: the directory " + + $"'{outDir}' cannot be searched.", stop.Exception.Message); + Assert.Contains($"The resume checkpoint at '{checkpoint}' records", + stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Grant access to that directory and run again to resume from it, or remove the " + + "checkpoint to export afresh.", + stop.Exception.Message); + + // Not the runtime's words for a mode on the file, and not the advice about it: that + // file is the one thing here the caller cannot reach to change or remove. + Assert.DoesNotContain("is denied", stop.Exception.Message); + Assert.DoesNotContain("Grant write access to that file", stop.Exception.Message); + // The reason carries no period of its own, so the sentence it joins is punctuated + // once. + Assert.DoesNotContain(".. ", stop.Exception.Message); + + // Not a reading of a file it never opened, and nothing acted on it. + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + Assert.Equal(servedBefore, handler.Served); + Assert.True(File.Exists(checkpoint), + "the position counting the items the advice recovers was deleted"); + Assert.Equal(position, File.ReadAllBytes(checkpoint)); + + // Follow it. The grant is on the directory, and the run that comes back resumes into + // the file the checkpoint records. + File.SetUnixFileMode(outDir, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + Assert.Equal(contents, File.ReadAllBytes(output)); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var resumed = ExportWarnings("/users", output, checkpoint); + Assert.DoesNotContain(resumed, w => w.Contains("from the beginning")); + Assert.False(File.Exists(checkpoint)); + } + finally + { + try + { + File.SetUnixFileMode(outDir, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + Directory.Delete(dir, true); + } + catch { } + } + } + + /// + /// A directory standing where the output should be. It is not a file, so File.Exists said + /// no and the run took the same route it takes for an output that is not there at all - + /// warning that the file no longer held the items, and deleting the position. Opening a + /// directory read-write is refused on both platforms, and that is an answer. + /// + [Fact] + public void A_directory_standing_at_the_output_path_stops_the_export() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + PromotedAndAppending(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + var position = File.ReadAllBytes(checkpoint); + + File.Delete(output); + Directory.CreateDirectory(output); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + // A directory is not a permission, whatever errno the open came back with: no grant + // makes one writable at an offset, and the way out is to take it off the path. + Assert.Equal(ErrorCategory.InvalidOperation, stop.CategoryInfo.Category); + Assert.Contains($"'{output}' could not be opened for writing: a directory stands at " + + "the path.", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Remove it (or point -OutputFile elsewhere) and run again to resume from it, or " + + "remove the checkpoint to export afresh.", + stop.Exception.Message); + Assert.DoesNotContain("Grant write access", stop.Exception.Message); + + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + Assert.Equal(servedBefore, handler.Served); + Assert.True(File.Exists(checkpoint), "the position was deleted over a directory"); + Assert.Equal(position, File.ReadAllBytes(checkpoint)); + Assert.True(Directory.Exists(output)); + Assert.Empty(Directory.GetFileSystemEntries(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The disk an interrupted fresh export leaves: its items are in a temp beside the output, + /// the checkpoint names that temp, and there is no output at all yet. The run after it + /// promotes the temp over the output and appends onto what it promoted. + /// + private static void InterruptedIntoATemp(ScriptedHandler handler, string output, + string checkpoint) + { + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", output, checkpoint); + } + + /// + /// The other route that ends in an append, one file over. A run whose checkpoint names a + /// temp promotes it over the output and then appends onto it - and used to hold nothing + /// between the move that put the rows there and the writer's own open a page fetch later. + /// The checkpoint it repoints in that gap names no temp, which is the shape a second run + /// over the same command line reads as an output it may cut back and append to. So the + /// promotion takes the file it promoted into, and the second run meets a file the first + /// one has. + /// + [Fact] + public void A_promoting_export_holds_the_output_it_promoted_into() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + (string[] Warnings, ErrorRecord[] Errors) aResult = default; + Exception? aFailed = null; + Thread? a = null; + using var arrived = new ManualResetEventSlim(false); + using var released = new ManualResetEventSlim(false); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + InterruptedIntoATemp(handler, output, checkpoint); + + // The premise: the items are in a temp, the checkpoint names it, and nothing is at + // -OutputFile for anyone to be holding. + var before = PaginationCheckpoint.Load(checkpoint)!; + Assert.NotNull(before.TempFile); + Assert.False(File.Exists(output)); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var servedBefore = handler.Served; + + a = new Thread(() => + { + try + { + aResult = ExportParkedAtResume("/users", output, checkpoint, + arrived, released); + } + catch (Exception ex) { aFailed = ex; } + }); + a.Start(); + Assert.True(arrived.Wait(TimeSpan.FromSeconds(60)), + "export A never reached the window between its promotion and its writer"); + + // Past the promotion: the rows are in the output, the temp is gone, and the position + // on disk names no temp - which is exactly what a second run reads as an output it + // may take. And short of the writer: no page has been asked for. + // + // Measured off the directory entry, since the run holds the file and a reader of it + // is refused until the run ends. What it holds is read at the end, when it does not. + var promotedBytes = ("{\"id\":\"u1\"}" + Environment.NewLine + + "{\"id\":\"u2\"}" + Environment.NewLine).Length; + Assert.Equal(promotedBytes, new FileInfo(output).Length); + var repointed = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(repointed.TempFile); + Assert.Equal(promotedBytes, repointed.DataLength); + Assert.Equal(["out.jsonl", "run.checkpoint"], + Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal)); + Assert.Equal(servedBefore, handler.Served); + + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.ResourceBusy, stop.CategoryInfo.Category); + Assert.Contains($"Another export is still writing '{output}'", stop.Exception.Message); + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + + // B sent no request and took nothing with it. + Assert.Equal(servedBefore, handler.Served); + Assert.True(File.Exists(checkpoint), "B deleted the position A is resuming from"); + + released.Set(); + Assert.True(a.Join(TimeSpan.FromSeconds(60)), "export A never finished"); + Assert.Null(aFailed); + Assert.Empty(aResult.Errors); + + // One run had the file. Every item once, in order, and the position spent. + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpoint), "the completed export left its position behind"); + } + finally + { + released.Set(); + a?.Join(TimeSpan.FromSeconds(30)); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A -OutputFile the open fails on for a reason no list of access errnos carries. The + /// failures were told apart the other way round - a short list meant "cannot be opened" and + /// everything else meant a second run - so a symlink loop at the output path stopped the + /// export with "Another export is still writing", and the caller was told to wait for a run + /// that could not have existed: nothing can open that path, this export included. + /// + [Fact] + public void A_symlink_loop_at_the_output_path_stops_the_export_as_unopenable() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + ResumedAndAppending(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + + // The output's name now points at itself, one link along. ELOOP, which is neither a + // sharing violation nor a permission the caller could grant. + File.Delete(output); + var other = Path.Combine(dir, "other.jsonl"); + File.CreateSymbolicLink(output, other); + File.CreateSymbolicLink(other, output); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var checkpointBefore = File.ReadAllBytes(checkpoint); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + // Not a permission, and not the category a caller reads as one: nothing about this + // path is granted, and InvalidOperation is what says so. + Assert.Equal(ErrorCategory.InvalidOperation, stop.CategoryInfo.Category); + Assert.Contains($"'{output}' could not be opened for writing:", stop.Exception.Message); + // The runtime's own ELOOP wording, not a guess at "on permissions" - and nothing in + // it a caller could grant, so the advice that follows does not offer to. + Assert.Contains("Too many levels of symbolic links", stop.Exception.Message); + // No trailing period on this reason, so trimming one off it is a no-op: the single + // period the sentence joins on is the one it always had. + Assert.Contains($"Too many levels of symbolic links : '{output}'. The resume " + + "checkpoint", stop.Exception.Message); + Assert.Contains($"The resume checkpoint at '{checkpoint}' records", + stop.Exception.Message); + Assert.DoesNotContain("Another export is still writing", stop.Exception.Message); + Assert.DoesNotContain("Grant write access", stop.Exception.Message); + Assert.Contains( + "Fix that and run again to resume from it, or remove it and the checkpoint to " + + "export afresh.", + stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + + // Nothing ran, and the position counting the items is still there. + Assert.Equal(servedBefore, handler.Served); + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other shape that got past both guards: a FIFO at the output path. It opens read-write + /// without blocking, so the claim answered Taken, and the first question asked of the handle + /// afterwards - how long is the file - threw a NotSupportedException out of the run with the + /// FIFO still open. A stream with no length and no offset to write at is a file this run + /// cannot resume into, decided at the open and let go there. + /// + [Fact] + public void A_fifo_at_the_output_path_stops_the_export_as_unopenable() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + ResumedAndAppending(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + + File.Delete(output); + Mkfifo(output); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var checkpointBefore = File.ReadAllBytes(checkpoint); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.InvalidOperation, stop.CategoryInfo.Category); + Assert.Contains($"'{output}' could not be opened for writing: the file has no " + + "offset to write at.", stop.Exception.Message); + Assert.Contains($"The resume checkpoint at '{checkpoint}' records", + stop.Exception.Message); + Assert.DoesNotContain("Grant write access", stop.Exception.Message); + Assert.Contains( + "Fix that and run again to resume from it, or remove it and the checkpoint to " + + "export afresh.", + stop.Exception.Message); + Assert.IsNotType(stop.Exception); + Assert.Equal(servedBefore, handler.Served); + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + + // And the handle it opened to find that out is closed: a claim of this test's own + // on the same FIFO succeeds, which it would not while the run still held one. + using var claimed = new FileStream(output, FileMode.Open, FileAccess.ReadWrite, + FileShare.None); + } + finally + { + try { File.Delete(output); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The promotion route's own claim on the file it replaces. A run whose checkpoint names a + /// temp puts those rows over the output and appends onto what it put there, and it asked for + /// the file only after the move - which a rename asks nothing about, on either side of it. So + /// the move went through over an output another run was holding and appending to: that run's + /// next write landed in an inode with no name left on it, and the claim taken a statement + /// later answered Taken about a file one statement old. The output is asked for first now, + /// and a held one is the whole answer. + /// + [Fact] + public void A_promotion_over_a_held_output_stops_and_replaces_nothing() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + InterruptedIntoATemp(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + var temp = Path.Combine(dir, cp.TempFile!); + Assert.True(File.Exists(temp), "the interrupted run left no temp to promote"); + Assert.False(File.Exists(output)); + + // A previous export's rows at -OutputFile, and the run that has them: appending, + // sharing reads, the way a resumed run holds its own output. + File.WriteAllLines(output, OldExport); + handler.Queue(HttpStatusCode.OK, UsersPage2); + using (var live = new StreamWriter( + new FileStream(output, FileMode.Append, FileAccess.Write, FileShare.Read)) + { AutoFlush = true }) + { + var before = ReadDisk(dir, checkpoint, temp, output); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.ResourceBusy, stop.CategoryInfo.Category); + Assert.Contains($"Another export is still writing '{output}'", + stop.Exception.Message); + // The temp, not the output, is where these items are counted on the promotion + // route, and going on replaces the output rather than cutting it back - the + // held sentence for a run that had already resumed says neither. + Assert.Contains($"The resume checkpoint at '{checkpoint}' records " + + $"{cp.ItemsCollected} items into the temp '{cp.TempFile}'", + stop.Exception.Message); + Assert.Contains($"going on would replace '{output}' with them under the run " + + "that holds it", stop.Exception.Message); + Assert.DoesNotContain("cut that file back", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", + stop.Exception.Message); + + // No request, no recovery, and nothing moved: the temp still holds the rows the + // checkpoint counts, the checkpoint still names it, and the staged file the move + // would have come from was taken back. + Assert.Equal(servedBefore, handler.Served); + Assert.DoesNotContain(warnings, w => w.Contains("Recovered")); + AssertUnmoved(before, ReadDisk(dir, checkpoint, temp, output)); + Assert.False(File.Exists($"{output}.adopt")); + + // And the same inode, not a new file wearing the name: the holder's next write + // is read back through the path. + live.WriteLine("{\"id\":\"old-0000004\"}"); + } + + Assert.Equal([.. OldExport, "{\"id\":\"old-0000004\"}"], File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same claim, refused the other way, on the route where the move would have done the + /// damage itself: a FIFO at -OutputFile is replaced by a rename, so the caller's pipe and + /// whatever reads it would simply be gone - and the run would then have appended to a regular + /// file nobody was reading. + /// + [Fact] + public void A_promotion_over_a_fifo_output_stops_and_leaves_the_pipe() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + InterruptedIntoATemp(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + var temp = Path.Combine(dir, cp.TempFile!); + Assert.False(File.Exists(output)); + Mkfifo(output); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var checkpointBefore = File.ReadAllBytes(checkpoint); + var tempBefore = File.ReadAllBytes(temp); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.InvalidOperation, stop.CategoryInfo.Category); + Assert.Contains($"'{output}' could not be opened for writing: the file has no " + + "offset to write at.", stop.Exception.Message); + // The temp, not the output, is where these items are counted on the promotion + // route - the fixed lead-in above no longer says so on its own. + Assert.Contains($"The resume checkpoint at '{checkpoint}' records " + + $"{cp.ItemsCollected} items into the temp '{cp.TempFile}'", stop.Exception.Message); + Assert.Contains($"cannot promote them into '{output}'", stop.Exception.Message); + Assert.DoesNotContain("Grant write access", stop.Exception.Message); + Assert.Contains( + "Fix that and run again to recover them, or remove it and the checkpoint to " + + "export afresh.", + stop.Exception.Message); + Assert.Equal(servedBefore, handler.Served); + Assert.DoesNotContain(warnings, w => w.Contains("Recovered")); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.False(File.Exists($"{output}.adopt")); + + // Still a pipe, and still nobody's: a stream with no offset in it is what says the + // first, and a claim of this test's own would be refused if the run had kept one. + using var stillAFifo = new FileStream(output, FileMode.Open, FileAccess.ReadWrite, + FileShare.None); + Assert.False(stillAFifo.CanSeek, "the promotion replaced the caller's FIFO"); + } + finally + { + try { File.Delete(output); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The disk a promotion leaves for the instant between the temp's unlink and the save that + /// repoints the position: the rows are in the output at the length recorded for them, the + /// temp is gone, and the checkpoint still names it. Written out here rather than raced for, + /// so what the route does with each shape can be asked one shape at a time. + /// + private static long PromotedButNotYetRepointed(string dir, string output, string checkpoint) + { + var cp = PaginationCheckpoint.Load(checkpoint)!; + var temp = Path.Combine(dir, cp.TempFile!); + var recorded = cp.DataLength!.Value; + File.WriteAllBytes(output, File.ReadAllBytes(temp)[..(int)recorded]); + File.Delete(temp); + return recorded; + } + + /// + /// The window the second run walked through. A checkpoint naming a temp that is not on disk + /// was read as items in no file at all: the run said so, deleted the position counting them, + /// exported from the beginning and moved its own temp over the output at the end - which was + /// the promoting run's output, held and being appended to. The output is asked about before + /// any of that now, and a held one is the same stop the other two routes reach. + /// + [Fact] + public void A_vanished_temp_over_a_held_output_stops_the_export() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + InterruptedIntoATemp(handler, output, checkpoint); + var recorded = PromotedButNotYetRepointed(dir, output, checkpoint); + Assert.Equal(recorded, new FileInfo(output).Length); + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.NotNull(cp.TempFile); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + using var live = new StreamWriter( + new FileStream(output, FileMode.Append, FileAccess.Write, FileShare.Read)); + + var before = ReadDisk(dir, checkpoint, output); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Contains($"Another export is still writing '{output}'", stop.Exception.Message); + // The temp this checkpoint named is gone, not held - so the items are recorded + // (past tense) into a temp the sentence does not go on to name, and going on means + // exporting fresh rather than cutting the output back. + Assert.Contains($"The resume checkpoint at '{checkpoint}' recorded " + + $"{cp.ItemsCollected} items into a temp that is gone", stop.Exception.Message); + Assert.Contains($"going on would export from the beginning and replace '{output}' " + + "under the run that holds it", stop.Exception.Message); + Assert.DoesNotContain("cut that file back", stop.Exception.Message); + Assert.DoesNotContain(warnings, w => w.Contains("missing or incomplete")); + Assert.Equal(servedBefore, handler.Served); + AssertUnmoved(before, ReadDisk(dir, checkpoint, output)); + Assert.True(File.Exists(checkpoint), + "the position the promoting run is resuming from was deleted"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other half of the vanished-temp claim, where there is no second export at all: the + /// output the promoting run left behind cannot be opened for writing by this account. It used + /// to be reported as a file that never held the recorded items - the same misreading an + /// unopenable output got on every route - and the position counting them deleted on the + /// strength of it. The advice is the only thing that recovers here, so the run giving it + /// leaves the checkpoint standing over a temp that is, in fact, already gone. + /// + [Fact] + public void A_vanished_temp_over_an_unopenable_output_stops_the_export() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + InterruptedIntoATemp(handler, output, checkpoint); + var recorded = PromotedButNotYetRepointed(dir, output, checkpoint); + Assert.Equal(recorded, new FileInfo(output).Length); + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.NotNull(cp.TempFile); + + // 0444: the owner cannot open it read-write either, which is the claim failing on + // permissions rather than on another handle. + File.SetUnixFileMode(output, UnixFileMode.UserRead); + + // Nothing is queued for this attempt: the stop is decided before any request, and a + // response queued here and never served would still be waiting for the fresh export + // below, ahead of the two pages that export actually asks for. + var before = ReadDisk(dir, checkpoint, output); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.PermissionDenied, stop.CategoryInfo.Category); + Assert.Contains($"'{output}' could not be opened for writing:", stop.Exception.Message); + Assert.Contains("is denied", stop.Exception.Message); + // The reason's own trailing period is trimmed before it is quoted, so the sentence + // this joins into is punctuated once, not twice. + Assert.DoesNotContain(".. ", stop.Exception.Message); + Assert.Contains("is denied. The resume checkpoint", stop.Exception.Message); + Assert.Contains($"The resume checkpoint at '{checkpoint}' recorded " + + $"{cp.ItemsCollected} items into a temp that is gone", stop.Exception.Message); + Assert.Contains($"cannot export a fresh copy into '{output}' either", + stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Grant write access to that file and run again, or remove it and the checkpoint " + + "to export afresh.", + stop.Exception.Message); + + Assert.DoesNotContain(warnings, w => w.Contains("missing or incomplete")); + Assert.Equal(servedBefore, handler.Served); + AssertUnmoved(before, ReadDisk(dir, checkpoint, output)); + Assert.True(File.Exists(checkpoint), + "the position counting the vanished temp's items was deleted"); + + // Follow it. Nothing was recoverable from the temp regardless, so the run that comes + // back exports fresh once it can reach the output at all. + File.SetUnixFileMode(output, UnixFileMode.UserRead | UnixFileMode.UserWrite); + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + var resumed = ExportWarnings("/users", output, checkpoint); + + Assert.Contains(resumed, w => w.Contains("missing or incomplete")); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpoint)); + } + finally + { + try + { + File.SetUnixFileMode(output, UnixFileMode.UserRead | UnixFileMode.UserWrite); + Directory.Delete(dir, true); + } + catch { } + } + } + + /// + /// And the shape that is no promotion at all. The temp is gone and the file at -OutputFile is + /// a previous export's, which nothing holds - and its length says nothing either way: two + /// rows of one shape are two rows of another shape's length, so a previous run's output + /// measures exactly what a promotion would have left about as often as a promoted one does. + /// This checkpoint counts its items into the temp, so resuming into that file would put rows + /// the caller has already consumed in front of this run's under no warning at all. It takes + /// the route it always took, which replaces the whole file and cannot leave a wrong one + /// behind. + /// + [Fact] + public void A_vanished_temp_over_an_unheld_output_still_exports_from_the_beginning() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + InterruptedIntoATemp(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + File.Delete(Path.Combine(dir, cp.TempFile!)); + + // A previous export's two rows, of the same shape and so of the same length as the + // two this checkpoint counts into the temp that has gone. + File.WriteAllLines(output, ["{\"id\":\"x1\"}", "{\"id\":\"x2\"}"]); + Assert.Equal(cp.DataLength, new FileInfo(output).Length); + + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + Assert.Empty(errors); + Assert.Contains(warnings, w => w.Contains("missing or incomplete")); + + // The whole file, and not the previous export's first two rows with this + // enumeration's remainder spliced onto them. + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The two runs the shapes above were taken from, released together on the promotion route. + /// Both completed: whichever lost the promotion found the temp gone, warned that the items it + /// counted were not on disk, deleted the position counting them, exported from the beginning + /// and moved its own temp over the file the other one was holding and appending to. One of + /// them ends the run instead now, and nothing it would have taken with it moves. + /// + [Fact] + public void Two_exports_released_together_on_the_promotion_route_leave_one_with_the_file() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var gate = new GatedUsersByUrlHandler(); + var results = new (string[] Warnings, ErrorRecord[] Errors)[2]; + var failures = new Exception?[2]; + var threads = new Thread[2]; + var ready = new[] { new ManualResetEventSlim(false), new ManualResetEventSlim(false) }; + using var go = new ManualResetEventSlim(false); + try + { + var setup = new ScriptedHandler(); + using (MgxTransportScope.Inject(setup)) + { + InterruptedIntoATemp(setup, output, checkpoint); + } + Assert.NotNull(PaginationCheckpoint.Load(checkpoint)!.TempFile); + Assert.False(File.Exists(output)); + + using var transport = MgxTransportScope.Inject(gate); + for (var i = 0; i < threads.Length; i++) + { + var slot = i; + threads[slot] = new Thread(() => + { + try + { + results[slot] = ExportReleasedWith("/users", output, checkpoint, + ready[slot], go); + } + catch (Exception ex) { failures[slot] = ex; } + }); + threads[slot].Start(); + } + + foreach (var r in ready) + Assert.True(r.Wait(TimeSpan.FromSeconds(60)), "an export never came up"); + go.Set(); + + // One of them got past the reconcile, so it holds the output and is parked on its + // first request. It stays there until the other has finished, which is what the run + // that stops does with nothing on the wire at all. + Assert.True(gate.Arrived.Wait(TimeSpan.FromSeconds(60)), + "neither export reached the wire"); + Assert.True( + SpinWait.SpinUntil(() => threads.Any(t => !t.IsAlive), TimeSpan.FromSeconds(60)), + "both exports were still running with one of them holding the output"); + + gate.Released.Set(); + foreach (var t in threads) + Assert.True(t.Join(TimeSpan.FromSeconds(60)), "an export never finished"); + Assert.All(failures, Assert.Null); + + // Exactly one run stopped, on one of the two files this checkpoint stands for, and + // it neither asked for a page nor said the items it was protecting were lost. + var stop = Assert.Single(results.SelectMany(r => r.Errors)); + Assert.StartsWith("Checkpoint", stop.FullyQualifiedErrorId, StringComparison.Ordinal); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.DoesNotContain(results.SelectMany(r => r.Warnings), + w => w.Contains("missing or incomplete")); + + // One run had the file: every item once, in order, and the position spent by the run + // that finished rather than by the run that gave up on it. + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpoint)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Empty(Directory.GetFiles(dir, "*.adopt")); + } + finally + { + gate.Released.Set(); + foreach (var t in threads) t?.Join(TimeSpan.FromSeconds(30)); + foreach (var r in ready) r.Dispose(); + gate.Arrived.Dispose(); + gate.Released.Dispose(); + gate.Dispose(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The write a promotion makes before it replaces anything, refused. A directory standing + /// where the staged copy goes is one of the ways it fails, and the answer used to come out of + /// the catch around the whole promotion as "nothing was promoted" - which the caller reads as + /// the temp having gone: it said the items it counted were not on disk, deleted the position + /// counting them, exported from the beginning and swept the temp that was holding every one. + /// The staging says so for itself now, and both files are where they were found. + /// + [Fact] + public void A_directory_at_the_staging_name_stops_the_export_and_changes_nothing() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + InterruptedIntoATemp(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + var temp = Path.Combine(dir, cp.TempFile!); + Assert.True(File.Exists(temp), "the interrupted run left no temp to promote"); + + var adopt = $"{output}.adopt"; + Directory.CreateDirectory(adopt); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var before = ReadDisk(dir, checkpoint, temp, output); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointStagingFailed", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.WriteError, stop.CategoryInfo.Category); + Assert.Contains($"The {cp.ItemsCollected} items the resume checkpoint at " + + $"'{checkpoint}' records are in '{cp.TempFile}', whole, but staging them into " + + $"'{output}' failed at '{adopt}': ", stop.Exception.Message); + Assert.Contains("Nothing was changed; fix that and run again to recover them, or " + + "remove the temp and the checkpoint to export afresh.", stop.Exception.Message); + + // Not the sentence for a temp that has gone, and not the delete that goes with it. + // The runtime's own words, punctuated once: the reason is quoted inside a sentence + // that ends it, and a message already ending in a period put two there. + Assert.DoesNotContain(".. Nothing was changed;", stop.Exception.Message); + + Assert.DoesNotContain(warnings, w => w.Contains("missing or incomplete")); + Assert.DoesNotContain(warnings, w => w.Contains("Recovered")); + Assert.Equal(servedBefore, handler.Served); + AssertUnmoved(before, ReadDisk(dir, checkpoint, temp, output)); + Assert.True(Directory.Exists(adopt), "the run removed the caller's directory"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same write refused the other way: another run holding the staging name, which is what + /// two exports over one command line reach a page apart. Nothing this run may take, and + /// nothing it may report as a temp that has gone. + /// + [Fact] + public void A_held_staging_name_stops_the_export_and_leaves_the_holders_file() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + InterruptedIntoATemp(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + var temp = Path.Combine(dir, cp.TempFile!); + var adopt = $"{output}.adopt"; + + // The run that has it, holding the staged copy the way a promotion holds its own. + using var holder = new FileStream(adopt, FileMode.Create, FileAccess.ReadWrite, + FileShare.None); + holder.Write("{\"id\":\"theirs\"}\n"u8); + holder.Flush(); + var holderBytes = holder.Length; + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var before = ReadDisk(dir, checkpoint, temp, output); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointStagingFailed", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.WriteError, stop.CategoryInfo.Category); + Assert.Contains($"staging them into '{output}' failed at '{adopt}': ", + stop.Exception.Message); + Assert.Contains("Nothing was changed;", stop.Exception.Message); + Assert.DoesNotContain(warnings, w => w.Contains("missing or incomplete")); + Assert.Equal(servedBefore, handler.Served); + + // The temp and the checkpoint as they were, and the holder's own bytes still theirs: + // a promotion that could not stage does not truncate what it could not open. + AssertUnmoved(before, ReadDisk(dir, checkpoint, temp, output)); + Assert.Equal(holderBytes, holder.Length); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The copy a promotion stages, left behind by a run killed between the write and the + /// rename. No glob of the sweep's shape reaches that name - it is the output's own name with + /// a suffix, not "{output}.{32-hex}.tmp" - so it stood beside the output until somebody + /// noticed it, holding a partial copy of an earlier run's rows. The sweep takes it by name + /// now, and asks for it first, the way it asks for a temp. + /// + [Fact] + public void A_staged_copy_an_interrupted_promotion_left_is_swept() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var adopt = $"{output}.adopt"; + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // What a killed promotion leaves: its staged copy, and no checkpoint counting it. + File.WriteAllLines(adopt, OldExport); + var orphanTemp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllLines(orphanTemp, OldExport); + + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + Export("/users", output, Path.Combine(dir, "run.checkpoint")); + + Assert.False(File.Exists(adopt), "the staged copy outlived the run that swept"); + Assert.False(File.Exists(orphanTemp)); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And the one the sweep may not take: a promotion running right now holds the name it has + /// just staged, and a file another run holds is not an orphan whichever name it wears. + /// + [Fact] + public void A_staged_copy_another_run_holds_is_left_alone() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var adopt = $"{output}.adopt"; + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + using var holder = new FileStream(adopt, FileMode.Create, FileAccess.ReadWrite, + FileShare.None); + holder.Write("{\"id\":\"theirs\"}\n"u8); + holder.Flush(); + var holderBytes = holder.Length; + + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + Export("/users", output, Path.Combine(dir, "run.checkpoint")); + + Assert.True(File.Exists(adopt), "the sweep took a staged copy another run was holding"); + Assert.Equal(holderBytes, holder.Length); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// Both streams of an export, for a scenario whose evidence is on the verbose one: whether + /// the run swept the temps beside its output is not something a warning says. + /// + private static (string[] Warnings, string[] Verbose) ExportStreams(string uri, + string outputPath, string checkpointPath) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", uri) + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("All") + .AddParameter("Verbose"); + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + return ([.. ps.Streams.Warning.Select(w => w.Message)], + [.. ps.Streams.Verbose.Select(v => v.Message)]); + } + + /// + /// A refusal leaves the checkpoint standing because the export that wrote it resumes from + /// exactly that, and the items it counted are in a temp beside this output. A checkpoint + /// written before the temp's name was recorded names none - the shape every release before + /// 2.1 wrote - so keyed on that name the sweep in the same run found nothing to spare and + /// deleted the file the spared position was pointing at, under a warning that had promised + /// to leave things alone. The refusal takes the whole sweep with it instead. + /// + [Fact] + public void A_refusal_of_a_checkpoint_naming_no_temp_holds_the_whole_sweep_off() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var orphan = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // A groups export killed partway: its two items in a temp beside this output, no + // output of its own yet, and a position counting them that names neither file. + var counted = "{\"id\":\"g1\"}\n{\"id\":\"g2\"}\n"; + File.WriteAllText(orphan, counted); + File.WriteAllText(checkpoint, """ + {"resource":"https://graph.microsoft.com/v1.0/groups?$top=999", + "nextLink":"https://graph.microsoft.com/v1.0/groups?$skiptoken=P2", + "itemsCollected":2,"pageItemsAlreadyWritten":0} + """); + + // A users export over the same -CheckpointPath. It refuses that position and dies + // on its first request, so it leaves no output and no temp of its own. + var (warnings, verbose) = ExportStreams("/users", output, checkpoint); + + Assert.Contains(warnings, w => w.Contains("belongs to a different export", + StringComparison.Ordinal)); + Assert.Contains(verbose, v => v.Contains( + $"Left the temp files beside '{output}' alone: one of them may hold the items " + + "of the checkpoint this run refused.", StringComparison.Ordinal)); + Assert.True(File.Exists(orphan), + "the sweep took the temp the refused checkpoint stands for"); + Assert.Equal(counted, File.ReadAllText(orphan)); + Assert.True(File.Exists(checkpoint), "the refused position was deleted"); + Assert.False(File.Exists(output), "the run that stopped left an output behind"); + + // And the export that owns both comes back to find them where it left them. + handler.Queue(HttpStatusCode.OK, GroupsPage2); + var (recovered, _) = ExportStreams("/groups", output, checkpoint); + + Assert.Contains(recovered, w => w.Contains( + "Recovered 2 items from an interrupted export's temp file.", + StringComparison.Ordinal)); + Assert.Equal(["{\"id\":\"g1\"}", "{\"id\":\"g2\"}", "{\"id\":\"g3\"}"], + File.ReadAllLines(output)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// An export run with a -WarningAction of the caller's choosing, both streams captured. A + /// warning is a terminating error under Stop and nothing at all under SilentlyContinue, so + /// where the run writes one decides what a caller is left holding. + /// + private static (string[] Warnings, string[] Verbose, ErrorRecord[] Errors) ExportWith( + string uri, string outputPath, string checkpointPath, string warningAction) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", uri) + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("All") + .AddParameter("Verbose") + .AddParameter("WarningAction", warningAction); + List errors = []; + try { ps.Invoke(); } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + // What -WarningAction Stop raises, and it is not a CmdletInvocationException: left + // uncaught it comes off the thread this runs on and takes the test host with it. + catch (ActionPreferenceStopException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + return ([.. ps.Streams.Warning.Select(w => w.Message)], + [.. ps.Streams.Verbose.Select(v => v.Message)], + [.. errors]); + } + + /// + /// The state a crash leaves partway through a checkpointed export, with a pipe standing at + /// the staging name: the previous export's output, this run's items in the temp beside it, + /// a checkpoint naming both, and an entry the recovery has to take off the name first. + /// + private static (string Temp, string Adopt) InterruptedWithAPipeAtTheStagingName(string dir, + string output, string checkpoint) + { + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = Path.GetFileName(temp), + OutputFile = output, + DataLength = new FileInfo(temp).Length, + }.Save(checkpoint); + var adopt = $"{output}.adopt"; + Mkfifo(adopt); + return (temp, adopt); + } + + /// + /// The sweep's removal warning was written where the deletion happens, which is above the + /// recovery it clears the way for. A warning is a terminating error under -WarningAction + /// Stop, so the run ended there: the pipe deleted, the items still in the temp, the output + /// still the previous export's, and a caller told only that something had been removed. The + /// warning is held until this run's own outcome for the recovery is known and written after + /// it, so a run stopped by it stops with the promotion made and the position repointed at + /// the file it landed in - resumable. + /// + [Fact] + public void A_sweep_under_WarningAction_Stop_leaves_the_recovery_made_and_resumable() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + string? adopt = null; + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + var (temp, adoptPath) = InterruptedWithAPipeAtTheStagingName(dir, output, checkpoint); + adopt = adoptPath; + + // On its own thread with a watchdog: an open of a pipe with no reader never comes + // back, and a test that called the export inline would hang the suite. + (string[] Warnings, string[] Verbose, ErrorRecord[] Errors) result = ([], [], []); + var running = new Thread(() => + result = ExportWith("/users", output, checkpoint, "Stop")) { IsBackground = true }; + running.Start(); + Assert.True(running.Join(TimeSpan.FromSeconds(30)), + "the run never returned: the staging open is waiting for a reader on the pipe"); + + // The run ended on a warning, which is what Stop makes of one. + var stop = Assert.Single(result.Errors); + Assert.StartsWith("ActionPreferenceStop", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + + // And what it ended with: the items promoted into the output, the temp spent, the + // pipe off the name, and a position that records the file they are in. + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"], File.ReadAllLines(output)); + Assert.False(File.Exists(temp), "the promoted temp was left on disk"); + Assert.False(File.Exists(adoptPath), "the pipe outlived the run"); + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + Assert.Equal(Path.GetFullPath(output), cp.OutputFile); + Assert.Equal(new FileInfo(output).Length, cp.DataLength); + Assert.Equal(0, handler.Served); + } + finally + { + if (adopt != null) { try { File.Delete(adopt); } catch { } } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other end of the same preference. -WarningAction SilentlyContinue takes the removal + /// off every stream a caller sees, and an entry beside their output was deleted with nothing + /// anywhere to say so. The verbose line stays where the deletion happens, as the record of + /// when it happened, so a caller who asks for verbose is told either way. + /// + [Fact] + public void A_sweep_under_WarningAction_SilentlyContinue_still_removes_and_says_so_in_verbose() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + string? adopt = null; + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + var (temp, adoptPath) = InterruptedWithAPipeAtTheStagingName(dir, output, checkpoint); + adopt = adoptPath; + handler.Queue(HttpStatusCode.OK, UsersPage2); + + (string[] Warnings, string[] Verbose, ErrorRecord[] Errors) result = ([], [], []); + var running = new Thread(() => + result = ExportWith("/users", output, checkpoint, "SilentlyContinue")) + { IsBackground = true }; + running.Start(); + Assert.True(running.Join(TimeSpan.FromSeconds(30)), + "the run never returned: the staging open is waiting for a reader on the pipe"); + + Assert.Empty(result.Errors); + Assert.Empty(result.Warnings); + Assert.Contains(result.Verbose, v => v.Contains( + $"Removed what stood at the staging name '{Path.GetFileName(adoptPath)}': " + + "an entry no copy could be staged in, such as a pipe or a socket.", + StringComparison.Ordinal)); + Assert.False(File.Exists(adoptPath), "the pipe outlived the run"); + Assert.False(File.Exists(temp), "the promoted temp was left on disk"); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpoint), "the completed export left its position behind"); + } + finally + { + if (adopt != null) { try { File.Delete(adopt); } catch { } } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A staging failure after the sweep took something off the name. The removal cannot be a + /// warning of its own there - a warning in front of a terminating error is what + /// -WarningAction Stop ends the run on instead of the stop - so the stop says it: a reader + /// told only that the staging failed goes looking for a file that is no longer there. The + /// two halves of one disk state, in one sentence, and the same phrase the warning uses. + /// + [Fact] + public void The_staging_stop_names_what_the_sweep_took_off_the_name() + { + var message = typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).GetMethod( + "CheckpointStagingStopMessage", + System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic)!; + var checkpoint = new PaginationCheckpoint { ItemsCollected = 7 }; + string Build(string? removedKind) => (string)message.Invoke(null, [ + Activator.CreateInstance( + typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetNestedType("StagingFailure", + System.Reflection.BindingFlags.NonPublic)!, + ["/d/out.jsonl.adopt", "the volume is full"])!, + "/d/run.checkpoint", "/d/out.jsonl", checkpoint, "out.jsonl.abc.tmp", removedKind, + "Nothing was changed;"])!; + + var swept = Build("a link, and not what it pointed at"); + Assert.Contains("failed at '/d/out.jsonl.adopt': the volume is full, after removing what " + + "stood at the staging name: a link, and not what it pointed at. Nothing was " + + "changed;", swept); + + // And where the name was clear, the sentence says nothing about a removal. + var clear = Build(null); + Assert.Contains("failed at '/d/out.jsonl.adopt': the volume is full. Nothing was changed;", + clear); + Assert.DoesNotContain("after removing", clear); + } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ExportWhatIfCheckpointTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ExportWhatIfCheckpointTests.cs index ee94e7f..526a61d 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/ExportWhatIfCheckpointTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/ExportWhatIfCheckpointTests.cs @@ -5,6 +5,7 @@ using System.Net; using System.Security; using System.Text; +using Mgx.Cmdlets.Base; using Mgx.Engine.Pagination; namespace Mgx.IntegrationTests; @@ -125,6 +126,51 @@ private static void Export(string output, string checkpoint) catch (CmdletInvocationException) { } } + /// + /// The applying run, with its warnings and its terminating record. A run that stops on the + /// staging name gets there before the client, so this needs no transport either - which is + /// what lets one test put the two passes over one disk state side by side. + /// + private static (string[] Warnings, ErrorRecord[] Errors) ExportRun(string output, + string checkpoint) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", output) + .AddParameter("CheckpointPath", checkpoint) + .AddParameter("All"); + List errors = []; + try { ps.Invoke(); } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + return ([.. ps.Streams.Warning.Select(w => w.Message)], [.. errors]); + } + + /// + /// A checkpoint from a release that recorded neither the temp's name nor its length, which + /// is the one shape adoption exists for: a line count, the newest temp beside the output, + /// and no output at all. + /// + private static void SaveOrphanCheckpoint(string checkpoint, long items) => + File.WriteAllText(checkpoint, $$""" + {"resource":"https://graph.microsoft.com/v1.0/users?$top=999", + "nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + "itemsCollected":{{items}},"pageItemsAlreadyWritten":0} + """); + + private static void Mkfifo(string path) + { + using var mkfifo = System.Diagnostics.Process.Start("/usr/bin/mkfifo", [path]); + Assert.NotNull(mkfifo); + mkfifo.WaitForExit(); + Assert.Equal(0, mkfifo.ExitCode); + } + /// /// What a hard kill leaves partway through a fresh checkpointed export: the previous /// export's output, this run's items in a temp beside it, and a checkpoint naming both. @@ -279,6 +325,182 @@ public void WhatIf_over_a_temp_a_running_export_holds_names_an_export() } } + /// + /// And what the preview says about it. A real run over these two files stops, which is not + /// one of the two actions the ShouldProcess gate names against -OutputFile - the gate says + /// which of append and export a run would do, and this run would do neither - so the + /// sentence goes out beside it, on the stream every other checkpoint refusal is written to. + /// In the future tense, and it does not throw: -WhatIf reports what a run would do without + /// ending the pipeline on it, and the caller may have piped a dozen more commands behind + /// this one. + /// + [Fact] + public void WhatIf_over_a_held_temp_says_the_run_would_stop() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + try + { + File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n"); + + using var live = new StreamWriter(temp, append: false); + live.WriteLine("{\"id\":\"u1\"}"); + live.WriteLine("{\"id\":\"u2\"}"); + live.Flush(); + + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = Path.GetFileName(temp), + OutputFile = output, + DataLength = new FileInfo(temp).Length, + }.Save(checkpoint); + + var outputBefore = File.ReadAllBytes(output); + var tempBefore = ReadShared(temp); + var checkpointBefore = File.ReadAllBytes(checkpoint); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf ended the pipeline on a run it only described"); + Assert.Contains(lines, l => + l.Contains("Another export is still writing the temp file") + && l.Contains("This run would stop here; nothing would be written.")); + Assert.DoesNotContain(lines, l => l.Contains("nothing was written")); + Assert.Equal(outputBefore, File.ReadAllBytes(output)); + Assert.Equal(tempBefore, ReadShared(temp)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.Equal([temp], Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same preview one file over. A checkpoint that names no temp counts its items into the + /// output itself, and a real run over one another export still holds stops there - so the + /// gate has to say so, in the tense of a pass that does none of it. The claim it needs is + /// taken and let go here like every other question above the gate: the output, the position + /// and the directory are exactly what they were. + /// + [Fact] + public void WhatIf_over_a_held_output_says_the_run_would_stop() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + File.WriteAllLines(output, ["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"]); + var recorded = new FileInfo(output).Length; + + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = output, + DataLength = recorded, + }.Save(checkpoint); + + // The run that owns it is still going: appending, sharing reads, one row past the + // length its last save recorded. + using var live = new StreamWriter( + new FileStream(output, FileMode.Append, FileAccess.Write, FileShare.Read)); + live.WriteLine("{\"id\":\"u3-live\"}"); + live.Flush(); + + var outputBefore = ReadShared(output); + var checkpointBefore = File.ReadAllBytes(checkpoint); + var entriesBefore = Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray(); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf ended the pipeline on a run it only described"); + Assert.Contains(lines, l => + l.Contains($"Another export is still writing '{output}'") + && l.Contains("This run would stop here; nothing would be written.")); + Assert.DoesNotContain(lines, l => l.Contains("nothing was written")); + Assert.Equal(outputBefore, ReadShared(output)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.Equal(entriesBefore, Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray()); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// What -WhatIf leaves of the directory itself, and not only of the files in it. The + /// recovery the gate reports on compares file names, and which spellings a directory keeps + /// apart is a question answered by creating a file in it and looking for it under the other + /// casing - so a run that reported it would change nothing was asking it by writing in the + /// caller's own output directory. The answer is read off the directory's entries instead, + /// which A_directory_holding_a_lettered_entry_decides_its_case_rule_without_writing pins at + /// the helper, and the entry list here is what says no probe was written on the way through. + /// + /// The entry list and not the write time: the preview makes the create the promotion makes + /// at its own staging name, since that create is what a directory this account cannot write + /// in refuses and asking only whether something already stood there previewed a recovery + /// the run would have stopped on. An entry made and taken back moves the directory's write + /// time, and what the preview owes the caller is that nothing is left of it. + /// + /// The answer this process has cached for the directory is forgotten first, so the question + /// is actually reached here rather than answered from the run that set the scene up. The + /// directory holds three lettered entries by then, which is what a run may read the answer + /// off instead - and reading takes nothing from either. + /// + [Fact] + public void WhatIf_leaves_the_output_directorys_entries_alone() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + try + { + File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n{\"id\":\"old3\"}\n"); + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = Path.GetFileName(temp), + OutputFile = output, + DataLength = new FileInfo(temp).Length, + }.Save(checkpoint); + + MgxCmdletBase.SetDirectoryCaseSensitivity(dir, null); + + var namesBefore = Directory.GetFileSystemEntries(dir).OrderBy(n => n).ToArray(); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf asked for a Graph connection"); + Assert.Contains(lines, l => l.Contains("Append JSONL data") && l.Contains(output)); + Assert.Equal(namesBefore, Directory.GetFileSystemEntries(dir).OrderBy(n => n).ToArray()); + } + finally + { + MgxCmdletBase.SetDirectoryCaseSensitivity(dir, null); + try { Directory.Delete(dir, true); } catch { } + } + } + // Reads a file the test's own live writer still holds open: on Windows the reader // must offer ReadWrite sharing or the holder's write access denies the open. private static byte[] ReadShared(string path) @@ -288,4 +510,409 @@ private static byte[] ReadShared(string path) fs.CopyTo(ms); return ms.ToArray(); } + + /// + /// The preview on the promotion route. A checkpoint naming a temp is recovered by replacing + /// the output with that temp's rows and appending onto them, and a real run over an output + /// another export holds stops there instead - so the gate has to say so. It never asked about + /// the output at all on this route, and named the append the run would not have got to. + /// + [Fact] + public void WhatIf_over_a_held_output_on_the_promotion_route_says_the_run_would_stop() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + try + { + File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n"); + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = Path.GetFileName(temp), + OutputFile = output, + DataLength = new FileInfo(temp).Length, + }.Save(checkpoint); + + // The export that has the output: appending, sharing reads. + using var live = new StreamWriter( + new FileStream(output, FileMode.Append, FileAccess.Write, FileShare.Read)); + live.WriteLine("{\"id\":\"old3-live\"}"); + live.Flush(); + + var outputBefore = ReadShared(output); + var tempBefore = File.ReadAllBytes(temp); + var checkpointBefore = File.ReadAllBytes(checkpoint); + var entriesBefore = Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray(); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf ended the pipeline on a run it only described"); + Assert.Contains(lines, l => + l.Contains($"Another export is still writing '{output}'") + && l.Contains("This run would stop here; nothing would be written.")); + Assert.DoesNotContain(lines, l => l.Contains("nothing was written")); + Assert.Equal(outputBefore, ReadShared(output)); + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.Equal(entriesBefore, Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray()); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The name the promotion stages its copy under, asked about too. A real run whose staging + /// name is another run's - or a directory - stops there with both files left as found, and + /// the gate could not see that at all: it named the recovery a run would have made if it had + /// got past a write it never would have made. + /// + [Fact] + public void WhatIf_over_a_held_staging_name_says_the_run_would_stop() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + var adopt = $"{output}.adopt"; + try + { + File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n"); + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = Path.GetFileName(temp), + OutputFile = output, + DataLength = new FileInfo(temp).Length, + }.Save(checkpoint); + + // The run that has the staging name, holding it the way a promotion holds the copy + // it has just written. + using var holder = new FileStream(adopt, FileMode.Create, FileAccess.ReadWrite, + FileShare.None); + holder.Write("{\"id\":\"theirs\"}\n"u8); + holder.Flush(); + var holderBytes = holder.Length; + + var outputBefore = File.ReadAllBytes(output); + var tempBefore = File.ReadAllBytes(temp); + var checkpointBefore = File.ReadAllBytes(checkpoint); + var entriesBefore = Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray(); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf ended the pipeline on a run it only described"); + Assert.Contains(lines, l => + l.Contains($"staging them into '{output}' failed at '{adopt}'") + && l.Contains("Nothing would be changed;")); + Assert.DoesNotContain(lines, l => l.Contains("Nothing was changed;")); + Assert.DoesNotContain(lines, l => l.Contains("Recovering")); + + Assert.Equal(outputBefore, File.ReadAllBytes(output)); + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.Equal(entriesBefore, Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray()); + Assert.Equal(holderBytes, holder.Length); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + private static void Chmod(string mode, string path) + { + using var chmod = System.Diagnostics.Process.Start("/bin/chmod", [mode, path]); + Assert.NotNull(chmod); + chmod.WaitForExit(); + Assert.Equal(0, chmod.ExitCode); + } + + /// + /// -WhatIf over an output directory this account may read and not write. Recovery stages a + /// copy beside the output, so a real run stops on a create it cannot make - and the gate + /// asked only whether something already stood at the staging name, found nothing, and named + /// the recovery the run would then have failed to make. It makes the create too, at no + /// length, so the preview says what the run would say. + /// + [Fact] + public void WhatIf_over_a_read_only_output_directory_says_the_run_would_stop() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + var adopt = $"{output}.adopt"; + try + { + File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n"); + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = Path.GetFileName(temp), + OutputFile = output, + DataLength = new FileInfo(temp).Length, + }.Save(checkpoint); + + var outputBefore = File.ReadAllBytes(output); + var tempBefore = File.ReadAllBytes(temp); + var checkpointBefore = File.ReadAllBytes(checkpoint); + var entriesBefore = Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray(); + Chmod("555", dir); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + Chmod("755", dir); + + Assert.False(hadErrors, "-WhatIf ended the pipeline on a run it only described"); + var stop = Assert.Single(lines.Where(l => + l.Contains($"staging them into '{output}' failed at '{adopt}'"))); + Assert.Contains("Nothing would be changed;", stop); + Assert.DoesNotContain("Nothing was changed;", stop); + Assert.DoesNotContain(lines, l => l.Contains("Recovering")); + + // The reason is quoted inside a sentence the caller punctuates, and the runtime's + // words for a refused create end in a period of their own. + Assert.DoesNotContain("..", stop.Replace(dir, "")); + + Assert.Equal(outputBefore, File.ReadAllBytes(output)); + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.Equal(entriesBefore, Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray()); + } + finally + { + try { Chmod("755", dir); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A FIFO at the staging name, under -WhatIf. The preview swept it, which made a deletion as + /// real as a run's on the one pass that is supposed to touch nothing: a caller asking what + /// would be changed had an entry beside their output removed in order to be told that + /// nothing would be. It is left standing now and named instead, in a sentence saying what + /// the run would take off the name and what it would put there - and the create test that + /// follows a clear name is skipped, since nothing at the name can be tested without taking + /// it off. + /// + [Fact] + public void WhatIf_leaves_a_fifo_at_the_staging_name_and_names_what_a_run_would_remove() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + var adopt = $"{output}.adopt"; + try + { + File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n"); + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = Path.GetFileName(temp), + OutputFile = output, + DataLength = new FileInfo(temp).Length, + }.Save(checkpoint); + + Mkfifo(adopt); + + var outputBefore = File.ReadAllBytes(output); + var tempBefore = File.ReadAllBytes(temp); + var checkpointBefore = File.ReadAllBytes(checkpoint); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf ended the pipeline on a run it only described"); + Assert.Contains(lines, l => l.Contains( + $"The run would remove what stands at the staging name '{Path.GetFileName(adopt)}' " + + "(an entry no copy could be staged in, such as a pipe or a socket) and stage " + + "its copy there.")); + Assert.Contains(lines, l => l.Contains("Append JSONL data") && l.Contains(output)); + Assert.DoesNotContain(lines, l => l.Contains("failed at")); + Assert.DoesNotContain(lines, l => l.Contains("Removed what stood at")); + + // The pipe is where the preview found it, and so is everything else. + Assert.True(File.Exists(adopt), "the preview removed what it said a run would"); + Assert.Equal(outputBefore, File.ReadAllBytes(output)); + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + } + finally + { + try { File.Delete(adopt); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A checkpoint from before the temp's name and length were recorded, with a directory at + /// the staging name. Adoption stages its copy under that name like a promotion does, and + /// the preview asked neither question: it reported the append the recovery would have + /// made, and the run then found the name unusable, said the output file was missing, + /// deleted the position counting the temp's items and exported from the beginning over it. + /// Both passes sweep and create there now, so the preview stops where the run stops and + /// says so in the run's own words. + /// + [Fact] + public void WhatIf_over_an_orphan_checkpoint_stopped_at_the_staging_name_says_the_run_would_stop() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + var adopt = $"{output}.adopt"; + try + { + // No output at all, which is the only state adoption runs in. + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + SaveOrphanCheckpoint(checkpoint, 2); + Directory.CreateDirectory(adopt); + File.WriteAllText(Path.Combine(adopt, "somebody-elses"), "keep\n"); + + var tempBefore = File.ReadAllBytes(temp); + var checkpointBefore = File.ReadAllBytes(checkpoint); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf ended the pipeline on a run it only described"); + var previewed = Assert.Single(lines.Where(l => + l.Contains($"records are in '{Path.GetFileName(temp)}', whole, but staging them " + + $"into '{output}' failed at '{adopt}': a directory stands at the " + + "path."))); + Assert.Contains("Nothing would be changed;", previewed); + Assert.DoesNotContain(lines, l => l.Contains("output file is missing")); + Assert.DoesNotContain(lines, l => l.Contains("Recovered")); + + // Nothing of the three moved, and the preview did not delete the position counting + // the temp's items. + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.False(File.Exists(output)); + Assert.Equal("keep\n", File.ReadAllText(Path.Combine(adopt, "somebody-elses"))); + + // And the run the preview described stops the same way, on the same sentence in the + // applying tense, with the checkpoint still there to come back to. + var (warnings, errors) = ExportRun(output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointStagingFailed", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.WriteError, stop.CategoryInfo.Category); + Assert.Contains($"records are in '{Path.GetFileName(temp)}', whole, but staging them " + + $"into '{output}' failed at '{adopt}': a directory stands at the path.", + stop.Exception.Message); + Assert.Contains("Nothing was changed;", stop.Exception.Message); + Assert.DoesNotContain(warnings, w => w.Contains("output file is missing")); + + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.False(File.Exists(output)); + Assert.Equal("keep\n", File.ReadAllText(Path.Combine(adopt, "somebody-elses"))); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same checkpoint with a FIFO at the staging name, on the adoption route. The preview + /// names what the run would take off the name and leaves the pipe where it is; the run that + /// follows takes it off and adopts. Leaving it is not what wedges a later run - the run's + /// own pass clears it before it opens anything - and removing it was a deletion made by a + /// pass whose whole promise is that it makes none. + /// + [Fact] + public void WhatIf_over_an_orphan_checkpoint_names_the_fifo_and_the_run_that_follows_adopts() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + var adopt = $"{output}.adopt"; + var kind = "an entry no copy could be staged in, such as a pipe or a socket"; + var removed = $"Removed what stood at the staging name '{Path.GetFileName(adopt)}': " + + kind + "."; + var wouldRemove = "The run would remove what stands at the staging name " + + $"'{Path.GetFileName(adopt)}' ({kind}) and stage its copy there."; + try + { + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + SaveOrphanCheckpoint(checkpoint, 2); + Mkfifo(adopt); + + var tempBefore = File.ReadAllBytes(temp); + var checkpointBefore = File.ReadAllBytes(checkpoint); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf ended the pipeline on a run it only described"); + Assert.Contains(lines, l => l.Contains(wouldRemove)); + Assert.Contains(lines, l => l.Contains("Append JSONL data") && l.Contains(output)); + Assert.DoesNotContain(lines, l => l.Contains("failed at")); + Assert.DoesNotContain(lines, l => l.Contains("Removed what stood at")); + + // The pipe is where the preview found it, and so are the two files the checkpoint + // stands for: the temp, and the output that is still not there. + Assert.True(File.Exists(adopt), "the preview removed what it said a run would"); + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.False(File.Exists(output)); + + // And the applying run over the state the preview left - the pipe still at the name + // - takes it off, warns, and adopts, on a thread with a watchdog: the failure this + // pins is a run that does not come back at all. + var handler = new CountingHandler(); + using var transport = MgxTransportScope.Inject(handler); + string[] warnings = []; + var running = new Thread(() => warnings = ExportRun(output, checkpoint).Warnings) + { IsBackground = true }; + running.Start(); + Assert.True(running.Join(TimeSpan.FromSeconds(30)), + "the run never returned: the staging open is waiting for a reader on the FIFO"); + + Assert.Contains(warnings, w => w.Contains(removed)); + Assert.Contains(warnings, w => w.Contains("Recovered 2 items")); + Assert.False(File.Exists(adopt)); + Assert.Equal( + ["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { File.Delete(adopt); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/NamedTempPromotionOwnershipTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/NamedTempPromotionOwnershipTests.cs index 0954fb7..2dc2473 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/NamedTempPromotionOwnershipTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/NamedTempPromotionOwnershipTests.cs @@ -67,7 +67,13 @@ private static string NewDir() => Directory.CreateDirectory( Path.Combine(Path.GetTempPath(), $"mgx-promote-owner-{Guid.NewGuid():N}")).FullName; - private sealed record RunResult(List Warnings, List Output); + /// + /// What a run leaves on the streams. Errors carries the terminating record a run that will + /// not touch a checkpoint's held or unopenable temp ends on, read from both places + /// PowerShell can put it. + /// + private sealed record RunResult(List Warnings, List Output, + List Errors); private static PowerShell Shell(bool withContext = false) { @@ -95,9 +101,29 @@ private static RunResult Export(string output, string checkpoint) .AddParameter("All"); // A run that dies is the subject here, so a terminating error is an expected outcome. List results = []; + List errors = []; try { results = [.. ps.Invoke()]; } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + return new RunResult([.. ps.Streams.Warning.Select(w => w.Message)], results, errors); + } + + /// + /// The same export with the verbose stream captured. What the stale-temp sweep left alone, + /// and why, is said there and nowhere else. + /// + private static List ExportVerbose(string output, string checkpoint) + { + using var ps = Shell(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", output) + .AddParameter("CheckpointPath", checkpoint) + .AddParameter("All") + .AddParameter("Verbose", true); + try { ps.Invoke(); } catch (CmdletInvocationException) { } - return new RunResult([.. ps.Streams.Warning.Select(w => w.Message)], results); + return [.. ps.Streams.Verbose.Select(v => v.Message)]; } private static RunResult Sync(string deltaPath, string checkpoint, string output) @@ -109,9 +135,11 @@ private static RunResult Sync(string deltaPath, string checkpoint, string output .AddParameter("CheckpointPath", checkpoint) .AddParameter("OutputFile", output); List results = []; + List errors = []; try { results = [.. ps.Invoke()]; } - catch (CmdletInvocationException) { } - return new RunResult([.. ps.Streams.Warning.Select(w => w.Message)], results); + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + return new RunResult([.. ps.Streams.Warning.Select(w => w.Message)], results, errors); } /// @@ -135,6 +163,10 @@ private static StreamWriter HoldLikeALiveRun(string tempPath) /// one has finished. The second run must not take the file the first is writing: it copied /// those rows into its own output, unlinked the temp, and the first run's closing move then /// failed with a FileNotFoundException reported against the output it never got to write. + /// + /// Nor may it export around them, which is what taking them looks like a page later: the + /// output it would replace is the one the checkpoint records, and the position it would save + /// goes to the same path. It stops, and says which of the two files to deal with. /// [Fact] public void An_export_does_not_promote_a_temp_a_running_export_still_holds() @@ -158,24 +190,29 @@ public void An_export_does_not_promote_a_temp_a_running_export_still_holds() // It is not dead. It still holds that temp, with its two rows in it. using var live = HoldLikeALiveRun(temp); - // The second run, same -OutputFile and same -CheckpointPath, which finishes. + // The second run, same -OutputFile and same -CheckpointPath, with the whole + // collection waiting for it on the wire. handler.Queue(HttpStatusCode.OK, Page1); handler.Queue(HttpStatusCode.OK, Page2); var second = Export(output, checkpoint); Assert.True(File.Exists(temp), "a running export's temp was unlinked under it"); Assert.DoesNotContain(second.Warnings, w => w.Contains("Recovered")); - Assert.Contains(second.Warnings, w => - w.Contains("Another export is still writing the temp file") - && w.Contains(checkpoint) - && w.Contains("exported from the beginning")); - - // Its output is what it enumerated itself, not the other run's rows. - Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], - File.ReadAllLines(output)); - - // And the first run goes on: the page it fetched since its last save lands, and - // the move it ends with finds the file it has been writing all along. + var stop = Assert.Single(second.Errors); + Assert.StartsWith("CheckpointTempHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Contains("Another export is still writing the temp file", stop.Exception.Message); + Assert.Contains(checkpoint, stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.DoesNotContain("cannot be opened for writing by this account", + stop.Exception.Message); + + // And it published no output of its own: the file at -OutputFile is still the one + // the checkpoint counts bytes of, which here is no file at all. + Assert.False(File.Exists(output), "the stopped run published an output"); + + // And the first run goes on, undisturbed: the page it fetched since its last save + // lands, and the move it ends with finds the file it has been writing all along. live.WriteLine("{\"id\":\"u3-live\"}"); live.Flush(); Assert.True(File.Exists(temp)); @@ -302,7 +339,8 @@ public void A_held_temp_does_not_change_who_a_checkpoint_without_an_output_belon handler.Queue(HttpStatusCode.OK, Page2); var run = Export(output, checkpoint); - Assert.Contains(run.Warnings, w => w.Contains("Another export is still writing the temp file")); + Assert.StartsWith("CheckpointTempHeld", Assert.Single(run.Errors).FullyQualifiedErrorId, + StringComparison.Ordinal); Assert.DoesNotContain(run.Warnings, w => w.Contains("no longer corroborate it")); Assert.True(File.Exists(temp), "a running export's temp was unlinked under it"); } @@ -313,7 +351,8 @@ public void A_held_temp_does_not_change_who_a_checkpoint_without_an_output_belon /// The same helper, reached the same way, from the other cmdlet. A sync has one thing more /// to lose: taking the live run's temp and then resuming past its items advances the delta /// token over changes that are in no file this run can produce, and a delta token that has - /// moved cannot be asked for them again. + /// moved cannot be asked for them again. Re-enumerating instead moves it just the same, one + /// completed page later, so the run ends before either. /// [Fact] public void A_sync_does_not_promote_a_temp_a_running_sync_still_holds() @@ -344,11 +383,17 @@ public void A_sync_does_not_promote_a_temp_a_running_sync_still_holds() Assert.True(File.Exists(temp), "a running sync's temp was unlinked under it"); Assert.DoesNotContain(second.Warnings, w => w.Contains("Recovered")); - Assert.Contains(second.Warnings, w => - w.Contains("Another sync is still writing the temp file") - && w.Contains("re-enumerates from the last saved delta token")); - Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], - File.ReadAllLines(output)); + var stop = Assert.Single(second.Errors); + Assert.StartsWith("CheckpointTempHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Contains("Another sync is still writing the temp file", stop.Exception.Message); + Assert.Contains("advance the delta token past them", stop.Exception.Message); + Assert.DoesNotContain("cannot be opened for writing by this account", + stop.Exception.Message); + + // Nothing of the second run's is on disk: the output is still the baseline's, and + // the token is still the one the interrupted run was working from. + Assert.Equal(["{\"id\":\"a1\"}", "{\"id\":\"a2\"}"], File.ReadAllLines(output)); live.WriteLine("{\"id\":\"b3-live\"}"); live.Flush(); @@ -360,6 +405,263 @@ public void A_sync_does_not_promote_a_temp_a_running_sync_still_holds() finally { try { Directory.Delete(dir, true); } catch { } } } + /// + /// Takes write access away from a file the way each platform expresses it. Unix mode bits + /// and a Windows deny entry are different mechanisms and neither exists on the other host, + /// so the choice is made per OS and each branch executes on the host it is for. + /// + /// The read-only attribute refuses the open the same way and is not the form used here, + /// because it refuses the unlink as well: one of these cases is a sweep that takes the name + /// back, and under the attribute the copy outlived it. A deny entry for the account the + /// tests run as refuses reading and writing the file and says nothing about deleting it, + /// which is the directory's to grant - so the entry the run meets is a file it cannot open + /// and can still take off the name, which is what mode 000 is on the other host. + /// + /// + private static void MakeUnwritable(string path) + { + if (OperatingSystem.IsWindows()) + { + // The premise itself, so a run that then opened the file would be measuring nothing. + Assert.Equal(0, Icacls(path, "/deny", $"{Environment.UserName}:(R,W)")); + } + else + { + File.SetUnixFileMode(path, + UnixFileMode.UserRead | UnixFileMode.GroupRead | UnixFileMode.OtherRead); + } + } + + /// Puts it back, so the directory can be removed on either platform. + private static void MakeWritable(string path) + { + try + { + if (!File.Exists(path)) return; + if (OperatingSystem.IsWindows()) + Icacls(path, "/remove:d", Environment.UserName); + else + File.SetUnixFileMode(path, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException + or System.ComponentModel.Win32Exception) { } + } + + /// + /// One icacls call over the file, waited for, with its exit code for the caller to read. + /// .NET's own access-control types are Windows-only and this assembly is built for a target + /// that has none of them, so the platform's own command is what edits an entry here. + /// + private static int Icacls(string path, params string[] arguments) + { + var start = new System.Diagnostics.ProcessStartInfo("icacls") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + }; + start.ArgumentList.Add(path); + foreach (var argument in arguments) start.ArgumentList.Add(argument); + using var icacls = System.Diagnostics.Process.Start(start); + Assert.NotNull(icacls); + icacls.WaitForExit(); + return icacls.ExitCode; + } + + /// + /// Whether this account can still open the file read-write after that. CI runs as an + /// ordinary user and cannot, which is the case these tests are about; a root shell opens a + /// 0444 file and reaches the ordinary recovery instead. Both answers are asserted, so the + /// test executes and counts whichever host it lands on rather than being skipped on one. + /// + private static bool StillOpenableReadWrite(string path) + { + try + { + using var _ = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite); + return true; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { return false; } + } + + /// + /// The other way the exclusive claim fails, which the refusal used to report as the first. + /// A temp this account cannot open read-write - mode bits, an ACL - stops the run for the + /// same reason and leaves the same two files behind, but there is no second export anywhere: + /// the sentence saying another run was still writing it sent the caller looking for + /// something that does not exist instead of at the file's permissions. + /// + [Fact] + public void An_export_names_a_temp_it_cannot_open_as_a_permissions_problem() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + string? temp = null; + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + handler.Queue(HttpStatusCode.OK, Page1); + Export(output, checkpoint); + temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + + MakeUnwritable(temp); + var openable = StillOpenableReadWrite(temp); + + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + var second = Export(output, checkpoint); + + if (openable) + { + // Running as root: the claim succeeds and this is the ordinary resume. + Assert.Contains(second.Warnings, w => + w.Contains("Recovered 2 items from an interrupted export's temp file")); + } + else + { + var stop = Assert.Single(second.Errors); + Assert.StartsWith("CheckpointTempUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Contains("cannot be opened for writing by this account", + stop.Exception.Message); + Assert.Contains(Path.GetFileName(temp), stop.Exception.Message); + Assert.Contains("Grant write access to that file and run again", + stop.Exception.Message); + Assert.DoesNotContain("Another export is still writing the temp file", + stop.Exception.Message); + Assert.True(File.Exists(temp), "the refused temp was swept by the run that stopped"); + Assert.False(File.Exists(output), "the stopped run published an output"); + } + } + finally + { + if (temp != null) MakeWritable(temp); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same distinction from the other cmdlet, which reaches the same claim. A sync stops + /// either way; what it can honestly say about the cause is what differs. + /// + [Fact] + public void A_sync_names_a_temp_it_cannot_open_as_a_permissions_problem() + { + var dir = NewDir(); + var delta = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + string? temp = null; + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + handler.Queue(HttpStatusCode.OK, Baseline); + Sync(delta, checkpoint, output); + handler.Queue(HttpStatusCode.OK, ChangesPage1); + Sync(delta, checkpoint, output); + temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + + MakeUnwritable(temp); + var openable = StillOpenableReadWrite(temp); + + handler.Queue(HttpStatusCode.OK, ChangesPage1); + handler.Queue(HttpStatusCode.OK, ChangesPage2); + var second = Sync(delta, checkpoint, output); + + if (openable) + { + Assert.Contains(second.Warnings, w => + w.Contains("Recovered 2 items from an interrupted sync's temp file")); + } + else + { + var stop = Assert.Single(second.Errors); + Assert.StartsWith("CheckpointTempUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Contains("cannot be opened for writing by this account", + stop.Exception.Message); + Assert.Contains(Path.GetFileName(temp), stop.Exception.Message); + Assert.Contains("Grant write access to that file and run again", + stop.Exception.Message); + Assert.DoesNotContain("Another sync is still writing the temp file", + stop.Exception.Message); + Assert.True(File.Exists(temp), "the refused temp was swept by the run that stopped"); + } + } + finally + { + if (temp != null) MakeWritable(temp); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The sweep's own account of a temp it passed over. It asks for the file exclusively and + /// leaves what it cannot claim, which is right for both ways the claim fails and honest + /// about only one of them: an orphan this account cannot open read-write has no second run + /// behind it, and calling it one sent the caller looking for an export that is not there. + /// + /// Which of the second kind it is is the open's own answer and not the line's to guess. Said + /// as "the open was refused on permissions, not on sharing", the same sentence went out for + /// a directory, for a pipe with no length to write at and for a path too long for the + /// volume - and a caller who went and granted write access to the file found nothing had + /// changed. The reason the open gave is quoted instead. + /// + /// No checkpoint here, so no resume is pending and the sweep runs over every temp beside + /// the output. The file is left where it is either way; only the sentence differs. + /// + [Fact] + public void The_sweep_names_a_temp_it_cannot_open_by_the_reason_the_open_gave() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var orphan = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + try + { + File.WriteAllLines(orphan, ["{\"id\":\"ORPHAN-1\"}"]); + MakeUnwritable(orphan); + var openable = StillOpenableReadWrite(orphan); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + + var verbose = ExportVerbose(output, checkpoint); + + if (openable) + { + // Running as root: the claim succeeds and the file is an ordinary orphan. + Assert.Contains(verbose, v => + v.Contains("Deleted an orphaned temp file") + && v.Contains(Path.GetFileName(orphan))); + } + else + { + var line = Assert.Single(verbose, v => v.Contains(Path.GetFileName(orphan))); + Assert.Contains("this run cannot open it for writing - ", line); + Assert.DoesNotContain("not on sharing", line); + Assert.DoesNotContain("another run is writing to it", line); + // The reason the open gave, quoted, and it names the file it was refused on. + Assert.Contains(orphan, line[line.IndexOf(" - ", StringComparison.Ordinal)..]); + Assert.True(File.Exists(orphan), "the sweep deleted a temp it could not claim"); + } + + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + MakeWritable(orphan); + try { Directory.Delete(dir, true); } catch { } + } + } + // Reads a file the test's own live writer still holds open: on Windows the reader // must offer ReadWrite sharing or the holder's write access denies the open. private static byte[] ReadShared(string path) @@ -369,4 +671,105 @@ private static byte[] ReadShared(string path) fs.CopyTo(ms); return ms.ToArray(); } + + /// + /// A FIFO at , which .NET has no call for. + /// + private static void Mkfifo(string path) + { + using var mkfifo = System.Diagnostics.Process.Start("/usr/bin/mkfifo", [path]); + Assert.NotNull(mkfifo); + mkfifo.WaitForExit(); + Assert.Equal(0, mkfifo.ExitCode); + } + + /// + /// The staging name in the stale-temp sweep, which reached it with the claim-then-delete a + /// temp takes. That claim answers for a regular file and for nothing else: a pipe and a + /// socket came back unopenable and were left standing - under a sentence calling them a file + /// this account could not open - and a dangling link was left as well, so the next + /// promotion met exactly the entry its own sweep exists to clear. Both go now, through that + /// sweep, and the run says which kind it took. + /// + [Theory] + [InlineData("pipe")] + [InlineData("dangling link")] + public void The_sweep_clears_what_stands_at_the_staging_name(string kind) + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var adopt = $"{output}.adopt"; + try + { + if (kind == "pipe") Mkfifo(adopt); + else File.CreateSymbolicLink(adopt, Path.Combine(dir, "nothing-is-here")); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + + var verbose = ExportVerbose(output, Path.Combine(dir, "run.checkpoint")); + + Assert.Contains(verbose, v => v.Contains( + $"Removed what stood at the staging name '{Path.GetFileName(adopt)}': " + + (kind == "pipe" + ? "an entry no copy could be staged in, such as a pipe or a socket." + : "a link, and not what it pointed at."), StringComparison.Ordinal)); + // File.Exists answers for a pipe and for a dangling link alike on this platform, so + // it is what says the entry itself is gone rather than what it pointed at. + Assert.False(File.Exists(adopt), "the entry outlived the sweep"); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { File.Delete(adopt); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// What the sweep of the staging name calls a regular file it cannot open. The only kinds it + /// told apart were a link and a copy nothing held, so a 0444 or a mode-000 file - the claim + /// refused, the unlink allowed by the directory it sits in - was reported as "an entry no + /// copy could be staged in, such as a pipe or a socket": a kind of entry it is not, and one + /// the caller cannot have made by the means they actually used. + /// + [Fact] + public void The_sweep_calls_a_staged_copy_it_cannot_open_what_it_is() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var adopt = $"{output}.adopt"; + try + { + File.WriteAllText(adopt, "{\"id\":\"half-staged\"}\n"); + MakeUnwritable(adopt); + var openable = StillOpenableReadWrite(adopt); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + + var verbose = ExportVerbose(output, Path.Combine(dir, "run.checkpoint")); + + // Running as root the claim succeeds and it is an ordinary leftover copy. + Assert.Contains(verbose, v => v.Contains( + $"Removed what stood at the staging name '{Path.GetFileName(adopt)}': " + + (openable + ? "a copy left by an interrupted promotion." + : "a copy this account cannot open."), StringComparison.Ordinal)); + Assert.DoesNotContain(verbose, v => v.Contains("such as a pipe or a socket")); + Assert.False(File.Exists(adopt), "the copy outlived the sweep"); + } + finally + { + MakeWritable(adopt); + try { Directory.Delete(dir, true); } catch { } + } + } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/OrphanTempOwnershipTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/OrphanTempOwnershipTests.cs index 44ae8e2..c6fa4ce 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/OrphanTempOwnershipTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/OrphanTempOwnershipTests.cs @@ -27,6 +27,18 @@ public class OrphanTempOwnershipTests typeof(MgxCmdletBase).GetMethod( "TryAdoptOrphanedTemp", BindingFlags.Static | BindingFlags.NonPublic)!; + /// + /// The two arguments, followed by an empty slot for each out parameter the method carries. + /// Sized off the method, since reflection wants a slot for every one of them. + /// + private static bool TryAdopt(string outputPath, long itemCount) + { + var slots = new object?[Adopt.GetParameters().Length]; + slots[0] = outputPath; + slots[1] = itemCount; + return (bool)Adopt.Invoke(null, slots)!; + } + private sealed class ScriptedHandler : HttpMessageHandler { private readonly Queue<(HttpStatusCode Status, string Body)> _steps = new(); @@ -98,7 +110,7 @@ public void A_temp_left_by_a_longer_named_export_is_not_adopted() var theirs = Path.Combine(dir, $"users.jsonl.{Guid.NewGuid():N}.tmp"); File.WriteAllLines(theirs, ["{\"id\":\"THEIRS-1\"}", "{\"id\":\"THEIRS-2\"}"]); - Assert.False((bool)Adopt.Invoke(null, [mine, 2L])!); + Assert.False(TryAdopt(mine, 2L)); Assert.False(File.Exists(mine)); Assert.True(File.Exists(theirs)); @@ -169,7 +181,7 @@ public void A_file_that_is_not_a_run_temp_is_not_adopted() var backup = Path.Combine(dir, "users.jsonl.backup.tmp"); File.WriteAllLines(backup, ["{\"id\":\"BACKUP-1\"}", "{\"id\":\"BACKUP-2\"}"]); - Assert.False((bool)Adopt.Invoke(null, [output, 2L])!); + Assert.False(TryAdopt(output, 2L)); Assert.False(File.Exists(output)); Assert.True(File.Exists(backup)); @@ -250,6 +262,84 @@ public void An_export_adopts_its_own_temp_when_the_output_is_spelled_differently finally { try { Directory.Delete(dir, true); } catch { } } } + /// + /// Whether the temps of two spellings cross is the filesystem's answer and not the + /// platform's. The comparison and the glob both took their rule from the operating system, + /// so a case-sensitive APFS volume - and a Windows directory flagged case-sensitive on its + /// own - was matched case-insensitively against a filesystem that keeps the two apart: + /// "Users.jsonl" and "users.jsonl" are two exports there, and one's temp was reachable by + /// the other's adoption and by its sweep. + /// + /// How this host gets there differs; what it asserts does not. Where the directory's own + /// probe answers case-sensitive the two files really are two, and where it does not the + /// answer is pinned so the same decision path still runs. Neither is skipped. + /// + [Fact] + public void An_export_does_not_adopt_the_other_spelling_of_its_output_where_case_is_kept() + { + var dir = NewDir(); + var pinned = !MgxCmdletBase.DirectoryNamesAreCaseSensitive(dir, mayWrite: true); + if (pinned) MgxCmdletBase.SetDirectoryCaseSensitivity(dir, true); + try + { + var mine = Path.Combine(dir, "users.jsonl"); + var theirs = Path.Combine(dir, $"Users.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllLines(theirs, ["{\"id\":\"THEIRS-1\"}", "{\"id\":\"THEIRS-2\"}"]); + + Assert.False(TryAdopt(mine, 2L)); + + Assert.False(File.Exists(mine)); + Assert.True(File.Exists(theirs)); + Assert.Equal(["{\"id\":\"THEIRS-1\"}", "{\"id\":\"THEIRS-2\"}"], + File.ReadAllLines(theirs)); + } + finally + { + if (pinned) MgxCmdletBase.SetDirectoryCaseSensitivity(dir, null); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same rule on the sweep, which is where it costs the other export its work rather + /// than this one its output: a fresh run deletes every "{output}.{guid}.tmp" beside it as + /// an orphan, and a temp whose name differs from this output's only in case is not beside + /// this output at all where the filesystem keeps the two apart. It was deleted from under + /// the run that was writing it. + /// + [Fact] + public void An_export_does_not_sweep_the_other_spelling_of_its_output_where_case_is_kept() + { + var dir = NewDir(); + var pinned = !MgxCmdletBase.DirectoryNamesAreCaseSensitive(dir, mayWrite: true); + if (pinned) MgxCmdletBase.SetDirectoryCaseSensitivity(dir, true); + try + { + var mine = Path.Combine(dir, "users.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var theirs = Path.Combine(dir, $"Users.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllLines(theirs, ["{\"id\":\"THEIRS-1\"}"]); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + + Export(mine, checkpoint); + + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(mine)); + Assert.True(File.Exists(theirs), + "the sweep took a temp that belongs to another output"); + Assert.Equal(["{\"id\":\"THEIRS-1\"}"], File.ReadAllLines(theirs)); + } + finally + { + if (pinned) MgxCmdletBase.SetDirectoryCaseSensitivity(dir, null); + try { Directory.Delete(dir, true); } catch { } + } + } + /// /// A second export running against the same output right now. Adoption picks the newest /// matching temp with only a line count to go on, and the file a live run is writing into @@ -302,6 +392,83 @@ public void An_export_does_not_adopt_a_temp_a_running_export_still_holds() finally { try { Directory.Delete(dir, true); } catch { } } } + /// + /// The probe file the case question is answered with is deleted by hand and by the handle + /// it was opened on - and on Unix that handle unlinks where the stream is disposed, so a + /// process killed outright leaves one behind, named after nothing any run comes back to. + /// The next run that writes here takes it back before writing its own. + /// + /// A leftover is put there by hand, since the state is only reachable by killing a process + /// mid-probe. The cached answer is forgotten first so the export below actually asks. + /// + [Fact] + public void A_run_reclaims_the_case_probe_file_an_earlier_run_left() + { + var dir = NewDir(); + var output = Path.Combine(dir, "users.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var litter = Path.Combine(dir, $".mgx-case-{Guid.NewGuid():N}"); + try + { + File.WriteAllText(litter, ""); + MgxCmdletBase.SetDirectoryCaseSensitivity(dir, null); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + + Export(output, checkpoint); + + Assert.False(File.Exists(litter), + "a leftover case probe outlived a run that wrote in the same directory"); + Assert.Empty(Directory.GetFiles(dir, ".mgx-case-*")); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + MgxCmdletBase.SetDirectoryCaseSensitivity(dir, null); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The question answered from what is already on disk. An entry that is there under one + /// spelling is there under the inverted spelling too exactly when the volume folds case, so + /// a directory holding any lettered name settles it with nothing written - which is what + /// keeps the answer out of the caller's directory's write time on every path, gated or not. + /// + /// What the volume actually does is taken with the test's own file first, so the assertion + /// is against this host rather than against its platform's default: either answer executes + /// here and neither is skipped. + /// + [Fact] + public void A_directory_holding_a_lettered_entry_decides_its_case_rule_without_writing() + { + var dir = NewDir(); + try + { + var insensitive = NamesAreCaseInsensitive(dir); + File.WriteAllText(Path.Combine(dir, "Entry.txt"), ""); + MgxCmdletBase.SetDirectoryCaseSensitivity(dir, null); + + var namesBefore = Directory.GetFileSystemEntries(dir).OrderBy(n => n).ToArray(); + var writtenBefore = Directory.GetLastWriteTimeUtc(dir); + + Assert.Equal(!insensitive, + MgxCmdletBase.DirectoryNamesAreCaseSensitive(dir, mayWrite: true)); + + Assert.Equal(namesBefore, Directory.GetFileSystemEntries(dir).OrderBy(n => n).ToArray()); + Assert.Equal(writtenBefore, Directory.GetLastWriteTimeUtc(dir)); + } + finally + { + MgxCmdletBase.SetDirectoryCaseSensitivity(dir, null); + try { Directory.Delete(dir, true); } catch { } + } + } + /// /// What reaches the caller's file. The checkpoint predates the recorded length, so adoption /// has only a line count to go on - and a temp cut inside the row that count reaches hands diff --git a/tests/Mgx.IntegrationTests/Cmdlets/RequestWhatIfCheckpointTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/RequestWhatIfCheckpointTests.cs new file mode 100644 index 0000000..e73f203 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/RequestWhatIfCheckpointTests.cs @@ -0,0 +1,574 @@ +using System.Globalization; +using System.Management.Automation; +using System.Management.Automation.Host; +using System.Management.Automation.Runspaces; +using System.Net; +using System.Security; +using System.Text; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// -WhatIf on a list run carrying -CheckpointPath. The read path saves the next position over +/// that file at every page boundary, deletes it on completion, and deletes it where it turns +/// out to describe another enumeration - so previewing a run destroyed the position the caller +/// resumes from. The reads are not what is gated: a GET under -WhatIf is sent, pages and emits +/// its objects, which is the convention Invoke-MgxRequest documents. +/// +[Collection("Pipeline")] +public class RequestWhatIfCheckpointTests +{ + private const string Page1 = """ + {"value":[{"id":"u1"},{"id":"u2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2"} + """; + private const string Page2 = """ + {"value":[{"id":"u3"}]} + """; + private const string NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2"; + + private const string EmptyPage = """ + {"value":[],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P3"} + """; + private const string LastPage = """ + {"value":[{"id":"u4"},{"id":"u5"}]} + """; + + /// + /// Two pages, plus a snapshot of the checkpoint file as each request arrives. The + /// page-boundary save lands between the first response and the second request and the + /// completion delete takes it away again, so mid-run is the only point from which a test + /// can see whether a run wrote a checkpoint at all. + /// + private sealed class PagingHandler(string? checkpointPath) : HttpMessageHandler + { + private readonly object _lock = new(); + public List Urls { get; } = []; + public List CheckpointExisted { get; } = []; + public List CheckpointContent { get; } = []; + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + lock (_lock) + { + Urls.Add(url); + var existed = checkpointPath != null && File.Exists(checkpointPath); + CheckpointExisted.Add(existed); + CheckpointContent.Add(existed ? File.ReadAllText(checkpointPath!) : null); + } + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent( + url.Contains("skiptoken", StringComparison.Ordinal) ? Page2 : Page1, + Encoding.UTF8, "application/json") + }); + } + } + + /// + /// Three pages with the middle one empty behind a nextLink, which is how a page boundary is + /// reached with no item yielded since the fetch. Every answer takes a real asynchronous hop: + /// a socket read always yields and Task.FromResult never does, and the thread the iterator + /// resumes on is what these pages are here to settle. + /// + private sealed class EmptyMiddlePageHandler : HttpMessageHandler + { + private readonly object _lock = new(); + public List Urls { get; } = []; + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + lock (_lock) { Urls.Add(url); } + + await Task.Yield(); + + var body = url.Contains("skiptoken=P3", StringComparison.Ordinal) ? LastPage + : url.Contains("skiptoken=P2", StringComparison.Ordinal) ? EmptyPage + : Page1; + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }; + } + } + + /// + /// Page 1 has items and a nextLink; whatever answers the request that follows is not JSON at + /// all. Awaits Task.Yield() for the same reason the empty-page handler does: every answer + /// takes a real asynchronous hop, so the page-boundary callback - which fires once page 1's + /// two items are both yielded, before the second request goes out - runs on whichever thread + /// that hop left it on rather than synchronously on the caller's. + /// + private sealed class MalformedSecondPageHandler : HttpMessageHandler + { + private readonly object _lock = new(); + public List Urls { get; } = []; + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + lock (_lock) { Urls.Add(url); } + + await Task.Yield(); + + var body = url.Contains("skiptoken", StringComparison.Ordinal) + ? "not a collection envelope at all" + : Page1; + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }; + } + } + + /// The lines -WhatIf writes go to the host, so a test that reads them needs one. + private sealed class RecordingHost : PSHost + { + private readonly Guid _id = Guid.NewGuid(); + public RecordingHostUI Recorder { get; } = new(); + public override string Name => "MgxTestHost"; + public override Version Version => new(1, 0); + public override Guid InstanceId => _id; + public override PSHostUserInterface UI => Recorder; + public override CultureInfo CurrentCulture => CultureInfo.InvariantCulture; + public override CultureInfo CurrentUICulture => CultureInfo.InvariantCulture; + public override void EnterNestedPrompt() { } + public override void ExitNestedPrompt() { } + public override void NotifyBeginApplication() { } + public override void NotifyEndApplication() { } + public override void SetShouldExit(int exitCode) { } + } + + private sealed class RecordingHostUI : PSHostUserInterface + { + public List Lines { get; } = []; + public override PSHostRawUserInterface? RawUI => null; + public override void Write(string value) => Lines.Add(value); + public override void Write(ConsoleColor f, ConsoleColor b, string value) => Lines.Add(value); + public override void WriteLine(string value) => Lines.Add(value); + public override void WriteErrorLine(string value) => Lines.Add(value); + public override void WriteDebugLine(string value) => Lines.Add(value); + public override void WriteVerboseLine(string value) => Lines.Add(value); + public override void WriteWarningLine(string value) => Lines.Add(value); + public override void WriteProgress(long sourceId, ProgressRecord record) { } + public override string ReadLine() => string.Empty; + public override SecureString ReadLineAsSecureString() => new(); + public override Dictionary Prompt( + string caption, string message, System.Collections.ObjectModel.Collection descriptions) => []; + public override int PromptForChoice( + string caption, string message, + System.Collections.ObjectModel.Collection choices, int defaultChoice) => defaultChoice; + public override PSCredential PromptForCredential( + string caption, string message, string userName, string targetName) => PSCredential.Empty; + public override PSCredential PromptForCredential( + string caption, string message, string userName, string targetName, + PSCredentialTypes allowedCredentialTypes, PSCredentialUIOptions options) => PSCredential.Empty; + } + + private static string NewDir() => + Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-req-whatif-{Guid.NewGuid():N}")).FullName; + + /// + /// Walks the collection and hands back the lines the host was given, how many objects the + /// run emitted, whether it errored, the warnings it wrote, and the ids of the errors it + /// wrote. + /// + private static (List Lines, int Items, bool HadErrors, List Warnings, List Errors) Enumerate( + string? checkpoint, bool whatIf) + { + var host = new RecordingHost(); + using var runspace = RunspaceFactory.CreateRunspace(host); + runspace.Open(); + using var ps = PowerShell.Create(); + ps.Runspace = runspace; + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + var cmd = ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("All") + .AddParameter("PageSize", 2); + if (checkpoint != null) cmd.AddParameter("CheckpointPath", checkpoint); + if (whatIf) cmd.AddParameter("WhatIf", true); + var output = ps.Invoke(); + return (host.Recorder.Lines, output.Count, ps.HadErrors, + [.. ps.Streams.Warning.Select(w => w.Message)], + [.. ps.Streams.Error.Select(e => e.FullyQualifiedErrorId)]); + } + + /// + /// The URL this enumeration asks for. A checkpoint is the run's own only when the resource + /// it records matches that string exactly, and writing the query shape into the test would + /// pin the URL builder instead of the gate. + /// + private static string EnumerationUrl() + { + var handler = new PagingHandler(checkpointPath: null); + using var transport = MgxTransportScope.Inject(handler); + Enumerate(checkpoint: null, whatIf: false); + return handler.Urls[0]; + } + + /// + /// A checkpoint another enumeration left over the same -CheckpointPath. A real run refuses + /// it, deletes it and pages from the start; the preview pages from the start too - same + /// requests, same objects - and leaves the caller's file exactly as it found it. + /// + [Fact] + public void WhatIf_leaves_another_enumerations_checkpoint_byte_identical() + { + var dir = NewDir(); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/groups?$top=2", + NextLink = "https://graph.microsoft.com/v1.0/groups?$skiptoken=P2", + ItemsCollected = 2, + }.Save(checkpoint); + var before = File.ReadAllBytes(checkpoint); + + var handler = new PagingHandler(checkpoint); + using var transport = MgxTransportScope.Inject(handler); + var (lines, items, hadErrors, _, _) = Enumerate(checkpoint, whatIf: true); + + Assert.False(hadErrors); + + // The reads are not what the gate withholds: both pages were fetched and every + // object went down the pipeline. + Assert.Equal(2, handler.Urls.Count); + Assert.Equal(3, items); + + // The file is what it withholds, and the caller's is where it was. + Assert.True(File.Exists(checkpoint), "-WhatIf deleted the caller's checkpoint"); + Assert.Equal(before, File.ReadAllBytes(checkpoint)); + + // Named on the way past, so the caller running the preview is told which file a + // real run would have written. + Assert.Contains(lines, + l => l.Contains("Save resume checkpoint") && l.Contains(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A completion marker (NextLink null) left by this enumeration's own previous run. A real + /// run takes that as "already finished" and deletes it before paging from the start again; + /// the preview takes the same route through the same Delete call, gated the same way as the + /// page-boundary save and the mismatched-resource branch above - and mayWriteCheckpoint is + /// the only thing standing between it and deleting a marker -WhatIf is not supposed to touch. + /// + [Fact] + public void WhatIf_leaves_a_completion_marker_checkpoint_byte_identical() + { + var dir = NewDir(); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + new PaginationCheckpoint + { + Resource = EnumerationUrl(), + NextLink = null, + ItemsCollected = 7, + }.Save(checkpoint); + var before = File.ReadAllBytes(checkpoint); + + var handler = new PagingHandler(checkpoint); + using var transport = MgxTransportScope.Inject(handler); + var (_, items, hadErrors, _, _) = Enumerate(checkpoint, whatIf: true); + + Assert.False(hadErrors); + + // A completion marker carries no position to resume from, so both pages are asked + // for and all three objects go down the pipeline either way. + Assert.Equal(2, handler.Urls.Count); + Assert.Equal(3, items); + + Assert.True(File.Exists(checkpoint), "-WhatIf deleted the caller's completion-marker checkpoint"); + Assert.Equal(before, File.ReadAllBytes(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// This enumeration's own checkpoint, but its nextLink is one NextLinkValidator refuses (here, + /// relative rather than https) - the SSRF gate, not a stale or foreign resource. A real run + /// cannot resume from a link it will not follow, so it deletes the checkpoint and pages from + /// the start; the preview takes the same branch through the same Delete call, and withholds it + /// the same way. + /// + [Fact] + public void WhatIf_leaves_a_refused_nextlink_checkpoint_byte_identical() + { + var dir = NewDir(); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + new PaginationCheckpoint + { + Resource = EnumerationUrl(), + NextLink = "pages/4", + ItemsCollected = 7, + }.Save(checkpoint); + var before = File.ReadAllBytes(checkpoint); + + var handler = new PagingHandler(checkpoint); + using var transport = MgxTransportScope.Inject(handler); + var (_, items, hadErrors, _, _) = Enumerate(checkpoint, whatIf: true); + + Assert.False(hadErrors); + + // The refused link carries no usable position either, so this run pages from the + // start the same as the completion-marker case above. + Assert.Equal(2, handler.Urls.Count); + Assert.Equal(3, items); + + Assert.True(File.Exists(checkpoint), + "-WhatIf deleted the caller's checkpoint with a nextLink the validator refuses"); + Assert.Equal(before, File.ReadAllBytes(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same run with nothing at -CheckpointPath to begin with. A preview that leaves a + /// checkpoint behind has written a resume position for an enumeration that never happened, + /// and one that writes and then deletes has still had the caller's file for the length of + /// the run - so the page boundary, not the end, is where this is settled. + /// + [Fact] + public void WhatIf_writes_no_checkpoint_of_its_own() + { + var dir = NewDir(); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new PagingHandler(checkpoint); + using var transport = MgxTransportScope.Inject(handler); + var (_, items, hadErrors, _, _) = Enumerate(checkpoint, whatIf: true); + + Assert.False(hadErrors); + Assert.Equal(3, items); + Assert.Equal(2, handler.CheckpointExisted.Count); + Assert.All(handler.CheckpointExisted, + existed => Assert.False(existed, "-WhatIf saved a checkpoint of its own")); + Assert.False(File.Exists(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A checkpoint this enumeration wrote. The preview is of the run the caller would make + /// next, and that run resumes - so the preview resumes too, and asks only for the page the + /// checkpoint stopped at. What it cannot do is finish by deleting the position it read. + /// + [Fact] + public void WhatIf_resumes_from_a_matching_checkpoint_without_consuming_it() + { + var dir = NewDir(); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + new PaginationCheckpoint + { + Resource = EnumerationUrl(), + NextLink = NextLink, + ItemsCollected = 2, + }.Save(checkpoint); + var before = File.ReadAllBytes(checkpoint); + + var handler = new PagingHandler(checkpoint); + using var transport = MgxTransportScope.Inject(handler); + var (_, items, hadErrors, _, _) = Enumerate(checkpoint, whatIf: true); + + Assert.False(hadErrors); + Assert.Single(handler.Urls); + Assert.Contains("skiptoken=P2", handler.Urls[0]); + Assert.Equal(1, items); + Assert.True(File.Exists(checkpoint), "-WhatIf deleted the checkpoint it resumed from"); + Assert.Equal(before, File.ReadAllBytes(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The run the previews above describe. Nothing about the checkpoint changes for a real + /// run: the next position is saved at the page boundary, and the file is gone once the + /// enumeration completes. + /// + [Fact] + public void A_real_run_saves_at_the_page_boundary_and_deletes_on_completion() + { + var dir = NewDir(); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new PagingHandler(checkpoint); + using var transport = MgxTransportScope.Inject(handler); + var (_, items, hadErrors, _, _) = Enumerate(checkpoint, whatIf: false); + + Assert.False(hadErrors); + Assert.Equal(3, items); + Assert.False(handler.CheckpointExisted[0]); + Assert.True(handler.CheckpointExisted[1], "no checkpoint was saved at the page boundary"); + Assert.Contains("skiptoken=P2", handler.CheckpointContent[1]!); + Assert.False(File.Exists(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same real run, against a checkpoint it cannot write. The page-boundary callback runs + /// on whichever thread the iterator resumed on, and a page whose value is empty resumes on + /// the thread pool with no yield in between - so a cmdlet API call from there throws + /// PSInvalidOperationException, and warning the caller about the disk ended the enumeration + /// at the empty page instead. The message goes onto the client's buffered warning channel + /// and the pipeline thread's drain writes it: one per failed save, and the pages behind the + /// empty one are still asked for and still emitted. + /// + [Fact] + public void A_failed_boundary_save_warns_and_the_enumeration_goes_on() + { + var dir = NewDir(); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + BlockSaves(dir, checkpoint); + + var handler = new EmptyMiddlePageHandler(); + using var transport = MgxTransportScope.Inject(handler); + var (_, items, hadErrors, warnings, _) = Enumerate(checkpoint, whatIf: false); + + Assert.False(hadErrors); + + // Both boundaries that carry a nextLink try to save, and both fail against a + // checkpoint that cannot be written. + var failures = warnings + .Where(w => w.StartsWith("Checkpoint save failed", StringComparison.Ordinal)) + .ToList(); + Assert.Equal(2, failures.Count); + + // The empty page sits behind the second request, so a run that died reporting its + // failed save never asks for the third - and the two items on it never arrive. + Assert.Equal(3, handler.Urls.Count); + Assert.Contains("skiptoken=P3", handler.Urls[2]); + Assert.Equal(4, items); + } + finally + { + RestoreMode(dir); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same blocked checkpoint, but the run this time does not reach the successful exit + /// the previous test measures: page 1's boundary save fails same as before, and what + /// answers the next request is not JSON at all. That exit is the JsonException catch, which + /// returns straight after writing its error record - there is no next item left to carry the + /// per-item drain, and until the enumeration loop's exits all shared a single finally, that + /// catch was one of three that returned without draining, so the buffered "Checkpoint save + /// failed" warning stayed on the queue and was never written. + /// + [Fact] + public void A_failed_boundary_save_warns_even_when_the_next_page_does_not_parse() + { + var dir = NewDir(); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + BlockSaves(dir, checkpoint); + + var handler = new MalformedSecondPageHandler(); + using var transport = MgxTransportScope.Inject(handler); + var (_, items, hadErrors, warnings, errors) = Enumerate(checkpoint, whatIf: false); + + Assert.True(hadErrors); + var errorId = Assert.Single(errors); + Assert.StartsWith("MalformedJsonResponse", errorId); + + var failures = warnings + .Where(w => w.StartsWith("Checkpoint save failed", StringComparison.Ordinal)) + .ToList(); + Assert.Single(failures); + + // Page 1's two items were already on the pipeline before the second request's body + // failed to parse. + Assert.Equal(2, items); + } + finally + { + RestoreMode(dir); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// Makes every checkpoint save into this directory fail, whatever the host allows. A mode + /// the account cannot write is the honest version of it, but root ignores mode bits and + /// Windows expresses this through ACLs; where a canary shows the mode does not bite, the + /// name Save stages its bytes under is taken by a directory instead, which no account can + /// write a file over. + /// + private static void BlockSaves(string dir, string checkpoint) + { + if (!OperatingSystem.IsWindows()) + { + File.SetUnixFileMode(dir, UnixFileMode.UserRead | UnixFileMode.UserExecute); + try + { + var canary = Path.Combine(dir, "writable.probe"); + File.WriteAllText(canary, ""); + File.Delete(canary); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return; + } + RestoreMode(dir); + } + + Directory.CreateDirectory(checkpoint + ".tmp"); + } + + private static void RestoreMode(string dir) + { + if (OperatingSystem.IsWindows()) return; + try + { + File.SetUnixFileMode(dir, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ResilienceWrapTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ResilienceWrapTests.cs index fb6a27b..e068d60 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/ResilienceWrapTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/ResilienceWrapTests.cs @@ -1,5 +1,7 @@ using System.Reflection; +using Mgx.Cmdlets.Base; using Mgx.Cmdlets.Cmdlets.Configuration; +using Mgx.Cmdlets.Models; using Mgx.Engine.Http; namespace Mgx.IntegrationTests; @@ -108,4 +110,307 @@ public void A_throttle_through_the_wrapper_is_visible_to_mgx() Assert.True(snapshot.ThrottleRetries > 0, $"mgx recorded {snapshot.ThrottleRetries} throttle retries for a 429 the wire served"); } + + // --- the preview --- + + /// + /// Puts the process where a successful Enable-MgxResilience leaves it, and then has the SDK + /// replace the wrapper - a Connect-MgGraph or a Set-MgRequestContext does that. It is the one + /// arrangement whose Enable does more than report itself already active, so it is the one that + /// says what a preview costs. + /// + private static HttpClient InjectThenLetTheSdkReplaceIt( + GraphSessionScope scope, HttpClient genuine, HttpClient replacement) + { + var wrapper = InjectAndLeaveItInstalled(scope, genuine); + scope.Session.GraphHttpClient = replacement; + return wrapper; + } + + /// + /// Puts the process where a successful Enable-MgxResilience leaves it and stops there: the + /// wrapper is still the client the SDK is sending through, so it is this module's to take off. + /// + private static HttpClient InjectAndLeaveItInstalled(GraphSessionScope scope, HttpClient genuine) + { + var wrapper = EnableMgxResilience.BuildResilientSdkClient(genuine, _ => { })!; + EnableMgxResilience.OriginalSdkClient = genuine; + EnableMgxResilience.ResilientSdkClient = wrapper; + EnableMgxResilience.IsEnabled = true; + scope.Session.GraphHttpClient = wrapper; + return wrapper; + } + + /// Runs a cmdlet where PowerShell runs it, and returns the warnings it wrote. + private static IReadOnlyList RunCmdlet(string name, bool whatIf = false) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(EnableMgxResilience).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + var command = ps.AddCommand(name); + if (whatIf) command.AddParameter("WhatIf", true); + ps.Invoke(); + + Assert.Empty(ps.Streams.Error.Select(e => e.FullyQualifiedErrorId)); + return [.. ps.Streams.Warning.Select(w => w.Message)]; + } + + private static HttpClient NewSdkClient() => + new() { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + + /// What Get-MgxResilience reports right now. + private static MgxResilienceOutput Reported() + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(EnableMgxResilience).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Get-MgxResilience"); + var results = ps.Invoke(); + return Assert.IsType(Assert.Single(results).BaseObject); + } + + /// + /// -WhatIf reports what a run would do; it does not do any of it. The injection is shared + /// state - the wrapper the SDK is sending through, the handler holding it, and the pipeline + /// factory's circuit-breaker history, rate limiter and learned pacing - and a session that + /// asks what would happen keeps all of it. + /// + [Fact] + public void A_preview_leaves_the_injection_and_the_pipeline_where_it_found_them() + { + using var scope = GraphSessionScope.Arm(); + using var genuine = NewSdkClient(); + using var replacement = NewSdkClient(); + var wrapper = InjectThenLetTheSdkReplaceIt(scope, genuine, replacement); + + var handler = EnableMgxResilience.ActiveHandler; + Assert.NotNull(handler); + var (pipeline, rateLimiter) = ResiliencePipelineFactory.GetOrCreate(MgxCmdletBase.s_clientOptions); + + Assert.Empty(RunCmdlet("Enable-MgxResilience", whatIf: true)); + + Assert.Same(wrapper, EnableMgxResilience.ResilientSdkClient); + Assert.Same(handler, EnableMgxResilience.ActiveHandler); + Assert.Same(genuine, EnableMgxResilience.OriginalSdkClient); + Assert.True(EnableMgxResilience.IsEnabled); + Assert.Same(replacement, scope.Session.GraphHttpClient); + + // The factory hands back what it is holding, so the same pipeline and the same limiter + // mean nothing rebuilt them - and rebuilding is how the history in them is lost. + var (pipelineAfter, rateLimiterAfter) = + ResiliencePipelineFactory.GetOrCreate(MgxCmdletBase.s_clientOptions); + Assert.Same(pipeline, pipelineAfter); + Assert.Same(rateLimiter, rateLimiterAfter); + } + + /// + /// The acceptance for the ShouldProcess gate says a preview performs none of the run's state + /// changes. Two of those changes are the pre-initialization of mgx's own client and the SDK + /// probe - and the probe is the one that sends a real Graph request and makes the SDK build a + /// client on the session. This drives the one arrangement where the probe would run: a + /// connected session whose GraphHttpClient is not built yet. + /// + [Fact] + public void A_preview_runs_neither_the_pre_init_nor_the_sdk_probe() + { + var marker = Path.Combine(Path.GetTempPath(), $"mgx-preview-probe-{Guid.NewGuid():N}.txt"); + + using var scope = GraphSessionScope.Arm( + graphHttpClient: null, + authContext: GraphSessionScope.AuthContextFor("tenant-a", "app-a")); + + var (pipeline, limiter) = ResiliencePipelineFactory.GetOrCreate(MgxCmdletBase.s_clientOptions); + + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(EnableMgxResilience).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + // Stands in for the SDK cmdlet the init probe shells out to. Its only job is to record + // that the probe reached the wire at all. + ps.AddScript( + "function global:Invoke-MgGraphRequest { param($Method,$Uri,$ErrorAction,$WarningAction) " + + $"Set-Content -LiteralPath '{marker}' -Value 'probe ran' }}"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Enable-MgxResilience").AddParameter("WhatIf", true); + var thrown = Record.Exception(() => ps.Invoke()); + + try + { + Assert.False(File.Exists(marker), + "-WhatIf ran the SDK init probe: it sent a Graph request and made the SDK build a client"); + Assert.Null(thrown); + Assert.Empty(ps.Streams.Error.Select(e => e.FullyQualifiedErrorId)); + Assert.Null(scope.Session.GraphHttpClient); + Assert.False(EnableMgxResilience.IsEnabled); + Assert.Null(EnableMgxResilience.ResilientSdkClient); + Assert.Null(EnableMgxResilience.ActiveHandler); + + var (pipelineAfter, limiterAfter) = + ResiliencePipelineFactory.GetOrCreate(MgxCmdletBase.s_clientOptions); + Assert.Same(pipeline, pipelineAfter); + Assert.Same(limiter, limiterAfter); + } + finally + { + if (File.Exists(marker)) File.Delete(marker); + } + } + + /// + /// Disable-MgxResilience deliberately leaves the pipeline factory alone: its circuit-breaker + /// history, rate limiter and learned pacing are shared with mgx's own cmdlets, which go on + /// running after the SDK injection comes off. A preview of the Enable must not do to that + /// state what the Disable itself refuses to do, and must leave the Disable the wrapper it is + /// there to release. + /// + [Fact] + public void A_disable_after_a_preview_releases_the_wrapper_the_preview_left() + { + using var scope = GraphSessionScope.Arm(); + using var genuine = NewSdkClient(); + using var replacement = NewSdkClient(); + var wrapper = InjectThenLetTheSdkReplaceIt(scope, genuine, replacement); + var (pipeline, rateLimiter) = ResiliencePipelineFactory.GetOrCreate(MgxCmdletBase.s_clientOptions); + + RunCmdlet("Enable-MgxResilience", whatIf: true); + + // What the Disable is about to release, still there for it to release. + Assert.Same(wrapper, EnableMgxResilience.ResilientSdkClient); + + RunCmdlet("Disable-MgxResilience"); + + Assert.Same(replacement, scope.Session.GraphHttpClient); + Assert.False(EnableMgxResilience.IsEnabled); + Assert.Null(EnableMgxResilience.ResilientSdkClient); + Assert.Null(EnableMgxResilience.ActiveHandler); + + var (pipelineAfter, rateLimiterAfter) = + ResiliencePipelineFactory.GetOrCreate(MgxCmdletBase.s_clientOptions); + Assert.Same(pipeline, pipelineAfter); + Assert.Same(rateLimiter, rateLimiterAfter); + } + + // --- whose client is on the session when the injection comes off --- + + /// + /// A Connect-MgGraph to another tenant builds a fresh SDK client and hands it to the session, + /// so the client Enable-MgxResilience saved is the previous tenant's. Installing that on the + /// way out puts those credentials back under every SDK cmdlet - and says the session was + /// restored to original behavior while it does it. The injection's own state still goes: what + /// the session is holding is not this module's, but the handler, the wrapper and the flag are. + /// + [Fact] + public void A_disable_leaves_a_client_the_sdk_replaced_after_the_injection() + { + using var scope = GraphSessionScope.Arm(); + using var genuine = NewSdkClient(); + using var replacement = NewSdkClient(); + InjectThenLetTheSdkReplaceIt(scope, genuine, replacement); + + var warnings = RunCmdlet("Disable-MgxResilience"); + + Assert.Same(replacement, scope.Session.GraphHttpClient); + Assert.Contains(warnings, w => w.Contains("left as it was found", StringComparison.Ordinal)); + Assert.False(EnableMgxResilience.IsEnabled); + Assert.Null(EnableMgxResilience.ResilientSdkClient); + Assert.Null(EnableMgxResilience.OriginalSdkClient); + Assert.Null(EnableMgxResilience.ActiveHandler); + + // The state Get-MgxResilience reports is the state the release left, not a session that + // still has something to re-inject. + var reported = Reported(); + Assert.False(reported.IsEnabled); + Assert.False(reported.IsActive); + Assert.Null(reported.Warning); + } + + /// + /// The other branch, unchanged: the wrapper is still the client the SDK is sending through, + /// so it is this module's to take off and the client it bridges to is the one to put back. + /// + [Fact] + public void A_disable_restores_the_original_while_the_wrapper_is_still_installed() + { + using var scope = GraphSessionScope.Arm(); + using var genuine = NewSdkClient(); + var wrapper = InjectAndLeaveItInstalled(scope, genuine); + + var warnings = RunCmdlet("Disable-MgxResilience"); + + Assert.Empty(warnings); + Assert.Same(genuine, scope.Session.GraphHttpClient); + Assert.False(EnableMgxResilience.IsEnabled); + Assert.Null(EnableMgxResilience.ResilientSdkClient); + Assert.Null(EnableMgxResilience.OriginalSdkClient); + Assert.Null(EnableMgxResilience.ActiveHandler); + + var reported = Reported(); + Assert.False(reported.IsEnabled); + Assert.False(reported.IsActive); + Assert.Null(reported.Warning); + } + + /// What the gate wrote when Disable-MgxResilience was previewed. + private static List PreviewTheDisable() => + RecordingHost.Preview( + typeof(EnableMgxResilience).Assembly, + ps => ps.AddCommand("Disable-MgxResilience").AddParameter("WhatIf", true)); + + private const string RestoreAction = "Restore original SDK HttpClient"; + private const string LeaveAloneAction = "leave the SDK HttpClient as found"; + + /// + /// A preview of the branch above. It restores nothing - the client the session is holding is + /// not this module's to take off - so a caller told a restore would happen was told the one + /// thing this run will not do, and the report is all -WhatIf gives them to act on. + /// + [Fact] + public void A_preview_over_a_client_the_sdk_replaced_names_the_leave_alone() + { + using var scope = GraphSessionScope.Arm(); + using var genuine = NewSdkClient(); + using var replacement = NewSdkClient(); + var wrapper = InjectThenLetTheSdkReplaceIt(scope, genuine, replacement); + + var lines = PreviewTheDisable(); + + Assert.Contains(lines, l => l.Contains(LeaveAloneAction, StringComparison.Ordinal)); + Assert.DoesNotContain(lines, l => l.Contains(RestoreAction, StringComparison.Ordinal)); + + // And it did none of it: the release the action names is still the Disable's to do. + Assert.Same(replacement, scope.Session.GraphHttpClient); + Assert.Same(wrapper, EnableMgxResilience.ResilientSdkClient); + Assert.True(EnableMgxResilience.IsEnabled); + } + + /// + /// The other branch previewed. Here the wrapper is still installed, so a restore is exactly + /// what a real run would do and exactly what the gate has to say. + /// + [Fact] + public void A_preview_over_the_wrapper_still_installed_names_the_restore() + { + using var scope = GraphSessionScope.Arm(); + using var genuine = NewSdkClient(); + var wrapper = InjectAndLeaveItInstalled(scope, genuine); + + var lines = PreviewTheDisable(); + + Assert.Contains(lines, l => l.Contains(RestoreAction, StringComparison.Ordinal)); + Assert.DoesNotContain(lines, l => l.Contains(LeaveAloneAction, StringComparison.Ordinal)); + + Assert.Same(wrapper, scope.Session.GraphHttpClient); + Assert.Same(genuine, EnableMgxResilience.OriginalSdkClient); + Assert.True(EnableMgxResilience.IsEnabled); + } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/SetOptionGuardTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/SetOptionGuardTests.cs index 0569e27..f8bad89 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/SetOptionGuardTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/SetOptionGuardTests.cs @@ -1,6 +1,11 @@ using System.Management.Automation; +using System.Net; +using System.Reflection; +using Mgx.Cmdlets; using Mgx.Cmdlets.Base; +using Mgx.Cmdlets.Models; using Mgx.Engine.Http; +using Microsoft.Graph.PowerShell.Authentication; namespace Mgx.IntegrationTests; @@ -82,4 +87,338 @@ public void Setting_NoRateLimit_beside_BatchItemsPerSecond_says_which_one_wins() reset.Invoke(); } } + + /// What Get-MgxOption reports right now. + private static MgxOptionOutput Reported() + { + using var ps = Shell(); + ps.AddCommand("Get-MgxOption"); + var results = ps.Invoke(); + return Assert.IsType(Assert.Single(results).BaseObject); + } + + /// What Get-MgxTelemetry reports right now. + private static MgxTelemetryOutput Telemetry() + { + using var ps = Shell(); + ps.AddCommand("Get-MgxTelemetry"); + var results = ps.Invoke(); + return Assert.IsType(Assert.Single(results).BaseObject); + } + + /// + /// No parameters bound (beyond the common ones PowerShell always adds) is the guard's + /// no-op case, and the gate now sits after that guard: a bare `Set-MgxOption` never reaches + /// ShouldProcess, so it previews nothing and would prompt for nothing under -Confirm either. + /// + [Fact] + public void No_parameters_previews_nothing() + { + var lines = RecordingHost.Preview( + typeof(Mgx.Cmdlets.Cmdlets.Configuration.SetMgxOption).Assembly, + ps => ps.AddCommand("Set-MgxOption") + .AddParameter("Verbose", true).AddParameter("WhatIf", true)); + + Assert.DoesNotContain(lines, l => l.Contains("What if:", StringComparison.Ordinal)); + } + + /// + /// The target a preview names is what the run would set. It was built from every bound + /// parameter, and PowerShell binds the common ones, so `Set-MgxOption -Verbose -WhatIf` + /// used to preview a change to "MgxOptions (Verbose, WhatIf)" - naming two parameters that + /// set no option at all, for a run the guard reports as unchanged. -Reset names what it + /// drops rather than what it sets, so a bound option shows up there as "ignoring", not as + /// part of the reset itself. + /// + [Theory] + [InlineData(false, "MaxRetryAttempts", 3, "MgxOptions (MaxRetryAttempts)")] + [InlineData(true, null, 0, "MgxOptions (reset to defaults)")] + [InlineData(true, "RateLimitBurst", 500, "MgxOptions (reset to defaults; ignoring RateLimitBurst)")] + public void A_preview_target_names_only_the_options_a_run_would_set( + bool reset, string? option, int value, string expected) + { + var lines = RecordingHost.Preview( + typeof(Mgx.Cmdlets.Cmdlets.Configuration.SetMgxOption).Assembly, + ps => + { + var command = ps.AddCommand("Set-MgxOption"); + if (reset) command.AddParameter("Reset", true); + if (option != null) command.AddParameter(option, value); + command.AddParameter("Verbose", true).AddParameter("WhatIf", true); + }); + + // The gate quotes its target, so the closing quote is what says the name ends there. + Assert.Contains(lines, l => l.Contains($"\"{expected}\"", StringComparison.Ordinal)); + } + + /// + /// -Reset beside a bound option drops the option - it always has - but used to do so without + /// a word about it, which reads as the option having taken effect over the defaults it did + /// not survive. The warning names what it ignored, and the value confirms it really was. + /// + [Fact] + public void Reset_beside_an_option_warns_and_names_it() + { + try + { + using var ps = Shell(); + ps.AddCommand("Set-MgxOption") + .AddParameter("Reset", true) + .AddParameter("RateLimitBurst", 500); + ps.Invoke(); + + var warned = string.Join(" ", ps.Streams.Warning.Select(w => w.Message)); + Assert.Contains("-Reset resets every option", warned); + Assert.Contains("-RateLimitBurst", warned); + Assert.Contains("ignored", warned); + + Assert.Equal(ResilientGraphClientOptions.Default.RateLimitBurst, Reported().RateLimitBurst); + } + finally + { + using var reset = Shell(); + reset.AddCommand("Set-MgxOption").AddParameter("Reset", true); + reset.Invoke(); + } + } + + /// + /// -NoRateLimit is sticky: it stays off until something turns it back on, and every later + /// call starts from the options it left behind. A caller who then tunes the limiter is + /// asking for a limiter, so a bound -RateLimitBurst or -RateLimitPerSecond clears it - + /// otherwise the burst lands on an option nothing reads and the session keeps running + /// unlimited with a rate on the books. + /// + [Theory] + [InlineData("RateLimitBurst", 777)] + [InlineData("RateLimitPerSecond", 77)] + public void A_rate_parameter_turns_rate_limiting_back_on(string parameter, int value) + { + try + { + using (var off = Shell()) + { + off.AddCommand("Set-MgxOption").AddParameter("NoRateLimit", true); + off.Invoke(); + } + + // What the second call has to clear. Without this the assertions below hold on a + // session that never turned the limiter off in the first place. + Assert.True(Reported().NoRateLimit, + "-NoRateLimit did not take, so nothing below is being cleared"); + + using (var tune = Shell()) + { + tune.AddCommand("Set-MgxOption").AddParameter(parameter, value); + tune.Invoke(); + } + + var reported = Reported(); + Assert.False(reported.NoRateLimit); + Assert.Equal(value, parameter == "RateLimitBurst" + ? reported.RateLimitBurst + : reported.RateLimitPerSecond); + } + finally + { + using var reset = Shell(); + reset.AddCommand("Set-MgxOption").AddParameter("Reset", true); + reset.Invoke(); + } + } + + /// + /// The counters are per-process too, and they are one session's record of what it sent: a + /// removal that leaves them set has Get-MgxTelemetry crediting a fresh import with the + /// requests of the import before it - the same shape as the options and the endpoint a + /// removal already puts back. + /// + [Fact] + public void A_module_removal_releases_the_telemetry_counters() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, "{\"value\":[]}"); + + MgxTelemetryCollector.Current.Reset(); + using (MgxTransportScope.Inject(wire)) + using (var request = Shell()) + { + request.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users"); + request.Invoke(); + Assert.Empty(request.Streams.Error.Select(e => e.FullyQualifiedErrorId)); + } + + // What the removal has to release: this import's own traffic, reported as current. + Assert.True(Telemetry().Requests > 0, + "the request the seam served was never counted, so the release below proves nothing"); + + // A removal is ReleaseStaticState and then the detach; an import is OnImport. + AlcInitializer.ReleaseStaticState(); + new AlcInitializer().OnImport(); + + Assert.Equal(0L, Telemetry().Requests); + } + + /// + /// The options a session sets are per-process state, and a removal that leaves them set gives + /// the next import somebody else's tuning under a fresh module - with Get-MgxOption reporting + /// it as current, so nothing says the reset the re-import looks like never happened. + /// + [Fact] + public void A_module_removal_puts_the_options_back_to_their_defaults() + { + try + { + using (var set = Shell()) + { + set.AddCommand("Set-MgxOption") + .AddParameter("RateLimitBurst", 999) + .AddParameter("RateLimitPerSecond", 42) + .AddParameter("NoRateLimit", true) + .AddParameter("RateLimitQueueLimit", 12345) + .AddParameter("NoAdaptivePacing", true) + .AddParameter("MaxRetryAfterSeconds", 7) + .AddParameter("MaxRetryAttempts", 3) + .AddParameter("TotalTimeoutSeconds", 111) + .AddParameter("AttemptTimeoutSeconds", 9) + .AddParameter("CircuitBreakerDurationSeconds", 8) + .AddParameter("CircuitBreakerFailureRatio", 0.55) + .AddParameter("CircuitBreakerMinThroughput", 6) + .AddParameter("CircuitBreakerSamplingDurationSeconds", 11) + .AddParameter("BatchChunkConcurrency", 4) + .AddParameter("BatchItemsPerSecond", 0); + set.Invoke(); + } + + // Every settable option away from its default, so the removal below has all of the + // option surface to put back - not just the one property this test used to read. + var changed = Reported(); + Assert.Equal(999, changed.RateLimitBurst); + Assert.Equal(42, changed.RateLimitPerSecond); + Assert.True(changed.NoRateLimit); + Assert.Equal(12345, changed.RateLimitQueueLimit); + Assert.True(changed.NoAdaptivePacing); + Assert.Equal(7, changed.MaxRetryAfterSeconds); + Assert.Equal(3, changed.MaxRetryAttempts); + Assert.Equal(111, changed.TotalTimeoutSeconds); + Assert.Equal(9, changed.AttemptTimeoutSeconds); + Assert.Equal(8, changed.CircuitBreakerDurationSeconds); + Assert.Equal(0.55, changed.CircuitBreakerFailureRatio); + Assert.Equal(6, changed.CircuitBreakerMinThroughput); + Assert.Equal(11, changed.CircuitBreakerSamplingDurationSeconds); + Assert.Equal(4, changed.BatchChunkConcurrency); + Assert.Equal(0, changed.BatchItemsPerSecond); + + // A removal is ReleaseStaticState and then the detach; an import is OnImport. + AlcInitializer.ReleaseStaticState(); + new AlcInitializer().OnImport(); + + // The whole surface back to Default, not only the property this test used to check. + var restored = Reported(); + var defaults = ResilientGraphClientOptions.Default; + Assert.Equal(defaults.RateLimitBurst, restored.RateLimitBurst); + Assert.Equal(defaults.RateLimitPerSecond, restored.RateLimitPerSecond); + Assert.Equal(defaults.NoRateLimit, restored.NoRateLimit); + Assert.Equal(defaults.RateLimitQueueLimit, restored.RateLimitQueueLimit); + Assert.Equal(defaults.NoAdaptivePacing, restored.NoAdaptivePacing); + Assert.Equal(defaults.MaxRetryAfterSeconds, restored.MaxRetryAfterSeconds); + Assert.Equal(defaults.MaxRetryAttempts, restored.MaxRetryAttempts); + Assert.Equal(defaults.TotalTimeoutSeconds, restored.TotalTimeoutSeconds); + Assert.Equal(defaults.AttemptTimeoutSeconds, restored.AttemptTimeoutSeconds); + Assert.Equal(defaults.CircuitBreakerDurationSeconds, restored.CircuitBreakerDurationSeconds); + Assert.Equal(defaults.CircuitBreakerFailureRatio, restored.CircuitBreakerFailureRatio); + Assert.Equal(defaults.CircuitBreakerMinThroughput, restored.CircuitBreakerMinThroughput); + Assert.Equal(defaults.CircuitBreakerSamplingDurationSeconds, restored.CircuitBreakerSamplingDurationSeconds); + Assert.Equal(defaults.BatchChunkConcurrency, restored.BatchChunkConcurrency); + Assert.Equal(defaults.BatchItemsPerSecond, restored.BatchItemsPerSecond); + } + finally + { + using var reset = Shell(); + reset.AddCommand("Set-MgxOption").AddParameter("Reset", true); + reset.Invoke(); + } + } + + /// + /// The timeout the last client build sized its transport to. MgxCmdletBase keeps it private, + /// and nothing above it reports what a build is comparing against, so the field is the state + /// under test. + /// + private static FieldInfo CachedTimeoutField() + { + var field = typeof(MgxCmdletBase).GetField( + "s_cachedTotalTimeoutSeconds", BindingFlags.Static | BindingFlags.NonPublic); + Assert.True(field != null, + "MgxCmdletBase no longer keeps the cached timeout in s_cachedTotalTimeoutSeconds"); + return field!; + } + + /// + /// The endpoint and that timeout are per-process too, and derived rather than configured: a + /// removal that leaves them set has the next import composing URLs against the cloud the + /// import before it connected to, and its first build comparing against a + /// TotalTimeoutSeconds nobody in the new session chose, until that build derives both again. + /// + [Fact] + public void A_module_removal_releases_the_endpoint_and_timeout_a_build_derived() + { + const string sovereign = "https://microsoftgraph.chinacloudapi.cn"; + var timeout = CachedTimeoutField(); + var previousEndpoint = MgxCmdletBase.s_graphEndpoint; + var previousTimeout = timeout.GetValue(null); + try + { + // Where a build against a sovereign cloud leaves the process. + MgxCmdletBase.s_graphEndpoint = sovereign; + timeout.SetValue(null, 900); + + // A removal is ReleaseStaticState and then the detach; an import is OnImport. + AlcInitializer.ReleaseStaticState(); + new AlcInitializer().OnImport(); + + Assert.Equal(MgxCmdletBase.DefaultGraphEndpoint, MgxCmdletBase.s_graphEndpoint); + Assert.Equal(0, timeout.GetValue(null)); + + // And the endpoint the next build derives comes off the session it finds, not off + // anything the removal left behind. Asserted on the static a build writes, through a + // build: GetGraphEndpoint on its own reads the session and nothing else, so it answers + // the same whether the release ran or not, and whether anything drives the + // re-derivation or not. + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, "{\"value\":[]}"); + using var sessionClient = new HttpClient(wire); + + // No transport seam: it sits above the auth probe in GetClient, and the endpoint is + // derived on the build below it. GetClient cannot build a client of its own against a + // session this fake, so it borrows the one on the session - which is the mock. + using var scope = GraphSessionScope.Arm( + sessionClient, GraphSessionScope.AuthContextFor( + "11111111-1111-1111-1111-111111111111", + "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")); + scope.Session.Environment = new FakeGraphEnvironment { GraphEndpoint = sovereign }; + + using (var request = Shell()) + { + request.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("WarningAction", ActionPreference.SilentlyContinue); + request.Invoke(); + Assert.Empty(request.Streams.Error.Select(e => e.FullyQualifiedErrorId)); + } + + Assert.Equal(sovereign, MgxCmdletBase.s_graphEndpoint); + + // The static is what the cmdlets compose from, so the request that went out is the + // reading of it that matters. + Assert.StartsWith( + $"{sovereign}/v1.0/users", + Assert.Single(wire.CapturedRequests).Uri, StringComparison.Ordinal); + } + finally + { + MgxCmdletBase.s_graphEndpoint = previousEndpoint; + timeout.SetValue(null, previousTimeout); + } + } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/StalledBodyTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/StalledBodyTests.cs index 789a7ba..3c52669 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/StalledBodyTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/StalledBodyTests.cs @@ -15,27 +15,6 @@ namespace Mgx.IntegrationTests; [Collection("Pipeline")] public class StalledBodyTests { - /// Sends one byte of body and then nothing, honoring only the read's own token. - private sealed class StallingContent : HttpContent - { - protected override async Task SerializeToStreamAsync(Stream stream, TransportContext? context) - => await SerializeToStreamAsync(stream, context, CancellationToken.None); - - protected override async Task SerializeToStreamAsync( - Stream stream, TransportContext? context, CancellationToken cancellationToken) - { - stream.WriteByte((byte)'{'); - await stream.FlushAsync(cancellationToken); - await Task.Delay(Timeout.Infinite, cancellationToken); - } - - protected override bool TryComputeLength(out long length) - { - length = 4096; - return true; - } - } - private sealed class StallingHandler(HttpStatusCode status) : HttpMessageHandler { protected override Task SendAsync( diff --git a/tests/Mgx.IntegrationTests/Engine/Batch/BatchWriteTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/BatchWriteTests.cs index 03e039f..1dd6099 100644 --- a/tests/Mgx.IntegrationTests/Engine/Batch/BatchWriteTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Batch/BatchWriteTests.cs @@ -1518,7 +1518,7 @@ public void RedactSensitiveFields_RedactsPasswordProfile() } """); - InvokeMgxBatchRequest.RedactSensitiveFields(node); + node = InvokeMgxBatchRequest.RedactSensitiveFields(node); var obj = node!.AsObject(); Assert.Equal("Test User", obj["displayName"]!.GetValue()); @@ -1537,7 +1537,7 @@ public void RedactSensitiveFields_RedactsNestedPassword() } """); - InvokeMgxBatchRequest.RedactSensitiveFields(node); + node = InvokeMgxBatchRequest.RedactSensitiveFields(node); var obj = node!.AsObject(); Assert.True(obj["accountEnabled"]!.GetValue()); @@ -1556,7 +1556,7 @@ public void RedactSensitiveFields_PreservesNonSensitiveFields() } """); - InvokeMgxBatchRequest.RedactSensitiveFields(node); + node = InvokeMgxBatchRequest.RedactSensitiveFields(node); var obj = node!.AsObject(); Assert.Equal("Test", obj["displayName"]!.GetValue()); @@ -1567,8 +1567,8 @@ public void RedactSensitiveFields_PreservesNonSensitiveFields() [Fact] public void RedactSensitiveFields_HandlesNullNode() { - // Should not throw - InvokeMgxBatchRequest.RedactSensitiveFields(null); + // Should not throw, and a body that was nothing is still nothing. + Assert.Null(InvokeMgxBatchRequest.RedactSensitiveFields(null)); } [Fact] @@ -1582,7 +1582,7 @@ public void RedactSensitiveFields_RedactsKeyAndPasswordCredentials() } """); - InvokeMgxBatchRequest.RedactSensitiveFields(node); + node = InvokeMgxBatchRequest.RedactSensitiveFields(node); var obj = node!.AsObject(); Assert.Equal("App", obj["displayName"]!.GetValue()); @@ -1599,7 +1599,7 @@ public void RedactSensitiveFields_DeepNesting_FiveLevels() } """); - InvokeMgxBatchRequest.RedactSensitiveFields(node); + node = InvokeMgxBatchRequest.RedactSensitiveFields(node); var d = node!["a"]!["b"]!["c"]!["d"]!.AsObject(); Assert.Equal("***REDACTED***", d["password"]!.GetValue()); @@ -1617,7 +1617,7 @@ public void RedactSensitiveFields_RedactsClientSecretAndAppPassword() } """); - InvokeMgxBatchRequest.RedactSensitiveFields(node); + node = InvokeMgxBatchRequest.RedactSensitiveFields(node); var obj = node!.AsObject(); Assert.Equal("App", obj["displayName"]!.GetValue()); @@ -1639,7 +1639,7 @@ public void RedactSensitiveFields_RedactsInsideArrays() } """); - InvokeMgxBatchRequest.RedactSensitiveFields(node); + node = InvokeMgxBatchRequest.RedactSensitiveFields(node); var users = node!["users"]!.AsArray(); Assert.Equal("Alice", users[0]!["displayName"]!.GetValue()); @@ -1648,6 +1648,40 @@ public void RedactSensitiveFields_RedactsInsideArrays() Assert.Equal("***REDACTED***", users[1]!["clientSecret"]!.GetValue()); } + /// + /// The URL rule reads path segments. Split naively, an absolute URL hands its scheme and + /// host to the same test as its path, and a tenant whose Graph host is called + /// keyvault.example.com loses every body it ever dead-letters - the file keeps its + /// Url/Method/Status and nothing to read. + /// + [Theory] + [InlineData("https://keyvault.example.com/v1.0/groups/g1", false)] + [InlineData("https://tokens.contoso.com/v1.0/users/u1", false)] + [InlineData("https://graph.microsoft.com/v1.0/users/u1/resetPassword", true)] + [InlineData("https://graph.microsoft.com/v1.0/applications/a1/addKey", true)] + [InlineData("/trustFramework/keySets/ks1/uploadSecret", true)] + [InlineData("/servicePrincipals/sp1/synchronization/secrets", true)] + [InlineData("/users/u1/resetPassword", true)] + [InlineData("/groups/g1?$select=displayName", false)] + // A query string, not a path. On Unix a leading slash parses as a file URI and the parser + // folds the query into the path, so these read as one more segment unless the scheme is + // checked - and they would then answer differently here and on Windows. + [InlineData("/users/u1?password=1", false)] + [InlineData("/servicePrincipals/x?$select=keyCredentials", false)] + [InlineData("users/u1?password=1", false)] + // %50 is 'P'. Graph reads the path decoded, so this is the same reset the two rows above + // are; the parser unescapes it in the absolute form and the batch envelope carries the + // relative one, which is where a name could hide behind an escape. + [InlineData("/users/u1/microsoft.graph.reset%50assword", true)] + [InlineData("https://graph.microsoft.com/v1.0/users/u1/microsoft.graph.reset%50assword", true)] + // An escape that decodes to nothing anyone named, and one that does not decode at all. + [InlineData("/groups/g1/%6Dembers", false)] + [InlineData("/groups/100%25", false)] + public void UrlNamesSecret_ReadsPathSegmentsOnly(string url, bool expected) + { + Assert.Equal(expected, InvokeMgxBatchRequest.UrlNamesSecret(url)); + } + [Fact] public void RedactSensitiveFields_RedactsRootLevelArray() { @@ -1660,7 +1694,7 @@ public void RedactSensitiveFields_RedactsRootLevelArray() ] """); - InvokeMgxBatchRequest.RedactSensitiveFields(node); + node = InvokeMgxBatchRequest.RedactSensitiveFields(node); var arr = node!.AsArray(); Assert.Equal("Alice", arr[0]!["displayName"]!.GetValue()); @@ -1669,6 +1703,1384 @@ public void RedactSensitiveFields_RedactsRootLevelArray() Assert.Equal("***REDACTED***", arr[1]!["passwordProfile"]!.GetValue()); } + // ═══════════════════════════════════════════════════════════════ + // Dead-letter: what a written line is allowed to carry + // ═══════════════════════════════════════════════════════════════ + + private const string OneItemRejected = """ + { "responses": [ + { "id": "1", "status": 400, "body": { "error": { "code": "BadRequest", "message": "rejected" } } } + ] } + """; + + private const string OneItemAccepted = """ + { "responses": [ + { "id": "1", "status": 200, "body": { "id": "u1" } } + ] } + """; + + private static System.Management.Automation.PowerShell CreateShell() + { + var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + return ps; + } + + /// + /// Reads the dead-letter file while a sibling run still holds it open to append to. Windows + /// checks the sharing both ways round - what the reader asks of the file, and what it will + /// let a handle already on it go on doing - so File.ReadAllLines, which asks for + /// FileShare.Read and nothing more, is refused by the run this test left holding it. The + /// sharing that holder asked for is offered back, which is what a reader of a file a live + /// run is appending to has to do on either platform. + /// + private static string[] ReadSharedLines(string path) + { + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, + FileShare.ReadWrite); + using var reader = new StreamReader(stream); + var lines = new List(); + while (reader.ReadLine() is { } line) lines.Add(line); + return [.. lines]; + } + + /// + /// Sends one item the server refuses and returns the line the dead-letter file got for it. + /// The assertions are about the file, not about the redactor: what a caller can read off + /// disk is the whole question, and the URL rule only exists in the writer. + /// + private static string DeadLetterLineFor(string url, string method, System.Collections.Hashtable body) + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, OneItemRejected); + using var transport = MgxTransportScope.Inject(handler); + + var path = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable + { + ["Url"] = url, + ["Method"] = method, + ["Body"] = body + } + }) + .AddParameter("DeadLetterPath", path); + ps.Invoke(); + + Assert.True(File.Exists(path), "dead-letter file was not written"); + return Assert.Single(File.ReadAllLines(path)); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void RedactedDeadLetterLine_OmitsANewPasswordAtTheTopOfTheBody() + { + var line = DeadLetterLineFor("/users/u1", "PATCH", new System.Collections.Hashtable + { + ["displayName"] = "Keep This", + ["newPassword"] = "sentinel-new-password" + }); + + Assert.DoesNotContain("sentinel-new-password", line); + Assert.Contains("***REDACTED***", line); + // The diagnostic half of the line is what the file is for; only the credential goes. + Assert.Contains("Keep This", line); + } + + [Fact] + public void RedactedDeadLetterLine_OmitsAPreSharedKeyAndASharedSecretMidBody() + { + var line = DeadLetterLineFor("/deviceManagement/deviceConfigurations", "POST", + new System.Collections.Hashtable + { + ["displayName"] = "Corp Wi-Fi", + ["wifiSettings"] = new System.Collections.Hashtable + { + ["preSharedKey"] = "sentinel-wifi-psk" + }, + ["vpnSettings"] = new System.Collections.Hashtable + { + ["sharedSecret"] = "sentinel-vpn-secret" + } + }); + + Assert.DoesNotContain("sentinel-wifi-psk", line); + Assert.DoesNotContain("sentinel-vpn-secret", line); + Assert.Contains("Corp Wi-Fi", line); + } + + [Fact] + public void RedactedDeadLetterLine_OmitsASecretTextNestedInAPasswordCredential() + { + var line = DeadLetterLineFor("/applications/app1", "PATCH", new System.Collections.Hashtable + { + ["displayName"] = "Corp App", + ["passwordCredential"] = new System.Collections.Hashtable + { + ["displayName"] = "rotation", + ["secretText"] = "sentinel-secret-text" + } + }); + + Assert.DoesNotContain("sentinel-secret-text", line); + Assert.Contains("Corp App", line); + } + + [Fact] + public void RedactedDeadLetterLine_OmitsAPasswordInsideAnArrayOfArrays() + { + // An array directly inside an array: no object between them, so a redactor that only + // steps from an array into an object never reaches the field. A -Body is whatever the + // caller built, so the writer cannot assume a shape. + var line = DeadLetterLineFor("/deviceManagement/configurationPolicies", "POST", + new System.Collections.Hashtable + { + ["name"] = "Corp Baseline", + ["settings"] = new object[] + { + new object[] + { + new System.Collections.Hashtable { ["password"] = "sentinel-nested-array" } + } + } + }); + + Assert.DoesNotContain("sentinel-nested-array", line); + Assert.Contains("Corp Baseline", line); + } + + [Fact] + public void RedactedDeadLetterLine_OmitsEveryNameTheExactListCovered() + { + var line = DeadLetterLineFor("/users/u1", "PATCH", new System.Collections.Hashtable + { + ["displayName"] = "Corp User", + ["passwordProfile"] = new System.Collections.Hashtable + { + ["password"] = "sentinel-passwordprofile" + }, + ["password"] = "sentinel-password", + ["secretText"] = "sentinel-secrettext", + ["keyCredentials"] = new object[] + { + new System.Collections.Hashtable { ["key"] = "sentinel-keycredentials" } + }, + ["passwordCredentials"] = new object[] + { + new System.Collections.Hashtable { ["secretText"] = "sentinel-passwordcredentials" } + }, + ["clientSecret"] = "sentinel-clientsecret", + ["appPassword"] = "sentinel-apppassword", + ["clientAssertion"] = "sentinel-clientassertion" + }); + + foreach (var sentinel in new[] + { + "sentinel-passwordprofile", "sentinel-password", "sentinel-secrettext", + "sentinel-keycredentials", "sentinel-passwordcredentials", "sentinel-clientsecret", + "sentinel-apppassword", "sentinel-clientassertion" + }) + { + Assert.DoesNotContain(sentinel, line); + } + + Assert.Contains("Corp User", line); + } + + [Fact] + public void RedactedDeadLetterLine_ReplacesTheWholeBodyWhenTheUrlNamesAnUploadSecret() + { + // The credential is the field called `k`; nothing about that name says so. + var line = DeadLetterLineFor("/trustFramework/keySets/ks1/uploadSecret", "POST", + new System.Collections.Hashtable + { + ["use"] = "sig", + ["k"] = "sentinel-upload-secret" + }); + + Assert.DoesNotContain("sentinel-upload-secret", line); + + var written = JsonNode.Parse(line)!.AsObject(); + Assert.Equal("***REDACTED***", written["Body"]!.GetValue()); + // What the line is still worth: everything a caller needs to decide on the item. + Assert.Equal("/trustFramework/keySets/ks1/uploadSecret", written["Url"]!.GetValue()); + Assert.Equal("POST", written["Method"]!.GetValue()); + Assert.Equal(400, written["Status"]!.GetValue()); + Assert.Contains("BadRequest", written["Error"]!.GetValue()); + } + + [Fact] + public void RedactedDeadLetterLine_ReplacesTheWholeBodyForSynchronizationSecrets() + { + // Here the credential is one `value` among a list of them. + var line = DeadLetterLineFor("/servicePrincipals/sp1/synchronization/secrets", "PUT", + new System.Collections.Hashtable + { + ["value"] = new object[] + { + new System.Collections.Hashtable + { + ["key"] = "ClientSecret", + ["value"] = "sentinel-sync-secret" + } + } + }); + + Assert.DoesNotContain("sentinel-sync-secret", line); + + var written = JsonNode.Parse(line)!.AsObject(); + Assert.Equal("***REDACTED***", written["Body"]!.GetValue()); + } + + /// + /// A dead-letter line re-piped the way -DeadLetterPath's recipe says to. A Body that is the + /// marker whole is refused already - the marker is not JSON - but a Body carrying it in one + /// field parses, and went to the tenant with the literal marker where the credential was. + /// + [Fact] + public void RePipedBody_CarryingTheRedactionMarkerInAField_IsRefusedAndNotSent() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, OneItemRejected); + using var transport = MgxTransportScope.Inject(handler); + + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable + { + ["Url"] = "/users/u1", + ["Method"] = "PATCH", + ["Body"] = new System.Collections.Hashtable + { + ["displayName"] = "Keep", + ["passwordProfile"] = "***REDACTED***" + } + }, + new System.Collections.Hashtable + { + ["Url"] = "/groups/g1", + ["Method"] = "PATCH", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "the-other-item" } + } + }); + ps.Invoke(); + + var error = Assert.Single(ps.Streams.Error, + e => e.FullyQualifiedErrorId.StartsWith("InvalidBatchItemBody", StringComparison.Ordinal)); + Assert.Contains("PATCH /users/u1", error.Exception.Message); + // The field is named: it says which value the file took, and so which one to supply. + Assert.Contains("passwordProfile", error.Exception.Message); + + // The rest of the file still goes out, and the marker never reaches the wire. + var envelope = Assert.Single(handler.Requests).Content! + .ReadAsStringAsync().GetAwaiter().GetResult(); + Assert.DoesNotContain("REDACTED", envelope); + Assert.Contains("the-other-item", envelope); + } + + /// + /// An ordinary call, never a dead-letter line: a hashtable body a caller wrote by hand, whose + /// own text happens to carry the marker. The refusal is the same one a re-piped line gets, + /// because nothing in the text tells the two apart, so the message has to hold for both - + /// naming the field, and both remedies: a re-piped line is rebuilt from its Url and Method, + /// and a body of the caller's own must not carry that text at all. + /// + [Fact] + public void OrdinaryBody_CarryingTheMarkerText_IsRefusedNamingTheFieldAndBothRemedies() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, """{"responses":[{"id":"1","status":200,"body":{}}]}"""); + using var transport = MgxTransportScope.Inject(handler); + + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable + { + ["Url"] = "/users/u1", + ["Method"] = "PATCH", + ["Body"] = new System.Collections.Hashtable + { + ["displayName"] = "Redaction demo ***REDACTED*** room" + } + } + }); + ps.Invoke(); + + var error = Assert.Single(ps.Streams.Error, + e => e.FullyQualifiedErrorId.StartsWith("InvalidBatchItemBody", StringComparison.Ordinal)); + Assert.Equal( + "Body for PATCH /users/u1 carries the redaction marker '***REDACTED***' at " + + "'displayName'. A dead-letter line is not the body that was sent: rebuild the " + + "request from its Url and Method, supplying the credential again. A body of your " + + "own must not carry that text.", + error.Exception.Message); + + // A body that never came from the file is refused the same way, before anything is sent. + Assert.Empty(handler.Requests); + } + + /// + /// The same recipe on a line whose marker is in a KEY, not a value. The writer cuts a + /// pre-authenticated URL used as a field name at its capability parameter and re-adds the + /// property under the cut name, suffixing #2 when two keys cut to the same string - so the + /// marker can be in a line with no string value carrying it. A guard reading values only + /// passed such a line through, and the tenant got the literal marker as a field name. + /// + [Fact] + public void RePipedBody_CarryingTheRedactionMarkerInAKey_IsRefusedAndNothingIsSent() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, OneItemRejected); + using var transport = MgxTransportScope.Inject(handler); + + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable + { + ["Url"] = "/users/u1", + ["Method"] = "PATCH", + ["Body"] = """{"https://h/f?sig=***REDACTED***#2":"1"}""" + } + }); + ps.Invoke(); + + var error = Assert.Single(ps.Streams.Error, + e => e.FullyQualifiedErrorId.StartsWith("InvalidBatchItemBody", StringComparison.Ordinal)); + Assert.Contains("PATCH /users/u1", error.Exception.Message); + // The key as written, collision suffix and all: it is what has to be supplied again. + Assert.Contains("https://h/f?sig=***REDACTED***#2", error.Exception.Message); + + // The only item was refused, so no chunk was built and nothing reached the wire. + Assert.Empty(handler.Requests); + } + + /// + /// The same recipe on the shape the URL rule leaves, where the marker sits INSIDE a value: + /// a pre-authenticated URL is cut after its capability parameter, so what stands in the + /// file is the URL prefix with the marker after it and never the marker alone. Comparing a + /// value to the marker whole let those lines through, and the tenant got a + /// "?sig=***REDACTED***" URL no host will honor in place of the one the caller believes it + /// supplied. Three shapes, because the writer cuts a URL in all three: a property's value, + /// one nested in an array of objects, and a bare array element with no property name at all. + /// + [Fact] + public void RePipedBody_CarryingACutCapabilityUrlInAValue_IsRefusedAndNothingIsSent() + { + (System.Collections.Hashtable Body, string Field)[] shapes = + [ + (new System.Collections.Hashtable + { + ["uploadUri"] = "https://s.blob.core.windows.net/c/b?sv=2021&sig=sentinel-sas" + }, "uploadUri"), + (new System.Collections.Hashtable + { + ["files"] = new object[] + { + new System.Collections.Hashtable + { + ["uploadUri"] = "https://h/f?tempauth=sentinel-jwt" + } + } + }, "files[0].uploadUri"), + (new System.Collections.Hashtable + { + ["a"] = new object[] { "https://h/f?sig=sentinel-bare" } + }, "a[0]") + ]; + + foreach (var (body, field) in shapes) + { + // A real failed item's line, read back the way -DeadLetterPath's recipe reads it. + var written = JsonNode.Parse(DeadLetterLineFor("/users/u1", "PATCH", body))!; + var rePiped = written["Body"]!.ToJsonString(); + Assert.Contains(InvokeMgxBatchRequest.RedactionMarker, rePiped, StringComparison.Ordinal); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, OneItemRejected); + using var transport = MgxTransportScope.Inject(handler); + + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable + { + ["Url"] = "/users/u1", + ["Method"] = "PATCH", + ["Body"] = rePiped + } + }); + ps.Invoke(); + + var error = Assert.Single(ps.Streams.Error, + e => e.FullyQualifiedErrorId.StartsWith("InvalidBatchItemBody", StringComparison.Ordinal)); + Assert.Contains("PATCH /users/u1", error.Exception.Message); + // The field is named at its own depth, which is what says what to supply again. + Assert.Contains(field, error.Exception.Message); + + // The only item was refused, so no chunk was built and nothing reached the wire. + Assert.Empty(handler.Requests); + } + } + + /// + /// A raw-string body the JSON parser accepts and the marker guard cannot read. An unpaired + /// surrogate escape is a legal string token whose value has no UTF-16 form, so the parse + /// succeeds and the read of that one string throws - and the guard reads every string of + /// every body. It is one item's body, so it is refused as one, like a body that is not JSON + /// at all: the guard is the only thing that decides whether a body may be sent, so a body + /// it cannot read is a body it cannot clear. + /// + [Fact] + public void Body_TheMarkerGuardCannotRead_IsRefusedAndTheOtherItemIsSent() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, + """{"responses":[{"id":"1","status":201,"body":{"id":"g1"}}]}"""); + using var transport = MgxTransportScope.Inject(handler); + + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable + { + ["Url"] = "/users", + ["Method"] = "POST", + ["Body"] = """{"displayName":"Jos\ud83d"}""" + }, + new System.Collections.Hashtable + { + ["Url"] = "/groups", + ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "the-other-item" } + } + }); + var output = ps.Invoke(); + + var error = Assert.Single(ps.Streams.Error, + e => e.FullyQualifiedErrorId.StartsWith("InvalidBatchItemBody", StringComparison.Ordinal)); + Assert.StartsWith( + "Body for POST /users could not be read for the redaction check: ", + error.Exception.Message, StringComparison.Ordinal); + // The runtime's own words for what it could not read, and the remedy. The reason's + // trailing period is trimmed rather than doubled onto the sentence quoting it. + Assert.Contains("surrogate", error.Exception.Message, StringComparison.Ordinal); + Assert.EndsWith(". Send it as a hashtable or a string this run can read.", + error.Exception.Message, StringComparison.Ordinal); + Assert.DoesNotContain("..", error.Exception.Message, StringComparison.Ordinal); + Assert.Equal("/users", error.TargetObject); + + // One item was refused; the other one went, was answered, and is in the output. The run + // used to end on the unreadable body with nothing sent and no record of anything. + var envelope = Assert.Single(handler.Requests).Content! + .ReadAsStringAsync().GetAwaiter().GetResult(); + Assert.Contains("the-other-item", envelope, StringComparison.Ordinal); + Assert.DoesNotContain("Jos", envelope, StringComparison.Ordinal); + + var result = Assert.Single(output).BaseObject as System.Collections.Hashtable; + Assert.NotNull(result); + Assert.Equal("/groups", result["Url"]); + Assert.Equal(201, result["Status"]); + } + + [Fact] + public void FindRedactionMarker_NamesTheFieldAtAnyDepth() + { + var nested = JsonSerializer.Deserialize( + """{"a":{"b":[{"c":"x"},{"keyCredentials":"***REDACTED***"}]}}"""); + Assert.Equal("a.b[1].keyCredentials", InvokeMgxBatchRequest.FindRedactionMarker(nested)); + + // A value is matched as a substring, not compared whole. The writer's cut leaves the + // URL up to the capability parameter in front of the marker, so a comparison saw only + // the cuts that took a whole value; and nothing in the text tells the file's marker + // from a caller's own, since the file's is a prefix plus the marker too. A caller who + // writes "***REDACTED***x" himself is refused with the rest - renaming a field costs + // less than a request the tenant cannot honor. + var ownText = JsonSerializer.Deserialize("""{"a":["***REDACTED***x"],"b":1}"""); + Assert.Equal("a[0]", InvokeMgxBatchRequest.FindRedactionMarker(ownText)); + + // The cut form the URL rule actually leaves in a value, at the depth the writer reaches. + var cutUrl = JsonSerializer.Deserialize( + """{"files":[{"uploadUri":"https://h/f?tempauth=***REDACTED***"}]}"""); + Assert.Equal("files[0].uploadUri", InvokeMgxBatchRequest.FindRedactionMarker(cutUrl)); + + // A name is matched the same way: a key carrying the marker is never the marker alone, + // and the path returned is the name as written. + var inAKey = JsonSerializer.Deserialize( + """{"keep":"v","https://h/f?sig=***REDACTED***#2":"1"}"""); + Assert.Equal("https://h/f?sig=***REDACTED***#2", + InvokeMgxBatchRequest.FindRedactionMarker(inAKey)); + + // Nested, so the path is built the same way the value case builds it. + var nestedKey = JsonSerializer.Deserialize( + """{"a":[{"https://h/f?tempauth=***REDACTED***":"1"}]}"""); + Assert.Equal("a[0].https://h/f?tempauth=***REDACTED***", + InvokeMgxBatchRequest.FindRedactionMarker(nestedKey)); + } + + /// + /// Intune's content-file body, on the line a caller reads off disk. The credential is the + /// URL: azureStorageUri and uploadUrl fetch and fill the blob on the strength of the SAS + /// alone, and no name in the body says so. A -Debug trace of the same body has hidden them + /// all along; BodyRedactionTests holds the two side by side. + /// + [Fact] + public void RedactedDeadLetterLine_CutsAPreAuthenticatedUrlAtItsCapabilityParameter() + { + var line = DeadLetterLineFor( + "/deviceAppManagement/mobileApps/a1/contentVersions/1/files/f1", "PATCH", + new System.Collections.Hashtable + { + ["azureStorageUri"] = "https://s.blob.core.windows.net/c/b?sv=2021-08-06&sig=sentinel-sas&se=2026", + ["@microsoft.graph.downloadUrl"] = "https://public.bl.files.1drv.com/y4msentinel-path/f.bin" + }); + + Assert.DoesNotContain("sentinel-sas", line); + Assert.DoesNotContain("sentinel-path", line); + + var body = JsonNode.Parse(line)!["Body"]!.AsObject(); + // The host and the parameter name stay. Which blob and which kind of URL is the half + // the file is read for, and losing it would leave the line saying only that one failed. + Assert.Equal("https://s.blob.core.windows.net/c/b?sv=2021-08-06&sig=***REDACTED***", + body["azureStorageUri"]!.GetValue()); + // The capability is in the path here, so the property name is what reaches it and the + // whole value goes. + Assert.Equal("***REDACTED***", body["@microsoft.graph.downloadUrl"]!.GetValue()); + } + + /// + /// A pre-authenticated URL used as the item URL rather than as a value inside a body. An + /// upload session's uploadUrl and a @microsoft.graph.downloadUrl are requests a caller + /// makes, so the URL a batch item names is as much a credential as one a body carries - and + /// the line wrote it verbatim while cutting the identical URL beside it in the Body. + /// + /// Both halves, from the run's own artifacts: the line on disk, and the envelope as the + /// trace formats it. NormalizeToRelativeUrl takes the scheme and host off before the POST, + /// so the two forms are not the same text and each needed its own rule. + /// + [Fact] + public void ACapabilityUrlUsedAsTheItemUrl_IsCutInTheFileAndInTheTrace() + { + const string itemUrl = "https://s.blob.core.windows.net/c/b?sv=2021&sig=sentinel-item-sas"; + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, OneItemRejected); + using var transport = MgxTransportScope.Inject(handler); + + var path = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + string line; + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable + { + ["Url"] = itemUrl, + ["Method"] = "PUT", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "Keep This" } + } + }) + .AddParameter("DeadLetterPath", path); + ps.Invoke(); + + Assert.True(File.Exists(path), "dead-letter file was not written"); + line = Assert.Single(File.ReadAllLines(path)); + } + finally + { + File.Delete(path); + } + + Assert.DoesNotContain("sentinel-item-sas", line); + var written = JsonNode.Parse(line)!.AsObject(); + // Cut where a body's value would be cut: the host and the parameter name stay, so the + // line still says which blob and which kind of URL failed. + Assert.Equal("https://s.blob.core.windows.net/c/b?sv=2021&sig=***REDACTED***", + written["Url"]!.GetValue()); + // The rest of the line is what the file is read for. + Assert.Equal("PUT", written["Method"]!.GetValue()); + Assert.Contains("Keep This", line); + + // The same URL on the wire, relative by then, in the trace of the envelope that carried + // it. A trace is what users paste into issue reports. + var captured = Assert.Single(handler.CapturedRequests); + var trace = GraphRequestTracer.FormatRequest( + new HttpRequestMessage(HttpMethod.Post, "https://graph.microsoft.com/v1.0/$batch"), + captured.Body, 1); + Assert.DoesNotContain("sentinel-item-sas", trace, StringComparison.Ordinal); + // The envelope's serializer escapes an ampersand, so the query in the trace is not the + // query in the file - "?sv=2021\u0026sig=". What both keep is the parameter name, which + // is what says which kind of URL went. + Assert.Contains("/c/b?sv=2021", trace, StringComparison.Ordinal); + Assert.Contains("sig=", trace, StringComparison.Ordinal); + } + + /// + /// The 429s a batch met and the time it spent in Retry-After sleeps, in the session view + /// after a run that -ErrorAction Stop ended on an item error. Those counters describe what + /// the wire did, which is settled by the time the batch returns; they used to be recorded at + /// the very end of the reporting, past the dead-letter write, the chunk failure and one + /// error record per failed item, so a run that stopped on any of those told Get-MgxTelemetry + /// nothing - and a throttled, retried batch is exactly the run whose numbers a caller opens + /// that view to find. + /// + /// Both preferences, because the point is that they agree: the stop must cost the caller + /// nothing the run already measured. + /// + [Theory] + [InlineData(System.Management.Automation.ActionPreference.Continue)] + [InlineData(System.Management.Automation.ActionPreference.Stop)] + public void SessionTelemetry_SurvivesAStopOnTheItemErrors( + System.Management.Automation.ActionPreference errorAction) + { + const string throttledAndRejected = """ + { "responses": [ + { "id": "1", "status": 429, "headers": { "Retry-After": "0" } }, + { "id": "2", "status": 400, "body": { "error": { "code": "BadRequest", "message": "rejected" } } } + ] } + """; + const string retrySucceeded = """ + { "responses": [ { "id": "1", "status": 201, "body": { "id": "u1" } } ] } + """; + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, throttledAndRejected); + handler.QueueResponse(HttpStatusCode.OK, retrySucceeded); + using var transport = MgxTransportScope.Inject(handler); + + MgxTelemetryCollector.Current.Reset(); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable { ["Url"] = "/users", ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "a" } }, + new System.Collections.Hashtable { ["Url"] = "/groups", ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "b" } } + }) + .AddParameter("ErrorAction", errorAction); + + if (errorAction == System.Management.Automation.ActionPreference.Stop) + Assert.ThrowsAny(() => ps.Invoke()); + else + ps.Invoke(); + + var session = MgxTelemetryCollector.Current.GetSummary(); + // One item was answered 429 and retried, and the retry waited. + Assert.Equal(1, session.BatchItemThrottles); + Assert.True(session.RetryDelayMs > 0, $"RetryDelayMs was {session.RetryDelayMs}"); + } + finally + { + MgxTelemetryCollector.Current.Reset(); + } + } + + private const string TwoItemsRejected = """ + { "responses": [ + { "id": "1", "status": 400, "body": { "error": { "code": "BadRequest", "message": "rejected" } } }, + { "id": "2", "status": 400, "body": { "error": { "code": "BadRequest", "message": "rejected" } } } + ] } + """; + + /// + /// Sends items the server refuses and returns every line the dead-letter file got, with the + /// warnings the run wrote. Nothing here asserts a line count: what a body the writer cannot + /// read costs the item and the items after it is the question. A run that ends on such a + /// body throws out of EndProcessing, so ps.Invoke() throwing is itself the failure - and it + /// names the exception, which is what the assertions below could not. + /// + private static (string[] Lines, string[] Warnings) DeadLetterRun(string response, params object[] items) + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, response); + using var transport = MgxTransportScope.Inject(handler); + + var path = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", items) + .AddParameter("DeadLetterPath", path); + ps.Invoke(); + + return (File.Exists(path) ? File.ReadAllLines(path) : [], + ps.Streams.Warning.Select(w => w.Message).ToArray()); + } + finally + { + File.Delete(path); + } + } + + /// + /// A raw-string body with the same property name twice. Graph takes it, and JsonNode.Parse + /// takes it; enumerating the JsonObject it built is what throws, so the failure lands in + /// the writer with the request already sent and answered. The line is owed either way. + /// + [Fact] + public void DeadLetterLine_ForABodyTheRedactorCannotWalk_CarriesTheMarkerAndWarns() + { + var (lines, warnings) = DeadLetterRun(TwoItemsRejected, + new System.Collections.Hashtable + { + ["Url"] = "/users", + ["Method"] = "POST", + ["Body"] = """{"displayName":"first","displayName":"second"}""" + }, + new System.Collections.Hashtable + { + ["Url"] = "/groups", + ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "the-second-item" } + }); + + Assert.Equal(2, lines.Length); + var first = JsonNode.Parse(lines[0])!.AsObject(); + Assert.Equal("/users", first["Url"]!.GetValue()); + Assert.Equal("***REDACTED***", first["Body"]!.GetValue()); + // The body was never read, so nothing in it may be written - not even the half a + // duplicate key left readable. + Assert.DoesNotContain("first", lines[0]); + + // The item after it still gets its line: one unreadable body is not the run. + Assert.Contains("the-second-item", lines[1]); + + // Not "Contains(withheld)" alone: the outcome warning now folds the withheld count + // into its own text too, so that predicate would match both. "POST /users" is what + // is unique to the per-item warning under test here. + var warning = Assert.Single(warnings, w => w.Contains("POST /users") && w.Contains("withheld")); + Assert.Contains("could not be read for redaction", warning); + } + + /// + /// The same body on the second of two items. The writer holds one StreamWriter open across + /// the loop, so an exception in the middle of it is what decides whether the lines already + /// written reach disk at all. + /// + [Fact] + public void DeadLetterFile_IsNotCutShortByABodyTheRedactorCannotWalk() + { + var (lines, warnings) = DeadLetterRun(TwoItemsRejected, + new System.Collections.Hashtable + { + ["Url"] = "/groups", + ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "the-first-item" } + }, + new System.Collections.Hashtable + { + ["Url"] = "/users", + ["Method"] = "POST", + ["Body"] = """{"displayName":"first","displayName":"second"}""" + }); + + Assert.Equal(2, lines.Length); + Assert.Contains("the-first-item", lines[0]); + Assert.Equal("***REDACTED***", JsonNode.Parse(lines[1])!["Body"]!.GetValue()); + Assert.Single(warnings, w => w.Contains("POST /users") && w.Contains("withheld")); + } + + /// + /// WriteWarning throws under -WarningAction Stop. The withheld-body warning used to be + /// written right after the dead-letter file closed - before the item errors and before + /// WriteBatchTelemetry - so that throw cut the run off before either ran: two refused items + /// produced two dead-letter lines and zero error records. The warning now runs last, so the + /// warning that actually stops this run is WriteBatchTelemetry's own outcome warning (it + /// still has one to give, since both items failed), and reaching it proves the item errors + /// and the telemetry it summarizes already ran. That outcome warning now folds the withheld + /// count in too, so it is what proves this run was stopped there rather than by the + /// dedicated per-item warning after it - the fold names the count without naming the item, + /// and the dedicated warning (never reached here) is the other way around. + /// + [Fact] + public void WarningActionStop_WithAnUnreadableBody_StillRecordsTheItemErrorsAndTheTelemetry() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TwoItemsRejected); + using var transport = MgxTransportScope.Inject(handler); + + var path = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable + { + ["Url"] = "/users", + ["Method"] = "POST", + ["Body"] = """{"displayName":"first","displayName":"second"}""" + }, + new System.Collections.Hashtable + { + ["Url"] = "/groups", + ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "the-second-item" } + } + }) + .AddParameter("DeadLetterPath", path) + .AddParameter("WarningAction", System.Management.Automation.ActionPreference.Stop); + + var ex = Assert.ThrowsAny(() => ps.Invoke()); + + // Reaching WriteBatchTelemetry's own outcome warning means the per-item errors and + // the telemetry summary it closes already happened. It is the outcome warning, not + // the dedicated per-item one, because it names the count rather than the item. + Assert.Contains("2 of 2 batch items failed", ex.Message); + Assert.Contains("1 body was withheld from the dead-letter file", ex.Message); + Assert.DoesNotContain("POST /users", ex.Message); + + var itemErrors = ps.Streams.Error + .Where(e => e.FullyQualifiedErrorId.StartsWith("BatchItemError", StringComparison.Ordinal)) + .ToList(); + Assert.Equal(2, itemErrors.Count); + + Assert.True(File.Exists(path), "dead-letter file was not written"); + Assert.Equal(2, File.ReadAllLines(path).Length); + } + finally + { + File.Delete(path); + } + } + + /// + /// A -DeadLetterPath the run cannot open, answered before the first POST. The open used to + /// happen where the lines are written, at the end, so a path like this was discovered with + /// the batch already applied: the caller had made every one of those changes and had no + /// record of which to retry. Nothing is sent now, and the message says so - which is what + /// makes the failure recoverable by fixing the path and running the same command again. + /// + /// An error record rather than a warning, at the default preference: a caller running + /// -WarningAction SilentlyContinue would otherwise lose the only line saying the file the + /// batch was to be recorded in is not there. + /// + [Fact] + public void DeadLetterPathThatCannotBeOpened_IsRefusedBeforeAnythingIsSent() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TwoItemsRejected); + using var transport = MgxTransportScope.Inject(handler); + + var dir = Path.Combine(Path.GetTempPath(), $"mgx-dl-dir-{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable + { + ["Url"] = "/users", + ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "a" } + }, + new System.Collections.Hashtable + { + ["Url"] = "/groups", + ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "b" } + } + }) + .AddParameter("DeadLetterPath", dir); + var output = ps.Invoke(); + + // One record, and no item errors behind it: there are no items to have errors. + var record = Assert.Single(ps.Streams.Error); + Assert.StartsWith("DeadLetterWriteFailed", record.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(System.Management.Automation.ErrorCategory.WriteError, + record.CategoryInfo.Category); + Assert.Equal(dir, record.TargetObject); + Assert.StartsWith("Failed to write dead-letter file '", record.Exception.Message, + StringComparison.Ordinal); + Assert.Contains(dir, record.Exception.Message); + Assert.EndsWith(" before the batch was sent; nothing was sent.", + record.Exception.Message, StringComparison.Ordinal); + // The reason the open refused is kept, not flattened into the text alone. + Assert.NotNull(record.Exception.InnerException); + + // Nothing went out, so there is no output and nothing to warn about. + Assert.Empty(handler.Requests); + Assert.Empty(output); + Assert.Empty(ps.Streams.Warning); + Assert.Empty(Directory.GetFiles(dir)); + } + finally + { + Directory.Delete(dir); + } + } + + /// + /// The same unusable path on a batch every item of which would have been applied. It is + /// refused just the same, and refused before the batch: the file is part of what the caller + /// asked for, and the run that could not have it is the run that has not yet spent anything. + /// The old order had no way to be this: the path was found out at the end, with the batch + /// applied, so refusing there would have told a scripted caller their batch failed when the + /// whole of it had succeeded, and the failure was softened to a warning to avoid saying so. + /// + [Fact] + public void DeadLetterPathThatCannotBeOpened_IsRefusedEvenWhenTheBatchWouldHaveSucceeded() + { + var dir = Path.Combine(Path.GetTempPath(), $"mgx-dl-dir-{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + try + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, BatchSuccessResponse); + using var transport = MgxTransportScope.Inject(handler); + + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] { "/users/u1", "/users/u2" }) + .AddParameter("DeadLetterPath", dir); + var output = ps.Invoke(); + + var record = Assert.Single(ps.Streams.Error); + Assert.StartsWith("DeadLetterWriteFailed", record.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.EndsWith(" before the batch was sent; nothing was sent.", + record.Exception.Message, StringComparison.Ordinal); + + // And it is true of the run: the two GETs never went. + Assert.Empty(handler.Requests); + Assert.Empty(output); + Assert.Empty(ps.Streams.Warning); + } + finally + { + Directory.Delete(dir); + } + } + + /// + /// The point of making it an error: under -ErrorAction Stop the run ends on it, and it ends + /// before the batch, so the caller is told the file is missing rather than told which items + /// to look for in it - and nothing has been applied to go looking for. The message names + /// the resolved path, which is the only place the path appears. + /// + [Fact] + public void ErrorActionStop_WithADeadLetterPathThatIsADirectory_TerminatesOnTheWriteFailure() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TwoItemsRejected); + using var transport = MgxTransportScope.Inject(handler); + + var dir = Path.Combine(Path.GetTempPath(), $"mgx-dl-dir-{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable + { + ["Url"] = "/users", + ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "a" } + }, + new System.Collections.Hashtable + { + ["Url"] = "/groups", + ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "b" } + } + }) + .AddParameter("DeadLetterPath", dir) + .AddParameter("ErrorAction", System.Management.Automation.ActionPreference.Stop); + + var ex = Assert.ThrowsAny( + () => ps.Invoke()); + + Assert.StartsWith("DeadLetterWriteFailed", + ex.ErrorRecord.FullyQualifiedErrorId, StringComparison.Ordinal); + // ActionPreferenceStopException wraps the record it stopped on, and its own message + // prefixes the preference text - the record's is the message the cmdlet wrote. + Assert.StartsWith("Failed to write dead-letter file '", + ex.ErrorRecord.Exception.Message, StringComparison.Ordinal); + Assert.Contains(dir, ex.ErrorRecord.Exception.Message); + + // Stopped on the open, so nothing was POSTed and there are no item errors to have + // been cut off - which is the whole difference: the caller can fix the path and run + // the same command again without deciding anything about duplicates. + Assert.EndsWith(" before the batch was sent; nothing was sent.", + ex.ErrorRecord.Exception.Message, StringComparison.Ordinal); + Assert.Empty(handler.Requests); + Assert.DoesNotContain(ps.Streams.Error, + e => e.FullyQualifiedErrorId.StartsWith("BatchItemError", StringComparison.Ordinal)); + } + finally + { + Directory.Delete(dir); + } + } + + /// + /// -DeadLetterPath on a provider that is not the file system. The path was resolved without + /// asking which provider answered, and every other provider resolves to something this run + /// cannot append to - "Env:\MGX_DEAD_LETTER" comes back as "MGX_DEAD_LETTER", a name + /// relative to nothing - so the run made a file of that name in whatever directory it was + /// standing in and reported the dead letters written to it. + /// + [Fact] + public void DeadLetterPathOnAnotherProvider_IsRefusedAndNothingIsSent() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TwoItemsRejected); + using var transport = MgxTransportScope.Inject(handler); + + // Where the resolved name lands when nothing refuses it: a bare name is relative to + // the current directory. Cleared first, because the file it names is exactly what a run + // without this check leaves behind. + var strayed = Path.Combine(Directory.GetCurrentDirectory(), "MGX_DEAD_LETTER"); + File.Delete(strayed); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] { "/users/u1", "/users/u2" }) + .AddParameter("DeadLetterPath", @"Env:\MGX_DEAD_LETTER"); + + var ex = Assert.ThrowsAny(() => ps.Invoke()); + + Assert.StartsWith("DeadLetterPathNotFileSystem", + ex.ErrorRecord.FullyQualifiedErrorId, StringComparison.Ordinal); + Assert.Equal(@"-DeadLetterPath 'Env:\MGX_DEAD_LETTER' is not a file system path.", + ex.ErrorRecord.Exception.Message); + Assert.Empty(handler.Requests); + Assert.False(File.Exists(strayed), $"a file was written to {strayed}"); + } + finally + { + File.Delete(strayed); + } + } + + /// + /// The same resolution on an empty path, which answers with the working directory itself - + /// so the run opened a directory to append to, or, where a provider gave it a name, made a + /// file nobody asked for. Refused at binding, which is where a parameter with nothing in it + /// belongs. + /// + [Fact] + public void DeadLetterPathThatIsEmpty_IsRefusedAndNothingIsSent() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TwoItemsRejected); + using var transport = MgxTransportScope.Inject(handler); + + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] { "/users/u1", "/users/u2" }) + .AddParameter("DeadLetterPath", string.Empty); + + var ex = Assert.ThrowsAny(() => ps.Invoke()); + + Assert.Contains("DeadLetterPath", ex.Message, StringComparison.Ordinal); + Assert.Empty(handler.Requests); + } + + /// + /// A batch in which nothing failed. The file is opened before the first POST, so it exists + /// by the time the run knows whether it has a line for it - and a run that has none used to + /// leave an empty file standing where a caller watching that path for failures would find + /// one after every clean run. It is removed, and nothing is said about it: there was nothing + /// in it to report. + /// + [Fact] + public void ACleanBatch_LeavesNoDeadLetterFileBehind() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, BatchSuccessResponse); + using var transport = MgxTransportScope.Inject(handler); + + var path = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] { "/users/u1", "/users/u2" }) + .AddParameter("DeadLetterPath", path); + var output = ps.Invoke(); + + Assert.Equal(2, output.Count); + Assert.False(File.Exists(path), "an empty dead-letter file was left behind"); + Assert.Empty(ps.Streams.Error); + Assert.Empty(ps.Streams.Warning); + } + finally + { + File.Delete(path); + } + } + + /// + /// The same clean run over a file that was already there. Those lines are an earlier run's + /// record of what is still outstanding, and this run has no business removing them because + /// its own items all succeeded. + /// + [Fact] + public void ACleanBatch_LeavesAnExistingDeadLetterFileAlone() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, BatchSuccessResponse); + using var transport = MgxTransportScope.Inject(handler); + + var path = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + File.WriteAllText(path, "{\"Url\":\"/users/earlier\"}" + Environment.NewLine); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] { "/users/u1", "/users/u2" }) + .AddParameter("DeadLetterPath", path); + ps.Invoke(); + + Assert.True(File.Exists(path), "an earlier run's dead-letter file was removed"); + Assert.Equal("{\"Url\":\"/users/earlier\"}", Assert.Single(File.ReadAllLines(path))); + } + finally + { + File.Delete(path); + } + } + + /// + /// Two batches over one -DeadLetterPath, which is what running them in parallel gives. A + /// opens the path first - creating it - and is held on the wire; B runs to completion with a + /// refused item and appends its line beside A's open handle, which the share mode admits; + /// then A is released with nothing of its own to write. + /// + /// A created the file and wrote no lines, so the removal is A's to consider - and B's dead + /// letter is in it. The removal asks the file rather than the run: the length A's own handle + /// reports is B's line, and the exclusive claim behind it is what a sibling still appending + /// refuses. The file stays, and nothing is said about it: A has nothing to report about a + /// record that is not its own. + /// + [Fact] + public void ABatchThatCreatedTheFile_LeavesASiblingsDeadLetterAlone() + { + var release = new ManualResetEventSlim(false); + var handler = new MockHttpHandler(); + handler.When(r => r.BodyText != null && r.BodyText.Contains("b-item"), "b") + .Respond(HttpStatusCode.OK, OneItemRejected); + handler.When(r => r.BodyText != null && r.BodyText.Contains("a-item"), "a") + .Respond(_ => + { + release.Wait(TimeSpan.FromSeconds(30)); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + OneItemAccepted, System.Text.Encoding.UTF8, "application/json") + }; + }); + using var transport = MgxTransportScope.Inject(handler); + + var path = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + var aWarnings = new List(); + var aErrors = new List(); + Exception? aFailure = null; + var a = new Thread(() => + { + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] { "/users/a-item" }) + .AddParameter("DeadLetterPath", path); + ps.Invoke(); + aWarnings.AddRange(ps.Streams.Warning.Select(w => w.Message)); + aErrors.AddRange(ps.Streams.Error.Select(e => e.FullyQualifiedErrorId)); + } + catch (Exception ex) { aFailure = ex; } + }); + + try + { + a.Start(); + + // A opens the file before its first POST, so its creation precedes the hold. + var deadline = DateTime.UtcNow.AddSeconds(20); + while (!File.Exists(path) && DateTime.UtcNow < deadline) Thread.Sleep(20); + Assert.True(File.Exists(path), "the held run never created the dead-letter file"); + + using (var ps = CreateShell()) + { + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] { "/users/b-item" }) + .AddParameter("DeadLetterPath", path); + ps.Invoke(); + } + Assert.Contains("b-item", Assert.Single(ReadSharedLines(path)), StringComparison.Ordinal); + + release.Set(); + Assert.True(a.Join(TimeSpan.FromSeconds(60)), "the held run did not finish"); + Assert.Null(aFailure); + + Assert.True(File.Exists(path), "a sibling's dead-letter file was removed"); + Assert.Contains("b-item", Assert.Single(ReadSharedLines(path)), StringComparison.Ordinal); + Assert.Empty(aErrors); + Assert.Empty(aWarnings); + } + finally + { + release.Set(); + a.Join(TimeSpan.FromSeconds(60)); + release.Dispose(); + if (File.Exists(path)) File.Delete(path); + } + } + + /// + /// The other half of the fold, at the default preference: an unreadable body, not a write + /// failure. Both the outcome warning's count and the per-item warning naming the item reach + /// the stream, since nothing here stops the run early. + /// + /// The text is asserted whole, because the fold is punctuation and a Contains on either + /// half of it passes either way. The base sentence used to close with its own period before + /// the clause was appended, and nothing closed the sentence after it: "...batch items + /// failed.; 1 body was withheld from the dead-letter file They failed after all retry + /// attempts." + /// + [Fact] + public void OutcomeWarning_CountsTheBodyItWithheld() + { + var (_, warnings) = DeadLetterRun(TwoItemsRejected, + new System.Collections.Hashtable + { + ["Url"] = "/users", + ["Method"] = "POST", + ["Body"] = """{"displayName":"first","displayName":"second"}""" + }, + new System.Collections.Hashtable + { + ["Url"] = "/groups", + ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "the-second-item" } + }); + + var outcome = Assert.Single(warnings, w => w.Contains("2 of 2 batch items failed")); + Assert.Equal( + "2 of 2 batch items failed; 1 body was withheld from the dead-letter file. " + + "They failed after all retry attempts. Check $Error for details on each item.", + outcome); + } + + /// + /// The duplicate key one level below where the old walk stopped. The array-of-array descent + /// reaches it, which is what turns a body that used to be written unredacted into one that + /// throws, so the two cases have to be pinned together. + /// + [Fact] + public void DeadLetterLine_ForADuplicateKeyInsideAnArrayOfArrays_CarriesTheMarkerAndWarns() + { + var (lines, warnings) = DeadLetterRun(OneItemRejected, + new System.Collections.Hashtable + { + ["Url"] = "/deviceManagement/configurationPolicies", + ["Method"] = "POST", + ["Body"] = """{"settings":[[{"displayName":"first","displayName":"second"}]]}""" + }); + + var line = Assert.Single(lines); + Assert.Equal("***REDACTED***", JsonNode.Parse(line)!["Body"]!.GetValue()); + Assert.DoesNotContain("settings", line); + Assert.Single(warnings, w => w.Contains("POST /deviceManagement/configurationPolicies") + && w.Contains("withheld")); + } + + /// + /// A body that is one bare string: a pre-authenticated URL and nothing else, which is a + /// legal JSON document and what a PUT to an upload session can carry. The walk steps into an + /// object's properties and an array's elements, and this body has neither, so it returned + /// having read nothing and the live signature was written to disk - while the identical URL + /// under any property name was cut, and a -Debug trace of the same body cut it too. + /// + [Fact] + public void DeadLetterLine_CutsABodyThatIsOneBareCapabilityUrl() + { + var (lines, warnings) = DeadLetterRun(OneItemRejected, + new System.Collections.Hashtable + { + ["Url"] = "/users/u1", + ["Method"] = "PATCH", + ["Body"] = "\"https://c-my.sharepoint.com/x?tempauth=sentinel-root-jwt\"" + }); + + var line = Assert.Single(lines); + Assert.DoesNotContain("sentinel-root-jwt", line); + // Cut, not withheld whole: the host and the parameter name are the half the file is read + // for, and nothing about this body was unreadable. + Assert.Equal("https://c-my.sharepoint.com/x?tempauth=***REDACTED***", + JsonNode.Parse(line)!["Body"]!.GetValue()); + Assert.DoesNotContain(warnings, w => w.Contains("withheld")); + } + + /// + /// The order the -DeadLetterPath doc states, on the line a caller parses field by field. + /// JsonObject writes properties in insertion order, so the order is the writer's to keep. + /// + [Fact] + public void DeadLetterLine_HoldsItsFieldsInTheDocumentedOrder() + { + var line = DeadLetterLineFor("/groups/g1", "PATCH", new System.Collections.Hashtable + { + ["displayName"] = "Corp Group" + }); + + Assert.Equal( + new[] { "Timestamp", "Url", "Method", "Status", "Body", "Error" }, + JsonNode.Parse(line)!.AsObject().Select(p => p.Key).ToArray()); + } + + [Fact] + public void RedactedDeadLetterLine_KeepsTheBodyWhenNeitherTheUrlNorTheNamesSayCredential() + { + // Over-redaction is the accepted cost, not the default: a body with nothing to hide + // reaches the file whole, which is what makes the file worth reading. + var line = DeadLetterLineFor("/groups/g1", "PATCH", new System.Collections.Hashtable + { + ["displayName"] = "Corp Group", + ["mailNickname"] = "corp-group" + }); + + Assert.DoesNotContain("***REDACTED***", line); + Assert.Contains("Corp Group", line); + Assert.Contains("corp-group", line); + } + // ═══════════════════════════════════════════════════════════════ // Additional batch edge case tests // ═══════════════════════════════════════════════════════════════ diff --git a/tests/Mgx.IntegrationTests/Engine/Batch/BulkWriteTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/BulkWriteTests.cs index 290143c..e970998 100644 --- a/tests/Mgx.IntegrationTests/Engine/Batch/BulkWriteTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Batch/BulkWriteTests.cs @@ -522,31 +522,13 @@ protected override Task SendAsync( /// /// Sends the headers of the given status and one byte of body, then nothing, honoring only /// the read's own token - the shape of a response whose body stalls. The status defaults to -/// a 500; a write is answered 201 and stalls exactly the same way. +/// a 500; a write is answered 201 and stalls exactly the same way. The body is the shared +/// , whose content read stream stalls too, so a reader that takes +/// the body as a stream reaches its read timeout rather than waiting on a buffer forever. /// public class StallingErrorHandler(HttpStatusCode status = HttpStatusCode.InternalServerError) : HttpMessageHandler { - private sealed class StallingContent : HttpContent - { - protected override async Task SerializeToStreamAsync(Stream stream, TransportContext? context) - => await SerializeToStreamAsync(stream, context, CancellationToken.None); - - protected override async Task SerializeToStreamAsync( - Stream stream, TransportContext? context, CancellationToken cancellationToken) - { - stream.WriteByte((byte)'{'); - await stream.FlushAsync(cancellationToken); - await Task.Delay(Timeout.Infinite, cancellationToken); - } - - protected override bool TryComputeLength(out long length) - { - length = 4096; - return true; - } - } - protected override Task SendAsync( HttpRequestMessage request, CancellationToken cancellationToken) { diff --git a/tests/Mgx.IntegrationTests/Engine/Batch/FaultInjectionBatchTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/FaultInjectionBatchTests.cs new file mode 100644 index 0000000..998ec58 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Batch/FaultInjectionBatchTests.cs @@ -0,0 +1,306 @@ +using System.Net; +using System.Net.Sockets; +using System.Text; +using System.Text.Json; +using Mgx.Engine.Errors; +using Mgx.Engine.Http; +using Mgx.Engine.Models; +using Polly.Timeout; + +namespace Mgx.IntegrationTests; + +/// +/// Every fault in the catalog, injected at subrequest 23 of a 25-operation batch - the third +/// subrequest of the second chunk, after the first chunk's twenty were answered. A status lands +/// inside the 200 envelope, which is the only place Graph can put one; a transport failure lands +/// on that chunk's own POST, because a sub-response has no way to carry one. What each case +/// expects comes from : the item loop asks it about the sub-response's +/// status with the item's own method, and the pipeline asks it about the POST, which is a POST. +/// +[Collection("Pipeline")] +public class FaultInjectionBatchTests +{ + private const int Operations = 25; + private const int ChunkSize = 20; + private const string TargetUrl = "/users/u23"; + + /// What every armed fault in this file is called, so a theory can assert by name + /// that does not carry it. + private const string FaultName = "fault-chunk-2"; + + private static readonly ResilientGraphClientOptions Options = new() + { + NoRateLimit = true, + NoAdaptivePacing = true, + MaxRetryAttempts = 1, + AttemptTimeoutSeconds = 1, + TotalTimeoutSeconds = 30, + }; + + private static readonly TimeSpan ShortBodyRead = TimeSpan.FromMilliseconds(250); + + /// The envelope stops in the middle of a sub-response. + private const string TruncatedEnvelope = """{"responses":[{"id":"1","status":200"""; + + private static List AllOperations() => + [.. Enumerable.Range(1, Operations).Select(i => new BatchOperation($"/users/u{i}"))]; + + /// + /// Answers a $batch envelope from the envelope that was sent: 200 for every subrequest, + /// except the one whose url is . Keyed on the url rather than the + /// sub-response id because a retry re-numbers the ids - the item that failed comes back as + /// id 1 - and the url is what identifies it across attempts. + /// + private static HttpResponseMessage Envelope(MockRequest request, string? faultUrl, int faultStatus) + { + using var sent = JsonDocument.Parse(request.BodyText!); + var items = new List(); + foreach (var subrequest in sent.RootElement.GetProperty("requests").EnumerateArray()) + { + var id = subrequest.GetProperty("id").GetString(); + var url = subrequest.GetProperty("url").GetString(); + var faulted = faultUrl != null && string.Equals(url, faultUrl, StringComparison.Ordinal); + var status = faulted ? faultStatus : 200; + // Retry-After on the throttled sub-response, so the item loop's backoff is the + // server's and not an exponential one this suite would have to wait out. + var headers = faulted && status == 429 ? ",\"headers\":{\"Retry-After\":\"0\"}" : ""; + var body = faulted + ? ",\"body\":{\"error\":{\"code\":\"Injected_" + status + "\"}}" + : ",\"body\":{\"id\":\"" + url!.Split('/')[^1] + "\"}"; + items.Add("{\"id\":\"" + id + "\",\"status\":" + status + headers + body + "}"); + } + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + "{\"responses\":[" + string.Join(",", items) + "]}", Encoding.UTF8, "application/json") + }; + } + + /// The chunk carrying the target subrequest, on any attempt - a retry re-sends it alone. + private static bool CarriesTheTarget(MockRequest request) => + request.BodyText?.Contains("\"" + TargetUrl + "\"", StringComparison.Ordinal) == true; + + private static MockHttpHandler TwoChunks(Action> armSecondChunk) + { + var handler = new MockHttpHandler(); + armSecondChunk(handler.When(CarriesTheTarget, FaultName).OnAttempt(1)); + handler.When(CarriesTheTarget).Respond(r => Envelope(r, faultUrl: null, faultStatus: 0)); + // Everything else: the first chunk, answered in full. + handler.When(_ => true).Respond(r => Envelope(r, faultUrl: null, faultStatus: 0)); + return handler; + } + + private static void ArmAtTheTargetSubrequest(MockResponseRule rule, FaultEntry entry) + { + switch (entry.Shape) + { + case FaultShape.Status: + // Graph answers $batch with 200 and puts the subrequest's failure in its own + // sub-response, so this is the only shape a status can take here. + rule.Respond(r => Envelope(r, TargetUrl, entry.Status)); + break; + case FaultShape.Transport: + rule.Throw(entry.Transport!()); + break; + case FaultShape.HeldPastAttemptTimeout: + rule.AfterDelay(TimeSpan.FromSeconds(5)) + .Respond(r => Envelope(r, faultUrl: null, faultStatus: 0)); + break; + case FaultShape.StalledBody: + rule.RespondStalled(HttpStatusCode.OK, prefix: """{"responses":["""); + break; + case FaultShape.TruncatedJson: + rule.Respond(HttpStatusCode.OK, TruncatedEnvelope); + break; + default: + throw new InvalidOperationException( + $"{entry.Kind} does not take a wire shape in a batch; it needs its own case."); + } + } + + private static async Task RunBatchAsync(MockHttpHandler handler) + { + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, Options) + { + BodyReadTimeout = ShortBodyRead + }; + var batchClient = new GraphBatchClient(client); + return await batchClient.ExecuteBatchIndexedAsync(AllOperations()); + } + + private static int StatusOf(BatchExecutionResult result, string url) => + result.Results.Single(r => r.Operation.Url == url).Response.Status; + + [Theory] + [MemberData(nameof(FaultCatalog.AllKinds), MemberType = typeof(FaultCatalog))] + public async Task Each_fault_at_the_target_subrequest(FaultKind kind) + { + var entry = FaultCatalog.Entry(kind); + var cell = entry.At(InjectionPoint.BatchSubrequest); + if (cell.Coverage == FaultCoverage.NotApplicable) + { + FaultInjectionRows.AssertNotApplicable(entry, InjectionPoint.BatchSubrequest); + return; + } + + ResiliencePipelineFactory.Reset(); + try + { + var handler = TwoChunks(rule => ArmAtTheTargetSubrequest(rule, entry)); + var result = await RunBatchAsync(handler); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + if (entry.Shape == FaultShape.Status) + AssertAFaultedSubResponse(entry, handler, result, cell.Coverage); + else + AssertARefusedChunk(entry, handler, result); + } + finally + { + ResiliencePipelineFactory.Reset(); + } + } + + /// + /// A status inside the envelope. The item loop asks the same policy the request pipeline + /// does, with the item's own method - GET here, so idempotent. + /// + private static void AssertAFaultedSubResponse( + FaultEntry entry, MockHttpHandler handler, BatchExecutionResult result, FaultCoverage coverage) + { + Assert.Null(result.ChunkFailure); + Assert.Equal(Operations, result.Results.Count); + Assert.Empty(result.NotSent); + + // Every subrequest but the target was answered 200, in both chunks. + foreach (var (operation, response) in result.Results.Where(r => r.Operation.Url != TargetUrl)) + Assert.InRange(response.Status, 200, 299); + + if (FaultCatalog.RetriedAsABatchItem(entry, "GET")) + { + // The retry is on the wire: chunk 1, chunk 2, and chunk 2 again carrying the one item. + Assert.Equal(3, handler.RequestCount); + Assert.Equal(1, result.Telemetry.ItemRetries); + Assert.InRange(StatusOf(result, TargetUrl), 200, 299); + Assert.Equal(Operations, result.Telemetry.Succeeded); + Assert.Equal(0, result.Telemetry.Failed); + Assert.Contains(TargetUrl, handler.CapturedRequests[2].BodyText); + Assert.DoesNotContain("/users/u21", handler.CapturedRequests[2].BodyText); + } + else + { + // One POST per chunk and no more: the item keeps the status the server gave it, and + // the batch-level retry pass does not pick up a status the policy will not retry. + Assert.Equal(2, handler.RequestCount); + Assert.Equal(0, result.Telemetry.ItemRetries); + Assert.Equal(0, result.Telemetry.BatchLevelRetries); + Assert.Equal(entry.Status, StatusOf(result, TargetUrl)); + Assert.Equal(Operations - 1, result.Telemetry.Succeeded); + Assert.Equal(1, result.Telemetry.Failed); + + if (coverage == FaultCoverage.Pinned) + AssertThePin(entry, handler); + } + } + + /// + /// What the two pinned status rows add over the plain non-retryable case: an expired token is + /// not refreshed and re-sent, and a read-after-write 404 is not waited out. + /// + private static void AssertThePin(FaultEntry entry, MockHttpHandler handler) + { + switch (entry.Kind) + { + case FaultKind.ExpiredToken: + Assert.Equal(MgxErrorClass.Authentication, entry.Class); + // Two POSTs, both answered 200 at the envelope level: nothing re-authenticated + // and sent the chunk again. + Assert.Equal([HttpStatusCode.OK, HttpStatusCode.OK], handler.ServedStatusCodes); + break; + case FaultKind.ConsistencyDelay: + // MgxErrorClass.Consistency is declared for the read-after-write window and has + // no producer; a 404 is a 404, and the item loop does not retry one. + Assert.Equal(MgxErrorClass.NotFound, entry.Class); + Assert.False(FaultCatalog.RetriedAsABatchItem(entry, "GET")); + break; + default: + Assert.Fail($"{entry.Kind} is pinned inside a batch envelope with nothing to record."); + break; + } + } + + /// + /// A transport failure on the chunk's own POST. The POST is not idempotent, so the policy + /// does not retry it whatever the class - and the chunk before it keeps everything the server + /// answered, which is the record of what it applied. + /// + private static void AssertARefusedChunk( + FaultEntry entry, MockHttpHandler handler, BatchExecutionResult result) + { + Assert.False(FaultCatalog.RetriedOnANonIdempotentRequest(entry)); + Assert.Equal(2, handler.RequestCount); + Assert.NotNull(result.ChunkFailure); + AssertTheFaultSurfaced(entry, result.ChunkFailure!); + Assert.Equal(Operations, result.Results.Count); + + var answered = result.Results.Take(ChunkSize).ToList(); + var refused = result.Results.Skip(ChunkSize).ToList(); + + // The first chunk's twenty are not unmade by the second chunk's failure. + Assert.All(answered, r => Assert.InRange(r.Response.Status, 200, 299)); + Assert.Equal(ChunkSize, result.Telemetry.Succeeded); + + // The refused chunk's POST went out, so none of its items may read as never sent: the + // difference between a request that may have reached the service and one that did not. + Assert.Equal(Operations - ChunkSize, refused.Count); + Assert.All(refused, r => + { + Assert.NotEqual(GraphBatchClient.NotSentStatus, r.Response.Status); + Assert.True(r.Response.Status >= 400, + $"{r.Operation.Url} came back {r.Response.Status}, which reads as a success"); + }); + Assert.Empty(result.NotSent); + } + + /// + /// The armed fault itself, as the caller gets it. A chunk failure is handed back on the + /// result rather than thrown, so this asks of + /// what the pagination theory asks of the exception the iterator raises: the kind that was + /// armed has to be the kind that surfaced, or the row proves only that something went wrong. + /// + private static void AssertTheFaultSurfaced(FaultEntry entry, Exception failure) + { + switch (entry.Shape) + { + case FaultShape.Transport: + // Straight through: the send throws, the POST is not retried, and the exception + // the mock raised is the one the result carries, reset SocketException and all. + var reset = Assert.IsType(failure); + Assert.Equal(entry.Transport!().Message, reset.Message); + Assert.IsType(reset.InnerException); + break; + case FaultShape.HeldPastAttemptTimeout: + // The pipeline's own per-attempt timeout, not the server's answer: the held + // response never arrives, so what surfaces names the timeout that gave up on it. + var timedOut = Assert.IsType(failure); + Assert.Equal(TimeSpan.FromSeconds(Options.AttemptTimeoutSeconds), timedOut.Timeout); + break; + case FaultShape.StalledBody: + // Headers arrived, so the timeout is the body read's and says so. + var stalled = Assert.IsType(failure); + Assert.Equal(ResilientGraphClient.BodyReadTimedOutMessage, stalled.Message); + break; + case FaultShape.TruncatedJson: + // The envelope is a 200 that stops mid-document; the parse is what fails, and + // it fails inside the responses array rather than at the top of the envelope. + var parse = Assert.IsAssignableFrom(failure); + Assert.Contains("responses", parse.Message, StringComparison.Ordinal); + break; + default: + Assert.Fail($"{entry.Kind} surfaced as {failure.GetType().Name} with no case to check it."); + break; + } + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs index 7ba6f16..e70f1e6 100644 --- a/tests/Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs @@ -1161,6 +1161,111 @@ public void The_gate_names_the_writes_a_batch_holds_not_the_batch_as_writes( Assert.Equal(0, requests); } + /// + /// Runs the batch under -WhatIf over per-item bodies and returns the gate's line, the error + /// records it wrote and what reached the wire. -WhatIf's whole purpose is saying what would + /// happen, so a gate that describes requests the run would refuse is the defect, and the + /// records naming them are half of the answer. + /// + private static (string Line, string[] Errors, int Requests) GateSaysOfBodies( + params object[] items) + { + var wire = new CountingHandler(); + using (MgxTransportScope.Inject(wire)) + { + var host = new GateHost(); + using var runspace = System.Management.Automation.Runspaces.RunspaceFactory.CreateRunspace(host); + runspace.Open(); + using var ps = System.Management.Automation.PowerShell.Create(); + ps.Runspace = runspace; + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", items) + .AddParameter("WhatIf", true); + ps.Invoke(); + + return ( + Assert.Single(host.Recorder.Lines, l => l.Contains("What if:", StringComparison.Ordinal)), + ps.Streams.Error.Select(e => e.Exception.Message).ToArray(), + wire.Requests); + } + } + + /// + /// The gate over a batch two of whose items will not be sent: one body the marker guard + /// refuses, one the JSON parser refuses. The count used to be everything that was piped + /// in, so the line announced three requests where the run would make one - at the one + /// surface whose purpose is saying what is about to happen - and said nothing at all about + /// the two it would refuse. The validation and the guard run before the gate now, so the + /// count is what would be sent and the refusals are named beside it. + /// + [Fact] + public void The_gate_names_the_count_that_will_be_sent_and_the_refusals() + { + var (line, errors, requests) = GateSaysOfBodies( + new System.Collections.Hashtable + { + ["Url"] = "/users/u1", ["Method"] = "PATCH", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "sent" } + }, + new System.Collections.Hashtable + { + ["Url"] = "/users/u2", ["Method"] = "PATCH", + ["Body"] = new System.Collections.Hashtable { ["passwordProfile"] = "***REDACTED***" } + }, + new System.Collections.Hashtable + { + ["Url"] = "/users/u3", ["Method"] = "PATCH", ["Body"] = "not json at all" + }); + + // One request, singular, and the two refusals as their own count. + Assert.Contains("Send batch\" on target \"PATCH 1 request via $batch; 2 refused\"", + line, StringComparison.Ordinal); + + // The refusals are error records, written under -WhatIf as without it: a body this run + // will not send is not an action the gate governs. + Assert.Equal(2, errors.Length); + Assert.Contains(errors, e => e.Contains("PATCH /users/u2", StringComparison.Ordinal) + && e.Contains("redaction marker", StringComparison.Ordinal)); + Assert.Contains(errors, e => e.Contains("PATCH /users/u3", StringComparison.Ordinal) + && e.Contains("not valid JSON", StringComparison.Ordinal)); + + Assert.Equal(0, requests); + } + + /// + /// Nothing refused, so nothing is named: the line a batch with no refusals gets is the one + /// it always got, and the clause is not appended empty. + /// + [Fact] + public void The_gate_names_no_refusals_when_there_are_none() + { + var (line, errors, requests) = GateSaysOfBodies( + new System.Collections.Hashtable + { + ["Url"] = "/users/u1", ["Method"] = "PATCH", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "a" } + }, + new System.Collections.Hashtable + { + ["Url"] = "/users/u2", ["Method"] = "PATCH", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "b" } + }); + + Assert.Contains("Send batch\" on target \"PATCH 2 requests via $batch\"", + line, StringComparison.Ordinal); + Assert.DoesNotContain("refused", line, StringComparison.Ordinal); + Assert.Empty(errors); + Assert.Equal(0, requests); + } + /// /// Several write methods alongside reads keep the shape they already had: the total, then /// the writes by method. It states a total and a subset of it, and nothing in it is a diff --git a/tests/Mgx.IntegrationTests/Engine/Errors/FaultInjectionCatalog.cs b/tests/Mgx.IntegrationTests/Engine/Errors/FaultInjectionCatalog.cs new file mode 100644 index 0000000..5a3eb03 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Errors/FaultInjectionCatalog.cs @@ -0,0 +1,482 @@ +using System.Net.Sockets; +using System.Text.Json; +using Mgx.Engine.Errors; +using Mgx.Engine.Http; +using Polly.Timeout; + +namespace Mgx.IntegrationTests; + +/// +/// The fault classes an operation has to survive, named once. Each one says how the mock +/// produces it and what makes of it; what the operation is +/// then expected to do comes from at the point of injection, +/// never from a status list restated in a test. +/// +public enum FaultKind +{ + Unauthorized, + Forbidden, + NotFound, + Conflict, + Throttled, + ServerError, + ConnectionReset, + AttemptTimeout, + BodyTimeout, + MalformedJson, + RepeatedNextLink, + UnusableNextLink, + ExpiredToken, + ChangedAuthContext, + ConsistencyDelay, +} + +/// How the mock produces a fault, which is also what decides what the classifier sees. +public enum FaultShape +{ + /// An HTTP status, on the request or inside a $batch envelope. + Status, + + /// An exception thrown instead of an answer. + Transport, + + /// An answer held past the per-attempt timeout. + HeldPastAttemptTimeout, + + /// Headers, then a body that delivers a prefix and stops. + StalledBody, + + /// A 200 whose body stops mid-document. + TruncatedJson, + + /// A page whose @odata.nextLink is its own URL. + SelfNextLink, + + /// A page whose @odata.nextLink is not a link mgx will follow. + UnusableNextLink, + + /// The session's AuthContext object replaced while the operation runs. + SwappedAuthContext, +} + +/// The four points a fault is injected at. +public enum InjectionPoint +{ + /// Page N of a pagination, after earlier pages were delivered. + Page, + + /// Item N of a fan-out, after earlier items succeeded. + FanOutItem, + + /// Subrequest N of a $batch, after earlier subrequests were answered. + BatchSubrequest, + + /// Mid-body of a content download, after earlier bytes arrived. + DownloadBody, +} + +public enum FaultCoverage +{ + /// The case runs and asserts the outcome the policy's decision implies. + Covered, + + /// The case runs and records today's behavior, which the policy does not decide. + Pinned, + + /// The fault cannot land at this point. says why. + NotApplicable, +} + +/// What one fault does at one injection point, and the shape it takes there. +public sealed record FaultCell(FaultCoverage Coverage, string Note); + +/// One row of the catalog: a fault, how it is produced, and where it can land. +public sealed record FaultEntry +{ + public required FaultKind Kind { get; init; } + + public required FaultShape Shape { get; init; } + + /// The status the fault carries, or 0 when it carries none. + public int Status { get; init; } + + /// The exception the fault is thrown as, for . + public Func? Transport { get; init; } + + /// + /// The class assigns, or null when the classifier never + /// sees this fault at all - a self-referencing nextLink and a swapped auth context are not + /// failures of a request. holds each non-null value + /// to what the classifier actually answers, so this column cannot drift away from it. + /// + public MgxErrorClass? Class { get; init; } + + /// How the mock produces it. + public required string Production { get; init; } + + public required IReadOnlyDictionary Cells { get; init; } + + public FaultCell At(InjectionPoint point) => Cells[point]; +} + +/// +/// The one fault catalog. The four injection-point theories read it; each of them iterates +/// every kind, so a cell that cannot be reached shows up as an explicit "not applicable" row +/// with its reason rather than as a case nobody wrote. +/// +public static class FaultCatalog +{ + private static FaultCell Covered(string note) => new(FaultCoverage.Covered, note); + + private static FaultCell Pinned(string note) => new(FaultCoverage.Pinned, note); + + private static FaultCell NotApplicable(string why) => new(FaultCoverage.NotApplicable, why); + + private const string StatusNotInABody = + "a status arrives on the response line, before the first byte of the body; once bytes " + + "are flowing there is no status left to change. Hop 2's own 401/403 - the expired " + + "pre-authenticated URL - is a request-start fault, held in GraphContentClientTests."; + + private const string NoLinksInABatch = + "the batch client never reads @odata.nextLink out of a sub-response; nothing in a $batch " + + "follows a link, so there is no link to poison."; + + private const string NoLinksInADownload = "a content download has no pages and follows no link."; + + private static IReadOnlyDictionary Cells( + FaultCell page, FaultCell fanOut, FaultCell batch, FaultCell download) => + new Dictionary + { + [InjectionPoint.Page] = page, + [InjectionPoint.FanOutItem] = fanOut, + [InjectionPoint.BatchSubrequest] = batch, + [InjectionPoint.DownloadBody] = download, + }; + + /// A plain status fault: the same shape at the first three points, impossible mid-body. + private static FaultEntry StatusEntry(FaultKind kind, int status, MgxErrorClass cls) => new() + { + Kind = kind, + Shape = FaultShape.Status, + Status = status, + Class = cls, + Production = $"the request is answered {status}", + Cells = Cells( + Covered($"page 3 is answered {status} after pages 1 and 2 were delivered"), + Covered($"the third url is answered {status} after the first two succeeded"), + Covered($"sub-response 23 carries status {status} inside the chunk's 200 envelope"), + NotApplicable(StatusNotInABody)), + }; + + public static IReadOnlyList Entries { get; } = + [ + StatusEntry(FaultKind.Unauthorized, 401, MgxErrorClass.Authentication), + StatusEntry(FaultKind.Forbidden, 403, MgxErrorClass.Authorization), + StatusEntry(FaultKind.NotFound, 404, MgxErrorClass.NotFound), + StatusEntry(FaultKind.Conflict, 409, MgxErrorClass.Conflict), + StatusEntry(FaultKind.Throttled, 429, MgxErrorClass.Throttle), + StatusEntry(FaultKind.ServerError, 503, MgxErrorClass.TransientServer), + + new FaultEntry + { + Kind = FaultKind.ConnectionReset, + Shape = FaultShape.Transport, + Transport = () => new HttpRequestException( + "The connection was reset.", new SocketException((int)SocketError.ConnectionReset)), + Class = MgxErrorClass.TransientTransport, + Production = "the send throws HttpRequestException over a reset SocketException", + Cells = Cells( + Covered("the page-3 GET is thrown on instead of answered"), + Covered("the third url's GET is thrown on instead of answered"), + Covered("the chunk carrying subrequest 23 has its own POST thrown on - a " + + "sub-response cannot carry a transport failure, only a status"), + Covered("the body throws IOException after delivering its first bytes; a reset " + + "mid-stream reaches the reader as an IOException, not as the " + + "HttpRequestException a failed send raises, and both classify the same")), + }, + + new FaultEntry + { + Kind = FaultKind.AttemptTimeout, + Shape = FaultShape.HeldPastAttemptTimeout, + Class = MgxErrorClass.TransientTransport, + Production = "the answer is held past AttemptTimeoutSeconds, so the pipeline's " + + "per-attempt timeout raises TimeoutRejectedException", + Cells = Cells( + Covered("page 3's answer is held past the attempt timeout"), + Covered("the third url's answer is held past the attempt timeout"), + Covered("the chunk carrying subrequest 23 has its POST held past the attempt timeout"), + NotApplicable( + "the per-attempt timeout ends when the response headers arrive; past that " + + "only the per-read idle timeout bounds the body, which is the body-timeout row")), + }, + + new FaultEntry + { + Kind = FaultKind.BodyTimeout, + Shape = FaultShape.StalledBody, + Class = MgxErrorClass.TransientTransport, + Production = "headers arrive, then the body delivers a prefix and stops without closing", + Cells = Cells( + Pinned("page 3's body stalls. The class is retryable for a GET, but the body is " + + "read after the resilience pipeline has already returned the response, so " + + "nothing retries it and the pagination ends at the stall"), + Pinned("the third url's body stalls, and for the same reason nothing retries it"), + Covered("the chunk's envelope stalls; the chunk POST is not idempotent, so the " + + "policy does not retry it either way"), + Pinned("the download body stalls mid-stream. ShouldRetryDownload answers true for " + + "the HttpRequestException the idle timeout raises, but the copy runs " + + "outside the download pipeline, so nothing retries it")), + }, + + new FaultEntry + { + Kind = FaultKind.MalformedJson, + Shape = FaultShape.TruncatedJson, + Class = MgxErrorClass.Permanent, + Production = "a 200 whose body stops in the middle of the value array", + Cells = Cells( + Covered("page 3 is answered 200 with a body that stops mid-array"), + Covered("the third url is answered 200 with a body that stops mid-array"), + Covered("the chunk's envelope stops mid-array"), + NotApplicable("download bytes are opaque; nothing on the content path parses them")), + }, + + new FaultEntry + { + Kind = FaultKind.RepeatedNextLink, + Shape = FaultShape.SelfNextLink, + Class = null, + Production = "a page whose @odata.nextLink is its own URL", + Cells = Cells( + Pinned("page 3 points back at page 3. Nothing stops a self-loop of non-empty " + + "pages today - only -Top / maxItems ends it; an empty self-loop ends at " + + "the consecutive-empty-page limit"), + Pinned("the third url's page points back at itself, ended the same two ways"), + NotApplicable(NoLinksInABatch), + NotApplicable(NoLinksInADownload)), + }, + + new FaultEntry + { + Kind = FaultKind.UnusableNextLink, + Shape = FaultShape.UnusableNextLink, + Class = MgxErrorClass.Permanent, + Production = "a page whose @odata.nextLink is not an absolute https URL, which " + + "NextLinkValidator refuses with an InvalidOperationException", + Cells = Cells( + Covered("page 3 carries a nextLink that is not a URL"), + Covered("the third url's page carries a nextLink that is not a URL"), + NotApplicable(NoLinksInABatch), + NotApplicable(NoLinksInADownload)), + }, + + new FaultEntry + { + Kind = FaultKind.ExpiredToken, + Shape = FaultShape.Status, + Status = 401, + Class = MgxErrorClass.Authentication, + Production = "a 401 that arrives only after earlier requests of the same operation " + + "were answered - the token expired while the operation was running", + Cells = Cells( + Pinned("page 3 is answered 401 after two pages were delivered: no retry, no " + + "re-authentication, and the pages already delivered stand"), + Pinned("the third url is answered 401 after the first two succeeded"), + Pinned("sub-response 23 carries 401 after the earlier subrequests were answered"), + NotApplicable(StatusNotInABody)), + }, + + new FaultEntry + { + Kind = FaultKind.ChangedAuthContext, + Shape = FaultShape.SwappedAuthContext, + Class = null, + Production = "the Graph session's AuthContext object is replaced with one carrying a " + + "different application while the operation is running", + Cells = Cells( + Pinned("the session's AuthContext is swapped while page 2 is being answered; the " + + "running invocation finishes on the client it resolved, and the next " + + "invocation is the one that notices"), + Pinned("the session's AuthContext is swapped while the second id is being " + + "answered, with the same outcome"), + NotApplicable( + "the auth context is read once per invocation, in GetClient, before the " + + "first request goes out; no chunk POST re-reads it, so a swap has no " + + "subrequest-level point to land at. What it does to a running invocation " + + "is pinned at the page and fan-out points."), + NotApplicable( + "hop 2 is token-free by construction - no Authorization header, no auth " + + "context - so nothing about the download body depends on one")), + }, + + new FaultEntry + { + Kind = FaultKind.ConsistencyDelay, + Shape = FaultShape.Status, + Status = 404, + Class = MgxErrorClass.NotFound, + Production = "a 404 on the first attempt that would be a 200 on any later one - the " + + "read-after-write window", + Cells = Cells( + Pinned("page 3 answers 404 first and 200 afterwards; the classifier assigns " + + "NotFound and the policy does not retry, so the 200 is never asked for"), + Pinned("the third url answers 404 first and 200 afterwards, with the same outcome"), + Pinned("sub-response 23 carries 404, which the batch item loop does not retry"), + NotApplicable(StatusNotInABody)), + }, + ]; + + public static FaultEntry Entry(FaultKind kind) => + Entries.SingleOrDefault(e => e.Kind == kind) + ?? throw new InvalidOperationException($"No catalog entry for {kind}."); + + /// Every kind, for a theory that has to show a row per kind at its own point. + public static TheoryData AllKinds + { + get + { + var data = new TheoryData(); + foreach (var kind in Enum.GetValues()) + data.Add(kind); + return data; + } + } + + // ── The policy's decisions, asked rather than restated ────────────────── + + /// What the retry policy says about this fault on an idempotent request (a GET page, + /// a fan-out read). + public static bool RetriedOnAnIdempotentRequest(FaultEntry entry) => + entry.Class is { } cls && MgxErrorPolicy.ShouldRetry(cls, isIdempotent: true); + + /// What it says on a non-idempotent request - the $batch POST itself. + public static bool RetriedOnANonIdempotentRequest(FaultEntry entry) => + entry.Class is { } cls && MgxErrorPolicy.ShouldRetry(cls, isIdempotent: false); + + /// What it says about a $batch sub-response carrying this fault's status. + public static bool RetriedAsABatchItem(FaultEntry entry, string method) => + entry.Class is { } cls && MgxErrorPolicy.ShouldRetry( + cls, isIdempotent: !string.Equals(method, "POST", StringComparison.OrdinalIgnoreCase)); + + /// What the download-host retry filter says about it. + public static bool RetriedByTheDownloadPipeline(FaultEntry entry, Exception? exception) => + entry.Class is { } cls + && MgxErrorPolicy.ShouldRetryDownload(new MgxErrorInfo(cls, entry.Status), exception); + + /// Whether this fault counts toward opening the circuit. + public static bool CountsAgainstTheBreaker(FaultEntry entry) => + entry.Class is { } cls && MgxErrorPolicy.CountsAsCircuitFailure(new MgxErrorInfo(cls, entry.Status)); + + /// + /// What the classifier answers for the fault this row produces. The + /// column is held to this, so the catalog cannot claim a class the classifier does not assign. + /// + public static MgxErrorClass? ClassifierAnswer(FaultEntry entry) => entry.Shape switch + { + FaultShape.Status => MgxErrorClassifier.Classify(entry.Status).Class, + FaultShape.Transport => MgxErrorClassifier.Classify(entry.Transport!(), cancellationRequested: false).Class, + // The pipeline's per-attempt timeout, as Polly raises it. + FaultShape.HeldPastAttemptTimeout => + MgxErrorClassifier.Classify(new TimeoutRejectedException(), cancellationRequested: false).Class, + // What ResilientGraphClient turns a timed-out body read into. + FaultShape.StalledBody => MgxErrorClassifier.Classify( + new HttpRequestException(ResilientGraphClient.BodyReadTimedOutMessage), + cancellationRequested: false).Class, + FaultShape.TruncatedJson => MgxErrorClassifier.Classify( + MalformedJsonException(), cancellationRequested: false).Class, + // What NextLinkValidator.ValidateOrThrow raises. + FaultShape.UnusableNextLink => MgxErrorClassifier.Classify( + new InvalidOperationException("Pagination stopped"), cancellationRequested: false).Class, + _ => null, + }; + + /// A real JsonException, from a real truncated document rather than a constructed one. + public static JsonException MalformedJsonException() + { + try + { + JsonDocument.Parse(TruncatedPageBody).Dispose(); + } + catch (JsonException ex) + { + return ex; + } + throw new InvalidOperationException("The truncated body in the catalog now parses."); + } + + /// A 200 body that stops in the middle of the value array. + public const string TruncatedPageBody = """{"value":[{"id":"u5"""; +} + +/// +/// The catalog's own guards: the table has a row per kind, every row's class is the class the +/// classifier assigns, and nothing is marked impossible everywhere. +/// +public class FaultInjectionCatalogTests +{ + [Theory] + [MemberData(nameof(FaultCatalog.AllKinds), MemberType = typeof(FaultCatalog))] + public void Every_kind_has_exactly_one_row(FaultKind kind) + { + var entry = FaultCatalog.Entry(kind); + Assert.Equal(kind, entry.Kind); + Assert.False(string.IsNullOrWhiteSpace(entry.Production)); + foreach (var point in Enum.GetValues()) + { + var cell = entry.At(point); + Assert.False(string.IsNullOrWhiteSpace(cell.Note), + $"{kind} at {point} has no note saying what shape it takes or why it cannot land there"); + } + } + + [Theory] + [MemberData(nameof(FaultCatalog.AllKinds), MemberType = typeof(FaultCatalog))] + public void The_class_a_row_claims_is_the_one_the_classifier_assigns(FaultKind kind) + { + // The catalog names the class so the table can be read on its own; this holds that + // column to MgxErrorClassifier rather than letting it become a second opinion. + // ErrorPolicyParityTests owns status-to-decision for every status; nothing here + // restates those rows. + var entry = FaultCatalog.Entry(kind); + Assert.Equal(entry.Class, FaultCatalog.ClassifierAnswer(entry)); + } + + [Theory] + [MemberData(nameof(FaultCatalog.AllKinds), MemberType = typeof(FaultCatalog))] + public void No_kind_is_impossible_at_every_point(FaultKind kind) + { + // "Where physically possible" has to be auditable from the table, which means a kind + // that is not applicable anywhere would be a kind nobody covers at all. + var entry = FaultCatalog.Entry(kind); + Assert.Contains(Enum.GetValues(), + point => entry.At(point).Coverage != FaultCoverage.NotApplicable); + } + + [Fact] + public void The_consistency_class_still_has_no_producer() + { + // MgxErrorClass.Consistency is declared for the read-after-write window and nothing + // assigns it yet, which is why the eventual-consistency rows below record a NotFound + // that is not retried instead of asserting a retry. The producer is 2.3.0 work. + for (var status = 100; status < 600; status++) + Assert.NotEqual(MgxErrorClass.Consistency, MgxErrorClassifier.Classify(status).Class); + + Exception[] shapes = + [ + new HttpRequestException(), + new SocketException((int)SocketError.ConnectionReset), + new IOException(), + new TaskCanceledException(), + new TimeoutRejectedException(), + new InvalidOperationException(), + ]; + foreach (var exception in shapes) + { + Assert.NotEqual(MgxErrorClass.Consistency, + MgxErrorClassifier.Classify(exception, cancellationRequested: false).Class); + Assert.NotEqual(MgxErrorClass.Consistency, + MgxErrorClassifier.Classify(exception, cancellationRequested: true).Class); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Http/BodyRedactionTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/BodyRedactionTests.cs index 9dcea33..e3d26a8 100644 --- a/tests/Mgx.IntegrationTests/Engine/Http/BodyRedactionTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Http/BodyRedactionTests.cs @@ -1,4 +1,8 @@ using System.Net; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Nodes; +using Mgx.Cmdlets.Cmdlets.Batch; using Mgx.Engine.Http; namespace Mgx.IntegrationTests.Engine.Http; @@ -51,6 +55,11 @@ public void Redacts_pre_authenticated_urls(string body, string secret) [InlineData("""{"u":"https://example.com/x?design=KEEPME"}""", "KEEPME")] [InlineData("""{"u":"https://example.com/x?config=KEEPME"}""", "KEEPME")] [InlineData("""{"u":"https://example.com/x?assign=KEEPME"}""", "KEEPME")] + // Relative paths, which the item-url pattern reads: a paging link and a $select naming a + // property that starts with "sig" are both the thing a trace is read for. + [InlineData("""{"url":"/users?$skiptoken=KEEPME"}""", "KEEPME")] + [InlineData("""{"url":"/users?$select=signInActivity,KEEPME"}""", "KEEPME")] + [InlineData("""{"url":"/me/drive/root:/KEEPME.xlsx:/content"}""", "KEEPME")] public void Does_not_redact_diagnostics_or_ordinary_urls(string body, string keep) { Assert.Contains(keep, Trace(body), StringComparison.Ordinal); @@ -63,12 +72,175 @@ public void Still_redacts_credential_named_properties() Assert.DoesNotContain("hunter2", trace, StringComparison.Ordinal); } + /// + /// The trace and the dead-letter file read one list of credential name fragments + /// (SensitiveNames), so a field the file hides is missing from -Debug output too. These + /// three are the fragments the file has that a name-by-name tracer would not: a federated + /// credential's assertion, a profile passphrase, a connection string. + /// + [Theory] + // A workload-identity JWT: the value IS the credential. + [InlineData("""{"clientAssertion":"eyJhbGci.LEAKED.c2ln"}""", "LEAKED")] + // An imported PFX and a Wi-Fi profile both spell it this way. + [InlineData("""{"passphrase":"LEAKED"}""", "LEAKED")] + // A connection string carries its credential inline, so the whole value goes. + [InlineData("""{"connectionString":"Server=db;User=svc;Password=LEAKED"}""", "LEAKED")] + public void Redacts_the_names_the_dead_letter_file_redacts(string body, string secret) + { + var trace = Trace(body); + Assert.DoesNotContain(secret, trace, StringComparison.Ordinal); + Assert.Contains("", trace); + } + /// /// Three items, not two. Sanitize makes two passes that can each match a downloadUrl - one /// by property name, one by value - so with only two items a first-match-only regression /// was masked: pass one redacted FIRST, pass two redacted SECOND, and the test stayed green /// while every third and subsequent capability URL leaked. /// + /// + /// The dead-letter writer's walk over the same body the tracer is given. It reads parsed + /// JSON rather than text, so the two cannot be one function, but the fragment, the + /// parameters and the cut are one definition in SensitiveNames - and a URL the trace hides + /// and the file writes is a live credential on disk, which is the worse half of the pair. + /// + /// Serialized the way the writer serializes it: the default encoder escapes an ampersand, + /// and the query string a caller reads off the line has to be the one that was sent. + /// + private static string FileLine(string body) + { + var node = JsonNode.Parse(body); + node = InvokeMgxBatchRequest.RedactSensitiveFields(node); + return node!.ToJsonString(new JsonSerializerOptions + { + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }); + } + + [Theory] + // Intune's content-file body. The SAS is the credential and neither name says so, so only + // the value rule reaches it; what survives the cut says which blob and which parameter. + [InlineData("""{"azureStorageUri":"https://s.blob.core.windows.net/c/b?sv=2021-08-06&sig=LEAKEDSAS&se=2026-12-31"}""", + "LEAKEDSAS", "https://s.blob.core.windows.net/c/b?sv=2021-08-06&sig=")] + [InlineData("""{"uploadUrl":"https://s.blob.core.windows.net/c/b?sv=2021&sig=LEAKEDSAS"}""", + "LEAKEDSAS", "https://s.blob.core.windows.net/c/b?sv=2021&sig=")] + [InlineData("""{"u":"https://c-my.sharepoint.com/x?tempauth=LEAKED&ApiVersion=2.0"}""", + "LEAKED", "https://c-my.sharepoint.com/x?tempauth=")] + // The capability sits in the path, so the property name is the only thing that reaches it. + [InlineData("""{"@microsoft.graph.downloadUrl":"https://public.bl.files.1drv.com/y4mLEAKED/file.bin"}""", + "LEAKED", null)] + [InlineData("""{"value":[{"@content.downloadUrl":"https://c-my.sharepoint.com/x?tempauth=LEAKED"}]}""", + "LEAKED", null)] + // An array element has no property name at all: the value rule is what is left. + [InlineData("""{"links":["https://c.sharepoint.com/d?guestaccesstoken=LEAKED"]}""", + "LEAKED", "https://c.sharepoint.com/d?guestaccesstoken=")] + // The capability sits in the property NAME rather than in any value: the tracer matches + // any quoted string in the text alike, key or value, but the file's walk used to read + // only values, so this key reached the file whole. + [InlineData("""{"https://h/f?sig=LEAKED":"v"}""", + "LEAKED", "https://h/f?sig=")] + // A capability URL used as a key whose text ALSO carries a SensitiveNames fragment - + // "guestaccesstoken" contains "token", "authkey" contains "key", "passwordreset" contains + // "password". The fragment check used to run first, take the branch that only clears the + // value beside the key, and leave the key - secret included - verbatim; the trace's own + // first pass has this same fragment-matches-the-key case, but its later, URL-only pass + // still finds and cuts the key as text, which is why the trace never leaked here even + // though the file did. + [InlineData("""{"https://h/f?guestaccesstoken=LEAKED":"v"}""", + "LEAKED", "https://h/f?guestaccesstoken=")] + [InlineData("""{"https://h/f?authkey=LEAKED":"v"}""", + "LEAKED", "https://h/f?authkey=")] + [InlineData("""{"https://h/passwordreset?sig=LEAKED":"v"}""", + "LEAKED", "https://h/passwordreset?sig=")] + // A capability URL as a key takes the value beside it, in the trace as in the file: the key + // IS the credential, so what a body files under one is what that URL was minted for. The + // trace's URL pass cuts such a key as text but stops at the key's closing quote, so a value + // the file cleared stood beside a key both of them had cut - and only where the key's own + // text carried no fragment, which is the signature deciding. + [InlineData("""{"https://h/f?sig=abc":"KEEPME"}""", + "KEEPME", "https://h/f?sig=")] + // The whole body is one string. There is no property and no element for the walk to step + // into, so it returned having read nothing and the URL reached the file entire; the tracer + // matches a quoted string wherever it stands and had cut this one all along. + [InlineData("\"https://s.blob.core.windows.net/c/b?sv=2021&sig=LEAKEDSAS\"", + "LEAKEDSAS", "https://s.blob.core.windows.net/c/b?sv=2021&sig=")] + [InlineData("\"https://c-my.sharepoint.com/x?tempauth=LEAKED\"", + "LEAKED", "https://c-my.sharepoint.com/x?tempauth=")] + // A capability URL with its scheme and host gone, which is the form a caller reaches for + // when they hand a batch item a URL Graph gave them relative - the item's own url, and any + // body value beside it. The tracer reads that form; the file's rule was anchored on the + // scheme, so the same value was cut in -Debug output and written to disk entire. + [InlineData("""{"u":"/drives/d/items/i/content?tempauth=LEAKEDJWT"}""", + "LEAKEDJWT", "/drives/d/items/i/content?tempauth=")] + [InlineData("""{"uploadUrl":"/c/b?sv=2021&sig=LEAKEDSAS&se=2026"}""", + "LEAKEDSAS", "/c/b?sv=2021&sig=")] + [InlineData("""{"links":["/d?guestaccesstoken=LEAKED"]}""", + "LEAKED", "/d?guestaccesstoken=")] + // The same form used as a property name, which the key rule reads through the same cut. + [InlineData("""{"/f?authkey=LEAKED":"v"}""", + "LEAKED", "/f?authkey=")] + public void The_dead_letter_file_hides_what_the_trace_hides(string body, string secret, string? kept) + { + var trace = Trace(body); + var line = FileLine(body); + + Assert.DoesNotContain(secret, trace, StringComparison.Ordinal); + Assert.DoesNotContain(secret, line, StringComparison.Ordinal); + + if (kept != null) + { + Assert.Contains(kept, trace, StringComparison.Ordinal); + Assert.Contains(kept, line, StringComparison.Ordinal); + } + } + + /// + /// The other direction, on the side that had no value rule at all. A paging link is a URL + /// the file is read for, and over-redacting it costs the same thing over-redacting a trace + /// costs. + /// + [Theory] + [InlineData("""{"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=RFNwdAoAAQAAA"}""", "RFNwdAoAAQAAA")] + [InlineData("""{"webUrl":"https://c.sharepoint.com/Shared%20Documents/KEEPME.xlsx"}""", "KEEPME")] + [InlineData("""{"u":"https://example.com/x?design=KEEPME"}""", "KEEPME")] + // The relative form the rule above now reads, on the other side of it: a paging link, a + // $select naming a property that starts with "sig", and a path with neither. + [InlineData("""{"url":"/users?$skiptoken=KEEPME"}""", "KEEPME")] + [InlineData("""{"url":"/users?$select=signInActivity,KEEPME"}""", "KEEPME")] + [InlineData("""{"url":"/me/drive/root:/KEEPME.xlsx:/content"}""", "KEEPME")] + public void The_dead_letter_file_keeps_the_urls_the_trace_keeps(string body, string keep) + { + Assert.Contains(keep, Trace(body), StringComparison.Ordinal); + Assert.Contains(keep, FileLine(body), StringComparison.Ordinal); + } + + /// + /// A $batch item's own url, as the envelope carries it. Invoke-MgxBatchRequest normalizes + /// each url to a relative path before the POST - that is the form Graph's batch format takes + /// - so a pre-authenticated URL a caller passed as an item URL has lost the scheme both URL + /// patterns anchor on by the time the request is traced, and the tempauth JWT stood in + /// -Debug output beside a dead-letter line that cut the same URL. + /// + /// Traced as a request, which is what a batch envelope is. + /// + [Theory] + [InlineData("""{"requests":[{"id":"1","method":"PUT","url":"/x?tempauth=LEAKEDJWT"}]}""", + "LEAKEDJWT", "/x?tempauth=")] + [InlineData("""{"requests":[{"id":"1","method":"PUT","url":"/c/b?sv=2021&sig=LEAKEDSAS&se=2026"}]}""", + "LEAKEDSAS", "/c/b?sv=2021&sig=")] + [InlineData("""{"requests":[{"id":"1","method":"GET","url":"/d?guestaccesstoken=LEAKED"}]}""", + "LEAKED", "/d?guestaccesstoken=")] + public void Redacts_a_capability_url_that_has_lost_its_scheme(string envelope, string secret, string kept) + { + var trace = GraphRequestTracer.FormatRequest( + new HttpRequestMessage(HttpMethod.Post, "https://graph.microsoft.com/v1.0/$batch"), + System.Text.Encoding.UTF8.GetBytes(envelope), 1); + + Assert.DoesNotContain(secret, trace, StringComparison.Ordinal); + // Which host is gone with the scheme, but which kind of URL still reads off the line. + Assert.Contains(kept, trace, StringComparison.Ordinal); + } + [Fact] public void Redacts_every_url_in_a_body_not_just_the_first() { @@ -77,4 +249,81 @@ public void Redacts_every_url_in_a_body_not_just_the_first() Assert.DoesNotContain("SECOND", trace, StringComparison.Ordinal); Assert.DoesNotContain("THIRD", trace, StringComparison.Ordinal); } + + /// + /// A capability-URL key whose only SensitiveNames fragment lies in the secret after the + /// capability parameter: "abc.token.def" carries "token", and "https://h/f?tempauth=" on + /// its own carries none. The cut takes the fragment away with the secret, so a fragment + /// check reading only the cut key found nothing and left the value beside the key whole - + /// a value under a pre-authenticated URL is as much a credential as one under + /// "guestaccesstoken", whose fragment happens to survive its own cut. The trace never had + /// the gap: its name pass reads the key as written, before its URL pass cuts anything. + /// + [Fact] + public void A_key_whose_fragment_is_only_in_the_secret_still_clears_the_value_beside_it() + { + const string body = """{"https://h/f?tempauth=abc.token.def":"SECRETVALUE"}"""; + + var trace = Trace(body); + Assert.DoesNotContain("abc.token.def", trace, StringComparison.Ordinal); + Assert.DoesNotContain("SECRETVALUE", trace, StringComparison.Ordinal); + + var line = FileLine(body); + Assert.DoesNotContain("abc.token.def", line, StringComparison.Ordinal); + Assert.DoesNotContain("SECRETVALUE", line, StringComparison.Ordinal); + + var cutKey = $"https://h/f?tempauth={InvokeMgxBatchRequest.RedactionMarker}"; + var obj = JsonNode.Parse(line)!.AsObject(); + Assert.True(obj.ContainsKey(cutKey), line); + Assert.Equal(InvokeMgxBatchRequest.RedactionMarker, obj[cutKey]!.GetValue()); + // Which host and which kind of URL still reads off the line, as with any other cut. + Assert.Contains("https://h/f?tempauth=", line, StringComparison.Ordinal); + } + + /// + /// A capability-URL key whose text carries no SensitiveNames fragment at all. The cut on the + /// key is what says the property is a credential - a pre-authenticated URL as a field name + /// makes whatever sits beside it the thing that URL was minted for - but the value beside it + /// was decided separately, on whether the key including its secret happened to read as a + /// credential name. So "?sig=abc" kept its value and "?sig=abcsecretdef" cleared it, and + /// which of the two a caller gets is the signature's spelling rather than the property's + /// meaning. + /// + [Fact] + public void A_capability_url_key_clears_the_value_beside_it_whatever_its_secret_spells() + { + var cutKey = $"https://h/f?sig={InvokeMgxBatchRequest.RedactionMarker}"; + + var line = FileLine("""{"https://h/f?sig=abc":"KEEPME"}"""); + var obj = JsonNode.Parse(line)!.AsObject(); + Assert.True(obj.ContainsKey(cutKey), line); + Assert.Equal(InvokeMgxBatchRequest.RedactionMarker, obj[cutKey]!.GetValue()); + Assert.DoesNotContain("KEEPME", line, StringComparison.Ordinal); + + // The same key with a fragment inside the signature, which was the only spelling that + // used to take the value with it. + var withFragment = JsonNode.Parse( + FileLine("""{"https://h/f?sig=abcsecretdef":"KEEPME"}"""))!.AsObject(); + Assert.Equal(InvokeMgxBatchRequest.RedactionMarker, withFragment[cutKey]!.GetValue()); + } + + /// + /// Two keys that cut to the identical redacted string - both are "?sig=", the same + /// capability parameter, differing only in the secret after it. JsonObject's indexer + /// replaces an existing key rather than refusing it, so re-adding the second under the + /// same cut name would silently drop the first property from the file. Both have to + /// survive, which means they cannot end up with the same name. + /// + [Fact] + public void Two_keys_that_redact_to_the_same_string_both_survive() + { + var line = FileLine("""{"https://h/f?sig=AAA":"1","https://h/f?sig=BBB":"2"}"""); + var obj = JsonNode.Parse(line)!.AsObject(); + + Assert.DoesNotContain("AAA", line, StringComparison.Ordinal); + Assert.DoesNotContain("BBB", line, StringComparison.Ordinal); + Assert.Equal(2, obj.Count); + Assert.True(obj.ContainsKey($"https://h/f?sig={InvokeMgxBatchRequest.RedactionMarker}")); + Assert.True(obj.ContainsKey($"https://h/f?sig={InvokeMgxBatchRequest.RedactionMarker}#2")); + } } diff --git a/tests/Mgx.IntegrationTests/Engine/Http/ConfigurableOptionsTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/ConfigurableOptionsTests.cs index f9189e3..40271f9 100644 --- a/tests/Mgx.IntegrationTests/Engine/Http/ConfigurableOptionsTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Http/ConfigurableOptionsTests.cs @@ -171,11 +171,43 @@ public void CircuitBreakerFailureRatio_AcceptsValidValues(double value) [Theory] [InlineData(0)] [InlineData(-1)] + [InlineData(1)] [InlineData(1001)] public void CircuitBreakerMinThroughput_RejectsInvalidValues(int value) { - Assert.Throws( + var ex = Assert.Throws( () => new ResilientGraphClientOptions { CircuitBreakerMinThroughput = value }); + // Refused in the option's own terms, with the range it will take. Polly refuses 1 as + // well, but it does so while a later call is building the pipeline. + Assert.Contains("between 2 and 1,000", ex.Message); + } + + [Theory] + [InlineData(2)] + [InlineData(40)] + [InlineData(1000)] + public void CircuitBreakerMinThroughput_EveryAcceptedValueBuildsAPipeline(int value) + { + // The floor, the default and the ceiling. A value the option takes but the pipeline + // cannot hold would fail on the next request instead of at the set site, which is the + // whole point of the option validating at all. + ResiliencePipelineFactory.Reset(); + try + { + var options = new ResilientGraphClientOptions + { + NoRateLimit = true, + CircuitBreakerMinThroughput = value + }; + + var (pipeline, _) = ResiliencePipelineFactory.GetOrCreate(options); + + Assert.NotNull(pipeline); + } + finally + { + ResiliencePipelineFactory.Reset(); + } } [Theory] diff --git a/tests/Mgx.IntegrationTests/Engine/Http/FaultInjectionDownloadTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/FaultInjectionDownloadTests.cs new file mode 100644 index 0000000..53179ef --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/FaultInjectionDownloadTests.cs @@ -0,0 +1,189 @@ +using System.Net; +using System.Text; +using Mgx.Engine.Errors; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// Every fault in the catalog, held against the middle of a content download: hop 1 redirects to +/// the download host, hop 2 answers 200, and the body faults after its first bytes have already +/// reached the caller. Two kinds can land there at all - a stall and a reset - and the rest are +/// explicit "not applicable" rows rather than tests nobody wrote: a status arrives on the +/// response line, a nextLink has no meaning here, and hop 2 is token-free by construction. +/// +[Collection("Pipeline")] +public class FaultInjectionDownloadTests : IDisposable +{ + private const string GraphContentUrl = + "https://graph.microsoft.com/v1.0/me/drive/items/01ABC/content"; + private const string CdnUrl = + "https://contoso-my.sharepoint.com/_layouts/15/download.aspx?UniqueId=abc&tempauth=SECRET"; + + /// The bytes the caller has already been handed when the fault lands. + private const string DeliveredPrefix = "the-first-bytes"; + + /// What the armed fault in this file is called, so the theory can assert by name + /// that does not carry it. + private const string FaultName = "fault-mid-body"; + + /// Short enough that a stalled body ends the case in a fraction of a second. + private static readonly TimeSpan IdleTimeout = TimeSpan.FromMilliseconds(200); + + private readonly MockHttpHandler _graphHandler = new(); + private readonly MockHttpHandler _cdnHandler = new(); + private readonly HttpClient _graphHttpClient; + private readonly ResilientGraphClient _client; + + public FaultInjectionDownloadTests() + { + ResiliencePipelineFactory.Reset(); + _graphHttpClient = new HttpClient(_graphHandler); + _client = new ResilientGraphClient(_graphHttpClient, + new ResilientGraphClientOptions { NoRateLimit = true, NoAdaptivePacing = true, MaxRetryAttempts = 1 }); + GraphContentClient.DownloadClientForTests = new HttpClient(_cdnHandler); + } + + public void Dispose() + { + GraphContentClient.DownloadClientForTests?.Dispose(); + GraphContentClient.DownloadClientForTests = null; + _client.Dispose(); + _graphHttpClient.Dispose(); + ResiliencePipelineFactory.Reset(); + GC.SuppressFinalize(this); + } + + /// + /// A body that delivers its prefix and then fails the read, the way a connection dropped + /// mid-transfer reaches a reader: an IOException out of the stream, not the + /// HttpRequestException a failed send raises. Overriding the content read stream is what + /// makes it a mid-body failure rather than a failure to produce a body at all. + /// + private sealed class ResetMidBodyContent : HttpContent + { + private readonly byte[] _prefix = Encoding.UTF8.GetBytes(DeliveredPrefix); + + protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context) + => throw new IOException("The response ended prematurely."); + + protected override Task CreateContentReadStreamAsync() + => Task.FromResult(new ResettingStream(_prefix)); + + protected override bool TryComputeLength(out long length) + { + length = 0; + return false; + } + + private sealed class ResettingStream(byte[] prefix) : Stream + { + private int _delivered; + + public override ValueTask ReadAsync( + Memory buffer, CancellationToken cancellationToken = default) + { + if (_delivered < prefix.Length) + { + var count = Math.Min(buffer.Length, prefix.Length - _delivered); + prefix.AsSpan(_delivered, count).CopyTo(buffer.Span); + _delivered += count; + return ValueTask.FromResult(count); + } + + throw new IOException("The response ended prematurely."); + } + + public override Task ReadAsync( + byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => ReadAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => _delivered; set => throw new NotSupportedException(); } + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + } + } + + private void ArmMidBody(FaultEntry entry) + { + _graphHandler.QueueResponse(HttpStatusCode.Found, null, new() { ["Location"] = CdnUrl }); + + switch (entry.Shape) + { + case FaultShape.StalledBody: + _cdnHandler.When(_ => true, FaultName).RespondStalled( + HttpStatusCode.OK, prefix: DeliveredPrefix, contentType: "application/octet-stream"); + break; + case FaultShape.Transport: + _cdnHandler.When(_ => true, FaultName).Respond(_ => new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ResetMidBodyContent() + }); + break; + default: + throw new InvalidOperationException( + $"{entry.Kind} does not take a wire shape mid-body; it needs its own case."); + } + } + + [Theory] + [MemberData(nameof(FaultCatalog.AllKinds), MemberType = typeof(FaultCatalog))] + public async Task Each_fault_mid_download_body(FaultKind kind) + { + var entry = FaultCatalog.Entry(kind); + var cell = entry.At(InjectionPoint.DownloadBody); + if (cell.Coverage == FaultCoverage.NotApplicable) + { + FaultInjectionRows.AssertNotApplicable(entry, InjectionPoint.DownloadBody); + return; + } + + ArmMidBody(entry); + + using var result = await _client.GetContentAsync(GraphContentUrl); + Assert.True(result.FromDownloadHost); + Assert.DoesNotContain(FaultName, _cdnHandler.UnansweredRules); + + using var destination = new MemoryStream(); + var thrown = await Record.ExceptionAsync(() => GraphContentClient.CopyWithIdleTimeoutAsync( + result.Content, destination, maxBytes: null, IdleTimeout, CancellationToken.None)); + + Assert.NotNull(thrown); + + // The bytes that arrived before the fault are the caller's; the copy does not unwrite them. + Assert.Equal(DeliveredPrefix, Encoding.UTF8.GetString(destination.ToArray())); + + // The classifier sees the failure the read raised, and it is the class the catalog claims. + Assert.Equal(entry.Class, + MgxErrorClassifier.Classify(thrown!, cancellationRequested: false).Class); + + // Whatever the download-host retry filter would say about it, nothing acts on it: the + // body copy runs after GetContentAsync returned, outside the pipeline that filter belongs + // to. A kind the filter would retry is therefore a pinned row, not a covered one. + var wouldRetry = FaultCatalog.RetriedByTheDownloadPipeline(entry, thrown); + Assert.Equal(wouldRetry ? FaultCoverage.Pinned : FaultCoverage.Covered, cell.Coverage); + Assert.Single(_cdnHandler.Requests); + Assert.Single(_graphHandler.Requests); + + switch (entry.Shape) + { + case FaultShape.StalledBody: + var stalled = Assert.IsType(thrown); + Assert.Contains("stalled", stalled.Message, StringComparison.OrdinalIgnoreCase); + break; + case FaultShape.Transport: + Assert.IsAssignableFrom(thrown); + break; + default: + Assert.Fail($"{entry.Kind} surfaced as {thrown!.GetType().Name} with no case to check it."); + break; + } + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Http/RetryTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/RetryTests.cs index ad1bdd2..e36827f 100644 --- a/tests/Mgx.IntegrationTests/Engine/Http/RetryTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Http/RetryTests.cs @@ -1,8 +1,11 @@ using System.Diagnostics; using System.Net; using System.Text; +using Mgx.Engine.Errors; using Mgx.Engine.Http; +using Mgx.Engine.Pagination; using Mgx.IntegrationTests.Fakes; +using Polly.Timeout; namespace Mgx.IntegrationTests; @@ -727,6 +730,396 @@ public async Task ResilientDelegatingHandler_AlwaysClonesRequest() ResiliencePipelineFactory.Reset(); } + /// + /// A keyed entry answers the request its predicate picks out and no other, so a fault sits + /// at a point in the operation instead of at a position in the send order. Everything the + /// predicate misses falls through to the default. + /// + [Fact] + public async Task Mock_KeyedResponse_AnswersOnlyTheMatchingRequest() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + handler.When(r => r.Uri.EndsWith("/users/user2", StringComparison.Ordinal)) + .Respond(HttpStatusCode.NotFound); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + + foreach (var id in new[] { "user1", "user2", "user3" }) + (await client.GetAsync($"https://graph.microsoft.com/v1.0/users/{id}")).Dispose(); + + Assert.Equal( + new[] { HttpStatusCode.OK, HttpStatusCode.NotFound, HttpStatusCode.OK }, + handler.ServedStatusCodes); + Assert.Equal(3, handler.RequestCount); + } + + /// + /// The attempt index counts the requests matching one entry's predicate, so a fault scoped + /// to the second leaves the first alone, and the retry the fault provokes falls through to + /// whatever answers next. + /// + [Fact] + public async Task Mock_KeyedResponse_ScopedToAttempt2_LeavesAttempt1Alone() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + handler.When(r => r.Uri.EndsWith("/users/user1", StringComparison.Ordinal)) + .OnAttempt(2) + .Respond((HttpStatusCode)429, null, new() { ["Retry-After"] = "0" }); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + + using var first = await client.GetAsync("https://graph.microsoft.com/v1.0/users/user1"); + Assert.Equal(HttpStatusCode.OK, first.StatusCode); + Assert.Equal(1, handler.RequestCount); + + using var second = await client.GetAsync("https://graph.microsoft.com/v1.0/users/user1"); + Assert.Equal(HttpStatusCode.OK, second.StatusCode); + + Assert.Equal( + new[] { HttpStatusCode.OK, (HttpStatusCode)429, HttpStatusCode.OK }, + handler.ServedStatusCodes); + } + + /// + /// A body that delivers a few bytes and then stops. The class is named rather than compared + /// against another exception of the same type, which no classification of a stalled body + /// could differ from: the contract is TransientTransport, the class a connection failure + /// carries, so the pipeline may retry an idempotent request and nothing reads the stall as + /// the server's final answer. + /// + [Fact] + public async Task Mock_StalledBody_SurfacesAsTheTransportClassTheClassifierAssigns() + { + var handler = new MockHttpHandler(); + handler.When(r => r.Uri.EndsWith("/users", StringComparison.Ordinal)) + .RespondStalled(HttpStatusCode.OK, prefix: """{"value":["""); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }) + { + BodyReadTimeout = TimeSpan.FromSeconds(2) + }; + + using var response = await client.GetAsync("https://graph.microsoft.com/v1.0/users"); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + var thrown = await Assert.ThrowsAsync( + () => client.ReadBodyAsStringAsync(response)); + Assert.Equal(ResilientGraphClient.BodyReadTimedOutMessage, thrown.Message); + + var stalled = MgxErrorClassifier.Classify(thrown, cancellationRequested: false); + Assert.Equal(MgxErrorClass.TransientTransport, stalled.Class); + } + + /// + /// An answer held past the attempt timeout never reaches the caller, so nothing is recorded + /// as served for it. Both attempts time out and the served list stays empty, which is what + /// makes it usable as evidence that a fault landed. + /// + [Fact] + public async Task Mock_HeldAnswer_TimedOutByTheAttempt_RecordsNothingAsServed() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + handler.When(r => r.Uri.EndsWith("/users/slow", StringComparison.Ordinal)) + .AfterDelay(TimeSpan.FromSeconds(5)) + .Respond(HttpStatusCode.OK, TestData.SingleUser); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions + { + NoRateLimit = true, + AttemptTimeoutSeconds = 1, + TotalTimeoutSeconds = 30, + MaxRetryAttempts = 1 + }); + + var thrown = await Record.ExceptionAsync( + () => client.GetAsync("https://graph.microsoft.com/v1.0/users/slow")); + + Assert.IsType(thrown); + Assert.Equal(2, handler.RequestCount); + Assert.Empty(handler.ServedStatusCodes); + Assert.Empty(handler.UnansweredRules); + } + + /// + /// Two answers, one held and one immediate, are recorded in the order the handler handed + /// them back rather than the order it chose them: the held one resolved first and arrives + /// last. + /// + [Fact] + public async Task Mock_HeldAnswer_IsRecordedWhenItIsHandedBack() + { + var handler = new MockHttpHandler(); + handler.When(r => r.Uri.EndsWith("/a", StringComparison.Ordinal)) + .AfterDelay(TimeSpan.FromSeconds(2)) + .Respond(HttpStatusCode.OK, """{"which":"a"}"""); + handler.When(r => r.Uri.EndsWith("/b", StringComparison.Ordinal)) + .Respond(HttpStatusCode.Accepted, """{"which":"b"}"""); + + using var httpClient = new HttpClient(handler); + var held = httpClient.GetAsync("https://graph.microsoft.com/v1.0/a"); + await Task.Delay(TimeSpan.FromMilliseconds(300)); + using var immediate = await httpClient.GetAsync("https://graph.microsoft.com/v1.0/b"); + using var released = await held; + + Assert.Equal(HttpStatusCode.OK, released.StatusCode); + Assert.Equal(HttpStatusCode.Accepted, immediate.StatusCode); + Assert.Equal( + new[] { HttpStatusCode.Accepted, HttpStatusCode.OK }, + handler.ServedStatusCodes); + } + + /// + /// A fault programmed after a generator that also matches its request never answers: the + /// first matching entry wins, the enumeration completes, every item arrives, and the run + /// looks exactly like one with no fault in it. The unanswered view is the only thing that + /// says so - the shadowed entry's own match counter still advanced. Programmed the other + /// way round the same fault lands. + /// + [Fact] + public async Task Mock_ShadowedFault_IsNamedInUnansweredRules() + { + const string FaultName = "page 3 fails once"; + var collection = new PagedCollection(total: 6, pageSize: 2); + + var shadowed = new MockHttpHandler(); + collection.ServeOn(shadowed); + ArmPageThreeFault(shadowed, FaultName); + + var injected = new MockHttpHandler(); + ArmPageThreeFault(injected, FaultName); + collection.ServeOn(injected); + + var expected = new[] { "user1", "user2", "user3", "user4", "user5", "user6" }; + Assert.Equal(expected, await EnumerateAsync(shadowed, collection)); + Assert.Equal(expected, await EnumerateAsync(injected, collection)); + + Assert.DoesNotContain(HttpStatusCode.ServiceUnavailable, shadowed.ServedStatusCodes); + Assert.Contains(HttpStatusCode.ServiceUnavailable, injected.ServedStatusCodes); + + Assert.Equal(new[] { FaultName }, shadowed.UnansweredRules); + Assert.Empty(injected.UnansweredRules); + + var shadowedFault = shadowed.ProgrammedRules.Single(rule => rule.Name == FaultName); + Assert.Equal(1, shadowedFault.Matches); + Assert.Equal(0, shadowedFault.Answers); + } + + private static void ArmPageThreeFault(MockHttpHandler handler, string name) => + handler.When(r => r.Uri.Contains("$skiptoken=page3", StringComparison.Ordinal), name) + .OnAttempt(1) + .Respond(HttpStatusCode.ServiceUnavailable, null, new() { ["Retry-After"] = "0" }); + + private static async Task> EnumerateAsync( + MockHttpHandler handler, PagedCollection collection) + { + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions + { + NoRateLimit = true, + AttemptTimeoutSeconds = 5, + TotalTimeoutSeconds = 60, + MaxRetryAttempts = 2 + }); + + var ids = new List(); + await foreach (var item in new PageIterator(client) + .StreamAllWithCountAsync(collection.InitialUrl, 0, null)) + { + ids.Add(item.GetProperty("id").GetString()!); + } + return ids; + } + + /// + /// The attempt index counts the requests that matched one entry, not the ones it answered. + /// The first request is answered by the first entry; the second entry counted that request + /// too, which is what puts its own fault on the second. + /// + [Fact] + public async Task Mock_AttemptIndex_CountsMatchesPerEntry_NotAnswers() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + handler.When(r => r.Uri.EndsWith("/users/user1", StringComparison.Ordinal), "first request") + .OnAttempt(1) + .Respond(HttpStatusCode.NotFound); + handler.When(r => r.Uri.EndsWith("/users/user1", StringComparison.Ordinal), "second request") + .OnAttempt(2) + .Respond(HttpStatusCode.Conflict); + + // The bare client: a 404 and a 409 are answers, and nothing here should retry them. + using var httpClient = new HttpClient(handler); + for (var i = 0; i < 2; i++) + (await httpClient.GetAsync("https://graph.microsoft.com/v1.0/users/user1")).Dispose(); + + Assert.Equal( + new[] { HttpStatusCode.NotFound, HttpStatusCode.Conflict }, + handler.ServedStatusCodes); + Assert.Empty(handler.UnansweredRules); + Assert.Equal(new[] { 2, 2 }, handler.ProgrammedRules.Select(rule => rule.Matches)); + Assert.Equal(new[] { 1, 1 }, handler.ProgrammedRules.Select(rule => rule.Answers)); + } + + /// + /// The attempt index over the retries of one request: the queue answers the first attempt, + /// the keyed entry the retry it provoked, and the retry after that falls through to the + /// default. One request, three attempts, three different answers. + /// + [Fact] + public async Task Mock_KeyedResponse_ScopedToAttempt2_AnswersTheRetryOfOneRequest() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + handler.QueueResponse(HttpStatusCode.ServiceUnavailable); + handler.When(r => r.Uri.EndsWith("/users/user1", StringComparison.Ordinal)) + .OnAttempt(2) + .Respond((HttpStatusCode)429, null, new() { ["Retry-After"] = "0" }); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions + { + NoRateLimit = true, + AttemptTimeoutSeconds = 5, + TotalTimeoutSeconds = 60, + MaxRetryAttempts = 4 + }); + + using var response = await client.GetAsync("https://graph.microsoft.com/v1.0/users/user1"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(3, handler.RequestCount); + Assert.All(handler.CapturedRequests, r => Assert.EndsWith("/users/user1", r.Uri, StringComparison.Ordinal)); + Assert.Equal( + new[] { HttpStatusCode.ServiceUnavailable, (HttpStatusCode)429, HttpStatusCode.OK }, + handler.ServedStatusCodes); + Assert.Empty(handler.UnansweredRules); + } + + /// A byte body comes back as programmed, content type and all. + [Fact] + public async Task Mock_RespondBytes_HandsBackTheBytesAndTheContentType() + { + var payload = new byte[] { 0x00, 0x01, 0xFE, 0xFF }; + var handler = new MockHttpHandler(); + handler.When(r => r.Uri.EndsWith("/content", StringComparison.Ordinal)) + .RespondBytes(HttpStatusCode.OK, payload, "application/octet-stream"); + + using var httpClient = new HttpClient(handler); + using var response = await httpClient.GetAsync("https://graph.microsoft.com/v1.0/content"); + + Assert.Equal(payload, await response.Content.ReadAsByteArrayAsync()); + Assert.Equal("application/octet-stream", response.Content.Headers.ContentType?.ToString()); + Assert.Equal(new[] { HttpStatusCode.OK }, handler.ServedStatusCodes); + } + + /// A status with no body at all, and the headers still arrive. + [Fact] + public async Task Mock_RespondEmpty_HandsBackAStatusAndNoBody() + { + var handler = new MockHttpHandler(); + handler.When(r => r.Method == HttpMethod.Delete) + .RespondEmpty(HttpStatusCode.NoContent, new() { ["request-id"] = "empty-1" }); + + using var httpClient = new HttpClient(handler); + using var response = await httpClient.DeleteAsync("https://graph.microsoft.com/v1.0/users/user1"); + + Assert.Equal(HttpStatusCode.NoContent, response.StatusCode); + Assert.Empty(await response.Content.ReadAsByteArrayAsync()); + Assert.Equal("empty-1", Assert.Single(response.Headers.GetValues("request-id"))); + Assert.Equal(new[] { HttpStatusCode.NoContent }, handler.ServedStatusCodes); + } + + /// + /// A thrown answer reaches the caller as itself and records no status: there was none. + /// + [Fact] + public async Task Mock_Throw_SurfacesTheExceptionAndRecordsNothingAsServed() + { + var handler = new MockHttpHandler(); + handler.When(r => r.Uri.EndsWith("/users", StringComparison.Ordinal)) + .Throw(new HttpRequestException("connection reset")); + + using var httpClient = new HttpClient(handler); + var thrown = await Assert.ThrowsAsync( + () => httpClient.GetAsync("https://graph.microsoft.com/v1.0/users")); + + Assert.Equal("connection reset", thrown.Message); + Assert.Equal(1, handler.RequestCount); + Assert.Empty(handler.ServedStatusCodes); + Assert.Empty(handler.UnansweredRules); + } + + /// + /// A computed answer: one entry answers every request in the collection, each from the + /// request itself, and the status it computed is what gets recorded. + /// + [Fact] + public async Task Mock_FactoryAnswer_ComputesTheResponseAndRecordsItsStatus() + { + var handler = new MockHttpHandler(); + handler.When(r => r.Uri.Contains("/users/", StringComparison.Ordinal)) + .Respond(r => + { + var id = r.Uri[(r.Uri.LastIndexOf('/') + 1)..]; + return new HttpResponseMessage( + id == "user2" ? HttpStatusCode.NotFound : HttpStatusCode.OK) + { + Content = new StringContent( + $$"""{"id":"{{id}}"}""", Encoding.UTF8, "application/json") + }; + }); + + using var httpClient = new HttpClient(handler); + var bodies = new List(); + foreach (var id in new[] { "user1", "user2" }) + { + using var response = await httpClient.GetAsync($"https://graph.microsoft.com/v1.0/users/{id}"); + bodies.Add(await response.Content.ReadAsStringAsync()); + } + + Assert.Equal(new[] { """{"id":"user1"}""", """{"id":"user2"}""" }, bodies); + Assert.Equal( + new[] { HttpStatusCode.OK, HttpStatusCode.NotFound }, + handler.ServedStatusCodes); + } + + /// + /// The scripted handler's keyed path, keyed on the $batch envelope: the predicate reads the + /// request body, so a fault can name the subrequest it is aimed at. The entry answers that + /// envelope and consumes nothing from the script. + /// + [Fact] + public async Task Stub_KeyedResponse_MatchesOnTheBatchBody() + { + var handler = new StubHttpMessageHandler(); + handler.When(r => r.BodyText?.Contains("""{"id":"7"}""", StringComparison.Ordinal) == true, + "subrequest 7 fails") + .Respond(HttpStatusCode.InternalServerError, """{"error":{"code":"aimed"}}"""); + handler.EnqueueJson(HttpStatusCode.OK, """{"responses":[]}"""); + + using var httpClient = new HttpClient(handler); + using var first = await PostBatchAsync(httpClient, """{"requests":[{"id":"1"}]}"""); + using var aimed = await PostBatchAsync(httpClient, """{"requests":[{"id":"7"}]}"""); + + Assert.Equal(HttpStatusCode.OK, first.StatusCode); + Assert.Equal(HttpStatusCode.InternalServerError, aimed.StatusCode); + Assert.Equal("""{"error":{"code":"aimed"}}""", await aimed.Content.ReadAsStringAsync()); + Assert.Equal(2, handler.RequestCount); + Assert.Empty(handler.UnansweredRules); + } + + private static Task PostBatchAsync(HttpClient client, string body) => + client.PostAsync( + "https://graph.microsoft.com/v1.0/$batch", + new StringContent(body, Encoding.UTF8, "application/json")); + private sealed class CapturingMockHandler(List captured) : HttpMessageHandler { protected override Task SendAsync( diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/AtomicSaveHygieneTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/AtomicSaveHygieneTests.cs index 146f0a0..7088a0b 100644 --- a/tests/Mgx.IntegrationTests/Engine/Pagination/AtomicSaveHygieneTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/AtomicSaveHygieneTests.cs @@ -40,6 +40,280 @@ public void A_failed_checkpoint_save_leaves_no_staging_file() } } +/// +/// The three scratch names the engine writes beside a file it is keeping: the write probe +/// ValidateWriteAccess makes, and the "<path>.tmp" each atomic save stages through. All +/// three were written with an open that creates or truncates, so whatever stood at the name was +/// what the run wrote into: a symlink there took the write into the link's target - a file +/// outside the run's business, truncated or filled with state - and a FIFO with no reader +/// blocked the open for as long as the process lived, in one case inside BeginProcessing before +/// the run had anything a cancellation could unwind. The name is cleared first now, by the same +/// primitive the promotion's staging name goes through, and created by an open that refuses to +/// use anything already there. +/// +[Collection("Pipeline")] +public class ScratchNameHygieneTests +{ + /// Which of the three names a case is about. + public enum Site + { + /// DeltaState.ValidateWriteAccess's "<path>.probe". + WriteProbe, + + /// DeltaState.Save's "<path>.tmp". + DeltaSave, + + /// PaginationCheckpoint.Save's "<path>.tmp". + CheckpointSave, + } + + private static string NewDir() => + Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-scratch-{Guid.NewGuid():N}")).FullName; + + /// + /// The scratch name a site writes through, and the write itself. The paths are the ones the + /// engine derives - Save normalizes before it appends the suffix - so a test cannot name a + /// scratch file the run does not use. + /// + private static (string Scratch, string Target, Action Write) SiteUnderTest(Site site, + string dir) + { + if (site == Site.CheckpointSave) + { + var target = Path.Combine(dir, "run.checkpoint"); + var checkpoint = new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + }; + return (Path.GetFullPath(target) + ".tmp", target, () => checkpoint.Save(target)); + } + + var statePath = Path.Combine(dir, "state.json"); + if (site == Site.DeltaSave) + { + var state = new DeltaState + { + DeltaLink = "https://graph.microsoft.com/v1.0/users/delta?$deltatoken=T", + Resource = "/users/delta", + }; + return (Path.GetFullPath(statePath) + ".tmp", statePath, () => state.Save(statePath)); + } + + return (statePath + ".probe", statePath, () => DeltaState.ValidateWriteAccess(statePath)); + } + + private static void Mkfifo(string path) + { + using var mkfifo = System.Diagnostics.Process.Start("/usr/bin/mkfifo", [path]); + Assert.NotNull(mkfifo); + mkfifo.WaitForExit(); + Assert.Equal(0, mkfifo.ExitCode); + } + + /// + /// A symlink at the scratch name. The link comes off the name and the file it pointed at is + /// not touched - which is the whole of what a run can honestly do about a link it did not + /// put there. + /// + [Theory] + [InlineData(Site.WriteProbe)] + [InlineData(Site.DeltaSave)] + [InlineData(Site.CheckpointSave)] + public void A_symlink_at_a_scratch_name_comes_off_it_and_its_target_is_untouched(Site site) + { + var dir = NewDir(); + try + { + var (scratch, target, write) = SiteUnderTest(site, dir); + var victim = Path.Combine(dir, "SOMEBODY-ELSES.txt"); + var kept = "line one\nline two\nline three\n"; + File.WriteAllText(victim, kept); + File.CreateSymbolicLink(scratch, victim); + + write(); + + Assert.Equal(kept, File.ReadAllText(victim)); + Assert.False(File.Exists(scratch), "the scratch name outlived the write"); + if (site != Site.WriteProbe) + Assert.True(File.Exists(target), "the file the save was keeping is not there"); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// A FIFO at the scratch name. Nothing is on the other end of it, so an open that waits for + /// a peer waits for as long as the process lives - past SIGINT, measured - and the write + /// runs on a background thread here so a regression cannot take the suite with it. + /// + [Theory] + [InlineData(Site.WriteProbe)] + [InlineData(Site.DeltaSave)] + [InlineData(Site.CheckpointSave)] + public void A_fifo_at_a_scratch_name_comes_off_it_without_wedging_the_write(Site site) + { + if (OperatingSystem.IsWindows()) return; + var dir = NewDir(); + try + { + var (scratch, target, write) = SiteUnderTest(site, dir); + Mkfifo(scratch); + + Exception? failed = null; + var writer = new Thread(() => + { + try { write(); } + catch (Exception ex) { failed = ex; } + }) + { + // A thread parked in an open cannot be interrupted, so it must not be one the + // process waits for on the way out. + IsBackground = true, + }; + writer.Start(); + + Assert.True(writer.Join(TimeSpan.FromSeconds(20)), + "the write is still waiting for a reader on the pipe at its scratch name"); + Assert.Null(failed); + Assert.False(File.Exists(scratch), "the pipe outlived the write"); + if (site != Site.WriteProbe) + Assert.True(File.Exists(target), "the file the save was keeping is not there"); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// Taking a scratch name back, under the handle that holds it. Released in front of the + /// delete, the name is decided and unheld for as long as the discard takes, and a second run + /// that sweeps it in that interval creates its own file at the name and has it unlinked from + /// under the handle it was holding it by. The entry goes and the handle goes with it. + /// + [Fact] + public void Discarding_a_scratch_name_takes_the_entry_and_the_handle_with_it() + { + var dir = NewDir(); + try + { + var path = Path.Combine(dir, "run.checkpoint.tmp"); + var held = new FileStream(path, FileMode.CreateNew, FileAccess.ReadWrite, + FileShare.None); + + ScratchName.Discard(path, held); + + Assert.False(File.Exists(path), "the entry outlived the discard"); + Assert.Throws(() => held.Length); + + // And a name nothing stands at is one a discard has nothing to say about. + ScratchName.Discard(path, null); + Assert.False(File.Exists(path)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// Two runs over one -DeltaPath, or one -OutputFile, reach the probe together as a matter of + /// course, and the name they both use is the same one. A file the other run holds there is + /// its own probe - taken out a moment ago, in this very directory, which is the question + /// this probe is asking - so it answers yes and leaves the file alone. Refused for the + /// other run's sake, the probe ended a run in BeginProcessing over nothing that was wrong + /// with the path. + /// + [Fact] + public void A_probe_another_run_is_holding_answers_the_question_it_asks() + { + var dir = NewDir(); + try + { + var (scratch, _, write) = SiteUnderTest(Site.WriteProbe, dir); + using var theirs = new FileStream(scratch, FileMode.CreateNew, FileAccess.ReadWrite, + FileShare.None); + + write(); + + Assert.True(File.Exists(scratch), "a live run's own probe was taken off the name"); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// Runs probing one path together. The probe name is derived from the path, so two runs over + /// one -DeltaPath - or two over one -OutputFile - meet at it as a matter of course, and the + /// probe is written to be passed by both: a name another run is holding answers the question + /// this one came to ask. The refusal a create came back with was read off a second look at + /// the name instead, which arrives after the run that made the entry has taken it away - so + /// the second run saw a clear name, called the create's refusal the path's own, and ended in + /// BeginProcessing over a path nothing was wrong with. + /// + /// Windows has a second way to refuse them, and the run is not allowed to stop on that one + /// either: a delete of the name that has been started and has not finished leaves the entry + /// there with every open refused, so the next create - and the next sweep's unlink - meets + /// ERROR_ACCESS_DENIED, which is also what a directory this account cannot write answers. + /// Both are waited out and only a refusal that outlasts the wait is the path's own. At this + /// density the state is reached constantly: over 40 runs of this shape it refused 154 creates + /// and 64 unlinks, and every run of the 40 ended in BeginProcessing over a path nothing was + /// wrong with. + /// + /// + [Fact] + public void Runs_probing_one_path_together_do_not_refuse_each_other() + { + var dir = NewDir(); + try + { + var (_, _, write) = SiteUnderTest(Site.WriteProbe, dir); + var failures = new Exception?[4]; + var threads = new Thread[failures.Length]; + using var go = new ManualResetEventSlim(false); + for (var i = 0; i < threads.Length; i++) + { + var slot = i; + threads[slot] = new Thread(() => + { + go.Wait(TimeSpan.FromSeconds(30)); + for (var n = 0; n < 200 && failures[slot] == null; n++) + { + try { write(); } + catch (Exception ex) { failures[slot] = ex; } + } + }); + threads[slot].Start(); + } + go.Set(); + foreach (var t in threads) + Assert.True(t.Join(TimeSpan.FromSeconds(60)), "a probing run never finished"); + + Assert.All(failures, f => Assert.Null(f)); + Assert.Empty(Directory.GetFiles(dir, "*.probe")); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// A directory at the scratch name is not the run's to delete and could hold anything, so + /// the write says so instead of failing on an open that cannot say what refused it. The + /// probe's refusal is the write-access error it has always been, naming the scratch file and + /// what stood there. + /// + [Fact] + public void A_directory_at_the_probe_name_is_the_write_access_refusal() + { + var dir = NewDir(); + try + { + var (scratch, target, write) = SiteUnderTest(Site.WriteProbe, dir); + Directory.CreateDirectory(scratch); + + var stop = Assert.Throws(write); + Assert.Contains($"Cannot write to delta state path '{target}'", stop.Message); + Assert.Contains($"a directory stands at '{Path.GetFileName(scratch)}'", stop.Message); + Assert.True(Directory.Exists(scratch), "the refusal deleted the directory"); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } +} + /// /// A delta state file from before this field existed carries no apiVersion, so the mismatch /// check is skipped and the run proceeds against whatever version the stored deltaLink names. diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/BoundarySizeTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/BoundarySizeTests.cs index 8624bcf..4ae05f3 100644 --- a/tests/Mgx.IntegrationTests/Engine/Pagination/BoundarySizeTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/BoundarySizeTests.cs @@ -1,8 +1,10 @@ +using System.Management.Automation; using System.Net; using System.Text; using System.Text.Json; using Mgx.Engine.Http; using Mgx.Engine.Pagination; +using Mgx.IntegrationTests.Fakes; namespace Mgx.IntegrationTests; @@ -34,6 +36,35 @@ private static string Page(int firstIndex, int count, int pageNumber, bool isLas return json.ToString(); } + /// + /// What a complete enumeration means, whatever armed the wire: every item, each exactly + /// once, from the first id to the last, over exactly the pages the nextLink chain describes. + /// Count alone does not say it - a loop that re-fetches a page it has already read arrives + /// at the right total whenever the last page is short by as much as it repeated. + /// + private static async Task AssertWholeCollection( + MockHttpHandler handler, string initialUrl, int total, int expectedPages) + { + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var iterator = new PageIterator(client); + + var ids = new List(total); + await foreach (var item in iterator.StreamAllWithCountAsync(initialUrl, maxItems: 0, onCount: null)) + { + ids.Add(item.GetProperty("id").GetString()!); + } + + Assert.Equal(total, ids.Count); + Assert.Distinct(ids); + Assert.Equal("user1", ids[0]); + Assert.Equal($"user{total}", ids[^1]); + Assert.Equal(expectedPages, handler.RequestCount); + Assert.All(handler.Requests.Skip(1).Select((r, i) => (r, i)), + x => Assert.Equal($"https://graph.microsoft.com/v1.0/users?$skiptoken=page{x.i + 2}", + x.r.RequestUri!.ToString())); + } + /// /// is the shape Graph actually returns when the /// collection divides evenly: the last full page still carries a nextLink, and following @@ -65,25 +96,164 @@ public async Task Every_item_comes_back_at_a_page_boundary_size(int total, int p expectedPages++; } + // MockHttpHandler answers in queue order regardless of URL, so the counts hold even for + // an iterator that re-sent the initial URL. The URL assertion is what pins the chain + // here; the generated cases below key the answer on the URL instead, so there a repeat + // hands back a page already seen and the distinctness assertion is what catches it. + await AssertWholeCollection(handler, + $"https://graph.microsoft.com/v1.0/users?$top={pageSize}", total, expectedPages); + } + + /// + /// The same seams over pages computed from the request rather than queued in advance, which + /// is what lets 100,000 items be a size like any other: the queue would need 101 responses + /// built before the first one is sent. + /// + [Theory] + [InlineData(100, 100, false)] + [InlineData(100, 100, true)] + [InlineData(101, 100, false)] + [InlineData(999, 999, false)] + [InlineData(999, 999, true)] + [InlineData(1000, 999, false)] + [InlineData(1001, 999, false)] + [InlineData(100_000, 999, false)] // 101 pages, the size no queue can hold + public async Task Every_item_comes_back_from_a_generated_collection(int total, int pageSize, bool trailingEmptyPage) + { + var pages = new PagedCollection(total, pageSize, trailingEmptyPage: trailingEmptyPage); + var handler = new MockHttpHandler(); + pages.ServeOn(handler); + + await AssertWholeCollection(handler, pages.InitialUrl, total, pages.PageCount); + } + + /// + /// One seam over the scripted handler as well. The generator is programmed on both of the + /// suite's handlers, and the scripted one resolves a keyed entry ahead of its own queue - + /// which it has none of here, so a page it did not key would be a request with nothing to + /// answer it rather than a page quietly repeated. + /// + [Fact] + public async Task Every_item_comes_back_from_a_generated_collection_over_the_scripted_handler() + { + const int total = 1001; + var pages = new PagedCollection(total, pageSize: 999); + var handler = new StubHttpMessageHandler(); + pages.ServeOn(handler); + using var httpClient = new HttpClient(handler); using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); var iterator = new PageIterator(client); - var items = new List(); - await foreach (var item in iterator.StreamAllWithCountAsync( - $"https://graph.microsoft.com/v1.0/users?$top={pageSize}", maxItems: 0, onCount: null)) + var ids = new List(total); + await foreach (var item in iterator.StreamAllWithCountAsync(pages.InitialUrl, maxItems: 0, onCount: null)) { - items.Add(item); + ids.Add(item.GetProperty("id").GetString()!); } - Assert.Equal(total, items.Count); - Assert.Equal(expectedPages, handler.RequestCount); - Assert.Equal("user1", items[0].GetProperty("id").GetString()); - Assert.Equal($"user{total}", items[^1].GetProperty("id").GetString()); - // MockHttpHandler answers in queue order regardless of URL, so the counts above hold - // even for an iterator that re-sent the initial URL. Read the URLs to pin the chain. + Assert.Equal(total, ids.Count); + Assert.Distinct(ids); + Assert.Equal("user1", ids[0]); + Assert.Equal($"user{total}", ids[^1]); + Assert.Equal(pages.PageCount, handler.RequestCount); Assert.All(handler.Requests.Skip(1).Select((r, i) => (r, i)), - x => Assert.Equal($"https://graph.microsoft.com/v1.0/users?$skiptoken=page{x.i + 2}", - x.r.RequestUri!.ToString())); + x => Assert.Equal(pages.NextLinkTo(x.i + 2), x.r.Uri)); + } + + /// + /// One pass of the largest size through the cmdlet, because the iterator's guarantee is not + /// the caller's: Export-MgxCollection writes each item to a temp as it arrives and moves + /// that over the output at the end, and a line written twice or a page lost between the + /// iterator and the file is invisible to every assertion above. -PageSize is left at its + /// default, so the first request also states what that default is worth. + /// + [Fact] + public void An_export_of_a_hundred_thousand_items_writes_each_one_once() + { + const int total = 100_000; + var pages = new PagedCollection(total, pageSize: 999); + var handler = new MockHttpHandler(); + pages.ServeOn(handler); + + using var transport = MgxTransportScope.Inject(handler); + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-boundary-{Guid.NewGuid():N}")).FullName; + var output = Path.Combine(dir, "out.jsonl"); + try + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", output) + .AddParameter("All") + .AddParameter("WarningAction", ActionPreference.SilentlyContinue); + ps.Invoke(); + + // Before the file is read at all: an export that wrote an error record produced a + // file that says nothing about what the run was asked to do, and counting its lines + // would report on the wrong thing. + Assert.Empty(ps.Streams.Error.Select(e => e.FullyQualifiedErrorId)); + + var seen = new HashSet(total); + var lines = 0; + foreach (var line in File.ReadLines(output)) + { + lines++; + using var item = JsonDocument.Parse(line); + var id = item.RootElement.GetProperty("id").GetString(); + Assert.True(seen.Add(id!), $"line {lines} repeats id {id}"); + } + + Assert.Equal(total, lines); + Assert.Equal(pages.PageCount, handler.RequestCount); + Assert.Equal("https://graph.microsoft.com/v1.0/users?$top=999", + handler.CapturedRequests[0].Uri); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } +} + +/// +/// The generator's own guards. It stands in for Graph at a page seam, so a shape the service +/// never returns has to be refused rather than served, and a page past the end of the collection +/// is the defect the boundary tests exist to catch rather than something to answer. +/// +public class PagedCollectionTests +{ + [Fact] + public void A_trailing_empty_page_is_refused_on_a_total_that_does_not_divide_evenly() + { + // 1001 items at 999 a page end on a short page of two. Graph's empty page comes behind + // a full one; behind a short one it is a nextLink no collection ever carries. + var refused = Assert.Throws( + () => new PagedCollection(1001, pageSize: 999, trailingEmptyPage: true)); + Assert.Equal("trailingEmptyPage", refused.ParamName); + + // The evenly dividing sizes the boundary theory passes keep their empty page. + Assert.Equal(2, new PagedCollection(999, pageSize: 999, trailingEmptyPage: true).PageCount); + Assert.Equal(3, new PagedCollection(1998, pageSize: 999, trailingEmptyPage: true).PageCount); + } + + [Fact] + public void A_page_past_the_end_is_refused_rather_than_answered() + { + var pages = new PagedCollection(1001, pageSize: 999); + Assert.Equal(2, pages.PageCount); + + // The last page is short and carries no nextLink, so nothing should ask for a third. + // An empty body here would let an iterator that asked anyway look correct. + var past = Assert.Throws(() => pages.BodyFor(pages.PageCount + 1)); + Assert.Contains("Page 3", past.Message, StringComparison.Ordinal); + Assert.Throws(() => pages.BodyFor(0)); } } diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/CheckpointRetentionTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/CheckpointRetentionTests.cs index 999d963..d2bf915 100644 --- a/tests/Mgx.IntegrationTests/Engine/Pagination/CheckpointRetentionTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/CheckpointRetentionTests.cs @@ -19,8 +19,17 @@ public class CheckpointRetentionTests typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( "TryAdoptOrphanedTemp", BindingFlags.Static | BindingFlags.NonPublic)!; - private static bool TryAdopt(string outputPath, long itemCount) => - (bool)Adopt.Invoke(null, [outputPath, itemCount])!; + /// + /// The two arguments, followed by an empty slot for each out parameter the method carries. + /// Sized off the method, since reflection wants a slot for every one of them. + /// + private static bool TryAdopt(string outputPath, long itemCount) + { + var slots = new object?[Adopt.GetParameters().Length]; + slots[0] = outputPath; + slots[1] = itemCount; + return (bool)Adopt.Invoke(null, slots)!; + } private static string NewDir() { diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/FaultInjectionFanOutTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/FaultInjectionFanOutTests.cs new file mode 100644 index 0000000..8ec1002 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/FaultInjectionFanOutTests.cs @@ -0,0 +1,400 @@ +using System.Management.Automation; +using System.Net; +using System.Text; +using System.Text.Json; +using Mgx.Cmdlets.Base; +using Mgx.Engine.Errors; +using Mgx.Engine.Http; +using Mgx.Engine.Models; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// Every fault in the catalog, injected at the third item of a fan-out whose first two items +/// have already succeeded. The rule the fan-out exists to keep is that one item failing does not +/// discard the others' data, so each case reads both: what happened to the third item, measured +/// against 's decision for its class, and that the first two are +/// still there. +/// +[Collection("Pipeline")] +public class FaultInjectionFanOutTests +{ + private const string Url1 = "https://graph.microsoft.com/v1.0/groups/g1/members"; + private const string Url2 = "https://graph.microsoft.com/v1.0/groups/g2/members"; + private const string Url3 = "https://graph.microsoft.com/v1.0/groups/g3/members"; + + /// What every armed fault in this file is called, so a theory can assert by name + /// that does not carry it. + private const string FaultName = "fault-item-3"; + + private static readonly ResilientGraphClientOptions Options = new() + { + NoRateLimit = true, + NoAdaptivePacing = true, + MaxRetryAttempts = 1, + AttemptTimeoutSeconds = 1, + TotalTimeoutSeconds = 30, + }; + + private static readonly TimeSpan ShortBodyRead = TimeSpan.FromMilliseconds(250); + + private static string PageBody(string[] ids, string? nextLink) + { + var value = string.Join(",", ids.Select(id => "{\"id\":\"" + id + "\"}")); + var link = nextLink == null ? "" : ",\"@odata.nextLink\":\"" + nextLink + "\""; + return "{\"value\":[" + value + "]" + link + "}"; + } + + private static string EntityBody(string id) => "{\"id\":\"" + id + "\"}"; + + private static Dictionary NoBackoff() => new() { ["Retry-After"] = "0" }; + + private static string ErrorBody(int status) => + "{\"error\":{\"code\":\"Injected_" + status + "\",\"message\":\"injected at item 3\"}}"; + + /// + /// Three urls of two items each, with the fault armed on the third url's first attempt only. + /// A later attempt at it is answered normally, so whether a retry happens is the policy's + /// decision rather than the mock's. + /// + private static MockHttpHandler ThreeUrls(Action> armThird) + { + var handler = new MockHttpHandler(); + handler.When(r => r.Uri.Contains("/g1/")).Respond(HttpStatusCode.OK, PageBody(["m1", "m2"], null)); + handler.When(r => r.Uri.Contains("/g2/")).Respond(HttpStatusCode.OK, PageBody(["m3", "m4"], null)); + armThird(handler.When(r => r.Uri.Contains("/g3/"), FaultName).OnAttempt(1)); + handler.When(r => r.Uri.Contains("/g3/")).Respond(HttpStatusCode.OK, PageBody(["m5", "m6"], null)); + return handler; + } + + private static void ArmAtTheThirdItem(MockResponseRule rule, FaultEntry entry) + { + switch (entry.Shape) + { + case FaultShape.Status: + rule.Respond((HttpStatusCode)entry.Status, ErrorBody(entry.Status), NoBackoff()); + break; + case FaultShape.Transport: + rule.Throw(entry.Transport!()); + break; + case FaultShape.HeldPastAttemptTimeout: + rule.AfterDelay(TimeSpan.FromSeconds(5)) + .Respond(HttpStatusCode.OK, PageBody(["m5", "m6"], null)); + break; + case FaultShape.StalledBody: + rule.RespondStalled(HttpStatusCode.OK, prefix: """{"value":["""); + break; + case FaultShape.TruncatedJson: + rule.Respond(HttpStatusCode.OK, FaultCatalog.TruncatedPageBody); + break; + case FaultShape.UnusableNextLink: + rule.Respond(HttpStatusCode.OK, PageBody(["m5", "m6"], "members?page=2")); + break; + default: + throw new InvalidOperationException( + $"{entry.Kind} does not take a wire shape at a fan-out item; it needs its own case."); + } + } + + private static async Task FanOutAsync(MockHttpHandler handler, int maxItemsPerUrl = 0) + { + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, Options) + { + BodyReadTimeout = ShortBodyRead + }; + // One at a time, so "the third item" is the third request on the wire and not a race. + var fanOut = new ConcurrentFanOut(client, maxConcurrency: 1); + return await fanOut.FetchAllAsync([Url1, Url2, Url3], maxItemsPerUrl); + } + + [Theory] + [MemberData(nameof(FaultCatalog.AllKinds), MemberType = typeof(FaultCatalog))] + public async Task Each_fault_at_the_third_fan_out_item(FaultKind kind) + { + var entry = FaultCatalog.Entry(kind); + var cell = entry.At(InjectionPoint.FanOutItem); + if (cell.Coverage == FaultCoverage.NotApplicable) + { + FaultInjectionRows.AssertNotApplicable(entry, InjectionPoint.FanOutItem); + return; + } + + ResiliencePipelineFactory.Reset(); + try + { + if (cell.Coverage == FaultCoverage.Pinned) + { + await PinAtTheThirdItem(entry); + return; + } + + var handler = ThreeUrls(rule => ArmAtTheThirdItem(rule, entry)); + var result = await FanOutAsync(handler); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + if (FaultCatalog.RetriedOnAnIdempotentRequest(entry)) + { + Assert.False(result.HasErrors); + Assert.Equal(3, result.Results.Count); + Assert.Equal(2, result.Results[Url3].Length); + Assert.Equal(4, handler.RequestCount); + } + else + { + AssertTheFirstTwoSurvived(handler, result); + Assert.Equal(3, handler.RequestCount); + var failed = Assert.Single(result.Errors); + Assert.Equal(Url3, failed.Key); + AssertTheFaultSurfaced(entry, failed.Value); + } + } + finally + { + ResiliencePipelineFactory.Reset(); + } + } + + /// + /// The whole point of the fan-out's per-url error bag: the two urls that worked keep their + /// data whatever happened to the third, and the third was the third request on the wire. + /// + private static void AssertTheFirstTwoSurvived(MockHttpHandler handler, FanOutResult result) + { + Assert.True(result.HasErrors); + Assert.Equal(2, result.Results.Count); + Assert.Equal(2, result.Results[Url1].Length); + Assert.Equal(2, result.Results[Url2].Length); + Assert.DoesNotContain(Url3, result.Results.Keys); + Assert.Contains("/g3/", handler.CapturedRequests[2].Uri); + } + + private static void AssertTheFaultSurfaced(FaultEntry entry, Exception thrown) + { + switch (entry.Shape) + { + case FaultShape.Status: + var graph = Assert.IsType(thrown); + Assert.Equal(entry.Status, (int)graph.StatusCode); + Assert.Equal(entry.Class, MgxErrorClassifier.Classify((int)graph.StatusCode).Class); + break; + case FaultShape.TruncatedJson: + Assert.IsAssignableFrom(thrown); + break; + case FaultShape.UnusableNextLink: + var refused = Assert.IsType(thrown); + Assert.Contains("nextLink", refused.Message, StringComparison.OrdinalIgnoreCase); + break; + case FaultShape.StalledBody: + var stalled = Assert.IsType(thrown); + Assert.Equal(ResilientGraphClient.BodyReadTimedOutMessage, stalled.Message); + break; + default: + Assert.Fail($"{entry.Kind} surfaced as {thrown.GetType().Name} with no case to check it."); + break; + } + } + + // ── The rows that record today's behavior rather than a policy decision ── + + private static async Task PinAtTheThirdItem(FaultEntry entry) + { + switch (entry.Kind) + { + case FaultKind.BodyTimeout: await PinAStalledItemBody(entry); return; + case FaultKind.RepeatedNextLink: await PinASelfReferencingNextLink(); return; + case FaultKind.ExpiredToken: await PinAnExpiredTokenAtTheThirdItem(entry); return; + case FaultKind.ConsistencyDelay: await PinAConsistencyDelay(entry); return; + case FaultKind.ChangedAuthContext: PinAnAuthContextSwapBetweenItems(); return; + default: + Assert.Fail($"{entry.Kind} is pinned at a fan-out item with nothing to run."); + return; + } + } + + private static async Task PinAStalledItemBody(FaultEntry entry) + { + var handler = ThreeUrls(rule => ArmAtTheThirdItem(rule, entry)); + var result = await FanOutAsync(handler); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + // The policy would retry this class on a GET. + Assert.True(FaultCatalog.RetriedOnAnIdempotentRequest(entry)); + + // Nothing does: the body is read after the pipeline handed the response back, so the + // third item gets one attempt and its stall lands in the error bag. + AssertTheFirstTwoSurvived(handler, result); + Assert.Equal(3, handler.RequestCount); + var stalled = Assert.IsType(Assert.Single(result.Errors).Value); + Assert.Equal(ResilientGraphClient.BodyReadTimedOutMessage, stalled.Message); + } + + private static async Task PinASelfReferencingNextLink() + { + // Non-empty pages: the third url's page points back at itself, and only the per-url item + // cap ends the walk. + var looping = new MockHttpHandler(); + looping.When(r => r.Uri.Contains("/g1/")).Respond(HttpStatusCode.OK, PageBody(["m1", "m2"], null)); + looping.When(r => r.Uri.Contains("/g2/")).Respond(HttpStatusCode.OK, PageBody(["m3", "m4"], null)); + looping.When(r => r.Uri.Contains("/g3/")).Respond(HttpStatusCode.OK, PageBody(["m5", "m6"], Url3)); + + var looped = await FanOutAsync(looping, maxItemsPerUrl: 6); + + Assert.False(looped.HasErrors); + Assert.Equal(6, looped.Results[Url3].Length); + Assert.Equal(5, looping.RequestCount); + Assert.Equal(3, looping.CapturedRequests.Count(r => r.Uri.Contains("/g3/"))); + + // Empty pages: the consecutive-empty-page limit is what does stop a self-loop today. + var empty = new MockHttpHandler(); + empty.When(r => r.Uri.Contains("/g1/")).Respond(HttpStatusCode.OK, PageBody(["m1", "m2"], null)); + empty.When(r => r.Uri.Contains("/g2/")).Respond(HttpStatusCode.OK, PageBody(["m3", "m4"], null)); + empty.When(r => r.Uri.Contains("/g3/")).Respond(HttpStatusCode.OK, PageBody([], Url3)); + + var stopped = await FanOutAsync(empty); + + Assert.False(stopped.HasErrors); + Assert.Empty(stopped.Results[Url3]); + Assert.Equal(5, empty.RequestCount); + } + + private static async Task PinAnExpiredTokenAtTheThirdItem(FaultEntry entry) + { + var handler = ThreeUrls(rule => ArmAtTheThirdItem(rule, entry)); + var result = await FanOutAsync(handler); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + Assert.Equal(MgxErrorClass.Authentication, entry.Class); + Assert.False(FaultCatalog.RetriedOnAnIdempotentRequest(entry)); + + AssertTheFirstTwoSurvived(handler, result); + var graph = Assert.IsType(Assert.Single(result.Errors).Value); + Assert.Equal(HttpStatusCode.Unauthorized, graph.StatusCode); + + // One attempt at the third item, and no fresh token asked for: the good page the mock is + // holding for a second attempt is never fetched. + Assert.Equal( + [HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.Unauthorized], + handler.ServedStatusCodes); + } + + private static async Task PinAConsistencyDelay(FaultEntry entry) + { + var handler = ThreeUrls(rule => ArmAtTheThirdItem(rule, entry)); + var result = await FanOutAsync(handler); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + // A read-after-write 404 classifies as NotFound, which the policy does not retry, so the + // 200 waiting behind it is never asked for. MgxErrorClass.Consistency, declared for this + // window, still has no producer. + Assert.Equal(MgxErrorClass.NotFound, entry.Class); + Assert.False(FaultCatalog.RetriedOnAnIdempotentRequest(entry)); + + AssertTheFirstTwoSurvived(handler, result); + var graph = Assert.IsType(Assert.Single(result.Errors).Value); + Assert.Equal(HttpStatusCode.NotFound, graph.StatusCode); + Assert.Equal( + [HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.NotFound], + handler.ServedStatusCodes); + } + + private static void PinAnAuthContextSwapBetweenItems() + { + const string Tenant = "11111111-1111-1111-1111-111111111111"; + const string AppA = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"; + const string AppB = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"; + + var handler = new MockHttpHandler(); + handler.When(r => r.Uri.Contains("/users/u1")).Respond(HttpStatusCode.OK, EntityBody("u1")); + handler.When(r => r.Uri.Contains("/users/u3")).Respond(HttpStatusCode.OK, EntityBody("u3")); + + using var wire = new HttpClient(handler); + // The transport seam sits above the auth probe, and the probe is what is under test, so + // this drives the real GetClient path against the session's own client. + using var scope = GraphSessionScope.Arm(wire, GraphSessionScope.AuthContextFor(Tenant, AppA)); + + // Swapped while the second id is being answered - one item into the fan-out. + handler.When(r => r.Uri.Contains("/users/u2")).OnAttempt(1).Respond(_ => + { + scope.Session.AuthContext = GraphSessionScope.AuthContextFor(Tenant, AppB); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(EntityBody("u2"), Encoding.UTF8, "application/json") + }; + }); + handler.When(r => r.Uri.Contains("/users/u2")).Respond(HttpStatusCode.OK, EntityBody("u2")); + + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + + var (firstCount, firstVerbose) = RunFanOut(ps); + Assert.Equal(3, firstCount); + Assert.DoesNotContain(firstVerbose, v => v.Contains("Graph identity changed")); + + var (secondCount, secondVerbose) = RunFanOut(ps); + Assert.Equal(3, secondCount); + Assert.Contains(secondVerbose, v => v.Contains("Graph identity changed")); + + static (int Output, List Verbose) RunFanOut(PowerShell ps) + { + ps.Commands.Clear(); + ps.Streams.ClearStreams(); + ps.AddScript( + "'u1','u2','u3' | Invoke-MgxRequest -Uri '/users/{id}' -Concurrency 1 " + + "-Verbose -WarningAction SilentlyContinue"); + var output = ps.Invoke(); + Assert.Empty(ps.Streams.Error.Select(e => e.FullyQualifiedErrorId)); + return (output.Count, [.. ps.Streams.Verbose.Select(v => v.Message)]); + } + } + + // ── The same faults through the {id} fan-out, over the transport seam ──── + + [Theory] + [InlineData(FaultKind.Throttled)] + [InlineData(FaultKind.Forbidden)] + public void A_fault_at_the_third_id_of_Invoke_MgxRequest(FaultKind kind) + { + var entry = FaultCatalog.Entry(kind); + var handler = new MockHttpHandler(); + handler.When(r => r.Uri.Contains("/users/u1")).Respond(HttpStatusCode.OK, EntityBody("u1")); + handler.When(r => r.Uri.Contains("/users/u2")).Respond(HttpStatusCode.OK, EntityBody("u2")); + handler.When(r => r.Uri.Contains("/users/u3"), FaultName).OnAttempt(1) + .Respond((HttpStatusCode)entry.Status, ErrorBody(entry.Status), NoBackoff()); + handler.When(r => r.Uri.Contains("/users/u3")).Respond(HttpStatusCode.OK, EntityBody("u3")); + + using (MgxTransportScope.Inject(handler, options: Options)) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript( + "'u1','u2','u3' | Invoke-MgxRequest -Uri '/users/{id}' -Concurrency 1 " + + "-WarningAction SilentlyContinue"); + var output = ps.Invoke(); + var errors = ps.Streams.Error.ToList(); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + if (FaultCatalog.RetriedOnAnIdempotentRequest(entry)) + { + Assert.Empty(errors); + Assert.Equal(3, output.Count); + } + else + { + // One failed id is one non-terminating record; the other two are still emitted. + Assert.Equal(2, output.Count); + var record = Assert.Single(errors); + Assert.Equal("u3", record.TargetObject); + Assert.Equal( + MgxErrorPresentation.Category(entry.Class!.Value), + record.CategoryInfo.Category); + } + } + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/FaultInjectionPaginationTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/FaultInjectionPaginationTests.cs new file mode 100644 index 0000000..f48f9bf --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/FaultInjectionPaginationTests.cs @@ -0,0 +1,534 @@ +using System.Management.Automation; +using System.Net; +using System.Text; +using System.Text.Json; +using Mgx.Cmdlets.Base; +using Mgx.Engine.Errors; +using Mgx.Engine.Http; +using Mgx.Engine.Models; +using Mgx.Engine.Pagination; +using Polly.CircuitBreaker; + +namespace Mgx.IntegrationTests; + +/// +/// Every fault in the catalog, injected at page 3 of a pagination that has already delivered +/// pages 1 and 2. What each case asserts comes from 's decision for +/// the class the classifier assigns: a retryable class has to show the mock serving the retry +/// and the pagination finishing, a non-retryable one exactly one attempt and the pages before +/// the fault still delivered. Nothing here restates a status table - the parity suite owns that. +/// +[Collection("Pipeline")] +public class FaultInjectionPaginationTests +{ + private const string CollectionUrl = "https://graph.microsoft.com/v1.0/users"; + private const string Page2Url = CollectionUrl + "?$skiptoken=page2"; + private const string Page3Url = CollectionUrl + "?$skiptoken=page3"; + + /// What every armed fault in this file is called, so a theory can assert by name + /// that does not carry it. + private const string FaultName = "fault-page-3"; + + /// + /// One retry is all any of these needs, and a one-second attempt timeout is what makes the + /// held-answer case fire without costing the suite anything. + /// + private static readonly ResilientGraphClientOptions Options = new() + { + NoRateLimit = true, + NoAdaptivePacing = true, + MaxRetryAttempts = 1, + AttemptTimeoutSeconds = 1, + TotalTimeoutSeconds = 30, + }; + + private static readonly TimeSpan ShortBodyRead = TimeSpan.FromMilliseconds(250); + + private static string PageBody(string[] ids, string? nextLink) + { + var value = string.Join(",", ids.Select(id => "{\"id\":\"" + id + "\"}")); + var link = nextLink == null ? "" : ",\"@odata.nextLink\":\"" + nextLink + "\""; + return "{\"value\":[" + value + "]" + link + "}"; + } + + private static Dictionary NoBackoff() => new() { ["Retry-After"] = "0" }; + + private static string ErrorBody(int status) => + "{\"error\":{\"code\":\"Injected_" + status + "\",\"message\":\"injected at page 3\"}}"; + + /// + /// Three pages of two items each, with the fault armed on page 3's first attempt only. The + /// same programming serves every kind: a later attempt at page 3 is answered normally, so + /// whether the retry happens at all is the policy's decision and not the mock's. + /// + private static MockHttpHandler ThreePages(Action> armPage3) + { + var handler = new MockHttpHandler(); + handler.When(r => r.Uri.Contains("skiptoken=page2")) + .Respond(HttpStatusCode.OK, PageBody(["u3", "u4"], Page3Url)); + armPage3(handler.When(r => r.Uri.Contains("skiptoken=page3"), FaultName).OnAttempt(1)); + handler.When(r => r.Uri.Contains("skiptoken=page3")) + .Respond(HttpStatusCode.OK, PageBody(["u5", "u6"], null)); + handler.SetDefaultResponse(HttpStatusCode.OK, PageBody(["u1", "u2"], Page2Url)); + return handler; + } + + private static void ArmAtPageThree(MockResponseRule rule, FaultEntry entry) + { + switch (entry.Shape) + { + case FaultShape.Status: + rule.Respond((HttpStatusCode)entry.Status, ErrorBody(entry.Status), NoBackoff()); + break; + case FaultShape.Transport: + rule.Throw(entry.Transport!()); + break; + case FaultShape.HeldPastAttemptTimeout: + rule.AfterDelay(TimeSpan.FromSeconds(5)) + .Respond(HttpStatusCode.OK, PageBody(["u5", "u6"], null)); + break; + case FaultShape.StalledBody: + rule.RespondStalled(HttpStatusCode.OK, prefix: """{"value":["""); + break; + case FaultShape.TruncatedJson: + rule.Respond(HttpStatusCode.OK, FaultCatalog.TruncatedPageBody); + break; + case FaultShape.UnusableNextLink: + // Not a URL at all, so the validator refuses it for a reason that has nothing to + // do with the host - SsrfValidationTests owns the cross-host refusal. + rule.Respond(HttpStatusCode.OK, PageBody(["u5", "u6"], "pages/4")); + break; + default: + throw new InvalidOperationException( + $"{entry.Kind} does not take a wire shape at a page; it needs its own case."); + } + } + + private static async Task<(List Items, Exception? Thrown)> PaginateAsync( + MockHttpHandler handler, int maxItems = 0, ResilientGraphClientOptions? options = null) + { + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, options ?? Options) + { + BodyReadTimeout = ShortBodyRead + }; + var iterator = new PageIterator(client); + var items = new List(); + var thrown = await Record.ExceptionAsync(async () => + { + await foreach (var item in iterator.StreamAllWithCountAsync(CollectionUrl, maxItems, null)) + items.Add(item); + }); + return (items, thrown); + } + + [Theory] + [MemberData(nameof(FaultCatalog.AllKinds), MemberType = typeof(FaultCatalog))] + public async Task Each_fault_at_page_three(FaultKind kind) + { + var entry = FaultCatalog.Entry(kind); + var cell = entry.At(InjectionPoint.Page); + if (cell.Coverage == FaultCoverage.NotApplicable) + { + FaultInjectionRows.AssertNotApplicable(entry, InjectionPoint.Page); + return; + } + + ResiliencePipelineFactory.Reset(); + try + { + if (cell.Coverage == FaultCoverage.Pinned) + { + await PinAtPageThree(entry); + return; + } + + var handler = ThreePages(rule => ArmAtPageThree(rule, entry)); + var (items, thrown) = await PaginateAsync(handler); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + if (FaultCatalog.RetriedOnAnIdempotentRequest(entry)) + { + Assert.Null(thrown); + Assert.Equal(6, items.Count); + // page 1, page 2, page 3 refused, page 3 served: the retry is on the wire. + Assert.Equal(4, handler.RequestCount); + } + else + { + Assert.NotNull(thrown); + // Everything delivered before the fault is still delivered - including page 3's + // own items when the fault is in the link that page carried rather than in the + // page: the items were handed over before the link was read. + Assert.Equal(entry.Shape == FaultShape.UnusableNextLink ? 6 : 4, items.Count); + Assert.Equal(3, handler.RequestCount); + AssertTheFaultSurfaced(entry, thrown!); + } + } + finally + { + ResiliencePipelineFactory.Reset(); + } + } + + private static void AssertTheFaultSurfaced(FaultEntry entry, Exception thrown) + { + switch (entry.Shape) + { + case FaultShape.Status: + var graph = Assert.IsType(thrown); + Assert.Equal(entry.Status, (int)graph.StatusCode); + Assert.Equal(entry.Class, MgxErrorClassifier.Classify((int)graph.StatusCode).Class); + break; + case FaultShape.TruncatedJson: + Assert.IsAssignableFrom(thrown); + break; + case FaultShape.UnusableNextLink: + var refused = Assert.IsType(thrown); + Assert.Contains("nextLink", refused.Message, StringComparison.OrdinalIgnoreCase); + break; + case FaultShape.StalledBody: + var stalled = Assert.IsType(thrown); + Assert.Equal(ResilientGraphClient.BodyReadTimedOutMessage, stalled.Message); + break; + default: + Assert.Fail($"{entry.Kind} surfaced as {thrown.GetType().Name} with no case to check it."); + break; + } + } + + // ── The rows that record today's behavior rather than a policy decision ── + + private static async Task PinAtPageThree(FaultEntry entry) + { + switch (entry.Kind) + { + case FaultKind.BodyTimeout: await PinAStalledPageBody(entry); return; + case FaultKind.RepeatedNextLink: await PinASelfReferencingNextLink(); return; + case FaultKind.ExpiredToken: await PinAnExpiredTokenAtPageThree(entry); return; + case FaultKind.ConsistencyDelay: await PinAConsistencyDelay(entry); return; + case FaultKind.ChangedAuthContext: PinAnAuthContextSwapBetweenPages(); return; + default: + Assert.Fail($"{entry.Kind} is pinned at a page with nothing to run."); + return; + } + } + + private static async Task PinAStalledPageBody(FaultEntry entry) + { + var handler = ThreePages(rule => ArmAtPageThree(rule, entry)); + var (items, thrown) = await PaginateAsync(handler); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + // The policy would retry this class on a GET. + Assert.True(FaultCatalog.RetriedOnAnIdempotentRequest(entry)); + + // Nothing does. The page body is read after the resilience pipeline has already handed + // the response back, so the retry predicate never sees the failure: one attempt at page + // 3, and the pagination ends there with the pages before it delivered. + var stalled = Assert.IsType(thrown); + Assert.Equal(ResilientGraphClient.BodyReadTimedOutMessage, stalled.Message); + Assert.Equal(4, items.Count); + Assert.Equal(3, handler.RequestCount); + } + + private static async Task PinASelfReferencingNextLink() + { + // Non-empty pages: the link points back at the page that produced it, every time, and + // nothing in the iterator notices. Only the item cap ends the walk - which is why this + // case has to pass one. + var looping = new MockHttpHandler(); + looping.When(r => r.Uri.Contains("skiptoken=page2")) + .Respond(HttpStatusCode.OK, PageBody(["u3", "u4"], Page3Url)); + looping.When(r => r.Uri.Contains("skiptoken=page3")) + .Respond(HttpStatusCode.OK, PageBody(["u5", "u6"], Page3Url)); + looping.SetDefaultResponse(HttpStatusCode.OK, PageBody(["u1", "u2"], Page2Url)); + + var (items, thrown) = await PaginateAsync(looping, maxItems: 10); + + Assert.Null(thrown); + Assert.Equal(10, items.Count); + Assert.Equal(5, looping.RequestCount); + Assert.Equal(3, looping.CapturedRequests.Count(r => r.Uri.Contains("skiptoken=page3"))); + + // Empty pages: the consecutive-empty-page limit is what does stop a self-loop today. + var empty = new MockHttpHandler(); + empty.When(r => r.Uri.Contains("skiptoken=page2")) + .Respond(HttpStatusCode.OK, PageBody(["u3", "u4"], Page3Url)); + empty.When(r => r.Uri.Contains("skiptoken=page3")) + .Respond(HttpStatusCode.OK, PageBody([], Page3Url)); + empty.SetDefaultResponse(HttpStatusCode.OK, PageBody(["u1", "u2"], Page2Url)); + + var (emptyItems, emptyThrown) = await PaginateAsync(empty); + + Assert.Null(emptyThrown); + Assert.Equal(4, emptyItems.Count); + Assert.Equal(5, empty.RequestCount); + } + + private static async Task PinAnExpiredTokenAtPageThree(FaultEntry entry) + { + var handler = ThreePages(rule => ArmAtPageThree(rule, entry)); + var (items, thrown) = await PaginateAsync(handler); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + Assert.Equal(MgxErrorClass.Authentication, entry.Class); + Assert.False(FaultCatalog.RetriedOnAnIdempotentRequest(entry)); + + var graph = Assert.IsType(thrown); + Assert.Equal(HttpStatusCode.Unauthorized, graph.StatusCode); + Assert.Equal(4, items.Count); + + // One attempt, and no fresh token asked for: the mock is holding a good page 3 that the + // run never comes back for. Mid-operation re-authentication is not something mgx does. + Assert.Equal( + [HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.Unauthorized], + handler.ServedStatusCodes); + } + + private static async Task PinAConsistencyDelay(FaultEntry entry) + { + var handler = ThreePages(rule => ArmAtPageThree(rule, entry)); + var (items, thrown) = await PaginateAsync(handler); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + // A read-after-write 404 classifies as NotFound, which the policy does not retry, so the + // 200 the mock would serve on a second attempt is never asked for. MgxErrorClass + // .Consistency is declared for this window and has no producer yet. + Assert.Equal(MgxErrorClass.NotFound, entry.Class); + Assert.False(FaultCatalog.RetriedOnAnIdempotentRequest(entry)); + + var graph = Assert.IsType(thrown); + Assert.Equal(HttpStatusCode.NotFound, graph.StatusCode); + Assert.Equal(4, items.Count); + Assert.Equal( + [HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.NotFound], + handler.ServedStatusCodes); + } + + private static void PinAnAuthContextSwapBetweenPages() + { + const string Tenant = "11111111-1111-1111-1111-111111111111"; + const string AppA = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"; + const string AppB = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"; + + var handler = new MockHttpHandler(); + handler.When(r => r.Uri.Contains("skiptoken=page3")) + .Respond(HttpStatusCode.OK, PageBody(["u5", "u6"], null)); + handler.SetDefaultResponse(HttpStatusCode.OK, PageBody(["u1", "u2"], Page2Url)); + + using var wire = new HttpClient(handler); + // No transport seam here: the seam sits above the auth probe, and the probe is the thing + // under test. GetClient falls back to the session's own client, so the mock is reached + // through the identity check rather than around it. + using var scope = GraphSessionScope.Arm(wire, GraphSessionScope.AuthContextFor(Tenant, AppA)); + + // The swap lands while page 2 is being answered - after page 2's request went out on the + // client the invocation resolved, and before page 3's does. + handler.When(r => r.Uri.Contains("skiptoken=page2")).OnAttempt(1).Respond(_ => + { + scope.Session.AuthContext = GraphSessionScope.AuthContextFor(Tenant, AppB); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + PageBody(["u3", "u4"], Page3Url), Encoding.UTF8, "application/json") + }; + }); + handler.When(r => r.Uri.Contains("skiptoken=page2")) + .Respond(HttpStatusCode.OK, PageBody(["u3", "u4"], Page3Url)); + + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + + var (firstCount, firstVerbose) = RunAll(ps); + Assert.Equal(6, firstCount); + Assert.DoesNotContain(firstVerbose, v => v.Contains("Graph identity changed")); + + var (secondCount, secondVerbose) = RunAll(ps); + Assert.Equal(6, secondCount); + Assert.Contains(secondVerbose, v => v.Contains("Graph identity changed")); + + static (int Output, List Verbose) RunAll(PowerShell ps) + { + ps.Commands.Clear(); + ps.Streams.ClearStreams(); + ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("All", true) + .AddParameter("Verbose", true) + .AddParameter("WarningAction", ActionPreference.SilentlyContinue); + var output = ps.Invoke(); + Assert.Empty(ps.Streams.Error.Select(e => e.FullyQualifiedErrorId)); + return (output.Count, [.. ps.Streams.Verbose.Select(v => v.Message)]); + } + } + + // ── The breaker dimension of the same faults ───────────────────────────── + + /// + /// The breaker's minimum throughput and sampling window at the floors the options allow, so + /// four attempts are enough to judge, and a long open window so nothing half-opens while the + /// runs behind the first one are being made. + /// + private static readonly ResilientGraphClientOptions BreakerOptions = new() + { + NoRateLimit = true, + NoAdaptivePacing = true, + MaxRetryAttempts = 1, + AttemptTimeoutSeconds = 1, + TotalTimeoutSeconds = 30, + CircuitBreakerMinThroughput = 2, + CircuitBreakerFailureRatio = 0.4, + CircuitBreakerSamplingDurationSeconds = 5, + CircuitBreakerDurationSeconds = 60, + }; + + /// How many runs the drive below makes before it gives up on the circuit opening. + private const int BreakerRuns = 3; + + /// Pages 1 and 2, then page 3 twice - its own attempt and the one retry. + private const int RequestsPerRun = 4; + + /// + /// The dimension the per-fault theory above does not ask about: whether repeating a fault + /// opens the circuit. answers it per class, + /// and the two classes that answer differently are what makes the answer worth anything - a + /// 503 is the service failing and a 429 is the service pacing us, and a circuit that opened + /// on the second would stop a run exactly when the right response is to slow down and keep + /// going. + /// + [Fact] + public async Task Repeating_a_page_three_fault_opens_the_circuit_only_for_a_counted_class() + { + var counted = FaultCatalog.Entry(FaultKind.ServerError); + var excluded = FaultCatalog.Entry(FaultKind.Throttled); + + var server = await DriveAgainstTheBreakerAsync(counted); + var throttle = await DriveAgainstTheBreakerAsync(excluded); + + // Each side is what the policy says of its own class. + Assert.Equal(FaultCatalog.CountsAgainstTheBreaker(counted), server.Opened); + Assert.Equal(FaultCatalog.CountsAgainstTheBreaker(excluded), throttle.Opened); + + // And the two answers differ, which neither assertion above can say on its own: a policy + // that counted throttles as well would agree with both of them and open the circuit on + // the service's own pacing. + Assert.NotEqual(server.Opened, throttle.Opened); + + // What the difference costs on the wire. Once the circuit is open the run behind it sends + // nothing at all - the refusal is the client's, before the request. The class the breaker + // ignores keeps sending every attempt, run after run, and what surfaces is still the + // status the server gave. + Assert.Equal(0, server.RequestsOnTheLastRun); + Assert.IsType(server.LastFailure); + Assert.Equal(RequestsPerRun, throttle.RequestsOnTheLastRun); + Assert.Equal(HttpStatusCode.TooManyRequests, + Assert.IsType(throttle.LastFailure).StatusCode); + } + + /// + /// Page 3 answered with this fault's status on every attempt, run and re-run against one + /// pipeline so the outcomes accumulate in a single sampling window. Returns whether any run + /// was refused by the open circuit, what the last run cost on the wire, and what it surfaced. + /// + private static async Task<(bool Opened, int RequestsOnTheLastRun, Exception LastFailure)> + DriveAgainstTheBreakerAsync(FaultEntry entry) + { + ResiliencePipelineFactory.Reset(); + try + { + var opened = false; + var requests = 0; + Exception? last = null; + for (var run = 0; run < BreakerRuns; run++) + { + var handler = new MockHttpHandler(); + // Not named and not asserted: once the circuit opens, a later run's client-side + // refusal never sends a request at all, so this entry is legitimately unanswered + // on that run - a blanket assertion here would fail the very outcome the test + // means to observe. + handler.When(r => r.Uri.Contains("skiptoken=page3")) + .Respond((HttpStatusCode)entry.Status, ErrorBody(entry.Status), NoBackoff()); + handler.When(r => r.Uri.Contains("skiptoken=page2")) + .Respond(HttpStatusCode.OK, PageBody(["u3", "u4"], Page3Url)); + handler.SetDefaultResponse(HttpStatusCode.OK, PageBody(["u1", "u2"], Page2Url)); + + var (_, thrown) = await PaginateAsync(handler, options: BreakerOptions); + + Assert.NotNull(thrown); + last = thrown; + opened |= thrown is BrokenCircuitException; + requests = handler.RequestCount; + } + return (opened, requests, last!); + } + finally + { + ResiliencePipelineFactory.Reset(); + } + } + + // ── The same faults through the cmdlet, over the transport seam ────────── + + [Theory] + [InlineData(FaultKind.Throttled)] + [InlineData(FaultKind.Forbidden)] + public void A_fault_at_page_three_of_Invoke_MgxRequest_All(FaultKind kind) + { + var entry = FaultCatalog.Entry(kind); + var handler = ThreePages(rule => ArmAtPageThree(rule, entry)); + + using (MgxTransportScope.Inject(handler, options: Options)) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("All", true) + .AddParameter("WarningAction", ActionPreference.SilentlyContinue); + var output = ps.Invoke(); + var errors = ps.Streams.Error.ToList(); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + if (FaultCatalog.RetriedOnAnIdempotentRequest(entry)) + { + Assert.Empty(errors); + Assert.Equal(6, output.Count); + } + else + { + // The contract: only a failure that makes the whole invocation meaningless + // terminates, so the pages already delivered are delivered and the page that + // failed is one non-terminating record. + Assert.Equal(4, output.Count); + var record = Assert.Single(errors); + Assert.Equal( + MgxErrorPresentation.Category(entry.Class!.Value), + record.CategoryInfo.Category); + } + } + } +} + +/// +/// What a theory says about a cell it cannot run. The acceptance is "every fault at each point +/// where it is physically possible", so a point that cannot host a fault has to say which fault +/// and why, in the same run as the ones it does host. +/// +internal static class FaultInjectionRows +{ + internal static void AssertNotApplicable(FaultEntry entry, InjectionPoint point) + { + var cell = entry.At(point); + Assert.Equal(FaultCoverage.NotApplicable, cell.Coverage); + Assert.False(string.IsNullOrWhiteSpace(cell.Note), + $"{entry.Kind} is marked impossible at {point} without saying why"); + Assert.Contains(Enum.GetValues(), + other => entry.At(other).Coverage != FaultCoverage.NotApplicable); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/HeldEntryTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/HeldEntryTests.cs new file mode 100644 index 0000000..4b2d3f1 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/HeldEntryTests.cs @@ -0,0 +1,149 @@ +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// Serializes the test classes that set HeldEntry's claim seam, which is one static for the +/// whole process: run in parallel, one class's assignment is the other's, and tests written +/// against the instant between a claim and its check would fire each other's. +/// +[CollectionDefinition("ClaimSeam")] +public class ClaimSeamCollection; + +/// +/// The question every claim on Unix is finished with: is the file this run holds still the file +/// standing at the name it asked for? A handle says nothing about that on its own - the lock +/// .NET takes for FileShare.None is granted several syscalls after the open that resolved the +/// name, and rename and unlink honor no lock - so the three ways a name moves under a handle are +/// asked about here, one test each. +/// +/// Those three are Unix's alone, and not because the answer would differ: Windows refuses to +/// unlink or rename a file another handle has open at all, so a name cannot move under a claim +/// there and the three states cannot be reached to be asked about. That refusal is the whole +/// reason the primitive answers Windows at the open and asks the platform nothing. +/// +/// +[Collection("ClaimSeam")] +public class HeldEntryTests +{ + private static string NewDir() => + Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-held-{Guid.NewGuid():N}")).FullName; + + private static FileStream Claim(string path) => + new(path, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None); + + /// A file nothing has touched since the claim: the name is still this handle's. + [Fact] + public void A_handle_on_the_file_at_the_name_still_stands_there() + { + var dir = NewDir(); + try + { + var path = Path.Combine(dir, "out.jsonl"); + using var held = Claim(path); + held.Write("{\"id\":\"a\"}\n"u8); + held.Flush(); + + Assert.True(HeldEntry.StillStandsAt(held, path)); + } + finally { Directory.Delete(dir, true); } + } + + /// + /// Unlinked under the handle, which is what a second run's sweep of the name does: the inode + /// is this run's for as long as it holds it and has no name left on it at all. The file still + /// reads and still takes writes, and every one of them is into a file no name reaches. + /// + [Fact] + public void A_file_unlinked_under_the_handle_no_longer_stands_at_the_name() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var path = Path.Combine(dir, "out.jsonl"); + using var held = Claim(path); + File.Delete(path); + + Assert.False(HeldEntry.StillStandsAt(held, path)); + } + finally { Directory.Delete(dir, true); } + } + + /// + /// Renamed away, which leaves the handle's own inode with a name - just not this one. Nothing + /// about the file has changed and the name reaches nothing. + /// + [Fact] + public void A_file_renamed_away_no_longer_stands_at_the_name() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var path = Path.Combine(dir, "out.jsonl"); + using var held = Claim(path); + File.Move(path, Path.Combine(dir, "elsewhere.jsonl")); + + Assert.False(HeldEntry.StillStandsAt(held, path)); + } + finally { Directory.Delete(dir, true); } + } + + /// + /// A second run's file standing at the name instead, with this run's own inode still named + /// somewhere else: the one case a link count cannot answer. The name exists, the handle is + /// open over a file with a name on it, and they are two different files - which is exactly + /// the state a promotion whose claim was granted late came away in. + /// + [Fact] + public void Another_file_at_the_name_is_not_the_one_the_handle_holds() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var path = Path.Combine(dir, "out.jsonl"); + using var held = Claim(path); + File.Move(path, Path.Combine(dir, "elsewhere.jsonl")); + File.WriteAllText(path, "{\"id\":\"theirs\"}\n"); + + Assert.False(HeldEntry.StillStandsAt(held, path)); + } + finally { Directory.Delete(dir, true); } + } + + /// + /// And the seam the deterministic tests of the rule are written against fires with the path + /// that was claimed, in the instant before the question is asked. One static for the whole + /// process, so it hears every claim any run in it takes: what a test written against it may + /// say anything about is its own name, which is what this counts. + /// + [Fact] + public void The_seam_fires_with_the_path_the_claim_was_taken_on() + { + var dir = NewDir(); + var seen = new List(); + try + { + var path = Path.Combine(dir, "out.jsonl"); + using var held = Claim(path); + HeldEntry.BeforeTheClaimIsVerified = claimed => + { + if (string.Equals(claimed, path, StringComparison.Ordinal)) seen.Add(claimed); + }; + + Assert.True(HeldEntry.StillStandsAt(held, path)); + Assert.Equal([path], seen); + } + finally + { + HeldEntry.BeforeTheClaimIsVerified = null; + Directory.Delete(dir, true); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/NamedTempPromotionTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/NamedTempPromotionTests.cs index 891ec81..5180acc 100644 --- a/tests/Mgx.IntegrationTests/Engine/Pagination/NamedTempPromotionTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/NamedTempPromotionTests.cs @@ -11,21 +11,86 @@ namespace Mgx.IntegrationTests.Engine.Pagination; /// found holding is an interrupted run's leftovers, and one that cannot be claimed belongs to a /// run that is still writing it. /// +[Collection("ClaimSeam")] public class NamedTempPromotionTests { + // An instance method, because what follows a promotion is an append onto the file it landed + // in: the run keeps that file from the move that puts the rows there until its writing ends, + // and the handle lives on the cmdlet. private static readonly MethodInfo Promote = typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( - "TryPromoteNamedTemp", BindingFlags.Static | BindingFlags.NonPublic)!; + "TryPromoteNamedTemp", BindingFlags.Instance | BindingFlags.NonPublic)!; private static readonly MethodInfo CanPromote = typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( "CanPromoteNamedTemp", BindingFlags.Static | BindingFlags.NonPublic)!; - private static bool Invoke(string outputPath, string tempFileName, long dataLength) => - (bool)Promote.Invoke(null, [outputPath, tempFileName, dataLength])!; + /// + /// The route a resumed run takes the output by, which is the shortest way to a claim whose + /// caller lets the handle go on its own path rather than in a finally: it asks for the file, + /// reads the claim's answer, and keeps nothing where the answer is not Taken. + /// + private static readonly MethodInfo Resume = + typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( + "TakeOutputForCheckpoint", BindingFlags.Instance | BindingFlags.NonPublic)!; + + /// + /// A directory a run may write in, which is how the tests below find every directory and how + /// they leave it. + /// + private const UnixFileMode Searchable = + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute; + + /// + /// And one that answers for every name in it and takes no new file: the mode a create is + /// refused by, with the stat of the name it was refused at still answering. + /// + private const UnixFileMode ReadAndSearch = UnixFileMode.UserRead | UnixFileMode.UserExecute; + + /// + /// The mode on a directory. Set through here rather than at each site, because the claim + /// seam below is a lambda and a lambda is its own body to the platform analyzer: it cannot + /// see the Windows guard the test opened with, and the guard has to be in front of the call. + /// + private static void SetMode(string dir, UnixFileMode mode) + { + if (OperatingSystem.IsWindows()) return; + File.SetUnixFileMode(dir, mode); + } + + /// + /// The promotion's answer, by name: the enum it returns is protected on the cmdlet base, so + /// a test outside that hierarchy cannot name the type. "Taken" is a promotion that landed + /// and kept the output it landed in; every other answer left both files where they were. + /// + private static string Invoke(Mgx.Cmdlets.Base.MgxCmdletBase run, + string outputPath, string tempFileName, long dataLength) => + Promote.Invoke(run, Slots(Promote, outputPath, tempFileName, dataLength))!.ToString()!; + + /// + /// The three arguments, followed by an empty slot for each out parameter the method carries. + /// Reflection wants a slot for every one of them and this suite asks about them by position + /// rather than by count - the reason a refusal carries at 3, the staging failure at 4 - + /// so the array is sized off the method itself. + /// + private static object?[] Slots(MethodInfo method, string outputPath, string tempFileName, + long dataLength) + { + var slots = new object?[method.GetParameters().Length]; + slots[0] = outputPath; + slots[1] = tempFileName; + slots[2] = dataLength; + return slots; + } - private static bool InvokeCan(string outputPath, string tempFileName, long dataLength) => - (bool)CanPromote.Invoke(null, [outputPath, tempFileName, dataLength])!; + /// + /// The preview's answer, by the same name. It reports what the applying pass would answer - + /// which of "Taken" and "NotPromoted" a temp claim gives it, and the two refusals the output + /// standing at the path gives it - and promotes nothing to find out. + /// + private static string InvokeCan(string outputPath, string tempFileName, long dataLength) => + CanPromote.Invoke(null, Slots(CanPromote, outputPath, tempFileName, dataLength))! + .ToString()!; private static string NewDir() { @@ -34,6 +99,24 @@ private static string NewDir() return d; } + /// + /// Reads back a file the run standing in for a second export still holds open to write it. + /// Windows checks the sharing both ways round - what the reader asks of the file, and what + /// the reader will let a handle already on it go on doing - so File.ReadAllLines, which asks + /// for FileShare.Read and nothing more, is refused by the writer this test put there itself. + /// The sharing the holder asked for is offered back, which is what a reader of a file a live + /// run is appending to has to do on either platform. + /// + private static string[] ReadSharedLines(string path) + { + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, + FileShare.ReadWrite); + using var reader = new StreamReader(stream); + var lines = new List(); + while (reader.ReadLine() is { } line) lines.Add(line); + return [.. lines]; + } + /// /// The state a crash leaves: the previous run's output, this run's items in the temp beside /// it, and a checkpoint naming both. The temp replaces the output, trimmed to the length @@ -53,8 +136,14 @@ public void Promotes_a_named_temp_nothing_holds() var counted = new FileInfo(temp).Length; File.AppendAllText(temp, "{\"id\":\"after-the-flush\"}\n"); - Assert.True(InvokeCan(output, Path.GetFileName(temp), counted)); - Assert.True(Invoke(output, Path.GetFileName(temp), counted)); + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + Assert.Equal("Taken", InvokeCan(output, Path.GetFileName(temp), counted)); + Assert.Equal("Taken", Invoke(run, output, Path.GetFileName(temp), counted)); + + // The rows are read once the run that took the file lets it go, which is what + // disposing it does: while the hold stands, a reader of that file is refused. + Assert.ThrowsAny(() => File.ReadAllLines(output)); + run.Dispose(); Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); Assert.Empty(Directory.GetFiles(dir, "*.tmp")); @@ -88,9 +177,13 @@ public void Refuses_a_named_temp_a_live_run_still_holds() var counted = new FileInfo(live).Length; Assert.True(counted > 0); - Assert.False(InvokeCan(output, Path.GetFileName(live), counted)); - Assert.False(Invoke(output, Path.GetFileName(live), counted)); + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + Assert.Equal("NotPromoted", InvokeCan(output, Path.GetFileName(live), counted)); + Assert.Equal("NotPromoted", + Invoke(run, output, Path.GetFileName(live), counted)); + // Nothing was promoted, so nothing was taken either: the output is the previous + // run's, unheld, and readable. Assert.Equal(["{\"id\":\"old\"}"], File.ReadAllLines(output)); Assert.False(File.Exists($"{output}.adopt")); Assert.True(File.Exists(live), "a running export's temp was promoted out from under it"); @@ -105,4 +198,1584 @@ public void Refuses_a_named_temp_a_live_run_still_holds() } finally { Directory.Delete(dir, true); } } + + /// + /// The third file. A promotion replaces the output and then appends onto what it replaced, + /// so the output another run is holding is one it must not replace at all - and a rename + /// asks nothing of the lock: the move went through, the holder's next write landed in an + /// inode with no name on it, and the claim taken afterwards answered Taken about a file one + /// statement old. The output is asked for before the move now, and a held one is the whole + /// answer. + /// + [Fact] + public void Refuses_an_output_another_run_holds_and_replaces_nothing() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"theirs\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBefore = File.ReadAllBytes(temp); + + // The run that has it: appending, sharing reads, the way a resumed run holds it. + using var live = new StreamWriter( + new FileStream(output, FileMode.Open, FileAccess.ReadWrite, FileShare.Read)) + { AutoFlush = true }; + live.BaseStream.Seek(0, SeekOrigin.End); + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + Assert.Equal("OutputHeld", InvokeCan(output, Path.GetFileName(temp), counted)); + Assert.Equal("OutputHeld", Invoke(run, output, Path.GetFileName(temp), counted)); + + // Nothing was replaced, nothing was staged, and the temp the checkpoint names is + // still there with the checkpoint's own bytes in it. + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.False(File.Exists($"{output}.adopt")); + + // The same inode, not a new file of the same name: the holder's next write is read + // back through the path. + live.WriteLine("{\"id\":\"theirs-2\"}"); + Assert.Equal( + ["{\"id\":\"theirs\"}", "{\"id\":\"theirs-2\"}"], + ReadSharedLines(output)); + } + finally { Directory.Delete(dir, true); } + } + + /// + /// The other refusal, which nothing could reach before: an output path that is not a file + /// this run can write at an offset. A FIFO opens read-write and answers no length, and the + /// move would have replaced it with a regular file - so the caller's pipe, and whatever is + /// reading it, would be gone. + /// + [Fact] + public void Refuses_an_output_it_cannot_open_and_replaces_nothing() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + try + { + using (var mkfifo = System.Diagnostics.Process.Start("/usr/bin/mkfifo", [output])) + { + Assert.NotNull(mkfifo); + mkfifo.WaitForExit(); + Assert.Equal(0, mkfifo.ExitCode); + } + + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBefore = File.ReadAllBytes(temp); + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + Assert.Equal("OutputUnopenable", InvokeCan(output, Path.GetFileName(temp), counted)); + Assert.Equal("OutputUnopenable", Invoke(run, output, Path.GetFileName(temp), counted)); + + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.False(File.Exists($"{output}.adopt")); + + // Still a pipe, and still nobody's: a stream with no offset in it is what says so, + // and a claim of this test's own would be refused if the run had kept one. + using var stillAFifo = new FileStream(output, FileMode.Open, FileAccess.ReadWrite, + FileShare.None); + Assert.False(stillAFifo.CanSeek, "the move replaced the caller's FIFO with a file"); + } + finally + { + try { File.Delete(output); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A temp big enough that the copy beside the output takes long enough to be watched, so a + /// question can be asked of the output path while the promotion is in the middle of staging. + /// + private static long WriteTempOfAtLeast(string temp, long bytes) + { + const string row = "{\"id\":\"0000000000000000000000000000000000000000000000000000\"}\n"; + var block = string.Concat(Enumerable.Repeat(row, 1024)); + using (var writer = new StreamWriter(temp, append: false)) + while (writer.BaseStream.Length < bytes) + writer.Write(block); + return new FileInfo(temp).Length; + } + + /// + /// The interval a claim released in front of the rename left open. The promotion asked for + /// the standing output and let it go, and only then renamed over it - and rename(2) honors no + /// lock on either side of it, so a run that took the output in between had it replaced under + /// it: its rows in an inode with no name left on them, and no warning on any stream. Widened + /// to 600 ms by a gate, a second process took the output there. + /// + /// The handle is kept now, from before the copy is staged through the move, so the interval + /// is one nothing can claim the output in. Measured over the whole of it: the staging file + /// appearing is what says the promotion has begun, and from that instant until it answers, + /// every claim on the output is refused. flock belongs to the open file description, so a + /// second open from this very process is refused exactly as another process's is - which is + /// also why the promotion never asks for the output again on the far side of the move. + /// + [Fact] + public void Nothing_can_claim_the_output_between_the_standing_claim_and_the_move() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + using var promoting = new ManualResetEventSlim(false); + Thread? racer = null; + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"theirs\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + var counted = WriteTempOfAtLeast(temp, 64L * 1024 * 1024); + var adopt = $"{output}.adopt"; + + // Spins on the output for as long as the promotion is staging or moving, and comes + // back with whether the path was ever anybody's to take. It starts asking when the + // staged file appears, which is the first instant a promotion has reached. + var claimed = false; + racer = new Thread(() => + { + while (!File.Exists(adopt) && !promoting.IsSet) Thread.SpinWait(64); + while (!promoting.IsSet) + { + try + { + using var mine = new FileStream(output, FileMode.Open, + FileAccess.ReadWrite, FileShare.None); + // A claim granted has to be a claim on the output, and one taken here + // is not always: the open and the lock behind it are two syscalls, and + // a thread descheduled between them comes back holding the inode the + // rename has since unnamed - a file at no path, whose lock the + // promotion let go on its way out. The staged file still standing and + // the claimed length still the path's are what say otherwise. + if (!File.Exists(adopt) || mine.Length != new FileInfo(output).Length) + continue; + claimed = true; + return; + } + catch (IOException) { } + catch (UnauthorizedAccessException) { } + } + }); + racer.Start(); + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var answer = Invoke(run, output, Path.GetFileName(temp), counted); + promoting.Set(); + Assert.True(racer.Join(TimeSpan.FromSeconds(30)), "the racing claim never finished"); + + Assert.Equal("Taken", answer); + Assert.False(claimed, + "the output was claimable between the standing claim and the rename"); + + // And the run that promoted has it: the rows are readable once it lets go. + run.Dispose(); + Assert.Equal(counted, new FileInfo(output).Length); + Assert.Empty(Directory.GetFiles(dir, "*.tmp")); + Assert.False(File.Exists(adopt)); + } + finally + { + promoting.Set(); + racer?.Join(TimeSpan.FromSeconds(30)); + Directory.Delete(dir, true); + } + } + + /// + /// A promotion that stops on the output staged the whole temp beside it first and then took + /// the copy back - 172 ms and a moved directory mtime for a 256 MiB temp, for a run that + /// replaces nothing. The output is asked for before any of it now, so a stop costs the + /// directory beside it nothing at all: no entry appears under the staging name, and the + /// directory's own mtime is where the run found it. + /// + [Fact] + public void A_held_output_stops_the_promotion_before_a_byte_is_staged() + { + var dir = NewDir(); + using var stopped = new ManualResetEventSlim(false); + Thread? watcher = null; + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"theirs\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + var counted = WriteTempOfAtLeast(temp, 64L * 1024 * 1024); + var adopt = $"{output}.adopt"; + var tempBefore = new FileInfo(temp).Length; + + // The run that has the output: appending, sharing reads, the way a resumed run holds + // it. Its claim is what the promotion stops on. + using var live = new StreamWriter( + new FileStream(output, FileMode.Open, FileAccess.ReadWrite, FileShare.Read)) + { AutoFlush = true }; + live.BaseStream.Seek(0, SeekOrigin.End); + + // Whether the staging name is ever on disk. A create and a delete inside one tick + // would leave the directory's mtime moved and nothing to see in it, so the entry is + // watched for as well as measured after. + var sawStaging = false; + watcher = new Thread(() => + { + while (!stopped.IsSet) + { + if (File.Exists(adopt)) { sawStaging = true; return; } + Thread.SpinWait(64); + } + }); + watcher.Start(); + + var directoryBefore = Directory.GetLastWriteTimeUtc(dir); + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var answer = Invoke(run, output, Path.GetFileName(temp), counted); + stopped.Set(); + Assert.True(watcher.Join(TimeSpan.FromSeconds(30)), "the staging watch never finished"); + + Assert.Equal("OutputHeld", answer); + Assert.False(sawStaging, "the stop staged a copy of the temp beside the output"); + Assert.Equal(directoryBefore, Directory.GetLastWriteTimeUtc(dir)); + Assert.False(File.Exists(adopt)); + Assert.Equal(tempBefore, new FileInfo(temp).Length); + + // The same inode, not a new file wearing the name. + live.WriteLine("{\"id\":\"theirs-2\"}"); + Assert.Equal(["{\"id\":\"theirs\"}", "{\"id\":\"theirs-2\"}"], ReadSharedLines(output)); + } + finally + { + stopped.Set(); + watcher?.Join(TimeSpan.FromSeconds(30)); + Directory.Delete(dir, true); + } + } + + /// + /// Which errno the claim reads as another run holding the file. .NET backs FileShare.None on + /// Unix with a non-blocking flock, and a refused LOCK_EX comes back EWOULDBLOCK - 35 on + /// macOS, 11 on Linux, where the name EAGAIN shares that value. Read as a union of the two, + /// each platform also read its own EDEADLK as a sharing violation: 11 is EDEADLK on macOS + /// and 35 is EDEADLK on Linux, so a deadlock the kernel had just refused to enter was + /// reported as an export to wait for. Each number is read on the platform that uses it. + /// + [Fact] + public void The_sharing_errno_is_the_one_this_platform_uses() + { + var isSharingViolation = typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( + "IsSharingViolation", BindingFlags.Static | BindingFlags.NonPublic)!; + bool Reads(int hResult) => + (bool)isSharingViolation.Invoke(null, [new IOException("errno", hResult)])!; + + Assert.Equal(OperatingSystem.IsMacOS(), Reads(35)); + Assert.Equal(OperatingSystem.IsLinux(), Reads(11)); + + // Windows carries the win32 code in the low half of an HRESULT, and those two are read + // wherever they turn up. + Assert.True(Reads(unchecked((int)0x80070020))); + Assert.True(Reads(unchecked((int)0x80070021))); + + // And the failures that are not a second run at all: ELOOP, ENOTSUP for a socket, and + // the win32-shaped HRESULT .NET maps an over-long path to by hand. + Assert.False(Reads(62)); + Assert.False(Reads(45)); + Assert.False(Reads(unchecked((int)0x800700CE))); + } + + /// + /// The staging failure a promotion answers with, beside its answer: the reason a caller + /// quotes in the stop it writes. + /// + private static (string Answer, string Reason) InvokeForStaging( + Mgx.Cmdlets.Base.MgxCmdletBase run, string outputPath, string tempFileName, long dataLength) + { + var args = Slots(Promote, outputPath, tempFileName, dataLength); + var answer = Promote.Invoke(run, args)!.ToString()!; + return (answer, args[4]?.ToString() ?? ""); + } + + /// + private static (string Answer, string Reason) InvokeCanForStaging( + string outputPath, string tempFileName, long dataLength) + { + var args = Slots(CanPromote, outputPath, tempFileName, dataLength); + var answer = CanPromote.Invoke(null, args)!.ToString()!; + return (answer, args[4]?.ToString() ?? ""); + } + + private static void Mkfifo(string path) + { + using var mkfifo = System.Diagnostics.Process.Start("/usr/bin/mkfifo", [path]); + Assert.NotNull(mkfifo); + mkfifo.WaitForExit(); + Assert.Equal(0, mkfifo.ExitCode); + } + + private static void Symlink(string target, string link) + { + using var ln = System.Diagnostics.Process.Start("/bin/ln", ["-s", target, link]); + Assert.NotNull(ln); + ln.WaitForExit(); + Assert.Equal(0, ln.ExitCode); + } + + /// + /// A second directory entry for a file's own inode. Not a symlink: nothing on the entry says + /// it is an alias, and unlinking it takes the name and leaves the file. + /// + private static void HardLink(string target, string link) + { + using var ln = System.Diagnostics.Process.Start("/bin/ln", [target, link]); + Assert.NotNull(ln); + ln.WaitForExit(); + Assert.Equal(0, ln.ExitCode); + } + + /// + /// A FIFO at the staging name. The copy was staged with FileMode.Create and + /// FileAccess.Write, which is O_WRONLY, and open(2) on a FIFO with no reader blocks until + /// one arrives - so the promotion never returned, holding the temp's claim and the output's + /// lock for as long as the process lived, with StopProcessing unable to reach it and a + /// second run refused on the output. The name is swept before it is created now, and a FIFO + /// standing there is one of the kinds that goes. + /// + [Fact] + public void Sweeps_a_fifo_off_the_staging_name_and_promotes_without_blocking() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var adopt = $"{output}.adopt"; + Mkfifo(adopt); + + // On its own thread with a watchdog: the failure this pins is a promotion that does + // not come back at all, and a test that called it inline would hang the suite. + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + string? answer = null; + var promoting = new Thread(() => + { + try { answer = Invoke(run, output, Path.GetFileName(temp), counted); } + catch (Exception ex) { answer = $"threw {ex.GetType().Name}"; } + }) { IsBackground = true }; + promoting.Start(); + Assert.True(promoting.Join(TimeSpan.FromSeconds(15)), + "the promotion never returned: the staging open is waiting for a reader on the FIFO"); + + Assert.Equal("Taken", answer); + run.Dispose(); + + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + Assert.False(File.Exists(adopt)); + Assert.Empty(Directory.GetFiles(dir, "*.tmp")); + + // A regular file at the output path, not the pipe that stood beside it: a seekable + // handle is what says so. + using var landed = new FileStream(output, FileMode.Open, FileAccess.Read); + Assert.True(landed.CanSeek); + } + finally + { + try { File.Delete($"{Path.Combine(dir, "out.jsonl")}.adopt"); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A symlink at the staging name, pointing at a file outside the output's directory. The + /// staging followed it: the copy went into the link's target, truncating a file no export + /// had ever written, and the rename then moved the LINK onto the output path - so the + /// output was a symlink to somebody's file and both passes answered as though the promotion + /// had landed. The sweep takes the link off the name, and never what it pointed at. + /// + [Fact] + public void Sweeps_a_symlink_off_the_staging_name_and_leaves_its_target_alone() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var elsewhere = NewDir(); + try + { + var victim = Path.Combine(elsewhere, "precious.conf"); + var victimBytes = "KEEP-THIS-EXACTLY\n"u8.ToArray(); + File.WriteAllBytes(victim, victimBytes); + + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var adopt = $"{output}.adopt"; + Symlink(victim, adopt); + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + Assert.Equal("Taken", Invoke(run, output, Path.GetFileName(temp), counted)); + run.Dispose(); + + Assert.Equal(victimBytes, File.ReadAllBytes(victim)); + Assert.True(File.Exists(victim), "the sweep deleted what the link pointed at"); + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + Assert.Null(new FileInfo(output).LinkTarget); + Assert.False(File.Exists(adopt)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + try { Directory.Delete(elsewhere, true); } catch { } + } + } + + /// + /// A dangling link at the staging name - the kind the sweep answered Absent about and left + /// standing, and the create then followed to a name in a directory that may not even be + /// there. Read off the entry rather than through an open, so a link with nothing behind it + /// is still a link. + /// + [Fact] + public void Sweeps_a_dangling_link_off_the_staging_name() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var adopt = $"{output}.adopt"; + Symlink(Path.Combine(dir, "no-such-file"), adopt); + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + Assert.Equal("Taken", Invoke(run, output, Path.GetFileName(temp), counted)); + run.Dispose(); + + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + Assert.False(File.Exists(Path.Combine(dir, "no-such-file")), + "the create made the file the dangling link pointed at"); + Assert.False(File.Exists(adopt)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// A directory at the staging name, which is the one kind the sweep does not remove: it is + /// not this run's to delete and could hold anything. Both passes stop there, and both say a + /// directory - the applying pass built its reason from the raw exception, so the same disk + /// state read "Access to the path ... is denied" in the run and "a directory stands at the + /// path" in the preview. + /// + [Fact] + public void A_directory_at_the_staging_name_stops_both_passes_saying_a_directory() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + var outputBytes = "{\"id\":\"old\"}\n"u8.ToArray(); + File.WriteAllBytes(output, outputBytes); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + var adopt = $"{output}.adopt"; + Directory.CreateDirectory(adopt); + File.WriteAllText(Path.Combine(adopt, "somebody-elses"), "keep\n"); + + var preview = InvokeCanForStaging(output, Path.GetFileName(temp), counted); + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + + Assert.Equal("StagingFailed", preview.Answer); + Assert.Equal("StagingFailed", applied.Answer); + Assert.Contains("a directory stands at the path", preview.Reason); + Assert.Equal(preview.Reason, applied.Reason); + + // The one wording carries no sentence-ending period of its own: the caller quotes + // it inside a sentence it punctuates itself. + Assert.DoesNotContain("denied", applied.Reason); + Assert.False(applied.Reason.TrimEnd().EndsWith('.')); + + // Nothing moved: not the output, not the temp, and not what the directory held. + Assert.Equal(outputBytes, File.ReadAllBytes(output)); + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + Assert.True(Directory.Exists(adopt)); + Assert.Equal("keep\n", File.ReadAllText(Path.Combine(adopt, "somebody-elses"))); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// A second run's handle on the staged copy. The discard was guarded on this run having + /// opened the name for writing, and FileMode.Create opened whatever stood there - so a + /// promotion that stopped unlinked the copy another run was holding across its own rename, + /// leaving that run writing into a file at no path. The create refuses a name that already + /// exists now, so the promotion stops before it has anything to discard. + /// + [Fact] + public void A_held_staging_name_stops_the_promotion_and_keeps_the_holder_s_file() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var adopt = $"{output}.adopt"; + + // The other run, holding the copy it has just written the way a promotion holds it + // across the rename. + using var holder = new FileStream(adopt, FileMode.CreateNew, FileAccess.ReadWrite, + FileShare.None); + holder.Write("{\"id\":\"theirs\"}\n"u8); + holder.Flush(); + var holderBytes = holder.Length; + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + + Assert.Equal("StagingFailed", applied.Answer); + Assert.Contains("another run has the staged copy open", applied.Reason); + + // The holder still has a file, at the name it wrote it under, with its own bytes. + Assert.True(File.Exists(adopt), "the stop unlinked the copy another run was holding"); + Assert.Equal(holderBytes, holder.Length); + + // Read back through the handle, since it is the only thing that can read the file: + // the hold is FileShare.None, which is what a promotion holds its copy under. + var theirs = new byte[holderBytes]; + holder.Position = 0; + Assert.Equal((int)holderBytes, holder.Read(theirs)); + Assert.Equal("{\"id\":\"theirs\"}\n"u8.ToArray(), theirs); + Assert.Equal(["{\"id\":\"old\"}"], File.ReadAllLines(output)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The other route that stages under the same name: the adoption of a temp left by a run + /// whose checkpoint predates the recorded name and length. It wrote its copy with + /// StreamWriter(path, append: false), which truncates whatever stands at the name and + /// follows a link there like every other open of a path. + /// + [Fact] + public void Adoption_sweeps_a_symlinked_staging_name_and_leaves_its_target_alone() + { + if (OperatingSystem.IsWindows()) return; + + var adopt = typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( + "TryAdoptOrphanedTemp", BindingFlags.Static | BindingFlags.NonPublic)!; + + var dir = NewDir(); + var elsewhere = NewDir(); + try + { + var victim = Path.Combine(elsewhere, "precious.conf"); + var victimBytes = "KEEP-THIS-EXACTLY\n"u8.ToArray(); + File.WriteAllBytes(victim, victimBytes); + + // No output at all, which is the only state adoption runs in. + var output = Path.Combine(dir, "out.jsonl"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + Symlink(victim, $"{output}.adopt"); + + var slots = new object?[adopt.GetParameters().Length]; + slots[0] = output; + slots[1] = 2L; + Assert.True((bool)adopt.Invoke(null, slots)!); + + Assert.Equal(victimBytes, File.ReadAllBytes(victim)); + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + Assert.Null(new FileInfo(output).LinkTarget); + Assert.False(File.Exists($"{output}.adopt")); + Assert.False(File.Exists(temp)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + try { Directory.Delete(elsewhere, true); } catch { } + } + } + + private static void Chmod(string mode, string path) + { + using var chmod = System.Diagnostics.Process.Start("/bin/chmod", [mode, path]); + Assert.NotNull(chmod); + chmod.WaitForExit(); + Assert.Equal(0, chmod.ExitCode); + } + + /// + /// An output directory this account may read and not write. The staging is a create in that + /// directory and a real run stops on it, and the preview only asked whether something + /// already stood at the staging name: it found nothing, answered that the run would recover + /// the items, and the run then stopped on the write the gate had just promised. The preview + /// makes the create too, at no length, and both passes fail on the same open with the same + /// reason. + /// + [Fact] + public void A_read_only_output_directory_stops_both_passes_on_the_create() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + var outputBytes = "{\"id\":\"old\"}\n"u8.ToArray(); + File.WriteAllBytes(output, outputBytes); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + var adopt = $"{output}.adopt"; + var entriesBefore = Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray(); + Chmod("555", dir); + + var preview = InvokeCanForStaging(output, Path.GetFileName(temp), counted); + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + + Assert.Equal("StagingFailed", preview.Answer); + Assert.Equal("StagingFailed", applied.Answer); + Assert.Equal(preview.Reason, applied.Reason); + Assert.Contains(adopt, applied.Reason); + + Chmod("755", dir); + Assert.Equal(outputBytes, File.ReadAllBytes(output)); + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + Assert.False(File.Exists(adopt)); + Assert.Equal(entriesBefore, Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray()); + } + finally + { + try { Chmod("755", dir); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The reason both passes carry has no sentence-ending period of its own, whichever site + /// built it: the caller quotes it inside a sentence and punctuates it there, and a + /// permission failure's runtime text ends in one - so the stop read "...is denied.. The + /// resume checkpoint..." wherever it was not taken off. One refinement reads the runtime's + /// words now, for the claim on a file and for the create a promotion stages with alike. + /// + [Fact] + public void The_staging_reason_carries_no_period_of_its_own() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + Chmod("555", dir); + + var preview = InvokeCanForStaging(output, Path.GetFileName(temp), counted); + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + Chmod("755", dir); + + // The record prints as "StagingFailure { Path = ..., Reason = ... }", so the reason + // is read out of it rather than off the end of the whole line. + const string opening = "Reason = "; + foreach (var (pass, printed) in new[] + { ("preview", preview.Reason), ("run", applied.Reason) }) + { + var at = printed.LastIndexOf(opening, StringComparison.Ordinal); + Assert.True(at >= 0, $"the {pass} carried no staging reason: {printed}"); + var reason = printed[(at + opening.Length)..].TrimEnd('}', ' '); + Assert.NotEmpty(reason); + Assert.False(reason.EndsWith('.'), + $"the {pass}'s staging reason ends in a period of its own: '{reason}'"); + } + } + finally + { + try { Chmod("755", dir); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// What the preview leaves at the staging name where the promotion would have gone through: + /// nothing. It makes the create a run makes - which is what tells it a directory it cannot + /// write in from one it can - and takes the entry straight back, so the directory holds + /// what it held and the two files the checkpoint stands for are untouched. + /// + [Fact] + public void The_preview_leaves_no_staged_copy_behind() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + var outputBytes = "{\"id\":\"old\"}\n"u8.ToArray(); + File.WriteAllBytes(output, outputBytes); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + var adopt = $"{output}.adopt"; + var entriesBefore = Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray(); + + Assert.Equal("Taken", InvokeCan(output, Path.GetFileName(temp), counted)); + + Assert.False(File.Exists(adopt)); + Assert.Equal(entriesBefore, Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray()); + Assert.Equal(outputBytes, File.ReadAllBytes(output)); + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + + // And the promotion the preview described still lands afterwards. + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + Assert.Equal("Taken", Invoke(run, output, Path.GetFileName(temp), counted)); + run.Dispose(); + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// A hard link at the staging name, standing for the output the promotion is about to + /// replace. Nothing on the entry says it is an alias - a hard link is a name, not a link - + /// so what the sweep asks about is the output's own inode, and the run had already claimed + /// that inode by the time it asked: a second open file description over a locked file is + /// refused even from this process, so the alias answered "another run has the staged copy + /// open" and the promotion stopped on a name nothing but itself was holding. The sweep runs + /// before either claim now, so the alias answers its claim and comes off the name - in the + /// run, which is the pass that clears the name; the preview names it and leaves it there. + /// + [Fact] + public void Sweeps_a_hard_link_to_the_output_off_the_staging_name_and_promotes() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + var outputBytes = "{\"id\":\"old\"}\n"u8.ToArray(); + File.WriteAllBytes(output, outputBytes); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + var adopt = $"{output}.adopt"; + + HardLink(output, adopt); + Assert.Equal("Taken", InvokeCan(output, Path.GetFileName(temp), counted)); + + // The preview holds neither file, so the alias answers a claim rather than reading + // as a staged copy another run has open - and it is left standing, since a preview + // removes nothing. What it names is where it was either way. + Assert.True(File.Exists(adopt), "the preview removed what a run would remove"); + Assert.Equal(outputBytes, File.ReadAllBytes(output)); + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + + Assert.Equal("Taken", applied.Answer); + Assert.Equal("", applied.Reason); + Assert.False(File.Exists(adopt)); + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + Assert.Empty(Directory.GetFiles(dir, "*.tmp")); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The same alias, one file over: a hard link at the staging name for the temp the + /// checkpoint names. The claim on the temp is the first one a promotion takes and the one it + /// holds longest, so this is the shape the old order refused hardest - and the temp itself + /// is never what the sweep removes, only the extra name. + /// + [Fact] + public void Sweeps_a_hard_link_to_the_temp_off_the_staging_name_and_promotes() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + var outputBytes = "{\"id\":\"old\"}\n"u8.ToArray(); + File.WriteAllBytes(output, outputBytes); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + var adopt = $"{output}.adopt"; + + HardLink(temp, adopt); + Assert.Equal("Taken", InvokeCan(output, Path.GetFileName(temp), counted)); + + Assert.True(File.Exists(adopt), "the preview removed what a run would remove"); + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + Assert.Equal(outputBytes, File.ReadAllBytes(output)); + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + + Assert.Equal("Taken", applied.Answer); + Assert.Equal("", applied.Reason); + Assert.False(File.Exists(adopt)); + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + Assert.Empty(Directory.GetFiles(dir, "*.tmp")); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// Two promotions over one output, released together, with a copy left at the staging name + /// by a run killed between the write and the rename. The sweep asked for that copy, let the + /// claim go, and unlinked the name afterwards - and what the other promotion does in between + /// is sweep the same copy, create its own at the name and hold it, so the stale unlink took + /// the second run's staged file out from under its handle and that run stopped on a name + /// that was no longer there. The claim is held across the unlink now, which refuses the + /// second sweep outright: whichever run gets there second either lands its own promotion or + /// stops with its temp where it was, and neither loses a file to the other. + /// + [Fact] + public void Two_promotions_over_one_leftover_never_unlink_each_others_files() + { + if (OperatingSystem.IsWindows()) return; + + for (var iteration = 0; iteration < 40; iteration++) + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var tempA = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + var tempB = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(tempA, "{\"id\":\"a1\"}\n{\"id\":\"a2\"}\n"); + File.WriteAllText(tempB, "{\"id\":\"b1\"}\n{\"id\":\"b2\"}\n"); + var countedA = new FileInfo(tempA).Length; + var countedB = new FileInfo(tempB).Length; + var adopt = $"{output}.adopt"; + File.WriteAllText(adopt, "LEFTOVER-FROM-AN-INTERRUPTED-PROMOTION\n"); + + var runA = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var runB = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + (string Answer, string Reason) a = default; + (string Answer, string Reason) b = default; + using var released = new Barrier(2); + var threads = new[] + { + new Thread(() => + { + released.SignalAndWait(); + a = InvokeForStaging(runA, output, Path.GetFileName(tempA), countedA); + }) { IsBackground = true }, + new Thread(() => + { + released.SignalAndWait(); + b = InvokeForStaging(runB, output, Path.GetFileName(tempB), countedB); + }) { IsBackground = true }, + }; + try + { + foreach (var t in threads) t.Start(); + foreach (var t in threads) + Assert.True(t.Join(TimeSpan.FromSeconds(30)), + "a promotion never returned"); + } + finally + { + runA.Dispose(); + runB.Dispose(); + } + + // One landed. Neither answer is a file that vanished: the run that stopped says + // so about a name another run holds or an output another run has, never about a + // copy of its own that is no longer there. + Assert.Single(new[] { a.Answer, b.Answer }.Where(x => x == "Taken")); + Assert.DoesNotContain("Could not find file", a.Reason); + Assert.DoesNotContain("Could not find file", b.Reason); + + // The output holds one run's rows and not a weave of both, the run that stopped + // still has the temp it would come back to, and the leftover is gone. + var landed = a.Answer == "Taken"; + Assert.Equal( + landed + ? ["{\"id\":\"a1\"}", "{\"id\":\"a2\"}"] + : ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], + File.ReadAllLines(output)); + Assert.True(File.Exists(landed ? tempB : tempA), + "the promotion that stopped lost the temp it was to come back to"); + Assert.False(File.Exists(adopt)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + } + + /// + /// A directory standing at the staging name where the create is what meets it, which is the + /// instant between the sweep and the create. The sweep calls that entry a directory; a + /// create refuses a name that already exists with the runtime's "already exists" whatever + /// kind of entry wears it, so one disk state came back in two wordings depending on which of + /// the two instructions reached it - and "already exists" reads as a file in the way, which + /// a caller answers by removing it. The refinement asks the filesystem on the create's route + /// too. + /// + [Fact] + public void A_directory_the_create_meets_reads_as_a_directory_and_not_as_a_file_in_the_way() + { + var reasonFor = typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( + "ReasonForUnopenable", BindingFlags.Static | BindingFlags.NonPublic)!; + + var dir = NewDir(); + try + { + var adopt = Path.Combine(dir, "out.jsonl.adopt"); + Directory.CreateDirectory(adopt); + + // The create a promotion stages its copy with, against the entry that appeared under + // it. The runtime's own words for it name no directory on either platform, and they + // are not the same words or even the same exception: a create meets a directory as a + // name that already exists on Unix and as an access failure on Windows. Which of the + // two the platform raises is not the question - both reach the refinement through + // the promotion's own catch, and what it makes of them is. + var raised = Assert.ThrowsAny(() => new FileStream( + adopt, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None)); + Assert.True(raised is IOException or UnauthorizedAccessException, + $"the create raised {raised.GetType()}, which the promotion does not catch"); + Assert.DoesNotContain("directory", raised.Message, StringComparison.OrdinalIgnoreCase); + + var slots = new object?[reasonFor.GetParameters().Length]; + slots[0] = adopt; + slots[1] = raised; + // Raised by a create, where the refinement is told which open it came from. + for (var i = 2; i < slots.Length; i++) slots[i] = true; + var refined = reasonFor.Invoke(null, slots)!.ToString()!; + + Assert.Contains("a directory stands at the path", refined); + Assert.DoesNotContain("already exists", refined); + + // And a failure from any other open still carries the runtime's words: the question + // the create's refusal cannot answer for itself is not asked of every exception. + slots[1] = new IOException("Too many levels of symbolic links."); + for (var i = 2; i < slots.Length; i++) slots[i] = false; + Assert.Contains("Too many levels of symbolic links", + reasonFor.Invoke(null, slots)!.ToString()!); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The claim a promotion takes on the output the move will replace, over an output that is + /// not there yet - which is the ordinary end of a fresh run interrupted into a temp. It + /// opened FileMode.Open and answered Absent with no handle at all, so nothing was held + /// through the staging or the rename: two runs with different temps over one output path both + /// got past this line, both renamed a copy onto it, and the first one's rows ended in an + /// unlinked inode with its own handle appending into a file at no path. The claim makes the + /// file in order to hold it now, and a second promotion is refused on it the way it is + /// refused on an output that was already there. + /// + [Fact] + public void The_claim_on_an_absent_output_makes_the_file_and_refuses_a_second_promotion() + { + if (OperatingSystem.IsWindows()) return; + + var claimStanding = typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( + "ClaimStandingOutput", BindingFlags.Static | BindingFlags.NonPublic)!; + + var dir = NewDir(); + FileStream? held = null; + try + { + // No output at all, and a temp for each of the two runs. + var output = Path.Combine(dir, "out.jsonl"); + var theirs = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(theirs, "{\"id\":\"t1\"}\n{\"id\":\"t2\"}\n"); + var theirsCounted = new FileInfo(theirs).Length; + var theirsBytes = File.ReadAllBytes(theirs); + var mine = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(mine, "{\"id\":\"m1\"}\n{\"id\":\"m2\"}\n"); + var mineCounted = new FileInfo(mine).Length; + Assert.False(File.Exists(output)); + + // The first run, stopped where its claim is taken: what it holds from before the + // staging through the rename, and nothing else of the promotion. + var slots = new object?[claimStanding.GetParameters().Length]; + slots[0] = output; + var answer = claimStanding.Invoke(null, slots)!.ToString()!; + held = (FileStream?)slots[1]; + var created = slots.Length > 3 && slots[3] is true; + + Assert.Equal("Taken", answer); + Assert.True(created, "the claim on an absent output reported making nothing"); + Assert.NotNull(held); + Assert.True(File.Exists(output), "the claim on an absent output made no file to hold"); + Assert.Equal(0, new FileInfo(output).Length); + + // The second run, over the same output path with its own temp, while the first is + // between its claim and its move. Refused on the output, with both of its files + // where it found them. + using (var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection()) + { + Assert.Equal("OutputHeld", Invoke(run, output, Path.GetFileName(theirs), + theirsCounted)); + } + + Assert.Equal(theirsBytes, File.ReadAllBytes(theirs)); + Assert.Equal(0, new FileInfo(output).Length); + Assert.False(File.Exists($"{output}.adopt")); + + // And once the first run lets go, a promotion lands in the file the claim made: the + // rename replaces it, so nothing of the empty one is left to remove. + held.Dispose(); + held = null; + using (var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection()) + { + Assert.Equal("Taken", Invoke(run, output, Path.GetFileName(mine), mineCounted)); + } + + Assert.Equal(["{\"id\":\"m1\"}", "{\"id\":\"m2\"}"], File.ReadAllLines(output)); + Assert.False(File.Exists(mine)); + } + finally + { + held?.Dispose(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// An output that is not there in a directory this account may read and not write. The claim + /// cannot make the file, and what refused it is the directory and not the file it would name + /// - so the answer is left to the staging create, in that same directory, which is the + /// question both passes ask and answer in one wording. Nothing of either pass is left at the + /// output path. + /// + [Fact] + public void A_read_only_directory_with_no_output_stops_both_passes_on_the_create() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + var entriesBefore = Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray(); + Chmod("555", dir); + + var preview = InvokeCanForStaging(output, Path.GetFileName(temp), counted); + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + Chmod("755", dir); + + Assert.Equal("StagingFailed", preview.Answer); + Assert.Equal("StagingFailed", applied.Answer); + Assert.Equal(preview.Reason, applied.Reason); + Assert.Contains($"{output}.adopt", applied.Reason); + + // The temp is where it was, no output was left standing at the path, and the + // directory holds exactly the entries it held. + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + Assert.False(File.Exists(output)); + Assert.Equal(entriesBefore, Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray()); + } + finally + { + try { Chmod("755", dir); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The interleaving two promotions over one output reach on Unix, at the instant that makes + /// it: the claim on the output is granted several syscalls after the open that resolved the + /// name - .NET takes the lock behind FileShare.None after open(2) - and a second run that + /// renames its own copy onto the name in between leaves this one holding an inode with no + /// name on it. Read as a claim, that is a promotion renaming its copy over a file another + /// run has just promoted, and both runs answering that they took the output. The seam below + /// is that instant, fired on every attempt, so the name never settles: the answer is the + /// held stop, and nothing at the path is replaced. + /// + [Fact] + public void An_output_replaced_under_the_claim_is_not_an_output_this_run_took() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + + // The second run's rename, landing in the instant between this run's claim being + // granted and the check that the name still reaches what it granted. A fresh file + // each time, so which one stands at the path says how many attempts were made. + var renames = 0; + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = claimed => + { + if (!string.Equals(claimed, output, StringComparison.Ordinal)) return; + renames++; + var theirs = Path.Combine(dir, $"theirs-{renames}.jsonl"); + File.WriteAllText(theirs, $"{{\"id\":\"theirs-{renames}\"}}\n"); + File.Move(theirs, output, overwrite: true); + }; + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + + // What the other run left at the path is whole, this run's temp is where it was, + // and no copy was staged beside either - the answer is the hold it is, and the + // claim was asked three times to reach it. + Assert.Equal("OutputHeld", applied.Answer); + Assert.Equal(["{\"id\":\"theirs-3\"}"], File.ReadAllLines(output)); + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + Assert.False(File.Exists($"{output}.adopt")); + Assert.Equal(3, renames); + } + finally + { + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = null; + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same instant in the sweep of the staging name, which is where the interleaving costs + /// the OTHER run its file: this run's claim on the leftover is granted after a second run + /// has taken that leftover off the name and created its own staged copy there, and the + /// unlink that follows a claim asks the directory - so it takes the second run's copy, and + /// that run's rename ends on a name that is no longer there. The sweep answers the hold it + /// met now, and the copy the other run is holding stays where it put it. + /// + [Fact] + public void A_staging_name_taken_under_the_sweep_s_claim_is_not_the_sweep_s_to_unlink() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + FileStream? theirs = null; + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + var adopt = $"{output}.adopt"; + File.WriteAllText(adopt, "LEFTOVER-FROM-AN-INTERRUPTED-PROMOTION\n"); + + // The second run, in that instant: it swept the same leftover, created its staged + // copy at the name and holds it there through the rename it is about to make. + var swept = 0; + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = claimed => + { + if (!string.Equals(claimed, adopt, StringComparison.Ordinal)) return; + if (swept++ > 0) return; + File.Delete(adopt); + theirs = new FileStream(adopt, FileMode.CreateNew, FileAccess.ReadWrite, + FileShare.None); + theirs.Write("{\"id\":\"theirs\"}\n"u8); + theirs.Flush(); + }; + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + + // The name is the other run's, which is the answer a name a second run holds always + // had here - and this run stopped on it with its own temp untouched. + Assert.Equal("StagingFailed", applied.Answer); + Assert.Contains("another run has the staged copy open", applied.Reason); + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + Assert.Equal(1, swept); + + // And the rename that copy is being held for lands, because the copy is still there. + File.Move(adopt, output, overwrite: true); + theirs!.Dispose(); + theirs = null; + Assert.Equal(["{\"id\":\"theirs\"}"], File.ReadAllLines(output)); + } + finally + { + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = null; + theirs?.Dispose(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The unlink at the end of a promotion that landed, which is the same question one file + /// later: the temp goes because this run read it, and what goes is the file this run holds - + /// not whatever wears its name by then. A second run that took the name in the interval + /// keeps its file. + /// + [Fact] + public void A_temp_replaced_at_its_name_is_not_the_temp_a_promotion_unlinks() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var moved = Path.Combine(dir, "moved-away.tmp"); + + // Two claims are taken on the temp's name in one promotion: the one it is read + // through, and the check in front of the unlink that spends it. The second run + // arrives between them. + var claims = 0; + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = claimed => + { + if (!string.Equals(claimed, temp, StringComparison.Ordinal)) return; + if (++claims != 2) return; + File.Move(temp, moved); + File.WriteAllText(temp, "{\"id\":\"theirs\"}\n"); + }; + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var answer = Invoke(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + + // The rows are in the output, and the file standing at the temp's name is the other + // run's, whole. The temp this promotion actually spent is where that run moved it, + // for the sweep that reaches it next. + Assert.Equal("Taken", answer); + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + Assert.Equal(["{\"id\":\"theirs\"}"], File.ReadAllLines(temp)); + Assert.True(File.Exists(moved)); + Assert.Equal(2, claims); + } + finally + { + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = null; + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And the empty output a promotion makes in order to hold it, taken back where the + /// promotion stops. That unlink is the same shape as the two above: what goes is the file + /// this claim made, and a second run standing at the name by then keeps what it put there. + /// + [Fact] + public void An_output_the_claim_made_is_not_taken_back_from_a_second_run() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + // Nothing at the output path, so the claim makes the file in order to hold it. + var output = Path.Combine(dir, "out.jsonl"); + var adopt = $"{output}.adopt"; + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + + var claims = 0; + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = claimed => + { + if (!string.Equals(claimed, output, StringComparison.Ordinal)) return; + if (++claims == 1) + { + // A directory at the staging name, in the instant after the sweep passed + // it: the create the copy is staged with meets it, and the promotion stops + // with the empty file it made still standing at the output path. + Directory.CreateDirectory(adopt); + return; + } + if (claims != 2) return; + + // And the second run, in the instant before that file is taken back: it is + // gone from the name and theirs stands there instead. + File.Delete(output); + File.WriteAllText(output, "{\"id\":\"theirs\"}\n"); + }; + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + + Assert.Equal(["{\"id\":\"theirs\"}"], File.ReadAllLines(output)); + Assert.Equal("StagingFailed", applied.Answer); + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + Assert.Equal(2, claims); + } + finally + { + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = null; + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A claim granted, and then a question about it the platform cannot answer: the output's + /// parent directory stops being searchable in the instant between the two, so the stat of + /// the name comes back EACCES - which is not the name resolving to nothing, and is not read + /// as "still there". A claim that cannot be verified is not a claim, so the caller is handed + /// a refusal in the shape it already reads every refusal in: the file could not be opened, + /// the reason says why, and the handle comes back with it for the caller to let go of the + /// way it lets go of one it kept. A resumed run asks that way, and keeps nothing. + /// + [Fact] + public void A_claim_the_platform_cannot_verify_is_refused_with_the_handle_handed_back() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var contents = File.ReadAllBytes(output); + + // The premise: a directory with no search bit answers for no name under it. Where + // the stat still answers, the account is root, a mode says nothing, and there is + // nothing here to measure. + SetMode(dir, 0); + var unsearchable = !File.Exists(output); + SetMode(dir, Searchable); + if (!unsearchable) return; + + var looks = 0; + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = claimed => + { + if (!string.Equals(claimed, output, StringComparison.Ordinal)) return; + looks++; + SetMode(dir, 0); + }; + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var args = new object?[] { output, (long)contents.Length, null }; + var answer = Resume.Invoke(run, args)!.ToString()!; + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = null; + SetMode(dir, Searchable); + + // The answer a run words its stop out of, and the reason it quotes in it: the call, + // the errno, and what could not be finished with them. + Assert.Equal("CannotBeOpened", answer); + Assert.Equal(1, looks); + Assert.Contains("cannot be verified", args[2]!.ToString()!); + + // The handle went with the answer, on the caller's own path. Asked for again the way + // every later claim asks for it, the file is granted at once - where a failure + // carried out of the claim with the stream still open left an exclusive lock on the + // caller's own output until the finalizer got around to it, with nothing + // deterministic able to say when. + using var next = new FileStream(output, FileMode.Open, FileAccess.ReadWrite, + FileShare.None); + Assert.Equal(contents.Length, next.Length); + } + finally + { + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = null; + try { SetMode(dir, Searchable); Directory.Delete(dir, true); } + catch { } + } + } + + /// + /// The same failed question at the one caller that lets its claim go in a finally. A + /// promotion over an output that is not there yet makes the file in order to hold it, and + /// takes that file away again on every answer but Taken - under the claim that made it, so + /// the name goes only while this run is still what stands at it. That take-back is armed by + /// the handle and by the claim's own answer about what it made, and a refusal that clears + /// either of them disarms it: the finally then holds nothing, unlinks nothing, and a file + /// with no rows in it that no run holds is left standing at the caller's -OutputFile. + /// + /// The parent answers again for the second look, which is the one the take-back is made + /// under: the failure being transient is what makes the empty file outlive it. + /// + /// + [Fact] + public void An_output_made_to_hold_a_claim_that_cannot_be_verified_is_taken_back() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + + // The premise, as above: a directory with no search bit answers for no name under + // it, and an account it does not refuse is one there is nothing here to measure on. + SetMode(dir, 0); + var unsearchable = !File.Exists(temp); + SetMode(dir, Searchable); + if (!unsearchable) return; + + var looks = 0; + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = claimed => + { + if (!string.Equals(claimed, output, StringComparison.Ordinal)) return; + + // The claim on the output, which cannot be verified; then the finally's look at + // the file it made, which can. + SetMode(dir, ++looks == 1 ? 0 : Searchable); + }; + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var args = Slots(Promote, output, Path.GetFileName(temp), counted); + var answer = Promote.Invoke(run, args)!.ToString()!; + run.Dispose(); + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = null; + SetMode(dir, Searchable); + + // Nothing of this run's making left at either name, and the temp holding every item + // the checkpoint counts exactly as it was found - so the run that comes back + // promotes the same bytes to the same length over the same output. + Assert.False(File.Exists(output), + "the empty output the claim made is left standing at the path"); + Assert.False(File.Exists($"{output}.adopt")); + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + + // Two looks: the claim's, which could not be answered, and the finally's, which is + // the one the take-back was made under. + Assert.Equal(2, looks); + Assert.Equal("OutputUnopenable", answer); + Assert.Contains("cannot be verified", args[3]!.ToString()!); + } + finally + { + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = null; + try { SetMode(dir, Searchable); Directory.Delete(dir, true); } + catch { } + } + } + + /// + /// The output leaving the name in the instant the claim on it was granted, and the directory + /// refusing a new file in the same one: the first look found a file, so the claim opened + /// rather than created; the file left the name, so the claim was dropped and asked again; + /// and the attempt that answers is a create the directory refused. Which of those two the + /// answer stands for decides who words the stop. A create the directory refused is the + /// staging create's own answer a few lines later, in that same directory, and the preview + /// asks it too - so the claim leaves the wording there rather than naming the output in a + /// sentence only the applying pass can reach. + /// + [Fact] + public void A_create_refused_at_an_output_that_went_is_left_to_the_staging_create_s_words() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var adopt = $"{output}.adopt"; + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + + // The premise: a directory this account may not write refuses a new file in it. + // Where it takes one, the account is root and there is nothing here to measure. + var probe = Path.Combine(dir, "probe"); + SetMode(dir, ReadAndSearch); + var refused = false; + try { File.WriteAllText(probe, "x"); } + catch (UnauthorizedAccessException) { refused = true; } + SetMode(dir, Searchable); + if (!refused) + { + File.Delete(probe); + return; + } + + var looks = 0; + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = claimed => + { + if (!string.Equals(claimed, output, StringComparison.Ordinal)) return; + if (looks++ > 0) return; + + // The file this claim was granted over, off the name - and the directory closed + // behind it, so the attempt that follows finds nothing at the name, asks for + // the create, and is refused it. + File.Delete(output); + SetMode(dir, ReadAndSearch); + }; + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = null; + + // The stop is the staging create's, in its own words, over the name it was making. + Assert.Equal("StagingFailed", applied.Answer); + Assert.Equal(1, looks); + Assert.Contains(adopt, applied.Reason); + Assert.Contains("is denied", applied.Reason); + + // And it is the wording the preview reaches over the same disk state, which is the + // whole reason the claim does not answer this one itself. + var preview = InvokeCanForStaging(output, Path.GetFileName(temp), counted); + Assert.Equal("StagingFailed", preview.Answer); + Assert.Equal(preview.Reason, applied.Reason); + + // Nothing was left at either name: no empty output of this run's making, and the + // temp holding the items exactly as it was found. + Assert.False(File.Exists(output)); + Assert.False(File.Exists(adopt)); + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + } + finally + { + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = null; + try { SetMode(dir, Searchable); Directory.Delete(dir, true); } + catch { } + } + } } diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/OrphanAdoptionTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/OrphanAdoptionTests.cs index af7b4f9..03ad736 100644 --- a/tests/Mgx.IntegrationTests/Engine/Pagination/OrphanAdoptionTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/OrphanAdoptionTests.cs @@ -19,11 +19,24 @@ public class OrphanAdoptionTests typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( "CanAdoptOrphanedTemp", BindingFlags.Static | BindingFlags.NonPublic)!; + /// + /// The two arguments, followed by an empty slot for each out parameter the method carries - + /// the temp it picked at 2, the staging failure at 3, what the sweep removed at 4. Sized off + /// the method, since reflection wants a slot for every one of them. + /// + private static object?[] Slots(MethodInfo method, string outputPath, long itemCount) + { + var slots = new object?[method.GetParameters().Length]; + slots[0] = outputPath; + slots[1] = itemCount; + return slots; + } + private static bool Invoke(string outputPath, long itemCount) => - (bool)Adopt.Invoke(null, [outputPath, itemCount])!; + (bool)Adopt.Invoke(null, Slots(Adopt, outputPath, itemCount))!; private static bool InvokeCan(string outputPath, long itemCount) => - (bool)CanAdopt.Invoke(null, [outputPath, itemCount])!; + (bool)CanAdopt.Invoke(null, Slots(CanAdopt, outputPath, itemCount))!; private static string NewDir() { diff --git a/tests/Mgx.IntegrationTests/Engine/SdkVersionTests.cs b/tests/Mgx.IntegrationTests/Engine/SdkVersionTests.cs index 14ff952..509acb3 100644 --- a/tests/Mgx.IntegrationTests/Engine/SdkVersionTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/SdkVersionTests.cs @@ -28,18 +28,32 @@ private static string ManifestVersion() return match.Groups[1].Value; } - /// Walk up from the test binaries until the repository-relative path exists. + /// The file that marks the repository root. + private const string RepositoryMarker = "Mgx.slnx"; + + /// + /// Resolve a repository-relative file under the repository root - the first ancestor of the + /// test binaries holding Mgx.slnx. Nothing above that root is consulted, so a module folder + /// beside a worktree checked out under it cannot supply the manifest this pins against. + /// private static string FindRepositoryFile(string relativePath) { var dir = new DirectoryInfo(AppContext.BaseDirectory); while (dir != null) { - var candidate = Path.Combine(dir.FullName, relativePath); - if (File.Exists(candidate)) return candidate; + if (File.Exists(Path.Combine(dir.FullName, RepositoryMarker))) + { + var candidate = Path.Combine(dir.FullName, relativePath); + if (File.Exists(candidate)) return candidate; + + throw new FileNotFoundException( + $"Could not locate '{relativePath}' under repository root {dir.FullName}"); + } dir = dir.Parent; } - throw new FileNotFoundException($"Could not locate '{relativePath}' above {AppContext.BaseDirectory}"); + throw new DirectoryNotFoundException( + $"Could not locate {RepositoryMarker} above {AppContext.BaseDirectory}"); } [Fact] diff --git a/tests/Mgx.IntegrationTests/Fakes/MockHttpHandler.cs b/tests/Mgx.IntegrationTests/Fakes/MockHttpHandler.cs index 36205f1..678f793 100644 --- a/tests/Mgx.IntegrationTests/Fakes/MockHttpHandler.cs +++ b/tests/Mgx.IntegrationTests/Fakes/MockHttpHandler.cs @@ -32,12 +32,13 @@ public sealed record CapturedRequest( /// public class MockHttpHandler : HttpMessageHandler { - private readonly Queue _responses = new(); + private readonly Queue _responses = new(); + private readonly MockRuleSet _rules = new(); private readonly List _requests = []; private readonly List _captured = []; private readonly List _served = []; private readonly object _lock = new(); - private MockResponse? _defaultResponse; + private MockReply? _defaultResponse; public int RequestCount { @@ -60,16 +61,53 @@ public List CapturedRequests /// /// The status codes handed back, in order. A test whose assertion is that nothing was - /// reported needs some way to know the wire reported something to begin with. + /// reported needs some way to know the wire reported something to begin with. An answer + /// that never reached the caller is not here: one thrown instead of answered, and one held + /// past the attempt timeout that canceled the send. /// public List ServedStatusCodes { get { lock (_lock) { return [.. _served]; } } } + /// + /// Program a response for the requests a predicate picks out, rather than for a position in + /// the queue. A fault can then be aimed at page 3, at one fan-out item, or at the second + /// attempt of a request, and it lands there however the requests interleave. + /// Precedence: a keyed entry that matches this request wins and consumes no queue entry; an + /// unmatched request takes the queue; an empty queue takes the default response. + /// + /// Among the keyed entries the first one that matches, in programming order, answers - so a + /// narrower entry has to be programmed before a broader one, and a fault registered after a + /// generator that also matches its request never answers at all. An entry that never + /// answered is named in , and a test that aims a fault should + /// assert that list is empty: nothing else about the run says the fault was not injected. + /// + /// is what those assertions call this entry, and defaults to its + /// position in programming order. + /// + public MockResponseRule When(Func predicate, string? name = null) + => _rules.When(this, predicate, name); + + /// Every keyed entry, with the requests it matched and the ones it answered. + public IReadOnlyList ProgrammedRules + { + get { lock (_lock) { return _rules.ProgrammedRules; } } + } + + /// + /// The keyed entries that never answered a request, by name. A fault an earlier entry + /// shadowed leaves no other trace: the operation completes exactly as it would have with + /// nothing programmed. + /// + public IReadOnlyList UnansweredRules + { + get { lock (_lock) { return _rules.UnansweredRules; } } + } + public void QueueResponse(HttpStatusCode statusCode, string? body = null, Dictionary? headers = null, string contentType = "application/json") { - _responses.Enqueue(new MockResponse(statusCode, body, headers, null, contentType, null)); + _responses.Enqueue(new MockReply { StatusCode = statusCode, Body = body, Headers = headers, ContentType = contentType }); } /// @@ -77,7 +115,7 @@ public void QueueResponse(HttpStatusCode statusCode, string? body = null, Dictio /// public void QueueBytes(HttpStatusCode statusCode, byte[] body, string contentType, Dictionary? headers = null) { - _responses.Enqueue(new MockResponse(statusCode, null, headers, null, contentType, body)); + _responses.Enqueue(new MockReply { StatusCode = statusCode, BodyBytes = body, Headers = headers, ContentType = contentType }); } /// @@ -85,12 +123,12 @@ public void QueueBytes(HttpStatusCode statusCode, byte[] body, string contentTyp /// public void QueueEmpty(HttpStatusCode statusCode, Dictionary? headers = null) { - _responses.Enqueue(new MockResponse(statusCode, null, headers, null, null, null)); + _responses.Enqueue(new MockReply { StatusCode = statusCode, Headers = headers }); } public void SetDefaultResponse(HttpStatusCode statusCode, string? body = null, Dictionary? headers = null) { - _defaultResponse = new MockResponse(statusCode, body, headers, null, "application/json", null); + _defaultResponse = new MockReply { StatusCode = statusCode, Body = body, Headers = headers, ContentType = "application/json" }; } /// @@ -109,7 +147,7 @@ public void QueueFailuresThenSuccess(int failCount, HttpStatusCode failStatus, s /// public void QueueException(Exception exception) { - _responses.Enqueue(new MockResponse(HttpStatusCode.OK, null, null, exception, null, null)); + _responses.Enqueue(new MockReply { Exception = exception }); } protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) @@ -126,42 +164,36 @@ protected override async Task SendAsync(HttpRequestMessage ?? new Dictionary(), bodyBytes); - MockResponse mock; + var keyed = new MockRequest(request, bodyBytes); + + MockReply mock; lock (_lock) { _requests.Add(request); _captured.Add(captured); - mock = _responses.Count > 0 ? _responses.Dequeue() : (_defaultResponse ?? new MockResponse(HttpStatusCode.OK, null, null, null, null, null)); - if (mock.Exception == null) _served.Add(mock.StatusCode); + // Keyed first, then the queue, then the default. A keyed answer leaves the queue + // where it stands, so a test can aim one fault and script the rest in order. + mock = _rules.Resolve(keyed) + ?? (_responses.Count > 0 ? _responses.Dequeue() : _defaultResponse ?? new MockReply()); } + // On the send's own token: an answer held past the attempt timeout has to end as a + // timeout, not as a mock that never returns. + if (mock.Delay > TimeSpan.Zero) + await Task.Delay(mock.Delay, cancellationToken); + if (mock.Exception != null) throw mock.Exception; - // Real transports (SocketsHttpHandler) set RequestMessage on the response; consumers - // like the pacer's OnRetry hook read the request URI off it. Mirror that here. - var response = new HttpResponseMessage(mock.StatusCode) { RequestMessage = request }; - if (mock.BodyBytes != null) - { - response.Content = new ByteArrayContent(mock.BodyBytes); - if (mock.ContentType != null) - response.Content.Headers.TryAddWithoutValidation("Content-Type", mock.ContentType); - } - else if (mock.Body != null) - { - response.Content = new StringContent(mock.Body, Encoding.UTF8, mock.ContentType ?? "application/json"); - } - - if (mock.Headers != null) - { - foreach (var (key, value) in mock.Headers) - response.Headers.TryAddWithoutValidation(key, value); - } + var response = mock.CreateResponse(keyed); + // Recorded here rather than where the answer was chosen, so the list says what the + // handler handed back: an answer held past an attempt timeout is never handed back at + // all, a thrown one never has a status, and two held answers come back in an order the + // one they resolved in does not predict. + lock (_lock) _served.Add(response.StatusCode); return response; } - - private record MockResponse(HttpStatusCode StatusCode, string? Body, Dictionary? Headers, Exception? Exception, string? ContentType, byte[]? BodyBytes); } public static class TestData diff --git a/tests/Mgx.IntegrationTests/Fakes/MockResponseRules.cs b/tests/Mgx.IntegrationTests/Fakes/MockResponseRules.cs new file mode 100644 index 0000000..4851110 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Fakes/MockResponseRules.cs @@ -0,0 +1,311 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; + +namespace Mgx.IntegrationTests; + +/// +/// One request as a rule predicate sees it: the message itself, plus the body bytes the +/// handler has already read. A predicate over a $batch envelope needs those bytes, and it +/// cannot await them - the handler reads the content once, so a predicate that read it again +/// would be reading a stream the client is entitled to have disposed. +/// +public sealed class MockRequest(HttpRequestMessage message, byte[]? body) +{ + /// The request as sent. Everything below is shorthand onto it. + public HttpRequestMessage Message => message; + + public HttpMethod Method => message.Method; + + public Uri? RequestUri => message.RequestUri; + + /// The URI as the caller wrote it, absolute or relative. + public string Uri => message.RequestUri?.OriginalString ?? string.Empty; + + public HttpRequestHeaders Headers => message.Headers; + + /// The request body, or null when the request carried no content. + public byte[]? Body => body; + + public string? BodyText => body == null ? null : Encoding.UTF8.GetString(body); +} + +/// +/// What a handler answers one request with, whether the entry came from the queue or from a +/// request-keyed rule. Both fakes render it the same way, so a queued response and a keyed +/// response are indistinguishable on the wire. +/// +public sealed record MockReply +{ + public HttpStatusCode StatusCode { get; init; } = HttpStatusCode.OK; + + public string? Body { get; init; } + + public byte[]? BodyBytes { get; init; } + + public string? ContentType { get; init; } + + public Dictionary? Headers { get; init; } + + /// Thrown instead of answering. Nothing is recorded as served. + public Exception? Exception { get; init; } + + /// + /// Held before the answer is handed back, so an attempt timeout has something to fire on. + /// The wait takes the send's own token, which is what makes it a timeout rather than a hang. + /// + public TimeSpan Delay { get; init; } + + /// + /// The body delivers or and then stops without + /// closing, leaving the body-read timeout as the only thing that ends the read. + /// + public bool StallBody { get; init; } + + /// + /// Computes the whole response from the request, for answers too numerous to enumerate in + /// advance. Takes precedence over every field here except . + /// + public Func? Factory { get; init; } + + internal HttpResponseMessage CreateResponse(MockRequest request) + { + if (Factory != null) + { + var computed = Factory(request); + computed.RequestMessage ??= request.Message; + return computed; + } + + // Real transports (SocketsHttpHandler) set RequestMessage on the response; consumers + // like the pacer's OnRetry hook read the request URI off it. Mirror that here. + var response = new HttpResponseMessage(StatusCode) { RequestMessage = request.Message }; + if (StallBody) + { + var stalling = new StallingContent(BodyBytes ?? Encoding.UTF8.GetBytes(Body ?? "{")); + stalling.Headers.TryAddWithoutValidation("Content-Type", ContentType ?? "application/json"); + response.Content = stalling; + } + else if (BodyBytes != null) + { + response.Content = new ByteArrayContent(BodyBytes); + if (ContentType != null) + response.Content.Headers.TryAddWithoutValidation("Content-Type", ContentType); + } + else if (Body != null) + { + response.Content = new StringContent(Body, Encoding.UTF8, ContentType ?? "application/json"); + } + + if (Headers != null) + { + foreach (var (key, value) in Headers) + response.Headers.TryAddWithoutValidation(key, value); + } + + return response; + } +} + +/// One request-keyed entry: which requests it answers, and with what. +internal sealed class MockRule +{ + public required Func Predicate { get; init; } + + /// + /// What an assertion calls this entry. A predicate cannot be printed, so the name is the + /// only thing a failure message can carry; null until the rule set fills in its position. + /// + public string? Name { get; set; } + + /// Null answers every matching request; N answers only the Nth, 1-based. + public int? Attempt { get; set; } + + public MockReply Reply { get; set; } = new(); + + /// How many requests have matched this entry's predicate so far. + public int Matches { get; set; } + + /// How many of those this entry was the one to answer. + public int Answers { get; set; } +} + +/// +/// One programmed entry as an assertion sees it: what it was called, how many requests matched +/// its predicate, and how many of those it answered. A matched entry that answered nothing was +/// shadowed by an earlier one, or scoped to an attempt that never came. +/// +public sealed record MockRuleReport(string Name, int Matches, int Answers) +{ + public override string ToString() => $"{Name} (matched {Matches}, answered {Answers})"; +} + +/// +/// One request-keyed entry under construction. OnAttempt and AfterDelay narrow +/// it and must come before the answer; a Respond or Throw call arms it and hands +/// back the handler, so entries chain. +/// +public sealed class MockResponseRule +{ + private readonly MockRuleSet _rules; + private readonly MockRule _rule; + private readonly TOwner _owner; + private TimeSpan _delay; + + internal MockResponseRule(MockRuleSet rules, MockRule rule, TOwner owner) + { + _rules = rules; + _rule = rule; + _owner = owner; + } + + /// + /// Answer only the th request matching this entry's predicate, + /// 1-based. The counting is per entry, so the requests another entry answered still count + /// here and nowhere else; every other attempt falls through to whatever answers next - a + /// later entry, the queue, or the default. + /// + public MockResponseRule OnAttempt(int attempt) + { + _rule.Attempt = attempt; + return this; + } + + /// Hold the answer this long before handing it back. + public MockResponseRule AfterDelay(TimeSpan delay) + { + _delay = delay; + return this; + } + + /// Answer with a status and an optional body. + public TOwner Respond( + HttpStatusCode statusCode, + string? body = null, + Dictionary? headers = null, + string contentType = "application/json") + => Arm(new MockReply + { + StatusCode = statusCode, + Body = body, + Headers = headers, + ContentType = contentType + }); + + /// Answer with a raw byte body, for binary and non-UTF8 cases. + public TOwner RespondBytes( + HttpStatusCode statusCode, + byte[] body, + string contentType, + Dictionary? headers = null) + => Arm(new MockReply + { + StatusCode = statusCode, + BodyBytes = body, + ContentType = contentType, + Headers = headers + }); + + /// Answer with no content at all - a 204, or a 200 with a zero-length body. + public TOwner RespondEmpty(HttpStatusCode statusCode, Dictionary? headers = null) + => Arm(new MockReply { StatusCode = statusCode, Headers = headers }); + + /// + /// Compute the answer from the request. What a page-generating fake is built on: 100,000 + /// items are a function of the skip token, not a hundred thousand queued responses. + /// + public TOwner Respond(Func factory) + => Arm(new MockReply { Factory = factory }); + + /// + /// Answer with headers and a body that delivers and then stops + /// without closing, so the body-read timeout is the only thing that ends the read. + /// + public TOwner RespondStalled( + HttpStatusCode statusCode = HttpStatusCode.OK, + string prefix = "{", + string contentType = "application/json", + Dictionary? headers = null) + => Arm(new MockReply + { + StatusCode = statusCode, + Body = prefix, + ContentType = contentType, + Headers = headers, + StallBody = true + }); + + /// Throw instead of answering, for the transport failures that never reach a status. + public TOwner Throw(Exception exception) => Arm(new MockReply { Exception = exception }); + + private TOwner Arm(MockReply reply) + { + _rule.Reply = reply with { Delay = _delay }; + _rules.Add(_rule); + return _owner; + } +} + +/// +/// The request-keyed entries a handler answers from, in the order they were programmed. Not +/// synchronized on its own: a handler resolves under the same lock it already holds for its +/// queue, so one request takes exactly one answer. +/// +public sealed class MockRuleSet +{ + private readonly List _rules = []; + + /// Nothing is programmed, so a handler can skip the work of preparing a match. + public bool IsEmpty => _rules.Count == 0; + + internal void Add(MockRule rule) + { + rule.Name ??= $"#{_rules.Count + 1}"; + _rules.Add(rule); + } + + /// + /// Start an entry keyed on . is what + /// assertions and failure messages call it, and defaults to its position in programming + /// order, 1-based, as #1. + /// + public MockResponseRule When( + TOwner owner, Func predicate, string? name = null) + => new(this, new MockRule { Predicate = predicate, Name = name }, owner); + + /// + /// Every entry in programming order, with the requests that matched it and the ones it + /// answered. Read after the operation under test has finished: the counts are live. + /// + public IReadOnlyList ProgrammedRules => + [.. _rules.Select(rule => new MockRuleReport(rule.Name!, rule.Matches, rule.Answers))]; + + /// + /// The entries that never answered a request, by name. An entry a broader one programmed + /// before it shadows is here and nowhere else: the operation it was aimed at completes, + /// every item arrives, and no other observation says the fault was never injected. + /// + public IReadOnlyList UnansweredRules => + [.. _rules.Where(rule => rule.Answers == 0).Select(rule => rule.Name!)]; + + /// + /// The first entry whose predicate matches and whose attempt scope selects this request, or + /// null when none does. Every matching entry's counter advances whether or not it is the one + /// that answers, so an entry's attempt index means the Nth request matching that entry and + /// nothing else - one entry's faults never renumber another's. + /// + internal MockReply? Resolve(MockRequest request) + { + MockReply? chosen = null; + foreach (var rule in _rules) + { + if (!rule.Predicate(request)) continue; + var matched = ++rule.Matches; + if (chosen != null) continue; + if (rule.Attempt is int scoped && scoped != matched) continue; + rule.Answers++; + chosen = rule.Reply; + } + return chosen; + } +} diff --git a/tests/Mgx.IntegrationTests/Fakes/PagedCollection.cs b/tests/Mgx.IntegrationTests/Fakes/PagedCollection.cs new file mode 100644 index 0000000..bdb297f --- /dev/null +++ b/tests/Mgx.IntegrationTests/Fakes/PagedCollection.cs @@ -0,0 +1,147 @@ +using System.Net; +using System.Text; +using Mgx.IntegrationTests.Fakes; + +namespace Mgx.IntegrationTests; + +/// +/// A Graph collection that exists only as a function of the request: every page is computed +/// from the skip token the previous page's nextLink carried, so a hundred thousand items are +/// one programmed entry rather than a hundred thousand queued responses. Nothing is enumerated +/// in advance, and nothing here is mutable, so the handler can answer from it under its lock or +/// outside it. +/// +/// It is registered through the request-keyed programming and takes exactly the precedence +/// every other entry does: a fault aimed at one page is an entry registered BEFORE this one - +/// the first matching entry answers - and the pages either side of it still come from here. +/// +/// +public sealed class PagedCollection +{ + /// What this generator's nextLinks carry, and what reads. + private const string SkipToken = "$skiptoken=page"; + + private readonly string _idPrefix; + + /// Items in the whole collection. + /// Items on every page but the last. + /// The collection's URL with no query string. Every request whose + /// URI is this, or this followed by a query, is one of these pages. + /// The shape Graph returns when the collection divides + /// evenly: the last full page still carries a nextLink, and following it yields an empty + /// page. False ends the collection with the last full page's absent nextLink instead, and + /// true is refused on a total that does not divide evenly. + /// Item ids are this followed by the item's 1-based position. + public PagedCollection( + int total, + int pageSize, + string resourceUrl = "https://graph.microsoft.com/v1.0/users", + bool trailingEmptyPage = false, + string idPrefix = "user") + { + ArgumentOutOfRangeException.ThrowIfNegative(total); + ArgumentOutOfRangeException.ThrowIfLessThan(pageSize, 1); + // Graph appends its empty page behind a full one. A collection that does not divide + // evenly ends at a short page, and a nextLink behind a short page is a shape the service + // never returns - served here it would make a boundary test pass against a wire no + // caller will ever meet. + if (trailingEmptyPage && total % pageSize != 0) + throw new ArgumentException( + $"A trailing empty page is the shape Graph returns when the collection divides " + + $"evenly; {total} items served {pageSize} at a time ends at a short page.", + nameof(trailingEmptyPage)); + + Total = total; + PageSize = pageSize; + ResourceUrl = resourceUrl; + _idPrefix = idPrefix; + + // An empty collection is still one page: a request has to be answered with something. + var full = Math.Max(1, (total + pageSize - 1) / pageSize); + PageCount = trailingEmptyPage ? full + 1 : full; + } + + public int Total { get; } + + public int PageSize { get; } + + public string ResourceUrl { get; } + + /// Pages a complete enumeration asks for, and so the requests it costs. + public int PageCount { get; } + + /// Where an enumeration starts, page size and all. + public string InitialUrl => $"{ResourceUrl}?$top={PageSize}"; + + /// The link page 's predecessor carries; is 2 or more. + public string NextLinkTo(int page) => $"{ResourceUrl}?{SkipToken}{page}"; + + /// + /// Which page a request asks for, 1-based: the number in the skip token, or 1 for the + /// initial URL, which carries none. + /// + public static int PageOf(string uri) + { + var token = uri.IndexOf(SkipToken, StringComparison.Ordinal); + if (token < 0) return 1; + + var start = token + SkipToken.Length; + var end = start; + while (end < uri.Length && char.IsAsciiDigit(uri[end])) end++; + return end > start ? int.Parse(uri[start..end]) : 1; + } + + /// Whether addresses this collection rather than some other resource. + private bool Addresses(string uri) => + uri.StartsWith(ResourceUrl, StringComparison.Ordinal) + && (uri.Length == ResourceUrl.Length || uri[ResourceUrl.Length] == '?'); + + /// The predicate a handler entry is keyed on. + public bool Matches(MockRequest request) => Addresses(request.Uri); + + /// + /// Page as a Graph collection response. A page past the end is a + /// thrown exception rather than an empty body: the iterator asking for one is the defect + /// these tests exist to catch, and an obliging answer would hide it. + /// + public string BodyFor(int page) + { + if (page < 1 || page > PageCount) + throw new InvalidOperationException( + $"Page {page} was requested from a collection of {Total} items served " + + $"{PageSize} at a time, which is {PageCount} page(s)."); + + var already = (page - 1) * PageSize; + var count = Math.Clamp(Total - already, 0, PageSize); + + var json = new StringBuilder(count * 24 + 128).Append("{\"value\":["); + for (var i = 0; i < count; i++) + { + if (i > 0) json.Append(','); + json.Append("{\"id\":\"").Append(_idPrefix).Append(already + i + 1).Append("\"}"); + } + json.Append(']'); + if (page < PageCount) + json.Append(",\"@odata.nextLink\":\"").Append(NextLinkTo(page + 1)).Append('"'); + return json.Append('}').ToString(); + } + + /// The answer to one request, for a handler entry armed with a factory. + public HttpResponseMessage Answer(MockRequest request) => + new(HttpStatusCode.OK) + { + Content = new StringContent( + BodyFor(PageOf(request.Uri)), Encoding.UTF8, "application/json") + }; + + /// + /// Program to serve this collection, and hand the handler back so + /// further entries chain. Entries registered first win, so a fault on one page goes on + /// before this call. + /// + public MockHttpHandler ServeOn(MockHttpHandler handler) => handler.When(Matches).Respond(Answer); + + /// + public StubHttpMessageHandler ServeOn(StubHttpMessageHandler handler) => + handler.When(Matches).Respond(Answer); +} diff --git a/tests/Mgx.IntegrationTests/Fakes/RecordingHost.cs b/tests/Mgx.IntegrationTests/Fakes/RecordingHost.cs new file mode 100644 index 0000000..8d4ea82 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Fakes/RecordingHost.cs @@ -0,0 +1,88 @@ +using System.Globalization; +using System.Management.Automation; +using System.Management.Automation.Host; +using System.Management.Automation.Runspaces; +using System.Reflection; +using System.Security; + +namespace Mgx.IntegrationTests; + +/// +/// A PSHost that keeps every line written to it. A ShouldProcess gate writes its "What if:" line +/// to the host rather than to one of the streams a PowerShell object exposes, so a test that +/// reads what a preview actually said needs a host of its own to read it off. +/// +internal sealed class RecordingHost : PSHost +{ + private readonly Guid _id = Guid.NewGuid(); + + public RecordingHostUI Recorder { get; } = new(); + + public override string Name => "MgxRecordingHost"; + public override Version Version => new(1, 0); + public override Guid InstanceId => _id; + public override PSHostUserInterface UI => Recorder; + public override CultureInfo CurrentCulture => CultureInfo.InvariantCulture; + public override CultureInfo CurrentUICulture => CultureInfo.InvariantCulture; + public override void EnterNestedPrompt() { } + public override void ExitNestedPrompt() { } + public override void NotifyBeginApplication() { } + public override void NotifyEndApplication() { } + public override void SetShouldExit(int exitCode) { } + + /// + /// Imports into a runspace over one of these hosts, runs the + /// command adds, and returns the lines the host was given. + /// + internal static List Preview(Assembly module, Action build) + { + var host = new RecordingHost(); + using var runspace = RunspaceFactory.CreateRunspace(host); + runspace.Open(); + using var ps = PowerShell.Create(); + ps.Runspace = runspace; + ps.AddCommand("Import-Module").AddParameter("Assembly", module); + ps.Invoke(); + ps.Commands.Clear(); + + build(ps); + ps.Invoke(); + + Assert.Empty(ps.Streams.Error.Select(e => e.FullyQualifiedErrorId)); + return host.Recorder.Lines; + } +} + +/// Every write, in the order it arrived, and nothing else. +internal sealed class RecordingHostUI : PSHostUserInterface +{ + public List Lines { get; } = []; + + public override PSHostRawUserInterface? RawUI => null; + public override void Write(string value) => Lines.Add(value); + public override void Write(ConsoleColor f, ConsoleColor b, string value) => Lines.Add(value); + public override void WriteLine(string value) => Lines.Add(value); + public override void WriteErrorLine(string value) => Lines.Add(value); + public override void WriteDebugLine(string value) => Lines.Add(value); + public override void WriteVerboseLine(string value) => Lines.Add(value); + public override void WriteWarningLine(string value) => Lines.Add(value); + public override void WriteProgress(long sourceId, ProgressRecord record) { } + public override string ReadLine() => string.Empty; + public override SecureString ReadLineAsSecureString() => new(); + + public override Dictionary Prompt( + string caption, string message, + System.Collections.ObjectModel.Collection descriptions) => []; + + public override int PromptForChoice( + string caption, string message, + System.Collections.ObjectModel.Collection choices, + int defaultChoice) => defaultChoice; + + public override PSCredential PromptForCredential( + string caption, string message, string userName, string targetName) => PSCredential.Empty; + + public override PSCredential PromptForCredential( + string caption, string message, string userName, string targetName, + PSCredentialTypes allowedCredentialTypes, PSCredentialUIOptions options) => PSCredential.Empty; +} diff --git a/tests/Mgx.IntegrationTests/Fakes/StallingContent.cs b/tests/Mgx.IntegrationTests/Fakes/StallingContent.cs new file mode 100644 index 0000000..fcdea0c --- /dev/null +++ b/tests/Mgx.IntegrationTests/Fakes/StallingContent.cs @@ -0,0 +1,93 @@ +using System.Net; +using System.Text; + +namespace Mgx.IntegrationTests; + +/// +/// A response body that delivers a prefix and then stops, without closing the stream, and ends +/// only when the read's own token is canceled. Requests go out with +/// HttpCompletionOption.ResponseHeadersRead, so neither HttpClient.Timeout nor the pipeline's +/// attempt timeout covers the content read at all - the engine's BodyReadTimeout is the only +/// thing that bounds it, and this is what makes that bound observable in-process. +/// +public sealed class StallingContent : HttpContent +{ + private readonly byte[] _prefix; + private readonly long _declaredLength; + + /// Delivered before the stall, so the body looks like one that started. + /// What Content-Length claims, so a reader keeps waiting for the rest. + public StallingContent(string prefix = "{", long declaredLength = 4096) + : this(Encoding.UTF8.GetBytes(prefix), declaredLength) + { + } + + public StallingContent(byte[] prefix, long declaredLength = 4096) + { + _prefix = prefix; + _declaredLength = declaredLength; + } + + protected override async Task SerializeToStreamAsync(Stream stream, TransportContext? context) + => await SerializeToStreamAsync(stream, context, CancellationToken.None); + + protected override async Task SerializeToStreamAsync( + Stream stream, TransportContext? context, CancellationToken cancellationToken) + { + await stream.WriteAsync(_prefix, cancellationToken); + await stream.FlushAsync(cancellationToken); + await Task.Delay(Timeout.Infinite, cancellationToken); + } + + /// + /// The stall as a reader on the content stream sees it. HttpContent's own implementation of + /// this buffers the whole body first and drops the cancellation token doing it, so a caller + /// that reads the body as a stream - a collection page, a $batch envelope, a content + /// download - waits on this content forever instead of hitting its read timeout. A real + /// response hands the stream over as soon as the headers are in and stalls on the read; + /// this does the same, on the read's own token. + /// + protected override Task CreateContentReadStreamAsync() + => Task.FromResult(new StallingStream(_prefix)); + + protected override bool TryComputeLength(out long length) + { + length = _declaredLength; + return true; + } + + private sealed class StallingStream(byte[] prefix) : Stream + { + private int _delivered; + + public override async ValueTask ReadAsync( + Memory buffer, CancellationToken cancellationToken = default) + { + if (_delivered < prefix.Length) + { + var count = Math.Min(buffer.Length, prefix.Length - _delivered); + prefix.AsSpan(_delivered, count).CopyTo(buffer.Span); + _delivered += count; + return count; + } + + await Task.Delay(Timeout.Infinite, cancellationToken); + return 0; + } + + public override Task ReadAsync( + byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => ReadAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => _delivered; set => throw new NotSupportedException(); } + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + } +} diff --git a/tests/Mgx.IntegrationTests/Fakes/StubHttpMessageHandler.cs b/tests/Mgx.IntegrationTests/Fakes/StubHttpMessageHandler.cs index ec57887..13b3df9 100644 --- a/tests/Mgx.IntegrationTests/Fakes/StubHttpMessageHandler.cs +++ b/tests/Mgx.IntegrationTests/Fakes/StubHttpMessageHandler.cs @@ -11,6 +11,7 @@ namespace Mgx.IntegrationTests.Fakes; public sealed class StubHttpMessageHandler : HttpMessageHandler { private readonly Queue> _responses = new(); + private readonly MockRuleSet _rules = new(); private Func? _last; private readonly Lock _sync = new(); @@ -22,6 +23,41 @@ public int RequestCount get { lock (_sync) return Requests.Count; } } + /// + /// Program a response for the requests a predicate picks out, rather than for a position in + /// the script, with the same precedence uses: a keyed entry + /// that matches this request wins and consumes no scripted response; an unmatched request + /// takes the script, whose last entry repeats as it always has. + /// + /// Among the keyed entries the first one that matches, in programming order, answers - so a + /// narrower entry has to be programmed before a broader one, and a fault registered after a + /// generator that also matches its request never answers at all. An entry that never + /// answered is named in , and a test that aims a fault should + /// assert that list is empty: nothing else about the run says the fault was not injected. + /// + /// is what those assertions call this entry, and defaults to its + /// position in programming order. + /// + public MockResponseRule When( + Func predicate, string? name = null) + => _rules.When(this, predicate, name); + + /// Every keyed entry, with the requests it matched and the ones it answered. + public IReadOnlyList ProgrammedRules + { + get { lock (_sync) return _rules.ProgrammedRules; } + } + + /// + /// The keyed entries that never answered a request, by name. A fault an earlier entry + /// shadowed leaves no other trace: the operation completes exactly as it would have with + /// nothing programmed. + /// + public IReadOnlyList UnansweredRules + { + get { lock (_sync) return _rules.UnansweredRules; } + } + public StubHttpMessageHandler Enqueue(Func factory) { _responses.Enqueue(factory); @@ -56,26 +92,48 @@ public StubHttpMessageHandler EnqueueRepeated(int count, Func SendAsync( + protected override async Task SendAsync( HttpRequestMessage request, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - Func factory; + // Only read when something is programmed to look: a predicate over a $batch envelope + // needs the bytes, and a handler with no keyed entries touches the request exactly as + // it did before there were any. + byte[]? bodyBytes = null; + if (!_rules.IsEmpty && request.Content != null) + bodyBytes = await request.Content.ReadAsByteArrayAsync(cancellationToken); + var keyed = new MockRequest(request, bodyBytes); + + MockReply? reply; + Func? factory = null; lock (_sync) { // Record method/uri rather than the message: HttpClient disposes the // request once the call completes, so the object is not safe to keep. Requests.Add((request.Method, request.RequestUri?.ToString() ?? string.Empty)); - if (_responses.Count > 0) - _last = _responses.Dequeue(); + reply = _rules.Resolve(keyed); + if (reply == null) + { + if (_responses.Count > 0) + _last = _responses.Dequeue(); - factory = _last ?? throw new InvalidOperationException( - "StubHttpMessageHandler received a request but no response was scripted."); + factory = _last ?? throw new InvalidOperationException( + "StubHttpMessageHandler received a request but no response was scripted."); + } } - var response = factory(request); - return Task.FromResult(response); + if (reply == null) + return factory!(request); + + // On the send's own token, so an answer held past the attempt timeout ends as a timeout. + if (reply.Delay > TimeSpan.Zero) + await Task.Delay(reply.Delay, cancellationToken); + + if (reply.Exception != null) + throw reply.Exception; + + return reply.CreateResponse(keyed); } } diff --git a/tests/Mgx.IntegrationTests/TestSetup/ModuleAssemblyClosureTests.cs b/tests/Mgx.IntegrationTests/TestSetup/ModuleAssemblyClosureTests.cs new file mode 100644 index 0000000..8869f23 --- /dev/null +++ b/tests/Mgx.IntegrationTests/TestSetup/ModuleAssemblyClosureTests.cs @@ -0,0 +1,360 @@ +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; +using System.Xml.Linq; + +namespace Mgx.IntegrationTests; + +/// +/// Everything the shipped assemblies reference has to be reachable from the module folder plus +/// the runtime the host already has. Nothing else is on offer: the ALC resolver serves +/// Dependencies/ only for assemblies found nowhere, so a reference that is neither ours, nor +/// staged, nor part of the framework src/ targets resolves to nothing and throws +/// FileNotFoundException at the first call that needs it - on a build agent, not here. +/// +/// The closure is computed by reading metadata, not by loading: start at the staged +/// Mgx.Cmdlets.dll and Mgx.Engine.dll and follow every assembly reference through whatever the +/// running runtime would resolve it to. That is deliberately the newest runtime rather than +/// net8.0's, because a host is free to be newer and the module has to survive it: PowerShell 7.6 +/// binds the engine's System.Text.Json reference to its own 10.0 copy, and that copy's own +/// references are as much a part of the module's closure as the engine's are. +/// +/// System.Management.Automation is where the walk stops. It is the host itself, it is never +/// shipped, and whatever it drags in is the host's to supply. +/// +/// Every path here resolves under the repository root - the directory holding Mgx.slnx - and +/// nothing above that root is ever consulted, so a module folder above the repository cannot be +/// measured in place of the staged one. +/// +public class ModuleAssemblyClosureTests +{ + /// The host. Present by definition, never staged, and not expanded. + private const string HostAssembly = "System.Management.Automation"; + + /// + /// A walk that stops early reports no unresolved references and reads exactly like a clean + /// one, so the size of the closure is asserted too. The floor carries slack because + /// references come and go; what it catches is a walk that has stopped walking. + /// + private const int ClosureFloor = 40; + + /// The file that marks the repository root. + private const string RepositoryMarker = "Mgx.slnx"; + + /// + /// Walk up from to the first ancestor containing Mgx.slnx + /// and return it as the repository root. Nothing above that directory is ever consulted, so + /// a decoy further up - a module folder beside a worktree checked out under it - cannot bind. + /// + private static string FindRepositoryRoot(string startDirectory) + { + var dir = new DirectoryInfo(startDirectory); + while (dir != null) + { + if (File.Exists(Path.Combine(dir.FullName, RepositoryMarker))) return dir.FullName; + dir = dir.Parent; + } + + throw new DirectoryNotFoundException( + $"Could not locate {RepositoryMarker} above {startDirectory}"); + } + + /// Resolve a repository-relative path under the root reached from . + private static string FindRepositoryPath(string relativePath, string startDirectory) + { + var root = FindRepositoryRoot(startDirectory); + var candidate = Path.Combine(root, relativePath); + if (File.Exists(candidate) || Directory.Exists(candidate)) return candidate; + + throw new FileNotFoundException( + $"Could not locate '{relativePath}' under repository root {root}"); + } + + /// FindRepositoryPath, starting from the test binaries. + private static string FindRepositoryPath(string relativePath) => + FindRepositoryPath(relativePath, AppContext.BaseDirectory); + + /// The staged module folder - the directory holding mgx.psd1. + private static string ModuleRoot() => + Path.GetDirectoryName(FindRepositoryPath(Path.Combine("module", "mgx.psd1")))!; + + /// + /// The staged Dependencies folder. Absent means the module has not been built, which is the + /// same demand the Pester harness makes: this test measures build output, so it fails rather + /// than skipping - a skipped test does not count toward the suite floor CI asserts on. + /// + private static string DependenciesDirectory() + { + var deps = Path.Combine(ModuleRoot(), "Dependencies"); + if (!Directory.Exists(deps)) + throw new DirectoryNotFoundException($"Dependencies folder not found at '{deps}'. Run ./build.ps1 first."); + + return deps; + } + + /// Simple assembly names of the DLLs in a folder, or an empty set if it is absent. + private static HashSet DllNamesIn(string directory) + { + var names = new HashSet(StringComparer.OrdinalIgnoreCase); + if (!Directory.Exists(directory)) return names; + + foreach (var file in Directory.EnumerateFiles(directory, "*.dll")) + names.Add(Path.GetFileNameWithoutExtension(file)); + + return names; + } + + /// The framework src/ compiles against, e.g. "net8.0". Read, not hardcoded, so a TFM bump moves this with it. + private static string ShippedTargetFramework() + { + var csproj = FindRepositoryPath(Path.Combine("src", "Mgx.Cmdlets", "Mgx.Cmdlets.csproj")); + var tfm = XDocument.Load(csproj).Descendants("TargetFramework").FirstOrDefault()?.Value; + + Assert.False(string.IsNullOrWhiteSpace(tfm), $"No in {csproj}"); + return tfm!; + } + + /// Directories that may hold a Microsoft.NETCore.App reference pack. + private static IEnumerable ReferencePackRoots() + { + // Restored as a package when the SDK building net8.0 is newer than 8.0 ... + var nugetRoot = Environment.GetEnvironmentVariable("NUGET_PACKAGES"); + if (string.IsNullOrEmpty(nugetRoot)) + nugetRoot = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".nuget", "packages"); + yield return Path.Combine(nugetRoot, "microsoft.netcore.app.ref"); + + // ... and bundled with the SDK when that SDK is installed alongside, which is what CI has. + var dotnetRoot = Environment.GetEnvironmentVariable("DOTNET_ROOT"); + if (string.IsNullOrEmpty(dotnetRoot)) + { + // /shared/Microsoft.NETCore.App// is where the core library lives. + var runtimeDirectory = RuntimeDirectory(); + dotnetRoot = Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(runtimeDirectory))); + } + + if (!string.IsNullOrEmpty(dotnetRoot)) + yield return Path.Combine(dotnetRoot, "packs", "Microsoft.NETCore.App.Ref"); + } + + /// + /// Simple names of every assembly the shared framework src/ targets provides, taken from that + /// framework's reference pack. This is the list a host is guaranteed to satisfy on its own; + /// anything outside it has to come from the module folder. + /// + private static HashSet SharedFrameworkAssemblyNames(string targetFramework) + { + var major = targetFramework["net".Length..].Split('.')[0] + "."; + var candidates = + from root in ReferencePackRoots() + where Directory.Exists(root) + from versionDirectory in Directory.EnumerateDirectories(root) + let name = Path.GetFileName(versionDirectory) + where name.StartsWith(major, StringComparison.Ordinal) + && Version.TryParse(name, out _) + let refDirectory = Path.Combine(versionDirectory, "ref", targetFramework) + where Directory.Exists(refDirectory) + orderby Version.Parse(name) + select refDirectory; + + var newest = candidates.LastOrDefault(); + Assert.True(newest is not null, + $"No Microsoft.NETCore.App reference pack for {targetFramework} under: " + + string.Join(", ", ReferencePackRoots()) + + ". The closure cannot be checked without the list of assemblies the framework supplies."); + + return DllNamesIn(newest!); + } + + /// Where the running shared framework's assemblies sit on disk. + private static string RuntimeDirectory() => Path.GetDirectoryName(typeof(object).Assembly.Location)!; + + /// + /// Assembly references recorded in a file's metadata. Reading rather than loading keeps the + /// walk honest about the staged copies: Mgx.Cmdlets, Polly.Core and the rest are already + /// loaded in this process from the test's own output folder, and loading them again from + /// module/ would just hand back the copy that is already there. + /// + private static IReadOnlyList ReferencesOf(string assemblyPath) + { + using var stream = File.OpenRead(assemblyPath); + using var pe = new PEReader(stream); + var metadata = pe.GetMetadataReader(); + + return [.. metadata.AssemblyReferences + .Select(handle => metadata.GetString(metadata.GetAssemblyReference(handle).Name))]; + } + + /// + /// The file a host would bind a reference to: the module folder first, then this process's own + /// folder, then the shared framework - app-local before framework, the order the runtime uses. + /// + private static string? ResolveReference(string simpleName, params string[] directories) + { + foreach (var directory in directories) + { + var candidate = Path.Combine(directory, $"{simpleName}.dll"); + if (File.Exists(candidate)) return candidate; + } + + return null; + } + + /// + /// The runtime's own implementation assemblies. They are in no reference pack because they + /// have no compile-time surface, and no package ships them - they exist wherever the runtime + /// does, so a host cannot be missing one. + /// + private static bool IsRuntimePrivate(string simpleName) => + simpleName.StartsWith("System.Private.", StringComparison.Ordinal); + + private sealed record Reached(string Name, string ReferencedBy, string? Path); + + /// + /// Breadth-first over assembly references from the two staged assemblies, recording where each + /// name was first reached from and which file it binds to. + /// + private static IReadOnlyList Closure() + { + var moduleRoot = ModuleRoot(); + var dependencies = DependenciesDirectory(); + var probeOrder = new[] { moduleRoot, dependencies, AppContext.BaseDirectory, RuntimeDirectory() }; + + var roots = new[] { "Mgx.Cmdlets", "Mgx.Engine" }; + var reached = new List(); + var seen = new HashSet(roots, StringComparer.OrdinalIgnoreCase); + var queue = new Queue<(string Name, string Path)>(); + + foreach (var root in roots) + { + var path = Path.Combine(moduleRoot, $"{root}.dll"); + Assert.True(File.Exists(path), $"'{path}' not found. Run ./build.ps1 first."); + queue.Enqueue((root, path)); + } + + while (queue.Count > 0) + { + var (from, path) = queue.Dequeue(); + foreach (var name in ReferencesOf(path)) + { + if (!seen.Add(name)) continue; + + var resolved = ResolveReference(name, probeOrder); + reached.Add(new Reached(name, from, resolved)); + + // The host and everything below it is the host's to supply. + if (name == HostAssembly) continue; + if (resolved is not null) queue.Enqueue((name, resolved)); + } + } + + return reached; + } + + [Fact] + public void Every_referenced_assembly_resolves_from_the_module_layout_or_the_framework() + { + var mgx = DllNamesIn(ModuleRoot()); + var staged = DllNamesIn(DependenciesDirectory()); + var framework = SharedFrameworkAssemblyNames(ShippedTargetFramework()); + var closure = Closure(); + + var unsatisfied = closure + .Where(r => !mgx.Contains(r.Name) + && !staged.Contains(r.Name) + && !framework.Contains(r.Name) + && !IsRuntimePrivate(r.Name) + && r.Name != HostAssembly) + .Select(r => $"{r.Name} (referenced by {r.ReferencedBy})") + .ToList(); + + Assert.True(unsatisfied.Count == 0, + "assemblies in the module's closure that a host is not guaranteed to have and the " + + "module does not ship - stage each into module/Dependencies/ (build.ps1 and " + + "Mgx.Cmdlets.csproj's StageModuleOutput target both list what is staged): " + + string.Join("; ", unsatisfied)); + + // Two ways the check above goes quiet: a walk that stops after the roots, and a reference + // that resolves to nothing so its own references are never read. Neither shows up as a + // failure on its own, so both are asserted. + var dead = closure + .Where(r => r.Path is null && r.Name != HostAssembly) + .Select(r => $"{r.Name} (referenced by {r.ReferencedBy})") + .ToList(); + + Assert.True(dead.Count == 0, + "references the walk could not bind to a file, so nothing below them was read: " + + string.Join("; ", dead)); + + Assert.True(closure.Count >= ClosureFloor, + $"the walk reached {closure.Count} assemblies - it is not following references"); + } + + [Fact] + public void The_walk_up_does_not_cross_the_repository_root() + { + // A decoy module/mgx.psd1 sits above the repository root here - the shape of a worktree + // checked out under a directory that holds a module folder of its own, where a walk-up + // taking the first ancestor with a module/mgx.psd1 under it measures the decoy's staging + // and reports on a module nobody built. Anchoring on Mgx.slnx means the decoy is never + // reached: the staged manifest resolves under the marked root, and a root without one + // resolves to nothing rather than to the decoy above it. + var sandbox = Path.Combine(Path.GetTempPath(), $"mgx-closure-root-test-{Guid.NewGuid():N}"); + var manifest = Path.Combine("module", "mgx.psd1"); + var repoRoot = Path.Combine(sandbox, "repo"); + var bareRoot = Path.Combine(sandbox, "bare"); + try + { + Directory.CreateDirectory(Path.Combine(sandbox, "module")); + File.WriteAllText(Path.Combine(sandbox, manifest), string.Empty); + + Directory.CreateDirectory(Path.Combine(repoRoot, "module")); + File.WriteAllText(Path.Combine(repoRoot, RepositoryMarker), string.Empty); + File.WriteAllText(Path.Combine(repoRoot, manifest), string.Empty); + var startDirectory = Path.Combine(repoRoot, "tests", "x", "bin", "Debug"); + Directory.CreateDirectory(startDirectory); + + Assert.Equal(Path.Combine(repoRoot, manifest), + FindRepositoryPath(manifest, startDirectory)); + + // The same tree with nothing staged under its root: the decoy above is still not an + // answer, and a walk that climbs past the marker to find one binds it here. + Directory.CreateDirectory(bareRoot); + File.WriteAllText(Path.Combine(bareRoot, RepositoryMarker), string.Empty); + var bareStart = Path.Combine(bareRoot, "tests", "x", "bin", "Debug"); + Directory.CreateDirectory(bareStart); + + try + { + var climbed = FindRepositoryPath(manifest, bareStart); + Assert.Fail($"a root with no '{manifest}' of its own resolved one at '{climbed}'"); + } + catch (FileNotFoundException) + { + // Nothing above the root was consulted, which is the whole of it. + } + } + finally + { + Directory.Delete(sandbox, recursive: true); + } + } + + [Fact] + public void The_staged_dependencies_are_carried_not_compiled_against() + { + // Dependencies/ holds two different kinds of file. Polly.Core and + // System.Threading.RateLimiting are ours: the engine compiles against them, and they are + // isolated there so a neighboring module's copy cannot win the bind. System.IO.Pipelines + // is nobody's: it is carried because the System.Text.Json a newer host supplies needs it + // and net8.0 does not include it. If it ever turned into a compile-time reference, the + // engine would be built against a surface the framework it targets does not have. + var moduleRoot = ModuleRoot(); + var referenced = ReferencesOf(Path.Combine(moduleRoot, "Mgx.Engine.dll")) + .Concat(ReferencesOf(Path.Combine(moduleRoot, "Mgx.Cmdlets.dll"))) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + Assert.DoesNotContain("System.IO.Pipelines", referenced); + Assert.Contains("Polly.Core", referenced); + Assert.Contains("System.Threading.RateLimiting", referenced); + } +} diff --git a/tests/Mgx.IntegrationTests/TestSetup/RegressionCorpusIndexTests.cs b/tests/Mgx.IntegrationTests/TestSetup/RegressionCorpusIndexTests.cs index 6cb35de..85a9d7a 100644 --- a/tests/Mgx.IntegrationTests/TestSetup/RegressionCorpusIndexTests.cs +++ b/tests/Mgx.IntegrationTests/TestSetup/RegressionCorpusIndexTests.cs @@ -5,7 +5,10 @@ namespace Mgx.IntegrationTests; /// /// Provenance is only useful if it can be trusted, and a doc comment naming an issue number is /// unverifiable on its own. Every issue id written anywhere under tests/ has to have a row in -/// tests/CORPUS.md saying what it guarantees and where. +/// the index saying what it guarantees and where. The index itself is private (tmp/CORPUS.md, +/// untracked), so a host without it - public CI - runs these as counted no-ops: the guard +/// binds wherever the index lives. Every path here resolves under the repository root - the +/// directory holding Mgx.slnx - and nothing above that root is ever consulted. /// /// The guard runs one way only. The reverse - every indexed id has a test - cannot hold while /// the index carries rows that name the release bringing their test, which is the honest way to @@ -13,7 +16,8 @@ namespace Mgx.IntegrationTests; /// public class RegressionCorpusIndexTests { - private const string IndexPath = "tests/CORPUS.md"; + private const string IndexPath = "tmp/CORPUS.md"; + private const string RepositoryMarker = "Mgx.slnx"; /// /// Matches an issue id and any ids abbreviated onto it: "M365DSC-5306/7175" is two ids, and @@ -33,21 +37,51 @@ private static IEnumerable IdsIn(string text) } } - /// Walk up from the test binaries until the repository-relative path exists. - private static string FindRepositoryPath(string relativePath) + /// + /// Walk up from to the first ancestor containing + /// Mgx.slnx and return it as the repository root. Nothing above that directory is ever + /// consulted, so a decoy further up - a worktree's own tmp/, a stranger's tests/ - cannot + /// bind. + /// + private static string FindRepositoryRoot(string startDirectory) { - var dir = new DirectoryInfo(AppContext.BaseDirectory); + var dir = new DirectoryInfo(startDirectory); while (dir != null) { - var candidate = Path.Combine(dir.FullName, relativePath); - if (File.Exists(candidate) || Directory.Exists(candidate)) return candidate; + if (File.Exists(Path.Combine(dir.FullName, RepositoryMarker))) return dir.FullName; dir = dir.Parent; } throw new DirectoryNotFoundException( - $"Could not locate '{relativePath}' above {AppContext.BaseDirectory}"); + $"Could not locate {RepositoryMarker} above {startDirectory}"); + } + + /// Resolve a repository-relative path under the root reached from . + private static string FindRepositoryPath(string relativePath, string startDirectory) + { + var root = FindRepositoryRoot(startDirectory); + var candidate = Path.Combine(root, relativePath); + if (File.Exists(candidate) || Directory.Exists(candidate)) return candidate; + + throw new DirectoryNotFoundException( + $"Could not locate '{relativePath}' under repository root {root}"); + } + + /// FindRepositoryPath, starting from the test binaries. + private static string FindRepositoryPath(string relativePath) => + FindRepositoryPath(relativePath, AppContext.BaseDirectory); + + /// FindRepositoryPath for a file allowed to be absent: null instead of a throw. + private static string? TryFindRepositoryPath(string relativePath, string startDirectory) + { + try { return FindRepositoryPath(relativePath, startDirectory); } + catch (DirectoryNotFoundException) { return null; } } + /// TryFindRepositoryPath, starting from the test binaries. + private static string? TryFindRepositoryPath(string relativePath) => + TryFindRepositoryPath(relativePath, AppContext.BaseDirectory); + /// /// Every source file under tests/, minus build output, the index itself, and this file - which /// carries fabricated ids on purpose and would otherwise fail its own guard. @@ -55,9 +89,10 @@ private static string FindRepositoryPath(string relativePath) private static IEnumerable CorpusSourceFiles() { var testsRoot = FindRepositoryPath("tests"); + // The index lives outside tests/ and needs no exclusion; this file carries fabricated + // ids on purpose and would otherwise fail its own guard. var excluded = new[] { - Path.GetFullPath(FindRepositoryPath(IndexPath)), Path.GetFullPath(FindRepositoryPath( Path.Combine("tests", "Mgx.IntegrationTests", "TestSetup", "RegressionCorpusIndexTests.cs"))) }; @@ -84,7 +119,8 @@ private static IReadOnlyList Unindexed(string indexText, string claiming [Fact] public void Every_issue_id_under_tests_has_a_row_in_the_index() { - var indexText = File.ReadAllText(FindRepositoryPath(IndexPath)); + if (TryFindRepositoryPath(IndexPath) is not string indexFile) return; + var indexText = File.ReadAllText(indexFile); var missing = new SortedDictionary>(StringComparer.Ordinal); var claimed = new HashSet(StringComparer.Ordinal); @@ -122,7 +158,8 @@ public void A_fabricated_id_in_a_test_file_fails_the_guard() { // What the guard above is worth: an id nobody has indexed has to be reported, including // one abbreviated onto an id that IS indexed. - var indexText = File.ReadAllText(FindRepositoryPath(IndexPath)); + if (TryFindRepositoryPath(IndexPath) is not string indexFile) return; + var indexText = File.ReadAllText(indexFile); Assert.Equal(["GraphSDK-9999"], Unindexed(indexText, "/// (Corpus: GraphSDK-9999, an issue that does not exist.)")); @@ -134,8 +171,37 @@ public void A_fabricated_id_in_a_test_file_fails_the_guard() [Fact] public void The_index_is_where_the_suite_says_it_is() { - // The file moved from TestSetup/ during 2.1.4. A stale path here would make both guards - // above throw rather than fail, which reads as a broken test rather than a broken index. - Assert.True(File.Exists(FindRepositoryPath(IndexPath))); + // A stale path here would read as an absent index and quietly turn all three guards + // into no-ops on every host; a host that carries the index pins the path with this. + if (TryFindRepositoryPath("tmp") is not string) return; + Assert.True(TryFindRepositoryPath(IndexPath) is not null); + } + + [Fact] + public void The_walk_up_does_not_cross_the_repository_root() + { + // A decoy tmp/ sits above the repository root here - the exact shape of a worktree + // checked out under the system temp directory, where the walk-up used to resolve the + // system temp directory itself as the repository's tmp/. Anchoring on Mgx.slnx means + // the decoy is never reached: "tests" resolves under the marked root, and "tmp" - which + // exists only above the root - resolves to nothing. + var sandbox = Path.Combine(Path.GetTempPath(), $"mgx-corpus-root-test-{Guid.NewGuid():N}"); + var repoRoot = Path.Combine(sandbox, "repo"); + var startDirectory = Path.Combine(repoRoot, "tests", "x", "bin", "Debug"); + try + { + Directory.CreateDirectory(Path.Combine(sandbox, "tmp")); + Directory.CreateDirectory(repoRoot); + File.WriteAllText(Path.Combine(repoRoot, "Mgx.slnx"), string.Empty); + Directory.CreateDirectory(Path.Combine(repoRoot, "tests")); + Directory.CreateDirectory(startDirectory); + + Assert.Equal(Path.Combine(repoRoot, "tests"), FindRepositoryPath("tests", startDirectory)); + Assert.Null(TryFindRepositoryPath("tmp", startDirectory)); + } + finally + { + Directory.Delete(sandbox, recursive: true); + } } } diff --git a/tests/TestHarness.psm1 b/tests/TestHarness.psm1 index e25ec60..97160b3 100644 --- a/tests/TestHarness.psm1 +++ b/tests/TestHarness.psm1 @@ -56,7 +56,8 @@ function Invoke-TestHarness Directory to search for *.Tests.ps1. Defaults to tests/Unit. .OUTPUTS - The Pester run object. Callers check $result.FailedCount. + The Pester run object. Callers check $result.Result (not 'Passed' when a test + failed OR a container could not run) and $result.FailedContainersCount. #> [CmdletBinding()] param @@ -123,7 +124,19 @@ function Invoke-TestHarness Write-Verbose -Message 'Code coverage is not collected for a binary module; continuing without it.' } - return Invoke-Pester -Configuration $configuration + $result = Invoke-Pester -Configuration $configuration + + # FailedCount counts tests only. A container that never ran at all - a parse error, a + # top-level throw, a dot-sourced helper that is gone - fails the container and leaves + # FailedCount at 0, so a caller checking FailedCount alone sees a clean pass. Name it here + # so a local run sees it even when the caller only logs the summary line. + foreach ($container in $result.FailedContainers) + { + $firstError = $container.ErrorRecord[0].Exception.Message + Write-Warning -Message "Container failed: $($container.Name) - $firstError" + } + + return $result } Export-ModuleMember -Function Invoke-TestHarness, Get-MgxTestPath diff --git a/tests/Unit/Benchmarks/Mgx.BenchBaseline.Tests.ps1 b/tests/Unit/Benchmarks/Mgx.BenchBaseline.Tests.ps1 new file mode 100644 index 0000000..d58ea94 --- /dev/null +++ b/tests/Unit/Benchmarks/Mgx.BenchBaseline.Tests.ps1 @@ -0,0 +1,882 @@ +#Requires -Modules Pester + +<# + Baseline promotion and comparison in tests/benchmarks/common.ps1. + + These cover the benchmark harness rather than the built module: no tenant, no Graph + session, and nothing read out of the real results/ directory - every fixture is written + under $TestDrive and every call is pointed at it. +#> + +BeforeAll { + $script:BenchRoot = Join-Path (Split-Path -Parent (Split-Path -Parent $PSScriptRoot)) 'benchmarks' + + # common.ps1 pins the thread to invariant culture so benchmark output never prints 39,2s. + # Right for a benchmark process, and not something to leave behind in a test run. + $script:CultureBefore = [System.Threading.Thread]::CurrentThread.CurrentCulture + . (Join-Path $script:BenchRoot 'common.ps1') + + function script:New-BenchEntry + { + param + ( + [Parameter(Mandatory)] [hashtable] $Result, + [datetime] $RecordedAt = (Get-Date) + ) + + return [pscustomobject]@{ + Result = [pscustomobject]$Result + Telemetry = $null + MgxVersion = '2.1.5' + SdkVersion = '2.34.0' + PSVersion = '7.6.4' + RecordedAt = $RecordedAt.ToString('o') + } + } + + function script:Write-BenchFixture + { + param + ( + [Parameter(Mandatory)] [string] $Directory, + [Parameter(Mandatory)] [string] $FileName, + [Parameter(Mandatory)] [object[]] $Entries + ) + + if (-not (Test-Path -Path $Directory)) + { + New-Item -ItemType Directory -Path $Directory -Force | Out-Null + } + ConvertTo-Json -InputObject @($Entries) -Depth 10 | + Set-Content -Path (Join-Path $Directory $FileName) + } + + # A 06 gauntlet result in the shape 06-fault-gauntlet.ps1 records. + function script:New-GauntletResult + { + param ([Parameter(Mandatory)] [int] $MgxMs) + + return @{ + mgx = [pscustomobject]@{ ElapsedMs = $MgxMs; Output = [pscustomobject]@{ ok = 1000; failed = 0 } } + sdkMgx = [pscustomobject]@{ ElapsedMs = 246715 } + } + } +} + +AfterAll { + [System.Threading.Thread]::CurrentThread.CurrentCulture = $script:CultureBefore +} + +Describe 'benchmark scenario table' { + It 'defines metrics for every benchmark run.ps1 executes' { + # Everything that writes or reads a baseline keys off this table. A scenario added to + # the suite without an entry here is pinned as nothing and read as unmeasured. + $runText = Get-Content -Path (Join-Path $script:BenchRoot 'run.ps1') -Raw + $suiteIds = [regex]::Matches($runText, "id\s*=\s*'(\d+)'") | ForEach-Object { $_.Groups[1].Value } + $suiteIds.Count | Should -BeGreaterThan 0 + + $known = (Get-BenchScenario).Id + foreach ($id in $suiteIds) + { + $known | Should -Contain $id -Because "run.ps1 runs benchmark $id" + } + } + + It 'names each scenario a headline metric first' { + foreach ($scenario in (Get-BenchScenario)) + { + $scenario.Metrics.Count | Should -BeGreaterThan 0 + $scenario.Metrics[0].Name | Should -Not -BeNullOrEmpty + } + } + + It 'skips an id it has no metrics for rather than throwing' { + Get-BenchScenario -Id '99' -WarningAction SilentlyContinue | Should -BeNullOrEmpty + } +} + +Describe 'Export-BenchBaseline' { + BeforeEach { + $script:Results = Join-Path $TestDrive 'results' + $script:Baseline = Join-Path $TestDrive 'baseline.json' + Remove-Item -Path $script:Results, $script:Baseline -Recurse -Force -ErrorAction SilentlyContinue + } + + It 'promotes the newest entry, its headline metric and its version block' { + Write-BenchFixture -Directory $script:Results -FileName '06-fault-gauntlet.json' -Entries @( + (New-BenchEntry -Result (New-GauntletResult -MgxMs 111111) -RecordedAt (Get-Date).AddDays(-2)), + (New-BenchEntry -Result (New-GauntletResult -MgxMs 134654)) + ) + + Export-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -Path $script:Baseline + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'06' + $pinned.Script | Should -Be '06-fault-gauntlet.ps1' + $pinned.Headline | Should -Be 'MgxFanoutMs' + $pinned.Metrics.MgxFanoutMs.Value | Should -Be 134654 + $pinned.Metrics.MgxFanoutMs.Unit | Should -Be 'ms' + $pinned.Metrics.MgxCompleted.Value | Should -Be 1000 + $pinned.MgxVersion | Should -Be '2.1.5' + $pinned.SdkVersion | Should -Be '2.34.0' + $pinned.PSVersion | Should -Be '7.6.4' + $pinned.RecordedAt | Should -Not -BeNullOrEmpty + } + + It 'leaves scenarios it was not asked about where they were' { + Write-BenchFixture -Directory $script:Results -FileName '06-fault-gauntlet.json' -Entries @( + (New-BenchEntry -Result (New-GauntletResult -MgxMs 134654)) + ) + Write-BenchFixture -Directory $script:Results -FileName '08-delta-sync.json' -Entries @( + (New-BenchEntry -Result @{ initial = [pscustomobject]@{ ElapsedMs = 145621; Output = [pscustomobject]@{ items = 130233 } } }) + ) + + Export-BenchBaseline -Benchmark '08' -ResultsDirectory $script:Results -Path $script:Baseline + Export-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -Path $script:Baseline + + $scenarios = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios + $scenarios.'08'.Metrics.InitialSyncMs.Value | Should -Be 145621 + $scenarios.'06'.Metrics.MgxFanoutMs.Value | Should -Be 134654 + } + + It 'pins no headline when the first metric of the scenario row is absent, rather than the next one' { + # 09's row names CheckpointedExportMs first. An entry with no 'checkpointed' node still + # carries four of its five metrics, and the next one that resolves is not a stand-in for + # the one it is read on - CheckpointOverheadPct is a percentage that crosses zero. + Write-BenchFixture -Directory $script:Results -FileName '09-kill-resume.json' -Entries @( + (New-BenchEntry -Result @{ + baseline = [pscustomobject]@{ ElapsedMs = 140322 } + resume = [pscustomobject]@{ ElapsedMs = 38104 } + verification = [pscustomobject]@{ overheadPct = -3.2; duplicateIds = 0 } + }) + ) + + Export-BenchBaseline -Benchmark '09' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningAction SilentlyContinue + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'09' + $pinned.Headline | Should -BeNullOrEmpty + $pinned.Metrics.BaselineExportMs.Value | Should -Be 140322 + $pinned.Metrics.CheckpointOverheadPct.Value | Should -Be -3.2 + + # And the comparison says so rather than reading whatever is first in the file. + $row = Compare-BenchBaseline -Benchmark '09' -ResultsDirectory $script:Results ` + -BaselinePath $script:Baseline + $row.Status | Should -Be 'not comparable' + $row.Note | Should -BeLike '*no headline*' + $row.Metric | Should -BeNullOrEmpty + $row.DeltaPct | Should -BeNullOrEmpty + } + + It 'tries a metric''s next source when the first one is not numeric' { + # 10's WallSeconds tries Arms.paced.Seconds before WallMs. An entry that recorded the + # scenario as inconclusive on its first source still carries a number on its second, + # and losing that to the first source's non-numeric value would cost WallSeconds its + # headline along with it. + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Mode = 'paced' + Arms = [pscustomobject]@{ paced = [pscustomobject]@{ Seconds = 'INCONCLUSIVE'; ThrottleRetries = 12 } } + WallMs = 39400 + }) + ) + + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningAction SilentlyContinue + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'10' + $pinned.Headline | Should -Be 'WallSeconds' + $pinned.Metrics.WallSeconds.Value | Should -Be 39.4 + $pinned.Metrics.WallSeconds.Path | Should -Be 'WallMs' + } + + It 'warns which metric was not recorded when none of its sources resolves' { + # WallSeconds tries Arms.paced.Seconds, then WallMs. An entry that leaves the first + # inconclusive and never writes the second must not be dropped in silence, and the + # warning it gets must name the value that was not a number rather than just listing + # every path tried - WallMs was never written, and a path with nothing at it is not + # part of the story. + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Mode = 'paced' + Arms = [pscustomobject]@{ + paced = [pscustomobject]@{ Seconds = 'INCONCLUSIVE'; ThrottleRetries = 12 } + unpaced = [pscustomobject]@{ Seconds = 40.3 } + } + }) + ) + + $warnings = $null + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningVariable warnings 3>$null + + "$warnings" | Should -BeLike "*10: 'WallSeconds' not recorded - 'Arms.paced.Seconds' holds 'INCONCLUSIVE', which is not a number*" + "$warnings" | Should -Not -BeLike '*WallMs*' + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'10' + $pinned.Headline | Should -BeNullOrEmpty + $pinned.Metrics.WallSeconds | Should -BeNullOrEmpty + $pinned.Metrics.ThrottleRetries.Value | Should -Be 12 + } + + It 'does not pin an empty string or a boolean as a measurement' { + # [double] coerces both without throwing - '' to 0, $true to 1 - so a source that + # never recorded a number could be pinned as though it had. Neither belongs in the + # baseline, and the warning must name what each source actually held. + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Mode = 'both' + Arms = [pscustomobject]@{ + paced = [pscustomobject]@{ Seconds = ''; ThrottleRetries = 3 } + unpaced = [pscustomobject]@{ Seconds = 40.3 } + } + WallMs = $true + }) + ) + + $warnings = $null + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningVariable warnings 3>$null + + "$warnings" | Should -BeLike "*'Arms.paced.Seconds' holds '', which is not a number*" + "$warnings" | Should -BeLike "*'WallMs' holds 'True', which is not a number*" + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'10' + $pinned.Headline | Should -BeNullOrEmpty + $pinned.Metrics.WallSeconds | Should -BeNullOrEmpty + $pinned.Metrics.UnpacedWallSeconds.Value | Should -Be 40.3 + $pinned.Metrics.ThrottleRetries.Value | Should -Be 3 + } + + It 'refuses "NaN", "Infinity" and "-Infinity" as measurements, named in the not-recorded warning' { + # NumberStyles.Float - the style [double]::TryParse is called with - parses these three + # strings as valid doubles: NaN, PositiveInfinity, NegativeInfinity. None is a + # measurement a baseline can hold a percentage delta against, so each must be refused + # and named rather than pinned. + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Mode = 'both' + Arms = [pscustomobject]@{ + paced = [pscustomobject]@{ Seconds = 'NaN'; ThrottleRetries = 3 } + unpaced = [pscustomobject]@{ Seconds = '-Infinity' } + } + WallMs = 'Infinity' + }) + ) + + $warnings = $null + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningVariable warnings 3>$null + + "$warnings" | Should -BeLike "*10: 'WallSeconds' not recorded - 'Arms.paced.Seconds' holds 'NaN', which is not a number; 'WallMs' holds 'Infinity', which is not a number.*" + "$warnings" | Should -BeLike "*10: 'UnpacedWallSeconds' not recorded - 'Arms.unpaced.Seconds' holds '-Infinity', which is not a number.*" + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'10' + $pinned.Headline | Should -BeNullOrEmpty + $pinned.Metrics.WallSeconds | Should -BeNullOrEmpty + $pinned.Metrics.UnpacedWallSeconds | Should -BeNullOrEmpty + $pinned.Metrics.ThrottleRetries.Value | Should -Be 3 + } + + It 'refuses a typed double NaN, not just the string it prints as' { + # ConvertTo-Json always re-quotes NaN and the infinities as strings, so the fixture + # helper above can only exercise the string arm of Test-BenchNumericValue. A result + # file written with JSON's non-standard bare NaN token round-trips through + # ConvertFrom-Json as a genuine [double] - the other arm - and that value must be + # refused too, not coerced into a measurement by the typed-value check alone. + New-Item -ItemType Directory -Path $script:Results -Force | Out-Null + $raw = @" +[ + { + "Result": { "Mode": "paced", "Arms": { "paced": { "Seconds": NaN, "ThrottleRetries": 12 } } }, + "Telemetry": null, + "MgxVersion": "2.1.5", + "SdkVersion": "2.34.0", + "PSVersion": "7.6.4", + "RecordedAt": "$((Get-Date).ToString('o'))" + } +] +"@ + Set-Content -Path (Join-Path $script:Results '10-pacing-under-real-throttling.json') -Value $raw + + $warnings = $null + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningVariable warnings 3>$null + + "$warnings" | Should -BeLike "*10: 'WallSeconds' not recorded - 'Arms.paced.Seconds' holds 'NaN', which is not a number*" + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'10' + $pinned.Headline | Should -BeNullOrEmpty + $pinned.Metrics.WallSeconds | Should -BeNullOrEmpty + $pinned.Metrics.ThrottleRetries.Value | Should -Be 12 + } + + It 'warns per rescued non-number and pins the metric from the source that resolved it' { + # A first source that recorded 'INCONCLUSIVE' before a second recorded the real figure + # used to fall back in silence. The rescue is now visible - one warning per source that + # was tried and skipped - and the metric still pins from the source that resolved it. + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Mode = 'paced' + Arms = [pscustomobject]@{ paced = [pscustomobject]@{ Seconds = 'INCONCLUSIVE'; ThrottleRetries = 12 } } + WallMs = 41200 + }) + ) + + $warnings = $null + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningVariable warnings 3>$null + + "$warnings" | Should -BeLike "*10: 'WallSeconds' - 'Arms.paced.Seconds' holds 'INCONCLUSIVE', which is not a number - trying the next source.*" + "$warnings" | Should -Not -BeLike '*not recorded*' + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'10' + $pinned.Headline | Should -Be 'WallSeconds' + $pinned.Metrics.WallSeconds.Value | Should -Be 41.2 + $pinned.Metrics.WallSeconds.Path | Should -Be 'WallMs' + } + + It 'does not warn about an optional source a single-arm entry never writes' { + # UnpacedWallSeconds' only source is Arms.unpaced.Seconds, written only when run.ps1 + # drives both arms. A standalone -Mode paced run never writes it - that is not a + # metric gone missing, it is one this entry was never going to carry, and every + # healthy single-arm run must promote in silence. + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Mode = 'paced' + Arms = [pscustomobject]@{ paced = [pscustomobject]@{ Seconds = 41.2; ThrottleRetries = 3 } } + }) + ) + + $warnings = $null + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningVariable warnings 3>$null + + $warnings | Should -BeNullOrEmpty + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'10' + $pinned.Headline | Should -Be 'WallSeconds' + $pinned.Metrics.UnpacedWallSeconds | Should -BeNullOrEmpty + $pinned.Metrics.WallSeconds.Value | Should -Be 41.2 + } + + It 'does not promote an entry older than the run that asked for it' { + # A benchmark that died before recording leaves the previous run's entry newest. + Write-BenchFixture -Directory $script:Results -FileName '06-fault-gauntlet.json' -Entries @( + (New-BenchEntry -Result (New-GauntletResult -MgxMs 134654) -RecordedAt (Get-Date).AddDays(-2)) + ) + + Export-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -Path $script:Baseline ` + -Since (Get-Date) -WarningAction SilentlyContinue + + Test-Path -Path $script:Baseline | Should -BeFalse + } + + It 'does not promote a run that recorded itself inconclusive, and names the scenario' { + # 10 records Conclusive false when its unpaced arm was never refused: the throttled + # regime it exists to measure was never reached, so the wall times beside the verdict + # are two unthrottled runs. They are numbers, and every later run would be read + # against them. + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Conclusive = $false + Arms = [pscustomobject]@{ + paced = [pscustomobject]@{ Seconds = 399.7; ThrottleRetries = 0 } + unpaced = [pscustomobject]@{ Seconds = 79.8 } + } + }) + ) + + $warnings = $null + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningVariable warnings 3>$null + + "$warnings" | Should -BeLike '*benchmark 10 recorded an inconclusive run (Conclusive is false) - not promoted*' + Test-Path -Path $script:Baseline | Should -BeFalse + } + + It 'quotes the reason an inconclusive entry carries, where it carries one' { + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Conclusive = $false + InconclusiveReason = 'the unpaced arm was never throttled' + Arms = [pscustomobject]@{ + paced = [pscustomobject]@{ Seconds = 399.7; ThrottleRetries = 0 } + unpaced = [pscustomobject]@{ Seconds = 79.8 } + } + }) + ) + + $warnings = $null + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningVariable warnings 3>$null + + "$warnings" | Should -BeLike '*(the unpaced arm was never throttled)*' + "$warnings" | Should -Not -BeLike '*Conclusive is false*' + } + + It 'promotes a run that recorded itself conclusive' { + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Conclusive = $true + Arms = [pscustomobject]@{ + paced = [pscustomobject]@{ Seconds = 61.4; ThrottleRetries = 3 } + unpaced = [pscustomobject]@{ Seconds = 88.2 } + } + }) + ) + + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'10' + $pinned.Headline | Should -Be 'WallSeconds' + $pinned.Metrics.WallSeconds.Value | Should -Be 61.4 + $pinned.Metrics.UnpacedWallSeconds.Value | Should -Be 88.2 + } + + It 'promotes a scenario that records no verdict at all' { + # The field is a claim a scenario may make, not one the promotion requires. Seven of + # the eight scenarios in the file never write it. + Write-BenchFixture -Directory $script:Results -FileName '06-fault-gauntlet.json' -Entries @( + (New-BenchEntry -Result (New-GauntletResult -MgxMs 134654)) + ) + + Export-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -Path $script:Baseline + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'06' + $pinned.Metrics.MgxFanoutMs.Value | Should -Be 134654 + } + + It 'leaves the row already pinned where it was when the next run is inconclusive' { + # A stale conclusive number is still a measurement, and the comparison reports its age. + # Dropping the row instead would report the scenario as unmeasured, which is the one + # thing the file exists to prevent. + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Conclusive = $true + Arms = [pscustomobject]@{ + paced = [pscustomobject]@{ Seconds = 61.4; ThrottleRetries = 3 } + unpaced = [pscustomobject]@{ Seconds = 88.2 } + } + }) + ) + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline + + # A later suite: 10 comes back inconclusive and 06 records normally, so the file is + # rewritten around the row that must not move. + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Conclusive = $false + Arms = [pscustomobject]@{ + paced = [pscustomobject]@{ Seconds = 399.7; ThrottleRetries = 0 } + unpaced = [pscustomobject]@{ Seconds = 79.8 } + } + }) + ) + Write-BenchFixture -Directory $script:Results -FileName '06-fault-gauntlet.json' -Entries @( + (New-BenchEntry -Result (New-GauntletResult -MgxMs 134654)) + ) + + $warnings = $null + Export-BenchBaseline -Benchmark '06', '10' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningVariable warnings 3>$null + + $scenarios = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios + $scenarios.'10'.Metrics.WallSeconds.Value | Should -Be 61.4 + $scenarios.'10'.Metrics.UnpacedWallSeconds.Value | Should -Be 88.2 + $scenarios.'06'.Metrics.MgxFanoutMs.Value | Should -Be 134654 + "$warnings" | Should -BeLike '*the row already in the file is left as it was*' + } +} + +Describe 'Get-BenchScenarioMetric' { + It 'refuses a source whose scaled value overflows, and tries the next source' { + # None of the table's own Scale values can overflow a source that was itself finite - + # 10's only one, WallMs at 0.001, shrinks - so this is exercised directly against a + # scenario shaped like the real ones rather than through Export-BenchBaseline. A + # source read within double's range is not the same thing as a source whose scaled + # value is: '1e308' is a legitimate double, and Scale 1000 pushes it past + # [double]::MaxValue. + $scenario = [pscustomobject]@{ + Id = 'S' + Discriminator = $null + Metrics = @( + @{ Name = 'ScaledSeconds'; Unit = 's'; Sources = @(@{ Path = 'Raw'; Scale = 1000.0 }, @{ Path = 'Fallback' }) } + ) + } + $entry = [pscustomobject]@{ Result = [pscustomobject]@{ Raw = '1e308'; Fallback = 41.5 } } + + $warnings = $null + $read = Get-BenchScenarioMetric -Scenario $scenario -Entry $entry -WarningVariable warnings 3>$null + + "$warnings" | Should -BeLike "*S: 'ScaledSeconds' - 'Raw' holds '1e308', which scaled by 1000 is not finite - trying the next source.*" + $read.Metrics.ScaledSeconds.Value | Should -Be 41.5 + $read.Metrics.ScaledSeconds.Path | Should -Be 'Fallback' + } + + It 'warns and pins nothing when the only source overflows once scaled' { + $scenario = [pscustomobject]@{ + Id = 'S' + Discriminator = $null + Metrics = @( + @{ Name = 'ScaledSeconds'; Unit = 's'; Sources = @(@{ Path = 'Raw'; Scale = 1000.0 }) } + ) + } + $entry = [pscustomobject]@{ Result = [pscustomobject]@{ Raw = '1e308' } } + + $warnings = $null + $read = Get-BenchScenarioMetric -Scenario $scenario -Entry $entry -WarningVariable warnings 3>$null + + "$warnings" | Should -BeLike "*S: 'ScaledSeconds' not recorded - 'Raw' holds '1e308', which scaled by 1000 is not finite.*" + $read.Metrics.ScaledSeconds | Should -BeNullOrEmpty + } + + It 'names each source by path when two of them overflow the same metric' { + # Before the path was named here, two overflowing sources on the same metric printed + # the identical "'1e308' scaled by is not finite" for both, and the record could + # not say which one was which. Different scales on the same raw value pin that down. + $scenario = [pscustomobject]@{ + Id = 'S' + Discriminator = $null + Metrics = @( + @{ Name = 'ScaledSeconds'; Unit = 's'; Sources = @(@{ Path = 'RawA'; Scale = 1000.0 }, @{ Path = 'RawB'; Scale = 1e10 }) } + ) + } + $entry = [pscustomobject]@{ Result = [pscustomobject]@{ RawA = '1e308'; RawB = '1e308' } } + + $warnings = $null + $read = Get-BenchScenarioMetric -Scenario $scenario -Entry $entry -WarningVariable warnings 3>$null + + "$warnings" | Should -BeLike "*'RawA' holds '1e308', which scaled by 1000 is not finite*" + "$warnings" | Should -BeLike "*'RawB' holds '1e308', which scaled by 10000000000 is not finite*" + $read.Metrics.ScaledSeconds | Should -BeNullOrEmpty + } + + It 'shows a non-number raw value the way the comparison does, not PowerShell''s default array join' { + # Interpolating an array or object directly prints PowerShell's own join - '1 2 3' for + # @(1,2,3), space-separated and unrecognizable as the source. ConvertTo-BenchDisplayValue + # is what Compare-BenchBaseline already reads a non-number baseline value through; the + # record path's warning is read the same way rather than through raw interpolation. + $scenario = [pscustomobject]@{ + Id = 'S' + Discriminator = $null + Metrics = @( + @{ Name = 'Metric'; Unit = 'ms'; Sources = @(@{ Path = 'Raw' }) } + ) + } + $entry = [pscustomobject]@{ Result = [pscustomobject]@{ Raw = @(1, 2, 3) } } + + $warnings = $null + $read = Get-BenchScenarioMetric -Scenario $scenario -Entry $entry -WarningVariable warnings 3>$null + + # -BeLike reads a bracketed substring as a wildcard character class, not literal text, + # so the exact message is compared instead - it also pins the whole sentence rather + # than just the fragment that changed. + "$warnings" | Should -Be "S: 'Metric' not recorded - 'Raw' holds '[1,2,3]', which is not a number." + "$warnings" | Should -Not -BeLike '*1 2 3*' + $read.Metrics.Metric | Should -BeNullOrEmpty + } +} + +Describe 'Compare-BenchBaseline' { + BeforeEach { + $script:Results = Join-Path $TestDrive 'results' + $script:Baseline = Join-Path $TestDrive 'baseline.json' + Remove-Item -Path $script:Results, $script:Baseline -Recurse -Force -ErrorAction SilentlyContinue + + Write-BenchFixture -Directory $script:Results -FileName '06-fault-gauntlet.json' -Entries @( + (New-BenchEntry -Result (New-GauntletResult -MgxMs 134654)) + ) + Export-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -Path $script:Baseline + } + + It 'reports a delta on the headline metric' { + $row = Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -BaselinePath $script:Baseline + $row.Scenario | Should -Be '06' + $row.Metric | Should -Be 'MgxFanoutMs' + $row.Status | Should -Be 'compared' + $row.DeltaPct | Should -Be 0 + $row.Flagged | Should -BeFalse + } + + It 'flags a headline metric past the threshold without failing' { + Write-BenchFixture -Directory $script:Results -FileName '06-fault-gauntlet.json' -Entries @( + (New-BenchEntry -Result (New-GauntletResult -MgxMs 269308)) + ) + + $row = Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -BaselinePath $script:Baseline + $row.DeltaPct | Should -Be 100 + $row.Flagged | Should -BeTrue + $row.Note | Should -Be 'REGRESSED' + } + + It 'reports a scenario with no baseline entry as unmeasured rather than skipping it' { + Write-BenchFixture -Directory $script:Results -FileName '08-delta-sync.json' -Entries @( + (New-BenchEntry -Result @{ initial = [pscustomobject]@{ ElapsedMs = 145621 } }) + ) + + $rows = Compare-BenchBaseline -Benchmark '06', '08' -ResultsDirectory $script:Results -BaselinePath $script:Baseline + $rows.Count | Should -Be 2 + ($rows | Where-Object { $_.Scenario -eq '08' }).Status | Should -Be 'unmeasured' + } + + It 'reports a scenario that recorded nothing this run' { + $rows = Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results ` + -BaselinePath $script:Baseline -Since (Get-Date).AddDays(1) + $rows.Status | Should -Be 'no result' + } + + It 'says so and returns nothing when there is no baseline' { + $rows = Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results ` + -BaselinePath (Join-Path $TestDrive 'absent.json') + $rows | Should -BeNullOrEmpty + } + + It 'refuses to compare two entries read from different paths' { + # 10 records a two-arm summary through run.ps1 and a single arm when run standalone. + # Seconds against milliseconds would read as a 1000x regression. + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ Arms = [pscustomobject]@{ + paced = [pscustomobject]@{ Seconds = 39.4; ThrottleRetries = 12 } + unpaced = [pscustomobject]@{ Seconds = 40.3 } } }) + ) + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline + + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ Mode = 'paced'; WallMs = 39400; ThrottleRetries = 12 }) + ) + + $row = Compare-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -BaselinePath $script:Baseline + $row.Status | Should -Be 'not comparable' + $row.DeltaPct | Should -BeNullOrEmpty + } + + It 'refuses a negative baseline, naming the sign' { + # No headline in the table can go negative today, which is what the guard is for: it + # keeps that true of the next one added. Over a negative baseline the percentage delta + # inverts, so -9ms against -4.5ms would be reported as a 100% regression by a figure + # that in fact doubled in the same direction. + $negative = [pscustomobject]@{ + SchemaVersion = 1 + GeneratedAt = (Get-Date).ToString('o') + Scenarios = [pscustomobject]@{ + '06' = [pscustomobject]@{ + Scenario = '06' + Script = '06-fault-gauntlet.ps1' + Headline = 'MgxFanoutMs' + Metrics = [pscustomobject]@{ + MgxFanoutMs = [pscustomobject]@{ Value = -4.5; Unit = 'ms'; Path = 'mgx.ElapsedMs' } + } + } + } + } + ConvertTo-Json -InputObject $negative -Depth 10 | Set-Content -Path $script:Baseline + + Write-BenchFixture -Directory $script:Results -FileName '06-fault-gauntlet.json' -Entries @( + (New-BenchEntry -Result (New-GauntletResult -MgxMs (-9))) + ) + + $row = Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results ` + -BaselinePath $script:Baseline + $row.Status | Should -Be 'not comparable' + $row.Note | Should -BeLike '*negative*' + $row.DeltaPct | Should -BeNullOrEmpty + } + + It 'refuses to compare a paced arm against an unpaced baseline' { + Write-BenchFixture -Directory $script:Results -FileName '07-adaptive-pacing.json' -Entries @( + (New-BenchEntry -Result @{ Mode = 'unpaced'; ElapsedMs = 388108; Output = [pscustomobject]@{ failed = 0 } }) + ) + Export-BenchBaseline -Benchmark '07' -ResultsDirectory $script:Results -Path $script:Baseline + + Write-BenchFixture -Directory $script:Results -FileName '07-adaptive-pacing.json' -Entries @( + (New-BenchEntry -Result @{ Mode = 'paced'; ElapsedMs = 388108; Output = [pscustomobject]@{ failed = 0 } }) + ) + + $row = Compare-BenchBaseline -Benchmark '07' -ResultsDirectory $script:Results -BaselinePath $script:Baseline + $row.Status | Should -Be 'not comparable' + } + + It 'refuses a baseline holding the string "NaN" or "Infinity", naming it as a non-number' { + # ConvertTo-Json quotes a non-finite double as a string - "NaN" and "Infinity" among + # them - so a baseline.json promoted before Export-BenchBaseline refused a non-finite + # measurement still holds one of those tokens today, as a string rather than a typed + # double. Test-BenchNumericValue refuses a "NaN" string the same as it refuses any + # other non-numeric one, so this is caught by the new gate ahead of the cast, not by + # the finiteness check after it - a stale row like this must still be caught somewhere + # or the comparison reports it "compared" on a NaN. + $stale = [pscustomobject]@{ + SchemaVersion = 1 + GeneratedAt = (Get-Date).ToString('o') + Scenarios = [pscustomobject]@{ + '06' = [pscustomobject]@{ + Scenario = '06' + Script = '06-fault-gauntlet.ps1' + Headline = 'MgxFanoutMs' + Metrics = [pscustomobject]@{ + MgxFanoutMs = [pscustomobject]@{ Value = 'NaN'; Unit = 'ms'; Path = 'mgx.ElapsedMs' } + } + } + } + } + ConvertTo-Json -InputObject $stale -Depth 10 | Set-Content -Path $script:Baseline + + $row = Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -BaselinePath $script:Baseline + $row.Status | Should -Be 'not comparable' + $row.Note | Should -Be "not comparable - the baseline value is 'NaN', which is not a number" + $row.DeltaPct | Should -BeNullOrEmpty + + (Get-Content -Path $script:Baseline -Raw).Replace('"NaN"', '"Infinity"') | Set-Content -Path $script:Baseline + + $row2 = Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -BaselinePath $script:Baseline + $row2.Status | Should -Be 'not comparable' + $row2.Note | Should -Be "not comparable - the baseline value is 'Infinity', which is not a number" + $row2.DeltaPct | Should -BeNullOrEmpty + } + + It 'still reports a bare JSON NaN or Infinity baseline token as non-finite, not as a non-number' { + # A typed double is exempted from the not-a-number gate above even when it is NaN or an + # infinity, because the gate itself would otherwise refuse it as a non-number - the same + # refusal a "NaN" string earns - and this arm exists specifically to say something more + # precise than that about a value that did arrive as a genuine double. The only way a + # baseline.json value round-trips through ConvertFrom-Json as a typed double rather than + # a string is a bare, unquoted token, which is not standard JSON but is what a pre-guard + # Export-BenchBaseline could still have written raw. + New-Item -ItemType Directory -Path (Split-Path -Parent $script:Baseline) -Force | Out-Null + $raw = @" +{ + "SchemaVersion": 1, + "GeneratedAt": "$((Get-Date).ToString('o'))", + "Scenarios": { + "06": { + "Scenario": "06", + "Script": "06-fault-gauntlet.ps1", + "Headline": "MgxFanoutMs", + "Metrics": { "MgxFanoutMs": { "Value": NaN, "Unit": "ms", "Path": "mgx.ElapsedMs" } } + } + } +} +"@ + Set-Content -Path $script:Baseline -Value $raw + + $row = Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -BaselinePath $script:Baseline + $row.Status | Should -Be 'not comparable' + $row.Note | Should -Be 'not comparable - the baseline value is NaN' + $row.DeltaPct | Should -BeNullOrEmpty + } + + It 'reports "INCONCLUSIVE", a boolean and an array as not comparable, naming each, with a healthy row still compared' { + # A stale baseline.json can hold anything a prior run's Value field once did - an + # inconclusive string, a boolean the old [double] cast would have silently read as 1, + # or an array or object nothing in the table ever wrote but nothing stopped from being + # written by hand. Three different shapes on three different scenarios, and a fourth, + # healthy scenario in the same call must still compare rather than the whole request + # failing on the first broken row. + Write-BenchFixture -Directory $script:Results -FileName '01-list-users.json' -Entries @( + (New-BenchEntry -Result @{ mgx = [pscustomobject]@{ Median = [pscustomobject]@{ ElapsedMs = 4200; PeakWorkingSetMB = 128 } } }) + ) + Write-BenchFixture -Directory $script:Results -FileName '02-fanout-lookup.json' -Entries @( + (New-BenchEntry -Result @{ mgx = [pscustomobject]@{ Median = [pscustomobject]@{ ElapsedMs = 6100; PeakWorkingSetMB = 96 } } }) + ) + Write-BenchFixture -Directory $script:Results -FileName '03-user-report.json' -Entries @( + (New-BenchEntry -Result @{ mgx = [pscustomobject]@{ Median = [pscustomobject]@{ ElapsedMs = 8300; PeakWorkingSetMB = 64 } } }) + ) + + # '06' rides along from the Describe's own BeforeEach, which already promoted a healthy + # baseline and result for it - carried forward here as the fourth, unbroken row rather + # than lost when the other three are written into the same file. + $healthy06 = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'06' + + $stale = [pscustomobject]@{ + SchemaVersion = 1 + GeneratedAt = (Get-Date).ToString('o') + Scenarios = [pscustomobject]@{ + '01' = [pscustomobject]@{ + Scenario = '01'; Script = '01-list-users.ps1'; Headline = 'MgxListMs' + Metrics = [pscustomobject]@{ MgxListMs = [pscustomobject]@{ Value = 'INCONCLUSIVE'; Unit = 'ms'; Path = 'mgx.Median.ElapsedMs' } } + } + '02' = [pscustomobject]@{ + Scenario = '02'; Script = '02-fanout-lookup.ps1'; Headline = 'MgxFanoutMs' + Metrics = [pscustomobject]@{ MgxFanoutMs = [pscustomobject]@{ Value = $true; Unit = 'ms'; Path = 'mgx.Median.ElapsedMs' } } + } + '03' = [pscustomobject]@{ + Scenario = '03'; Script = '03-user-report.ps1'; Headline = 'MgxReportMs' + Metrics = [pscustomobject]@{ MgxReportMs = [pscustomobject]@{ Value = @(1, 2); Unit = 'ms'; Path = 'mgx.Median.ElapsedMs' } } + } + '06' = $healthy06 + } + } + ConvertTo-Json -InputObject $stale -Depth 10 | Set-Content -Path $script:Baseline + + $rows = Compare-BenchBaseline -Benchmark '01', '02', '03', '06' -ResultsDirectory $script:Results -BaselinePath $script:Baseline + + $byId = @{} + foreach ($row in $rows) { $byId[$row.Scenario] = $row } + + $byId['01'].Status | Should -Be 'not comparable' + $byId['01'].Note | Should -Be "not comparable - the baseline value is 'INCONCLUSIVE', which is not a number" + $byId['02'].Status | Should -Be 'not comparable' + $byId['02'].Note | Should -Be "not comparable - the baseline value is 'True', which is not a number" + $byId['03'].Status | Should -Be 'not comparable' + $byId['03'].Note | Should -Be "not comparable - the baseline value is '[1,2]', which is not a number" + + $byId['06'].Status | Should -Be 'compared' + $byId['06'].DeltaPct | Should -Be 0 + } + + It 'refuses a current value that is NaN, naming it' { + # Get-BenchScenarioMetric now refuses every path that could hand back a non-finite + # pinned value - a raw non-number before scaling, an overflow after it - so nothing + # this release ships can make a live run's Current arrive here non-finite. The guard + # below exists for the same reason the baseline one does: a value that arrives + # non-finite by some path not yet closed must be refused rather than compared, not + # silently trusted because today's callers happen not to produce one. Standing in for + # Get-BenchScenarioMetric is the only way to reach that guard, since nothing else can. + Mock -CommandName Get-BenchScenarioMetric -MockWith { + [pscustomobject]@{ + Metrics = [ordered]@{ + MgxFanoutMs = [pscustomobject]@{ Value = [double]::NaN; Unit = 'ms'; Path = 'mgx.ElapsedMs' } + } + Discriminator = $null + } + } + + $row = Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -BaselinePath $script:Baseline + $row.Status | Should -Be 'not comparable' + $row.Note | Should -Be "not comparable - this run's value is NaN" + $row.DeltaPct | Should -BeNullOrEmpty + } + + It 'refuses a baseline with no Scenarios, naming the file, instead of indexing into null' { + # $baseline.Scenarios.PSObject.Properties[...] used to be reached unguarded: a missing + # key and an explicit null both read back as $null, and indexing into that threw + # "Cannot index into a null array" without naming the file or which field was absent. + foreach ($text in @( + '{"SchemaVersion":1,"GeneratedAt":"2026-01-01T00:00:00Z"}', + '{"SchemaVersion":1,"GeneratedAt":"2026-01-01T00:00:00Z","Scenarios":null}' + )) { + Set-Content -Path $script:Baseline -Value $text + + $thrown = $null + try { Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -BaselinePath $script:Baseline } + catch { $thrown = $_ } + + $thrown.Exception.Message | Should -Be "Baseline '$script:Baseline' carries no Scenarios; record one with -RecordBaseline before comparing." + } + } + + It 'refuses a baseline whose SchemaVersion is not the one Export-BenchBaseline writes' { + $scenarios = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios + $wrongSchema = [pscustomobject]@{ + SchemaVersion = 2 + GeneratedAt = (Get-Date).ToString('o') + Scenarios = $scenarios + } + ConvertTo-Json -InputObject $wrongSchema -Depth 10 | Set-Content -Path $script:Baseline + + $thrown = $null + try { Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -BaselinePath $script:Baseline } + catch { $thrown = $_ } + + $thrown.Exception.Message | Should -Be "Baseline '$script:Baseline' is schema 2; this suite reads schema 1." + } +} diff --git a/tests/Unit/Benchmarks/Mgx.BenchClampSummary.Tests.ps1 b/tests/Unit/Benchmarks/Mgx.BenchClampSummary.Tests.ps1 new file mode 100644 index 0000000..933ddae --- /dev/null +++ b/tests/Unit/Benchmarks/Mgx.BenchClampSummary.Tests.ps1 @@ -0,0 +1,145 @@ +#Requires -Modules Pester + +<# + Get-ClampSummary and the functions it composes (Get-ClampClockGap, Get-ClampBuckets, + Get-ClampOverall, Get-ClampHeadline) in tests/benchmarks/common.ps1 - the summary + 18-spo-latency-clamp.ps1 builds from its checkpointed rows.jsonl once a load finishes. + + No tenant and no live load: every case writes a synthetic rows.jsonl under $TestDrive and + calls Get-ClampSummary on its path directly, the same call the script makes. +#> + +BeforeAll { + $script:BenchRoot = Join-Path (Split-Path -Parent (Split-Path -Parent $PSScriptRoot)) 'benchmarks' + + # common.ps1 pins the thread to invariant culture so benchmark output never prints 39,2s. + # Right for a benchmark process, and not something to leave behind in a test run. + $script:CultureBefore = [System.Threading.Thread]::CurrentThread.CurrentCulture + . (Join-Path $script:BenchRoot 'common.ps1') + + # One row, in the shape 18 writes to its rows.jsonl. ElapsedMs and ErrorText are the two + # fields each case varies; the rest are fixed because nothing here reads them. + function script:Write-ClampRow { + param( + [Parameter(Mandatory)] [string] $Path, + [Parameter(Mandatory)] [datetime] $Timestamp, + [Parameter(Mandatory)] [int] $Status, + [object] $ElapsedMs = $null, + [string] $ErrorText = $null, + [int] $Worker = 1 + ) + $row = [pscustomobject]@{ + Timestamp = $Timestamp.ToString('o'); Worker = $Worker; Label = 'item'; Uri = '/drives/x/items/1' + Attempt = 1; Attempts = 1; Status = $Status; ElapsedMs = $ElapsedMs + CallElapsedMs = 100; PacingWaitMs = $null + ResourceUnit = $null; ThrottleLimitPercentage = $null; ThrottleScope = $null + RetryAfter = $null; RateLimitHeader = $false + Error = $ErrorText + } + Add-Content -Path $Path -Value (ConvertTo-Json -InputObject $row -Compress -Depth 4) + } +} + +AfterAll { + [System.Threading.Thread]::CurrentThread.CurrentCulture = $script:CultureBefore +} + +Describe 'Get-ClampSummary' { + BeforeEach { + $script:RowsPath = Join-Path $TestDrive "$([guid]::NewGuid()).jsonl" + $script:Started = [datetime]'2026-01-01T00:00:00Z' + } + + Context 'the clock gap' { + It 'refuses and names the gap when the wall clock outran the stopwatch by more than a bucket' { + # 20 minutes wall clock, a 5-minute Stopwatch - the 5757093 run's own shape, a + # machine sleep the Stopwatch does not advance through. + Write-ClampRow -Path $script:RowsPath -Timestamp $script:Started.AddMinutes(1) -Status 200 -ElapsedMs 100 + $ended = $script:Started.AddMinutes(20) + + $summary = Get-ClampSummary -RowsPath $script:RowsPath -StartedAt $script:Started -EndedAt $ended ` + -WallMs 300000 -Minutes 20 -BucketMinutes 5 + + $summary.Refused | Should -BeTrue + $summary.GapMinutes | Should -Be 15 + $summary.RefusalReason | Should -Be 'the machine was not awake for 15 minute(s) of the window; rows kept, summary withheld' + $summary.Buckets | Should -BeNullOrEmpty + $summary.Overall | Should -BeNullOrEmpty + } + + It 'does not refuse when the wall clock and the stopwatch agree within a bucket' { + Write-ClampRow -Path $script:RowsPath -Timestamp $script:Started.AddMinutes(1) -Status 200 -ElapsedMs 100 + $ended = $script:Started.AddMinutes(2) + + # A 1-second wall/Stopwatch drift - ordinary scheduling noise, not a sleep. + $summary = Get-ClampSummary -RowsPath $script:RowsPath -StartedAt $script:Started -EndedAt $ended ` + -WallMs 119000 -Minutes 2 -BucketMinutes 5 + + $summary.Refused | Should -BeFalse + $summary.Overall | Should -Not -BeNullOrEmpty + } + } + + Context 'buckets' { + It 'marks a bucket with no rows rather than omitting it, filling every index in the window' { + # Rows land in minute 1 (bucket 0) and minute 12 (bucket 2); bucket 1 (minutes + # 5-10) gets none, the way buckets 2 and 3 got none across the 5757093 run's sleep. + Write-ClampRow -Path $script:RowsPath -Timestamp $script:Started.AddMinutes(1) -Status 200 -ElapsedMs 100 + Write-ClampRow -Path $script:RowsPath -Timestamp $script:Started.AddMinutes(12) -Status 200 -ElapsedMs 100 + $ended = $script:Started.AddMinutes(15) + + $summary = Get-ClampSummary -RowsPath $script:RowsPath -StartedAt $script:Started -EndedAt $ended ` + -WallMs ([long]($ended - $script:Started).TotalMilliseconds) -Minutes 15 -BucketMinutes 5 + + $summary.Refused | Should -BeFalse + $summary.Buckets.Count | Should -Be 3 + $summary.Buckets[1].Bucket | Should -Be 1 + $summary.Buckets[1].Attempts | Should -Be 0 + $summary.Buckets[1].Note | Should -Be 'no rows' + $summary.Buckets[0].Note | Should -BeNullOrEmpty + $summary.Buckets[2].Note | Should -BeNullOrEmpty + } + } + + Context 'errors' { + It 'counts a Status-0 refusal in Errors and groups it by the text before its first colon' { + Write-ClampRow -Path $script:RowsPath -Timestamp $script:Started.AddMinutes(1) -Status 200 -ElapsedMs 100 + 1..3 | ForEach-Object { + Write-ClampRow -Path $script:RowsPath -Timestamp $script:Started.AddMinutes(1) -Status 0 ` + -ErrorText 'Circuit breaker tripped: too many failures caused Mgx to temporarily stop requests. Wait 15s.' + } + Write-ClampRow -Path $script:RowsPath -Timestamp $script:Started.AddMinutes(1) -Status 0 -ErrorText 'no response traced' + $ended = $script:Started.AddMinutes(2) + + $summary = Get-ClampSummary -RowsPath $script:RowsPath -StartedAt $script:Started -EndedAt $ended ` + -WallMs ([long]($ended - $script:Started).TotalMilliseconds) -Minutes 2 -BucketMinutes 5 + + $summary.Overall.Errors | Should -Be 4 + $summary.Buckets[0].Errors | Should -Be 4 + $summary.Overall.Refusals['Circuit breaker tripped'] | Should -Be 3 + $summary.Overall.Refusals['no response traced'] | Should -Be 1 + } + } + + Context 'the headline' { + It 'takes the headline from the bucket distribution, not a single late sample' { + # Bucket 0: nine calls at 200ms, then one late straggler at 1864ms - the shape of + # the 5757093 run's own last row, which PacingState quoted as though it were the + # headline. Nearest-rank p90 of ten values with one high outlier is still 200. + 1..9 | ForEach-Object { Write-ClampRow -Path $script:RowsPath -Timestamp $script:Started.AddMinutes(1) -Status 200 -ElapsedMs 200 } + Write-ClampRow -Path $script:RowsPath -Timestamp $script:Started.AddMinutes(1) -Status 200 -ElapsedMs 1864 + # Bucket 1: nine calls at 900ms - the worst bucket, and the one the headline names. + 1..9 | ForEach-Object { Write-ClampRow -Path $script:RowsPath -Timestamp $script:Started.AddMinutes(6) -Status 200 -ElapsedMs 900 } + $ended = $script:Started.AddMinutes(10) + + $summary = Get-ClampSummary -RowsPath $script:RowsPath -StartedAt $script:Started -EndedAt $ended ` + -WallMs ([long]($ended - $script:Started).TotalMilliseconds) -Minutes 10 -BucketMinutes 5 + + $summary.Buckets[0].P90ElapsedMs | Should -Be 200 + $summary.Headline.WorstBucketFromMinute | Should -Be 5 + $summary.Headline.WorstBucketP90ElapsedMs | Should -Be 900 + $summary.Headline.Sentence | Should -BeLike '*900*' + $summary.Headline.Sentence | Should -Not -BeLike '*1864*' + } + } +} diff --git a/tests/Unit/Benchmarks/Mgx.BenchLoadShape.Tests.ps1 b/tests/Unit/Benchmarks/Mgx.BenchLoadShape.Tests.ps1 new file mode 100644 index 0000000..18b7f8d --- /dev/null +++ b/tests/Unit/Benchmarks/Mgx.BenchLoadShape.Tests.ps1 @@ -0,0 +1,175 @@ +#Requires -Modules Pester + +<# + The load shape benchmark 10 issues, and whether it can reach the throttled regime, in + tests/benchmarks/common.ps1. + + The thresholds are passed in rather than read from results/, so these run the same + arithmetic on every machine whatever any local calibration says. The numbers used below + are the ones benchmark 13 measured against the seeded tenant: 22,954 RU of burst allowance + and 731 RU/s served while refusing. +#> + +BeforeAll { + $script:BenchRoot = Join-Path (Split-Path -Parent (Split-Path -Parent $PSScriptRoot)) 'benchmarks' + + $script:CultureBefore = [System.Threading.Thread]::CurrentThread.CurrentCulture + . (Join-Path $script:BenchRoot 'common.ps1') + + $script:Seeded = @{ BurstRu = 22954; SustainedRuPerSecond = 731 } + + function Get-Shape + { + param + ( + [int] $Count = 60000, + [int] $Pipelines = 12, + [int] $Concurrency = 128, + [int] $IdPool = 5000 + ) + + return Get-BenchPacingLoadShape -Count $Count -Pipelines $Pipelines ` + -Concurrency $Concurrency -IdPool $IdPool @script:Seeded + } + + # The defaults in the script's own param block. A default that stops being admissible is the + # failure this benchmark shipped once already, so the test reads them out of the file. + $script:ScriptDefaults = @{} + $text = Get-Content -Path (Join-Path $script:BenchRoot '10-pacing-under-real-throttling.ps1') -Raw + foreach ($name in 'Count', 'Pipelines', 'Concurrency', 'IdPool') + { + $match = [regex]::Match($text, "\[int\]\s*\`$$name\s*=\s*(\d+)") + if ($match.Success) { $script:ScriptDefaults[$name] = [int]$match.Groups[1].Value } + } +} + +AfterAll { + [System.Threading.Thread]::CurrentThread.CurrentCulture = $script:CultureBefore +} + +Describe 'Get-BenchPacingLoadShape' { + It 'issues distinct ids only, so the request count is Pipelines x IdsPerPipeline' { + # Invoke-MgxRequest deduplicates the ids it is piped: a slice that repeats one issues a + # single request for it, and the arm reports a volume it never sent. + $shape = Get-Shape -Count 60000 -Pipelines 12 -IdPool 5000 + $shape.IdsPerPipeline | Should -Be 5000 + $shape.TotalRequests | Should -Be 60000 + $shape.TotalRequests | Should -Be ($shape.IdsPerPipeline * $shape.Pipelines) + $shape.Clamped | Should -BeFalse + } + + It 'clamps a -Count the pool cannot cover and says it clamped' { + # 120,000 over 8 pipelines of a 5,000 id pool is 40,000 requests, whatever was asked for. + $shape = Get-Shape -Count 120000 -Pipelines 8 -IdPool 5000 + $shape.Requested | Should -Be 120000 + $shape.IdsPerPipeline | Should -Be 5000 + $shape.TotalRequests | Should -Be 40000 + $shape.Clamped | Should -BeTrue + } + + It 'divides a -Count under the ceiling evenly and does not clamp' { + $shape = Get-Shape -Count 24000 -Pipelines 12 -IdPool 5000 + $shape.IdsPerPipeline | Should -Be 2000 + $shape.TotalRequests | Should -Be 24000 + $shape.Clamped | Should -BeFalse + } + + It 'scales the projected draw with the pipelines and the concurrency' { + (Get-Shape -Pipelines 12 -Concurrency 128).ExpectedRuPerSecond | Should -Be 2160 + (Get-Shape -Pipelines 8 -Concurrency 128).ExpectedRuPerSecond | Should -Be 1440 + (Get-Shape -Pipelines 12 -Concurrency 64).ExpectedRuPerSecond | Should -Be 1080 + } + + It 'refuses a set whose rate cannot drain the allowance, whatever its volume' { + # The point the first version of the benchmark missed: no -Count exhausts a burst + # allowance that refills faster than the load draws it down. + $shape = Get-Shape -Count 500000 -Pipelines 2 -IdPool 250000 + $shape.TotalRequests | Should -Be 500000 + $shape.ExpectedRuPerSecond | Should -Be 360 + $shape.RateClears | Should -BeFalse + $shape.Admissible | Should -BeFalse + $shape.Reasons -join ' ' | Should -Match 'volume cannot buy a rate' + } + + It 'refuses a set that never spends past the allowance' { + $shape = Get-Shape -Count 12000 -Pipelines 12 -IdPool 5000 + $shape.RateClears | Should -BeTrue + $shape.VolumeClears | Should -BeFalse + $shape.Admissible | Should -BeFalse + $shape.ThrottledSeconds | Should -Be 0 + } + + It 'admits a set that clears both, and times the two phases' { + $shape = Get-Shape -Count 60000 -Pipelines 12 -IdPool 5000 + $shape.RateClears | Should -BeTrue + $shape.VolumeClears | Should -BeTrue + $shape.Admissible | Should -BeTrue + $shape.Reasons | Should -BeNullOrEmpty + + # 22,954 RU at 2,160 RU/s, then the remaining 37,046 at the 731 RU/s served while refusing. + $shape.BurstSeconds | Should -Be 10.6 + $shape.ThrottledSeconds | Should -Be 50.7 + $shape.EstimatedSeconds | Should -Be 61.3 + } +} + +Describe 'Get-BenchPipelineSlice' { + BeforeAll { + $script:Pool = 1..5000 | ForEach-Object { "id-$_" } + } + + It 'gives every pipeline its own count of ids, none of them repeated' { + # The invariant the arm's request count rests on. A repeated id inside one pipeline is + # deduplicated away by Invoke-MgxRequest and never sent. + $slices = Get-BenchPipelineSlice -Pool $script:Pool -Pipelines 12 -IdsPerPipeline 5000 + + $slices.Count | Should -Be 12 + foreach ($slice in $slices) + { + $slice.Count | Should -Be 5000 + @($slice | Sort-Object -Unique).Count | Should -Be 5000 + } + } + + It 'staggers where the pipelines start so they are not all on one object' { + $slices = Get-BenchPipelineSlice -Pool $script:Pool -Pipelines 12 -IdsPerPipeline 100 + + @($slices | ForEach-Object { $_[0] } | Sort-Object -Unique).Count | Should -Be 12 + } + + It 'holds the invariant for a short slice and for a pool smaller than the pipeline count' { + foreach ($case in @( + @{ Pipelines = 12; Ids = 2000 }, + @{ Pipelines = 3; Ids = 5000 }, + @{ Pipelines = 1; Ids = 1 })) + { + $slices = Get-BenchPipelineSlice -Pool $script:Pool -Pipelines $case.Pipelines -IdsPerPipeline $case.Ids + foreach ($slice in $slices) { @($slice | Sort-Object -Unique).Count | Should -Be $case.Ids } + } + + $tiny = Get-BenchPipelineSlice -Pool @('a', 'b', 'c') -Pipelines 8 -IdsPerPipeline 3 + $tiny.Count | Should -Be 8 + foreach ($slice in $tiny) { @($slice | Sort-Object -Unique).Count | Should -Be 3 } + } + + It 'refuses to hand out more distinct ids than the pool holds' { + { Get-BenchPipelineSlice -Pool @('a', 'b') -Pipelines 2 -IdsPerPipeline 3 } | + Should -Throw -ExpectedMessage '*deduplicates*' + } +} + +Describe 'benchmark 10 defaults' { + It 'reads four defaults out of the script' { + $script:ScriptDefaults.Keys.Count | Should -Be 4 + } + + It 'ships a default run that is admissible on the calibrated tenant' { + $shape = Get-BenchPacingLoadShape -Count $script:ScriptDefaults.Count ` + -Pipelines $script:ScriptDefaults.Pipelines ` + -Concurrency $script:ScriptDefaults.Concurrency ` + -IdPool $script:ScriptDefaults.IdPool @script:Seeded + + $shape.Clamped | Should -BeFalse -Because 'a default that asks for more than it can issue reports a volume it never sent' + $shape.Admissible | Should -BeTrue -Because 'run.ps1 passes no arguments, so the defaults are what gets measured' + } +} diff --git a/tests/Unit/Benchmarks/Mgx.BenchResult.Tests.ps1 b/tests/Unit/Benchmarks/Mgx.BenchResult.Tests.ps1 new file mode 100644 index 0000000..014da36 --- /dev/null +++ b/tests/Unit/Benchmarks/Mgx.BenchResult.Tests.ps1 @@ -0,0 +1,373 @@ +#Requires -Modules Pester + +<# + What a recorded benchmark result says about the run that produced it, in + tests/benchmarks/common.ps1: the documented budget tiers, and the tenant and tree identity + Write-BenchResult stamps on every entry. + + No tenant and no Graph session: Get-MgContext, the count call and the telemetry snapshot + are redefined in the scope common.ps1 is dot-sourced into, which is where Write-BenchResult + resolves them from, and every entry is written under $TestDrive. +#> + +BeforeAll { + $script:BenchRoot = Join-Path (Split-Path -Parent (Split-Path -Parent $PSScriptRoot)) 'benchmarks' + + # common.ps1 pins the thread to invariant culture so benchmark output never prints 39,2s. + # Right for a benchmark process, and not something to leave behind in a test run. + $script:CultureBefore = [System.Threading.Thread]::CurrentThread.CurrentCulture + . (Join-Path $script:BenchRoot 'common.ps1') + + # The faked tenant and the faked counters, each with CmdletBinding because the callers + # below pass -ErrorAction. Defined after the dot-source and in the same scope, so these + # are the definitions the real code finds. + $script:FakeTenantId = '00000000-1111-2222-3333-444444444444' + $script:FakeUserCount = 19 + + function Get-MgContext + { + [CmdletBinding()] + param () + + if (-not $script:FakeSession) { return $null } + return [pscustomobject]@{ TenantId = $script:FakeTenantId; ClientId = 'probe'; AuthType = 'AppOnly' } + } + + function Get-BenchDirectoryObjectCount + { + [CmdletBinding()] + param () + + $script:CountCalls++ + return [long]$script:FakeUserCount + } + + # 1,750 units over a 5,000 ms wall clock is 350 RU/s exactly, so the recorded fraction + # names its own denominator: 1.0 against the S tier's 350, 0.438 against the 800 of a + # tenant whose tier is not known. + function Get-MgxTelemetry + { + [CmdletBinding()] + param () + + return [pscustomobject]@{ + ResourceUnitsConsumed = 1750 + Requests = 1750 + Succeeded = 1750 + Failed = 0 + ThrottleRetries = 0 + OtherRetries = 0 + AdaptivePacingWaitMs = 0 + AdaptivePacingActivations = 0 + RateLimiterWaitMs = 0 + LastThrottlePercentage = -1.0 + PacingState = 'directory: latency 100ms (1.0x of 100ms baseline)' + } + } + + function Write-BenchProbe + { + param + ( + [Parameter(Mandatory)] [string] $Name, + [switch] $Session, + [long] $UserCount = 19, + [long] $WallMs = 5000, + [string] $TenantId = '00000000-1111-2222-3333-444444444444', + [object] $Identity + ) + + $script:FakeSession = [bool]$Session + $script:FakeUserCount = $UserCount + $script:FakeTenantId = $TenantId + $script:CountCalls = 0 + Get-BenchTenantFact -Refresh | Out-Null + + # Passed on only when the caller passed one, so the probe covers both the supplied and + # the unsupplied path rather than turning $null into an identity of five nulls. + $supplied = @{} + if ($PSBoundParameters.ContainsKey('Identity')) { $supplied['Identity'] = $Identity } + + Write-BenchResult -Benchmark $Name -ResultsDirectory $TestDrive -WallMs $WallMs ` + -Result ([pscustomobject]@{ Probe = $Name }) @supplied | Out-Null + + return @(Get-Content -Path (Join-Path $TestDrive "$Name.json") -Raw | ConvertFrom-Json)[-1] + } +} + +AfterAll { + [System.Threading.Thread]::CurrentThread.CurrentCulture = $script:CultureBefore +} + +Describe 'Get-BenchDocumentedBudget' { + It 'puts users in tier at RU/s' -ForEach @( + @{ UserCount = 0; Tier = 'S'; RuPerSecond = 350 } + @{ UserCount = 19; Tier = 'S'; RuPerSecond = 350 } + @{ UserCount = 49; Tier = 'S'; RuPerSecond = 350 } + @{ UserCount = 50; Tier = 'M'; RuPerSecond = 500 } + @{ UserCount = 500; Tier = 'M'; RuPerSecond = 500 } + @{ UserCount = 501; Tier = 'L'; RuPerSecond = 800 } + @{ UserCount = 130233; Tier = 'L'; RuPerSecond = 800 } + ) { + $budget = Get-BenchDocumentedBudget -UserCount $UserCount + $budget.Tier | Should -Be $Tier + $budget.RuPerSecond | Should -Be $RuPerSecond + $budget.RuPer10Seconds | Should -Be ($RuPerSecond * 10) + $budget.UserCount | Should -Be $UserCount + $budget.Band | Should -Not -BeNullOrEmpty + } + + It 'answers null for a count it does not have' { + # An assumed tier is worse than none: it would put a denominator nobody measured into + # the ratio and label it documented. + Get-BenchDocumentedBudget -UserCount $null | Should -BeNullOrEmpty + Get-BenchDocumentedBudget -UserCount -1 | Should -BeNullOrEmpty + } +} + +Describe 'Write-BenchResult metadata' { + It 'records the tenant, its size and its documented budget' { + $entry = Write-BenchProbe -Name 'tenant' -Session -UserCount 19 + + $entry.TenantId | Should -Be $script:FakeTenantId + $entry.DirectoryObjectCount | Should -Be 19 + $entry.DocumentedBudgetRuPerSecond | Should -Be 350 + } + + It 'treats a session whose tenant id is not a GUID as no tenant' { + # 06 connects a mock Graph and names the tenant 'gauntlet-mock-tenant'. There is no + # directory behind that to count, and the benchmark has to stay off the network. + $entry = Write-BenchProbe -Name 'mocktenant' -Session -TenantId 'gauntlet-mock-tenant' + + $entry.TenantId | Should -BeNullOrEmpty + $entry.DirectoryObjectCount | Should -BeNullOrEmpty + $script:CountCalls | Should -Be 0 -Because 'a mock session must not be counted against a real tenant' + } + + It 'records those three as null rather than failing with no session' { + # 06 and the mock-server benchmarks never connect. They still have to record. + $entry = Write-BenchProbe -Name 'notenant' + + $entry.PSObject.Properties.Name | Should -Contain 'TenantId' + $entry.TenantId | Should -BeNullOrEmpty + $entry.DirectoryObjectCount | Should -BeNullOrEmpty + $entry.DocumentedBudgetRuPerSecond | Should -BeNullOrEmpty + $entry.Result.Probe | Should -Be 'notenant' + } + + It 'records the commit of the tree that ran, with a dirty marker' { + $entry = Write-BenchProbe -Name 'commit' + + # Null outside a checkout, which CI is not - so assert the shape wherever there is one. + if ($entry.Commit) + { + $entry.Commit | Should -Match '^[0-9a-f]{7,40}(-dirty)?$' + } + } + + It 'records the wall clock as metadata, not only inside the telemetry block' { + $entry = Write-BenchProbe -Name 'wall' -WallMs 5000 + $entry.WallMs | Should -Be 5000 + + $noWall = Write-BenchProbe -Name 'nowall' -WallMs 0 + $noWall.WallMs | Should -BeNullOrEmpty + } + + It 'measures the draw against the documented budget and says so' { + # 350 RU/s against the S tier's 350 RU/s is the whole budget. + $entry = Write-BenchProbe -Name 'documented' -Session -UserCount 19 + + $entry.Telemetry.RuPerSecond | Should -Be 350 + $entry.Telemetry.BudgetFraction | Should -Be 1 + $entry.Telemetry.BudgetBasis | Should -Be 'documented' + } + + It 'measures the same draw against a larger tenant as a smaller fraction' { + $entry = Write-BenchProbe -Name 'documented-large' -Session -UserCount 130233 + + $entry.Telemetry.BudgetFraction | Should -Be 0.438 + $entry.Telemetry.BudgetBasis | Should -Be 'documented' + } + + It 'falls back to 800 RU/s with no documented tier and labels the fallback' { + $entry = Write-BenchProbe -Name 'assumed' + + $entry.Telemetry.BudgetFraction | Should -Be 0.438 + $entry.Telemetry.BudgetBasis | Should -Be 'assumed-800' + } +} + +Describe 'Write-BenchResult -Identity' { + It 'stamps the entry with the identity it was handed, not this process''s' { + # The case this exists for: a comparison whose arms ran as child processes writes its + # combined entry from a parent that imported no local module and connected to nothing. + # A session is faked here so the process HAS a tenant of its own to leak - the entry + # must carry the arms' tenant, not that one. + $entry = Write-BenchProbe -Name 'identity' -Session -UserCount 19 -Identity ([pscustomobject]@{ + MgxVersion = '2.1.4' + SdkVersion = '2.34.0' + TenantId = 'e5011376-aaaa-bbbb-cccc-dddddddddddd' + DirectoryObjectCount = 100000 + DocumentedBudgetRuPerSecond = 800 + }) + + $entry.MgxVersion | Should -Be '2.1.4' + $entry.SdkVersion | Should -Be '2.34.0' + $entry.TenantId | Should -Be 'e5011376-aaaa-bbbb-cccc-dddddddddddd' + $entry.TenantId | Should -Not -Be $script:FakeTenantId + $entry.DirectoryObjectCount | Should -Be 100000 + $entry.DocumentedBudgetRuPerSecond | Should -Be 800 + } + + It 'takes a hashtable as readily as an object' { + $entry = Write-BenchProbe -Name 'identity-hash' -Identity @{ + MgxVersion = '2.1.4'; SdkVersion = '2.34.0'; TenantId = 'e5011376' + DirectoryObjectCount = 100000; DocumentedBudgetRuPerSecond = 800 + } + + $entry.MgxVersion | Should -Be '2.1.4' + $entry.DirectoryObjectCount | Should -Be 100000 + } + + It 'records a field the identity does not carry as null rather than falling back' { + # Get-BenchEntryIdentity answers null for a field the arms disagreed on. Substituting + # this process's value there would put back exactly what the parameter exists to keep + # out, and label a disagreement as a measurement. + $entry = Write-BenchProbe -Name 'identity-partial' -Session -UserCount 19 ` + -Identity ([pscustomobject]@{ MgxVersion = '2.1.4'; TenantId = $null }) + + $entry.MgxVersion | Should -Be '2.1.4' + $entry.TenantId | Should -BeNullOrEmpty + $entry.SdkVersion | Should -BeNullOrEmpty + $entry.DirectoryObjectCount | Should -BeNullOrEmpty + $entry.DocumentedBudgetRuPerSecond | Should -BeNullOrEmpty + } + + It 'leaves the run''s own fields to the process that wrote them' { + $entry = Write-BenchProbe -Name 'identity-run' -WallMs 5000 -Identity ([pscustomobject]@{ + MgxVersion = '2.1.4'; PSVersion = '1.0'; WallMs = 42 + }) + + $entry.PSVersion | Should -Be $PSVersionTable.PSVersion.ToString() + $entry.WallMs | Should -Be 5000 + $entry.RecordedAt | Should -Not -BeNullOrEmpty + } + + It 'reads this process with no identity supplied' { + $entry = Write-BenchProbe -Name 'identity-absent' -Session -UserCount 19 + + $entry.MgxVersion | Should -Be (Get-Module Mgx -ErrorAction SilentlyContinue)?.Version?.ToString() + $entry.TenantId | Should -Be $script:FakeTenantId + $entry.DirectoryObjectCount | Should -Be 19 + $entry.DocumentedBudgetRuPerSecond | Should -Be 350 + } +} + +Describe 'Get-BenchEntryIdentity' { + BeforeAll { + function script:New-ArmEntry + { + param + ( + [string] $MgxVersion = '2.1.4', + [string] $SdkVersion = '2.34.0', + [object] $TenantId = 'e5011376-aaaa-bbbb-cccc-dddddddddddd', + [object] $DirectoryObjectCount = 100000, + [object] $DocumentedBudgetRuPerSecond = 800 + ) + + return [pscustomobject]@{ + MgxVersion = $MgxVersion + SdkVersion = $SdkVersion + TenantId = $TenantId + DirectoryObjectCount = $DirectoryObjectCount + DocumentedBudgetRuPerSecond = $DocumentedBudgetRuPerSecond + } + } + } + + It 'answers the five fields the arms agree on' { + $identity = Get-BenchEntryIdentity -Entries @((New-ArmEntry), (New-ArmEntry)) + + $identity.MgxVersion | Should -Be '2.1.4' + $identity.SdkVersion | Should -Be '2.34.0' + $identity.TenantId | Should -Be 'e5011376-aaaa-bbbb-cccc-dddddddddddd' + $identity.DirectoryObjectCount | Should -Be 100000 + $identity.DocumentedBudgetRuPerSecond | Should -Be 800 + } + + It 'counts a long and an int holding the same number as agreement' { + # The arms record through the same writer, but one of them reaching the count as a long + # and the other as an int is not a disagreement about which tenant ran. + $identity = Get-BenchEntryIdentity -Entries @( + (New-ArmEntry -DirectoryObjectCount ([long]100000)), + (New-ArmEntry -DirectoryObjectCount ([int]100000)) + ) + + $identity.DirectoryObjectCount | Should -Be 100000 + } + + It 'answers null for a field the arms disagree on, naming it and both values' { + $warnings = $null + $identity = Get-BenchEntryIdentity -WarningVariable warnings 3>$null -Entries @( + (New-ArmEntry -MgxVersion '2.1.4'), + (New-ArmEntry -MgxVersion '2.1.1') + ) + + $identity.MgxVersion | Should -BeNullOrEmpty + "$warnings" | Should -BeLike '*disagree on MgxVersion (2.1.4, 2.1.1)*' + $identity.TenantId | Should -Be 'e5011376-aaaa-bbbb-cccc-dddddddddddd' -Because 'a field they agree on is still answered' + } + + It 'names a field one arm recorded and the other did not as a disagreement' { + # An arm with no tenant and an arm with one did not run against the same thing, and + # taking the one that has a value would say they did. + $warnings = $null + $identity = Get-BenchEntryIdentity -WarningVariable warnings 3>$null -Entries @( + (New-ArmEntry), + (New-ArmEntry -TenantId $null) + ) + + $identity.TenantId | Should -BeNullOrEmpty + "$warnings" | Should -BeLike '*disagree on TenantId (e5011376-aaaa-bbbb-cccc-dddddddddddd, (none))*' + } + + It 'answers null without warning when neither arm recorded a field' { + $warnings = $null + $identity = Get-BenchEntryIdentity -WarningVariable warnings 3>$null -Entries @( + (New-ArmEntry -TenantId $null -DocumentedBudgetRuPerSecond $null), + (New-ArmEntry -TenantId $null -DocumentedBudgetRuPerSecond $null) + ) + + $identity.TenantId | Should -BeNullOrEmpty + $identity.DocumentedBudgetRuPerSecond | Should -BeNullOrEmpty + "$warnings" | Should -BeNullOrEmpty + } + + It 'reads a single arm''s entry as that arm''s identity' { + $identity = Get-BenchEntryIdentity -Entries @((New-ArmEntry -MgxVersion '2.1.5')) + + $identity.MgxVersion | Should -Be '2.1.5' + } +} + +Describe 'Export-BenchBaseline over a recorded entry' { + It 'still reads the version block off an entry carrying the new fields' { + # The promotion reads MgxVersion/SdkVersion/PSVersion/RecordedAt by name. The tenant + # fields extend that block; they must not have moved what was already in it. + $results = Join-Path $TestDrive 'promote' + Write-BenchResult -Benchmark '06-fault-gauntlet' -ResultsDirectory $results -Result ([pscustomobject]@{ + mgx = [pscustomobject]@{ ElapsedMs = 134654; Output = [pscustomobject]@{ ok = 1000; failed = 0 } } + sdkMgx = [pscustomobject]@{ ElapsedMs = 246715 } + }) | Out-Null + + $baseline = Join-Path $TestDrive 'promoted-baseline.json' + Export-BenchBaseline -Benchmark '06' -ResultsDirectory $results -Path $baseline + + $pinned = (Get-Content -Path $baseline -Raw | ConvertFrom-Json).Scenarios.'06' + $pinned.Headline | Should -Be 'MgxFanoutMs' + $pinned.Metrics.MgxFanoutMs.Value | Should -Be 134654 + $pinned.PSVersion | Should -Be $PSVersionTable.PSVersion.ToString() + $pinned.RecordedAt | Should -Not -BeNullOrEmpty + } +} diff --git a/tests/Unit/Benchmarks/Mgx.BenchWatchdog.Tests.ps1 b/tests/Unit/Benchmarks/Mgx.BenchWatchdog.Tests.ps1 new file mode 100644 index 0000000..ee2b6f8 --- /dev/null +++ b/tests/Unit/Benchmarks/Mgx.BenchWatchdog.Tests.ps1 @@ -0,0 +1,65 @@ +#Requires -Modules Pester + +<# + Invoke-WatchdoggedContender in tests/benchmarks/common.ps1. + + Each case spawns a real child pwsh process, so these run at the watchdog's own 10-second + poll granularity rather than instantly - the same cost 16 and 17 already pay to use it. +#> + +BeforeAll { + $script:BenchRoot = Join-Path (Split-Path -Parent (Split-Path -Parent $PSScriptRoot)) 'benchmarks' + + # common.ps1 pins the thread to invariant culture; not something to leave behind in a + # test run. + $script:CultureBefore = [System.Threading.Thread]::CurrentThread.CurrentCulture + . (Join-Path $script:BenchRoot 'common.ps1') +} + +AfterAll { + [System.Threading.Thread]::CurrentThread.CurrentCulture = $script:CultureBefore +} + +Describe 'Invoke-WatchdoggedContender' { + BeforeEach { + $script:ResultFile = Join-Path $TestDrive 'result.json' + $script:HeartbeatFile = Join-Path $TestDrive 'heartbeat' + Remove-Item -Path $script:ResultFile, $script:HeartbeatFile -ErrorAction SilentlyContinue + } + + It 'records a Crashed row naming the exit code when the child dies before its first heartbeat' { + # An import or connect failure inside the child's first poll: it exits before ever + # touching $HeartbeatFile, which is what used to fall back to $p.StartTime and read + # back null once the child had already exited - and used to come back Hung, which is + # not what happened: nothing here waited for progress that never came, the child quit + # on its own within the first poll. + $child = Join-Path $TestDrive 'dies-early.ps1' + Set-Content -Path $child -Value 'exit 3' + + $row = Invoke-WatchdoggedContender -Name 'dies-early' -ScriptPath $child ` + -ResultFile $script:ResultFile -HeartbeatFile $script:HeartbeatFile -StallSeconds 60 + + $row.Hung | Should -BeFalse + $row.Crashed | Should -BeTrue + $row.ExitCode | Should -Be 3 + $row.Output.note | Should -BeLike '*exited 3*' + } + + It 'reports Hung = false and Crashed = false for a child that heartbeats and then completes' { + $child = Join-Path $TestDrive 'completes.ps1' + Set-Content -Path $child -Value @' +param([string] $HeartbeatFile, [string] $ResultFile) +Set-Content -Path $HeartbeatFile -Value ([datetime]::UtcNow.Ticks) +[pscustomobject]@{ Name = 'completes'; ElapsedMs = 1 } | ConvertTo-Json | Set-Content -Path $ResultFile +exit 0 +'@ + + $row = Invoke-WatchdoggedContender -Name 'completes' -ScriptPath $child ` + -ArgumentList @('-HeartbeatFile', $script:HeartbeatFile, '-ResultFile', $script:ResultFile) ` + -ResultFile $script:ResultFile -HeartbeatFile $script:HeartbeatFile -StallSeconds 60 + + $row.Hung | Should -BeFalse + $row.Crashed | Should -BeFalse + $row.Name | Should -Be 'completes' + } +} diff --git a/tests/Unit/Benchmarks/Mgx.MockPlan.Tests.ps1 b/tests/Unit/Benchmarks/Mgx.MockPlan.Tests.ps1 new file mode 100644 index 0000000..15101a9 --- /dev/null +++ b/tests/Unit/Benchmarks/Mgx.MockPlan.Tests.ps1 @@ -0,0 +1,131 @@ +#Requires -Modules Pester + +<# + Set-Plan's window validation, served counting under a plan fault on the entity route, + and slowBody's chunks: 1 timing, all in tests/benchmarks/mock-graph-server.ps1. + + A real child process on a real socket, the same way 06 and the pathological gauntlets use + it: the plan screen, the disposition a fault hands back, and a trickled body are exactly + the things no in-process fake stands in for. +#> + +BeforeAll { + $script:BenchRoot = Join-Path (Split-Path -Parent (Split-Path -Parent $PSScriptRoot)) 'benchmarks' + + # common.ps1 pins the thread to invariant culture; not something to leave behind in a + # test run. + $script:CultureBefore = [System.Threading.Thread]::CurrentThread.CurrentCulture + . (Join-Path $script:BenchRoot 'common.ps1') + + $script:Server = Start-BenchMockServer -Port 9970 + $script:Base = $script:Server.BaseUrl + + # Posts a plan without Invoke-RestMethod's default throw-on-4xx, so a refusal is a value + # to assert on rather than an exception to catch. Read inside this function's own scope + # rather than relied on to cross into the caller's, so the answer does not depend on + # exactly how -StatusCodeVariable propagates. Defined here, in BeforeAll, rather than at + # the script's top level: Pester's discovery pass does not carry a plain top-level + # function into the It blocks that run afterward. + function Set-MockPlan([hashtable[]] $Rules) { + $body = @{ rules = $Rules } | ConvertTo-Json -Depth 6 -Compress + $response = Invoke-RestMethod "$script:Base/plan" -Method Post -ContentType 'application/json' ` + -Body $body -SkipHttpErrorCheck -StatusCodeVariable 'statusCode' + [pscustomobject]@{ StatusCode = $statusCode; Body = $response } + } +} + +AfterAll { + Stop-BenchMockServer $script:Server + [System.Threading.Thread]::CurrentThread.CurrentCulture = $script:CultureBefore +} + +Describe 'Set-Plan refuses a zero window or count' { + BeforeEach { + $null = Invoke-RestMethod "$script:Base/plan" -Method Post -ContentType 'application/json' -Body '{"rules":[]}' + } + + It 'refuses an explicit from: 0' { + $result = Set-MockPlan @(@{ route = '^/v1\.0/users/1$'; kind = 'outage'; from = 0 }) + $result.StatusCode | Should -Be 400 + $result.Body.error.code | Should -Be 'MockPlanRejected' + $result.Body.error.message | Should -Be "rule 1: 'from' must be at least 1, got 0" + } + + It 'refuses an explicit count: 0' { + $result = Set-MockPlan @(@{ route = '^/v1\.0/users/1$'; kind = 'outage'; count = 0 }) + $result.StatusCode | Should -Be 400 + $result.Body.error.message | Should -Be "rule 1: 'count' must be at least 1, got 0" + } + + It 'refuses an explicit to: 0' { + $result = Set-MockPlan @(@{ route = '^/v1\.0/users/1$'; kind = 'outage'; to = 0 }) + $result.StatusCode | Should -Be 400 + $result.Body.error.message | Should -Be "rule 1: 'to' must be at least 1, got 0" + } + + It 'refuses an explicit seconds: 0' { + $result = Set-MockPlan @(@{ route = '^/v1\.0/users/1$'; kind = 'delayedVisibility'; seconds = 0 }) + $result.StatusCode | Should -Be 400 + $result.Body.error.message | Should -Be "rule 1: 'seconds' must be at least 1, got 0" + } + + It 'names the failing rule by its 1-based position' { + $result = Set-MockPlan @( + @{ route = '^/v1\.0/users/2$'; kind = 'outage' } + @{ route = '^/v1\.0/users/3$'; kind = 'outage'; count = 0 } + ) + $result.StatusCode | Should -Be 400 + $result.Body.error.message | Should -Be "rule 2: 'count' must be at least 1, got 0" + } + + It 'still accepts a rule with no window fields at all, meaning no end' { + $result = Set-MockPlan @(@{ route = '^/v1\.0/users/9$'; kind = 'outage' }) + $result.StatusCode | Should -Be 200 + $result.Body.ok | Should -BeTrue + } + + It 'still accepts from and count set at their floor of 1' { + $result = Set-MockPlan @(@{ route = '^/v1\.0/users/9$'; kind = 'outage'; from = 1; count = 1 }) + $result.StatusCode | Should -Be 200 + $result.Body.ok | Should -BeTrue + } +} + +Describe 'served counts a plan-faulted entity read the way pagesServed counts a cut page' { + BeforeEach { + $null = Invoke-RestMethod "$script:Base/plan" -Method Post -ContentType 'application/json' -Body '{"rules":[]}' + $null = Invoke-RestMethod "$script:Base/reset" + } + + It 'leaves served unchanged while a throttleStorm rule is firing' { + $null = Set-MockPlan @(@{ route = '^/v1\.0/users/42$'; kind = 'throttleStorm'; count = 3 }) + $before = Invoke-RestMethod "$script:Base/stats" + 1..3 | ForEach-Object { Invoke-RestMethod "$script:Base/v1.0/users/42" -SkipHttpErrorCheck | Out-Null } + $after = Invoke-RestMethod "$script:Base/stats" + ($after.served - $before.served) | Should -Be 0 + ($after.throttleStorm - $before.throttleStorm) | Should -Be 3 + } + + It 'raises served by one once the window closes and the entity is answered whole' { + $null = Set-MockPlan @(@{ route = '^/v1\.0/users/43$'; kind = 'throttleStorm'; count = 3 }) + $before = Invoke-RestMethod "$script:Base/stats" + 1..4 | ForEach-Object { Invoke-RestMethod "$script:Base/v1.0/users/43" -SkipHttpErrorCheck | Out-Null } + $after = Invoke-RestMethod "$script:Base/stats" + ($after.served - $before.served) | Should -Be 1 + } +} + +Describe 'Send-Slow honors bodyMs with a single chunk' { + BeforeEach { + $null = Invoke-RestMethod "$script:Base/plan" -Method Post -ContentType 'application/json' -Body '{"rules":[]}' + } + + It 'takes about bodyMs, not an instant write, when chunks is 1' { + $null = Set-MockPlan @(@{ route = '^/v1\.0/users/777$'; kind = 'slowBody'; chunks = 1; bodyMs = 1500 }) + $sw = [System.Diagnostics.Stopwatch]::StartNew() + Invoke-RestMethod "$script:Base/v1.0/users/777" | Out-Null + $sw.Stop() + $sw.ElapsedMilliseconds | Should -BeGreaterThan 1300 + $sw.ElapsedMilliseconds | Should -BeLessThan 5000 + } +} diff --git a/tests/Unit/Mgx.Behaviour.Tests.ps1 b/tests/Unit/Mgx.Behaviour.Tests.ps1 index 48cef8a..7d7654b 100644 --- a/tests/Unit/Mgx.Behaviour.Tests.ps1 +++ b/tests/Unit/Mgx.Behaviour.Tests.ps1 @@ -1,15 +1,21 @@ #Requires -Modules Pester <# - Mgx Pester Tests (v0.3.0) + The behavior of the built module as PowerShell sees it: the cmdlet and parameter contract, + and what the cmdlets decide before a request leaves the process. - Prerequisites: - 1. Run build.ps1 first - 2. For Live tests: Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All" + Run ./build.ps1 first, then the harness, which is what CI runs too: - Usage: - Invoke-Pester ./tests/Mgx.Tests.ps1 -Output Detailed - Invoke-Pester ./tests/Mgx.Tests.ps1 -Output Detailed -Tag 'Live' + Import-Module ./tests/TestHarness.psm1; Invoke-TestHarness + + The harness excludes the Live tag, so a clone with no Graph session passes. What a cmdlet + does against a real tenant is proven in tests/Live, run on its own: + + Invoke-Pester -Path ./tests/Live + + The one Live-tagged block left here is the sustained auth run at the bottom: it needs an + hour of directory traffic to cross a token boundary, which is not something tests/Live can + ask of anyone running the suite. Pass -IncludeLive to the harness to reach it. #> BeforeAll { @@ -187,20 +193,6 @@ Describe 'Expand-MgxRelation Parameter Compatibility' { $err.Exception.Message | Should -Not -BeLike '*ConsistencyLevel*' } } - - # R3-11: Buffer size warning at 50k items - It 'Should warn when buffer exceeds 50k items' -Tag 'Slow' { - $objects = 1..50001 | ForEach-Object { [PSCustomObject]@{id = "id-$_"} } - $w = @() - try { - $objects | Expand-MgxRelation '/users/{id}/manager' -As Manager ` - -WarningVariable w 3>$null -ErrorAction Stop - } catch { - # Expected: auth error from EndProcessing. Warning fires in ProcessRecord before this. - } - $bufferWarning = $w | Where-Object { $_ -match 'buffer' -or $_ -match '50.000' } - $bufferWarning | Should -Not -BeNullOrEmpty - } } Describe 'Invoke-MgxRequest Parameter Compatibility' { @@ -806,6 +798,10 @@ Describe 'Set-MgxOption Pipeline Parameters' { { Set-MgxOption -CircuitBreakerMinThroughput 0 } | Should -Throw } + It 'Should reject -CircuitBreakerMinThroughput 1' { + { Set-MgxOption -CircuitBreakerMinThroughput 1 } | Should -Throw + } + It 'Should reject -RateLimitQueueLimit -1' { { Set-MgxOption -RateLimitQueueLimit -1 } | Should -Throw } @@ -1342,177 +1338,6 @@ Describe 'NormalizeSelect (via reflection)' { } } -Describe 'Live API Tests' -Tag 'Live' { - BeforeAll { - $script:liveConnected = $null -ne (Get-MgContext -ErrorAction SilentlyContinue) - } - - BeforeEach { - if (-not $script:liveConnected) { - Set-ItResult -Skipped -Because 'Not connected to Microsoft Graph' - } - } - - It 'Invoke-MgxRequest /users -Top 5 should return users' { - $users = Invoke-MgxRequest /users -Top 5 - $users | Should -Not -BeNullOrEmpty - $users.Count | Should -BeLessOrEqual 5 - $users[0].id | Should -Not -BeNullOrEmpty - } - - It 'Invoke-MgxRequest /users -All | Select -First 5 should stream' { - $sw = [System.Diagnostics.Stopwatch]::StartNew() - $firstUser = Invoke-MgxRequest /users -All | Select-Object -First 1 - $sw.Stop() - - $firstUser | Should -Not -BeNullOrEmpty - # Streaming means first item arrives fast - $sw.ElapsedMilliseconds | Should -BeLessThan 30000 - } - - It 'Invoke-MgxRequest /users/ should return single entity' { - $firstUser = Invoke-MgxRequest /users -Top 1 - $user = Invoke-MgxRequest "/users/$($firstUser.id)" - $user | Should -Not -BeNullOrEmpty - $user.id | Should -Be $firstUser.id - } - - It 'Invoke-MgxRequest /users -Filter should work' { - $users = Invoke-MgxRequest /users -Filter "accountEnabled eq true" -Top 3 - $users | Should -Not -BeNullOrEmpty - } - - It 'Fan-out: $ids | Invoke-MgxRequest /users/{id} should return entities with _MgxSourceId' { - $topUsers = Invoke-MgxRequest /users -Top 3 -Property id - $ids = $topUsers | ForEach-Object { $_.id } - $results = $ids | Invoke-MgxRequest '/users/{id}' - $results | Should -Not -BeNullOrEmpty - $results.Count | Should -Be $ids.Count - $results[0]._MgxSourceId | Should -Not -BeNullOrEmpty - } - - It 'Collection fan-out: $ids | Invoke-MgxRequest /groups/{id}/members -All should work' { - $groups = Invoke-MgxRequest /groups -Top 2 -Property id - if ($groups) { - $ids = $groups | ForEach-Object { $_.id } - $results = $ids | Invoke-MgxRequest '/groups/{id}/members' -Top 5 - # May be empty if groups have no members, but should not error - $results | Should -Not -BeNullOrEmpty -Because 'At least some groups should have members' - } else { - Set-ItResult -Skipped -Because 'No groups in tenant' - } - } - - It 'Invoke-MgxRequest with {id} but no pipeline should error' { - { Invoke-MgxRequest '/users/{id}' } | Should -Throw '*pipeline*' - } - - It '@odata.type should be preserved verbatim' { - # Use group members (polymorphic endpoint that reliably returns @odata.type) - $groups = Invoke-MgxRequest /groups -Top 5 -Property id - $members = @() - foreach ($g in $groups) { - $members = @(Invoke-MgxRequest "/groups/$($g.id)/members" -Top 3) - if ($members.Count -gt 0) { break } - } - if ($members.Count -eq 0) { - Set-ItResult -Skipped -Because 'No group members found to test @odata.type' - return - } - $withType = $members | Where-Object { $_['@odata.type'] } - $withType | Should -Not -BeNullOrEmpty - } - - It 'DateTime properties should be DateTimeOffset' { - $user = Invoke-MgxRequest /users -Top 1 - if ($user.createdDateTime) { - $user.createdDateTime | Should -BeOfType [System.DateTimeOffset] - } - } - - It 'Output parity: Invoke-MgxRequest vs Get-MgUser' { - $mgx = Invoke-MgxRequest /users -Top 3 -Property id, displayName, userPrincipalName - $mg = Get-MgUser -Top 3 -Property Id, DisplayName, UserPrincipalName - - $diff = Compare-Object ($mg.Id | Sort-Object) ($mgx.id | Sort-Object) - $diff | Should -BeNullOrEmpty - } - - It 'Invoke-MgxRequest -ApiVersion beta should use beta endpoint' { - $users = Invoke-MgxRequest /users -ApiVersion beta -Top 3 - $users | Should -Not -BeNullOrEmpty - } - - It 'Set-MgxOption round-trip: NoRateLimit then re-enable with RateLimitBurst' { - Set-MgxOption -NoRateLimit - Set-MgxOption -RateLimitBurst 100 - # Should implicitly clear NoRateLimit; verify by running a request - $user = Invoke-MgxRequest /users -Top 1 - $user | Should -Not -BeNullOrEmpty - } - - It 'Export-MgxCollection should export JSONL file' { - $outFile = Join-Path ([System.IO.Path]::GetTempPath()) "mgx-test-export-$(Get-Random).jsonl" - try { - $result = Export-MgxCollection /users -OutputFile $outFile -Top 10 - $result | Should -Not -BeNullOrEmpty - $result.ItemCount | Should -BeGreaterThan 0 - $result.ItemCount | Should -BeLessOrEqual 10 - $result.OutputFile | Should -Be $outFile - Test-Path $outFile | Should -BeTrue - $lines = Get-Content $outFile - $lines.Count | Should -Be $result.ItemCount - # Each line should be valid JSON - $lines | ForEach-Object { $_ | ConvertFrom-Json } | Should -Not -BeNullOrEmpty - } finally { - if (Test-Path $outFile) { Remove-Item $outFile -Force } - } - } - - It 'Export-MgxCollection checkpoint should be deleted on completion' { - $outFile = Join-Path ([System.IO.Path]::GetTempPath()) "mgx-test-cp-export-$(Get-Random).jsonl" - $cpFile = Join-Path ([System.IO.Path]::GetTempPath()) "mgx-test-cp-$(Get-Random).json" - try { - Export-MgxCollection /users -OutputFile $outFile -Top 5 -CheckpointPath $cpFile - Test-Path $cpFile | Should -BeFalse -Because 'Checkpoint should be deleted on successful completion' - } finally { - if (Test-Path $outFile) { Remove-Item $outFile -Force } - if (Test-Path $cpFile) { Remove-Item $cpFile -Force } - } - } - - It 'Enable-MgxResilience should inject resilience and SDK should still work' { - Enable-MgxResilience - $state = Get-MgxResilience - $state.IsEnabled | Should -BeTrue - $state.IsActive | Should -BeTrue - - # SDK cmdlet should work with resilience injected - $user = Get-MgUser -Top 1 - $user | Should -Not -BeNullOrEmpty - - Disable-MgxResilience - $state2 = Get-MgxResilience - $state2.IsEnabled | Should -BeFalse - $state2.IsActive | Should -BeFalse - - # SDK should still work after disable - $user2 = Get-MgUser -Top 1 - $user2 | Should -Not -BeNullOrEmpty - } -} - -Describe 'Expand-MgxRelation Buffer Warning' { - It 'Source code contains 50k buffer warning guard' { - # The actual live test (piping 50k items) requires Graph connection and takes ~15 min - # due to fan-out HTTP calls. Verify the guard exists in source instead. - $source = Get-Content (Join-Path $PSScriptRoot '../../src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs') -Raw - $source | Should -Match '50_000' - $source | Should -Match 'WriteWarning' - $source | Should -Match 'buffer|Buffer' - } -} - Describe 'HttpClient Reset Leaves A Held Client Alone' { # A replaced transport is dropped, not closed. Every ResilientGraphClient built on it # captured it in its constructor and keeps sending through it for as long as its @@ -1657,195 +1482,6 @@ finally { } } -Describe 'Sync-MgxDelta ExecuteDeltaSync Tests' -Tag 'Live' { - BeforeAll { - $script:liveConnected = $null -ne (Get-MgContext -ErrorAction SilentlyContinue) - $script:deltaTestDir = Join-Path ([System.IO.Path]::GetTempPath()) "mgx-delta-live-$(New-Guid)" - New-Item -ItemType Directory -Path $script:deltaTestDir -Force | Out-Null - } - - BeforeEach { - if (-not $script:liveConnected) { - Set-ItResult -Skipped -Because 'Not connected to Microsoft Graph' - } - } - - AfterAll { - if (Test-Path $script:deltaTestDir) { Remove-Item $script:deltaTestDir -Recurse -Force } - } - - # --- #1: 410 Gone retry flow --- - It '410 Gone: deletes state and performs full re-sync' { - $dp = Join-Path $script:deltaTestDir '410-test.json' - # Create a delta state with a fabricated expired token - # Graph will return 410 Gone for this invalid token - @{ - deltaLink = "https://graph.microsoft.com/v1.0/users/delta?`$deltatoken=EXPIRED_FAKE_TOKEN_$(New-Guid)" - select = '' - filter = $null - resource = '/users/delta' - lastSync = (Get-Date).AddDays(-30).ToString('o') - itemCount = 10 - graphEndpoint = 'https://graph.microsoft.com' - } | ConvertTo-Json | Set-Content $dp - - # The cmdlet should catch the 410, delete state, and re-sync - $results = Sync-MgxDelta /users/delta -DeltaPath $dp -Property id -Verbose -WarningVariable warnings *>&1 - $warningTexts = $warnings | ForEach-Object { "$_" } - $items = $results | Where-Object { $_ -isnot [System.Management.Automation.VerboseRecord] -and $_ -isnot [System.Management.Automation.WarningRecord] } - - # Should have items (full re-sync happened) - $items.Count | Should -BeGreaterThan 0 - # Delta state should exist (new token saved after re-sync) - Test-Path $dp | Should -BeTrue - # Should have warned about 410 - $warningTexts | Where-Object { $_ -like '*410*' -or $_ -like '*expired*' -or $_ -like '*re-sync*' } | Should -Not -BeNullOrEmpty - } - - # --- #2: 410 on second attempt falls through --- - # Cannot reliably trigger two consecutive 410s against live Graph. - # The first 410 causes a full re-sync which gets a fresh token. - # The only way to get a second 410 is if Graph is broken, which we can't simulate. - # Tested indirectly: test #1 proves the retry loop works once, and the for-loop - # has attempt < 2 with when(attempt == 0) guard, so attempt 1 cannot catch 410. - - # --- #3: OutputFile JSONL mode --- - It 'OutputFile: writes items as one-JSON-per-line' { - $dp = Join-Path $script:deltaTestDir 'jsonl-test.json' - $outFile = Join-Path $script:deltaTestDir 'output.jsonl' - - Sync-MgxDelta /users/delta -DeltaPath $dp -Property id,displayName -OutputFile $outFile - - Test-Path $outFile | Should -BeTrue - $lines = Get-Content $outFile - $lines.Count | Should -BeGreaterThan 0 - # Each line should be valid JSON - foreach ($line in $lines) { - { $line | ConvertFrom-Json } | Should -Not -Throw - } - # First item should have an id property - ($lines[0] | ConvertFrom-Json).id | Should -Not -BeNullOrEmpty - } - - # --- #4: OutputFile temp file cleanup on error --- - It 'OutputFile: no orphan temp files after successful sync' { - $dp = Join-Path $script:deltaTestDir 'tempclean-test.json' - $outFile = Join-Path $script:deltaTestDir 'tempclean-output.jsonl' - - Sync-MgxDelta /users/delta -DeltaPath $dp -Property id -OutputFile $outFile - - # No .tmp files should remain - $tmpFiles = Get-ChildItem $script:deltaTestDir -Filter '*.tmp' - $tmpFiles | Should -BeNullOrEmpty - } - - # --- #5: OutputFile atomic rename on success --- - It 'OutputFile: final file exists, no temp file remains' { - $dp = Join-Path $script:deltaTestDir 'atomic-test.json' - $outFile = Join-Path $script:deltaTestDir 'atomic-output.jsonl' - - Sync-MgxDelta /users/delta -DeltaPath $dp -Property id -OutputFile $outFile - - Test-Path $outFile | Should -BeTrue - # Verify it's a complete file (not truncated) by parsing last line - $lines = Get-Content $outFile - { $lines[-1] | ConvertFrom-Json } | Should -Not -Throw - } - - # --- #6: Cancellation does NOT save delta state --- - It 'Cancellation: delta state unchanged when sync is cancelled' { - $dp = Join-Path $script:deltaTestDir 'cancel-state.json' - - # First: do a successful sync to create a valid delta state - Sync-MgxDelta /users/delta -DeltaPath $dp -Property id - - $stateBefore = Get-Content $dp -Raw - $lastSyncBefore = ($stateBefore | ConvertFrom-Json).lastSync - - # Now start a sync with a very short timeout to force cancellation - $job = Start-Job -ScriptBlock { - param($modulePath, $dp) - Import-Module $modulePath -Force - # This will be killed before completion - Sync-MgxDelta /users/delta -DeltaPath $dp -Property id - } -ArgumentList $ModulePath, $dp - - Start-Sleep -Milliseconds 500 - Stop-Job $job -ErrorAction SilentlyContinue - Remove-Job $job -Force -ErrorAction SilentlyContinue - - # Delta state should still have the ORIGINAL lastSync (not updated by cancelled run) - # Note: the job runs in a separate process so it can't write to the same delta file - # unless it reaches the save point. A 500ms window should be too short for full pagination. - $stateAfter = Get-Content $dp | ConvertFrom-Json - $stateAfter.lastSync | Should -Be $lastSyncBefore - } - - # --- #7: Cancellation cleans up temp file --- - # Tested indirectly by #4 and #6. A cancelled sync in a subprocess - # may leave temp files if killed hard, but the cmdlet's catch block - # does cleanup. Cannot reliably test kill-during-write from Pester. - - # --- #8: Delta state NOT saved on error --- - It 'Error: delta state not saved when Graph returns error' { - $dp = Join-Path $script:deltaTestDir 'error-nosave.json' - if (Test-Path $dp) { Remove-Item $dp } - - # Use a nonexistent endpoint that will return 404 - $Error.Clear() - Sync-MgxDelta /nonexistent/delta -DeltaPath $dp -ErrorAction SilentlyContinue - - # Delta state should NOT exist (error before save) - Test-Path $dp | Should -BeFalse - } - - # --- #9: GraphServiceException (non-410) writes error record --- - It 'Error: 404 from Graph produces ErrorRecord' { - $dp = Join-Path $script:deltaTestDir 'errorrecord-test.json' - $Error.Clear() - - Sync-MgxDelta /nonexistent/delta -DeltaPath $dp -ErrorAction SilentlyContinue - - $Error.Count | Should -BeGreaterThan 0 - # Graph may return 400 or 404 for nonexistent endpoints - $Error[0].FullyQualifiedErrorId | Should -Not -BeNullOrEmpty - } - - # --- #10: BrokenCircuitException writes error record --- - # Cannot reliably trigger circuit breaker from Pester against live Graph - # without sending hundreds of requests that all fail. This would abuse - # the test tenant's quota. The circuit breaker behavior is tested in - # xUnit (CircuitBreakerTests.cs, 15 tests). - - # --- #11: HttpRequestException writes error record --- - # Cannot trigger network errors against a live tenant from Pester. - # Tested in xUnit (Delta_NetworkError_ThrowsHttpRequestException). - - # --- #12: IOException writes error record --- - It 'Error: unwritable OutputFile produces terminating error' { - $dp = Join-Path $script:deltaTestDir 'ioerror-test.json' - $outFile = '/System/locked-output.jsonl' - if (-not $IsMacOS) { $outFile = '/proc/locked-output.jsonl' } - - # ValidateWriteAccess throws a terminating error in BeginProcessing - { Sync-MgxDelta /users/delta -DeltaPath $dp -Property id -OutputFile $outFile -ErrorAction Stop } | - Should -Throw '*Cannot write*' - } - - # --- #13: No delta token received warning --- - It 'Warning: non-delta endpoint produces no-deltaLink warning' { - $dp = Join-Path $script:deltaTestDir 'nodelta-warn.json' - $warnings = $null - - Sync-MgxDelta /users -DeltaPath $dp -Property id -WarningVariable warnings -ErrorAction SilentlyContinue 3>$null - - $warningTexts = $warnings | ForEach-Object { "$_" } - # Should warn about no delta token received (endpoint doesn't return deltaLink) - # OR warn about URI not containing /delta (from BeginProcessing) - $warningTexts | Should -Not -BeNullOrEmpty - } -} - # Sustained auth pipeline test. Exercises token refresh under load. # MSAL's AuthenticationHandler refreshes tokens proactively (5 min before expiry). # Azure AD tokens expire at 60-90 min, so crossing a token boundary requires ~65+ min. diff --git a/tests/Unit/Mgx.Harness.Tests.ps1 b/tests/Unit/Mgx.Harness.Tests.ps1 new file mode 100644 index 0000000..1279a65 --- /dev/null +++ b/tests/Unit/Mgx.Harness.Tests.ps1 @@ -0,0 +1,147 @@ +#Requires -Modules Pester + +<# + Invoke-TestHarness's own result contract, from tests/TestHarness.psm1: a container that + cannot run at all - a parse error, a top-level throw - must not read as a clean pass. + Pester already reports this on the object it hands back (Result, FailedContainersCount); + these pin that it keeps doing so, and that the harness's own warning names the file. + + Each case drives Invoke-TestHarness in a child pwsh process rather than in-process: + Invoke-TestHarness re-imports Pester with -Force on every call, and Pester does not + support that while a run it started is still on the call stack - a second such call inside + this container's own run hangs rather than fails, so the nested run is given a process of + its own. +#> + +BeforeAll { + $script:HarnessModule = Join-Path (Split-Path -Parent $PSScriptRoot) 'TestHarness.psm1' + + # Written once for the container and driven with the arguments below: a scenario is just + # the fixture files plus the path this runs Invoke-TestHarness against. + $script:ChildRunner = Join-Path $TestDrive 'Invoke-HarnessChild.ps1' + Set-Content -Path $script:ChildRunner -Value @' +param( + [Parameter(Mandatory)] [string] $HarnessModule, + [Parameter(Mandatory)] [string] $TestPath, + [Parameter(Mandatory)] [string] $TestResultsFile, + [Parameter(Mandatory)] [string] $ResultFile +) +Import-Module -Name $HarnessModule -Force +$warnings = $null +$result = Invoke-TestHarness -TestPath $TestPath -TestResultsFile $TestResultsFile ` + -IgnoreCodeCoverage -WarningVariable warnings -WarningAction SilentlyContinue +[pscustomobject]@{ + Result = $result.Result + FailedCount = $result.FailedCount + FailedContainersCount = $result.FailedContainersCount + PassedCount = $result.PassedCount + Warnings = @($warnings | ForEach-Object { $_.ToString() }) +} | ConvertTo-Json -Depth 5 | Set-Content -Path $ResultFile -Encoding utf8 +'@ + + function Invoke-HarnessChild + { + <# + .SYNOPSIS + Runs Invoke-TestHarness -TestPath $TestPath in its own pwsh process and + returns the JSON result it wrote, so the harness's re-import of Pester never + runs nested inside this run. + #> + [CmdletBinding()] + param ([Parameter(Mandatory)] [string] $TestPath) + + $stamp = [Guid]::NewGuid().ToString('N') + $resultFile = Join-Path $TestDrive "result-$stamp.json" + $testResultsFile = Join-Path $TestDrive "nunit-$stamp.xml" + $stdout = Join-Path $TestDrive "stdout-$stamp.log" + $stderr = Join-Path $TestDrive "stderr-$stamp.log" + + $arguments = @( + '-NoProfile', '-NonInteractive', '-File', $script:ChildRunner, + '-HarnessModule', $script:HarnessModule, + '-TestPath', $TestPath, + '-TestResultsFile', $testResultsFile, + '-ResultFile', $resultFile + ) + $process = Start-Process -FilePath 'pwsh' -ArgumentList $arguments -PassThru -NoNewWindow ` + -RedirectStandardOutput $stdout -RedirectStandardError $stderr + if (-not $process.WaitForExit(60000)) + { + try { $process.Kill() } catch { } + throw "Invoke-TestHarness child process did not exit within 60s for '$TestPath'." + } + if (-not (Test-Path -Path $resultFile)) + { + $errorText = Get-Content -Path $stderr -Raw -ErrorAction SilentlyContinue + throw "Invoke-TestHarness child process (exit $($process.ExitCode)) wrote no result for '$TestPath'. Stderr: $errorText" + } + return Get-Content -Path $resultFile -Raw | ConvertFrom-Json + } +} + +Describe 'Invoke-TestHarness result contract' { + Context 'a test path with a parse error' { + BeforeAll { + $script:Dir = Join-Path $TestDrive 'parse-error' + New-Item -ItemType Directory -Path $script:Dir -Force | Out-Null + Set-Content -Path (Join-Path $script:Dir 'Ok.Tests.ps1') -Value 'Describe "ok" { It "passes" { $true | Should -BeTrue } }' + # Unclosed block: Pester's own discovery fails to parse the file, the same way a + # dropped brace or a moved helper does. + Set-Content -Path (Join-Path $script:Dir 'Broken.Tests.ps1') -Value 'Describe "broken" {' + $script:Result = Invoke-HarnessChild -TestPath $script:Dir + } + + It 'reports Result Failed' { + $script:Result.Result | Should -Be 'Failed' + } + + It 'counts the container as failed, not a test' { + $script:Result.FailedContainersCount | Should -Be 1 + $script:Result.FailedCount | Should -Be 0 + } + + It 'warns naming the broken file' { + ($script:Result.Warnings -join "`n") | Should -Match ([Regex]::Escape('Broken.Tests.ps1')) + } + } + + Context 'a test path with a top-level throw' { + BeforeAll { + $script:Dir = Join-Path $TestDrive 'top-level-throw' + New-Item -ItemType Directory -Path $script:Dir -Force | Out-Null + Set-Content -Path (Join-Path $script:Dir 'Ok.Tests.ps1') -Value 'Describe "ok" { It "passes" { $true | Should -BeTrue } }' + Set-Content -Path (Join-Path $script:Dir 'Broken.Tests.ps1') -Value 'throw "dot-sourced helper is gone"' + $script:Result = Invoke-HarnessChild -TestPath $script:Dir + } + + It 'reports Result Failed' { + $script:Result.Result | Should -Be 'Failed' + } + + It 'counts the container as failed, not a test' { + $script:Result.FailedContainersCount | Should -Be 1 + $script:Result.FailedCount | Should -Be 0 + } + + It 'warns naming the broken file' { + ($script:Result.Warnings -join "`n") | Should -Match ([Regex]::Escape('Broken.Tests.ps1')) + } + } + + Context 'a clean test path' { + BeforeAll { + $script:Dir = Join-Path $TestDrive 'clean' + New-Item -ItemType Directory -Path $script:Dir -Force | Out-Null + Set-Content -Path (Join-Path $script:Dir 'Ok.Tests.ps1') -Value 'Describe "ok" { It "passes" { $true | Should -BeTrue } }' + $script:Result = Invoke-HarnessChild -TestPath $script:Dir + } + + It 'reports Result Passed' { + $script:Result.Result | Should -Be 'Passed' + } + + It 'writes no warning' { + $script:Result.Warnings.Count | Should -Be 0 + } + } +} diff --git a/tests/Unit/Mgx.HelpFreshness.Tests.ps1 b/tests/Unit/Mgx.HelpFreshness.Tests.ps1 index edc13d5..536674b 100644 --- a/tests/Unit/Mgx.HelpFreshness.Tests.ps1 +++ b/tests/Unit/Mgx.HelpFreshness.Tests.ps1 @@ -8,9 +8,23 @@ rejects. The existing surface tests compare example COUNT and parameter PRESENCE, so none of it registered. - This compares the text. It is deliberately coarse - descriptions are reflowed and links are - rendered differently by the generator - so it checks that the MAML contains no parameter - description that the markdown has since changed, using a normalized comparison. + This compares the text. Every piece of prose is compared whole - the synopsis, the + description, each example's prose and every parameter description - normalized for what the + generator does to it: it reflows prose and it renders a markdown link as its text followed by + its target. Runs of whitespace then collapse to a single space on both sides, and MAML's + separate elements join with a space rather than nothing, so a real edit inside + the prose still tells the two sides apart - only where a paragraph happens to break is out of + the comparison, because a break reads the same as an ordinary space once joined. The one + further allowance is the space after a rendered link's closing ")": platyPS drops it there + when it folds the paragraph that follows a link into the same one. Normalize marks that one + ")" - on the markdown side where it renders the link, and on the MAML side where the link + already reads as its rendered "text (target)" - with a sentinel no help text contains, so + the fold finds exactly that position and nowhere else; an ordinary parenthetical's ")" is + never marked and keeps its space. Measured with New-ExternalHelp (0.14.2): a plain + list item compiles to its own , a wrapped continuation folds into the para of the + item it continues, and whether a break lands before the next bullet depends on whether the + line before it was indented - none of which is where a documentation edit lives, so that + shape is pinned separately, para for para, rather than compared here. #> BeforeAll { @@ -18,14 +32,214 @@ BeforeAll { $script:Maml = Join-Path $repo 'module/en-US/Mgx.Cmdlets.dll-Help.xml' $script:HelpDir = Join-Path $repo 'module/help' + # The mark Normalize leaves on a rendered link's closing ")" - a private-use code point no + # help text contains - so FoldLinkSpace can find that one position later and nothing else. + $script:LinkMark = [char]0xE000 + $script:LinkMarkPattern = [regex]::Escape($script:LinkMark) + function Normalize([string] $t) { if (-not $t) { return '' } - # markdown link -> its text; code ticks and whitespace collapse - $t = [regex]::Replace($t, '\[([^\]]+)\]\([^)]+\)', '$1') + # markdown link -> text followed by its target, the way the generator renders one; code + # ticks and whitespace collapse. The target stays in the comparison, so a link retargeted + # in the markdown is a difference and not a wash. The link's closing ")" is marked so + # FoldLinkSpace can find exactly that position later. + $t = [regex]::Replace($t, '\[([^\]]+)\]\(([^)]+)\)', ('$1 ($2)' + $script:LinkMark)) + # The MAML side never carries that markdown syntax - by the time its text reaches here + # platyPS has already rendered the link to "text (target)" - so the same mark goes on a + # parenthetical whose target is itself a URL, which is what every markdown link in + # module/help points at. The lookahead skips a ")" the line above already marked, so a + # link is never marked twice. + $t = $t -replace "(\(https?://[^)]*\))(?!$script:LinkMarkPattern)", ('$1' + $script:LinkMark) $t = $t -replace '`', '' ($t -replace '\s+', ' ').Trim() } + # Drops the whitespace after the ")" Normalize marked as a rendered link's, and the mark + # itself - the fold platyPS performs when it merges the paragraph after a link into the one + # the link is in. A ")" with no mark - every ordinary parenthetical - is untouched, so a real + # edit that drops its own following space still reads as a difference. + function FoldLinkSpace([string] $t) { + $t -replace "$script:LinkMarkPattern\s*", '' + } + + # An excerpt of $t from $from, marked where it is cut. + function Excerpt([string] $t, [int] $from, [int] $length = 80) { + if ($from -ge $t.Length) { return '' } + $lead = if ($from -gt 0) { '...' } else { '' } + $take = [Math]::Min($length, $t.Length - $from) + $trail = if ($t.Length - $from -gt $length) { '...' } else { '' } + "$lead$($t.Substring($from, $take))$trail" + } + + # Both sides from where they part company: an appended sentence differs only at the end, and + # a message quoting the two openings would print the same words twice. + function Divergence([string] $want, [string] $got) { + $shared = 0 + $common = [Math]::Min($want.Length, $got.Length) + while ($shared -lt $common -and $want[$shared] -eq $got[$shared]) { $shared++ } + $from = [Math]::Max(0, $shared - 20) + "markdown says '$(Excerpt $want $from)' MAML says '$(Excerpt $got $from)'" + } + + # A section's prose with whitespace collapsed to a single space, so a paragraph break folds + # the same way an ordinary space between words does and where the generator happens to break + # or fold one never enters the comparison - only the sequence of prose, marker and + # punctuation characters does. Where that break lands does not track markdown's blank lines: + # a plain list item gets its own , a wrapped continuation folds into the para of + # the item it continues, and a lead-in line folds into the first item that follows it - that + # shape is pinned separately, below, rather than depended on here. The one further fold is the + # space after a rendered link's ")": platyPS drops it there when the paragraph that follows a + # link joins the one the link is in. Normalize already marked that ")", so FoldLinkSpace finds + # only that position - an ordinary parenthetical was never marked and keeps its space. + function Paragraphs([string[]] $lines) { + FoldLinkSpace (Normalize ($lines -join ' ')) + } + + # The paragraphs a MAML node carries at $xpath, joined with a space and folded the way + # Paragraphs folds markdown's. local-name() throughout: the document declares three + # namespaces and the prefixes are not worth registering to read a . + function MamlParas($node, [string] $xpath) { + FoldLinkSpace ((@($node.SelectNodes($xpath) | ForEach-Object { Normalize $_.InnerText } | + Where-Object { $_ })) -join ' ') + } + + # The same paragraphs MamlParas joins, kept apart instead - for pinning the LIST a MAML node + # carries, para for para, rather than the one string MamlParas flattens them to. + function MamlParaList($node, [string] $xpath) { + @($node.SelectNodes($xpath) | ForEach-Object { Normalize $_.InnerText } | Where-Object { $_ } | + ForEach-Object { FoldLinkSpace $_ }) + } + + # The markdown's own list of paragraphs, grouped the way New-ExternalHelp (0.14.2) groups + # them into elements - which does not track markdown's blank lines. A bullet line + # ("-" or "*" then a space, however far indented - a nested item under another still opens + # one) opens a new paragraph, unless the line before it was an indented continuation, in which + # case the break that would otherwise land before a bullet does not. An indented line with no + # bullet marker is always a continuation and always joins the paragraph before it. A lead-in + # line - unindented prose before the first bullet has appeared - joins the paragraph of the + # first bullet after it. Each group is folded the way Paragraphs folds one, so the result + # lines up against MamlParaList's, position for position. + function MarkdownParaList([string[]] $lines) { + $groups = [System.Collections.Generic.List[System.Collections.Generic.List[string]]]::new() + $sawBullet = $false + $prevContinuation = $false + $prevLeadIn = $false + foreach ($line in $lines) { + $bullet = $line -match '^\s*[-*]\s' + $continuation = -not $bullet -and $line -match '^\s+\S' + $suppress = $continuation -or $prevContinuation -or ($prevLeadIn -and -not $sawBullet) + if ($groups.Count -eq 0 -or -not $suppress) { + $groups.Add([System.Collections.Generic.List[string]]::new()) + } + $groups[$groups.Count - 1].Add($line) + if ($bullet) { $sawBullet = $true } + $prevContinuation = $continuation + $prevLeadIn = -not $bullet -and -not $continuation + } + @($groups | ForEach-Object { Paragraphs $_ }) + } + + # A node carrying one per string in $paras, so a pinned MAML shape can + # be read back through MamlParas itself rather than a second copy of what it does. + function AsMamlParas([string[]] $paras) { + $xml = [xml]'' + foreach ($p in $paras) { + $el = $xml.CreateElement('para') + $el.InnerText = $p + [void]$xml.DocumentElement.AppendChild($el) + } + $xml.DocumentElement + } + + # The lines of one '## ' section, up to the next heading of any level. A line + # inside a ``` fence never ends a section: a fenced example's own '# comment' opens with + # the same character a heading does, so fence state is tracked and heading detection is + # suspended for as long as one is open - the fenced lines themselves are code, not prose, + # so they are dropped rather than carried into the section like MarkdownExamples drops them. + function SectionLines([string[]] $lines, [string] $heading) { + $out = [System.Collections.Generic.List[string]]::new() + $in = $false + $fenced = $false + foreach ($line in $lines) { + if ($line -match "^## $heading\s*$") { $in = $true; continue } + if (-not $in) { continue } + if ($line -match '^```') { $fenced = -not $fenced; continue } + if ($fenced) { continue } + if ($line -match '^#{1,3} ') { break } + $out.Add($line) + } + $out + } + + # Every '### Example' heading with its title and the prose around its code fences. Prose + # stops at the second fence, the way the generator's does: a fence count + # resets with each example, and once it reaches two, nothing further joins the prose - only + # the code between the fences was ever remarks, not whatever trails the second one. + function MarkdownExamples([string[]] $lines) { + $examples = [System.Collections.Generic.List[object]]::new() + $title = $null + $prose = [System.Collections.Generic.List[string]]::new() + $fenced = $false + $fences = 0 + foreach ($line in $lines) { + # The fence is read before anything else, and nothing inside one is read as markdown: + # a PowerShell comment opens with the '#' a heading opens with, and most of these + # examples carry one. + if ($line -match '^```') { + $fenced = -not $fenced + if ($fenced) { $fences++ } + continue + } + if ($fenced) { continue } + if ($line -match '^### Example') { + if ($title) { $examples.Add([pscustomobject]@{ Title = $title; Prose = Paragraphs $prose }) } + $title = Normalize ($line -replace '^### ', '') + $prose.Clear() + $fences = 0 + continue + } + if (-not $title) { continue } + if ($line -match '^#{1,3} ') { + $examples.Add([pscustomobject]@{ Title = $title; Prose = Paragraphs $prose }) + $title = $null + continue + } + if ($fences -ge 2) { continue } + $prose.Add($line) + } + if ($title) { $examples.Add([pscustomobject]@{ Title = $title; Prose = Paragraphs $prose }) } + $examples + } + + # Every '### ' entry under OUTPUTS, with the prose beneath it - the type from the + # heading and the prose from the lines under it, read the same way MarkdownExamples reads an + # example: entering and leaving on '## ' headings of its own, a '### ' line starting a new + # entry rather than ending the section, and a fence dropped rather than read as markdown. + function MarkdownOutputs([string[]] $lines) { + $outputs = [System.Collections.Generic.List[object]]::new() + $in = $false + $type = $null + $prose = [System.Collections.Generic.List[string]]::new() + $fenced = $false + foreach ($line in $lines) { + if ($line -match '^## OUTPUTS\s*$') { $in = $true; continue } + if (-not $in) { continue } + if ($line -match '^```') { $fenced = -not $fenced; continue } + if ($fenced) { continue } + if ($line -match '^## ') { break } + if ($line -match '^### (.+)$') { + if ($type) { $outputs.Add([pscustomobject]@{ Type = $type; Prose = Paragraphs $prose }) } + $type = $Matches[1].Trim() + $prose.Clear() + continue + } + if (-not $type) { continue } + $prose.Add($line) + } + if ($type) { $outputs.Add([pscustomobject]@{ Type = $type; Prose = Paragraphs $prose }) } + $outputs + } + $script:MamlXml = if (Test-Path $script:Maml) { [xml](Get-Content $script:Maml -Raw) } else { $null } } @@ -42,6 +256,7 @@ Describe 'Compiled help matches its markdown source' { @{ Text = 'instead of PSObjects'; Why = '-Raw returns raw JSON instead of hashtables' } @{ Text = 'ephemeral and deleted on success'; Why = 'Sync-MgxDelta gained -CheckpointPath in 2.1.0' } @{ Text = 'belongs to a chunk that was refused'; Why = 'after a chunk failure the batch-level retry pass is skipped for every candidate, including ones a cleanly answered chunk left failing' } + @{ Text = 'its request went out and may have been applied'; Why = 'a refusal status also covers a POST an open circuit or a rate limiter stopped before it left' } ) $raw = Get-Content $script:Maml -Raw $hits = $stale | Where-Object { $raw -match [regex]::Escape($_.Text) } @@ -51,9 +266,13 @@ Describe 'Compiled help matches its markdown source' { It 'carries the synopsis, description, outputs and examples its markdown gives' { # The parameter check below was the whole guard, so a SYNOPSIS saying the wrong thing - # entirely, a replaced OUTPUTS section, or a renamed example all shipped green. Sections - # are compared by their opening sentence: the generator reflows prose, so a prefix is - # what can be compared honestly. + # entirely, a replaced OUTPUTS section, or a renamed example all shipped green. Comparing + # a section by its opening sentence closed only half of that: the opening survives a + # rewrite of everything under it, so a DESCRIPTION whose remaining paragraphs said + # "deletes every user in the tenant" passed, and so did an example whose prose described + # something its code does not do, and so did a mutated OUTPUTS type. Prose is compared + # whole, including each OUTPUTS entry's - a cmdlet can document more than one return type, + # and each carries its own type and its own prose. $mismatches = [System.Collections.Generic.List[string]]::new() foreach ($md in Get-ChildItem $script:HelpDir -Filter '*.md') { @@ -63,51 +282,52 @@ Describe 'Compiled help matches its markdown source' { Where-Object { $_.details.name.Trim() -eq $cmdlet } | Select-Object -First 1 if (-not $node) { $mismatches.Add("$cmdlet : absent from MAML"); continue } - function SectionText([string[]] $l, [string] $heading) { - $out = [System.Collections.Generic.List[string]]::new() - $in = $false - foreach ($line in $l) { - if ($line -match "^## $heading\s*$") { $in = $true; continue } - if ($in -and $line -match '^#{1,3} ') { break } - if ($in -and $line.Trim()) { $out.Add($line) } - } - Normalize ($out -join ' ') - } - function Head([string] $t, [int] $n = 50) { - if ($t.Length -gt $n) { $t.Substring(0, $n) } else { $t } - } - - $wantSyn = SectionText $lines 'SYNOPSIS' - $gotSyn = Normalize $node.details.description.para - if ($wantSyn -and $gotSyn -notlike "*$(Head $wantSyn)*") { - $mismatches.Add("$cmdlet SYNOPSIS : markdown '$(Head $wantSyn)...' MAML '$(Head $gotSyn)...'") - } - - $wantOut = SectionText $lines 'OUTPUTS' - $gotOut = Normalize (($node.returnValues.returnValue.type.name | ForEach-Object { $_ }) -join ' ') - if ($wantOut -and $gotOut) { - # OUTPUTS opens with the type name under a ### heading; compare that token. - $wantType = ($lines | Where-Object { $_ -match '^### ' } | ForEach-Object { $_ } | - Select-Object -First 0) - $typeLine = $null - $inOut = $false - foreach ($line in $lines) { - if ($line -match '^## OUTPUTS\s*$') { $inOut = $true; continue } - if ($inOut -and $line -match '^## ') { break } - if ($inOut -and $line -match '^### (.+)$') { $typeLine = $Matches[1].Trim(); break } + $wantSyn = Paragraphs (SectionLines $lines 'SYNOPSIS') + $gotSyn = MamlParas $node "*[local-name()='details']/*[local-name()='description']/*[local-name()='para']" + if ($wantSyn -ne $gotSyn) { + $mismatches.Add("$cmdlet SYNOPSIS : $(Divergence $wantSyn $gotSyn)") + } + + $wantDesc = Paragraphs (SectionLines $lines 'DESCRIPTION') + $gotDesc = MamlParas $node "*[local-name()='description']/*[local-name()='para']" + if ($wantDesc -ne $gotDesc) { + $mismatches.Add("$cmdlet DESCRIPTION : $(Divergence $wantDesc $gotDesc)") + } + + $mdOutputs = @(MarkdownOutputs $lines) + $mamlOutputs = @($node.SelectNodes("*[local-name()='returnValues']/*[local-name()='returnValue']")) + for ($i = 0; $i -lt [Math]::Min($mdOutputs.Count, $mamlOutputs.Count); $i++) { + $wantType = Normalize $mdOutputs[$i].Type + $gotType = Normalize $mamlOutputs[$i].SelectSingleNode("*[local-name()='type']/*[local-name()='name']").InnerText + if ($wantType -ne $gotType) { + $mismatches.Add("$cmdlet OUTPUTS $($i + 1) type : markdown '$wantType' MAML '$gotType'") } - if ($typeLine -and $gotOut -notlike "*$typeLine*") { - $mismatches.Add("$cmdlet OUTPUTS : markdown '$typeLine' MAML '$(Head $gotOut 60)'") + + $wantOutProse = $mdOutputs[$i].Prose + $gotOutProse = MamlParas $mamlOutputs[$i] "*[local-name()='description']/*[local-name()='para']" + if ($wantOutProse -ne $gotOutProse) { + $mismatches.Add("$cmdlet OUTPUTS $($i + 1) prose : $(Divergence $wantOutProse $gotOutProse)") } } + if ($mdOutputs.Count -ne $mamlOutputs.Count) { + $mismatches.Add("$cmdlet OUTPUTS : markdown names $($mdOutputs.Count) type(s), MAML carries $($mamlOutputs.Count)") + } - $mdExamples = @($lines | Where-Object { $_ -match '^### Example' }) - $mamlExamples = @($node.examples.example) + $mdExamples = @(MarkdownExamples $lines) + $mamlExamples = @($node.SelectNodes("*[local-name()='examples']/*[local-name()='example']")) for ($i = 0; $i -lt [Math]::Min($mdExamples.Count, $mamlExamples.Count); $i++) { - $want = Normalize ($mdExamples[$i] -replace '^### ', '') - $got = Normalize $mamlExamples[$i].title - if ($got -notlike "*$want*") { - $mismatches.Add("$cmdlet example $($i + 1) : markdown '$want' MAML '$got'") + # The generator pads a title out to a fixed width with dashes, so the title is + # the one thing here compared by containment rather than whole. + $wantTitle = $mdExamples[$i].Title + $gotTitle = Normalize $mamlExamples[$i].SelectSingleNode("*[local-name()='title']").InnerText + if ($gotTitle -notlike "*$wantTitle*") { + $mismatches.Add("$cmdlet example $($i + 1) : markdown '$wantTitle' MAML '$gotTitle'") + } + + $wantProse = $mdExamples[$i].Prose + $gotProse = MamlParas $mamlExamples[$i] "*[local-name()='remarks']/*[local-name()='para']" + if ($wantProse -ne $gotProse) { + $mismatches.Add("$cmdlet example $($i + 1) prose : $(Divergence $wantProse $gotProse)") } } } @@ -115,7 +335,78 @@ Describe 'Compiled help matches its markdown source' { ($mismatches -join "`n") | Should -BeNullOrEmpty -Because 'run ./build.ps1 to regenerate the compiled help' } + It 'reads a list the way the generator compiles it, seams and all' { + # No compared section carries a list today, so nothing above exercises this. Each shape + # pins the para LIST New-ExternalHelp (0.14.2) actually compiled its markdown to on a + # scratch help file: a plain item gets its own para, a wrapped continuation folds into + # the para of the item it continues, and a lead-in line folds into the first item after + # it - a different break position per shape. MarkdownParaList rebuilds that same list from + # the markdown side independently, so the two are compared para for para - count and each + # para's text - rather than joined into one string where a fixture that merged or split a + # shape's paras would still read as a match. + $shapes = @( + @{ Name = 'plain two items'; Markdown = @('- Item one', '- Item two') + Maml = @('- Item one', '- Item two') } + @{ Name = 'plain three items'; Markdown = @('- Item one', '- Item two', '- Item three') + Maml = @('- Item one', '- Item two', '- Item three') } + @{ Name = '* items'; Markdown = @('* Item one', '* Item two') + Maml = @('* Item one', '* Item two') } + @{ Name = 'a nested item' + Markdown = @('- Item one', '- Item two', ' - Nested under two') + Maml = @('- Item one', '- Item two', ' - Nested under two') } + @{ Name = 'a wrapped item' + Markdown = @('- Item one', ' continued here.', '- Item two') + Maml = @('- Item one continued here. - Item two') } + @{ Name = 'a wrapped first item' + Markdown = @('- Item one', ' continued here.', '- Item two', '- Item three') + Maml = @('- Item one continued here. - Item two', '- Item three') } + @{ Name = 'a lead-in line' + Markdown = @('Lead prose.', '- Item one', '- Item two') + Maml = @('Lead prose. - Item one', '- Item two') } + ) + + foreach ($shape in $shapes) { + $want = @(MarkdownParaList $shape.Markdown) + $got = @(MamlParaList (AsMamlParas $shape.Maml) "*[local-name()='para']") + $got.Count | Should -Be $want.Count -Because "the $($shape.Name) shape should compile to $($want.Count) para(s)" + for ($i = 0; $i -lt $want.Count; $i++) { + $got[$i] | Should -Be $want[$i] -Because "the $($shape.Name) shape's para $($i + 1) should match para for para" + } + } + } + + It 'still catches a changed word once the seams are out of the comparison' { + # Stripping whitespace must not blind the comparison to the prose itself - only to where + # the generator chose to break it. + $want = Paragraphs @('- Item one', '- Item two') + $got = MamlParas (AsMamlParas @('- Item one', '- Item deux')) "*[local-name()='para']" + $want | Should -Not -Be $got + } + + It 'still catches a parenthetical that lost its own space' { + # module/help/Enable-MgxResilience.md, measured: nothing about "(Get-MgUser, Get-MgGroup, + # etc.)" tells platyPS to fold the sentence after it into the same paragraph, so a MAML + # that lost the space there is a real drift, not the one platyPS itself produces. + $want = Paragraphs @('After calling this cmdlet, all SDK cmdlets (Get-MgUser, Get-MgGroup, etc.) automatically gain resilience with zero script changes.') + $got = MamlParas (AsMamlParas @('After calling this cmdlet, all SDK cmdlets (Get-MgUser, Get-MgGroup, etc.)automatically gain resilience with zero script changes.')) "*[local-name()='para']" + $want | Should -Not -Be $got -Because 'only a rendered link''s own ")" may lose its following space' + } + + It 'tolerates the space a rendered link''s fold actually drops' { + # module/help/Invoke-MgxBatchRequest.md DESCRIPTION, measured with New-ExternalHelp + # (0.14.2): the paragraph after a link folds into the link's own with no + # space at the seam, so the markdown's two paragraphs and the MAML's one must still + # compare equal. + $want = Paragraphs @('Source: [Combine multiple HTTP requests using JSON batching](https://learn.microsoft.com/en-us/graph/json-batching)', '', 'Pipeline input can be string URLs.') + $got = MamlParas (AsMamlParas @('Source: Combine multiple HTTP requests using JSON batching (https://learn.microsoft.com/en-us/graph/json-batching)Pipeline input can be string URLs.')) "*[local-name()='para']" + $want | Should -Be $got -Because "platyPS's own fold at a link boundary is not a drift" + } + It 'documents every parameter with the description its markdown gives' { + # Whole descriptions, not their openings. A comparison of the first 60 characters passes + # every edit that appends to a parameter which already said something true - a widened + # range, a sentence naming what a gate covers, a sentence naming what a redaction reaches + # - and those are most of what a help edit is. $mismatches = [System.Collections.Generic.List[string]]::new() foreach ($md in Get-ChildItem $script:HelpDir -Filter '*.md') { @@ -128,7 +419,11 @@ Describe 'Compiled help matches its markdown source' { $param = $Matches[1] $desc = [System.Collections.Generic.List[string]]::new() for ($j = $i + 1; $j -lt $lines.Count; $j++) { - if ($lines[$j] -match '^```' -or $lines[$j] -match '^### ' -or $lines[$j] -match '^## ') { break } + # The yaml block ends the description. A fence that is not it belongs to the + # description and is compared with it: the generator renders a fenced block + # verbatim and eats the asterisks of anything written into prose, so a + # description naming a literal like ***REDACTED*** has to fence it. + if ($lines[$j] -match '^```yaml' -or $lines[$j] -match '^### ' -or $lines[$j] -match '^## ') { break } if ($lines[$j].Trim()) { $desc.Add($lines[$j]) } } $want = Normalize ($desc -join ' ') @@ -142,16 +437,151 @@ Describe 'Compiled help matches its markdown source' { if (-not $node) { $mismatches.Add("$cmdlet -$param : absent from MAML"); continue } $got = Normalize (($node.description.para | ForEach-Object { $_ }) -join ' ') - # Coarse: the MAML must open with what the markdown says. - $head = if ($want.Length -gt 60) { $want.Substring(0, 60) } else { $want } - if ($got -notlike "*$head*") { - $mismatches.Add("$cmdlet -$param : markdown says '$head...' MAML says '$($got.Substring(0,[Math]::Min(60,$got.Length)))...'") + if ($got -ne $want) { + $mismatches.Add("$cmdlet -$param : $(Divergence $want $got)") } } } ($mismatches -join "`n") | Should -BeNullOrEmpty -Because 'run ./build.ps1 to regenerate the compiled help' } + + It 'shows every parameter of the cmdlet in its SYNTAX block' { + # The checks above compare sections and parameter descriptions, and a parameter can have + # a section of its own while the SYNTAX block does not list it - which is the block a + # reader looks at first, and the only place that says which parameters go together. + # -BatchChunkConcurrency had a full section and a range the cmdlet enforces, and was + # missing from the line above it. + # + # -ProgressAction is off both sides. PowerShell 7.4 made it a common parameter, so + # Get-Command reports it among those, while platyPS 0.14.2 does not know it and writes it + # into the block as an ordinary one: comparing it would say every block is wrong. + $repo = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent + Import-Module "$repo/module/mgx.psd1" -Force + $common = [System.Management.Automation.PSCmdlet]::CommonParameters + $mismatches = [System.Collections.Generic.List[string]]::new() + + foreach ($md in Get-ChildItem $script:HelpDir -Filter '*.md') { + $cmdInfo = Get-Command -Name $md.BaseName -ErrorAction Stop + $cmdInfo | Should -Not -BeNullOrEmpty -Because "$($md.BaseName) must resolve for this comparison to mean anything" + + # Each fenced block under ## SYNTAX, keyed by the parameter set the heading above it + # names. A single-set cmdlet has no heading, and Get-Command calls that set + # __AllParameterSets. + $blocks = @{} + $inSyntax = $false + $inFence = $false + $setName = $null + $buffer = $null + foreach ($line in Get-Content $md.FullName) { + if ($line -match '^## SYNTAX\s*$') { $inSyntax = $true; continue } + if ($inSyntax -and $line -match '^## ') { break } + if (-not $inSyntax) { continue } + if ($line -match '^### (.+)$') { + $setName = ($Matches[1] -replace '\s*\(Default\)\s*$', '').Trim() + continue + } + if ($line -match '^```') { + if ($inFence) { + $key = if ($setName) { $setName } else { '__AllParameterSets' } + $blocks[$key] = $buffer -join ' ' + $inFence = $false + $buffer = $null + } + else { $inFence = $true; $buffer = @() } + continue + } + if ($inFence) { $buffer += $line } + } + + if ($blocks.Count -eq 0) { + $mismatches.Add("$($md.BaseName) : no SYNTAX block") + continue + } + + foreach ($key in $blocks.Keys) { + $set = $cmdInfo.ParameterSets | Where-Object { $_.Name -eq $key } + if (-not $set) { + $mismatches.Add("$($md.BaseName) SYNTAX [$key] : the cmdlet has no such parameter set (it has $(($cmdInfo.ParameterSets.Name) -join ', '))") + continue + } + + # A '-' that opens a parameter follows a '[' or a space, which is what keeps the + # one in the cmdlet's own name out of the list. + $shown = @([regex]::Matches($blocks[$key], '(?:^|[\s\[])-([A-Za-z]\w*)') | + ForEach-Object { $_.Groups[1].Value } | + Where-Object { $_ -ne 'ProgressAction' } | Sort-Object -Unique) + $takes = @($set.Parameters.Name | + Where-Object { $_ -notin $common -and $_ -ne 'ProgressAction' } | + Sort-Object -Unique) + + $absent = @($takes | Where-Object { $_ -notin $shown }) + $extra = @($shown | Where-Object { $_ -notin $takes }) + if ($absent) { $mismatches.Add("$($md.BaseName) SYNTAX [$key] : absent $($absent -join ', ')") } + if ($extra) { $mismatches.Add("$($md.BaseName) SYNTAX [$key] : names $($extra -join ', '), which the cmdlet does not take") } + } + } + + ($mismatches -join "`n") | Should -BeNullOrEmpty -Because 'a SYNTAX block should list the parameters its cmdlet takes' + } + + It 'gives every parameter one yaml block naming the type its cmdlet declares' { + # The block under a parameter's heading is the only place the markdown states the type, + # the parameter sets, the default and the pipeline binding, and platyPS builds both the + # compiled parameter entry and the SYNTAX line out of it. A section with no block leaves + # Get-Help reporting no type at all - -ConsistencyLevel had none, and the block that + # belonged to it sat under -ThrottlePriority's, where nothing reads it - and a section + # with two makes the second dead text. The checks above compare prose, so neither showed. + # + # Type: is the .NET type's short name, what Type.Name gives: Int32, String, + # SwitchParameter, ActionPreference, Hashtable, and an array keeps its brackets, String[]. + # Never the namespace-qualified name. + $repo = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent + Import-Module "$repo/module/mgx.psd1" -Force + $mismatches = [System.Collections.Generic.List[string]]::new() + + foreach ($md in Get-ChildItem $script:HelpDir -Filter '*.md') { + $cmdInfo = Get-Command -Name $md.BaseName -ErrorAction Stop + $lines = Get-Content $md.FullName + + for ($i = 0; $i -lt $lines.Count; $i++) { + if ($lines[$i] -notmatch '^### -(\w+)$') { continue } + $param = $Matches[1] + + # Only a ```yaml fence opens one. A description may carry a fence of its own - + # the generator eats the asterisks of a literal written into prose - and its + # closing fence is bare, so counting openings keeps the two apart. + $blocks = 0 + $inYaml = $false + $types = [System.Collections.Generic.List[string]]::new() + for ($j = $i + 1; $j -lt $lines.Count; $j++) { + if ($lines[$j] -match '^### ' -or $lines[$j] -match '^## ') { break } + if ($lines[$j] -match '^```yaml\s*$') { $blocks++; $inYaml = $true; continue } + if ($inYaml -and $lines[$j] -match '^```\s*$') { $inYaml = $false; continue } + if ($inYaml -and $lines[$j] -match '^Type:\s*(.+)$') { $types.Add($Matches[1].Trim()) } + } + + if ($blocks -ne 1) { + $mismatches.Add("$($md.BaseName) -$param : $blocks yaml blocks under the heading, not 1") + continue + } + if (-not $cmdInfo.Parameters.ContainsKey($param)) { + $mismatches.Add("$($md.BaseName) -$param : the cmdlet takes no such parameter") + continue + } + + $declared = $cmdInfo.Parameters[$param].ParameterType.Name + if ($types.Count -ne 1) { + $mismatches.Add("$($md.BaseName) -$param : the yaml block names no Type; the cmdlet declares $declared") + } + elseif ($types[0] -ne $declared) { + $mismatches.Add("$($md.BaseName) -$param : the yaml block says Type: $($types[0]), the cmdlet declares $declared") + } + } + } + + ($mismatches -join "`n") | Should -BeNullOrEmpty -Because "a parameter's yaml block should name the type its cmdlet declares" + } } Describe 'Documented output types match the cmdlets' { diff --git a/tests/benchmarks/06-fault-gauntlet.ps1 b/tests/benchmarks/06-fault-gauntlet.ps1 index 3e17847..ba02c14 100644 --- a/tests/benchmarks/06-fault-gauntlet.ps1 +++ b/tests/benchmarks/06-fault-gauntlet.ps1 @@ -8,6 +8,10 @@ # 4. mgx - ids | Invoke-MgxRequest '/users/{id}' (fan-out, full resilience stack) # The server's /reset is called between contenders so each faces the identical schedule. # Fault profile: 15% of ids throttle once (429, Retry-After 1s); 3% fail twice with 503. +# +# The server also takes a programmable fault plan over POST /plan. 06 posts none, so what +# it measures is the default schedule above and nothing else; a scenario wanting a plan +# gets its own script rather than growing this one. param( [int] $N = 1000, [int] $Port = 8787 @@ -23,34 +27,14 @@ Import-Module Microsoft.Graph.Users # claiming no pipeline input. Real Graph ids are strings, so this matches actual usage. $ids = @(1..$N | ForEach-Object { "$_" }) -# Find a free port (a crashed prior run can leave a zombie listener behind) -$Port = ($Port..($Port + 20)) | Where-Object { - -not (Test-Connection -TargetName localhost -TcpPort $_ -TimeoutSeconds 1 -Quiet) -} | Select-Object -First 1 -if (-not $Port) { throw 'no free port found in range' } -$base = "http://localhost:$Port" - -# --- Start mock server as a child process --- -$server = Start-Process pwsh -PassThru ` - -ArgumentList '-NoProfile', '-File', (Join-Path $PSScriptRoot 'mock-graph-server.ps1'), '-Port', $Port +# --- Start mock server as a child process, on a port nothing else is holding --- +$server = Start-BenchMockServer -Port $Port +$base = $server.BaseUrl try { - $up = $false - foreach ($i in 1..50) { - try { $null = Invoke-RestMethod "$base/ping" -TimeoutSec 1; $up = $true; break } catch { Start-Sleep -Milliseconds 200 } - } - if (-not $up) { throw "mock server did not come up on port $Port" } - Write-Host "mock server up (pid $($server.Id))" - # --- Point the whole Graph stack at the mock --- - if (-not (Get-MgEnvironment -Name MgxBench -ErrorAction SilentlyContinue)) { - Add-MgEnvironment -Name MgxBench -GraphEndpoint $base -AzureADEndpoint 'https://login.microsoftonline.com' | Out-Null - } - Disconnect-MgGraph -ErrorAction SilentlyContinue | Out-Null - Connect-MgGraph -Environment MgxBench ` - -AccessToken (ConvertTo-SecureString 'mock-token-not-validated' -AsPlainText -Force) -NoWelcome - # -AccessToken auth leaves AuthContext.TenantId empty, and Mgx's auth fingerprint - # treats an empty TenantId as "not connected" (1.0.4 identity fix). Give it a value. - [Microsoft.Graph.PowerShell.Authentication.GraphSession]::Instance.AuthContext.TenantId = 'gauntlet-mock-tenant' + # This run's port, every run: the environment is written into the user profile, so one + # left standing from a previous run names a port nothing is listening on now. + Connect-BenchMockGraph -GraphEndpoint $base function Reset-Server { $null = Invoke-RestMethod "$base/reset" } function Get-ServerStats { Invoke-RestMethod "$base/stats" } @@ -121,5 +105,6 @@ try { } finally { Disconnect-MgGraph -ErrorAction SilentlyContinue | Out-Null - if ($server -and -not $server.HasExited) { $server.Kill() } + Remove-BenchGraphEnvironment + Stop-BenchMockServer $server } diff --git a/tests/benchmarks/10-pacing-under-real-throttling.ps1 b/tests/benchmarks/10-pacing-under-real-throttling.ps1 index 7a197e9..5e98a5f 100644 --- a/tests/benchmarks/10-pacing-under-real-throttling.ps1 +++ b/tests/benchmarks/10-pacing-under-real-throttling.ps1 @@ -19,22 +19,44 @@ # happened to finish first. # # Reaching the regime needs more in flight than one pipeline holds: Invoke-MgxRequest caps -# -Concurrency at 128, and 128 in flight measured ~180 req/s. The work is therefore split across -# N pipelines in parallel runspaces of ONE process. The pacer, the telemetry collector and the -# HTTP client are all process-wide statics, so those runspaces share a single pacer - which is -# the point: one workload bucket, N producers, the shape a real fan-out has. +# -Concurrency at 128, and 128 in flight measured ~180 req/s - a quarter of the rate this tenant +# keeps serving at while refusing. Rate is the lever, and volume cannot substitute for it. The +# work is therefore split across N pipelines in parallel runspaces of ONE process. The pacer, the +# telemetry collector and the HTTP client are all process-wide statics, so those runspaces share +# a single pacer - which is the point: one workload bucket, N producers, the shape a real fan-out +# has. The roof is the transport's: 20 pooled connections multiplexing HTTP/2 streams hold +# roughly 2,000 requests in flight, and the 12 x 128 default sits under it. +# +# The ids handed to a pipeline have to be distinct. Invoke-MgxRequest deduplicates the ids it is +# piped, by design, so a slice that repeats an id issues one request for it - and a -Count above +# what the pool can cover was inert: the arm reported the volume it was asked for and issued a +# fraction of it. Each pipeline now draws IdsPerPipeline distinct ids off the shared pool, from a +# rotated start so the pipelines are not all on the same object at once, and the arm refuses to +# run if any slice holds a duplicate. Requests issued is then the number of ids handed out, and a +# -Count above Pipelines x IdPool is reported as clamped rather than quietly dropped. +# +# -Plan prints that shape and the admissibility arithmetic against 13's thresholds and exits +# without connecting, which is how a parameter set is checked before spending an hour on it. # # Requests are /users/{id}?$select=id, 1 RU each, the same shape 13 calibrated the thresholds # with, so request counts and unit counts are the same number. # # Reads only: throttling is not write-specific, so this needs no write consent. +# +# CmdletBinding so an argument this script does not define is refused rather than collected into +# $args and ignored: a run that silently drops the parameter it was given is the same class of +# wrong answer as one that silently drops the requests it was given. +[CmdletBinding()] param( [ValidateSet('both', 'paced', 'unpaced')] [string] $Mode = 'both', - [int] $Count = 120000, - [int] $Pipelines = 8, + # 12 x 5,000: 60,000 units at a projected ~2,160 RU/s, which is ~10s inside the seeded + # tenant's 23,000 RU allowance and ~51s past it at the 731 RU/s it serves while refusing. + [ValidateRange(1, [int]::MaxValue)] [int] $Count = 60000, + [ValidateRange(1, 512)] [int] $Pipelines = 12, [ValidateRange(1, 128)] [int] $Concurrency = 128, - [int] $IdPool = 5000, - [int] $CooldownSeconds = 300 + [ValidateRange(1, [int]::MaxValue)] [int] $IdPool = 5000, + [int] $CooldownSeconds = 300, + [switch] $Plan ) . "$PSScriptRoot/common.ps1" @@ -70,17 +92,58 @@ function Get-ThrottleThreshold { } $threshold = Get-ThrottleThreshold +$shape = Get-BenchPacingLoadShape -Count $Count -Pipelines $Pipelines -Concurrency $Concurrency ` + -IdPool $IdPool -BurstRu $threshold.BurstRu -SustainedRuPerSecond $threshold.Sustained -if ($Mode -eq 'both') { +function Show-LoadShape { + <# + What each arm will issue, and whether that can reach the regime - printed before + anything connects, because the answer is arithmetic and does not need a tenant. + #> Write-Host ("throttling threshold from {0}: burst {1} RU, sustained {2} RU/s" -f ` $threshold.Source, $threshold.BurstRu, $threshold.Sustained) - if ($Count -lt $threshold.BurstRu * 3) { - Write-Host (" WARNING: -Count $Count is under 3x the burst allowance ($($threshold.BurstRu) RU).") -ForegroundColor Yellow - Write-Host (" The run may finish inside the allowance and measure nothing. Raise -Count.") -ForegroundColor Yellow + Write-Host '=== LOAD SHAPE (per arm) ===' + Write-Host (" distinct ids per pipeline {0,10} (pool of {1}, shared across pipelines)" -f ` + $shape.IdsPerPipeline, $shape.IdPool) + Write-Host (" pipelines {0,10} x concurrency {1} = {2} requests in flight" -f ` + $shape.Pipelines, $shape.Concurrency, $shape.InFlight) + Write-Host (" requests issued {0,10} = resource units (1 RU each)" -f $shape.TotalRequests) + if ($shape.Clamped) { + Write-Host (" CLAMPED: -Count {0} exceeds {1} pipelines x {2} pool ids; the arm issues {3}." -f ` + $shape.Requested, $shape.Pipelines, $shape.IdPool, $shape.TotalRequests) -ForegroundColor Yellow + } + Write-Host (" projected draw {0,10} RU/s (from 180 RU/s per pipeline at 128)" -f ` + $shape.ExpectedRuPerSecond) + Write-Host (" time inside the allowance {0,10}s then {1}s past it at {2} RU/s" -f ` + $shape.BurstSeconds, $shape.ThrottledSeconds, $shape.SustainedRuPerSecond) + Write-Host '' + if ($shape.Admissible) { + Write-Host ("ADMISSIBLE: {0} RU/s over the {1} RU/s ceiling, and {2} of {3} estimated seconds spent past the allowance." -f ` + $shape.ExpectedRuPerSecond, $shape.SustainedRuPerSecond, $shape.ThrottledSeconds, $shape.EstimatedSeconds) } + else { + Write-Host 'INADMISSIBLE: this parameter set cannot reach the throttled regime.' -ForegroundColor Yellow + foreach ($reason in $shape.Reasons) { Write-Host " $reason" -ForegroundColor Yellow } + } +} + +if ($Plan) { + Show-LoadShape + Write-Host '' + Write-Host '-Plan: nothing was run and no tenant was contacted.' + return +} + +if ($Mode -eq 'both') { + Show-LoadShape + Write-Host '' $resultFile = Join-Path $PSScriptRoot 'results/10-pacing-under-real-throttling.json' $arms = [ordered]@{} + # The arms' own entries, kept whole beside the figures read out of them: this process starts + # the two rounds and reads their files, and never imports the local module or connects to + # anything, so the identity of the runs being compared is only in what they recorded. + $armEntries = @() $roundIndex = 0 foreach ($m in 'paced', 'unpaced') { $roundIndex++ @@ -105,6 +168,7 @@ if ($Mode -eq 'both') { $entries = @(Get-Content $resultFile -Raw | ConvertFrom-Json) $entry = @($entries | Select-Object -Skip $before | Where-Object { $_.Result.Mode -eq $m } | Select-Object -Last 1)[0] if (-not $entry) { throw "round '$m' recorded no result in $resultFile" } + $armEntries += $entry $r = $entry.Result $arms[$m] = @{ Seconds = [math]::Round($r.WallMs / 1000, 1) @@ -115,11 +179,13 @@ if ($Mode -eq 'both') { PacingWaitMs = [long]$r.PacingWaitMs ResourceUnits = [long]$r.ResourceUnits RuPerSecond = if ($r.WallMs -gt 0) { [math]::Round($r.ResourceUnits / ($r.WallMs / 1000), 1) } else { 0 } + ReachedRegime = [int]$r.ThrottleRetries -gt 0 } } Write-Host '' - Write-Host "=== ADAPTIVE PACING AGAINST REAL THROTTLING ($Count reads, $Pipelines x $Concurrency in flight) ===" + Write-Host ("=== ADAPTIVE PACING AGAINST REAL THROTTLING ({0} reads, {1} x {2} in flight) ===" -f ` + $shape.TotalRequests, $shape.Pipelines, $shape.Concurrency) Write-Host ('{0,-28} {1,8} {2,8} {3,8} {4,10} {5,12} {6,10}' -f ` 'Arm', 'Wall', 'Ok', 'Failed', '429 retries', 'RetryDelay', 'RU/s') foreach ($k in 'paced', 'unpaced') { @@ -129,17 +195,31 @@ if ($Mode -eq 'both') { $label, $a.Seconds, $a.Ok, $a.Failed, $a.ThrottleRetries, ($a.RetryDelayMs / 1000), $a.RuPerSecond) } + # Admissibility, stated per arm rather than inferred from the wall times: what each arm + # actually drew against the rate this tenant was measured to keep serving at, and whether it + # was ever refused. A paced arm that is never refused is the outcome pacing exists to + # produce; an UNPACED arm that is never refused means the load never got there. + Write-Host '' + Write-Host ('=== ADMISSIBILITY (thresholds from {0}) ===' -f $threshold.Source) + foreach ($k in 'paced', 'unpaced') { + $a = $arms[$k] + Write-Host (' {0,-8} {1,8} RU at {2,7:F1} RU/s ({3,5:P0} of the {4} RU/s ceiling), {5} throttle retries - {6}' -f ` + $k, $a.ResourceUnits, $a.RuPerSecond, ($a.RuPerSecond / $threshold.Sustained), + $threshold.Sustained, $a.ThrottleRetries, + $(if ($a.ReachedRegime) { 'reached the throttled regime' } else { 'never refused' })) + } + # The verdict, and the conditions under which there is not one. An unpaced arm that was never - # refused did not reach the regime this benchmark exists to measure, and the wall times below + # refused did not reach the regime this benchmark exists to measure, and the wall times above # it are then a comparison of two unthrottled runs - which is benchmark 14's question, already # answered there. Saying so is the difference between no result and a wrong one. Write-Host '' - $conclusive = $arms.unpaced.ThrottleRetries -gt 0 + $conclusive = $arms.unpaced.ReachedRegime if (-not $conclusive) { Write-Host 'INCONCLUSIVE: the unpaced arm was never throttled, so there was nothing for pacing to avoid.' -ForegroundColor Yellow Write-Host (" It spent {0} RU at {1} RU/s; the tenant serves {2} RU/s while refusing and allows {3} RU of burst." -f ` $arms.unpaced.ResourceUnits, $arms.unpaced.RuPerSecond, $threshold.Sustained, $threshold.BurstRu) -ForegroundColor Yellow - Write-Host ' Raise -Count (spend further past the burst) or -Pipelines (offer a higher rate), then re-run.' -ForegroundColor Yellow + Write-Host ' Raise -Pipelines (offer a higher rate) and -Count with it, then re-run; -Plan checks a set first.' -ForegroundColor Yellow } else { $delta = $arms.unpaced.Seconds - $arms.paced.Seconds @@ -154,11 +234,16 @@ if ($Mode -eq 'both') { } } - Write-BenchResult -Benchmark '10-pacing-under-real-throttling' -Result ([pscustomobject]@{ + Write-BenchResult -Benchmark '10-pacing-under-real-throttling' ` + -Identity (Get-BenchEntryIdentity -Entries $armEntries) -Result ([pscustomobject]@{ Count = $Count Pipelines = $Pipelines Concurrency = $Concurrency Threshold = $threshold + # The load the arms were actually asked for and the arithmetic that said it could reach + # the regime, recorded beside the verdict: a comparison is only readable against the + # shape that produced it. + Shape = $shape Conclusive = $conclusive Arms = $arms }) @@ -168,15 +253,28 @@ if ($Mode -eq 'both') { # --- single arm --------------------------------------------------------------------------- +Show-LoadShape +Write-Host '' + Import-MgxLocal Connect-MgxBenchmark -# A bounded id pool, cycled: the tenant does not need 120,000 users for the workload to draw -# 120,000 units, and a fixed pre-materialized list keeps the work bounded with no loop that can -# run away. +# A bounded id pool, shared: the tenant does not need 60,000 users for the workload to draw +# 60,000 units, because each pipeline may issue the whole pool once. A fixed pre-materialized +# list keeps the work bounded with no loop that can run away. Write-Host "collecting up to $IdPool ids..." $pool = Get-BenchUserIds -Count $IdPool -Write-Host " got $($pool.Count); cycling to $Count requests" +Write-Host " got $($pool.Count) ids" + +# The pool can come back short - a tenant with fewer users than -IdPool, or fewer seeded ones - +# and the shape is then not the one printed above. Recompute it against what the tenant gave. +if ($pool.Count -lt $IdPool) { + $shape = Get-BenchPacingLoadShape -Count $Count -Pipelines $Pipelines -Concurrency $Concurrency ` + -IdPool $pool.Count -BurstRu $threshold.BurstRu -SustainedRuPerSecond $threshold.Sustained + Write-Host (" pool short of -IdPool $IdPool - the shape this arm will issue:") -ForegroundColor Yellow + Show-LoadShape + Write-Host '' +} # The token-bucket limiter would cap us well under the RU budget and we would never reach the # throttling regime this benchmark exists to measure. Lift it; the pacer (or its absence) is the @@ -190,9 +288,12 @@ if ($Mode -eq 'unpaced') { Write-Host ' adaptive pacing ON (2.1 default)' } -$script:slices = [System.Collections.Generic.List[object]]::new() -for ($i = 0; $i -lt $Pipelines; $i++) { $script:slices.Add([System.Collections.Generic.List[string]]::new()) } -for ($i = 0; $i -lt $Count; $i++) { $script:slices[$i % $Pipelines].Add($pool[$i % $pool.Count]) } +# Distinct ids per pipeline, and the slicer refuses rather than hand back a slice that repeats +# one: Invoke-MgxRequest deduplicates its pipeline ids, so a repeat is a request this arm would +# report and never send - which is how the published figures came to describe a regime the load +# never entered. +$script:slices = Get-BenchPipelineSlice -Pool $pool -Pipelines $Pipelines -IdsPerPipeline $shape.IdsPerPipeline +Write-Host (" $Pipelines pipelines x $($shape.IdsPerPipeline) distinct ids = $($shape.TotalRequests) requests") $script:commonPath = Join-Path $PSScriptRoot 'common.ps1' $script:armConcurrency = $Concurrency @@ -237,11 +338,20 @@ Write-Host (" pacing wait {0,8:F1}s over {1} activations" -f ($t.Adaptiv Write-Host (" resource units {0,8} ({1:F1} RU/s)" -f $t.ResourceUnitsConsumed, ($t.ResourceUnitsConsumed / $wallSeconds)) Write-Host (" pacing state {0}" -f $t.PacingState) +# The admissibility check for this arm on its own, in the same terms the plan stated: what it +# drew against the rate the tenant was measured to keep serving at, and whether it was refused. +$drawn = if ($wallSeconds -gt 0) { $t.ResourceUnitsConsumed / $wallSeconds } else { 0 } +Write-Host (" admissibility {0,8:F0} RU/s of the {1} RU/s ceiling, {2} refusal(s) - {3}" -f ` + $drawn, $threshold.Sustained, $t.ThrottleRetries, + $(if ($t.ThrottleRetries -gt 0) { 'reached the throttled regime' } else { 'never refused' })) +Write-Host (" issued {0,8} of {1} planned" -f $result.Output.ok, $shape.TotalRequests) + if ($Mode -eq 'unpaced' -and $t.ThrottleRetries -eq 0) { Write-Host '' - Write-Host (" NOTE: no refusal at {0:F0} RU/s over {1} units. This arm did not reach the throttled" -f ` - ($t.ResourceUnitsConsumed / $wallSeconds), $t.ResourceUnitsConsumed) -ForegroundColor Yellow - Write-Host (" regime, so it sets no baseline for the paced arm to beat.") -ForegroundColor Yellow + Write-Host (" NOTE: no refusal at {0:F0} RU/s over {1} units, against a {2} RU/s ceiling and {3} RU of" -f ` + $drawn, $t.ResourceUnitsConsumed, $threshold.Sustained, $threshold.BurstRu) -ForegroundColor Yellow + Write-Host (" burst. This arm did not reach the throttled regime, so it sets no baseline for the") -ForegroundColor Yellow + Write-Host (" paced arm to beat. Raise -Pipelines; -Plan checks a set without spending a run on it.") -ForegroundColor Yellow } # WallMs is read back by Write-BenchResult for the RU rate: telemetry's own elapsed total is the @@ -254,6 +364,10 @@ Write-BenchResult -Benchmark '10-pacing-under-real-throttling' -Result ([pscusto Count = $Count Pipelines = $Pipelines Concurrency = $Concurrency + # The shape this arm issued and the verdict on whether it reached the regime, so the entry + # is readable on its own rather than only through the summary the parent writes. + Shape = $shape + ReachedRegime = $t.ThrottleRetries -gt 0 WallMs = $result.ElapsedMs Ok = $result.Output.ok Failed = $result.Output.failed diff --git a/tests/benchmarks/13-resource-unit-rate.ps1 b/tests/benchmarks/13-resource-unit-rate.ps1 index 780b0c8..a8267ed 100644 --- a/tests/benchmarks/13-resource-unit-rate.ps1 +++ b/tests/benchmarks/13-resource-unit-rate.ps1 @@ -24,6 +24,21 @@ param( Import-MgxLocal Connect-MgxBenchmark +# What the tenant is DOCUMENTED to allow, read before the first round measures what it actually +# allows. Recorded per round beside the measured numbers: the burst allowance and the sustained +# ceiling only mean something against the budget the tier promises, and the tier is banded on +# the user count, so the same two numbers off two tenants are two different findings. +$tenant = Get-BenchTenantFact +$documented = $tenant.Budget +if ($documented) { + Write-Host ("tenant {0}: {1} users, documented tier {2} ({3}) = {4} RU/s ({5} RU / 10s)" -f ` + $tenant.TenantId, $documented.UserCount, $documented.Tier, $documented.Band, + $documented.RuPerSecond, $documented.RuPer10Seconds) +} +else { + Write-Host 'documented budget unknown (no user count) - rounds record it as null.' +} + # Enough ids to avoid hammering one object; requests cycle through them. $ids = Get-BenchUserIds -Count 5000 Write-Host "using $($ids.Count) user ids; $($Rates.Count) round(s) of ${Seconds}s" @@ -123,6 +138,10 @@ foreach ($rate in $Rates) { RuBeforeFirst429 = $okBeforeFirst429 # 1 RU each, so units == successes ServedRateAfter429 = $served RuHeaderSample = $ruHeaderSample + # The documented tier the two measured numbers above are to be read against. + DocumentedBudgetRuPerSecond = $documented?.RuPerSecond + DocumentedTier = $documented?.Tier + DocumentedTierBand = $documented?.Band } if ($null -eq $first429Seconds) { @@ -140,6 +159,10 @@ foreach ($rate in $Rates) { # --- Report ------------------------------------------------------------------------------- Write-Host '' Write-Host "=== RESOURCE-UNIT RATE (1-RU requests, no retry) ===" +if ($documented) { + Write-Host ("documented for this tenant: tier {0}, {1}, {2} RU/s" -f ` + $documented.Tier, $documented.Band, $documented.RuPerSecond) +} Write-Host ('{0,10} {1,10} {2,9} {3,11} {4,13} {5,15}' -f ` 'Target', 'Achieved', 'Sent', 'First 429', 'RU before it', 'Served after it') foreach ($k in $results.Keys) { diff --git a/tests/benchmarks/16-pathological-gauntlet.ps1 b/tests/benchmarks/16-pathological-gauntlet.ps1 new file mode 100644 index 0000000..14601e3 --- /dev/null +++ b/tests/benchmarks/16-pathological-gauntlet.ps1 @@ -0,0 +1,330 @@ +# Benchmark 16: the pathological class. No tenant required. +# +# 06 measures the mock's default schedule - a fault an entity meets once or twice and then +# never again, which is what ordinary Graph throttling looks like. This one measures the +# faults that do not go away by themselves. Each is posted to the mock as a fault plan +# (POST /plan) before its contenders run, and cleared after: +# +# storm every request 429s with Retry-After for -StormSeconds, then the storm ends +# outage every attempt 503s, for as long as the scenario lasts; the contender has to +# give up, and how long it takes to admit that is the measurement +# abort the collection's body is killed halfway through, connection dropped +# visibility a resource 404s for -VisibilitySeconds after it is first asked for and then +# appears - the read-after-create shape, where nothing in either stack retries +# a 404 and the caller is the one that has to poll +# +# The same four contenders as 06, in 06's order: a naive Invoke-RestMethod loop with no +# retry, the bare Graph SDK, the SDK with Enable-MgxResilience injected, and Mgx. Each runs +# in its own child pwsh under common.ps1's watchdog, which buys two things: a hang is killed +# and recorded as Hung instead of stalling the run (two of these scenarios exist to produce +# one), and "bare SDK" means a process Mgx was never loaded into, so its Telemetry is null +# because there is none rather than because it read zero. The server's /reset runs before +# every contender, so a storm or an outage lands on the same request ordinal for each of +# them, and its /stats delta is recorded per contender beside what the contender itself saw. +# +# The abort scenario asks for one large page rather than a long paged enumeration, because +# the mock speaks http and Mgx refuses to follow an @odata.nextLink that is not https +# (NextLinkValidator, an SSRF guard): a paged enumeration is a fault the Mgx arm would never +# reach, and a row reading "25 of 1000" against contenders that met the fault is worse than +# no row. One page every contender does read, cut in the middle, measures the same thing. +# +# Standalone: not in run.ps1, like 11-15. Next-link cycles are out of scope until the +# detection they exercise exists. +param( + [int] $N = 200, + [int] $Port = 8787, + # Storm and visibility are wall-clock windows, so they bound their scenario whatever the + # item count is. A window of 0 is not a short window, it is no window - the title would + # read "throttled for 0s" or "404s for 0s" and the plan would have nothing to arm. + [ValidateRange(1, [int]::MaxValue)] [int] $StormSeconds = 15, + [ValidateRange(1, [int]::MaxValue)] [int] $VisibilitySeconds = 8, + [int] $VisibilityBudgetSeconds = 40, + [int] $PollMs = 500, + # Under the outage every attempt fails and every failure costs a contender its whole + # retry budget, so that scenario is measured on a handful of items rather than on N. + [int] $OutageItems = 10, + # One page big enough that half of it is a body worth dying in the middle of. + [int] $PageSize = 1000, + [int] $StallSeconds = 120, + # child-mode plumbing (internal) + [ValidateSet('', 'storm', 'outage', 'abort', 'visibility')] [string] $Scenario = '', + [ValidateSet('', 'naive', 'sdk', 'sdkMgx', 'mgx')] [string] $Contender = '', + [string] $Base, + [string] $ResultFile, + [string] $HeartbeatFile +) + +. "$PSScriptRoot/common.ps1" + +# Named for the stack under the call, not for the call: the entity scenarios ask for one +# user at a time and the abort scenario enumerates a collection, and the row is read on +# which stack answered. +$labels = [ordered]@{ + naive = 'naive Invoke-RestMethod' + sdk = 'Graph SDK' + sdkMgx = 'Graph SDK + MgxResilience' + mgx = 'mgx' +} +# The id the visibility scenario reads. Nothing else asks for it, so the plan can govern +# that one path and leave every other route on the default schedule. +$visibilityId = '999' + +# ---------------- child mode: one contender against one scenario ---------------- +if ($Scenario -and $Contender) { + switch ($Contender) { + 'naive' { } # no Graph module at all: it is a REST loop + 'sdk' { Import-Module Microsoft.Graph.Users } + default { Import-MgxLocal } + } + function Beat { Set-Content -Path $HeartbeatFile -Value ([datetime]::UtcNow.Ticks) } + Beat + if ($Contender -ne 'naive') { Connect-BenchMockGraph -GraphEndpoint $Base } + if ($Contender -eq 'sdkMgx') { Enable-MgxResilience } + + # The heartbeat is touched every few items, and a failed item is an item: under the + # outage a contender makes steady progress without producing a single object, and a + # watchdog reading output alone would call that a hang. + $script:steps = 0 + function Step { $script:steps++; if ($script:steps % 5 -eq 0) { Beat } } + + # The record's id and category, not its message: the message carries this run's port and + # would differ between two runs of the same scenario. + function Get-ErrorFacts($record) { + if ($null -eq $record) { return $null } + [ordered]@{ + id = [string]$record.FullyQualifiedErrorId + category = [string]$record.CategoryInfo.Category + exception = $record.Exception.GetType().Name + inner = $(if ($record.Exception.InnerException) { $record.Exception.InnerException.GetType().Name } else { $null }) + } + } + + # One user at a time, the way each contender asks for one. + function Invoke-EntityWork([string[]] $Items) { + $ok = 0; $failed = 0; $first = $null + switch ($Contender) { + 'naive' { + foreach ($id in $Items) { + try { $null = Invoke-RestMethod "$Base/v1.0/users/$id" -TimeoutSec 30; $ok++ } + catch { $failed++; if (-not $first) { $first = $_ } } + Step + } + } + 'mgx' { + # 2>&1 rather than -ErrorVariable: the error records reach the pipeline as + # they happen, so a scenario where every item fails still beats. + $Items | Invoke-MgxRequest '/users/{id}' -ErrorAction Continue 2>&1 | ForEach-Object { + if ($_ -is [System.Management.Automation.ErrorRecord]) { + $failed++; if (-not $first) { $first = $_ } + } + else { $ok++ } + Step + } + } + default { + foreach ($id in $Items) { + $u = Get-MgUser -UserId "$id" -ErrorAction SilentlyContinue -ErrorVariable ev + if ($u) { $ok++ } else { $failed++; if (-not $first -and $ev) { $first = $ev[0] } } + Step + } + } + } + @{ ok = $ok; failed = $failed; error = (Get-ErrorFacts $first) } + } + + # The collection, page by page, until it ends or something cuts it off. + function Invoke-ListWork { + $ok = 0; $err = $null + switch ($Contender) { + 'naive' { + $uri = "$Base/v1.0/users" + try { + while ($uri) { + $page = Invoke-RestMethod $uri -TimeoutSec 30 + $ok += @($page.value).Count + Beat + $uri = $page.'@odata.nextLink' + } + } + catch { $err = $_ } + } + 'mgx' { + try { Invoke-MgxRequest '/users' -All -ErrorAction Stop | ForEach-Object { $ok++; Step } } + catch { $err = $_ } + } + default { + try { Get-MgUser -All -ErrorAction Stop | ForEach-Object { $ok++; Step } } + catch { $err = $_ } + } + } + @{ ok = $ok; failed = $(if ($err) { 1 } else { 0 }); error = (Get-ErrorFacts $err) } + } + + # Read-after-create. No layer in any of these stacks retries a 404, so what is measured + # is the caller's own polling: how many reads, and how long before the resource answers. + function Invoke-VisibilityWork { + $sw = [System.Diagnostics.Stopwatch]::StartNew() + $attempts = 0; $visibleMs = $null; $first = $null + while ($sw.Elapsed.TotalSeconds -lt $VisibilityBudgetSeconds) { + $attempts++ + $got = switch ($Contender) { + 'naive' { + try { $null = Invoke-RestMethod "$Base/v1.0/users/$visibilityId" -TimeoutSec 30; $true } + catch { if (-not $first) { $first = $_ }; $false } + } + 'mgx' { + $r = @($visibilityId | Invoke-MgxRequest '/users/{id}' -ErrorAction SilentlyContinue -ErrorVariable ev) + if (-not $first -and $ev) { $first = $ev[0] } + [bool]$r.Count + } + default { + $u = Get-MgUser -UserId $visibilityId -ErrorAction SilentlyContinue -ErrorVariable ev + if (-not $first -and $ev) { $first = $ev[0] } + [bool]$u + } + } + if ($got) { $visibleMs = $sw.ElapsedMilliseconds; break } + Start-Sleep -Milliseconds $PollMs + Beat + } + # $visibleMs is truthy-tested nowhere below: 0 is a legitimate elapsed reading (visible + # on the very first read) and PowerShell treats 0 as $false, which used to record a + # resource visible at 0ms the same as one that never became visible at all. + @{ ok = $(if ($null -ne $visibleMs) { 1 } else { 0 }); failed = $(if ($null -ne $visibleMs) { 0 } else { 1 }) + attempts = $attempts; visibleAfterMs = $visibleMs; error = (Get-ErrorFacts $first) } + } + + $ids = @(1..$N | ForEach-Object { "$_" }) + $outageIds = @($ids | Select-Object -First $OutageItems) + $pass = switch ($Scenario) { + 'storm' { Measure-BenchPass -Name $labels[$Contender] -Script { Invoke-EntityWork $ids } } + 'outage' { Measure-BenchPass -Name $labels[$Contender] -Script { Invoke-EntityWork $outageIds } } + 'abort' { Measure-BenchPass -Name $labels[$Contender] -Script { Invoke-ListWork } } + 'visibility' { Measure-BenchPass -Name $labels[$Contender] -Script { Invoke-VisibilityWork } } + } + if ($Contender -eq 'sdkMgx') { Disable-MgxResilience } + Disconnect-MgGraph -ErrorAction SilentlyContinue | Out-Null + $pass | ConvertTo-Json -Depth 8 | Set-Content $ResultFile + exit 0 +} + +# ---------------- parent mode ---------------- +Import-MgxLocal + +$server = Start-BenchMockServer -Port $Port +$base = $server.BaseUrl +$tmp = [System.IO.Path]::GetTempPath() +try { + function Set-Plan([hashtable] $Document, [string] $Label) { + $json = ConvertTo-Json -InputObject $Document -Depth 6 -Compress + $posted = @($Document.rules).Count + $response = Invoke-RestMethod "$base/plan" -Method Post -Body $json -ContentType 'application/json' + # The server answers with the rule count it actually armed. A mismatch means this + # scenario is about to run against a plan that is not the one just posted - worth + # stopping for, since every row after it would print as a result against no fault. + if ($response.rules -ne $posted) { + throw "plan for '$Label' posted $posted rule(s) but the server reports $($response.rules)" + } + } + function Reset-Server { $null = Invoke-RestMethod "$base/reset" } + function Get-ServerStats { Invoke-RestMethod "$base/stats" } + # What this contender cost the server, not what the server has counted since it started. + function Get-StatsDelta($Before, $After) { + $delta = [ordered]@{} + foreach ($property in $After.PSObject.Properties) { + $was = $Before.PSObject.Properties[$property.Name] + $delta[$property.Name] = $property.Value - $(if ($was) { $was.Value } else { 0 }) + } + [pscustomobject]$delta + } + function Show-Row([string] $Id, $Row) { + $o = $Row.Output + $telemetry = if ($Row.MgxTelemetry) { + 'req {0}, retries {1}' -f $Row.MgxTelemetry.Requests, + ($Row.MgxTelemetry.ThrottleRetries + $Row.MgxTelemetry.OtherRetries) + } else { 'no telemetry' } + # A crashed contender's ElapsedMs is watchdog poll noise (up to 10s), not a + # measurement - it never got the chance to be one - so the wall-time slot names the + # exit instead of printing a number nobody asked for. + $wall = if ($Row.Crashed) { "EXITED $($Row.ExitCode) without a result" } else { '{0:F1}s' -f ($Row.ElapsedMs / 1000) } + $note = if ($Row.Hung) { 'HUNG (killed by watchdog)' } + elseif ($null -ne $o.visibleAfterMs) { 'visible after {0:F1}s in {1} reads' -f ($o.visibleAfterMs / 1000), $o.attempts } + elseif ($null -ne $o.attempts) { 'never visible in {0} reads' -f $o.attempts } + elseif ($o.error) { '{0} / {1}' -f $o.error.exception, $o.error.id } + else { '' } + Write-Host ("{0,-10} {1,-26} completed {2,5} / failed {3,5} wall {4,8} {5,-20} {6}" -f ` + $Id, $Row.Name, $o.ok, $o.failed, $wall, $telemetry, $note) + } + + $scenarios = [ordered]@{ + storm = @{ + Title = "429 storm: every request throttled for ${StormSeconds}s, Retry-After 2, then clean ($N ids)" + Plan = @{ rules = @(@{ route = '^/(v1\.0|beta)/users'; kind = 'throttleStorm' + scope = 'server'; seconds = $StormSeconds; retryAfter = 2 }) } + } + outage = @{ + Title = "never-recovering 5xx: every attempt 503, no window ($OutageItems ids)" + Plan = @{ rules = @(@{ route = '^/(v1\.0|beta)/users'; kind = 'outage'; retryAfter = 1 }) } + } + abort = @{ + Title = "connection death mid-body: one page of $PageSize users, killed halfway through the body" + Plan = @{ pages = 1; pageSize = $PageSize + rules = @(@{ route = '^/(v1\.0|beta)/users$'; kind = 'abort' + from = 1; count = 1 }) } + } + visibility = @{ + Title = "delayed visibility: /users/$visibilityId 404s for ${VisibilitySeconds}s, then appears" + Plan = @{ rules = @(@{ route = ('^/(v1\.0|beta)/users/{0}$' -f $visibilityId) + kind = 'delayedVisibility'; seconds = $VisibilitySeconds }) } + } + } + + $results = [ordered]@{} + foreach ($id in $scenarios.Keys) { + $definition = $scenarios[$id] + Write-Host '' + Write-Host ("### {0}: {1}" -f $id, $definition.Title) + Set-Plan $definition.Plan $id + $results[$id] = [ordered]@{} + foreach ($arm in $labels.Keys) { + Reset-Server + $before = Get-ServerStats + $row = Invoke-WatchdoggedContender -Name $labels[$arm] -ScriptPath $PSCommandPath ` + -ArgumentList @( + '-Scenario', $id, '-Contender', $arm, '-Base', $base, + '-N', $N, '-OutageItems', $OutageItems, + '-VisibilityBudgetSeconds', $VisibilityBudgetSeconds, '-PollMs', $PollMs, + '-ResultFile', (Join-Path $tmp "bench16-$id-$arm.json"), + '-HeartbeatFile', (Join-Path $tmp "bench16-$id-$arm.beat")) ` + -ResultFile (Join-Path $tmp "bench16-$id-$arm.json") ` + -HeartbeatFile (Join-Path $tmp "bench16-$id-$arm.beat") ` + -StallSeconds $StallSeconds + $row | Add-Member -NotePropertyName ServerStats -NotePropertyValue (Get-StatsDelta $before (Get-ServerStats)) -Force + $results[$id][$arm] = $row + Show-Row $id $row + } + # Cleared between scenarios so the next one's plan is the only thing in force. + Set-Plan @{ rules = @() } "$id (clear)" + } + + Write-Host '' + Write-Host ("=== PATHOLOGICAL GAUNTLET (N={0}) ===" -f $N) + foreach ($id in $results.Keys) { + foreach ($arm in $results[$id].Keys) { Show-Row $id $results[$id][$arm] } + } + Write-Host '' + Write-Host 'server-side truth: the /stats delta each contender left behind' + foreach ($id in $results.Keys) { + foreach ($arm in $results[$id].Keys) { + $s = $results[$id][$arm].ServerStats + Write-Host ("{0,-10} {1,-26} served {2,5} pages {3,4} 429 {4,4} 503 {5,4} abort {6,3} 404 {7,4} errors {8,3}" -f ` + $id, $labels[$arm], $s.served, $s.pagesServed, $s.throttleStorm, $s.outage, + $s.abort, $s.delayedVisibility, $s.serverErrors) + } + } + Write-BenchResult -Benchmark '16-pathological-gauntlet' -Result $results +} +finally { + Remove-BenchGraphEnvironment + Stop-BenchMockServer $server +} diff --git a/tests/benchmarks/17-pathological-environment.ps1 b/tests/benchmarks/17-pathological-environment.ps1 new file mode 100644 index 0000000..490ec90 --- /dev/null +++ b/tests/benchmarks/17-pathological-environment.ps1 @@ -0,0 +1,336 @@ +# Benchmark 17: the pathological faults that act on the session rather than on the socket. +# No tenant required. +# +# 16's faults are properties of the server. These two are properties of the process the +# operation is running in, and they arrive while it is running: +# +# auth a second runspace calls Connect-MgGraph again, with a different access token +# and a different tenant id, while a read is in flight. GraphSession.Instance is +# a process-wide singleton, so the runspace boundary does not protect anything: +# what the operation holds is replaced under it. +# modules a second runspace imports Az.Accounts and then PnP.PowerShell while a read is +# in flight. Module load and assembly resolution are irreversible in a process, +# and Az.Accounts brings its own System.Text.Json. +# +# This records what happens today. Recovering from a session replaced mid-operation is 2.3.0 +# work, so there is no pass or fail here and nothing is asserted: the rows are the before +# picture that work will be measured against. Every row says "recorded". +# +# Contenders: mgx ($ids | Invoke-MgxRequest '/users/{id}', one cmdlet invocation that spans +# the perturbation) and the bare Graph SDK (Get-MgUser per id, a sequence of invocations). +# The naive Invoke-RestMethod loop is not a contender here: it holds no session and imports +# nothing, so neither perturbation has anything to act on. +# +# The read is a paced sequence of entity reads rather than a paged enumeration, for the same +# reason 16's abort scenario is one page: the mock speaks http, and Mgx refuses an +# @odata.nextLink that is not https, so a paged enumeration is not a shape both contenders +# can be measured on. What is needed here is an operation that is still running when the +# perturbation lands, and 200 reads paced at -PaceMs give that for both. +# +# The timeline is the consumer's: an item counts as received when the pipeline handed it +# over, which for the fan-out can lag its own HTTP by the width of the fan. +# +# Standalone: not in run.ps1. +param( + [int] $N = 200, + [int] $Port = 8787, + # 200 reads at 100ms is a twenty-second operation, which leaves room for a perturbation + # to land in the middle of it and for the read to carry on afterwards. + [int] $PaceMs = 100, + # Counted from the read's first item, not from the start of the child. + [int] $PerturbAfterMs = 5000, + [int] $StallSeconds = 180, + # child-mode plumbing (internal) + [ValidateSet('', 'auth', 'modules')] [string] $Scenario = '', + [ValidateSet('', 'sdk', 'mgx')] [string] $Contender = '', + [string] $Base, + [string] $ResultFile, + [string] $HeartbeatFile +) + +. "$PSScriptRoot/common.ps1" + +$labels = [ordered]@{ + sdk = 'Graph SDK' + mgx = 'mgx' +} + +# ---------------- child mode: one contender under one perturbation ---------------- +if ($Scenario -and $Contender) { + if ($Contender -eq 'sdk') { Import-Module Microsoft.Graph.Users } else { Import-MgxLocal } + function Beat { Set-Content -Path $HeartbeatFile -Value ([datetime]::UtcNow.Ticks) } + Beat + Connect-BenchMockGraph -GraphEndpoint $Base + + $script:steps = 0 + function Step { $script:steps++; if ($script:steps % 10 -eq 0) { Beat } } + + function Get-ErrorFacts($record) { + if ($null -eq $record) { return $null } + [ordered]@{ + id = [string]$record.FullyQualifiedErrorId + category = [string]$record.CategoryInfo.Category + exception = $record.Exception.GetType().Name + inner = $(if ($record.Exception.InnerException) { $record.Exception.InnerException.GetType().Name } else { $null }) + } + } + + function Get-SessionFacts { + $context = Get-MgContext + [ordered]@{ + tenantId = [string]$context.TenantId + authType = [string]$context.AuthType + scopes = @($context.Scopes).Count + } + } + + # Every id, one read at a time, paced so the operation is still running when the + # perturbation lands. Each item is stamped as the pipeline hands it over, which is what + # makes "before, during and after" answerable afterwards. + function Invoke-PacedRead { + $received = [System.Collections.Generic.List[object]]::new() + $first = $null + $startedAt = [datetime]::UtcNow + switch ($Contender) { + 'mgx' { + $ids | Invoke-MgxRequest '/users/{id}' -ErrorAction Continue 2>&1 | ForEach-Object { + if ($_ -is [System.Management.Automation.ErrorRecord]) { + if (-not $first) { $first = $_ } + } + else { + $received.Add([pscustomobject]@{ id = [string]$_.id; at = [datetime]::UtcNow }) + if (-not $sync.ReadStarted) { $sync.ReadStarted = [datetime]::UtcNow } + } + Step + Start-Sleep -Milliseconds $PaceMs + } + } + default { + foreach ($id in $ids) { + $user = Get-MgUser -UserId $id -ErrorAction SilentlyContinue -ErrorVariable ev + if ($user) { + $received.Add([pscustomobject]@{ id = [string]$user.Id; at = [datetime]::UtcNow }) + if (-not $sync.ReadStarted) { $sync.ReadStarted = [datetime]::UtcNow } + } + elseif (-not $first -and $ev) { $first = $ev[0] } + Step + Start-Sleep -Milliseconds $PaceMs + } + } + } + @{ startedAt = $startedAt; endedAt = [datetime]::UtcNow; received = $received + error = (Get-ErrorFacts $first) } + } + + # The perturbation runs in a second runspace of THIS process, because that is what makes + # it a perturbation: a child process would share neither the session singleton nor the + # assemblies. It waits for the read's first item and then for -PerturbAfterMs, so it + # lands the same distance into the read whichever contender is running - the two do not + # take the same time to hand over their first item, and a delay counted from anywhere + # else lands before the mgx arm has read anything. + $perturbations = @{ + auth = @' +param($DelayMs, $Token, $TenantId, $AuthModule) +$log = [ordered]@{ action = 'Connect-MgGraph again, different access token and tenant'; steps = @() } +while (-not $sync.ReadStarted) { Start-Sleep -Milliseconds 50 } +Start-Sleep -Milliseconds $DelayMs +$log.startedAt = [datetime]::UtcNow +try { + # By path, not by name. A new runspace has no modules, and the name resolves to the + # newest Microsoft.Graph.Authentication installed - which is not necessarily the one + # this process already loaded, and loading a second version of it throws instead of + # perturbing anything. The path is the one the read is running against. + Import-Module $AuthModule -ErrorAction Stop + Connect-MgGraph -Environment MgxBench -NoWelcome -ErrorAction Stop ` + -AccessToken (ConvertTo-SecureString $Token -AsPlainText -Force) + [Microsoft.Graph.PowerShell.Authentication.GraphSession]::Instance.AuthContext.TenantId = $TenantId + $log.steps += "connected as $TenantId" +} +catch { $log.steps += "connect failed: $($_.Exception.Message)" } +$log.finishedAt = [datetime]::UtcNow +[pscustomobject]$log +'@ + modules = @' +param($DelayMs) +$log = [ordered]@{ action = 'Import-Module Az.Accounts, then PnP.PowerShell'; steps = @() } +while (-not $sync.ReadStarted) { Start-Sleep -Milliseconds 50 } +Start-Sleep -Milliseconds $DelayMs +$log.startedAt = [datetime]::UtcNow +foreach ($module in 'Az.Accounts', 'PnP.PowerShell') { + try { + Import-Module $module -ErrorAction Stop + $log.steps += "$module imported at $([datetime]::UtcNow.ToString('HH:mm:ss.fff'))" + } + catch { $log.steps += "$module failed: $($_.Exception.Message)" } +} +$log.finishedAt = [datetime]::UtcNow +[pscustomobject]$log +'@ + } + + $ids = @(1..$N | ForEach-Object { "$_" }) + $sessionBefore = Get-SessionFacts + + # Shared across both runspaces of this process: the read stamps it when its first item + # arrives, and the perturbation blocks on it. + $sync = [hashtable]::Synchronized(@{ ReadStarted = $null }) + $runspace = [runspacefactory]::CreateRunspace() + $runspace.Open() + $runspace.SessionStateProxy.SetVariable('sync', $sync) + $shell = [powershell]::Create() + $shell.Runspace = $runspace + $null = $shell.AddScript($perturbations[$Scenario]) + $null = if ($Scenario -eq 'auth') { + $shell.AddParameters(@{ DelayMs = $PerturbAfterMs + Token = 'replacement-token-not-validated' + TenantId = 'gauntlet-replacement-tenant' + AuthModule = (Get-Module Microsoft.Graph.Authentication | + Select-Object -First 1).Path }) + } else { + $shell.AddParameters(@{ DelayMs = $PerturbAfterMs }) + } + $handle = $shell.BeginInvoke() + + $pass = Measure-BenchPass -Name $labels[$Contender] -Script { Invoke-PacedRead } + + $perturbation = $null + try { $perturbation = @($shell.EndInvoke($handle))[0] } + catch { $perturbation = [pscustomobject]@{ action = 'perturbation runspace'; steps = @("threw: $($_.Exception.Message)") } } + $runspaceErrors = @($shell.Streams.Error | + ForEach-Object { '{0}: {1}' -f $_.CategoryInfo.Category, $_.Exception.Message } | + Where-Object { $_ -notmatch '^\w+: *$' }) + if ($runspaceErrors.Count) { + $perturbation | Add-Member -NotePropertyName runspaceErrors -NotePropertyValue $runspaceErrors -Force + } + $shell.Dispose(); $runspace.Close(); $runspace.Dispose() + + # What the operation got, against what the mock holds. The ids are the mock's own, so a + # loss and a duplicate are both countable without a tenant. + $out = $pass.Output + $received = @($out.received) + $distinct = @($received.id | Sort-Object -Unique) + $windowStart = $perturbation.startedAt + $windowEnd = $perturbation.finishedAt + $summary = [ordered]@{ + items = $received.Count + distinctItems = $distinct.Count + duplicates = $received.Count - $distinct.Count + lost = $N - $distinct.Count + itemsBefore = @($received | Where-Object { $windowStart -and $_.at -lt $windowStart }).Count + itemsDuring = @($received | Where-Object { $windowStart -and $windowEnd -and $_.at -ge $windowStart -and $_.at -le $windowEnd }).Count + itemsAfter = @($received | Where-Object { $windowEnd -and $_.at -gt $windowEnd }).Count + firstId = $(if ($received.Count) { $received[0].id } else { $null }) + lastId = $(if ($received.Count) { $received[-1].id } else { $null }) + readStartedAt = $out.startedAt.ToString('o') + readEndedAt = $out.endedAt.ToString('o') + error = $out.error + sessionBefore = $sessionBefore + sessionAfter = (Get-SessionFacts) + perturbation = $perturbation + # The proof that this row measured anything: the perturbation has to have started + # after the read did and before it ended. + perturbedInFlight = [bool]($windowStart -and $windowStart -ge $out.startedAt -and $windowStart -le $out.endedAt) + } + # The per-item list is what the summary was derived from; it is not what anyone reads. + $pass.Output = $summary + + # No teardown: the process exits here, and the parent takes the Graph environment out. + # A session this scenario deliberately replaced is not worth disconnecting cleanly, and + # a throw on the way out would lose the row. + $pass | ConvertTo-Json -Depth 8 | Set-Content $ResultFile + exit 0 +} + +# ---------------- parent mode ---------------- +Import-MgxLocal + +$server = Start-BenchMockServer -Port $Port +$base = $server.BaseUrl +$tmp = [System.IO.Path]::GetTempPath() +try { + function Show-Record([string] $Id, $Row) { + $o = $Row.Output + if ($Row.Hung) { + Write-Host ("{0,-8} {1,-11} HUNG (killed by watchdog) after {2:F1}s" -f $Id, $Row.Name, ($Row.ElapsedMs / 1000)) + return + } + if ($Row.Crashed) { + # ElapsedMs here is watchdog poll noise, not a measurement, so it is left out + # rather than printed as though it were a wall time. + Write-Host ("{0,-8} {1,-11} EXITED {2} without a result" -f $Id, $Row.Name, $Row.ExitCode) + return + } + $telemetry = if ($Row.MgxTelemetry) { + 'req {0}, retries {1}' -f $Row.MgxTelemetry.Requests, + ($Row.MgxTelemetry.ThrottleRetries + $Row.MgxTelemetry.OtherRetries) + } else { 'no telemetry' } + Write-Host ("{0,-8} {1,-11} recorded {2,4} items ({3} before / {4} during / {5} after the perturbation), {6} duplicated, {7} lost, wall {8:F1}s, {9}" -f ` + $Id, $Row.Name, $o.items, $o.itemsBefore, $o.itemsDuring, $o.itemsAfter, + $o.duplicates, $o.lost, ($Row.ElapsedMs / 1000), $telemetry) + Write-Host ("{0,-8} {1,-11} perturbation landed in flight: {2} - {3}" -f ` + $Id, $Row.Name, $o.perturbedInFlight, $o.perturbation.action) + foreach ($step in @($o.perturbation.steps)) { + Write-Host ("{0,-8} {1,-11} step: {2}" -f $Id, $Row.Name, $step) + } + foreach ($runspaceError in @($o.perturbation.runspaceErrors | Where-Object { $_ })) { + Write-Host ("{0,-8} {1,-11} runspace error: {2}" -f $Id, $Row.Name, $runspaceError) + } + $surfaced = if ($o.error) { '{0} / {1}' -f $o.error.exception, $o.error.id } else { 'none' } + Write-Host ("{0,-8} {1,-11} error surfaced: {2}; session tenant {3} -> {4}" -f ` + $Id, $Row.Name, $surfaced, $o.sessionBefore.tenantId, $o.sessionAfter.tenantId) + } + + # A rule whose window never opens governs the entity route without ever firing, which + # takes the route off the mock's default 429/503 schedule: the perturbation is then the + # only thing that can go wrong, and a lost item is the perturbation's doing. + $plan = @{ rules = @(@{ route = '^/(v1\.0|beta)/users/'; kind = 'outage'; from = 1000000 }) } + $planResponse = Invoke-RestMethod "$base/plan" -Method Post -ContentType 'application/json' ` + -Body (ConvertTo-Json -InputObject $plan -Depth 6 -Compress) + # The server answers with the rule count it actually armed; a mismatch means the entity + # route is not off the default schedule the way this whole file assumes it is. + $postedRules = @($plan.rules).Count + if ($planResponse.rules -ne $postedRules) { + throw "plan setup posted $postedRules rule(s) but the server reports $($planResponse.rules)" + } + + Write-Host '' + Write-Host 'The naive Invoke-RestMethod loop is not a contender here: it holds no session and' + Write-Host 'imports nothing, so neither perturbation has anything of its to act on.' + + $scenarios = [ordered]@{ + auth = 'a second runspace connects again, with a different token and tenant, mid-read' + modules = 'a second runspace imports Az.Accounts and PnP.PowerShell mid-read' + } + + $results = [ordered]@{} + foreach ($id in $scenarios.Keys) { + Write-Host '' + Write-Host ("### {0}: {1}" -f $id, $scenarios[$id]) + $results[$id] = [ordered]@{} + foreach ($arm in $labels.Keys) { + $null = Invoke-RestMethod "$base/reset" + $row = Invoke-WatchdoggedContender -Name $labels[$arm] -ScriptPath $PSCommandPath ` + -ArgumentList @( + '-Scenario', $id, '-Contender', $arm, '-Base', $base, + '-N', $N, '-PaceMs', $PaceMs, '-PerturbAfterMs', $PerturbAfterMs, + '-ResultFile', (Join-Path $tmp "bench17-$id-$arm.json"), + '-HeartbeatFile', (Join-Path $tmp "bench17-$id-$arm.beat")) ` + -ResultFile (Join-Path $tmp "bench17-$id-$arm.json") ` + -HeartbeatFile (Join-Path $tmp "bench17-$id-$arm.beat") ` + -StallSeconds $StallSeconds + $results[$id][$arm] = $row + Show-Record $id $row + } + } + + Write-Host '' + Write-Host ("=== PATHOLOGICAL ENVIRONMENT (N={0}, paced {1}ms, perturbed at {2}ms) ===" -f $N, $PaceMs, $PerturbAfterMs) + foreach ($id in $results.Keys) { + foreach ($arm in $results[$id].Keys) { Show-Record $id $results[$id][$arm] } + } + Write-BenchResult -Benchmark '17-pathological-environment' -Result $results +} +finally { + Remove-BenchGraphEnvironment + Stop-BenchMockServer $server +} diff --git a/tests/benchmarks/18-spo-latency-clamp.ps1 b/tests/benchmarks/18-spo-latency-clamp.ps1 new file mode 100644 index 0000000..795ca39 --- /dev/null +++ b/tests/benchmarks/18-spo-latency-clamp.ps1 @@ -0,0 +1,427 @@ +# Benchmark 18: does SPO clamp a drive workload by latency instead of by 429? +# +# After heavy traffic a SharePoint-backed drive has been seen to serve the same GET five times +# slower or worse for about an hour, with no 429 anywhere in the run and no throttle header on +# any response. Nothing in the pacer reacts to that: the AIMD cap is armed by a 429, the +# proximity damping by x-ms-throttle-limit-percentage, and a clamp shows neither. The pacer +# already keeps a slow EMA of drive latency and reports the ratio in Get-MgxTelemetry's +# PacingState - telemetry only, deliberately not a pacing input (AdaptiveRequestPacer +# RecordLatency). Whether it should become one is a measurement, and this is the measurement. +# +# So the recorded metric is NOT throughput. It is the latency distribution over the window, +# per five-minute bucket, beside the throttle signals present or absent on each call: the +# resource unit, the throttle-limit percentage, the throttle scope, Retry-After, and whether +# any RateLimit-* header appeared at all. A clamp is a run whose late buckets sit several times +# the early ones on p90 while the throttle-signal columns stay empty. The absence of the +# signals is half the finding, which is why every call records them as absent rather than the +# summary reporting only the ones that showed up. +# +# What is recorded per call, and from where. Two figures, never conflated: +# - ElapsedMs is the HTTP round trip, read from the request tracer's own response line +# ("[Mgx] Response: 200 OK in 41 ms"). This is the latency the clamp is about. +# - CallElapsedMs is the whole Invoke-MgxRequest call, which includes the pacer's proactive +# wait and any Retry-After sleep. PacingWaitMs, from the pacer's verbose line, says how +# much of the gap is mgx's own. Reading a paced call's wall time as SPO latency would +# manufacture a clamp out of the client's own back-off. +# The headers come from the same trace: the tracer keeps Retry-After, x-ms-resource-unit and +# every RateLimit-* and x-ms-throttle-* header, which is exactly the set this measurement +# needs. It is on when $DebugPreference is Continue, which the workers set - the -Debug switch +# itself asks ShouldContinue and would stall a headless run. No raw HttpClient probe is needed +# and none is taken: the subject is mgx under pacing, not a bare socket, and a parallel probe +# would be a different request on a different connection answered under a different queue. +# Get-MgxTelemetry supplies the run-level counters (resource units, throttle retries, pacing +# wait, PacingState with the drive bucket's latency ratio), sampled once per bucket - those are +# process-wide totals, so they describe the window and never one call. +# +# Read-only throughout: GET only, against the drive the operator names. There is no default +# drive, because a default is how a benchmark ends up pointed at production. The load itself is +# heavy by design - $Concurrency workers issuing back-to-back single-item GETs for $Minutes - +# and heavy read traffic against a drive is what the clamp is reported to follow, so it draws +# on the tenant's budget and may leave the drive slow for other callers for the following hour. +# That is the point of the experiment and the reason it wants an uncontended machine, a window +# longer than the hour the clamp is said to last, and a tenant nobody is working in. +# +# Rows checkpoint to results/18-spo-latency-clamp.rows.jsonl as they arrive, flushed per line, +# so a run killed at minute 50 leaves the fifty minutes it collected. The summary is written +# only by a completed window: a partial summary in the results log would be promoted as a +# baseline by the next -RecordBaseline, and half a window is not a distribution. +# +# The short smoke, which must run end to end before a long run is worth starting: +# ./18-spo-latency-clamp.ps1 -DriveId -Minutes 2 -Concurrency 2 ` +# -PrecedingActivity 'smoke, drive idle' +# It writes rows and a summary like the long form, on the same paths. +param( + # The drive to measure. One of the two, and no default: -DriveId b!xxxx, or the same drive + # as a URI in the /drives/ shape. Anything else is refused rather than guessed at. + [string] $DriveId, + [string] $DriveUri, + # Longer than the hour the clamp is reported to hold, so a default run outlasts it. + [int] $Minutes = 75, + [int] $Concurrency = 8, + [int] $BucketMinutes = 5, + # How many item ids to cycle through. One item repeatedly would measure that item's cache. + [int] $ItemPoolSize = 200, + # What the drive had just been through, in the operator's words ("2h export at c=64", + # "idle overnight"). The distribution means nothing without it, so it is recorded verbatim. + [string] $PrecedingActivity = '' +) + +. "$PSScriptRoot/common.ps1" + +# Get-ClampPercentile and the rest of the summary - buckets, totals, refusals-by-kind, the +# headline - live in common.ps1, dot-sourced above: they read the checkpointed rows file back +# rather than trust a live accumulator, and a Pester test calls the identical functions over a +# synthetic rows.jsonl. See the comment above Get-ClampSummary in common.ps1. + +# --- Preconditions: refuse, never measure nothing quietly ----------------------------------- + +if ($Minutes -le 0) { throw "18-spo-latency-clamp: -Minutes is $Minutes; a window has to be at least a minute long." } +if ($Concurrency -le 0) { throw "18-spo-latency-clamp: -Concurrency is $Concurrency; a paced load needs at least one worker." } +if ($BucketMinutes -le 0) { throw "18-spo-latency-clamp: -BucketMinutes is $BucketMinutes; a bucket has to span at least a minute." } +if ($ItemPoolSize -le 0) { throw "18-spo-latency-clamp: -ItemPoolSize is $ItemPoolSize; the load needs at least one item to read." } + +$drivePath = $null +if ($DriveId) { + if ($DriveId -notmatch '^[^/?#\s]+$') { + throw "18-spo-latency-clamp: -DriveId '$DriveId' is not a drive id. Pass the id alone, or pass the whole thing as -DriveUri /drives/." + } + $drivePath = "/drives/$DriveId" +} +if ($DriveUri) { + if ($DriveUri -notmatch '^/drives/[^/?#\s]+$') { + throw "18-spo-latency-clamp: -DriveUri '$DriveUri' is not in the /drives/ shape." + } + if ($drivePath -and $drivePath -ne $DriveUri) { + throw "18-spo-latency-clamp: -DriveId and -DriveUri name different drives ('$drivePath' and '$DriveUri'). Pass one." + } + $drivePath = $DriveUri +} +if (-not $drivePath) { + throw '18-spo-latency-clamp: name the drive to measure - -DriveId or -DriveUri /drives/. There is no default: a default drive is how a read benchmark ends up pointed at production.' +} + +Import-MgxLocal +Connect-MgxBenchmark + +# One probe GET. Every reason this run cannot happen is named here, before anything is +# measured: a drive that does not answer, a tenant with no SharePoint at all, a session whose +# token carries no Files/Sites read. +$probe = $null +$probeSw = [Diagnostics.Stopwatch]::StartNew() +try { + $probe = Invoke-MgxRequest -Uri $drivePath -Property id, driveType, name -ErrorAction Stop +} +catch { + $ex = $_.Exception + $status = 0 + if ($null -ne $ex.StatusCode) { try { $status = [int]$ex.StatusCode } catch { $status = 0 } } + $text = "$($ex.Message)" + + # The documented shape of a tenant with no SharePoint: every sites/drives call answers + # BadRequest, and only the message says why. No permission grant changes it. + if ($text -match 'SPO license') { + throw "18-spo-latency-clamp: the tenant reports no SPO license, so $drivePath cannot be measured on it - '$text'. This benchmark needs a tenant with SharePoint; no role grant substitutes for the license." + } + if ($status -eq 401 -or $status -eq 403) { + throw "18-spo-latency-clamp: the session cannot read $drivePath (HTTP $status - '$text'). Connect with Files.Read.All or Sites.Read.All and try again." + } + throw "18-spo-latency-clamp: $drivePath did not answer a probe GET$(if ($status) { " (HTTP $status)" }) - '$text'. Nothing was measured." +} +$probeSw.Stop() +if (-not $probe -or -not $probe.id) { + throw "18-spo-latency-clamp: $drivePath answered a probe GET without a drive id, so there is no drive there to measure." +} +Write-Host ("drive {0} ({1}) answered a probe in {2} ms" -f $probe.id, ($probe.driveType ?? 'unknown type'), $probeSw.ElapsedMilliseconds) + +# The item pool. One item read over and over measures a cache; a pool the size of a page does +# not. A drive with no children at root is still measurable - /root is a drive item too - so +# an empty listing narrows the load rather than refusing it. +$items = @() +try { + $items = @(Invoke-MgxRequest -Uri "$drivePath/root/children" -Property id -Top $ItemPoolSize -ErrorAction Stop | + ForEach-Object { $_.id } | Where-Object { $_ }) +} +catch { + Write-Warning "listing $drivePath/root/children failed ('$($_.Exception.Message)') - the load falls back to /root." +} +$targets = if ($items.Count -gt 0) { + @($items | Select-Object -First $ItemPoolSize | ForEach-Object { [pscustomobject]@{ Label = 'item'; Uri = "$drivePath/items/$_" } }) +} else { + @([pscustomobject]@{ Label = 'root'; Uri = "$drivePath/root" }) +} +Write-Host ("load reads {0} target(s) in rotation, {1} worker(s), for {2} minute(s)" -f $targets.Count, $Concurrency, $Minutes) +if ($PrecedingActivity) { Write-Host "preceding activity, as stated: $PrecedingActivity" } +else { Write-Warning 'no -PrecedingActivity given; the distribution is uninterpretable without knowing what the drive had just been through.' } + +# --- The load ------------------------------------------------------------------------------- + +$resultsDir = Join-Path $PSScriptRoot 'results' +if (-not (Test-Path $resultsDir)) { New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null } +$rowsPath = Join-Path $resultsDir '18-spo-latency-clamp.rows.jsonl' + +$modulePath = Join-Path $PSScriptRoot '../../module/mgx.psd1' +if (-not (Test-Path $modulePath)) { $modulePath = 'Mgx' } +$startedAt = Get-Date +$deadline = [datetime]::UtcNow.AddMinutes($Minutes) + +# A rolling figure for the live progress line only - the real summary is read back from +# $rowsPath by Get-ClampSummary once the load below finishes, never from this list. +$liveLatencies = [System.Collections.Generic.List[double]]::new() +$attempts = 0 +$pacingSamples = @() +$lastBucket = -1 + +# AutoFlush, append: the file is the checkpoint. A run that dies at minute 50 has minute 50 on +# disk, not a buffer that never reached it. +$writer = [System.IO.StreamWriter]::new($rowsPath, $true, [System.Text.UTF8Encoding]::new($false)) +$writer.AutoFlush = $true +$loadSw = [Diagnostics.Stopwatch]::StartNew() +try { + 1..$Concurrency | ForEach-Object -ThrottleLimit $Concurrency -Parallel { + $worker = $_ + $deadline = $using:deadline + $targets = $using:targets + $modulePath = $using:modulePath + + # Staggered: eight runspaces importing the same binary module in the same millisecond is + # a race nobody needs, and the ramp is seconds against a window of an hour or more. + Start-Sleep -Milliseconds (($worker - 1) * 400) + $ErrorActionPreference = 'Stop' + Import-Module $modulePath -ErrorAction Stop + # The request tracer, which is where the response headers come from, is on whenever + # DebugPreference is not silent. The -Debug switch asks ShouldContinue on a compiled + # cmdlet and would block a headless run; the preference does not. + $DebugPreference = 'Continue' + $VerbosePreference = 'Continue' + + $index = 0 + while ([datetime]::UtcNow -lt $deadline) { + $target = $targets[$index % $targets.Count] + $index++ + $sentAt = (Get-Date).ToString('o') + $sw = [Diagnostics.Stopwatch]::StartNew() + $streams = $null + $failure = $null + try { + $streams = Invoke-MgxRequest -Uri $target.Uri -Property id -ErrorAction Stop 5>&1 4>&1 + } + catch { + $failure = "$($_.Exception.Message)" + } + $sw.Stop() + + # One row per attempt on the wire: a call retried past a 429 met the service twice, + # and the second meeting has its own latency and its own headers. + $traces = @() + $pacingWaitMs = $null + foreach ($record in @($streams)) { + if ($record -is [System.Management.Automation.VerboseRecord]) { + if ("$($record.Message)" -match 'Adaptive pacing: waited (\d+)ms') { + $pacingWaitMs = [int]$Matches[1] + } + continue + } + if ($record -isnot [System.Management.Automation.DebugRecord]) { continue } + $lines = "$($record.Message)" -split "`r?`n" + if ($lines[0] -notmatch '^\[Mgx\] Response: (\d+) .* in (\d+) ms') { continue } + $trace = [ordered]@{ + Status = [int]$Matches[1] + ElapsedMs = [int]$Matches[2] + ResourceUnit = $null + ThrottleLimitPercentage = $null + ThrottleScope = $null + RetryAfter = $null + RateLimitHeader = $false + } + foreach ($line in $lines) { + # The body follows the header block and is not indented; a four-space indent + # is a header and nothing else is. + if ($line -match '^ Body \(') { break } + if ($line -notmatch '^ ([^:]+):\s*(.*)$') { continue } + $name = $Matches[1].Trim(); $value = $Matches[2].Trim() + switch -Regex ($name) { + '^(?i)x-ms-resource-unit$' { $trace.ResourceUnit = $value } + '^(?i)x-ms-throttle-limit-percentage$' { $trace.ThrottleLimitPercentage = $value } + '^(?i)x-ms-throttle-scope$' { $trace.ThrottleScope = $value } + '^(?i)Retry-After$' { $trace.RetryAfter = $value } + '^(?i)RateLimit-' { $trace.RateLimitHeader = $true } + } + } + $traces += , $trace + } + + if ($traces.Count -eq 0) { + # No response reached the tracer: a transport failure, or a refusal raised before + # the request went out. Recorded, never dropped - a window with a hole in it is + # not a distribution, and a silent drop is how one gets a hole. + [pscustomobject]@{ + Timestamp = $sentAt; Worker = $worker; Label = $target.Label; Uri = $target.Uri + Attempt = 1; Attempts = 1; Status = 0; ElapsedMs = $null + CallElapsedMs = [int]$sw.ElapsedMilliseconds; PacingWaitMs = $pacingWaitMs + ResourceUnit = $null; ThrottleLimitPercentage = $null; ThrottleScope = $null + RetryAfter = $null; RateLimitHeader = $false + Error = $(if ($failure) { $failure } else { 'no response traced' }) + } + continue + } + + $n = 0 + foreach ($trace in $traces) { + $n++ + [pscustomobject]@{ + Timestamp = $sentAt; Worker = $worker; Label = $target.Label; Uri = $target.Uri + Attempt = $n; Attempts = $traces.Count; Status = $trace.Status; ElapsedMs = $trace.ElapsedMs + CallElapsedMs = [int]$sw.ElapsedMilliseconds; PacingWaitMs = $pacingWaitMs + ResourceUnit = $trace.ResourceUnit + ThrottleLimitPercentage = $trace.ThrottleLimitPercentage + ThrottleScope = $trace.ThrottleScope + RetryAfter = $trace.RetryAfter + RateLimitHeader = $trace.RateLimitHeader + Error = $(if ($failure -and $n -eq $traces.Count) { $failure } else { $null }) + } + } + } + } | ForEach-Object { + $row = $_ + $writer.WriteLine((ConvertTo-Json -InputObject $row -Compress -Depth 4)) + $attempts++ + if ($null -ne $row.ElapsedMs -and $row.Status -ge 200 -and $row.Status -lt 300) { + $liveLatencies.Add([double]$row.ElapsedMs) + } + + # Clamped with a comparison, not [math]::Max: Max(0, ) binds the int overload + # and rounds the minute away before the floor sees it, which halves the first bucket + # and invents a last one. + $minutesIn = ((Get-Date $row.Timestamp) - $startedAt).TotalMinutes + if ($minutesIn -lt 0) { $minutesIn = 0 } + $index = [int][math]::Floor($minutesIn / $BucketMinutes) + + # Once per bucket, not once per row: these are process-wide totals, and the point of + # sampling them on the clock is to see the drive bucket's latency ratio move. Forward + # only: rows arrive from every worker at once and their timestamps interleave across a + # boundary, so a late row from the previous bucket must not re-sample. The p90 printed + # here is $liveLatencies' rolling figure, for this progress line only - the summary + # Write-BenchResult records below is read back from $rowsPath, not from it. + if ($index -gt $lastBucket) { + $lastBucket = $index + $t = $null + try { $t = Get-MgxTelemetry -ErrorAction Stop } catch { $t = $null } + if ($t) { + $pacingSamples += [pscustomobject]@{ + AtMinute = $index * $BucketMinutes + Requests = $t.Requests + ResourceUnits = $t.ResourceUnitsConsumed + ThrottleRetries = $t.ThrottleRetries + PacingWaitMs = $t.AdaptivePacingWaitMs + PacingActivations = $t.AdaptivePacingActivations + LastThrottlePercentage = $t.LastThrottlePercentage + DrivePacingState = @($t.PacingState -split ';' | Where-Object { $_ -match '^\s*drive:' } | + ForEach-Object { $_.Trim() })[0] + } + } + Write-Host (" minute {0,4}: {1} attempt(s) so far, p90 {2} ms" -f ` + ($index * $BucketMinutes), $attempts, (Get-ClampPercentile -Values $liveLatencies -Percentile 90)) + } + } +} +finally { + $writer.Flush() + $writer.Dispose() + $loadSw.Stop() +} +Write-Host ("rows written to {0} ({1} attempt row(s))" -f $rowsPath, $attempts) + +# --- Report --------------------------------------------------------------------------------- +# +# Read back from $rowsPath rather than carried over from the loop above: the checkpoint is the +# one artifact a killed run still has, and a summary built from anything else can disagree with +# what re-running Get-ClampSummary against the same file would say. It is also what the Pester +# cases over this benchmark call, so the table below and the assertions over it read the same +# computation, not two implementations of it. + +$endedAt = Get-Date +$wallMs = [long]$loadSw.ElapsedMilliseconds + +Write-Host '' +Write-Host '=== SPO LATENCY CLAMP (paced drive reads, read-only) ===' +Write-Host ("drive {0}, {1} worker(s), {2} target(s)" -f $drivePath, $Concurrency, $targets.Count) +# Both clocks, always: a Stopwatch that lost time to a suspended machine only shows up next to +# the wall clock it should agree with, never in either figure alone. +Write-Host ("window measured: {0:F1} minute(s) wall clock (StartedAt to EndedAt), {1:F1} minute(s) stopwatch" -f ` + ($endedAt - $startedAt).TotalMinutes, ($wallMs / 60000.0)) +Write-Host ("preceding activity: {0}" -f $(if ($PrecedingActivity) { $PrecedingActivity } else { 'NOT STATED' })) + +$summary = Get-ClampSummary -RowsPath $rowsPath -StartedAt $startedAt -EndedAt $endedAt ` + -WallMs $wallMs -Minutes $Minutes -BucketMinutes $BucketMinutes + +if ($summary.Refused) { + # Named in $summary.RefusalReason, printed verbatim: rows are already safe on disk (written + # above, independent of this check), but a distribution computed across a gap the machine + # was not awake for is not a distribution, and nothing is appended to the results log for a + # later -RecordBaseline to pick up. + Write-Host $summary.RefusalReason + exit 1 +} + +$bucketRows = $summary.Buckets +$overall = $summary.Overall + +Write-Host ('{0,7} {1,8} {2,8} {3,8} {4,8} {5,6} {6,7} {7,7} {8,7} {9,7} {10,7}' -f ` + 'minute', 'calls', 'p50', 'p90', 'p99', '429s', 'errors', 'pct', 'scope', 'retry', 'ratelim') +foreach ($b in $bucketRows) { + $mark = if ($b.Note) { " <- $($b.Note)" } else { '' } + Write-Host ('{0,7} {1,8} {2,7}ms {3,7}ms {4,7}ms {5,6} {6,7} {7,7} {8,7} {9,7} {10,7}{11}' -f ` + $b.FromMinute, $b.Calls, ($b.P50ElapsedMs ?? '-'), ($b.P90ElapsedMs ?? '-'), ($b.P99ElapsedMs ?? '-'), + $b.Throttled429, $b.Errors, $b.ThrottlePercentageSeen, $b.ThrottleScopeSeen, $b.RetryAfterSeen, $b.RateLimitSeen, $mark) +} +Write-Host '' +Write-Host $summary.Headline.Sentence +if ($overall.Refusals.Count -gt 0) { + Write-Host 'refusals by kind:' + foreach ($kind in $overall.Refusals.Keys) { + Write-Host (" {0}: {1}" -f $kind, $overall.Refusals[$kind]) + } +} +if ($overall.Throttled429 -eq 0 -and $overall.ThrottlePercentageSeen -eq 0 -and $overall.ThrottleScopeSeen -eq 0 -and $overall.RateLimitSeen -eq 0) { + Write-Host 'no 429 and no throttle-signal header anywhere in the window: whatever the latency did, nothing in the response said so.' +} +else { + Write-Host ("throttle signals present: {0} 429(s), {1} percentage header(s), {2} scope header(s), {3} RateLimit-* header(s)" -f ` + $overall.Throttled429, $overall.ThrottlePercentageSeen, $overall.ThrottleScopeSeen, $overall.RateLimitSeen) +} + +$telemetry = $null +try { $telemetry = Get-MgxTelemetry -ErrorAction Stop } catch { $telemetry = $null } +# Labeled, and never the headline above: PacingState is the pacer's one most-recent latency +# sample, which is exactly the figure that used to be read as this benchmark's verdict. +if ($telemetry?.PacingState) { Write-Host "last pacer sample: $($telemetry.PacingState)" } +foreach ($sample in $pacingSamples) { + if ($sample.DrivePacingState) { Write-Host (" minute {0,4}: {1}" -f $sample.AtMinute, $sample.DrivePacingState) } +} + +$result = [ordered]@{ + DriveUri = $drivePath + DriveType = $probe.driveType + Minutes = $Minutes + Concurrency = $Concurrency + BucketMinutes = $BucketMinutes + Targets = $targets.Count + StartedAt = $startedAt.ToString('o') + EndedAt = $endedAt.ToString('o') + PrecedingActivity = $PrecedingActivity + # Recorded, not assumed: an unpaced run measures a different thing and must not be read + # against a paced baseline without the difference being visible in the entry. + AdaptivePacing = -not (Get-MgxOption).NoAdaptivePacing + RowsFile = '18-spo-latency-clamp.rows.jsonl' + ProbeElapsedMs = [int]$probeSw.ElapsedMilliseconds + Overall = $overall + # The verdict, taken from the buckets - see Get-ClampHeadline in common.ps1. PacingState + # below is the pacer's own last snapshot, kept for diagnosis and never read as this. + Headline = $summary.Headline.Sentence + Buckets = $bucketRows + PacingSamples = $pacingSamples + PacingState = $telemetry?.PacingState + WallMs = $wallMs +} + +Write-BenchResult -Benchmark '18-spo-latency-clamp' -Result $result -WallMs $wallMs diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md index 3b459da..13ad241 100644 --- a/tests/benchmarks/README.md +++ b/tests/benchmarks/README.md @@ -1,6 +1,6 @@ # Mgx benchmark suite -Reproduces every number in the main README's Benchmarks section. Each script is self-contained; `run.ps1` runs the full suite and regenerates the README tables. +Reproduces every number in the main README's Benchmarks section. Each script is self-contained; `run.ps1` runs the full suite. The main README's tables are written by hand from what the scripts print - nothing here generates them. ## Resource units @@ -21,7 +21,10 @@ there is no session telemetry to read - expected, not a failure. `~/.mgx-bench/app.json` (`{ tenantId, appId, clientSecret }`) — the app needs `User.ReadWrite.All`, `Group.ReadWrite.All`, `Directory.ReadWrite.All`, `AuditLog.Read.All` - A seeded test tenant (~100k users, ~15k groups) for the tenant benchmarks. - `06-fault-gauntlet.ps1` needs **no tenant** — it runs against a local mock. + `06-fault-gauntlet.ps1`, `16-pathological-gauntlet.ps1` and + `17-pathological-environment.ps1` need **no tenant** — they run against a local mock. + 17 also needs `Az.Accounts` and `PnP.PowerShell` installed: importing them mid-read is + the perturbation. Never point these scripts at a production tenant. Benchmarks 04 and 07 create and delete objects; 07 and 10 deliberately provoke throttling. @@ -44,10 +47,24 @@ Never point these scripts at a production tenant. Benchmarks 04 and 07 create an | 13 | `13-resource-unit-rate.ps1` | Sustained RU ceiling vs burst allowance, at held send rates | yes | | 14 | `14-pacing-cold-cost.ps1` | What pacing costs a short run that finishes during the ramp | yes | | 15 | `15-fanout-concurrency-scaling.ps1` | What `-Concurrency` buys, and whether the connection pool is the ceiling | yes | +| 16 | `16-pathological-gauntlet.ps1` | Faults that never clear: storm, outage, death mid-body, delayed visibility | no | +| 17 | `17-pathological-environment.ps1` | What a read does when the session is replaced, or a competing module loaded, under it | no | +| 18 | `18-spo-latency-clamp.ps1` | Whether SPO stretches drive latency with no 429 and no throttle header - latency-as-pacing-input evidence | yes, with SPO | -11-15 are not in `run.ps1`. 11-13 each deliberately drive the tenant to 429s, so they run +11-18 are not in `run.ps1`. 11-13 each deliberately drive the tenant to 429s, so they run standalone (10 also throttles on purpose, which is why `run.ps1` puts it last). 14 and 15 are cheap and answer one question each; they are read on their own rather than as part of a sweep. +16 and 17 need no tenant, but they are minutes of deliberate pathology and record what each +stack does rather than a number the README quotes, so they are read on their own too. 17 has +no pass or fail at all: recovering from a session replaced mid-operation is 2.3.0 work, and +what it records is the before picture. +18 is read-only and still not cheap: it holds sustained paced reads against one drive for over +an hour. It needs the drive named on the command line - `-DriveId` or `-DriveUri /drives/`, +with no default - a tenant with a SharePoint license, an uncontended machine, and a window +longer than the hour the clamp is reported to last; it refuses to start rather than measure +nothing when any of those is missing. Its per-call rows land in +`results/18-spo-latency-clamp.rows.jsonl` as they are collected, and `-RecordBaseline` promotes +it only when it is named, since `run.ps1` never runs it. ## Methodology @@ -59,4 +76,7 @@ cheap and answer one question each; they are read on their own rather than as pa - Memory numbers are peak working set sampled at 200ms plus managed-heap delta; streaming claims are measured with streaming consumers (piped, never assigned). - Results append to `results/.json` with Mgx/SDK/PS versions and a - timestamp; the README tables are generated from the latest entries. + timestamp. That directory stays local; `run.ps1 -RecordBaseline` promotes the + latest entries into the committed `baseline.json`, and `-CompareBaseline` + reads a later run against it. The main README's tables are written by hand + from the same entries. diff --git a/tests/benchmarks/baseline.json b/tests/benchmarks/baseline.json new file mode 100644 index 0000000..cbd8d46 --- /dev/null +++ b/tests/benchmarks/baseline.json @@ -0,0 +1,227 @@ +{ + "SchemaVersion": 1, + "GeneratedAt": "2026-09-10T17:11:39.2385100+02:00", + "Scenarios": { + "01": { + "Scenario": "01", + "Script": "01-list-users.ps1", + "Headline": "MgxListMs", + "Discriminator": null, + "Metrics": { + "MgxListMs": { + "Value": 56924.0, + "Unit": "ms", + "Path": "mgx.Median.ElapsedMs" + }, + "MgxTimeToFirstMs": { + "Value": 663.0, + "Unit": "ms", + "Path": "ttfrMgx.Median.ElapsedMs" + }, + "MgxPeakWorkingSetMB": { + "Value": 314.6, + "Unit": "MB", + "Path": "mgx.Median.PeakWorkingSetMB" + } + }, + "MgxVersion": "2.1.4", + "SdkVersion": "2.34.0", + "PSVersion": "7.6.4", + "RecordedAt": "2026-09-10T13:56:49.347805+02:00", + "PromotedAt": "2026-09-10T15:50:28.117482+02:00" + }, + "02": { + "Scenario": "02", + "Script": "02-fanout-lookup.ps1", + "Headline": "MgxFanoutMs", + "Discriminator": null, + "Metrics": { + "MgxFanoutMs": { + "Value": 127303.0, + "Unit": "ms", + "Path": "mgx.Median.ElapsedMs" + }, + "MgxPeakWorkingSetMB": { + "Value": 330.1, + "Unit": "MB", + "Path": "mgx.Median.PeakWorkingSetMB" + } + }, + "MgxVersion": "2.1.4", + "SdkVersion": "2.34.0", + "PSVersion": "7.6.4", + "RecordedAt": "2026-09-10T14:40:49.214531+02:00", + "PromotedAt": "2026-09-10T15:50:28.122807+02:00" + }, + "04": { + "Scenario": "04", + "Script": "04-batch-create.ps1", + "Headline": "MgxBatchCreateMs", + "Discriminator": null, + "Metrics": { + "MgxBatchCreateMs": { + "Value": 503002.0, + "Unit": "ms", + "Path": "mgxCreate.ElapsedMs" + }, + "MgxBatchPatchMs": { + "Value": 49333.0, + "Unit": "ms", + "Path": "batchPatch.ElapsedMs" + }, + "MgxFanoutPatchMs": { + "Value": 28851.0, + "Unit": "ms", + "Path": "fanoutPatch.ElapsedMs" + } + }, + "MgxVersion": "2.1.4", + "SdkVersion": "2.34.0", + "PSVersion": "7.6.4", + "RecordedAt": "2026-09-10T17:06:39.0267+02:00", + "PromotedAt": "2026-09-10T17:11:39.2330620+02:00" + }, + "05": { + "Scenario": "05", + "Script": "05-memory-export.ps1", + "Headline": "MgxExportPeakWorkingSetMB", + "Discriminator": null, + "Metrics": { + "MgxExportPeakWorkingSetMB": { + "Value": 311.6, + "Unit": "MB", + "Path": "mgxExport.PeakWorkingSetMB" + }, + "MgxExportMs": { + "Value": 47303.0, + "Unit": "ms", + "Path": "mgxExport.ElapsedMs" + }, + "MgxExportManagedHeapDeltaMB": { + "Value": 24.8, + "Unit": "MB", + "Path": "mgxExport.ManagedHeapDeltaMB" + } + }, + "MgxVersion": "2.1.4", + "SdkVersion": "2.34.0", + "PSVersion": "7.6.4", + "RecordedAt": "2026-09-10T13:59:39.157651+02:00", + "PromotedAt": "2026-09-10T15:50:28.121054+02:00" + }, + "08": { + "Scenario": "08", + "Script": "08-delta-sync.ps1", + "Headline": "InitialSyncMs", + "Discriminator": null, + "Metrics": { + "InitialSyncMs": { + "Value": 255781.0, + "Unit": "ms", + "Path": "initial.ElapsedMs" + }, + "IncrementalSyncMs": { + "Value": 452.0, + "Unit": "ms", + "Path": "incremental.ElapsedMs" + }, + "SteadySyncMs": { + "Value": 147.0, + "Unit": "ms", + "Path": "steady.ElapsedMs" + }, + "InitialItems": { + "Value": 140261.0, + "Unit": "", + "Path": "initial.Output.items" + } + }, + "MgxVersion": "2.1.4", + "SdkVersion": "2.34.0", + "PSVersion": "7.6.4", + "RecordedAt": "2026-09-10T14:54:54.564289+02:00", + "PromotedAt": "2026-09-10T15:50:28.124234+02:00" + }, + "09": { + "Scenario": "09", + "Script": "09-kill-resume.ps1", + "Headline": "CheckpointedExportMs", + "Discriminator": null, + "Metrics": { + "CheckpointedExportMs": { + "Value": 48406.0, + "Unit": "ms", + "Path": "checkpointed.ElapsedMs" + }, + "BaselineExportMs": { + "Value": 45995.0, + "Unit": "ms", + "Path": "baseline.ElapsedMs" + }, + "ResumeMs": { + "Value": 26640.0, + "Unit": "ms", + "Path": "resume.ElapsedMs" + }, + "CheckpointOverheadPct": { + "Value": 5.2, + "Unit": "%", + "Path": "verification.overheadPct" + }, + "DuplicateIds": { + "Value": 0.0, + "Unit": "", + "Path": "verification.duplicateIds" + } + }, + "MgxVersion": "2.1.4", + "SdkVersion": "2.34.0", + "PSVersion": "7.6.4", + "RecordedAt": "2026-09-10T14:57:23.266194+02:00", + "PromotedAt": "2026-09-10T15:50:28.126139+02:00" + }, + "18": { + "Scenario": "18", + "Script": "18-spo-latency-clamp.ps1", + "Headline": "P90ElapsedMs", + "Discriminator": "live suite content tests about 25 minutes earlier; nothing else of ours on this tenant since", + "Metrics": { + "P90ElapsedMs": { + "Value": 225.0, + "Unit": "ms", + "Path": "Overall.P90ElapsedMs" + }, + "P50ElapsedMs": { + "Value": 171.0, + "Unit": "ms", + "Path": "Overall.P50ElapsedMs" + }, + "P99ElapsedMs": { + "Value": 517.0, + "Unit": "ms", + "Path": "Overall.P99ElapsedMs" + }, + "P90CallElapsedMs": { + "Value": 295.0, + "Unit": "ms", + "Path": "Overall.P90CallElapsedMs" + }, + "ClampRatio": { + "Value": 1.0, + "Unit": "x", + "Path": "Overall.ClampRatio" + }, + "Throttled429": { + "Value": 203.0, + "Unit": "", + "Path": "Overall.Throttled429" + } + }, + "MgxVersion": "2.1.4", + "SdkVersion": "2.34.0", + "PSVersion": "7.6.4", + "RecordedAt": "2026-09-10T14:31:37.737793+02:00", + "PromotedAt": "2026-09-10T15:11:15.980556+02:00" + } + } +} diff --git a/tests/benchmarks/common.ps1 b/tests/benchmarks/common.ps1 index 9041373..15c17c1 100644 --- a/tests/benchmarks/common.ps1 +++ b/tests/benchmarks/common.ps1 @@ -193,6 +193,89 @@ function Import-MgxLocal { else { Import-Module Mgx -Force } } +# The gauntlets point the whole Graph stack at a local mock through a named Graph +# environment. Add-MgEnvironment writes that name into the user profile, so an environment +# one run adds outlives it: the next run finds MgxBench already there, keeps whatever +# GraphEndpoint the first run wrote, and aims its SDK contenders at a port this run's mock +# does not hold - which reports no completions, or, when an orphaned listener still holds +# that port, counts answered by a server this run never started. Point it at this run's +# endpoint every time, and take it out again when the run ends. +function Set-BenchGraphEnvironment { + param( + [Parameter(Mandatory)] [string] $GraphEndpoint, + [string] $Name = 'MgxBench' + ) + Remove-BenchGraphEnvironment -Name $Name + Add-MgEnvironment -Name $Name -GraphEndpoint $GraphEndpoint ` + -AzureADEndpoint 'https://login.microsoftonline.com' | Out-Null + $Name +} + +function Remove-BenchGraphEnvironment { + param([string] $Name = 'MgxBench') + # Remove-MgEnvironment raises "Environment not found" as a terminating error that + # -ErrorAction does not downgrade, so the name is looked up first: a finally block has to + # run on a machine where the environment was never added, and on the second call of a run + # that already removed it. + if (Get-MgEnvironment -Name $Name -ErrorAction SilentlyContinue) { + Remove-MgEnvironment -Name $Name | Out-Null + } +} + +# Starts mock-graph-server.ps1 as a child process on the first free port at or above $Port, +# waits for it to answer /ping, and returns the process, the port and the base URL. The port +# is searched rather than assumed: a crashed run can leave a listener behind, and a second +# gauntlet may be running on the same machine. The caller stops it in its finally. +function Start-BenchMockServer { + param( + [int] $Port = 8787, + [int] $Range = 20, + [int] $ReadySeconds = 10 + ) + $free = ($Port..($Port + $Range)) | Where-Object { + -not (Test-Connection -TargetName localhost -TcpPort $_ -TimeoutSeconds 1 -Quiet) + } | Select-Object -First 1 + if (-not $free) { throw 'no free port found in range' } + $base = "http://localhost:$free" + $process = Start-Process pwsh -PassThru ` + -ArgumentList '-NoProfile', '-File', (Join-Path $PSScriptRoot 'mock-graph-server.ps1'), '-Port', $free + $up = $false + foreach ($i in 1..($ReadySeconds * 5)) { + try { $null = Invoke-RestMethod "$base/ping" -TimeoutSec 1; $up = $true; break } + catch { Start-Sleep -Milliseconds 200 } + } + if (-not $up) { + if (-not $process.HasExited) { $process.Kill() } + throw "mock server did not come up on port $free" + } + Write-Host "mock server up (pid $($process.Id))" + [pscustomobject]@{ Process = $process; Port = $free; BaseUrl = $base } +} + +# By pid, and only ever by pid: another gauntlet's server is the same executable under the +# same name, and a pattern kill takes it with this one. +function Stop-BenchMockServer { + param([object] $Server) + if ($Server -and $Server.Process -and -not $Server.Process.HasExited) { $Server.Process.Kill() } +} + +# Points the whole Graph stack at a mock endpoint: the SDK's own cmdlets, and through the +# session the clients Mgx builds. The token is a string the mock does not read. The tenant id +# it is given is read - by Mgx: -AccessToken auth leaves AuthContext.TenantId empty, and the +# auth fingerprint treats an empty TenantId as "not connected" (1.0.4 identity fix). +function Connect-BenchMockGraph { + param( + [Parameter(Mandatory)] [string] $GraphEndpoint, + [string] $Token = 'mock-token-not-validated', + [string] $TenantId = 'gauntlet-mock-tenant' + ) + $null = Set-BenchGraphEnvironment -GraphEndpoint $GraphEndpoint + Disconnect-MgGraph -ErrorAction SilentlyContinue | Out-Null + Connect-MgGraph -Environment MgxBench ` + -AccessToken (ConvertTo-SecureString $Token -AsPlainText -Force) -NoWelcome + [Microsoft.Graph.PowerShell.Authentication.GraphSession]::Instance.AuthContext.TenantId = $TenantId +} + # Runs one measured pass: telemetry snapshot around $Script, wall time, peak working set # sampled from a background thread. Returns a result object; does not write anywhere. function Measure-BenchPass { @@ -315,14 +398,19 @@ function Invoke-WatchdoggedContender { $sw = [System.Diagnostics.Stopwatch]::StartNew() $p = Start-Process pwsh -PassThru -NoNewWindow ` -ArgumentList (@('-NoProfile', '-File', $ScriptPath) + $ArgumentList) + # Captured now, not read off $p later: Process.StartTime reads back null on Unix once the + # child has exited, and a child that dies before its first Beat (an import or connect + # failure inside the first poll) has usually done exactly that by the time the fallback + # below is needed. + $launchedAt = Get-Date while (-not $p.HasExited) { Start-Sleep -Seconds 10 - $lastBeat = if (Test-Path $HeartbeatFile) { (Get-Item $HeartbeatFile).LastWriteTime } else { $p.StartTime } + $lastBeat = if (Test-Path $HeartbeatFile) { (Get-Item $HeartbeatFile).LastWriteTime } else { $launchedAt } if (((Get-Date) - $lastBeat).TotalSeconds -gt $StallSeconds) { $p.Kill() Write-Host (" WATCHDOG: '{0}' no progress for {1}s - killed after {2:F0}s total" -f $Name, $StallSeconds, $sw.Elapsed.TotalSeconds) return [pscustomobject]@{ - Name = $Name; Hung = $true; ElapsedMs = [long]$sw.ElapsedMilliseconds + Name = $Name; Hung = $true; Crashed = $false; ElapsedMs = [long]$sw.ElapsedMilliseconds Output = [pscustomobject]@{ note = "HUNG - no progress for ${StallSeconds}s, killed by watchdog" } } } @@ -330,12 +418,378 @@ function Invoke-WatchdoggedContender { if (Test-Path $ResultFile) { $r = Get-Content $ResultFile -Raw | ConvertFrom-Json $r | Add-Member -NotePropertyName Hung -NotePropertyValue $false -Force + $r | Add-Member -NotePropertyName Crashed -NotePropertyValue $false -Force return $r } + # Exited on its own, within the poll granularity above, before ever touching + # $HeartbeatFile or $ResultFile - an import or connect failure, not a stall. Hung stays + # false: nothing here was killed for want of progress, so the elapsed time is watchdog + # poll noise (up to 10s) rather than a measurement, and Crashed is what callers should + # branch on instead of reading Hung as a catch-all for "no result". + [pscustomobject]@{ + Name = $Name; Hung = $false; Crashed = $true; ExitCode = $p.ExitCode + ElapsedMs = [long]$sw.ElapsedMilliseconds + Output = [pscustomobject]@{ note = "exited $($p.ExitCode) without a result" } + } +} + +function Get-BenchPacingLoadShape { + <# + .SYNOPSIS + The load a pacing arm will issue, and whether it can reach the throttled regime. + + .DESCRIPTION + Invoke-MgxRequest deduplicates the ids it is piped, by design, so a slice that repeats + an id issues one request for it: the request count of an arm is the count of DISTINCT + ids handed to each pipeline, never the volume asked for. A pipeline can be handed the + shared pool at most once, so an arm can issue at most Pipelines x IdPool requests and + anything above that is clamped - reported, because an arm that believes it issued more + than it did describes a regime it never entered. + + Admissibility has two halves, and a run needs both: + + rate the offered draw must exceed the rate the tenant keeps serving at while it + refuses, or the burst allowance never drains and nothing is ever refused. + Volume cannot buy this: it is set by how much is in flight. + volume the run must then spend long enough past the allowance for the throttled + phase to be at least as long as the unthrottled one, or the comparison is + mostly a comparison of two unthrottled runs. + + The rate here is a projection, from the 180 RU/s that 128 requests in flight measured + against the seeded tenant; the arms report what they actually drew, and the verdict is + taken from that. One process can hold roughly 2,000 requests in flight - the transport + pools 20 connections and multiplexes HTTP/2 streams over them - so a pipeline count + whose product with the concurrency passes that will not offer the rate projected here. + + .PARAMETER BurstRu + Resource units the tenant serves before its first refusal. + + .PARAMETER SustainedRuPerSecond + The rate it keeps serving at once it is refusing. + + .PARAMETER RuPerSecondPerPipeline + Measured draw of one pipeline at concurrency 128. Scaled linearly with -Concurrency. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [ValidateRange(1, [int]::MaxValue)] [int] $Count, + [Parameter(Mandatory)] [ValidateRange(1, 512)] [int] $Pipelines, + [Parameter(Mandatory)] [ValidateRange(1, 128)] [int] $Concurrency, + [Parameter(Mandatory)] [ValidateRange(1, [int]::MaxValue)] [int] $IdPool, + [Parameter(Mandatory)] [ValidateRange(1, [int]::MaxValue)] [int] $BurstRu, + [Parameter(Mandatory)] [ValidateRange(1, [int]::MaxValue)] [int] $SustainedRuPerSecond, + [double] $RuPerSecondPerPipeline = 180 + ) + + $idsPerPipeline = [math]::Min([int][math]::Ceiling($Count / $Pipelines), $IdPool) + $total = $idsPerPipeline * $Pipelines + $inFlight = $Pipelines * $Concurrency + $expectedRate = [math]::Round($Pipelines * $RuPerSecondPerPipeline * ($Concurrency / 128), 0) + + $burstSeconds = [math]::Round($BurstRu / $expectedRate, 1) + $throttledSeconds = [math]::Round([math]::Max(0, $total - $BurstRu) / $SustainedRuPerSecond, 1) + + $rateClears = $expectedRate -gt $SustainedRuPerSecond + $volumeClears = $total -gt $BurstRu -and $throttledSeconds -ge $burstSeconds + + $reasons = @() + if (-not $rateClears) { + $reasons += ("offered draw {0} RU/s does not exceed the {1} RU/s the tenant serves while refusing - raise -Pipelines (volume cannot buy a rate)" -f ` + $expectedRate, $SustainedRuPerSecond) + } + if (-not $volumeClears) { + $reasons += ("{0} requests leaves {1}s in the throttled regime against {2}s inside the {3} RU burst allowance - raise -Count, and -IdPool or -Pipelines with it" -f ` + $total, $throttledSeconds, $burstSeconds, $BurstRu) + } + + [pscustomobject]@{ + Requested = $Count + Pipelines = $Pipelines + Concurrency = $Concurrency + IdPool = $IdPool + IdsPerPipeline = $idsPerPipeline + TotalRequests = $total + Clamped = $total -lt $Count + InFlight = $inFlight + ExpectedRuPerSecond = $expectedRate + BurstRu = $BurstRu + SustainedRuPerSecond = $SustainedRuPerSecond + BurstSeconds = $burstSeconds + ThrottledSeconds = $throttledSeconds + EstimatedSeconds = [math]::Round($burstSeconds + $throttledSeconds, 1) + RateClears = $rateClears + VolumeClears = $volumeClears + Admissible = $rateClears -and $volumeClears + Reasons = $reasons + } +} + +function Get-BenchPipelineSlice { + <# + .SYNOPSIS + The ids each pipeline of a fan-out arm issues: distinct within a pipeline, staggered + across them. + + .DESCRIPTION + Every pipeline may issue the whole shared pool once, and each starts at its own rotated + offset so the pipelines are not all pulling the same object at the same moment. The walk + takes fewer steps than the pool is long, so it never wraps onto its own start and no + slice can hold a repeat - which is the invariant the arm's request count rests on, and + which this checks before handing the slices back rather than trusting the arithmetic. + + .OUTPUTS + One list of ids per pipeline, in pipeline order. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string[]] $Pool, + [Parameter(Mandatory)] [ValidateRange(1, 512)] [int] $Pipelines, + [Parameter(Mandatory)] [ValidateRange(1, [int]::MaxValue)] [int] $IdsPerPipeline + ) + + if ($Pool.Count -eq 0) { throw 'no ids to slice across the pipelines.' } + if ($IdsPerPipeline -gt $Pool.Count) { + throw ("a pipeline cannot issue $IdsPerPipeline distinct ids from a pool of $($Pool.Count); " + + 'Invoke-MgxRequest deduplicates its pipeline ids, so the repeats would never be sent.') + } + + $slices = [System.Collections.Generic.List[object]]::new() + $stride = [math]::Max(1, [int][math]::Floor($Pool.Count / $Pipelines)) + for ($i = 0; $i -lt $Pipelines; $i++) { + $slice = [System.Collections.Generic.List[string]]::new() + $start = ($i * $stride) % $Pool.Count + for ($j = 0; $j -lt $IdsPerPipeline; $j++) { $slice.Add($Pool[($start + $j) % $Pool.Count]) } + + $distinct = @($slice | Sort-Object -Unique).Count + if ($distinct -ne $slice.Count) { + throw ("pipeline $i was given $($slice.Count - $distinct) duplicate id(s) of $($slice.Count); " + + 'the arm would issue fewer requests than it reports.') + } + $slices.Add($slice) + } + , $slices +} + +# --- Which tenant, and which tree, produced a number ----------------------------------------- +# +# Mgx, SDK and PowerShell versions plus a timestamp do not identify a measurement. Two runs +# against two tenants are the same entry with different numbers in it, so a cross-tenant claim +# cannot be audited from results/; and every working tree that has not bumped the version +# carries the same version string, so a slow number cannot be attributed to the code or to the +# tenant after the fact. Everything below answers null rather than throwing when its source is +# absent: the no-tenant benchmarks must still record. + +function Get-BenchDocumentedBudget { + <# + .SYNOPSIS + Graph's documented resource-unit budget for an application+tenant pair, by tenant size. + + .DESCRIPTION + The budget is banded on the tenant's user count rather than fixed: + + S under 50 users 3,500 ResourceUnits / 10s 350 RU/s + M 50 to 500 users 5,000 ResourceUnits / 10s 500 RU/s + L above 500 users 8,000 ResourceUnits / 10s 800 RU/s + + Source: https://learn.microsoft.com/en-us/graph/throttling-limits - "Identity and access + service limits", whose application+tenant pair row carries those three quotas and whose + note reads "The tenant sizes are defined as follows: S - under 50 users, M - between 50 + and 500 users, and L - above 500 users." + + One function, because the denominator a result records its draw against and the tier a + calibration run reports have to be the same number or the two artifacts disagree about + what a tenant is allowed. + + .PARAMETER UserCount + The tenant's user count. Null (unknown) yields null: an assumed tier is worse than none. + #> + [CmdletBinding()] + param([object] $UserCount) + + if ($null -eq $UserCount) { return $null } + $users = [long]$UserCount + if ($users -lt 0) { return $null } + + $tier = if ($users -lt 50) { @{ Tier = 'S'; Band = 'under 50 users'; RuPer10Seconds = 3500 } } + elseif ($users -le 500) { @{ Tier = 'M'; Band = '50 to 500 users'; RuPer10Seconds = 5000 } } + else { @{ Tier = 'L'; Band = 'above 500 users'; RuPer10Seconds = 8000 } } + [pscustomobject]@{ - Name = $Name; Hung = $true; ElapsedMs = [long]$sw.ElapsedMilliseconds - Output = [pscustomobject]@{ note = "child exited $($p.ExitCode) without writing a result" } + Tier = $tier.Tier + Band = $tier.Band + UserCount = $users + RuPer10Seconds = $tier.RuPer10Seconds + RuPerSecond = $tier.RuPer10Seconds / 10 + } +} + +function Get-BenchDirectoryObjectCount { + <# + .SYNOPSIS + The tenant's user count, in one request. + + .DESCRIPTION + $count=true on a $top=1 page carries @odata.count for the whole collection, which is the + cheapest exact count Graph offers - the /users/$count segment needs the same + ConsistencyLevel header and answers in text/plain, which the SDK does not surface. + + Through the SDK's own cmdlet rather than Invoke-MgxRequest: this runs as a result is + being written, sometimes moments after a benchmark that deliberately drove the tenant + into refusing, and it must not sit in mgx's retry pipeline waiting out a Retry-After + that belongs to no measurement. The uri is relative, so it resolves against whatever + endpoint the session was connected to rather than assuming the worldwide cloud. + #> + [CmdletBinding()] + param() + + $response = Invoke-MgGraphRequest -Method GET -OutputType HashTable -ErrorAction Stop ` + -Uri 'v1.0/users?$top=1&$count=true' ` + -Headers @{ ConsistencyLevel = 'eventual' } + $count = $response['@odata.count'] + if ($null -eq $count) { return $null } + [long]$count +} + +function Get-BenchTenantFact { + <# + .SYNOPSIS + Which tenant a benchmark ran against, how big it is, and what it is budgeted at. + + .DESCRIPTION + Computed once per process and cached: every benchmark records at least one result and + several record many, and none of them should pay for the same count twice. + + All three are null when there is no Graph session - the ordinary state of the no-tenant + benchmarks, and of the parent process of a benchmark whose arms connect in children. + A tenant that answers the context but not the count records the tenant and null for the + rest, because a partial fact is still an auditable one. + + An Entra tenant id is a GUID, and a session whose id is not one is not a directory any + of this describes: 06 connects a mock Graph under a made-up tenant name and must stay + off the network, so a non-GUID id records as no tenant rather than as a count request + nobody asked for. + + .PARAMETER Refresh + Recompute rather than answer from the cache. + #> + [CmdletBinding()] + param([switch] $Refresh) + + if ($script:BenchTenantFact -and -not $Refresh) { return $script:BenchTenantFact } + + $fact = [pscustomobject]@{ + TenantId = $null + DirectoryObjectCount = $null + Budget = $null + } + try { + $ctx = Get-MgContext -ErrorAction Stop + $tenantGuid = [guid]::Empty + if ($ctx -and $ctx.TenantId -and [guid]::TryParse($ctx.TenantId, [ref] $tenantGuid)) { + $fact.TenantId = $ctx.TenantId + $fact.DirectoryObjectCount = Get-BenchDirectoryObjectCount + $fact.Budget = Get-BenchDocumentedBudget -UserCount $fact.DirectoryObjectCount + } + } + catch { + # No Microsoft.Graph.Authentication, no session, or a tenant that would not answer the + # count. A recorded null says "not known"; a thrown error would lose the whole result. + Write-Verbose "tenant facts unavailable: $($_.Exception.Message)" + } + $script:BenchTenantFact = $fact + $fact +} + +function Get-BenchCommit { + <# + .SYNOPSIS + The short commit of the tree the suite ran from, with a dirty marker. + + .DESCRIPTION + The marker carries more than the hash does: a number measured over uncommitted edits is + not reproducible from the hash alone, and saying so is the difference between an entry + that can be re-run and one that only looks like it can. Null outside a git checkout. + #> + [CmdletBinding()] + param() + + if ($script:BenchCommitRead) { return $script:BenchCommit } + $script:BenchCommitRead = $true + $script:BenchCommit = $null + try { + $hash = & git -C $PSScriptRoot rev-parse --short HEAD 2>$null + if ($hash) { + $commit = "$hash".Trim() + if (@(& git -C $PSScriptRoot status --porcelain 2>$null).Count -gt 0) { + $commit = "$commit-dirty" + } + $script:BenchCommit = $commit + } + } + catch { + # No git, or not a checkout. Recording nothing is right; guessing is not. + $script:BenchCommit = $null } + $script:BenchCommit +} + +# The five fields of a recorded entry that say WHERE its numbers came from: which module and +# which SDK ran, against which tenant, how big that tenant is, and what the documentation +# budgets a tenant that size at. Everything else in an entry describes the run itself. +$script:BenchIdentityField = @( + 'MgxVersion' + 'SdkVersion' + 'TenantId' + 'DirectoryObjectCount' + 'DocumentedBudgetRuPerSecond' +) + +# One identity field off one entry. Get-BenchResultValue resolves a path through PSObject +# properties, which is every entry read back out of a results file but not a hashtable a caller +# assembled by hand, so a dictionary is read as one. A blank string is read as no value: an arm +# that recorded an empty tenant did not record a tenant, and pairing that against an arm that +# did is a disagreement, not a name to print with nothing after it. +function Get-BenchIdentityField { + param( + [object] $Node, + [Parameter(Mandatory)] [string] $Name + ) + $value = if ($Node -is [System.Collections.IDictionary]) { $Node[$Name] } + else { Get-BenchResultValue -Node $Node -Path $Name } + if ($value -is [string] -and [string]::IsNullOrWhiteSpace($value)) { return $null } + $value +} + +# Reads that identity off the entries the arms of a comparison recorded, for a caller writing a +# combined entry from a process that did none of the measuring itself. +# +# Arms that disagree on a field have no shared answer to give, so the field is recorded as null +# and the disagreement is named: two arms measured against two tenants, or under two module +# versions, are a comparison of two different things, and the entry has to say so rather than +# pick whichever arm was read first. Values are compared as they print, so a count recorded as +# a long by one arm and an int by the other still agrees. +function Get-BenchEntryIdentity { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [object[]] $Entries + ) + $identity = [ordered]@{} + foreach ($name in $script:BenchIdentityField) { + $seen = [ordered]@{} + foreach ($entry in $Entries) { + $value = Get-BenchIdentityField -Node $entry -Name $name + $key = if ($null -eq $value) { '(none)' } else { [string]$value } + if (-not $seen.Contains($key)) { $seen[$key] = $value } + } + if ($seen.Count -le 1) { + $identity[$name] = @($seen.Values)[0] + } + else { + $identity[$name] = $null + Write-Warning "the runs disagree on ${name} ($((@($seen.Keys)) -join ', ')) - recorded as null." + } + } + [pscustomobject]$identity } # Appends a result object to results/.json (one JSON doc per file, array of entries). @@ -345,10 +799,24 @@ function Write-BenchResult { [Parameter(Mandatory)] [object] $Result, # The run's wall clock. Only needed for the RU rate below; picked up from $Result.WallMs # when the caller already records one. - [long] $WallMs = 0 + [long] $WallMs = 0, + # Where the per-run log lives. Named for the same reason Export-BenchBaseline and + # Compare-BenchBaseline name theirs: so a test can be pointed somewhere that is not the + # suite's own measurements. + [string] $ResultsDirectory = (Join-Path $PSScriptRoot 'results'), + # Who measured, for a benchmark whose arms ran somewhere other than this process. The + # five fields below are read from this process by default, which is right wherever this + # process did the work; a comparison that starts its arms as child runs records from a + # parent that imported no local module and connected to no tenant, and the default then + # describes the parent - whichever copy of Mgx happened to autoload there, and no tenant + # at all. Get-BenchEntryIdentity reads this off the arm entries. Supplied, it is taken + # whole and a field it does not carry is recorded as null, because falling back to this + # process for that one field is exactly the substitution the parameter exists to stop. + # Everything else in the entry still describes the process that writes it. + [object] $Identity ) - $dir = Join-Path $PSScriptRoot 'results' - if (-not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir | Out-Null } + $dir = $ResultsDirectory + if (-not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir -Force | Out-Null } $file = Join-Path $dir "$Benchmark.json" $entries = @() if (Test-Path $file) { $entries = @(Get-Content $file -Raw | ConvertFrom-Json) } @@ -359,8 +827,21 @@ function Write-BenchResult { $wall = if ($WallMs -gt 0) { $WallMs } elseif ($Result.WallMs -and [long]$Result.WallMs -gt 0) { [long]$Result.WallMs } else { 0 } - try { - $t = Get-MgxTelemetry -ErrorAction Stop + # The counters are snapshotted before the tenant facts below, which cost one request the + # first time a process records: what gets recorded has to describe the benchmark and + # nothing else. + $t = $null + try { $t = Get-MgxTelemetry -ErrorAction Stop } + catch { + # A benchmark that does not load Mgx (the bare-SDK comparison arms) has no telemetry. + # That is expected; record its absence rather than failing the run. + $t = $null + } + + $fact = Get-BenchTenantFact + $documentedBudget = $fact.Budget?.RuPerSecond + + if ($t) { $telemetry = [pscustomobject]@{ ResourceUnits = $t.ResourceUnitsConsumed TotalRequests = $t.Requests @@ -406,26 +887,51 @@ function Write-BenchResult { RuPerSecond = $(if ($wall -gt 0) { [math]::Round($t.ResourceUnitsConsumed / ($wall / 1000), 1) } else { $null }) - # The documented budget is 8,000 RU per 10s per application+tenant pair for tenants - # above 500 users, i.e. 800 RU/s. Recorded as a ratio so a run that approaches the - # ceiling is obvious without re-deriving the arithmetic each time. + # Recorded as a ratio so a run that approaches the ceiling is obvious without + # re-deriving the arithmetic each time. Against this tenant's documented budget + # where the tier is known, and against the 800 RU/s of a large tenant where it is + # not - BudgetBasis says which, because the same ratio over two denominators is two + # different claims. BudgetFraction = $(if ($wall -gt 0) { - [math]::Round(($t.ResourceUnitsConsumed / ($wall / 1000)) / 800, 3) + $denominator = if ($null -ne $documentedBudget) { $documentedBudget } else { 800 } + [math]::Round(($t.ResourceUnitsConsumed / ($wall / 1000)) / $denominator, 3) } else { $null }) + BudgetBasis = $(if ($wall -le 0) { $null } + elseif ($null -ne $documentedBudget) { 'documented' } + else { 'assumed-800' }) } } - catch { - # A benchmark that does not load Mgx (the bare-SDK comparison arms) has no telemetry. - # That is expected; record its absence rather than failing the run. - $telemetry = $null + + # The telemetry block above keeps this process's own denominator whatever -Identity says: + # its numerator is this process's counters, and a draw measured here against a budget + # documented elsewhere is neither run's number. + $stamp = $Identity + if (-not $PSBoundParameters.ContainsKey('Identity')) { + $stamp = [pscustomobject]@{ + MgxVersion = (Get-Module Mgx -ErrorAction SilentlyContinue)?.Version?.ToString() + SdkVersion = (Get-Module Microsoft.Graph.Authentication -ErrorAction SilentlyContinue)?.Version?.ToString() + TenantId = $fact.TenantId + DirectoryObjectCount = $fact.DirectoryObjectCount + DocumentedBudgetRuPerSecond = $documentedBudget + } } $meta = [pscustomobject]@{ Result = $Result Telemetry = $telemetry - MgxVersion = (Get-Module Mgx -ErrorAction SilentlyContinue)?.Version?.ToString() - SdkVersion = (Get-Module Microsoft.Graph.Authentication -ErrorAction SilentlyContinue)?.Version?.ToString() + MgxVersion = Get-BenchIdentityField -Node $stamp -Name 'MgxVersion' + SdkVersion = Get-BenchIdentityField -Node $stamp -Name 'SdkVersion' PSVersion = $PSVersionTable.PSVersion.ToString() + # Which tenant produced the number, how big it is, and what the documentation budgets a + # tenant that size at. Null together when there was no session behind the numbers. + TenantId = Get-BenchIdentityField -Node $stamp -Name 'TenantId' + DirectoryObjectCount = Get-BenchIdentityField -Node $stamp -Name 'DirectoryObjectCount' + DocumentedBudgetRuPerSecond = Get-BenchIdentityField -Node $stamp -Name 'DocumentedBudgetRuPerSecond' + # Which tree produced it. Versions are identical across working trees. + Commit = Get-BenchCommit + # First-class, not only inside the telemetry block: a scenario that loads no Mgx records + # no telemetry at all, and its wall clock is still the figure it is read on. + WallMs = $(if ($wall -gt 0) { $wall } else { $null }) RecordedAt = (Get-Date).ToString('o') } # -InputObject (not pipeline) so a single-element array still serializes as a JSON array @@ -433,3 +939,1029 @@ function Write-BenchResult { ConvertTo-Json -InputObject $all -Depth 12 | Set-Content -Path $file Write-Host " result appended to $file" } + +# --- The baseline snapshot ------------------------------------------------------------------ +# +# results/.json is the append-only per-run log: every run adds an entry and nothing +# is ever promoted out of it, so a number measured on a laptop in August is indistinguishable +# from the one a release was signed off against - and .gitignore keeps that whole directory +# local, so nobody else sees either. baseline.json beside it is the pinned snapshot: the newest +# entry of each named scenario, reduced to the figures that scenario exists to produce, and +# committed, so a later run has something to be read against. +# +# One table names those figures. The FIRST metric of a scenario is its headline - the figure +# that scenario is read on; the rest ride along so the committed file says what the run looked +# like without anyone re-deriving it from a log. Everything that writes or reads a baseline goes +# through this table, so a scenario cannot be pinned on one figure and read on another. +# +# Every headline is lower-is-better and strictly positive. That is deliberate: a percentage +# delta on a figure that can cross zero flips its own sign - 09's checkpoint overhead comes out +# slightly negative on a quiet tenant - so such a figure is recorded and never made a headline. +function Get-BenchScenario { + <# + .SYNOPSIS + The benchmark scenarios that can be promoted into baseline.json, and the metrics of each. + + .DESCRIPTION + Each scenario carries the script that produces it and an ordered list of metrics. A + metric names its unit and one or more Sources: dotted paths into the recorded Result + object, tried in order, the first that resolves winning. Scenario 10 needs the list - + run through run.ps1 it records a two-arm summary, run standalone with -Mode it records + one arm - and a Scale brings a path recorded in another unit onto the metric's own. + + Discriminator, where a scenario has one, is a path whose value is recorded beside the + metrics. 07 and 10 each record a Mode, and a number off the paced arm is not the same + number as one off the unpaced arm: reading one against the other is a wrong answer + rather than a missing one. + + .PARAMETER Id + Scenario ids to return, in the order given. Omit for all of them. An id with no + definition is skipped with a warning rather than throwing: a promotion at the end of a + two-hour suite must not lose the scenarios it does know because of one it does not. + #> + [CmdletBinding()] + param([string[]] $Id) + + $table = [ordered]@{ + '01' = @{ + Script = '01-list-users.ps1' + Metrics = @( + @{ Name = 'MgxListMs'; Unit = 'ms'; Sources = @(@{ Path = 'mgx.Median.ElapsedMs' }) } + @{ Name = 'MgxTimeToFirstMs'; Unit = 'ms'; Sources = @(@{ Path = 'ttfrMgx.Median.ElapsedMs' }) } + @{ Name = 'MgxPeakWorkingSetMB'; Unit = 'MB'; Sources = @(@{ Path = 'mgx.Median.PeakWorkingSetMB' }) } + ) + } + '02' = @{ + Script = '02-fanout-lookup.ps1' + Metrics = @( + @{ Name = 'MgxFanoutMs'; Unit = 'ms'; Sources = @(@{ Path = 'mgx.Median.ElapsedMs' }) } + @{ Name = 'MgxPeakWorkingSetMB'; Unit = 'MB'; Sources = @(@{ Path = 'mgx.Median.PeakWorkingSetMB' }) } + ) + } + '03' = @{ + Script = '03-user-report.ps1' + Metrics = @( + @{ Name = 'MgxReportMs'; Unit = 'ms'; Sources = @(@{ Path = 'mgx.Median.ElapsedMs' }) } + @{ Name = 'MgxPeakWorkingSetMB'; Unit = 'MB'; Sources = @(@{ Path = 'mgx.Median.PeakWorkingSetMB' }) } + ) + } + '04' = @{ + Script = '04-batch-create.ps1' + Metrics = @( + @{ Name = 'MgxBatchCreateMs'; Unit = 'ms'; Sources = @(@{ Path = 'mgxCreate.ElapsedMs' }) } + @{ Name = 'MgxBatchPatchMs'; Unit = 'ms'; Sources = @(@{ Path = 'batchPatch.ElapsedMs' }) } + @{ Name = 'MgxFanoutPatchMs'; Unit = 'ms'; Sources = @(@{ Path = 'fanoutPatch.ElapsedMs' }) } + ) + } + '05' = @{ + # Memory is the claim here, not speed, so peak working set is the headline and the + # wall time rides along. The managed-heap delta is the figure the README quotes for + # the streaming arms, and it sits near zero by design - too near to carry a + # percentage, which is why it is recorded rather than diffed. + Script = '05-memory-export.ps1' + Metrics = @( + @{ Name = 'MgxExportPeakWorkingSetMB'; Unit = 'MB'; Sources = @(@{ Path = 'mgxExport.PeakWorkingSetMB' }) } + @{ Name = 'MgxExportMs'; Unit = 'ms'; Sources = @(@{ Path = 'mgxExport.ElapsedMs' }) } + @{ Name = 'MgxExportManagedHeapDeltaMB'; Unit = 'MB'; Sources = @(@{ Path = 'mgxExport.ManagedHeapDeltaMB' }) } + ) + } + '06' = @{ + Script = '06-fault-gauntlet.ps1' + Metrics = @( + @{ Name = 'MgxFanoutMs'; Unit = 'ms'; Sources = @(@{ Path = 'mgx.ElapsedMs' }) } + @{ Name = 'SdkMgxMs'; Unit = 'ms'; Sources = @(@{ Path = 'sdkMgx.ElapsedMs' }) } + @{ Name = 'MgxCompleted'; Unit = ''; Sources = @(@{ Path = 'mgx.Output.ok' }) } + @{ Name = 'MgxFailed'; Unit = ''; Sources = @(@{ Path = 'mgx.Output.failed' }) } + ) + } + '07' = @{ + Script = '07-adaptive-pacing.ps1' + Discriminator = 'Mode' + Metrics = @( + @{ Name = 'ArmMs'; Unit = 'ms'; Sources = @(@{ Path = 'ElapsedMs' }) } + @{ Name = 'ArmFailed'; Unit = ''; Sources = @(@{ Path = 'Output.failed' }) } + ) + } + '08' = @{ + Script = '08-delta-sync.ps1' + Metrics = @( + @{ Name = 'InitialSyncMs'; Unit = 'ms'; Sources = @(@{ Path = 'initial.ElapsedMs' }) } + @{ Name = 'IncrementalSyncMs'; Unit = 'ms'; Sources = @(@{ Path = 'incremental.ElapsedMs' }) } + @{ Name = 'SteadySyncMs'; Unit = 'ms'; Sources = @(@{ Path = 'steady.ElapsedMs' }) } + @{ Name = 'InitialItems'; Unit = ''; Sources = @(@{ Path = 'initial.Output.items' }) } + ) + } + '09' = @{ + # The checkpointed export is the headline: it is what checkpointing costs, in the + # same units as everything else. The overhead percentage the script derives from it + # reads better and is recorded, but it can land either side of zero on a quiet + # tenant, so it is not what gets diffed. + Script = '09-kill-resume.ps1' + Metrics = @( + @{ Name = 'CheckpointedExportMs'; Unit = 'ms'; Sources = @(@{ Path = 'checkpointed.ElapsedMs' }) } + @{ Name = 'BaselineExportMs'; Unit = 'ms'; Sources = @(@{ Path = 'baseline.ElapsedMs' }) } + @{ Name = 'ResumeMs'; Unit = 'ms'; Sources = @(@{ Path = 'resume.ElapsedMs' }) } + @{ Name = 'CheckpointOverheadPct'; Unit = '%'; Sources = @(@{ Path = 'verification.overheadPct' }) } + @{ Name = 'DuplicateIds'; Unit = ''; Sources = @(@{ Path = 'verification.duplicateIds' }) } + ) + } + '10' = @{ + Script = '10-pacing-under-real-throttling.ps1' + Discriminator = 'Mode' + Metrics = @( + @{ Name = 'WallSeconds'; Unit = 's'; Sources = @(@{ Path = 'Arms.paced.Seconds' }, @{ Path = 'WallMs'; Scale = 0.001 }) } + @{ Name = 'UnpacedWallSeconds'; Unit = 's'; Sources = @(@{ Path = 'Arms.unpaced.Seconds' }) } + @{ Name = 'ThrottleRetries'; Unit = ''; Sources = @(@{ Path = 'Arms.paced.ThrottleRetries' }, @{ Path = 'ThrottleRetries' }) } + ) + } + '18' = @{ + # A latency distribution, so the headline is the whole window's p90 of the HTTP + # round trip - lower is better and it cannot reach zero. The clamp ratio the script + # derives from the five-minute buckets is the figure the claim is stated in, and it + # rides along rather than leading: a window that never stretched reports 1.0, which + # is the good outcome and would read as an improvement against anything pinned. + # P90CallElapsedMs is the same window measured at the caller, pacer waits included; + # it is here so the two are never mistaken for one another. + # + # Discriminated on the preceding activity rather than on the drive: the whole point + # of the measurement is that the same drive answers differently depending on what it + # has just been through, so two runs are comparable only where the operator states + # the same history for both. The drive itself would be the other candidate and is + # deliberately not used - baseline.json is committed, and a drive id in it would put + # a tenant's identifiers in the repository, which is why the live suite passes one + # through MGX_LIVE_CONTENT_URI instead. The entry's own Result records the drive. + Script = '18-spo-latency-clamp.ps1' + Discriminator = 'PrecedingActivity' + Metrics = @( + @{ Name = 'P90ElapsedMs'; Unit = 'ms'; Sources = @(@{ Path = 'Overall.P90ElapsedMs' }) } + @{ Name = 'P50ElapsedMs'; Unit = 'ms'; Sources = @(@{ Path = 'Overall.P50ElapsedMs' }) } + @{ Name = 'P99ElapsedMs'; Unit = 'ms'; Sources = @(@{ Path = 'Overall.P99ElapsedMs' }) } + @{ Name = 'P90CallElapsedMs'; Unit = 'ms'; Sources = @(@{ Path = 'Overall.P90CallElapsedMs' }) } + @{ Name = 'ClampRatio'; Unit = 'x'; Sources = @(@{ Path = 'Overall.ClampRatio' }) } + @{ Name = 'Throttled429'; Unit = ''; Sources = @(@{ Path = 'Overall.Throttled429' }) } + ) + } + } + + $wanted = if ($Id) { $Id } else { @($table.Keys) } + foreach ($key in $wanted) { + if (-not $table.Contains($key)) { + Write-Warning "benchmark '$key' has no baseline metrics in Get-BenchScenario - skipped." + continue + } + $s = $table[$key] + [pscustomobject]@{ + Id = $key + Script = $s.Script + # Write-BenchResult names the file after the benchmark, and every benchmark passes + # its own script name without the extension. + ResultFile = [System.IO.Path]::GetFileNameWithoutExtension($s.Script) + '.json' + Discriminator = $s.Discriminator + Metrics = $s.Metrics + } + } +} + +# Walks a dotted path into a recorded Result object. Returns $null for any segment that is not +# there, which is how a metric reports itself absent rather than throwing halfway through a +# promotion. +function Get-BenchResultValue { + param( + [object] $Node, + [Parameter(Mandatory)] [string] $Path + ) + foreach ($segment in ($Path -split '\.')) { + if ($null -eq $Node) { return $null } + $property = $Node.PSObject.Properties[$segment] + if (-not $property) { return $null } + $Node = $property.Value + } + $Node +} + +# The newest entry of one benchmark's results file, or $null. -Since rejects an entry older than +# the run that asked for it: a benchmark that died before recording leaves the previous run's +# entry newest, and promoting or comparing that reports a stale number as a fresh one. +function Get-BenchLatestResult { + param( + [Parameter(Mandatory)] [string] $ResultFile, + [datetime] $Since = [datetime]::MinValue + ) + if (-not (Test-Path $ResultFile)) { return $null } + $entries = @(Get-Content $ResultFile -Raw | ConvertFrom-Json) + if ($entries.Count -eq 0) { return $null } + $entry = $entries[-1] + + # ConvertFrom-Json turns a round-trip timestamp into a DateTime on its own; a file written + # by another host may still hand back the string it was written as. + $recorded = $entry.RecordedAt + if ($recorded -isnot [datetime]) { + try { + $recorded = [datetime]::Parse([string]$recorded, [cultureinfo]::InvariantCulture, + [System.Globalization.DateTimeStyles]::RoundtripKind) + } + catch { $recorded = [datetime]::MinValue } + } + if ($recorded -lt $Since) { return $null } + $entry +} + +# A value is a measurement only when it is a finite number: one of PowerShell's numeric types, +# or a string that parses as one under the invariant culture the benchmark process is pinned +# to. [double] coerces almost anything without throwing - '' becomes 0, $true becomes 1 - so a +# source that never recorded a number would otherwise be pinned as though it had. NaN and the +# infinities pass both tests unless they are refused explicitly: a typed double or single can +# already hold one mid-computation, and NumberStyles.Float parses the strings "NaN", +# "Infinity" and "-Infinity" as valid doubles, so each arm is checked for +# [double]::IsFinite before the value is accepted. +function Test-BenchNumericValue([object] $Value) { + if ($Value -is [int] -or $Value -is [long] -or $Value -is [decimal]) { + return $true + } + if ($Value -is [double] -or $Value -is [single]) { + return [double]::IsFinite([double]$Value) + } + $parsed = 0.0 + $Value -is [string] -and [double]::TryParse($Value, [System.Globalization.NumberStyles]::Float, + [System.Globalization.CultureInfo]::InvariantCulture, [ref] $parsed) -and [double]::IsFinite($parsed) +} + +# Names what a value not accepted by Test-BenchNumericValue actually held, for a warning or a +# comparison note to quote. A string or a boolean prints the way PowerShell already prints it +# elsewhere in this file ('INCONCLUSIVE', 'True'). An array, an object, or JSON null read back +# off a stale baseline.json has no such print of its own worth trusting - $null and '' would +# both print as nothing, and an array joins its elements with spaces - so each renders the way +# ConvertTo-Json -Compress would, the same shape the file was written in. +function ConvertTo-BenchDisplayValue([object] $Value) { + if ($null -eq $Value -or $Value -is [array] -or $Value -is [System.Collections.IDictionary] -or $Value -is [pscustomobject]) { + return ConvertTo-Json -InputObject $Value -Compress -Depth 5 + } + "$Value" +} + +# Reads one scenario's metrics out of a recorded entry: an ordered map of +# name -> @{ Value; Unit; Path }, plus the discriminator value where the scenario has one. +function Get-BenchScenarioMetric { + param( + [Parameter(Mandatory)] [object] $Scenario, + [Parameter(Mandatory)] [object] $Entry + ) + $metrics = [ordered]@{} + foreach ($m in $Scenario.Metrics) { + $sources = @($m.Sources) + $resolved = $false + $notNumbers = [System.Collections.Generic.List[object]]::new() + for ($i = 0; $i -lt $sources.Count; $i++) { + $source = $sources[$i] + $value = Get-BenchResultValue -Node $Entry.Result -Path $source.Path + if ($null -eq $value) { continue } + if (-not (Test-BenchNumericValue $value)) { + $notNumbers.Add([pscustomobject]@{ + Description = "'$($source.Path)' holds '$(ConvertTo-BenchDisplayValue $value)', which is not a number" + }) + continue + } + $scale = if ($source.ContainsKey('Scale')) { [double]$source.Scale } else { 1.0 } + $number = if ($value -is [string]) { [double]::Parse($value, [System.Globalization.CultureInfo]::InvariantCulture) } else { [double]$value } + $scaled = $number * $scale + # A source that read within double's range can still overflow once Scale is + # applied - the table's own Scale (10's WallMs, 0.001) shrinks, so this cannot + # happen through it today, but a larger Scale would push a source that was itself + # finite past it. What must be finite is the multiply's result, not the raw + # value, so it is checked after scaling rather than before - and a source that + # fails here is a source that held a non-number just as surely as 'INCONCLUSIVE' + # is: the walk tries the next one and says so. Named the same way its sibling + # above names a raw non-number, so two overflowing sources on the same metric + # can still be told apart by path. + if (-not [double]::IsFinite($scaled)) { + $notNumbers.Add([pscustomobject]@{ + Description = "'$($source.Path)' holds '$(ConvertTo-BenchDisplayValue $value)', which scaled by $scale is not finite" + }) + continue + } + $metrics[$m.Name] = [pscustomobject]@{ + Value = [math]::Round($scaled, 3) + Unit = $m.Unit + Path = $source.Path + } + $resolved = $true + break + } + if ($resolved) { + # Every source ahead of the one that resolved held something that was not a + # number - a first source that recorded 'INCONCLUSIVE' before a second recorded + # the real figure, say. The next source covering for it should not read as + # silence, so each earns its own warning naming what it held. + foreach ($n in $notNumbers) { + Write-Warning "$($Scenario.Id): '$($m.Name)' - $($n.Description) - trying the next source." + } + } + # A source with nothing at its path was never written - 10's single-arm entries never + # write the other arm's, and that is not a metric gone missing, it is one this entry + # was never going to carry. Only a source that held something, and held a value that + # is not a number, earns a warning - naming what it held. + elseif ($notNumbers.Count -gt 0) { + $held = ($notNumbers | ForEach-Object { $_.Description }) -join '; ' + Write-Warning "$($Scenario.Id): '$($m.Name)' not recorded - $held." + } + } + $discriminator = $null + if ($Scenario.Discriminator) { + $discriminator = Get-BenchResultValue -Node $Entry.Result -Path $Scenario.Discriminator + } + [pscustomobject]@{ Metrics = $metrics; Discriminator = $discriminator } +} + +function Export-BenchBaseline { + <# + .SYNOPSIS + Promotes the newest results entry of each named benchmark into baseline.json. + + .DESCRIPTION + Writes one record per scenario: its id, the script that produced it, the metrics + Get-BenchScenario names for it, and the Mgx/SDK/PowerShell versions and timestamp + Write-BenchResult already recorded alongside the numbers. Scenarios already in the file + and not named here are left as they are, so promoting one scenario does not discard the + six the release baseline is otherwise made of. + + A scenario whose entry records Result.Conclusive false is reported and left out: the run + itself says it did not measure what the scenario exists to measure, and whatever row was + already pinned for it stays. Where such an entry also carries Result.InconclusiveReason, + the warning quotes it. + + .PARAMETER Benchmark + Scenario ids to promote. + + .PARAMETER Since + Ignore entries recorded before this. run.ps1 passes the moment the suite started, so a + benchmark that recorded nothing is reported rather than promoted from a previous run. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string[]] $Benchmark, + [string] $ResultsDirectory = (Join-Path $PSScriptRoot 'results'), + [string] $Path = (Join-Path $PSScriptRoot 'baseline.json'), + [datetime] $Since = [datetime]::MinValue + ) + + $scenarios = [ordered]@{} + if (Test-Path $Path) { + $existing = Get-Content $Path -Raw | ConvertFrom-Json + if ($existing.Scenarios) { + foreach ($p in $existing.Scenarios.PSObject.Properties) { $scenarios[$p.Name] = $p.Value } + } + } + + $promoted = 0 + foreach ($scenario in (Get-BenchScenario -Id $Benchmark)) { + $file = Join-Path $ResultsDirectory $scenario.ResultFile + $entry = Get-BenchLatestResult -ResultFile $file -Since $Since + if (-not $entry) { + Write-Warning "benchmark $($scenario.Id) recorded no result in $file - not promoted." + continue + } + # A run that says it measured nothing is not a baseline. 10 is the shape of it: it + # records Result.Conclusive false when its unpaced arm was never refused, which means + # the throttled regime it exists to compare against was never reached - and the wall + # times it did record are then two unthrottled runs wearing a throttling benchmark's + # name. Promoting that pins a number every later run is read against, so the row it + # would overwrite is kept instead: a stale conclusive figure is still a measurement, + # and none at all is what the comparison already reports as unmeasured. A scenario that + # records no verdict at all promotes as before - the field is a claim, not a + # requirement. + $conclusive = Get-BenchResultValue -Node $entry.Result -Path 'Conclusive' + if ($null -ne $conclusive -and -not $conclusive) { + $why = Get-BenchResultValue -Node $entry.Result -Path 'InconclusiveReason' + if ([string]::IsNullOrWhiteSpace([string]$why)) { $why = 'Conclusive is false' } + $kept = if ($scenarios.Contains($scenario.Id)) { + '; the row already in the file is left as it was' + } else { '' } + Write-Warning "benchmark $($scenario.Id) recorded an inconclusive run ($why) - not promoted$kept." + continue + } + $read = Get-BenchScenarioMetric -Scenario $scenario -Entry $entry + if ($read.Metrics.Count -eq 0) { + Write-Warning "benchmark $($scenario.Id) recorded an entry carrying none of its metrics - not promoted." + continue + } + # The headline is the first metric of the scenario's own row, and only where that + # metric resolved. Taking the first metric that HAPPENED to resolve promotes whichever + # one came next instead - a different figure, in a different unit, read as though it + # were the one the scenario is pinned on. 09 is the shape of it: with its leading + # sources absent from an entry, the next resolvable metric is CheckpointOverheadPct, a + # percentage that lands either side of zero on a quiet tenant. An entry that did not + # record its own headline is pinned with the metrics it did record and no headline, so + # the comparison reports it rather than reading something else in its place. + $tableHeadline = @($scenario.Metrics)[0].Name + $headline = if ($read.Metrics.Contains($tableHeadline)) { $tableHeadline } else { $null } + $scenarios[$scenario.Id] = [pscustomobject]@{ + Scenario = $scenario.Id + Script = $scenario.Script + Headline = $headline + Discriminator = $read.Discriminator + Metrics = $read.Metrics + MgxVersion = $entry.MgxVersion + SdkVersion = $entry.SdkVersion + PSVersion = $entry.PSVersion + RecordedAt = $entry.RecordedAt + PromotedAt = (Get-Date).ToString('o') + } + $promoted++ + if ($headline) { + Write-Host (" {0}: {1} = {2}{3}" -f $scenario.Id, $headline, + $read.Metrics[$headline].Value, $read.Metrics[$headline].Unit) + } + else { + Write-Warning "benchmark $($scenario.Id) recorded no '$tableHeadline' - pinned with its other metrics and no headline; the comparison will report it as not comparable." + } + } + + if ($promoted -eq 0) { + Write-Warning "nothing promoted; $Path left as it was." + return + } + + $sorted = [ordered]@{} + foreach ($key in (@($scenarios.Keys) | Sort-Object)) { $sorted[$key] = $scenarios[$key] } + $document = [pscustomobject]@{ + SchemaVersion = 1 + GeneratedAt = (Get-Date).ToString('o') + Scenarios = $sorted + } + ConvertTo-Json -InputObject $document -Depth 10 | Set-Content -Path $Path + Write-Host " baseline written to $Path ($promoted promoted, $($sorted.Count) scenario(s) in the file)" +} + +function Compare-BenchBaseline { + <# + .SYNOPSIS + Reads a finished run against baseline.json and prints a delta per scenario. + + .DESCRIPTION + Reports; never fails the run. The tenant's own throttling ceiling has moved 40% between + consecutive days, so a hard gate on these numbers is noise wearing a verdict's clothes. + A scenario with no baseline entry prints as unmeasured, and one whose fresh entry is + missing or not comparable prints as that - a silent skip is how a regression hides. + + Emits one row object per scenario as well as printing them, so a caller can assert. + + .PARAMETER ThresholdPercent + How far the headline metric may move before the row is flagged. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string[]] $Benchmark, + [string] $ResultsDirectory = (Join-Path $PSScriptRoot 'results'), + [string] $BaselinePath = (Join-Path $PSScriptRoot 'baseline.json'), + [datetime] $Since = [datetime]::MinValue, + # Proposed, not derived: 50%. The tenant's throttling ceiling has been seen to move 40% + # between consecutive days, so anything tighter flags the weather. 50% sits just above + # that band, which makes a flag mean "further than the worst environmental swing anyone + # has recorded here" - a weak claim, but a true one. Once baseline.json has been + # re-recorded a few times, a per-scenario band derived from those runs replaces it. + [double] $ThresholdPercent = 50 + ) + + if (-not (Test-Path $BaselinePath)) { + Write-Host '' + Write-Host '=== BASELINE COMPARISON ===' + Write-Host "no baseline at $BaselinePath - nothing to read this run against." + Write-Host 'record one with ./run.ps1 -RecordBaseline when a run is worth pinning.' + return + } + + $baseline = Get-Content $BaselinePath -Raw | ConvertFrom-Json + if (-not $baseline.Scenarios) { + throw "Baseline '$BaselinePath' carries no Scenarios; record one with -RecordBaseline before comparing." + } + if ($baseline.SchemaVersion -ne 1) { + throw "Baseline '$BaselinePath' is schema $($baseline.SchemaVersion); this suite reads schema 1." + } + Write-Host '' + Write-Host ('=== BASELINE COMPARISON (flagged past {0:F0}%) ===' -f $ThresholdPercent) + Write-Host ('{0,-4} {1,-28} {2,14} {3,14} {4,9} {5}' -f 'id', 'metric', 'baseline', 'this run', 'delta', 'note') + + $rows = @() + foreach ($scenario in (Get-BenchScenario -Id $Benchmark)) { + $pinnedProperty = $baseline.Scenarios.PSObject.Properties[$scenario.Id] + $pinned = if ($pinnedProperty) { $pinnedProperty.Value } else { $null } + $entry = Get-BenchLatestResult -ResultFile (Join-Path $ResultsDirectory $scenario.ResultFile) -Since $Since + $fresh = if ($entry) { Get-BenchScenarioMetric -Scenario $scenario -Entry $entry } else { $null } + + $row = [pscustomobject]@{ + Scenario = $scenario.Id + Metric = $null + Unit = $null + Baseline = $null + Current = $null + DeltaPct = $null + Status = $null + Flagged = $false + Note = $null + } + + if (-not $pinned) { + $row.Status = 'unmeasured' + $row.Note = 'unmeasured - no baseline entry' + } + elseif (-not $fresh) { + $row.Status = 'no result' + $row.Note = 'no result recorded by this run' + } + elseif (-not $pinned.Headline) { + # Pinned without one: the promoted run did not record the metric its scenario is + # read on. The metrics beside it are not substitutes - each is a different figure - + # so the row says there is nothing to compare instead of comparing something else. + $row.Status = 'not comparable' + $row.Note = 'not comparable - the baseline entry has no headline, so its run never recorded the metric this scenario is read on' + } + else { + $row.Metric = $pinned.Headline + $baselineProperty = $pinned.Metrics.PSObject.Properties[$pinned.Headline] + $b = if ($baselineProperty) { $baselineProperty.Value } else { $null } + $f = $fresh.Metrics[$pinned.Headline] + + if (-not $b -or -not $f) { + $row.Status = 'unmeasured' + $row.Note = "unmeasured - '$($pinned.Headline)' is not in both" + } + elseif ($b.Path -ne $f.Path -or $b.Unit -ne $f.Unit) { + $row.Status = 'not comparable' + $row.Note = "not comparable - baseline read $($b.Path), this run $($f.Path)" + } + elseif ($scenario.Discriminator -and + [string]$pinned.Discriminator -ne [string]$fresh.Discriminator) { + $row.Status = 'not comparable' + $row.Note = "not comparable - $($scenario.Discriminator) was '$($pinned.Discriminator)', this run '$($fresh.Discriminator)'" + } + else { + $row.Unit = $b.Unit + # [double] coerces almost anything without throwing - '' becomes 0, $true + # becomes 1 - so a stale baseline.json holding something that was never a + # number would cast into a plausible-looking figure instead of being refused. + # Test-BenchNumericValue is asked first, on the raw value, before either side + # is cast - the same gate a raw source reading already goes through in + # Get-BenchScenarioMetric. It also refuses NaN and the infinities outright, so + # a value already typed [double] or [single] is exempted from it here: a bare + # JSON NaN or Infinity token round-trips through ConvertFrom-Json as a genuine + # typed double, and that is a number for the cast and the finiteness check + # below to refuse, not one for this gate to call not a number at all. + $bIsTyped = $b.Value -is [double] -or $b.Value -is [single] + $fIsTyped = $f.Value -is [double] -or $f.Value -is [single] + if (-not $bIsTyped -and -not (Test-BenchNumericValue $b.Value)) { + $row.Status = 'not comparable' + $row.Note = "not comparable - the baseline value is '$(ConvertTo-BenchDisplayValue $b.Value)', which is not a number" + } + elseif (-not $fIsTyped -and -not (Test-BenchNumericValue $f.Value)) { + $row.Status = 'not comparable' + $row.Note = "not comparable - this run's value is '$(ConvertTo-BenchDisplayValue $f.Value)', which is not a number" + } + else { + # [double]$b.Value already reads "NaN", "Infinity" and "-Infinity" as the + # doubles they name - PowerShell's own numeric conversion, not + # [double]::Parse, accepts those three tokens under any culture - so a + # stale baseline.json recorded before Export-BenchBaseline refused a + # non-finite measurement, or a bare JSON NaN or Infinity token the gate + # above let through untested, casts here to a genuine non-finite double + # rather than throwing. Nothing pins a non-finite Current today, but the + # same cast is used for it, so both are checked before either reaches the + # arithmetic below. + $row.Baseline = [double]$b.Value + $row.Current = [double]$f.Value + if (-not [double]::IsFinite($row.Baseline)) { + $row.Status = 'not comparable' + $row.Note = "not comparable - the baseline value is $($row.Baseline)" + } + elseif (-not [double]::IsFinite($row.Current)) { + $row.Status = 'not comparable' + $row.Note = "not comparable - this run's value is $($row.Current)" + } + elseif ([math]::Abs($row.Baseline) -lt 1e-9) { + $row.Status = 'not comparable' + $row.Note = 'not comparable - the baseline value is zero' + } + elseif ($row.Baseline -lt 0) { + # A percentage delta over a negative denominator inverts its own sign: a + # figure that moved further below the baseline comes out positive and reads + # as the bad direction, and one that improved reads as a gain. Every + # headline in the table is meant to be strictly positive, so a negative one + # is a figure that was never fit to be a headline - which is a thing to say, + # not a delta to compute. + $row.Status = 'not comparable' + $row.Note = "not comparable - the baseline value is negative ($($row.Baseline)$($row.Unit)); a percentage delta over it would invert its own sign" + } + else { + $row.DeltaPct = [math]::Round((($row.Current - $row.Baseline) / $row.Baseline) * 100, 1) + $row.Status = 'compared' + # Lower is better for every headline in the table, so a positive delta is the + # bad direction. A move the other way is flagged too: a scenario that halves + # has usually measured less work rather than done the same work faster. + if ($row.DeltaPct -gt $ThresholdPercent) { + $row.Flagged = $true + $row.Note = 'REGRESSED' + } + elseif ($row.DeltaPct -lt -$ThresholdPercent) { + $row.Flagged = $true + $row.Note = 'IMPROVED - check it measured the same work' + } + } + } + } + } + + $show = { + param($value, $unit) + if ($null -eq $value) { '-' } else { '{0:F1}{1}' -f $value, $unit } + } + $color = if ($row.Flagged) { 'Yellow' } elseif ($row.Status -ne 'compared') { 'DarkGray' } else { 'Gray' } + Write-Host ('{0,-4} {1,-28} {2,14} {3,14} {4,9} {5}' -f ` + $row.Scenario, + $(if ($row.Metric) { $row.Metric } else { '-' }), + (& $show $row.Baseline $row.Unit), + (& $show $row.Current $row.Unit), + $(if ($null -ne $row.DeltaPct) { '{0:+0.0;-0.0;0.0}%' -f $row.DeltaPct } else { '-' }), + $(if ($row.Note) { $row.Note } else { $row.Status })) -ForegroundColor $color + + $rows += $row + } + + Write-Host 'reported, not gated: these numbers move with the tenant, so nothing here fails the run.' + $rows +} + +# --- Benchmark 18: the SPO latency-clamp summary, read from its checkpointed rows ------------- +# +# 18-spo-latency-clamp.ps1 used to build its buckets and totals while the load ran, from +# whatever rows happened to be in memory at each step. Measured against the run recorded +# 2026-09-10 (StartedAt 13:16:34, EndedAt 14:31:36, a 75-minute window with a 15-minute machine +# sleep in it): the Stopwatch the header timed itself with does not advance while suspended, so +# it printed "59.6 minute(s)" for a window that ran 75; the two buckets the sleep spanned were +# never created at all, because a live accumulator only ever holds a key for a minute a row +# actually arrived in; the printed table had no Errors column, so the 1,005 rows the sleep's +# breaker trip wrote were invisible next to the 429 count everyone was already looking at; and +# the result's PacingState is the pacer's one most-recent latency sample - that run's own last +# row happened to be a slow one, so the figure sitting beside Overall.P90ElapsedMs and +# ClampRatio read as the verdict when it was one call out of 5,919 in its bucket. +# +# The functions below read the rows file back rather than trust an accumulator that was live +# through all of that: every row the run checkpointed is on disk before the summary is built, +# so a bucket, a count or a headline computed from it is the same whether it runs while the load +# is finishing or an hour later against a copy. Get-ClampSummary is what the script calls once +# its load completes, and what a Pester test calls against a synthetic rows.jsonl it wrote +# itself - the same computation either way, rather than a live one and a tested one that can +# drift apart. + +function Get-ClampPercentile { + <# + .SYNOPSIS + Nearest-rank percentile: the value the tenant actually served, never an interpolation. + #> + [CmdletBinding()] + param([object[]] $Values, [double] $Percentile) + if (-not $Values -or @($Values).Count -eq 0) { return $null } + $sorted = @($Values | Sort-Object) + $rank = [int][math]::Ceiling(($Percentile / 100.0) * $sorted.Count) + if ($rank -lt 1) { $rank = 1 } + if ($rank -gt $sorted.Count) { $rank = $sorted.Count } + [math]::Round([double]$sorted[$rank - 1], 1) +} + +function Get-ClampClockGap { + <# + .SYNOPSIS + Whether the wall clock outran the run's Stopwatch by more than a bucket - a suspended + machine, not a pacer wait or a breaker hold, because a Stopwatch can only fail to + advance by not running. + + .PARAMETER StartedAt + .PARAMETER EndedAt + Get-Date at the start and end of the load, wall time. + + .PARAMETER WallMs + The load's own Stopwatch, in milliseconds - ElapsedMilliseconds, read after Stop(). + + .PARAMETER BucketMinutes + The run's bucket width, and the gap's own tolerance: a wall/Stopwatch drift inside one + bucket is ordinary scheduling noise, not a sleep. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [datetime] $StartedAt, + [Parameter(Mandatory)] [datetime] $EndedAt, + [Parameter(Mandatory)] [long] $WallMs, + [Parameter(Mandatory)] [ValidateRange(1, [int]::MaxValue)] [int] $BucketMinutes + ) + + $wallSpanMs = ($EndedAt - $StartedAt).TotalMilliseconds + $gapMs = $wallSpanMs - $WallMs + $gapMinutes = [math]::Round($gapMs / 60000.0, 1) + $refused = $gapMs -gt ($BucketMinutes * 60000.0) + + [pscustomobject]@{ + Refused = $refused + GapMinutes = $gapMinutes + Reason = $(if ($refused) { + "the machine was not awake for $gapMinutes minute(s) of the window; rows kept, summary withheld" + } else { $null }) + } +} + +function Get-ClampRows { + <# + .SYNOPSIS + Parses 18's checkpointed rows.jsonl - one call attempt per line - back into objects. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $Path) + + if (-not (Test-Path $Path)) { throw "Get-ClampRows: no rows file at '$Path'." } + @(Get-Content -Path $Path | Where-Object { $_ } | ForEach-Object { ConvertFrom-Json -InputObject $_ }) +} + +function Get-ClampRefusalKind { + <# + .SYNOPSIS + Groups a refusal message to what comes before its first colon - two "Circuit breaker + tripped: too many failures..." trips with different wait times group under the same + "Circuit breaker tripped" - or the whole message when it carries no colon at all, such + as "no response traced". + #> + [CmdletBinding()] + param([AllowNull()] [AllowEmptyString()] [string] $Message) + + if (-not $Message) { return $null } + (($Message -split ':', 2)[0]).Trim() +} + +function Get-ClampBuckets { + <# + .SYNOPSIS + 18's per-bucket table: every index from 0 through Ceiling(Minutes/BucketMinutes)-1, so + a bucket the run never wrote a row to is reported rather than silently missing. + + .DESCRIPTION + A row's own minute-offset from StartedAt decides its bucket, exactly as the live run + computes it - so a bucket built here from the checkpoint agrees with the one the run + would have shown if its accumulator had covered the whole window. The index range is + computed from Minutes and BucketMinutes rather than from the rows themselves: the rows + say which buckets got traffic, not how many the run was asked to cover, and the gap + between the two is exactly what a marked "no rows" bucket exists to say. + #> + [CmdletBinding()] + param( + [AllowNull()] [object[]] $Rows, + [Parameter(Mandatory)] [datetime] $StartedAt, + [Parameter(Mandatory)] [ValidateRange(1, [int]::MaxValue)] [int] $Minutes, + [Parameter(Mandatory)] [ValidateRange(1, [int]::MaxValue)] [int] $BucketMinutes + ) + + $buckets = [ordered]@{} + foreach ($row in @($Rows)) { + $minutesIn = ((Get-Date $row.Timestamp) - $StartedAt).TotalMinutes + if ($minutesIn -lt 0) { $minutesIn = 0 } + $index = [int][math]::Floor($minutesIn / $BucketMinutes) + $key = "b$index" + if (-not $buckets.Contains($key)) { + $buckets[$key] = [ordered]@{ + Bucket = $index; FromMinute = $index * $BucketMinutes; Calls = 0; Attempts = 0 + Ok = 0; Throttled429 = 0; Errors = 0 + ThrottlePercentageSeen = 0; ThrottleScopeSeen = 0; RetryAfterSeen = 0; RateLimitSeen = 0 + Latencies = [System.Collections.Generic.List[double]]::new() + Note = $null + } + } + $b = $buckets[$key] + $b.Attempts++ + if ($row.Attempt -eq 1) { $b.Calls++ } + if ($row.Status -ge 200 -and $row.Status -lt 300) { $b.Ok++ } + elseif ($row.Status -eq 429) { $b.Throttled429++ } + if ($row.Status -eq 0 -or $row.Error) { $b.Errors++ } + if ($row.ThrottleLimitPercentage) { $b.ThrottlePercentageSeen++ } + if ($row.ThrottleScope) { $b.ThrottleScopeSeen++ } + if ($row.RetryAfter) { $b.RetryAfterSeen++ } + if ($row.RateLimitHeader) { $b.RateLimitSeen++ } + if ($null -ne $row.ElapsedMs -and $row.Status -ge 200 -and $row.Status -lt 300) { + $b.Latencies.Add([double]$row.ElapsedMs) + } + } + + $bucketCount = [int][math]::Ceiling($Minutes / [double]$BucketMinutes) + for ($index = 0; $index -lt $bucketCount; $index++) { + $key = "b$index" + if (-not $buckets.Contains($key)) { + $buckets[$key] = [ordered]@{ + Bucket = $index; FromMinute = $index * $BucketMinutes; Calls = 0; Attempts = 0 + Ok = 0; Throttled429 = 0; Errors = 0 + ThrottlePercentageSeen = 0; ThrottleScopeSeen = 0; RetryAfterSeen = 0; RateLimitSeen = 0 + Latencies = [System.Collections.Generic.List[double]]::new() + Note = 'no rows' + } + } + } + + @( + foreach ($key in $buckets.Keys) { + $b = $buckets[$key] + [pscustomobject]@{ + Bucket = $b.Bucket + FromMinute = $b.FromMinute + Calls = $b.Calls + Attempts = $b.Attempts + Ok = $b.Ok + Throttled429 = $b.Throttled429 + Errors = $b.Errors + P50ElapsedMs = Get-ClampPercentile -Values $b.Latencies -Percentile 50 + P90ElapsedMs = Get-ClampPercentile -Values $b.Latencies -Percentile 90 + P99ElapsedMs = Get-ClampPercentile -Values $b.Latencies -Percentile 99 + ThrottlePercentageSeen = $b.ThrottlePercentageSeen + ThrottleScopeSeen = $b.ThrottleScopeSeen + RetryAfterSeen = $b.RetryAfterSeen + RateLimitSeen = $b.RateLimitSeen + Note = $b.Note + } + } + ) | Sort-Object Bucket +} + +function Get-ClampOverall { + <# + .SYNOPSIS + 18's whole-window totals: outcome counts, percentiles, throttle-signal counts, the + clamp ratio, and refusals grouped by kind. + + .PARAMETER Buckets + Get-ClampBuckets' own output, for the clamp ratio - worst bucket P90 over the first + bucket's - which has to read the same buckets the table beside it prints. + #> + [CmdletBinding()] + param( + [AllowNull()] [object[]] $Rows, + [Parameter(Mandatory)] [AllowNull()] [object[]] $Buckets + ) + + $rowList = @($Rows) + $latencies = [System.Collections.Generic.List[double]]::new() + $callLatencies = [System.Collections.Generic.List[double]]::new() + $perWorker = @{} + $callCount = 0; $okCount = 0; $throttled = 0; $errorCount = 0 + $sawPercentage = 0; $sawScope = 0; $sawRetryAfter = 0; $sawRateLimit = 0 + $refusals = [ordered]@{} + + foreach ($row in $rowList) { + if ($row.Attempt -eq 1) { $callCount++ } + if ($row.Status -ge 200 -and $row.Status -lt 300) { $okCount++ } + elseif ($row.Status -eq 429) { $throttled++ } + if ($row.Status -eq 0 -or $row.Error) { + $errorCount++ + $kind = Get-ClampRefusalKind -Message $row.Error + if ($kind) { + if (-not $refusals.Contains($kind)) { $refusals[$kind] = 0 } + $refusals[$kind]++ + } + } + if ($row.ThrottleLimitPercentage) { $sawPercentage++ } + if ($row.ThrottleScope) { $sawScope++ } + if ($row.RetryAfter) { $sawRetryAfter++ } + if ($row.RateLimitHeader) { $sawRateLimit++ } + if (-not $perWorker.ContainsKey($row.Worker)) { $perWorker[$row.Worker] = 0 } + $perWorker[$row.Worker]++ + if ($null -ne $row.ElapsedMs -and $row.Status -ge 200 -and $row.Status -lt 300) { + $latencies.Add([double]$row.ElapsedMs) + } + if ($row.Attempt -eq 1 -and $null -ne $row.CallElapsedMs) { $callLatencies.Add([double]$row.CallElapsedMs) } + } + + $bucketList = @($Buckets) + $firstP90 = @($bucketList | Where-Object { $null -ne $_.P90ElapsedMs })[0]?.P90ElapsedMs + $worstP90 = ($bucketList | Where-Object { $null -ne $_.P90ElapsedMs } | + Measure-Object -Property P90ElapsedMs -Maximum).Maximum + $clampRatio = if ($firstP90 -and $firstP90 -gt 0 -and $null -ne $worstP90) { + [math]::Round($worstP90 / $firstP90, 2) + } else { $null } + + [ordered]@{ + Calls = $callCount + Attempts = $rowList.Count + Ok = $okCount + Throttled429 = $throttled + Errors = $errorCount + Workers = $perWorker.Count + P50ElapsedMs = Get-ClampPercentile -Values $latencies -Percentile 50 + P90ElapsedMs = Get-ClampPercentile -Values $latencies -Percentile 90 + P99ElapsedMs = Get-ClampPercentile -Values $latencies -Percentile 99 + P90CallElapsedMs = Get-ClampPercentile -Values $callLatencies -Percentile 90 + ThrottlePercentageSeen = $sawPercentage + ThrottleScopeSeen = $sawScope + RetryAfterSeen = $sawRetryAfter + RateLimitSeen = $sawRateLimit + ClampRatio = $clampRatio + Refusals = $refusals + } +} + +function Get-ClampHeadline { + <# + .SYNOPSIS + The one-line verdict, read from the buckets: whole-window p50/p90 and the worst + bucket's own p90, never a single live sample. + + .DESCRIPTION + Get-MgxTelemetry's PacingState carries the pacer's most recent latency sample - useful + for watching the pacer live, and exactly one call wide. Printed beside a headline that + is actually labeled as one, it stops reading as the run's verdict by default; this + function is that headline, and the caller prints PacingState only as "last pacer + sample", never in its place. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [object] $Overall, + [AllowNull()] [object[]] $Buckets + ) + + $worstBucket = @($Buckets) | Where-Object { $null -ne $_.P90ElapsedMs } | + Sort-Object P90ElapsedMs -Descending | Select-Object -First 1 + + $sentence = "whole window: p50 {0}ms, p90 {1}ms over {2} answered attempt(s); worst bucket (minute {3}) p90 {4}ms" -f ` + ($Overall.P50ElapsedMs ?? '-'), ($Overall.P90ElapsedMs ?? '-'), $Overall.Ok, ` + $(if ($worstBucket) { $worstBucket.FromMinute } else { '-' }), ` + $(if ($worstBucket) { $worstBucket.P90ElapsedMs } else { '-' }) + if ($null -ne $Overall.ClampRatio) { + $sentence += (" ({0:F2}x the first bucket's)" -f $Overall.ClampRatio) + } + + [pscustomobject]@{ + P50ElapsedMs = $Overall.P50ElapsedMs + P90ElapsedMs = $Overall.P90ElapsedMs + WorstBucketFromMinute = $(if ($worstBucket) { $worstBucket.FromMinute } else { $null }) + WorstBucketP90ElapsedMs = $(if ($worstBucket) { $worstBucket.P90ElapsedMs } else { $null }) + ClampRatio = $Overall.ClampRatio + Sentence = $sentence + } +} + +function Get-ClampSummary { + <# + .SYNOPSIS + 18's whole summary, built from its checkpointed rows file: the clock-gap refusal, every + bucket, the whole-window totals and refusals-by-kind, and the headline. + + .DESCRIPTION + Callable on a rows file path alone, with the run's own clocks and window - the same call + the script makes once its load finishes, and the one a test makes against a synthetic + rows.jsonl it wrote itself. When StartedAt/EndedAt and the Stopwatch disagree by more + than a bucket (Get-ClampClockGap), the rows stay checkpointed on disk, but Buckets, + Overall and Headline all come back $null: a summary built across a gap the machine was + not awake for is not a distribution, and the caller is expected to print + Refused/RefusalReason and stop rather than read the rest of this object. + + .PARAMETER RowsPath + 18's checkpointed rows.jsonl. + + .PARAMETER StartedAt + .PARAMETER EndedAt + Wall time (Get-Date) at the start and end of the load. + + .PARAMETER WallMs + The load's own Stopwatch, in milliseconds. + + .PARAMETER Minutes + .PARAMETER BucketMinutes + The window and bucket width the run was asked for - fixes the bucket count independent + of which buckets the rows happen to touch. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $RowsPath, + [Parameter(Mandatory)] [datetime] $StartedAt, + [Parameter(Mandatory)] [datetime] $EndedAt, + [Parameter(Mandatory)] [long] $WallMs, + [Parameter(Mandatory)] [ValidateRange(1, [int]::MaxValue)] [int] $Minutes, + [Parameter(Mandatory)] [ValidateRange(1, [int]::MaxValue)] [int] $BucketMinutes + ) + + $gap = Get-ClampClockGap -StartedAt $StartedAt -EndedAt $EndedAt -WallMs $WallMs -BucketMinutes $BucketMinutes + if ($gap.Refused) { + return [pscustomobject]@{ + Refused = $true + RefusalReason = $gap.Reason + GapMinutes = $gap.GapMinutes + Buckets = $null + Overall = $null + Headline = $null + } + } + + $rows = Get-ClampRows -Path $RowsPath + $buckets = Get-ClampBuckets -Rows $rows -StartedAt $StartedAt -Minutes $Minutes -BucketMinutes $BucketMinutes + $overall = Get-ClampOverall -Rows $rows -Buckets $buckets + $headline = Get-ClampHeadline -Overall $overall -Buckets $buckets + + [pscustomobject]@{ + Refused = $false + RefusalReason = $null + GapMinutes = $gap.GapMinutes + Buckets = $buckets + Overall = $overall + Headline = $headline + } +} diff --git a/tests/benchmarks/mock-graph-server.ps1 b/tests/benchmarks/mock-graph-server.ps1 index 7f7cce3..ed7fb07 100644 --- a/tests/benchmarks/mock-graph-server.ps1 +++ b/tests/benchmarks/mock-graph-server.ps1 @@ -1,23 +1,136 @@ -# Minimal mock Graph endpoint for the fault-injection gauntlet (06-fault-gauntlet.ps1). -# Serves GET /v1.0/users/{id} with deterministic transient faults keyed on the entity id, -# so every contender faces the identical fault schedule regardless of request order: -# id % 100 0-14 -> first attempt gets 429 (Retry-After: 1), then success -# id % 100 15-17 -> first two attempts get 503 (Retry-After: 1), then success -# otherwise -> success -# Control endpoints: GET /ping (liveness), GET /reset (clear attempt state between -# contenders), GET /stats (server-side truth: requests served, faults injected). +# Mock Graph endpoint for the benchmark gauntlets. Started and stopped by the benchmark +# that needs it (06-fault-gauntlet.ps1 and the pathological gauntlets after it). +# +# Division of labor. Two kinds of fault live here and nowhere else: +# - faults every contender must meet identically. The gauntlets compare mgx against the +# Graph SDK and against a bare Invoke-RestMethod loop, so a fault injected inside mgx +# is not a fault the other two can face; it has to be a property of the server. +# - faults that need a real socket: a connection dropped mid-body, a body trickled over +# seconds, a body that stops mid-document under an honest Content-Length. +# Everything asserted in-process belongs to the xUnit fakes instead - the handlers, sessions +# and canned collections under tests/Mgx.IntegrationTests/Fakes, which answer a request +# without a socket. Those are cheaper, they run in the unit suite, and they can see the +# request objects. Do not grow this server for a fault one of them already holds. +# +# Routes: +# GET /(v1.0|beta)/users/{id} The entity. With no plan in force it carries the default +# schedule, keyed on the entity id so request order cannot change what a contender +# meets: +# id % 100 0-14 -> first attempt gets 429 (Retry-After: 1), then success +# id % 100 15-17 -> first two attempts get 503 (Retry-After: 1), then success +# otherwise -> success +# GET /(v1.0|beta)/users The collection. One page of mock users by default; with +# "pages" in the plan, that many pages joined by a real @odata.nextLink the client +# has to follow, items shaped like the entity route's. +# anything else A benign empty page. An unknown route is not a fault, so +# a benchmark that pages one measures nothing rather than failing - which is why a +# scenario needing a real collection asks for one of the routes above by name. A +# plan rule can still target it; the empty page is the body it works on. +# +# Control endpoints. Matched before any route, so a plan whose pattern matches every path +# cannot lock the server out: +# GET /ping liveness +# GET /reset clears attempt counters and rule clocks; KEEPS the plan, because 06 +# resets between contenders and every contender must meet the same plan +# GET /stats server-side truth (below) +# GET /plan the plan in force +# POST /plan sets it, replacing whatever was there; {"rules":[]} clears it. Screened +# where it is written, not on the request that would have met it: the document's +# own top-level fields must be ones the document can have, it must be an object +# whose 'rules' is an array, every rule's field names must be ones a rule can +# have, the route must compile as a regex, the kind must be one of the seven, +# the scope must be path or server, and every numeric field must be a number of +# at least zero. A plan failing any of those is refused 400 MockPlanRejected +# naming the field, and the plan already in force stays in force. +# +# The plan. POST /plan takes: +# { +# "pages": 3, # list routes serve this many pages (default 1) +# "pageSize": 5, # items per list page (default 25) +# "rules": [ { +# "route": "^/v1\\.0/users$", # .NET regex over the URL path; the query is not +# # part of it, so every page of one enumeration +# # is another attempt on the same route +# "kind": "abort", # one of the kinds below +# "scope": "path", # count attempts per path (default), or "server" +# # for one counter across every matching path - +# # what a token expiry or a tenant-wide storm is +# "from": 3, # first attempt it fires on (default 1) +# "count": 1, # how many attempts it fires for (default: no end) +# "to": 3, # the same thing said as a last attempt +# "seconds": 10, # and fire only this long after the rule's first +# # matching request; combines with the attempt +# # window rather than replacing it +# "bytes": 64, # abort, truncatedJson: body bytes to send +# "bodyMs": 2000, # slowBody: how long the body takes +# "chunks": 10, # slowBody: how many pieces to trickle it in +# "retryAfter": 1 # throttleStorm, outage: the header value +# } ] +# } +# The first rule whose route matches governs the request, and a route the plan governs does +# not also get the default schedule: outside a rule's window it simply succeeds. Kinds: +# abort headers and part of the body, then the connection dies +# slowBody the whole body, trickled over bodyMs +# truncatedJson a complete response whose JSON stops mid-document +# expiredToken 401 with WWW-Authenticate, the shape an expired token shows +# delayedVisibility 404 while the window is open, then the resource appears +# throttleStorm 429 with Retry-After for every request in the window +# outage 503 for every attempt; with no window it never recovers +# +# /stats reports the three fields it always did - served, a body the entity route served +# whole, and faulted429/faulted503, which stay the default schedule's own tally - then one +# counter per kind under the kind's own name, then pagesServed, a list page whose body went +# out whole, and serverErrors, a request this server itself failed to answer. Every counter +# is raised after its response has been written, so /stats never affirms a fault that never +# reached the wire. A plan fault counts under its kind only - not served, and not +# pagesServed - so a run's default-schedule numbers stay comparable across releases. +# +# The accept loop is single-threaded, as it always has been: one request is answered +# before the next is read. A slowBody rule therefore holds up whatever else is in flight, +# which is the honest shape of a stalled response on a shared connection - but it means a +# scenario should trickle one body, not two hundred. param( [int] $Port = 8787 ) $ErrorActionPreference = 'Stop' +# Retry-After and duration values are rendered into headers and JSON, so the process must +# not be able to print 1,5 for 1.5 on a comma-decimal machine. +[System.Threading.Thread]::CurrentThread.CurrentCulture = [System.Globalization.CultureInfo]::InvariantCulture + $listener = [System.Net.HttpListener]::new() $listener.Prefixes.Add("http://localhost:$Port/") $listener.Start() Write-Host "mock graph listening on http://localhost:$Port/" -$attempts = @{} -$stats = @{ served = 0; faulted429 = 0; faulted503 = 0 } +$attempts = @{} # default schedule: entity id -> attempts +$ruleState = @{} # plan: "|" -> @{ n; first } +$plan = @{ pages = 1; pageSize = 25; rules = @() } +$kinds = 'abort', 'slowBody', 'truncatedJson', 'expiredToken', + 'delayedVisibility', 'throttleStorm', 'outage' +$stats = [ordered]@{ + served = 0; faulted429 = 0; faulted503 = 0 + abort = 0; slowBody = 0; truncatedJson = 0; expiredToken = 0 + delayedVisibility = 0; throttleStorm = 0; outage = 0 + pagesServed = 0; serverErrors = 0 +} + +# The numeric plan fields, and what each must convert to. Every one of them is cast where it +# is used, deep inside a response a client is already waiting on, so they are screened here +# instead - see Test-PlanNumber. +$ruleNumbers = [ordered]@{ + from = 'int'; count = 'int'; to = 'int'; bytes = 'int' + bodyMs = 'int'; chunks = 'int'; retryAfter = 'int'; seconds = 'double' +} +# Every field a rule may carry: the numeric ones above, plus the three structural fields no +# kind owns. Built from $ruleNumbers rather than listed again by hand, so a numeric field +# added there is automatically a known field here too. +$ruleFieldNames = @($ruleNumbers.Keys) + @('route', 'kind', 'scope') +# The plan document's own top-level fields. A name outside this list - "pagez" for +# "pageSize" - is not an error Get-Field can see: it just looks like a field the document +# does not set, and the default is used in its place. Screened the same way a rule's own +# fields are, so a typo here fails the same way a typo there does. +$planFieldNames = @('rules', 'pages', 'pageSize') function Send-Json($ctx, [int]$status, [string]$json, [hashtable]$headers) { $ctx.Response.StatusCode = $status @@ -29,6 +142,292 @@ function Send-Json($ctx, [int]$status, [string]$json, [hashtable]$headers) { $ctx.Response.Close() } +# The honest length is announced and only part of the body delivered, then the connection +# is killed: the client sees the transport end, not a status code. +function Send-Aborted($ctx, [string]$json, [int]$bytes) { + $full = [System.Text.Encoding]::UTF8.GetBytes($json) + if ($bytes -le 0 -or $bytes -ge $full.Length) { $bytes = [math]::Max(1, [int]($full.Length / 2)) } + $ctx.Response.StatusCode = 200 + $ctx.Response.ContentType = 'application/json' + $ctx.Response.ContentLength64 = $full.Length + $ctx.Response.OutputStream.Write($full, 0, $bytes) + $ctx.Response.OutputStream.Flush() + $ctx.Response.Abort() +} + +# A complete, well-formed HTTP response - Content-Length matches what is sent, the +# connection closes cleanly - whose JSON stops mid-document. The client fails parsing, +# not reading. +function Send-Truncated($ctx, [string]$json, [int]$bytes) { + $full = [System.Text.Encoding]::UTF8.GetBytes($json) + if ($bytes -le 0 -or $bytes -ge $full.Length) { $bytes = [math]::Max(1, [int]($full.Length / 2)) } + $ctx.Response.StatusCode = 200 + $ctx.Response.ContentType = 'application/json' + $ctx.Response.ContentLength64 = $bytes + $ctx.Response.OutputStream.Write($full, 0, $bytes) + $ctx.Response.Close() +} + +# The whole body, delivered in $chunks pieces spaced to spend about $ms total between the +# first and the last. One chunk is still a body the client waited $ms for, but the loop below +# only sleeps in the gap between writes - and there is no gap when there is one write - so +# that case sleeps before it instead. +function Send-Slow($ctx, [string]$json, [int]$ms, [int]$chunks) { + $full = [System.Text.Encoding]::UTF8.GetBytes($json) + if ($chunks -lt 1) { $chunks = 10 } + if ($chunks -gt $full.Length) { $chunks = $full.Length } + $ctx.Response.StatusCode = 200 + $ctx.Response.ContentType = 'application/json' + $ctx.Response.ContentLength64 = $full.Length + if ($chunks -eq 1) { + if ($ms -gt 0) { Start-Sleep -Milliseconds $ms } + $ctx.Response.OutputStream.Write($full, 0, $full.Length) + $ctx.Response.OutputStream.Flush() + $ctx.Response.Close() + return + } + $size = [math]::Max(1, [int][math]::Ceiling($full.Length / $chunks)) + $gap = [math]::Max(1, [int]($ms / [math]::Max(1, $chunks - 1))) + $off = 0 + while ($off -lt $full.Length) { + $n = [math]::Min($size, $full.Length - $off) + $ctx.Response.OutputStream.Write($full, $off, $n) + $ctx.Response.OutputStream.Flush() + $off += $n + if ($off -lt $full.Length) { Start-Sleep -Milliseconds $gap } + } + $ctx.Response.Close() +} + +# [ordered], not a plain hashtable: a hashtable hands ConvertTo-Json its keys in whatever +# order that process happened to hash them, so the same id answered by two servers is two +# different bodies - and a server whose point is determinism cannot serve those. +function New-MockUser([int]$id) { + [ordered]@{ + id = "00000000-0000-0000-0000-{0:D12}" -f $id + displayName = "Mock User $id" + userPrincipalName = "mock.u$id@mock.local" + mail = "mock.u$id@mock.local" + department = 'Mockestration' + jobTitle = 'Test Subject' + } +} + +function Get-ListPage([string]$version, [string]$origin, [int]$page) { + $size = if ($plan.pageSize -gt 0) { [int]$plan.pageSize } else { 25 } + $pages = if ($plan.pages -gt 0) { [int]$plan.pages } else { 1 } + if ($page -lt 1) { $page = 1 } + $first = ($page - 1) * $size + 1 + $doc = [ordered]@{ + '@odata.context' = "$origin/$version/`$metadata#users" + value = @($first..($first + $size - 1) | ForEach-Object { New-MockUser $_ }) + } + if ($page -lt $pages) { + $doc['@odata.nextLink'] = "$origin/$version/users?`$skiptoken=p$($page + 1)" + } + ConvertTo-Json -InputObject $doc -Depth 5 -Compress +} + +function Get-Field($obj, [string]$name, $default) { + if ($null -eq $obj) { return $default } + $p = $obj.PSObject.Properties[$name] + if ($p -and $null -ne $p.Value) { return $p.Value } + $default +} + +# A numeric plan field that is not a number throws where it is cast, which is halfway +# through answering a request: the client is left holding an empty 200 and the counter for a +# fault that never went out. Screened here, the same typo is a 400 naming the field. +function Test-PlanNumber($container, [string]$name, [string]$type, [string]$where) { + $value = Get-Field $container $name $null + if ($null -eq $value) { return } + try { $number = if ($type -eq 'double') { [double]$value } else { [int]$value } } + catch { throw "$where '$name' must be a number, not '$value'" } + if ($number -lt 0 -or -not [double]::IsFinite([double]$number)) { + throw "$where '$name' must be a number of at least 0, not '$value'" + } +} + +# from, to, count and seconds describe a firing window, and Test-RuleFires treats a zero on +# any of them as no bound rather than a bound at zero: an explicit from: 0 clears the lower +# bound instead of matching nothing, and to/count/seconds: 0 clears the upper bound the same +# way leaving the field out does - so a rule meant to fire briefly fires forever instead. +# Called after Test-PlanNumber has already confirmed the value is a number of at least 0, so +# the only thing left to refuse here is exactly 0. +function Test-PlanWindowField($container, [string]$name, [int]$ruleNumber) { + $value = Get-Field $container $name $null + if ($null -eq $value -or [double]$value -ge 1) { return } + throw "rule ${ruleNumber}: '$name' must be at least 1, got $value" +} + +# True when this attempt falls inside the rule's window. The attempt window always +# applies; a "seconds" duration narrows it further, measured from the rule's first +# matching request so a scenario's clock starts when its traffic does, not when the plan +# was posted. +function Test-RuleFires([int]$index, $rule, [string]$path) { + $key = if ([string](Get-Field $rule 'scope' 'path') -eq 'server') { "$index|*" } else { "$index|$path" } + if (-not $ruleState.ContainsKey($key)) { $ruleState[$key] = @{ n = 0; first = [DateTime]::UtcNow } } + $st = $ruleState[$key] + $st.n++ + + $from = [int](Get-Field $rule 'from' 1) + $to = [int](Get-Field $rule 'to' 0) + if ($to -le 0) { + $count = [int](Get-Field $rule 'count' 0) + if ($count -gt 0) { $to = $from + $count - 1 } + } + if ($st.n -lt $from) { return $false } + if ($to -gt 0 -and $st.n -gt $to) { return $false } + + $seconds = [double](Get-Field $rule 'seconds' 0) + if ($seconds -gt 0 -and ([DateTime]::UtcNow - $st.first).TotalSeconds -ge $seconds) { return $false } + $true +} + +# Returns 'whole' when the route's own body went out complete (trickled counts: all of it +# arrives), 'partial' when a fault cut it short, 'error' when a status fault replaced it, and +# $null when no rule governs this path - in which case the caller carries on with the default +# schedule. A caller counting a served body - pagesServed, served - counts 'whole' and +# nothing else; see Test-DispositionServed. +function Invoke-PlanRule($ctx, [string]$path, [string]$body) { + if ($plan.rules.Count -eq 0) { return $null } + $index = -1 + for ($i = 0; $i -lt $plan.rules.Count; $i++) { + if ($path -match [string]$plan.rules[$i].route) { $index = $i; break } + } + if ($index -lt 0) { return $null } + + $rule = $plan.rules[$index] + if (-not (Test-RuleFires $index $rule $path)) { Send-Json $ctx 200 $body $null; return 'whole' } + + $retry = [string][int](Get-Field $rule 'retryAfter' 1) + switch ([string]$rule.kind) { + 'abort' { + Send-Aborted $ctx $body ([int](Get-Field $rule 'bytes' 0)) + $stats.abort++ + return 'partial' + } + 'slowBody' { + Send-Slow $ctx $body ([int](Get-Field $rule 'bodyMs' 1000)) ([int](Get-Field $rule 'chunks' 10)) + $stats.slowBody++ + return 'whole' + } + 'truncatedJson' { + Send-Truncated $ctx $body ([int](Get-Field $rule 'bytes' 0)) + $stats.truncatedJson++ + return 'partial' + } + 'expiredToken' { + Send-Json $ctx 401 '{"error":{"code":"InvalidAuthenticationToken","message":"Access token has expired."}}' ` + @{ 'WWW-Authenticate' = 'Bearer realm="", error="invalid_token", error_description="The access token has expired."' } + $stats.expiredToken++ + return 'error' + } + 'delayedVisibility' { + Send-Json $ctx 404 '{"error":{"code":"Request_ResourceNotFound","message":"Resource does not exist or one of its queried reference-property objects are not present."}}' $null + $stats.delayedVisibility++ + return 'error' + } + 'throttleStorm' { + Send-Json $ctx 429 '{"error":{"code":"TooManyRequests","message":"mock throttle storm"}}' @{ 'Retry-After' = $retry } + $stats.throttleStorm++ + return 'error' + } + 'outage' { + Send-Json $ctx 503 '{"error":{"code":"ServiceUnavailable","message":"mock outage"}}' @{ 'Retry-After' = $retry } + $stats.outage++ + return 'error' + } + default { + # Unreachable while Set-Plan screens kinds, and loud rather than benign if that + # ever stops being true: a scenario must not report success against a fault + # that never fired. + Send-Json $ctx 500 (@{ error = @{ code = 'MockPlanError' + message = "unknown fault kind '$($rule.kind)'" } } | + ConvertTo-Json -Compress -Depth 3) $null + return 'error' + } + } +} + +# The predicate every disposition-counting caller shares: only 'whole' is a body this route +# actually served. 'partial' and 'error' reached the wire too, but under a fault's own shape, +# and are counted under that fault's kind instead - never here as well. +function Test-DispositionServed([string]$disposition) { + $disposition -eq 'whole' +} + +function Set-Plan($ctx) { + $raw = [System.IO.StreamReader]::new($ctx.Request.InputStream, [System.Text.Encoding]::UTF8).ReadToEnd() + try { + # -NoEnumerate, or a one-element array document - [{"rules":[]}] - comes back out of + # ConvertFrom-Json as that one element, not the array it was: PSCustomObject passes the + # shape check below on the object the array held, and a document that is not the + # documented shape is accepted as though it were. + $new = if ($raw.Trim()) { $raw | ConvertFrom-Json -NoEnumerate } else { $null } + # A document that is not an object at all (an array, a bare string, a number) has no + # properties to read 'rules' off, and Get-Field's empty default for a missing property + # is indistinguishable from {"rules":[]} - the documented way to clear the plan on + # purpose. So the shape is checked directly, on the object graph ConvertFrom-Json + # already built, before anything reads a field out of it. + if ($new -isnot [System.Management.Automation.PSCustomObject]) { + $shape = if ($null -eq $new) { 'empty' } else { $new.GetType().Name } + throw "plan document must be a JSON object with a 'rules' array, not $shape" + } + # $rulesProp.Value is read by direct member access everywhere below, never assigned + # through an if/else branch: passing an array through a script block's trailing + # expression - even just to pick a default - re-enumerates it onto the pipeline, and a + # one-rule plan comes back out the other side as that one rule instead of an array of + # it. @() on the way into the foreach would paper over the read, not fix it. + $rulesProp = $new.PSObject.Properties['rules'] + if (-not $rulesProp -or $rulesProp.Value -isnot [array]) { + $shape = if (-not $rulesProp -or $null -eq $rulesProp.Value) { 'missing' } else { $rulesProp.Value.GetType().Name } + throw "plan 'rules' must be an array, not $shape" + } + $rules = $rulesProp.Value + # Screened here, not on the first request that would have matched: a scenario whose + # plan does not mean what it says must fail where it is written. Nothing below is + # replaced until the whole document has passed, so a refused plan leaves the one in + # force untouched rather than half of it. + foreach ($name in $new.PSObject.Properties.Name) { + if ($name -notin $script:planFieldNames) { + throw "plan has an unknown field '$name'; expected one of $($script:planFieldNames -join ', ')" + } + } + Test-PlanNumber $new 'pages' 'int' 'plan' + Test-PlanNumber $new 'pageSize' 'int' 'plan' + for ($i = 0; $i -lt $rules.Count; $i++) { + $r = $rules[$i] + if (-not $r.route -or -not $r.kind) { throw 'every rule needs a route and a kind' } + $null = [regex]::new([string]$r.route) + if ([string]$r.kind -notin $script:kinds) { + throw "unknown fault kind '$($r.kind)'; expected one of $($script:kinds -join ', ')" + } + foreach ($name in $r.PSObject.Properties.Name) { + if ($name -notin $script:ruleFieldNames) { + throw "rule '$($r.kind)' has an unknown field '$name'; expected one of $($script:ruleFieldNames -join ', ')" + } + } + $scope = [string](Get-Field $r 'scope' 'path') + if ($scope -notin 'path', 'server') { throw "scope must be path or server, not '$scope'" } + foreach ($field in $script:ruleNumbers.Keys) { + Test-PlanNumber $r $field $script:ruleNumbers[$field] "rule '$($r.kind)'" + } + foreach ($field in 'from', 'to', 'count', 'seconds') { + Test-PlanWindowField $r $field ($i + 1) + } + } + $plan.rules = $rules + $plan.pages = [int](Get-Field $new 'pages' 1) + $plan.pageSize = [int](Get-Field $new 'pageSize' 25) + $ruleState.Clear() + Send-Json $ctx 200 (@{ ok = $true; rules = $rules.Count } | ConvertTo-Json -Compress) $null + } + catch { + Send-Json $ctx 400 (@{ error = @{ code = 'MockPlanRejected'; message = $_.Exception.Message } } | ConvertTo-Json -Compress) $null + } +} + while ($listener.IsListening) { $ctx = $listener.GetContext() $path = $ctx.Request.Url.AbsolutePath @@ -36,47 +435,74 @@ while ($listener.IsListening) { try { switch -Regex ($path) { '^/ping$' { Send-Json $ctx 200 '{"ok":true}' $null; continue } - '^/reset$' { $attempts.Clear(); Send-Json $ctx 200 '{"reset":true}' $null; continue } - '^/stats$' { - Send-Json $ctx 200 (@{ served = $stats.served; faulted429 = $stats.faulted429; faulted503 = $stats.faulted503 } | ConvertTo-Json -Compress) $null + '^/reset$' { $attempts.Clear(); $ruleState.Clear(); Send-Json $ctx 200 '{"reset":true}' $null; continue } + '^/stats$' { Send-Json $ctx 200 (ConvertTo-Json -InputObject $stats -Compress) $null; continue } + '^/plan$' { + if ($ctx.Request.HttpMethod -in 'POST', 'PUT') { Set-Plan $ctx } + else { Send-Json $ctx 200 (ConvertTo-Json -InputObject $plan -Depth 5 -Compress) $null } continue } '^/(v1\.0|beta)/users/(?[^/?]+)$' { - $stats.served++ $id = [int]($Matches['id'] -replace '\D', '0') + $user = New-MockUser $id | ConvertTo-Json -Compress + $disposition = Invoke-PlanRule $ctx $path $user + if ($null -ne $disposition) { + # A plan fault does not count as served, same predicate as the list + # route's pagesServed: only a 'whole' disposition is a body this route + # actually served. + if (Test-DispositionServed $disposition) { $stats.served++ } + continue + } + if (-not $attempts.ContainsKey($id)) { $attempts[$id] = 0 } $attempts[$id]++ $n = $attempts[$id] $bucket = $id % 100 if ($bucket -lt 15 -and $n -eq 1) { - $stats.faulted429++ Send-Json $ctx 429 '{"error":{"code":"TooManyRequests","message":"mock throttle"}}' @{ 'Retry-After' = '1' } + $stats.faulted429++ } elseif ($bucket -ge 15 -and $bucket -lt 18 -and $n -le 2) { - $stats.faulted503++ Send-Json $ctx 503 '{"error":{"code":"ServiceUnavailable","message":"mock outage"}}' @{ 'Retry-After' = '1' } + $stats.faulted503++ } else { - $user = @{ - id = "00000000-0000-0000-0000-{0:D12}" -f $id - displayName = "Mock User $id" - userPrincipalName = "mock.u$id@mock.local" - mail = "mock.u$id@mock.local" - department = 'Mockestration' - jobTitle = 'Test Subject' - } | ConvertTo-Json -Compress Send-Json $ctx 200 $user $null } + $stats.served++ + continue + } + '^/(v1\.0|beta)/users$' { + # The list route counts under pagesServed and nowhere else: served is the + # entity route's tally, and a 2.1.4 run has to stay comparable with this one. + $version = $Matches[1] + $origin = $ctx.Request.Url.GetLeftPart([System.UriPartial]::Authority) + $page = if ($ctx.Request.Url.Query -match '\$skiptoken=p(\d+)') { [int]$Matches[1] } else { 1 } + $body = Get-ListPage $version $origin $page + $disposition = Invoke-PlanRule $ctx $path $body + if ($null -eq $disposition) { Send-Json $ctx 200 $body $null; $disposition = 'whole' } + # A page cut off mid-body is not a page served; it is counted by the kind + # that cut it. + if (Test-DispositionServed $disposition) { $stats.pagesServed++ } continue } default { - # Anything else (org probe, list endpoints): benign empty page - Send-Json $ctx 200 '{"value":[]}' $null + # Anything else (org probe, unimplemented endpoints): benign empty page + $empty = '{"value":[]}' + if ($null -eq (Invoke-PlanRule $ctx $path $empty)) { Send-Json $ctx 200 $empty $null } } } } catch { - try { $ctx.Response.Close() } catch { } + # A closed response nothing was written to reaches the client as 200 with an empty + # body, which a benchmark reads as an answer. A harness bug says so instead. + $stats.serverErrors++ + try { + Send-Json $ctx 500 (@{ error = @{ code = 'MockServerError' + message = $_.Exception.Message } } | + ConvertTo-Json -Compress -Depth 3) $null + } + catch { try { $ctx.Response.Close() } catch { } } } } diff --git a/tests/benchmarks/run.ps1 b/tests/benchmarks/run.ps1 index a292695..f922b42 100644 --- a/tests/benchmarks/run.ps1 +++ b/tests/benchmarks/run.ps1 @@ -7,13 +7,22 @@ # ./run.ps1 # everything except the slow ones # ./run.ps1 -IncludeSlow # everything, including 04 full baselines (~2h) # ./run.ps1 -Only 01,05 # just those benchmarks +# ./run.ps1 -RecordBaseline # pin what the run produced into baseline.json +# ./run.ps1 -CompareBaseline # read the run against the pinned baseline.json param( [string[]] $Only, [switch] $IncludeSlow, - [int] $CooldownSeconds = 300 + [int] $CooldownSeconds = 300, + [switch] $RecordBaseline, + [switch] $CompareBaseline ) $ErrorActionPreference = 'Stop' +# Export-BenchBaseline, Compare-BenchBaseline and the table of headline metrics they share live +# here. It is also what forces invariant culture, so the delta lines below print 12.5% and +# never 12,5%. +. (Join-Path $PSScriptRoot 'common.ps1') + $logDir = Join-Path $PSScriptRoot 'results/logs' if (-not (Test-Path $logDir)) { New-Item -ItemType Directory -Path $logDir -Force | Out-Null } @@ -33,14 +42,38 @@ $suite = @( @{ id = '10'; script = '10-pacing-under-real-throttling.ps1'; args = @(); cooldown = $false } ) +$promoteOnly = @() if ($Only) { # pwsh -File passes "05,02" as one literal string - split so both call styles work $Only = @($Only | ForEach-Object { $_ -split ',' } | Where-Object { $_ }) $suite = @($suite | Where-Object { $Only -contains $_.id }) + + # An id Get-BenchScenario knows the metrics for but that never runs inside this suite - + # 18, a live-tenant scenario driven standalone through -DriveId/-DriveUri - is not the + # same thing as an id nobody has ever heard of. Without -RecordBaseline it is refused by + # name rather than folded into the generic "no benchmarks matched"; with it, it is + # accepted and promoted below from its own latest recorded result, never run here. + $matchedIds = @($suite | ForEach-Object { $_.id }) + $unmatched = @($Only | Where-Object { $matchedIds -notcontains $_ }) + if ($unmatched.Count -gt 0) { + $standalone = @(Get-BenchScenario -Id $unmatched -WarningAction SilentlyContinue) + $promoteOnly = @($standalone | ForEach-Object { $_.Id }) + if ($promoteOnly.Count -gt 0 -and -not $RecordBaseline) { + $sentences = $standalone | ForEach-Object { + "$($_.Id) is not in the suite; run tests/benchmarks/$($_.Script) directly, then promote it with -RecordBaseline -Only $($_.Id)." + } + throw ($sentences -join ' ') + } + } } -if ($suite.Count -eq 0) { throw "no benchmarks matched -Only $($Only -join ',')" } +if ($suite.Count -eq 0 -and $promoteOnly.Count -eq 0) { throw "no benchmarks matched -Only $($Only -join ',')" } $stamp = Get-Date -Format 'yyyyMMdd-HHmmss' +# Both the promotion and the comparison read results/.json back, which is an +# append-only log: without a floor they would read the newest entry a benchmark ever wrote, +# and a benchmark that died before recording would be pinned or compared from a run that +# happened last month. +$runStart = Get-Date $outcomes = @() foreach ($b in $suite) { $log = Join-Path $logDir "$($b.id)-$stamp.log" @@ -62,3 +95,30 @@ Write-Host '' Write-Host '=== SUITE COMPLETE ===' $outcomes | Format-Table -AutoSize Write-Host "per-benchmark results in $(Join-Path $PSScriptRoot 'results')" + +$ran = @($suite | ForEach-Object { $_.id }) + +if ($RecordBaseline) { + # The seven scenarios the release baseline is made of. A full run also executes 03, 06 and + # 07, which are read on their own merits rather than pinned - naming one with -Only pins it + # anyway, which is how the no-tenant gauntlet gets into a baseline on a laptop. + $baselineDefault = @('01', '02', '04', '05', '08', '09', '10') + $promote = if ($Only) { $ran } else { @($ran | Where-Object { $baselineDefault -contains $_ }) } + Write-Host '' + Write-Host '=== RECORDING BASELINE ===' + if ($promote.Count -eq 0 -and $promoteOnly.Count -eq 0) { + Write-Warning "none of the benchmarks this run executed are baseline scenarios ($($baselineDefault -join ', ')) - nothing promoted." + } + else { + if ($promote.Count -gt 0) { Export-BenchBaseline -Benchmark $promote -Since $runStart } + # $promoteOnly never ran in this invocation, so there is no fresh entry to floor with + # -Since $runStart - it promotes from whatever its own standalone run last recorded, + # however long before this call that was. + if ($promoteOnly.Count -gt 0) { Export-BenchBaseline -Benchmark $promoteOnly } + } +} + +if ($CompareBaseline) { + # Reports, never gates: exit code is the suite's, not the comparison's. + $null = Compare-BenchBaseline -Benchmark $ran -Since $runStart +} diff --git a/tests/benchmarks/tenant-resilience-comparison.ps1 b/tests/benchmarks/tenant-resilience-comparison.ps1 deleted file mode 100644 index 4d485c0..0000000 --- a/tests/benchmarks/tenant-resilience-comparison.ps1 +++ /dev/null @@ -1,102 +0,0 @@ -# Benchmark: Microsoft.Graph SDK vs Mgx resilience vs Invoke-MgxRequest. -# -# Compares wall-clock time and memory for the same operation across three -# approaches. Run this on your own tenant to see real numbers. -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All" - -Import-Module Microsoft.Graph.Users -Import-Module Mgx - -$property = "id,displayName,mail,department,jobTitle,accountEnabled" -$selectArr = @("id","displayName","mail","department","jobTitle","accountEnabled") - -Write-Host "Benchmarking against your tenant..." -Write-Host "" - -# --- 1. Bare SDK: Get-MgUser -All --- -Write-Host "1) Get-MgUser -All (bare SDK, no resilience)" -[GC]::Collect() -$memBefore = [GC]::GetTotalMemory($true) -$sw = [System.Diagnostics.Stopwatch]::StartNew() - -$sdkUsers = Get-MgUser -All -Property $selectArr -$sw.Stop() - -$memAfter = [GC]::GetTotalMemory($false) -$sdkCount = $sdkUsers.Count -$sdkMs = $sw.ElapsedMilliseconds -$sdkMem = [math]::Round(($memAfter - $memBefore) / 1MB, 1) -Write-Host " $sdkCount users in ${sdkMs}ms (~${sdkMem} MB)" -$sdkUsers = $null - -# --- 2. SDK + Enable-MgxResilience --- -Write-Host "" -Write-Host "2) Get-MgUser -All (with Enable-MgxResilience)" -Enable-MgxResilience -[GC]::Collect() -$memBefore = [GC]::GetTotalMemory($true) -$sw = [System.Diagnostics.Stopwatch]::StartNew() - -$resilientUsers = Get-MgUser -All -Property $selectArr -$sw.Stop() - -$memAfter = [GC]::GetTotalMemory($false) -$resilientCount = $resilientUsers.Count -$resilientMs = $sw.ElapsedMilliseconds -$resilientMem = [math]::Round(($memAfter - $memBefore) / 1MB, 1) -Write-Host " $resilientCount users in ${resilientMs}ms (~${resilientMem} MB)" -Write-Host " (retry + circuit breaker + rate limiting active)" -Disable-MgxResilience -$resilientUsers = $null - -# --- 3. Invoke-MgxRequest -All (streaming pagination) --- -Write-Host "" -Write-Host "3) Invoke-MgxRequest -All (streaming pagination)" -[GC]::Collect() -$memBefore = [GC]::GetTotalMemory($true) -$sw = [System.Diagnostics.Stopwatch]::StartNew() - -$mgxUsers = Invoke-MgxRequest /users -All -Property $selectArr -$sw.Stop() - -$memAfter = [GC]::GetTotalMemory($false) -$mgxCount = $mgxUsers.Count -$mgxMs = $sw.ElapsedMilliseconds -$mgxMem = [math]::Round(($memAfter - $memBefore) / 1MB, 1) -Write-Host " $mgxCount users in ${mgxMs}ms (~${mgxMem} MB)" -$mgxUsers = $null - -# --- 4. Export-MgxCollection (constant memory, disk streaming) --- -Write-Host "" -Write-Host "4) Export-MgxCollection (JSONL to disk, constant memory)" -$outFile = Join-Path ([System.IO.Path]::GetTempPath()) "mgx-bench.jsonl" -[GC]::Collect() -$memBefore = [GC]::GetTotalMemory($true) -$sw = [System.Diagnostics.Stopwatch]::StartNew() - -$exportResult = Export-MgxCollection /users -OutputFile $outFile -All -Property $selectArr -$sw.Stop() - -$memAfter = [GC]::GetTotalMemory($false) -$exportMs = $sw.ElapsedMilliseconds -$exportMem = [math]::Round(($memAfter - $memBefore) / 1MB, 1) -Write-Host " $($exportResult.ItemCount) users in ${exportMs}ms (~${exportMem} MB)" -Remove-Item $outFile -ErrorAction SilentlyContinue - -# --- Summary --- -Write-Host "" -Write-Host "=== Summary ===" -Write-Host "" - -$results = @( - [PSCustomObject]@{ Approach = "Get-MgUser (bare SDK)"; Users = $sdkCount; TimeMs = $sdkMs; MemMB = $sdkMem; Resilience = "None" } - [PSCustomObject]@{ Approach = "Get-MgUser + MgxResilience"; Users = $resilientCount; TimeMs = $resilientMs; MemMB = $resilientMem; Resilience = "Retry + CB + RL" } - [PSCustomObject]@{ Approach = "Invoke-MgxRequest -All"; Users = $mgxCount; TimeMs = $mgxMs; MemMB = $mgxMem; Resilience = "Full stack" } - [PSCustomObject]@{ Approach = "Export-MgxCollection (JSONL)"; Users = $exportResult.ItemCount; TimeMs = $exportMs; MemMB = $exportMem; Resilience = "Full stack" } -) -$results | Format-Table -AutoSize - -# Show telemetry for the Mgx operations -Write-Host "Telemetry (Mgx operations only):" -Get-MgxTelemetry From d1334c3eebeb1bc6ab5cc5625f39b0713cdc610d Mon Sep 17 00:00:00 2001 From: Fabien Tschanz Date: Mon, 21 Sep 2026 22:15:12 +0200 Subject: [PATCH 07/10] Return inner Graph error when detected --- CHANGELOG.md | 6 ++ Directory.Build.props | 2 +- Modules/M365DSC.mgx/M365DSC.mgx.psd1 | 2 +- .../Models/GraphServiceException.cs | 39 ++++++++++++- .../Engine/ErrorBodyShapeTests.cs | 55 +++++++++++++++++++ 5 files changed, 100 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c4370b..32ef2fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 2.1.2 + +### Fixed + +- A Graph error reported only its top level message, which for a validation failure is only `The request is invalid.`, while the property that was actually wrong sat in the `innerError` of the response and was discarded. `GraphServiceException` now appends that message and exposes it as `InnerErrorMessage`. + ## 2.1.1 ### Fixed diff --git a/Directory.Build.props b/Directory.Build.props index 5e1fa40..01a02c5 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,7 +1,7 @@ - 2.1.1 + 2.1.2 Thomas Maillo Grome, Fabien Tschanz Mgx (c) 2026 Thomas Maillo Grome, (c) 2026 Fabien Tschanz. All rights reserved. diff --git a/Modules/M365DSC.mgx/M365DSC.mgx.psd1 b/Modules/M365DSC.mgx/M365DSC.mgx.psd1 index 48c7668..0ff47fd 100644 --- a/Modules/M365DSC.mgx/M365DSC.mgx.psd1 +++ b/Modules/M365DSC.mgx/M365DSC.mgx.psd1 @@ -1,6 +1,6 @@ @{ RootModule = 'M365DSC.mgx.psm1' - ModuleVersion = '2.1.1' + ModuleVersion = '2.1.2' GUID = 'f978315f-75c0-48f5-b929-ca7a7757d1d2' Author = 'Thomas Maillo Grome, Fabien Tschanz' CompanyName = 'Mgx' diff --git a/src/Mgx.Engine/Models/GraphServiceException.cs b/src/Mgx.Engine/Models/GraphServiceException.cs index 22304b3..0da2e4c 100644 --- a/src/Mgx.Engine/Models/GraphServiceException.cs +++ b/src/Mgx.Engine/Models/GraphServiceException.cs @@ -12,16 +12,20 @@ public class GraphServiceException : Exception public HttpStatusCode StatusCode { get; } public string? ErrorCode { get; } + public string? InnerErrorMessage { get; } + public GraphServiceException(HttpStatusCode statusCode, string responseBody) - : base(FormatAndExtract(statusCode, responseBody, out var code)) + : base(FormatAndExtract(statusCode, responseBody, out var code, out var innerMessage)) { StatusCode = statusCode; ErrorCode = code; + InnerErrorMessage = innerMessage; } - private static string FormatAndExtract(HttpStatusCode statusCode, string responseBody, out string? errorCode) + private static string FormatAndExtract(HttpStatusCode statusCode, string responseBody, out string? errorCode, out string? innerErrorMessage) { errorCode = null; + innerErrorMessage = null; if (string.IsNullOrEmpty(responseBody)) return $"HTTP {(int)statusCode}: {statusCode}"; @@ -44,6 +48,7 @@ private static string FormatAndExtract(HttpStatusCode statusCode, string respons var code = AsString(errorObj, "code"); var message = AsString(errorObj, "message"); errorCode = code; + innerErrorMessage = ExtractInnerErrorMessage(errorObj); // Build formatted message from whatever Graph provided var formatted = !string.IsNullOrEmpty(code) @@ -52,6 +57,12 @@ private static string FormatAndExtract(HttpStatusCode statusCode, string respons ? message : $"HTTP {(int)statusCode}: {statusCode}"; + if (!string.IsNullOrEmpty(innerErrorMessage) + && !string.Equals(innerErrorMessage, message, StringComparison.Ordinal)) + { + formatted += $"\n{innerErrorMessage}"; + } + var guidance = GetGuidanceForCode(code); if (guidance != null) formatted += $"\nHint: {guidance}"; @@ -66,6 +77,30 @@ private static string FormatAndExtract(HttpStatusCode statusCode, string respons return $"HTTP {(int)statusCode}: {statusCode}"; } + private static string? ExtractInnerErrorMessage(JsonElement errorObj) + { + string? message = null; + var current = errorObj; + + for (var depth = 0; depth < 8; depth++) + { + if (!current.TryGetProperty("innerError", out var inner) || inner.ValueKind != JsonValueKind.Object) + { + break; + } + + var innerMessage = AsString(inner, "message"); + if (!string.IsNullOrWhiteSpace(innerMessage)) + { + message = innerMessage.Trim(); + } + + current = inner; + } + + return message; + } + private static string? AsString(JsonElement obj, string name) { if (!obj.TryGetProperty(name, out var v)) return null; diff --git a/tests/Mgx.IntegrationTests/Engine/ErrorBodyShapeTests.cs b/tests/Mgx.IntegrationTests/Engine/ErrorBodyShapeTests.cs index e9214a1..0b9c6bc 100644 --- a/tests/Mgx.IntegrationTests/Engine/ErrorBodyShapeTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/ErrorBodyShapeTests.cs @@ -70,4 +70,59 @@ public void Falls_back_to_the_status_line_when_the_shape_is_wrong() var ex = new GraphServiceException(HttpStatusCode.ServiceUnavailable, """{"error":404}"""); Assert.Contains("503", ex.Message); } + + [Fact] + public void Carries_the_inner_error_message() + { + var ex = new GraphServiceException( + HttpStatusCode.BadRequest, + """{"error":{"code":"","message":"The request is invalid.","innerError":{"message":"configuration.DisplayName : The DisplayName field is required.","date":"2026-09-21T19:08:59"}}}"""); + + Assert.Equal("configuration.DisplayName : The DisplayName field is required.", ex.InnerErrorMessage); + Assert.Contains("The request is invalid.", ex.Message); + Assert.Contains("configuration.DisplayName", ex.Message); + } + + [Fact] + public void Takes_the_deepest_message_of_a_nested_inner_error() + { + var ex = new GraphServiceException( + HttpStatusCode.BadRequest, + """{"error":{"code":"BadRequest","message":"top","innerError":{"message":"middle","innerError":{"message":"deepest"}}}}"""); + + Assert.Equal("deepest", ex.InnerErrorMessage); + Assert.Contains("deepest", ex.Message); + } + + [Fact] + public void Leaves_the_message_alone_when_the_inner_error_carries_no_message() + { + var ex = new GraphServiceException( + HttpStatusCode.BadRequest, + """{"error":{"code":"Request_BadRequest","message":"Invalid request","innerError":{"date":"2024-01-01T00:00:00","request-id":"12345"}}}"""); + + Assert.Null(ex.InnerErrorMessage); + Assert.StartsWith("Request_BadRequest: Invalid request", ex.Message); + } + + [Fact] + public void Does_not_repeat_an_inner_error_that_matches_the_message() + { + var ex = new GraphServiceException( + HttpStatusCode.BadRequest, + """{"error":{"code":"BadRequest","message":"same text","innerError":{"message":"same text"}}}"""); + + Assert.Equal("same text", ex.InnerErrorMessage); + Assert.Equal(1, ex.Message.Split("same text").Length - 1); + } + + [Fact] + public void Stops_walking_a_chain_nested_past_any_real_depth() + { + var ex = new GraphServiceException( + HttpStatusCode.BadRequest, + """{"error":{"code":"BadRequest","message":"top","innerError":{"message":"a","innerError":{"message":"b","innerError":{"message":"c","innerError":{"message":"d","innerError":{"message":"e","innerError":{"message":"f","innerError":{"message":"g","innerError":{"message":"h","innerError":{"message":"i"}}}}}}}}}}}"""); + + Assert.False(string.IsNullOrWhiteSpace(ex.Message)); + } } From 0b6c07f43ebf983a31ece61293f1fdf3de98af4a Mon Sep 17 00:00:00 2001 From: Fabien Tschanz Date: Tue, 22 Sep 2026 00:53:42 +0200 Subject: [PATCH 08/10] Ship System.IO.Pipelines again after the upstream merge --- .gitignore | 8 +++++++- src/Mgx.Cmdlets/Mgx.Cmdlets.csproj | 17 ++++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 1d1f951..380c6f2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -# ignore +# ignore .vs/ .vscode/ .DS_Store @@ -19,6 +19,12 @@ Modules/M365DSC.mgx/*.json Modules/M365DSC.mgx/Dependencies/ Modules/M365DSC.mgx/.build-hash +# The csproj stages an IDE build into module/ as well. Only the manifest there is source. +module/*.dll +module/*.pdb +module/*.json +module/Dependencies/ + # Test output TestResults/ tests/TestResults.xml diff --git a/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj b/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj index c4e16cd..44d5c54 100644 --- a/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj +++ b/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj @@ -1,4 +1,4 @@ - + net10.0 @@ -6,6 +6,11 @@ enable Mgx.Cmdlets true + + false $(WarningsAsErrors);CA1416 @@ -28,7 +33,7 @@ resolver has only Dependencies/ to offer and nothing put it there. This reference is what puts it there, and one 10.0.x file answers both requests - a Resolving handler's answer is taken as given. ModuleAssemblyClosureTests pins the whole class of gap. --> - + @@ -46,7 +51,13 @@ A Debug build now leaves module/ alone. To stage Debug on purpose, ask for it: ./build.ps1 -Configuration Debug, which copies explicitly and does not rely on this. --> - + + + + + + $(MSBuildThisFileDirectory)../../module $(ModuleRoot)/Dependencies From ac3f58451ed5ac9a434a48fd9fa4bcfe362b16ea Mon Sep 17 00:00:00 2001 From: Fabien Tschanz Date: Tue, 22 Sep 2026 06:56:00 +0200 Subject: [PATCH 09/10] Release 2.1.7 with an envelope switch, a clamped batch backoff and a faster suite --- CHANGELOG.md | 12 +++- Directory.Build.props | 2 +- Modules/M365DSC.mgx/M365DSC.mgx.psd1 | 2 +- .../en-US/Mgx.Cmdlets.dll-Help.xml | 23 ++++++ Modules/M365DSC.mgx/help/Invoke-MgxRequest.md | 17 ++++- src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs | 17 ++++- src/Mgx.Engine/Http/GraphBatchClient.cs | 7 +- .../Cmdlets/ExportSharedCheckpointTests.cs | 72 +++++++++++++------ .../InvokeMgxRequestCollectionTests.cs | 57 ++++++++++++++- .../Engine/Batch/BatchWriteTests.cs | 18 ++--- .../Engine/Batch/PartialBatchResultTests.cs | 16 ++--- 11 files changed, 198 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74a581a..836a0c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,14 @@ -# Changelog +# Changelog + +## 2.1.7 + +### Added + +- `Invoke-MgxRequest -Envelope` emits the payload as the service sent it, collection envelope and all, instead of the items inside it. Without it a collection holding one item and a single entity arrive as the same thing - one object - which a caller that has to tell them apart cannot do. `-All` keeps aggregating pages and ignores the switch. Upstream does not carry this. + +### Fixed + +- The batch retry loop waited out its exponential fallback without regard for `MaxRetryAfterSeconds`, which caps only the delays a server asks for. Both now answer to the same ceiling. ## 2.1.6 diff --git a/Directory.Build.props b/Directory.Build.props index 43c799a..adba6ed 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,7 +1,7 @@ - 2.1.6 + 2.1.7 Thomas Maillo Grome, Fabien Tschanz Mgx (c) 2026 Thomas Maillo Grome, (c) 2026 Fabien Tschanz. All rights reserved. diff --git a/Modules/M365DSC.mgx/M365DSC.mgx.psd1 b/Modules/M365DSC.mgx/M365DSC.mgx.psd1 index 6011f15..7209d4c 100644 --- a/Modules/M365DSC.mgx/M365DSC.mgx.psd1 +++ b/Modules/M365DSC.mgx/M365DSC.mgx.psd1 @@ -1,6 +1,6 @@ @{ RootModule = 'M365DSC.mgx.psm1' - ModuleVersion = '2.1.6' + ModuleVersion = '2.1.7' GUID = 'f978315f-75c0-48f5-b929-ca7a7757d1d2' Author = 'Thomas Maillo Grome, Fabien Tschanz' CompanyName = 'Mgx' diff --git a/Modules/M365DSC.mgx/en-US/Mgx.Cmdlets.dll-Help.xml b/Modules/M365DSC.mgx/en-US/Mgx.Cmdlets.dll-Help.xml index 1b5a241..0e15df7 100644 --- a/Modules/M365DSC.mgx/en-US/Mgx.Cmdlets.dll-Help.xml +++ b/Modules/M365DSC.mgx/en-US/Mgx.Cmdlets.dll-Help.xml @@ -2612,6 +2612,17 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } None + + Envelope + + Return the payload as the service sent it, collection envelope and all, instead of the items inside it. Without it a collection holding one item and a single entity are both one object, which a caller that has to tell them apart cannot do. -All aggregates pages and ignores this. + + + SwitchParameter + + + False + Raw @@ -2912,6 +2923,18 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } None + + Envelope + + Return the payload as the service sent it, collection envelope and all, instead of the items inside it. Without it a collection holding one item and a single entity are both one object, which a caller that has to tell them apart cannot do. -All aggregates pages and ignores this. + + SwitchParameter + + SwitchParameter + + + False + Raw diff --git a/Modules/M365DSC.mgx/help/Invoke-MgxRequest.md b/Modules/M365DSC.mgx/help/Invoke-MgxRequest.md index a711eb4..250246e 100644 --- a/Modules/M365DSC.mgx/help/Invoke-MgxRequest.md +++ b/Modules/M365DSC.mgx/help/Invoke-MgxRequest.md @@ -1,4 +1,4 @@ ---- +--- external help file: Mgx.Cmdlets.dll-Help.xml Module Name: Mgx online version: https://github.com/Microsoft365DSC/mgx/blob/main/Modules/M365DSC.mgx/help/Invoke-MgxRequest.md @@ -393,6 +393,21 @@ Accept pipeline input: False Accept wildcard characters: False ``` +### -Envelope +Return the payload as the service sent it, collection envelope and all, instead of the items inside it. Without it a collection holding one item and a single entity are both one object, which a caller that has to tell them apart cannot do. -All aggregates pages and ignores this. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + ### -Raw Return raw JSON strings instead of hashtables. Useful for piping to ConvertFrom-Json or writing to files. diff --git a/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs b/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs index 1098509..98120a4 100644 --- a/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs +++ b/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs @@ -1,4 +1,4 @@ -using System.Collections; +using System.Collections; using System.Management.Automation; using System.Net; using System.Net.Http.Headers; @@ -60,6 +60,15 @@ public class InvokeMgxRequest : MgxCmdletBase [Parameter] public SwitchParameter Raw { get; set; } + /// + /// Emit the payload as the service sent it, collection envelope and all, instead of the items + /// inside it. A caller that has to tell one item in a collection from a single entity cannot do + /// it from the unwrapped output: both are one object. -All keeps aggregating pages and ignores + /// this. + /// + [Parameter] + public SwitchParameter Envelope { get; set; } + #endregion #region List parameters @@ -1140,6 +1149,12 @@ private void NoteNonJsonStringBody() private void OutputPayload(JsonElement json, string? sourceId) { + if (Envelope.IsPresent) + { + OutputItem(json, sourceId); + return; + } + var items = TryUnwrapCollection(json, out var truncated); if (items == null) { diff --git a/src/Mgx.Engine/Http/GraphBatchClient.cs b/src/Mgx.Engine/Http/GraphBatchClient.cs index eed8d38..0de62cb 100644 --- a/src/Mgx.Engine/Http/GraphBatchClient.cs +++ b/src/Mgx.Engine/Http/GraphBatchClient.cs @@ -1,4 +1,4 @@ -using Mgx.Engine.Errors; +using Mgx.Engine.Errors; using System.Collections.Concurrent; using System.Diagnostics; using System.Text; @@ -972,9 +972,12 @@ await SendBatchWithRetryAsync(chunk, cancellationToken, // batch open for a Retry-After the run has already stopped honoring. if (stopRequested.IsCancellationRequested) break; + // The exponential fallback answers to the same ceiling as a server-sent + // Retry-After: a cap on how long one attempt may hold the batch open is not + // a cap on only the delays the server asked for. var baseDelaySeconds = maxRetryAfterSeconds > 0 ? maxRetryAfterSeconds - : (int)Math.Pow(2, attempt); + : Math.Min((int)Math.Pow(2, attempt), _maxRetryAfterSeconds); // C4: Add 0-50% jitter to prevent thundering herd on batch retries var jitter = baseDelaySeconds * Random.Shared.NextDouble() * 0.5; var retrySw = Stopwatch.StartNew(); diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ExportSharedCheckpointTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ExportSharedCheckpointTests.cs index acd11b3..76df0b2 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/ExportSharedCheckpointTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/ExportSharedCheckpointTests.cs @@ -1,4 +1,4 @@ -using System.Management.Automation; +using System.Management.Automation; using System.Net; using System.Text; using Mgx.Engine.Pagination; @@ -2437,8 +2437,9 @@ public void A_vanished_temp_over_an_unheld_output_still_exports_from_the_beginni /// The two runs the shapes above were taken from, released together on the promotion route. /// Both completed: whichever lost the promotion found the temp gone, warned that the items it /// counted were not on disk, deleted the position counting them, exported from the beginning - /// and moved its own temp over the file the other one was holding and appending to. One of - /// them ends the run instead now, and nothing it would have taken with it moves. + /// and moved its own temp over the file the other one was holding and appending to. A run + /// that loses a file ends there instead now, and nothing it would have taken with it moves - + /// whichever of the two files it lost, and whether one of them lost one or both did. /// [Fact] public void Two_exports_released_together_on_the_promotion_route_leave_one_with_the_file() @@ -2482,34 +2483,63 @@ public void Two_exports_released_together_on_the_promotion_route_leave_one_with_ Assert.True(r.Wait(TimeSpan.FromSeconds(60)), "an export never came up"); go.Set(); - // One of them got past the reconcile, so it holds the output and is parked on its - // first request. It stays there until the other has finished, which is what the run - // that stops does with nothing on the wire at all. - Assert.True(gate.Arrived.Wait(TimeSpan.FromSeconds(60)), - "neither export reached the wire"); + // A run that got past the reconcile holds the output and is parked on its first + // request. It stays there until the other has finished, which is what a run that + // stops does with nothing on the wire at all - and which file it loses, the output + // or the name the promotion stages into, is the race. Both of them losing one is an + // outcome of it: then no run has the output and none of them asks for a page. Assert.True( - SpinWait.SpinUntil(() => threads.Any(t => !t.IsAlive), TimeSpan.FromSeconds(60)), - "both exports were still running with one of them holding the output"); + SpinWait.SpinUntil(() => gate.Arrived.IsSet || threads.All(t => !t.IsAlive), + TimeSpan.FromSeconds(60)), + "neither export reached the wire or finished"); + if (gate.Arrived.IsSet) + { + Assert.True( + SpinWait.SpinUntil(() => threads.Any(t => !t.IsAlive), + TimeSpan.FromSeconds(60)), + "both exports were still running with one of them holding the output"); + } gate.Released.Set(); foreach (var t in threads) Assert.True(t.Join(TimeSpan.FromSeconds(60)), "an export never finished"); Assert.All(failures, Assert.Null); - // Exactly one run stopped, on one of the two files this checkpoint stands for, and - // it neither asked for a page nor said the items it was protecting were lost. - var stop = Assert.Single(results.SelectMany(r => r.Errors)); - Assert.StartsWith("Checkpoint", stop.FullyQualifiedErrorId, StringComparison.Ordinal); - Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + // Every run that stopped did so on one of the files this checkpoint stands for, + // saying it wrote nothing and changed nothing, and none of them said the items it + // was protecting were lost. + var stops = results.SelectMany(r => r.Errors).ToArray(); + Assert.All(stops, s => Assert.StartsWith("Checkpoint", s.FullyQualifiedErrorId, + StringComparison.Ordinal)); + Assert.All(stops, s => Assert.True( + s.Exception.Message.Contains("This run stops here; nothing was written.") + || s.Exception.Message.Contains("Nothing was changed;"), + s.Exception.Message)); Assert.DoesNotContain(results.SelectMany(r => r.Warnings), w => w.Contains("missing or incomplete")); - // One run had the file: every item once, in order, and the position spent by the run - // that finished rather than by the run that gave up on it. - Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], - File.ReadAllLines(output)); - Assert.False(File.Exists(checkpoint)); - Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + if (gate.Arrived.IsSet) + { + // One run had the file: every item once, in order, and the position spent by + // the run that finished rather than by the run that gave up on it. + Assert.Single(stops); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpoint)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + else + { + // Neither had it: the two files they were released on are the two files they + // leave, with every item the checkpoint counts still in the temp it names. + Assert.Equal(2, stops.Length); + Assert.False(File.Exists(output)); + var left = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(left!.TempFile); + Assert.True(File.Exists(Path.Combine(dir, left.TempFile!)), + "a run that changed nothing took the temp with it"); + } + Assert.Empty(Directory.GetFiles(dir, "*.adopt")); } finally diff --git a/tests/Mgx.IntegrationTests/Cmdlets/InvokeMgxRequestCollectionTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/InvokeMgxRequestCollectionTests.cs index 203a688..11c2ad5 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/InvokeMgxRequestCollectionTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/InvokeMgxRequestCollectionTests.cs @@ -1,4 +1,4 @@ -using System.Collections; +using System.Collections; using System.Management.Automation; using System.Net; using System.Text; @@ -51,6 +51,61 @@ public void Raw_emits_the_json_text_rather_than_a_hashtable() Assert.Contains("\"id\"", Assert.IsType(Assert.Single(result.Output).BaseObject)); } + [Fact] + public void Envelope_keeps_a_one_item_collection_a_collection() + { + using var host = new MgxTestHost(Json(Page("u1"))); + + var result = host.Run(ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users?$filter=id eq 'u1'") + .AddParameter("Envelope")); + + var payload = (Hashtable)Assert.Single(result.Output).BaseObject; + var items = Assert.IsAssignableFrom(payload["value"]).Cast().ToList(); + Assert.Equal("u1", (string)((Hashtable)Assert.Single(items))["id"]!); + } + + [Fact] + public void Envelope_leaves_an_entity_payload_as_it_is() + { + using var host = new MgxTestHost(Json("""{ "id": "u1" }""")); + + var result = host.Run(ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users/u1") + .AddParameter("Envelope")); + + Assert.Equal("u1", Id(Assert.Single(result.Output))); + } + + [Fact] + public void Envelope_holds_every_item_and_still_strips_the_transport_metadata() + { + using var host = new MgxTestHost(Json( + """{ "@odata.count": 7, "value": [ { "id": "u1" }, { "id": "u2" } ] }""")); + + var result = host.Run(ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("Envelope")); + + var payload = (Hashtable)Assert.Single(result.Output).BaseObject; + Assert.Equal(2, Assert.IsAssignableFrom(payload["value"]).Cast().Count()); + Assert.False(payload.ContainsKey("@odata.count")); + } + + [Fact] + public void Envelope_with_raw_emits_the_payload_whole() + { + using var host = new MgxTestHost(Json(Page("u1"))); + + var result = host.Run(ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("Envelope") + .AddParameter("Raw")); + + var text = Assert.IsType(Assert.Single(result.Output).BaseObject); + Assert.Contains("\"value\"", text); + Assert.Contains("\"u1\"", text); + } [Fact] public void A_collection_payload_from_an_entity_request_is_unwrapped_and_flagged_as_truncated() { diff --git a/tests/Mgx.IntegrationTests/Engine/Batch/BatchWriteTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/BatchWriteTests.cs index 1dd6099..99455e4 100644 --- a/tests/Mgx.IntegrationTests/Engine/Batch/BatchWriteTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Batch/BatchWriteTests.cs @@ -1,4 +1,4 @@ -using System.Net; +using System.Net; using System.Text.Json; using System.Text.Json.Nodes; using Mgx.Cmdlets.Cmdlets.Batch; @@ -630,7 +630,7 @@ public async Task BatchGet_RetryExhausted_ReturnsLastError() using var httpClient = new HttpClient(handler); using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); - var batchClient = new GraphBatchClient(client); + var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); var operations = new List { new("/users/user1", "GET") }; var result = await batchClient.ExecuteBatchIndexedAsync(operations); @@ -783,7 +783,7 @@ public async Task BatchGet_BatchLevelRetry_RecoverAfterChunkExhaustion() using var httpClient = new HttpClient(handler); using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); - var batchClient = new GraphBatchClient(client); + var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); var operations = new List { new("/users/user1", "GET") }; var result = await batchClient.ExecuteBatchIndexedAsync(operations); @@ -826,7 +826,7 @@ public async Task BatchPost_BatchLevelRetry_429Recovery() using var httpClient = new HttpClient(handler); using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); - var batchClient = new GraphBatchClient(client); + var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); var body = JsonSerializer.Deserialize("""{"displayName":"Test"}"""); var operations = new List { new("/users", "POST", body) }; @@ -846,7 +846,9 @@ public async Task BatchPost_BatchLevelRetry_429Recovery() [Fact] public async Task BatchGet_VerboseWriter_LogsClampEvent() { - // Server requests 300s Retry-After, client clamps to 120s (default). + // Server requests 300s Retry-After, client clamps it to the cap it was built with. + // The cap is 1s rather than the 120s default so the clamp is observable without the + // test sleeping out the default. // VerboseWriter should receive a clamping message. var throttledResponse = """ { @@ -869,7 +871,7 @@ public async Task BatchGet_VerboseWriter_LogsClampEvent() using var httpClient = new HttpClient(handler); using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); - var batchClient = new GraphBatchClient(client); + var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); var verboseMessages = new List(); batchClient.VerboseWriter = msg => verboseMessages.Add(msg); @@ -1197,7 +1199,7 @@ public async Task R2_3a_AllItemsFail_20Items_ExhaustsRetries() using var httpClient = new HttpClient(handler); using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); - var batchClient = new GraphBatchClient(client); + var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); var ops = Enumerable.Range(1, 20) .Select(i => new BatchOperation($"/users/{i}")) @@ -1326,7 +1328,7 @@ public async Task R2_3d_Phase2_TransportError_IsReportedWithTheResults() AttemptTimeoutSeconds = 5, TotalTimeoutSeconds = 30 }); - var batchClient = new GraphBatchClient(client); + var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); var ops = new[] { diff --git a/tests/Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs index 288747c..eee555d 100644 --- a/tests/Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs @@ -1,4 +1,4 @@ -using System.Management.Automation; +using System.Management.Automation; using System.Management.Automation.Host; using System.Net; using Mgx.Engine.Http; @@ -113,7 +113,7 @@ public async Task A_chunk_whose_retry_is_refused_keeps_the_items_the_server_answ using var httpClient = new HttpClient(wire); using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - var batchClient = new GraphBatchClient(client); + var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); var ops = Enumerable.Range(1, 20) .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) @@ -192,7 +192,7 @@ public async Task Operations_that_were_never_sent_count_as_neither_succeeded_nor using var httpClient = new HttpClient(new FirstChunkOnlyHandler()); using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - var batchClient = new GraphBatchClient(client); + var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); var ops = Enumerable.Range(1, 60) .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) @@ -362,7 +362,7 @@ public async Task An_item_answered_without_a_status_is_not_handed_back_as_never_ using var httpClient = new HttpClient(wire); using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - var batchClient = new GraphBatchClient(client); + var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); var ops = Enumerable.Range(1, 2) .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) @@ -416,7 +416,7 @@ public async Task A_chunk_answered_with_a_garbled_envelope_keeps_the_chunk_befor using var httpClient = new HttpClient(new SecondChunkGarbledHandler()); using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - var batchClient = new GraphBatchClient(client); + var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); var ops = Enumerable.Range(1, 40) .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) @@ -501,7 +501,7 @@ public async Task A_refused_retry_pass_keeps_what_the_run_already_knew() using var httpClient = new HttpClient(new RefusesTheRetryPassHandler(garbled: false)); using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - var batchClient = new GraphBatchClient(client); + var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); var ops = Enumerable.Range(1, 40) .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) @@ -580,7 +580,7 @@ public async Task A_chunk_refused_with_a_3xx_gives_its_items_a_failing_status() using var httpClient = new HttpClient(new NotModifiedHandler()); using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - var batchClient = new GraphBatchClient(client); + var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); var ops = Enumerable.Range(1, 20) .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) @@ -748,7 +748,7 @@ public async Task Items_the_retry_pass_applied_keep_that_answer_when_its_follow_ using var httpClient = new HttpClient(wire); using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - var batchClient = new GraphBatchClient(client); + var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); var ops = Enumerable.Range(1, 20) .Select(i => new BatchOperation("/users", "POST", From 04f53123abdd4ddbe4ddc1f6a3c2378e2b1598a7 Mon Sep 17 00:00:00 2001 From: Fabien Tschanz Date: Tue, 22 Sep 2026 06:56:00 +0200 Subject: [PATCH 10/10] Run the Pester suite against this fork, not against upstream's layout --- .gitignore | 5 - Modules/M365DSC.mgx/M365DSC.mgx.psd1 | 10 +- .../en-US/Mgx.Cmdlets.dll-Help.xml | 47 ++++++++++ .../help/Invoke-MgxBatchRequest.md | 33 ++++++- Modules/M365DSC.mgx/help/Invoke-MgxRequest.md | 2 + module/mgx.psd1 | 70 -------------- .../Cmdlets/Export/ExportMgxCollection.cs | 2 +- src/Mgx.Cmdlets/Mgx.Cmdlets.csproj | 2 +- tests/Ecosystem/Mgx.Ecosystem.Tests.ps1 | 8 +- .../Cmdlets/DeltaSharedCheckpointTests.cs | 91 ++++++++++++------- .../TestSetup/ModuleAssemblyClosureTests.cs | 7 +- tests/TestHarness.psm1 | 15 ++- .../Unit/Benchmarks/Mgx.BenchResult.Tests.ps1 | 2 +- tests/Unit/Mgx.Behaviour.Tests.ps1 | 26 +++--- tests/Unit/Mgx.Harness.Tests.ps1 | 12 +-- tests/Unit/Mgx.HelpFreshness.Tests.ps1 | 4 +- tests/benchmarks/18-spo-latency-clamp.ps1 | 2 +- tests/benchmarks/common.ps1 | 4 +- 18 files changed, 198 insertions(+), 144 deletions(-) delete mode 100644 module/mgx.psd1 diff --git a/.gitignore b/.gitignore index 380c6f2..59e6264 100644 --- a/.gitignore +++ b/.gitignore @@ -19,11 +19,6 @@ Modules/M365DSC.mgx/*.json Modules/M365DSC.mgx/Dependencies/ Modules/M365DSC.mgx/.build-hash -# The csproj stages an IDE build into module/ as well. Only the manifest there is source. -module/*.dll -module/*.pdb -module/*.json -module/Dependencies/ # Test output TestResults/ diff --git a/Modules/M365DSC.mgx/M365DSC.mgx.psd1 b/Modules/M365DSC.mgx/M365DSC.mgx.psd1 index 7209d4c..63a14e0 100644 --- a/Modules/M365DSC.mgx/M365DSC.mgx.psd1 +++ b/Modules/M365DSC.mgx/M365DSC.mgx.psd1 @@ -1,4 +1,4 @@ -@{ +@{ RootModule = 'M365DSC.mgx.psm1' ModuleVersion = '2.1.7' GUID = 'f978315f-75c0-48f5-b929-ca7a7757d1d2' @@ -49,6 +49,14 @@ LicenseUri = 'https://github.com/Microsoft365DSC/mgx/blob/main/LICENSE' ProjectUri = 'https://github.com/Microsoft365DSC/mgx' ReleaseNotes = @' +v2.1.7 +- Added Invoke-MgxRequest -Envelope, emitting the payload as the service sent it, collection envelope and all, so a collection holding one item can be told from a single entity. +- Fixed the batch retry loop waiting out its exponential fallback without regard for MaxRetryAfterSeconds, which capped only the delays a server asked for. + +v2.1.6 +- Merged upstream gromedev/mgx 2.1.2 through 2.1.5. +- Fixed a Graph error reporting only its top level message: the property that was actually wrong sat in the innerError of the response and was discarded. + v2.1.1 - Fixed Invoke-MgxRequest failing with a missing System.IO.Pipelines assembly on PowerShell 7.6 hosts that do not ship it. diff --git a/Modules/M365DSC.mgx/en-US/Mgx.Cmdlets.dll-Help.xml b/Modules/M365DSC.mgx/en-US/Mgx.Cmdlets.dll-Help.xml index 0e15df7..8e23239 100644 --- a/Modules/M365DSC.mgx/en-US/Mgx.Cmdlets.dll-Help.xml +++ b/Modules/M365DSC.mgx/en-US/Mgx.Cmdlets.dll-Help.xml @@ -2084,6 +2084,29 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP False + + FollowNextLink + + Drain `@odata.nextLink` in sub-response bodies, merging the pages into each result. Off by default, where a sub-response that carries a next link is returned with the items of its first page and the link beside them. Follow-up pages are submitted as further batches, so N partial collections drain in ceil(N/20) requests per page rather than N. + + + SwitchParameter + + + False + + + MaxPage + + Ceiling on the pages drained per sub-request when -FollowNextLink is on. 0, the default, is unlimited. A sub-request stopped at the ceiling with more to read is reported as such rather than returned as though it were whole. + + Int32 + + Int32 + + + 0 + ProgressAction @@ -2227,6 +2250,30 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP False + + FollowNextLink + + Drain `@odata.nextLink` in sub-response bodies, merging the pages into each result. Off by default, where a sub-response that carries a next link is returned with the items of its first page and the link beside them. Follow-up pages are submitted as further batches, so N partial collections drain in ceil(N/20) requests per page rather than N. + + SwitchParameter + + SwitchParameter + + + False + + + MaxPage + + Ceiling on the pages drained per sub-request when -FollowNextLink is on. 0, the default, is unlimited. A sub-request stopped at the ceiling with more to read is reported as such rather than returned as though it were whole. + + Int32 + + Int32 + + + 0 + ProgressAction diff --git a/Modules/M365DSC.mgx/help/Invoke-MgxBatchRequest.md b/Modules/M365DSC.mgx/help/Invoke-MgxBatchRequest.md index 9a5da17..68a72b5 100644 --- a/Modules/M365DSC.mgx/help/Invoke-MgxBatchRequest.md +++ b/Modules/M365DSC.mgx/help/Invoke-MgxBatchRequest.md @@ -1,4 +1,4 @@ ---- +--- external help file: Mgx.Cmdlets.dll-Help.xml Module Name: Mgx online version: https://github.com/gromedev/mgx/blob/main/module/help/Invoke-MgxBatchRequest.md @@ -15,6 +15,7 @@ Bundle multiple Graph API requests into /$batch calls. ``` Invoke-MgxBatchRequest [-Uri] [-Method ] [-Body ] [-ConsistencyLevel ] [-Headers ] [-ThrottlePriority ] [-ApiVersion ] [-DeadLetterPath ] + [-FollowNextLink] [-MaxPage ] [-ProgressAction ] [-WhatIf] [-Confirm] [] ``` @@ -298,6 +299,36 @@ Accept pipeline input: False Accept wildcard characters: False ``` +### -FollowNextLink +Drain `@odata.nextLink` in sub-response bodies, merging the pages into each result. Off by default, where a sub-response that carries a next link is returned with the items of its first page and the link beside them. Follow-up pages are submitted as further batches, so N partial collections drain in ceil(N/20) requests per page rather than N. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -MaxPage +Ceiling on the pages drained per sub-request when -FollowNextLink is on. 0, the default, is unlimited. A sub-request stopped at the ceiling with more to read is reported as such rather than returned as though it were whole. + +```yaml +Type: Int32 +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: 0 +Accept pipeline input: False +Accept wildcard characters: False +``` + ### -ProgressAction Determines how the cmdlet responds to progress updates. diff --git a/Modules/M365DSC.mgx/help/Invoke-MgxRequest.md b/Modules/M365DSC.mgx/help/Invoke-MgxRequest.md index 250246e..d41de63 100644 --- a/Modules/M365DSC.mgx/help/Invoke-MgxRequest.md +++ b/Modules/M365DSC.mgx/help/Invoke-MgxRequest.md @@ -16,6 +16,7 @@ General-purpose resilient client for any Microsoft Graph endpoint. ``` Invoke-MgxRequest [-Uri] [-Method ] [-Body ] [-Property ] [-ExpandProperty ] [-ConsistencyLevel ] [-Headers ] [-ApiVersion ] [-Raw] + [-Envelope] [-Filter ] [-Sort ] [-Search ] [-Skip ] [-All] [-Top ] [-PageSize ] [-CountVariable ] [-CheckpointPath ] [-NoPageSize] [-ProgressAction ] [-WhatIf] [-Confirm] [] @@ -25,6 +26,7 @@ Invoke-MgxRequest [-Uri] [-Method ] [-Body ] [-Property ``` Invoke-MgxRequest [-Uri] [-Method ] [-Body ] [-Property ] [-ExpandProperty ] [-ConsistencyLevel ] [-Headers ] [-ApiVersion ] [-Raw] + [-Envelope] [-Filter ] [-Sort ] [-Search ] [-Skip ] [-All] [-Top ] [-PageSize ] [-CountVariable ] [-CheckpointPath ] [-NoPageSize] [-InputObject ] [-Concurrency ] [-SkipNotFound] [-SkipForbidden] [-ProgressAction ] [-WhatIf] diff --git a/module/mgx.psd1 b/module/mgx.psd1 deleted file mode 100644 index 9fee977..0000000 --- a/module/mgx.psd1 +++ /dev/null @@ -1,70 +0,0 @@ -@{ - RootModule = 'mgx.psm1' - ModuleVersion = '2.1.5' - GUID = 'a3f7e8d2-5b4c-4a1e-9f6d-2c8b0e3a7d5f' - Author = 'Thomas Maillo Grome' - CompanyName = 'Mgx' - Copyright = '(c) 2026 Thomas Maillo Grome. All rights reserved.' - Description = 'Resilient companion for Microsoft.Graph PowerShell. Adds retry, circuit breaker, rate limiting, streaming pagination, batching, and fan-out to any Graph API endpoint.' - - PowerShellVersion = '7.4' - CompatiblePSEditions = @('Core') - - FormatsToProcess = @('mgx.Format.ps1xml') - - # Pre-load Mgx.Engine.dll so it resolves into the same load context - # as Mgx.Cmdlets.dll. Without this, MgxTelemetrySummary (a record type - # returned by MgxTelemetryCollector.GetSummary()) fails to load at JIT - # time with TypeLoadException when Get-MgxTelemetry is called. - RequiredAssemblies = @('Mgx.Engine.dll') - - # Microsoft.Graph.Authentication is deliberately NOT in RequiredModules. - # - # Auth is discovered reflectively at call time (GraphSession.Instance, falling back to - # Get-MgContext), so nothing here links against the SDK and the module imports without it. - # Declaring it would force the dependency on every consumer - including hosts that supply - # their own Graph auth and only want the resilience layer - and would install a second copy - # alongside whatever they already load. Cmdlets that need a token raise - # GraphAuthModuleNotLoaded with install instructions when it is genuinely absent. - - CmdletsToExport = @( - 'Invoke-MgxRequest' - 'Invoke-MgxBatchRequest' - 'Export-MgxCollection' - 'Expand-MgxRelation' - 'Set-MgxOption' - 'Get-MgxOption' - 'Enable-MgxResilience' - 'Disable-MgxResilience' - 'Get-MgxResilience' - 'Get-MgxTelemetry' - 'Sync-MgxDelta' - 'Get-MgxContent' - ) - - FunctionsToExport = @() - VariablesToExport = @() - AliasesToExport = @() - - PrivateData = @{ - PSData = @{ - Tags = @('Microsoft', 'Graph', 'MicrosoftGraph', 'API', 'Azure', 'EntraID', 'Resilience', 'PowerShell', 'Polly', 'Retry', 'RateLimit', 'Batch', 'Delta', 'Throttling', 'Pagination') - LicenseUri = 'https://github.com/gromedev/mgx/blob/main/LICENSE' - ProjectUri = 'https://github.com/gromedev/mgx' - ReleaseNotes = @' -v2.1.5 -Fixed -- Two runs resuming or recovering one interrupted export or sync no longer write into the same file: the second stops and names the file the first holds. -- Recovering an interrupted run never follows a link, waits on a pipe, or starts over on a copy that failed; every stop says what going on would have done and leaves both files as found. -- A checkpoint from before 2.1 keeps the temp file it stands for when another run refuses it, and Sync-MgxDelta recovers that shape as Export-MgxCollection does. -- Invoke-MgxBatchRequest refuses an item it cannot read instead of ending the run, cuts every pre-authenticated URL from the dead-letter file, checks the file before sending, and keeps item errors and telemetry under a Stop preference. -- Set-MgxOption previews and prompts only for a change it makes; Invoke-MgxRequest asks its checkpoint question once per run; -WhatIf previews on every cmdlet report what the run would do. -- Removing the module returns every option, the endpoint and the telemetry counters to their defaults. -Added -- A committed benchmark baseline with a comparison run, a programmable fault plan for the mock Graph server, and pathological and interference gauntlets. -- Fault injection at every point of an operation, pagination checks at page-size boundaries, and a help-freshness test over the compiled help. -See CHANGELOG.md for the full list. -'@ - } - } -} diff --git a/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs b/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs index a60c248..50f06bb 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs @@ -17,7 +17,7 @@ namespace Mgx.Cmdlets.Cmdlets.Export; /// to prevent duplicate items on crash resume (H6 dedup fix). /// [Cmdlet(VerbsData.Export, "MgxCollection", SupportsShouldProcess = true)] -[OutputType(typeof(PSObject))] +[OutputType(typeof(Models.MgxExportResult))] public class ExportMgxCollection : MgxCmdletBase { [Parameter(Mandatory = true, Position = 0)] diff --git a/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj b/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj index 44d5c54..d6030ff 100644 --- a/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj +++ b/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj @@ -59,7 +59,7 @@ - $(MSBuildThisFileDirectory)../../module + $(MSBuildThisFileDirectory)../../Modules/M365DSC.mgx $(ModuleRoot)/Dependencies diff --git a/tests/Ecosystem/Mgx.Ecosystem.Tests.ps1 b/tests/Ecosystem/Mgx.Ecosystem.Tests.ps1 index 755feb2..9758a6e 100644 --- a/tests/Ecosystem/Mgx.Ecosystem.Tests.ps1 +++ b/tests/Ecosystem/Mgx.Ecosystem.Tests.ps1 @@ -24,7 +24,7 @@ BeforeDiscovery { BeforeAll { $repo = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent - $script:Manifest = Join-Path $repo 'module/mgx.psd1' + $script:Manifest = Join-Path $repo 'Modules/M365DSC.mgx/M365DSC.mgx.psd1' if (-not (Test-Path $script:Manifest)) { throw "Built module not found at $script:Manifest - run ./build.ps1 first." } # The battery runs after the permutation's imports. Order inside matters: the side-load @@ -36,7 +36,7 @@ function Check([string]$name, [scriptblock]$body) { try { $results[$name] = @{ ok = [bool](& $body); detail = '' } } catch { $results[$name] = @{ ok = $false; detail = $_.Exception.Message } } } -Check 'cmdletsResolve' { (Get-Command -Module mgx -CommandType Cmdlet).Count -eq 12 } +Check 'cmdletsResolve' { (Get-Command -Module M365DSC.mgx -CommandType Cmdlet).Count -eq 12 } Check 'noGraphSideLoad' { @(Get-Module 'Microsoft.Graph*').Count -eq 0 } Check 'optionRoundTrip' { Set-MgxOption -MaxRetryAttempts 4; (Get-MgxOption).MaxRetryAttempts -eq 4 } Check 'telemetry' { $null -ne (Get-MgxTelemetry) } @@ -49,9 +49,9 @@ Check 'depsResolve' { ($null -ne $polly) -and ($null -ne $rate) -and ($null -ne $stj) } Check 'removeReimport' { - Remove-Module mgx -Force + Remove-Module M365DSC.mgx -Force Import-Module '__MANIFEST__' -ErrorAction Stop - (Get-Command -Module mgx -CommandType Cmdlet).Count -eq 12 -and $null -ne (Get-MgxOption) + (Get-Command -Module M365DSC.mgx -CommandType Cmdlet).Count -eq 12 -and $null -ne (Get-MgxOption) } Check 'resilienceNamesTheSession' { # ThrowTerminatingError is statement-terminating; no -ErrorAction downgrades it, diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DeltaSharedCheckpointTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DeltaSharedCheckpointTests.cs index c7de015..29e86e2 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/DeltaSharedCheckpointTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/DeltaSharedCheckpointTests.cs @@ -1,4 +1,4 @@ -using System.Reflection; +using System.Reflection; using System.Management.Automation; using System.Net; using System.Text; @@ -2659,8 +2659,9 @@ public void A_vanished_temp_over_a_held_output_stops_the_sync() /// Two syncs released together on the promotion route. Both completed: whichever lost the /// promotion found the temp gone, warned that the changes it counted were not on disk, deleted /// the position counting them, re-enumerated over the output the other one was holding, and - /// moved the delta token past work that was no longer in any file. One of them ends the run - /// instead now, and the token moves once. + /// moved the delta token past work that was no longer in any file. A run that loses a file + /// ends there instead now - whichever of the files it lost, and whether one of them lost one + /// or both did - and the token moves once, or not at all. /// [Fact] public void Two_syncs_released_together_on_the_promotion_route_advance_the_token_once() @@ -2708,14 +2709,22 @@ public void Two_syncs_released_together_on_the_promotion_route_advance_the_token Assert.True(r.Wait(TimeSpan.FromSeconds(60)), "a sync never came up"); go.Set(); - // One of them got past the reconcile, so it holds the output and is parked on its - // first request. It stays there until the other has finished, which is what the run - // that stops does with nothing on the wire at all. - Assert.True(gate.Arrived.Wait(TimeSpan.FromSeconds(60)), - "neither sync reached the wire"); + // A run that got past the reconcile holds the output and is parked on its first + // request. It stays there until the other has finished, which is what a run that + // stops does with nothing on the wire at all - and both of them losing a file is an + // outcome of the race too: then no sync has the output and none of them asks for a + // page. Assert.True( - SpinWait.SpinUntil(() => threads.Any(t => !t.IsAlive), TimeSpan.FromSeconds(60)), - "both syncs were still running with one of them holding the output"); + SpinWait.SpinUntil(() => gate.Arrived.IsSet || threads.All(t => !t.IsAlive), + TimeSpan.FromSeconds(60)), + "neither sync reached the wire or finished"); + if (gate.Arrived.IsSet) + { + Assert.True( + SpinWait.SpinUntil(() => threads.Any(t => !t.IsAlive), + TimeSpan.FromSeconds(60)), + "both syncs were still running with one of them holding the output"); + } gate.Released.Set(); foreach (var t in threads) @@ -2734,33 +2743,53 @@ public void Two_syncs_released_together_on_the_promotion_route_advance_the_token // whichever route the run took, and what is asserted of the run itself is the same // either way, below: one winner, every change once, and nothing of this one's left // on the disk. - var stop = Assert.Single(results.SelectMany(r => r.Errors)); - Assert.StartsWith("Checkpoint", stop.FullyQualifiedErrorId, StringComparison.Ordinal); - if (stop.FullyQualifiedErrorId.StartsWith("CheckpointStagingFailed", - StringComparison.Ordinal)) + var stops = results.SelectMany(r => r.Errors).ToArray(); + Assert.All(stops, s => Assert.StartsWith("Checkpoint", s.FullyQualifiedErrorId, + StringComparison.Ordinal)); + foreach (var stop in stops) + { + if (stop.FullyQualifiedErrorId.StartsWith("CheckpointStagingFailed", + StringComparison.Ordinal)) + { + // The promoter's own staged copy, reached between its create and its rename. + // Named as a staging that failed and not as an entry nothing could be made + // of, and the sentence says the changes are still whole in the temp the + // checkpoint names. Which refusal the open came back with is the platform's + // and the moment's, so the reason it quotes is not asserted. + Assert.Contains("whole, but staging them into", stop.Exception.Message); + Assert.Contains("Nothing was changed;", stop.Exception.Message); + } + else + { + Assert.Contains("This run stops here; nothing was written.", + stop.Exception.Message); + } + } + Assert.DoesNotContain(results.SelectMany(r => r.Warnings), + w => w.Contains("missing or incomplete")); + + if (gate.Arrived.IsSet) { - // The promoter's own staged copy, held from the create through the rename. Named - // as that and not as an entry nothing could be made of, and the sentence says - // the changes are still whole in the temp the checkpoint names. - Assert.Contains("another run has the staged copy open", stop.Exception.Message); - Assert.Contains("whole, but staging them into", stop.Exception.Message); - Assert.Contains("Nothing was changed;", stop.Exception.Message); + // One run had the file: every change once, in order, the position spent by the + // run that finished, and the token moved exactly once. + Assert.Single(stops); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpointPath)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); } else { - Assert.Contains("This run stops here; nothing was written.", - stop.Exception.Message); + // Neither had it: the files they were released on are the files they leave, and + // the token stands where the interrupted run left it. + Assert.Equal(2, stops.Length); + Assert.False(File.Exists(output)); + var left = PaginationCheckpoint.Load(checkpointPath); + Assert.NotNull(left!.TempFile); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); } - Assert.DoesNotContain(results.SelectMany(r => r.Warnings), - w => w.Contains("missing or incomplete")); - // One run had the file: every change once, in order, the position spent by the run - // that finished, and the token moved exactly once. - Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], - File.ReadAllLines(output)); - Assert.False(File.Exists(checkpointPath)); - Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); - Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); Assert.Empty(Directory.GetFiles(dir, "*.adopt")); } finally diff --git a/tests/Mgx.IntegrationTests/TestSetup/ModuleAssemblyClosureTests.cs b/tests/Mgx.IntegrationTests/TestSetup/ModuleAssemblyClosureTests.cs index 8869f23..f79bacf 100644 --- a/tests/Mgx.IntegrationTests/TestSetup/ModuleAssemblyClosureTests.cs +++ b/tests/Mgx.IntegrationTests/TestSetup/ModuleAssemblyClosureTests.cs @@ -1,4 +1,4 @@ -using System.Reflection.Metadata; +using System.Reflection.Metadata; using System.Reflection.PortableExecutable; using System.Xml.Linq; @@ -73,9 +73,10 @@ private static string FindRepositoryPath(string relativePath, string startDirect private static string FindRepositoryPath(string relativePath) => FindRepositoryPath(relativePath, AppContext.BaseDirectory); - /// The staged module folder - the directory holding mgx.psd1. + /// The staged module folder - the directory holding the module manifest. private static string ModuleRoot() => - Path.GetDirectoryName(FindRepositoryPath(Path.Combine("module", "mgx.psd1")))!; + Path.GetDirectoryName(FindRepositoryPath( + Path.Combine("Modules", "M365DSC.mgx", "M365DSC.mgx.psd1")))!; /// /// The staged Dependencies folder. Absent means the module has not been built, which is the diff --git a/tests/TestHarness.psm1 b/tests/TestHarness.psm1 index 6f684e9..4f51a20 100644 --- a/tests/TestHarness.psm1 +++ b/tests/TestHarness.psm1 @@ -1,4 +1,4 @@ -#Requires -Version 7.0 +#Requires -Version 7.0 <# .SYNOPSIS @@ -323,7 +323,18 @@ function Invoke-PesterTest $configuration.CodeCoverage.Enabled = $false Write-Host -Object 'Running all mgx PowerShell Unit Tests' - return Invoke-Pester -Configuration $configuration + $results = Invoke-Pester -Configuration $configuration + + foreach ($container in $results.Containers | Where-Object { -not $_.Passed }) + { + if ($container.Result -eq 'Failed' -and $container.Tests.Count -eq 0) + { + $name = if ($container.Item -is [System.IO.FileInfo]) { $container.Item.FullName } else { "$($container.Item)" } + Write-Warning -Message "The test file '$name' could not be run at all, so none of its tests were: $($container.ErrorRecord.Exception.Message)" + } + } + + return $results } <# diff --git a/tests/Unit/Benchmarks/Mgx.BenchResult.Tests.ps1 b/tests/Unit/Benchmarks/Mgx.BenchResult.Tests.ps1 index 014da36..69c2940 100644 --- a/tests/Unit/Benchmarks/Mgx.BenchResult.Tests.ps1 +++ b/tests/Unit/Benchmarks/Mgx.BenchResult.Tests.ps1 @@ -256,7 +256,7 @@ Describe 'Write-BenchResult -Identity' { It 'reads this process with no identity supplied' { $entry = Write-BenchProbe -Name 'identity-absent' -Session -UserCount 19 - $entry.MgxVersion | Should -Be (Get-Module Mgx -ErrorAction SilentlyContinue)?.Version?.ToString() + $entry.MgxVersion | Should -Be (Get-Module M365DSC.mgx -ErrorAction SilentlyContinue)?.Version?.ToString() $entry.TenantId | Should -Be $script:FakeTenantId $entry.DirectoryObjectCount | Should -Be 19 $entry.DocumentedBudgetRuPerSecond | Should -Be 350 diff --git a/tests/Unit/Mgx.Behaviour.Tests.ps1 b/tests/Unit/Mgx.Behaviour.Tests.ps1 index 7d7654b..7198a2e 100644 --- a/tests/Unit/Mgx.Behaviour.Tests.ps1 +++ b/tests/Unit/Mgx.Behaviour.Tests.ps1 @@ -1,4 +1,4 @@ -#Requires -Modules Pester +#Requires -Modules Pester <# The behavior of the built module as PowerShell sees it: the cmdlet and parameter contract, @@ -19,16 +19,16 @@ #> BeforeAll { - $ModulePath = Join-Path $PSScriptRoot '../../module/mgx.psd1' + $ModulePath = Join-Path $PSScriptRoot '../../Modules/M365DSC.mgx/M365DSC.mgx.psd1' Import-Module $ModulePath -Force } Describe 'Module Loading' { It 'Should import Mgx module' { - $module = Get-Module Mgx + $module = Get-Module M365DSC.mgx $module | Should -Not -BeNullOrEmpty # Read expected version from manifest to avoid hardcoded values breaking on version bumps - $manifest = Import-PowerShellDataFile (Join-Path $PSScriptRoot '../../module/mgx.psd1') + $manifest = Import-PowerShellDataFile (Join-Path $PSScriptRoot '../../Modules/M365DSC.mgx/M365DSC.mgx.psd1') $module.Version | Should -Be $manifest.ModuleVersion } @@ -36,7 +36,7 @@ Describe 'Module Loading' { # Filtered to -CommandType Cmdlet on purpose: Get-Command -Module also counts # exported functions, so an unfiltered count would drift for reasons that have # nothing to do with the compiled surface. - $commands = (Get-Command -Module Mgx -CommandType Cmdlet).Name | Sort-Object + $commands = (Get-Command -Module M365DSC.mgx -CommandType Cmdlet).Name | Sort-Object $commands | Should -Contain 'Invoke-MgxRequest' $commands | Should -Contain 'Invoke-MgxBatchRequest' $commands | Should -Contain 'Export-MgxCollection' @@ -64,12 +64,12 @@ Describe 'Module Loading' { # # Runs in a child process because the precondition is "Polly.Core has never # been loaded", which the parent suite has already violated by this point. - $modulePath = Join-Path $PSScriptRoot '../../module/mgx.psd1' + $modulePath = Join-Path $PSScriptRoot '../../Modules/M365DSC.mgx/M365DSC.mgx.psd1' $probe = @" Import-Module '$modulePath' -Force -try { Remove-Module mgx -ErrorAction Stop } +try { Remove-Module M365DSC.mgx -ErrorAction Stop } catch { Write-Output ('THREW: ' + `$_.Exception.Message.Split([char]10)[0]); exit 1 } -if (Get-Module mgx) { Write-Output 'STILL LOADED'; exit 1 } +if (Get-Module M365DSC.mgx) { Write-Output 'STILL LOADED'; exit 1 } Write-Output 'OK' "@ # This assertion needs a FRESH session, so it must spawn a child host. Under a @@ -541,9 +541,9 @@ Describe 'Disable-MgxResilience Parameter Compatibility' { } Describe 'Get-MgxResilience Parameter Compatibility' { - It 'Should have OutputType of PSObject' { + It 'Should have the OutputType its help documents' { $outputType = (Get-Command Get-MgxResilience).OutputType - $outputType.Name | Should -Contain 'System.Management.Automation.PSObject' + $outputType.Name | Should -Contain 'Mgx.Cmdlets.Models.MgxResilienceOutput' } It 'Should have no mandatory parameters' { @@ -1096,7 +1096,7 @@ Describe 'Sync-MgxDelta Session Endpoint' { # connected session without a tenant, and the child process keeps that connection (and # the module's static endpoint state) out of this one. BeforeAll { - $script:modulePath = Join-Path $PSScriptRoot '../../module/mgx.psd1' + $script:modulePath = Join-Path $PSScriptRoot '../../Modules/M365DSC.mgx/M365DSC.mgx.psd1' $script:testDir = Join-Path ([System.IO.Path]::GetTempPath()) "mgx-endpoint-tests-$(New-Guid)" New-Item -ItemType Directory -Path $script:testDir -Force | Out-Null @@ -1346,7 +1346,7 @@ Describe 'HttpClient Reset Leaves A Held Client Alone' { BeforeAll { # Ensure Mgx.Engine types are loaded (transitive dep of Mgx.Cmdlets) - $enginePath = Join-Path $PSScriptRoot '../../module/Mgx.Engine.dll' + $enginePath = Join-Path $PSScriptRoot '../../Modules/M365DSC.mgx/Mgx.Engine.dll' [System.Reflection.Assembly]::LoadFrom($enginePath) | Out-Null $script:CmdletBaseType = [Mgx.Cmdlets.Base.MgxCmdletBase] @@ -1425,7 +1425,7 @@ Describe 'Enable-MgxResilience SDK Client Wrap' { return } - $modulePath = Join-Path $PSScriptRoot '../../module/mgx.psd1' + $modulePath = Join-Path $PSScriptRoot '../../Modules/M365DSC.mgx/M365DSC.mgx.psd1' # Add-MgEnvironment PERSISTS custom environments, so the name is unique per run # and removed again in the probe's finally block. $envName = 'MgxWrap' + [guid]::NewGuid().ToString('N').Substring(0, 10) diff --git a/tests/Unit/Mgx.Harness.Tests.ps1 b/tests/Unit/Mgx.Harness.Tests.ps1 index 1279a65..568a999 100644 --- a/tests/Unit/Mgx.Harness.Tests.ps1 +++ b/tests/Unit/Mgx.Harness.Tests.ps1 @@ -1,4 +1,4 @@ -#Requires -Modules Pester +#Requires -Modules Pester <# Invoke-TestHarness's own result contract, from tests/TestHarness.psm1: a container that @@ -29,12 +29,12 @@ param( Import-Module -Name $HarnessModule -Force $warnings = $null $result = Invoke-TestHarness -TestPath $TestPath -TestResultsFile $TestResultsFile ` - -IgnoreCodeCoverage -WarningVariable warnings -WarningAction SilentlyContinue + -IgnoreCodeCoverage -SkipDotNetTests -WarningVariable warnings -WarningAction SilentlyContinue [pscustomobject]@{ - Result = $result.Result - FailedCount = $result.FailedCount - FailedContainersCount = $result.FailedContainersCount - PassedCount = $result.PassedCount + Result = $result.Pester.Result + FailedCount = $result.Pester.FailedCount + FailedContainersCount = $result.Pester.FailedContainersCount + PassedCount = $result.Pester.PassedCount Warnings = @($warnings | ForEach-Object { $_.ToString() }) } | ConvertTo-Json -Depth 5 | Set-Content -Path $ResultFile -Encoding utf8 '@ diff --git a/tests/Unit/Mgx.HelpFreshness.Tests.ps1 b/tests/Unit/Mgx.HelpFreshness.Tests.ps1 index bc2ad12..8b4eaa9 100644 --- a/tests/Unit/Mgx.HelpFreshness.Tests.ps1 +++ b/tests/Unit/Mgx.HelpFreshness.Tests.ps1 @@ -457,7 +457,7 @@ Describe 'Compiled help matches its markdown source' { # Get-Command reports it among those, while platyPS 0.14.2 does not know it and writes it # into the block as an ordinary one: comparing it would say every block is wrong. $repo = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent - Import-Module "$repo/module/mgx.psd1" -Force + Import-Module "$repo/Modules/M365DSC.mgx/M365DSC.mgx.psd1" -Force $common = [System.Management.Automation.PSCmdlet]::CommonParameters $mismatches = [System.Collections.Generic.List[string]]::new() @@ -537,7 +537,7 @@ Describe 'Compiled help matches its markdown source' { # SwitchParameter, ActionPreference, Hashtable, and an array keeps its brackets, String[]. # Never the namespace-qualified name. $repo = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent - Import-Module "$repo/module/mgx.psd1" -Force + Import-Module "$repo/Modules/M365DSC.mgx/M365DSC.mgx.psd1" -Force $mismatches = [System.Collections.Generic.List[string]]::new() foreach ($md in Get-ChildItem $script:HelpDir -Filter '*.md') { diff --git a/tests/benchmarks/18-spo-latency-clamp.ps1 b/tests/benchmarks/18-spo-latency-clamp.ps1 index 795ca39..b77d97b 100644 --- a/tests/benchmarks/18-spo-latency-clamp.ps1 +++ b/tests/benchmarks/18-spo-latency-clamp.ps1 @@ -159,7 +159,7 @@ $resultsDir = Join-Path $PSScriptRoot 'results' if (-not (Test-Path $resultsDir)) { New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null } $rowsPath = Join-Path $resultsDir '18-spo-latency-clamp.rows.jsonl' -$modulePath = Join-Path $PSScriptRoot '../../module/mgx.psd1' +$modulePath = Join-Path $PSScriptRoot '../../Modules/M365DSC.mgx/M365DSC.mgx.psd1' if (-not (Test-Path $modulePath)) { $modulePath = 'Mgx' } $startedAt = Get-Date $deadline = [datetime]::UtcNow.AddMinutes($Minutes) diff --git a/tests/benchmarks/common.ps1 b/tests/benchmarks/common.ps1 index 15c17c1..a48acfe 100644 --- a/tests/benchmarks/common.ps1 +++ b/tests/benchmarks/common.ps1 @@ -188,7 +188,7 @@ function Import-MgxLocal { # different-versioned Users then fails with "assembly with same name already loaded". if (-not (Get-Module Microsoft.Graph.Users)) { Import-Module Microsoft.Graph.Users } # Local build first (repo checkout), gallery module as fallback. - $local = Join-Path $PSScriptRoot '../../module/mgx.psd1' + $local = Join-Path $PSScriptRoot '../../Modules/M365DSC.mgx/M365DSC.mgx.psd1' if (Test-Path $local) { Import-Module $local -Force } else { Import-Module Mgx -Force } } @@ -908,7 +908,7 @@ function Write-BenchResult { $stamp = $Identity if (-not $PSBoundParameters.ContainsKey('Identity')) { $stamp = [pscustomobject]@{ - MgxVersion = (Get-Module Mgx -ErrorAction SilentlyContinue)?.Version?.ToString() + MgxVersion = (Get-Module M365DSC.mgx -ErrorAction SilentlyContinue)?.Version?.ToString() SdkVersion = (Get-Module Microsoft.Graph.Authentication -ErrorAction SilentlyContinue)?.Version?.ToString() TenantId = $fact.TenantId DirectoryObjectCount = $fact.DirectoryObjectCount