diff --git a/.gitignore b/.gitignore index 37c093a..59e6264 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -# ignore +# ignore .vs/ .vscode/ .DS_Store @@ -6,6 +6,7 @@ # tmp bin/ obj/ +tmp/ publish/ *.jsonl *.json @@ -18,6 +19,7 @@ Modules/M365DSC.mgx/*.json Modules/M365DSC.mgx/Dependencies/ Modules/M365DSC.mgx/.build-hash + # Test output TestResults/ tests/TestResults.xml @@ -28,3 +30,12 @@ artifacts/ # Re-included from the blanket *.json above, since it pins the ReportGenerator version CI restores !.config/dotnet-tools.json + +# results/ above stays local: it is the append-only per-run log, and one machine's numbers are +# not another's. The baseline beside it is the pinned snapshot a later run is read against, and +# it is committed, so it has to be re-included after *.json has had its say. Nothing here +# re-includes results/ itself. +!tests/benchmarks/baseline.json + +# Generated by the CI compat job when run locally +compat-surface-tests.ps1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c4370b..836a0c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,36 @@ -# Changelog +# Changelog + +## 2.1.7 + +### Added + +- `Invoke-MgxRequest -Envelope` emits the payload as the service sent it, collection envelope and all, instead of the items inside it. Without it a collection holding one item and a single entity arrive as the same thing - one object - which a caller that has to tell them apart cannot do. `-All` keeps aggregating pages and ignores the switch. Upstream does not carry this. + +### Fixed + +- The batch retry loop waited out its exponential fallback without regard for `MaxRetryAfterSeconds`, which caps only the delays a server asks for. Both now answer to the same ceiling. + +## 2.1.6 + +Merges upstream `gromedev/mgx` 2.1.2 through 2.1.5. The module name, version line, target framework, CI and examples remain this fork's. The version jumps from 2.1.1 to 2.1.6 so the two version lines stop colliding: upstream had already released its own 2.1.2. + +### Fixed + +- A Graph error reported only its top level message, which for a validation failure is only `The request is invalid.`, while the property that was actually wrong sat in the `innerError` of the response and was discarded. `GraphServiceException` now appends that message and exposes it as `InnerErrorMessage`. Upstream does not carry this. + +### From upstream + +- Request bodies serialize correctly for enums, byte arrays, `TimeSpan`, `DateTime` without a `Kind`, `PSCustomObject` members and non-ASCII text, and refuse `SecureString`, credentials, script blocks and self-referencing objects with an explicit error instead of reflection noise or a stack overflow. +- URI and header handling: no double encoding of pre-encoded `-Filter`, `-Search`, `-Property`, `-Sort` and `-ExpandProperty`, `#` no longer truncates a path, absolute `-Uri` inputs are preserved, array `-Headers` values are sent, content headers are kept, and a caller's `client-request-id` is left alone. +- Response handling: 204 and empty bodies, HTML and other non-JSON bodies, unparseable JSON, a UTF-8 BOM and a declared charset are all handled without an unhandled exception. +- Failure classification is unified across retries, circuit breaking, batch retry, content download and adaptive pacing, and error records carry a specific `ErrorCategory` rather than `NotSpecified`. +- `Invoke-MgxBatchRequest` writes a per-item error record for every failed item, halts on a chunk refusal, reports never-sent items as such, and keeps applied server writes out of the dead-letter file. +- Dead-letter files and `-Debug` traces redact the same pre-authenticated URLs and sensitive names, and an unwritable dead-letter path is refused before the batch is sent. +- `Export-MgxCollection` and `Sync-MgxDelta` no longer let a second run resume the same interrupted operation, reject checkpoints belonging to another run or an older release, and stop with the files as found rather than re-enumerating over them. + +### Changed + +- The engine and cmdlet sources now follow upstream's wording again. The fork's earlier comment rewrites were cosmetic and cost a conflict on every upstream release. ## 2.1.1 diff --git a/Directory.Build.props b/Directory.Build.props index 5e1fa40..adba6ed 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,7 +1,7 @@ - 2.1.1 + 2.1.7 Thomas Maillo Grome, Fabien Tschanz Mgx (c) 2026 Thomas Maillo Grome, (c) 2026 Fabien Tschanz. All rights reserved. diff --git a/Modules/M365DSC.mgx/M365DSC.mgx.psd1 b/Modules/M365DSC.mgx/M365DSC.mgx.psd1 index 48c7668..63a14e0 100644 --- a/Modules/M365DSC.mgx/M365DSC.mgx.psd1 +++ b/Modules/M365DSC.mgx/M365DSC.mgx.psd1 @@ -1,6 +1,6 @@ -@{ +@{ RootModule = 'M365DSC.mgx.psm1' - ModuleVersion = '2.1.1' + ModuleVersion = '2.1.7' GUID = 'f978315f-75c0-48f5-b929-ca7a7757d1d2' Author = 'Thomas Maillo Grome, Fabien Tschanz' CompanyName = 'Mgx' @@ -49,6 +49,14 @@ LicenseUri = 'https://github.com/Microsoft365DSC/mgx/blob/main/LICENSE' ProjectUri = 'https://github.com/Microsoft365DSC/mgx' ReleaseNotes = @' +v2.1.7 +- Added Invoke-MgxRequest -Envelope, emitting the payload as the service sent it, collection envelope and all, so a collection holding one item can be told from a single entity. +- Fixed the batch retry loop waiting out its exponential fallback without regard for MaxRetryAfterSeconds, which capped only the delays a server asked for. + +v2.1.6 +- Merged upstream gromedev/mgx 2.1.2 through 2.1.5. +- Fixed a Graph error reporting only its top level message: the property that was actually wrong sat in the innerError of the response and was discarded. + v2.1.1 - Fixed Invoke-MgxRequest failing with a missing System.IO.Pipelines assembly on PowerShell 7.6 hosts that do not ship it. diff --git a/Modules/M365DSC.mgx/en-US/Mgx.Cmdlets.dll-Help.xml b/Modules/M365DSC.mgx/en-US/Mgx.Cmdlets.dll-Help.xml index 900bbb4..8e23239 100644 --- a/Modules/M365DSC.mgx/en-US/Mgx.Cmdlets.dll-Help.xml +++ b/Modules/M365DSC.mgx/en-US/Mgx.Cmdlets.dll-Help.xml @@ -10,7 +10,7 @@ - Disable-MgxResilience removes the Polly resilience injection from the Microsoft.Graph SDK's HTTP transport, restoring the original SDK HttpClient that was saved by Enable-MgxResilience. + Disable-MgxResilience removes the Polly resilience injection from the Microsoft.Graph SDK's HTTP transport, restoring the original SDK HttpClient that was saved by Enable-MgxResilience. A client the SDK replaced after the injection went on - a Connect-MgGraph to another tenant builds one - is not this module's to restore and is left as it was found, with a warning saying so; the injection's own state comes off either way. After calling this cmdlet, SDK cmdlets revert to their default retry behavior. @@ -759,7 +759,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP CheckpointPath - Path to a JSON checkpoint file for resumable exports. Deleted on successful completion. + Path to a JSON checkpoint file for resumable exports. Deleted on successful completion. A run stops - with nothing written to the output, the checkpoint or a temp - when the temp file the checkpoint names, or the output it would append to or replace, is open in another export or cannot be opened for writing (a permission, a directory or a pipe at the path, a link that cannot be followed, a path too long for the file system, a socket, a directory above it this account cannot search), or when the copy it stages before promoting a temp fails, in which case both files are left as found: the rows it counts are in that file and this checkpoint is the only thing that counts them, and exporting anyway would write over both. This holds whether the run would append, promote a temp, or export from the beginning after its temp vanished. The copy a promotion stages is written beside the output under a name the run creates itself, and the name is cleared first: a link, a pipe, a socket, a copy left by an interrupted promotion or a copy this account cannot open standing there is deleted and the removal reported as a warning, while a directory or a copy another run holds stops the run instead. A -WhatIf pass removes nothing: where something stands at that name it says what the run would remove there and leaves it, and where the name is clear it makes the same create at no length before taking it back - so a preview stops where the run would and deletes nothing on the way. Wait for that export to finish, give this run its own -OutputFile and -CheckpointPath, or grant write access to that file - or to the directory holding it, where that is what the run names - and run again. A resumed or promoting run holds the output open for as long as it is appending, which is what stops a second run over the same command line; a run recovering a checkpoint older than 2.1 opens the output as it always has. On Linux and macOS a PowerShell reader of a held file fails with a sharing violation until the run ends, while tail and cat read it as before. String @@ -975,7 +975,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP CheckpointPath - Path to a JSON checkpoint file for resumable exports. Deleted on successful completion. + Path to a JSON checkpoint file for resumable exports. Deleted on successful completion. A run stops - with nothing written to the output, the checkpoint or a temp - when the temp file the checkpoint names, or the output it would append to or replace, is open in another export or cannot be opened for writing (a permission, a directory or a pipe at the path, a link that cannot be followed, a path too long for the file system, a socket, a directory above it this account cannot search), or when the copy it stages before promoting a temp fails, in which case both files are left as found: the rows it counts are in that file and this checkpoint is the only thing that counts them, and exporting anyway would write over both. This holds whether the run would append, promote a temp, or export from the beginning after its temp vanished. The copy a promotion stages is written beside the output under a name the run creates itself, and the name is cleared first: a link, a pipe, a socket, a copy left by an interrupted promotion or a copy this account cannot open standing there is deleted and the removal reported as a warning, while a directory or a copy another run holds stops the run instead. A -WhatIf pass removes nothing: where something stands at that name it says what the run would remove there and leaves it, and where the name is clear it makes the same create at no length before taking it back - so a preview stops where the run would and deletes nothing on the way. Wait for that export to finish, give this run its own -OutputFile and -CheckpointPath, or grant write access to that file - or to the directory holding it, where that is what the run names - and run again. A resumed or promoting run holds the output open for as long as it is appending, which is what stops a second run over the same command line; a run recovering a checkpoint older than 2.1 opens the output as it always has. On Linux and macOS a PowerShell reader of a held file fails with a sharing violation until the run ends, while tail and cat read it as before. String @@ -1936,7 +1936,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP Invoke-MgxBatchRequest bundles multiple Microsoft Graph API requests into /$batch calls, sending up to 20 requests per HTTP round-trip (the Graph API maximum). This is 3-4x faster than individual requests for bulk operations. Supports GET, POST, PATCH, PUT, and DELETE methods with optional request bodies. Auto-chunks input into 20-request batches. - Source: Combine multiple HTTP requests using JSON batching (https://learn.microsoft.com/en-us/graph/json-batching)Pipeline input can be string URLs (for GET, or combined with -Method/-Body for the same operation on all) or hashtables or PSCustomObjects with Url, Method, and Body members for per-item control. An optional Id member is echoed back on the matching result, so you can correlate on your own key instead of on URL or position. + Source: Combine multiple HTTP requests using JSON batching (https://learn.microsoft.com/en-us/graph/json-batching)Pipeline input can be string URLs (for GET, or combined with -Method/-Body for the same operation on all) or hashtables or PSCustomObjects with Url, Method, and Body members for per-item control. Failed items are surfaced as PowerShell ErrorRecords. Use -ErrorAction Stop to halt on the first failure, or inspect $Error after completion. @@ -1945,7 +1945,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP Uri - Graph API URLs to batch. Accepts absolute URLs (https://graph.microsoft.com/v1.0/users/id) or relative URLs (/users/id). Also accepts hashtables or PSCustomObjects with Url, Method, and Body members for per-item control, plus an optional Id that is echoed back on the matching result. + Graph API URLs to batch. Accepts absolute URLs (https://graph.microsoft.com/v1.0/users/id) or relative URLs (/users/id). Also accepts hashtables or PSCustomObjects with Url, Method, and Body members for per-item control. Object[] @@ -1974,6 +1974,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP Body Request body for all requests when piping string URLs. Ignored when pipeline input carries its own Body member. + Bodies follow the same serialization contract as Invoke-MgxRequest -Body. An item whose body serialization refuses a value (a SecureString, NaN) fails on its own, like an item whose body is not valid JSON; the rest of the batch is sent. So does an item whose body the redaction check cannot read: a raw-string body carrying an unpaired surrogate escape - `{"displayName":"Jos\ud83d"}` - is a document the JSON parser accepts and the check cannot read a string out of, and the check is what decides whether a body may be sent, so that item is refused naming the reason and the rest of the batch is sent. Object @@ -1983,35 +1984,37 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP None - FollowNextLink + DeadLetterPath - Drain `@odata.nextLink` in sub-response bodies, merging the pages into each result. Off by default. - A batched sub-request against a collection returns only its first page, and `/$batch` does not page. Without this switch each such result is a partial collection, and you cannot tell: the conversion to a hashtable strips `@odata.nextLink` along with the rest of the `@odata.*` metadata, so a truncated read looks exactly like a complete one. - Follow-up pages are submitted as further batches rather than per-item GETs, so N partial collections drain in ceil(N/20) requests per page. - If a page cannot be read - an HTTP failure, or a `nextLink` that fails host validation - that one sub-request is marked failed: `Status` becomes the failing page's status, `PagingIncomplete` is set to `$true`, and a non-terminating error is written (`BatchPagingFailed`, `BatchNextLinkRefused`, or `BatchPagingTruncated`). The items already collected stay in `Body`, so you can use them deliberately. One failed link never discards the other results in the batch. + Path to a JSONL file where the batch's failed and never-sent items are appended: anything the server answered with a status >= 400, the items of a chunk whose own POST failed, which carry that failure's status, and the items of chunks that were never POSTed, which carry status 0. Nothing the server confirmed is ever written, so the file holds only work that is still outstanding. Each line holds Timestamp, Url, Method and Status, then Body where the request had one, with sensitive fields redacted, then Error where the item's own response carried an error message: a never-sent item has no response to read one from, and neither does an item of a refused chunk the server did not answer, so their lines end at Body. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Any field whose name carries password, secret, credential, key, token, assertion, passphrase or connectionstring is replaced by a redaction marker, and so is any field whose name carries downloadUrl, because a pre-authenticated URL fetches the file with no token of its own. A value that is a URL carrying a capability parameter - `sig`, `tempauth`, `guestaccesstoken`, `authkey`, `X-Amz-Signature` - is cut after that parameter and the rest replaced by the marker, the way a -Debug trace cuts it, so the host and the parameter name still say which URL failed; an ordinary link is left whole, because `@odata.nextLink` and a `webUrl` are half of what the line is read for. Where the URL itself names the secret - resetPassword, uploadSecret, addKey, synchronization/secrets - the whole Body is that marker, because those bodies hold the credential under a name that says nothing. A body the redaction cannot walk at all - one naming the same property twice is such a body - gets the marker whole for the same reason, and the run writes a warning naming the item whose body was withheld. What stands in the file is: + + ***REDACTED*** + + Read it to decide what to resubmit, and supply the sensitive fields again yourself. Re-piping the file - `Get-Content dead.jsonl | ConvertFrom-Json | Invoke-MgxBatchRequest` - resends the lines that carry no marker, and refuses the ones that do as an error record before the first chunk leaves: a Body that is the marker whole is not JSON, and a Body carrying it in a field parses but is no longer the body that was sent, so that one is refused naming the field. Any body carrying that marker text is refused the same way before the first chunk leaves, whether it came from the file or not. Under -ErrorAction Stop - which this cmdlet's description recommends - the pipeline then ends with nothing sent at all: re-pipe at the default error preference, or filter the marker lines out first. A refused item carries that status whether its POST reached the server and went unanswered or never left at all - an open circuit, a rate limiter with no permit for it - and the status does not separate the two. Read it as the first: the write may have been applied, so resending one is a decision about duplicates rather than a free retry. Only status 0 says nothing was sent. + The file is opened before the first request goes out and held open for the run, so a path this run cannot write - a directory standing at it, a parent directory that is not there, a file this account may not write - is refused with nothing sent at all, and the error says so: fix the path and run the same command again. It has to be a file system path with something in it. `Env:\X` resolves to a bare name relative to nothing and an empty path resolves to the working directory, so both are refused rather than written somewhere nobody named. A run with no failed and no never-sent items has no lines for the file, and one it created for them is removed again, so a batch that succeeded whole leaves nothing behind; a file that was already there holds an earlier run's outstanding work and is left as it was, and so is one a second run appending to the same path has put a line in. + String - SwitchParameter + String - False + None - - MaxPage + + Confirm - Ceiling on the number of pages drained per sub-request when `-FollowNextLink` is used. The default, 0, is unlimited. Hitting the ceiling marks the result `PagingIncomplete` and writes a `BatchPagingTruncated` error, so a bounded read is never mistaken for a complete one. + Prompts you for confirmation before running the cmdlet. - Int32 - Int32 + SwitchParameter - 0 + False - DeadLetterPath + ConsistencyLevel - Path to a JSONL file where failed batch items (status >= 400) are appended, along with any that were never sent because an earlier chunk failed. Each line contains Url, Method, Body (with sensitive fields redacted), Status, Error, Timestamp, and the Id if one was supplied. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Read it to decide what to resubmit, and supply the sensitive fields again yourself. + ConsistencyLevel header added to each individual batch item. Required when any batch item URL contains $search (Graph advanced query capabilities). Takes precedence over the same key in -Headers. Source: Advanced query capabilities on Microsoft Entra ID objects (https://learn.microsoft.com/en-us/graph/aad-advanced-queries) String @@ -2020,17 +2023,6 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP None - - Confirm - - Prompts you for confirmation before running the cmdlet. - - - SwitchParameter - - - False - Headers @@ -2083,7 +2075,19 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP WhatIf - Shows what would happen if the cmdlet runs. The cmdlet is not run. + Shows what would happen if the cmdlet runs. The cmdlet is not run. The gate covers reads as well as writes: an all-GET batch is described and not sent, because even a read batch spends resource units, can be throttled, and emits objects into the pipeline. Invoke-MgxRequest makes the other choice and sends its reads under -WhatIf. + The count it names is what would be sent. Bodies are validated and checked for the redaction marker before the gate, so an item either of those refuses is left out of that count and named beside it - `PATCH 1 request via $batch; 2 refused` - and each refusal is written as the error record it is, under -WhatIf as without it: a refusal is not an action the gate governs. + + + SwitchParameter + + + False + + + FollowNextLink + + Drain `@odata.nextLink` in sub-response bodies, merging the pages into each result. Off by default, where a sub-response that carries a next link is returned with the items of its first page and the link beside them. Follow-up pages are submitted as further batches, so N partial collections drain in ceil(N/20) requests per page rather than N. SwitchParameter @@ -2091,6 +2095,18 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP False + + MaxPage + + Ceiling on the pages drained per sub-request when -FollowNextLink is on. 0, the default, is unlimited. A sub-request stopped at the ceiling with more to read is reported as such rather than returned as though it were whole. + + Int32 + + Int32 + + + 0 + ProgressAction @@ -2122,6 +2138,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP Body Request body for all requests when piping string URLs. Ignored when pipeline input carries its own Body member. + Bodies follow the same serialization contract as Invoke-MgxRequest -Body. An item whose body serialization refuses a value (a SecureString, NaN) fails on its own, like an item whose body is not valid JSON; the rest of the batch is sent. So does an item whose body the redaction check cannot read: a raw-string body carrying an unpaired surrogate escape - `{"displayName":"Jos\ud83d"}` - is a document the JSON parser accepts and the check cannot read a string out of, and the check is what decides whether a body may be sent, so that item is refused naming the reason and the rest of the batch is sent. Object @@ -2130,37 +2147,15 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP None - - FollowNextLink - - Drain `@odata.nextLink` in sub-response bodies, merging the pages into each result. Off by default. - A batched sub-request against a collection returns only its first page, and `/$batch` does not page. Without this switch each such result is a partial collection, and you cannot tell: the conversion to a hashtable strips `@odata.nextLink` along with the rest of the `@odata.*` metadata, so a truncated read looks exactly like a complete one. - Follow-up pages are submitted as further batches rather than per-item GETs, so N partial collections drain in ceil(N/20) requests per page. - If a page cannot be read - an HTTP failure, or a `nextLink` that fails host validation - that one sub-request is marked failed: `Status` becomes the failing page's status, `PagingIncomplete` is set to `$true`, and a non-terminating error is written (`BatchPagingFailed`, `BatchNextLinkRefused`, or `BatchPagingTruncated`). The items already collected stay in `Body`, so you can use them deliberately. One failed link never discards the other results in the batch. - - SwitchParameter - - SwitchParameter - - - False - - - MaxPage - - Ceiling on the number of pages drained per sub-request when `-FollowNextLink` is used. The default, 0, is unlimited. Hitting the ceiling marks the result `PagingIncomplete` and writes a `BatchPagingTruncated` error, so a bounded read is never mistaken for a complete one. - - Int32 - - Int32 - - - 0 - DeadLetterPath - Path to a JSONL file where failed batch items (status >= 400) are appended, along with any that were never sent because an earlier chunk failed. Each line contains Url, Method, Body (with sensitive fields redacted), Status, Error, Timestamp, and the Id if one was supplied. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Read it to decide what to resubmit, and supply the sensitive fields again yourself. + Path to a JSONL file where the batch's failed and never-sent items are appended: anything the server answered with a status >= 400, the items of a chunk whose own POST failed, which carry that failure's status, and the items of chunks that were never POSTed, which carry status 0. Nothing the server confirmed is ever written, so the file holds only work that is still outstanding. Each line holds Timestamp, Url, Method and Status, then Body where the request had one, with sensitive fields redacted, then Error where the item's own response carried an error message: a never-sent item has no response to read one from, and neither does an item of a refused chunk the server did not answer, so their lines end at Body. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Any field whose name carries password, secret, credential, key, token, assertion, passphrase or connectionstring is replaced by a redaction marker, and so is any field whose name carries downloadUrl, because a pre-authenticated URL fetches the file with no token of its own. A value that is a URL carrying a capability parameter - `sig`, `tempauth`, `guestaccesstoken`, `authkey`, `X-Amz-Signature` - is cut after that parameter and the rest replaced by the marker, the way a -Debug trace cuts it, so the host and the parameter name still say which URL failed; an ordinary link is left whole, because `@odata.nextLink` and a `webUrl` are half of what the line is read for. Where the URL itself names the secret - resetPassword, uploadSecret, addKey, synchronization/secrets - the whole Body is that marker, because those bodies hold the credential under a name that says nothing. A body the redaction cannot walk at all - one naming the same property twice is such a body - gets the marker whole for the same reason, and the run writes a warning naming the item whose body was withheld. What stands in the file is: + + ***REDACTED*** + + Read it to decide what to resubmit, and supply the sensitive fields again yourself. Re-piping the file - `Get-Content dead.jsonl | ConvertFrom-Json | Invoke-MgxBatchRequest` - resends the lines that carry no marker, and refuses the ones that do as an error record before the first chunk leaves: a Body that is the marker whole is not JSON, and a Body carrying it in a field parses but is no longer the body that was sent, so that one is refused naming the field. Any body carrying that marker text is refused the same way before the first chunk leaves, whether it came from the file or not. Under -ErrorAction Stop - which this cmdlet's description recommends - the pipeline then ends with nothing sent at all: re-pipe at the default error preference, or filter the marker lines out first. A refused item carries that status whether its POST reached the server and went unanswered or never left at all - an open circuit, a rate limiter with no permit for it - and the status does not separate the two. Read it as the first: the write may have been applied, so resending one is a decision about duplicates rather than a free retry. Only status 0 says nothing was sent. + The file is opened before the first request goes out and held open for the run, so a path this run cannot write - a directory standing at it, a parent directory that is not there, a file this account may not write - is refused with nothing sent at all, and the error says so: fix the path and run the same command again. It has to be a file system path with something in it. `Env:\X` resolves to a bare name relative to nothing and an empty path resolves to the working directory, so both are refused rather than written somewhere nobody named. A run with no failed and no never-sent items has no lines for the file, and one it created for them is removed again, so a batch that succeeded whole leaves nothing behind; a file that was already there holds an earlier run's outstanding work and is left as it was, and so is one a second run appending to the same path has put a line in. String @@ -2181,14 +2176,14 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP False - + ConsistencyLevel ConsistencyLevel header added to each individual batch item. Required when any batch item URL contains $search (Graph advanced query capabilities). Takes precedence over the same key in -Headers. Source: Advanced query capabilities on Microsoft Entra ID objects (https://learn.microsoft.com/en-us/graph/aad-advanced-queries) - + String - + String None @@ -2233,7 +2228,7 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP Uri - Graph API URLs to batch. Accepts absolute URLs (https://graph.microsoft.com/v1.0/users/id) or relative URLs (/users/id). Also accepts hashtables or PSCustomObjects with Url, Method, and Body members for per-item control, plus an optional Id that is echoed back on the matching result. + Graph API URLs to batch. Accepts absolute URLs (https://graph.microsoft.com/v1.0/users/id) or relative URLs (/users/id). Also accepts hashtables or PSCustomObjects with Url, Method, and Body members for per-item control. Object[] @@ -2245,7 +2240,20 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP WhatIf - Shows what would happen if the cmdlet runs. The cmdlet is not run. + Shows what would happen if the cmdlet runs. The cmdlet is not run. The gate covers reads as well as writes: an all-GET batch is described and not sent, because even a read batch spends resource units, can be throttled, and emits objects into the pipeline. Invoke-MgxRequest makes the other choice and sends its reads under -WhatIf. + The count it names is what would be sent. Bodies are validated and checked for the redaction marker before the gate, so an item either of those refuses is left out of that count and named beside it - `PATCH 1 request via $batch; 2 refused` - and each refusal is written as the error record it is, under -WhatIf as without it: a refusal is not an action the gate governs. + + SwitchParameter + + SwitchParameter + + + False + + + FollowNextLink + + Drain `@odata.nextLink` in sub-response bodies, merging the pages into each result. Off by default, where a sub-response that carries a next link is returned with the items of its first page and the link beside them. Follow-up pages are submitted as further batches, so N partial collections drain in ceil(N/20) requests per page rather than N. SwitchParameter @@ -2254,6 +2262,18 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP False + + MaxPage + + Ceiling on the pages drained per sub-request when -FollowNextLink is on. 0, the default, is unlimited. A sub-request stopped at the ceiling with more to read is reported as such rather than returned as though it were whole. + + Int32 + + Int32 + + + 0 + ProgressAction @@ -2283,15 +2303,15 @@ $report | Expand-MgxRelation -Uri '/users/{id}/licenseDetails' -As Licenses -IdP System.Collections.Hashtable - Per-request results with Url, Method, Status, and Body keys, plus Id when the request supplied one, NotSent on any operation that was never sent because an earlier chunk failed, and PagingIncomplete when -FollowNextLink could not drain the collection. Body here is the RESPONSE body - for a failure that is the error envelope, not the request - so piping results straight back resubmits the wrong thing. Use Url and Method to rebuild the requests you want to retry. + Per-request results with Url, Method, Status, and Body keys. Status says what became of the operation: an item the server answered keeps the status it answered with; an item whose chunk was refused carries the refusal's status (>= 400), which covers both a POST that went out unanswered and one an open circuit or a rate limiter stopped before it left, so read it as a write that may have been applied; only an item in a chunk that was never POSTed gets status 0, and only those carry NotSent - another chunk of the batch failed, not necessarily one before it, since chunks can run in parallel. Body here is the RESPONSE body - for a failure that is the error envelope, not the request - so piping results straight back resubmits the wrong thing. Use Url and Method to rebuild the requests you want to retry. - Each batch item is retried individually on 429 (throttled) or 5xx errors (for idempotent methods). POST requests only retry on 429 because POST is non-idempotent - retrying a failed POST on 5xx could create duplicates if the server processed the request before the error. This matches the Kiota SDK retry behavior. Source: Microsoft Graph error responses and resource types (https://learn.microsoft.com/en-us/graph/errors)Items that exhaust per-chunk retries get one additional batch-level retry pass. + Each batch item is retried individually on 429 (throttled) or 5xx errors (for idempotent methods). POST requests only retry on 429 because POST is non-idempotent - retrying a failed POST on 5xx could create duplicates if the server processed the request before the error. This matches the Kiota SDK retry behavior. Source: Microsoft Graph error responses and resource types (https://learn.microsoft.com/en-us/graph/errors)Items that exhaust per-chunk retries get one additional batch-level retry pass, except after a chunk failure: the run has stopped sending, and that pass is a send. It is skipped for every item it would have picked up, including items in chunks that were POSTed and answered in full, and they are handed back with the status the server gave them rather than resent. Use -Verbose to see retry counts, throttle encounters, and timing. - Batching reduces HTTP round-trips (20 operations per request instead of 1), but Graph counts each item inside a batch individually against the server-side write quota (3,000 writes / 2.5 min per app+tenant). Sustained write throughput caps at ~20/sec regardless of batching. See Set-MgxOption (Set-MgxOption.md) for the full throttle limits table and tuning guidance. Source: [Microsoft Graph service-specific throttling limits](https://learn.microsoft.com/en-us/graph/throttling-limits)Pipe all items into a single Invoke-MgxBatchRequest call rather than calling it in a loop. One call chunks the items itself, so it makes far fewer round-trips than a loop of single-item calls and reuses one HTTP client, one rate limiter and one circuit breaker. Repeated invocation is safe at any rate the host can drive. + Batching reduces HTTP round-trips (20 operations per request instead of 1), but Graph counts each item inside a batch individually against the server-side write quota (3,000 writes / 2.5 min per app+tenant). Sustained write throughput caps at ~20/sec regardless of batching. See Set-MgxOption (Set-MgxOption.md) for the full throttle limits table and tuning guidance. Source: [Microsoft Graph service-specific throttling limits](https://learn.microsoft.com/en-us/graph/throttling-limits)For best performance and stability, always pipe all items into a single Invoke-MgxBatchRequest call rather than calling it in a loop. At 200+ rapid invocations per second, PowerShell's internal pipeline thread safety can race between cmdlet instances. Piping all items into one call avoids this entirely and is significantly faster. @@ -2373,9 +2393,12 @@ $users | ForEach-Object { "/users/$($_.id)" } | Invoke-MgxBatchRequest -Method D ------ Example 8: Capture failures to a dead-letter file ------ 1..1000 | ForEach-Object { [PSCustomObject]@{ Url = "/users"; Method = "POST"; Body = @{ displayName = "User-$_"; mailNickname = "user$_"; userPrincipalName = "user$_@contoso.com"; passwordProfile = @{ password = "P@ss$(Get-Random -Minimum 10000)!" } } } -} | Invoke-MgxBatchRequest -DeadLetterPath ./failed-users.jsonl +} | Invoke-MgxBatchRequest -DeadLetterPath ./failed-users.jsonl + +# The redacted password field, as failed-users.jsonl holds it: +# "passwordProfile":"***REDACTED***" - Creates 1000 users via batch. Any failures (status >= 400) are appended to the JSONL dead-letter file with Url, Method, Body (passwords redacted), Status, and Error. Sensitive fields like passwordProfile are automatically replaced with *REDACTED *. + Creates 1000 users via batch. Anything the server answered with a status >= 400 is appended to the JSONL dead-letter file, and so is anything that was never sent, which carries status 0; each line holds Timestamp, Url, Method, Status and Body, then Error where the item's own response carried an error message - a never-sent item has no response to read one from, and neither does an item of a refused chunk the server did not answer, so those lines end at Body. The Body is redacted by field name, by field value and by URL: a field whose name carries password, secret, credential, key, token, assertion, passphrase or connectionstring is replaced by the marker shown above, which is what passwordProfile gets here; a value that is a pre-authenticated URL is cut after its capability parameter, the way a -Debug trace cuts it; and a request whose own path names the secret it carries - resetPassword, uploadSecret, addKey - has its whole Body replaced by that marker instead, as does an item whose body could not be read for redaction. @@ -2384,7 +2407,7 @@ $users | ForEach-Object { "/users/$($_.id)" } | Invoke-MgxBatchRequest -Method D $urls = 1..10000 | ForEach-Object { "/users/$_" } $urls | Invoke-MgxBatchRequest -# Avoid: calling in a tight loop - many more round-trips than one chunked call +# Avoid: calling in a tight loop - slower and can cause threading errors at 200+ iterations foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Always pipe all items into a single Invoke-MgxBatchRequest call. This is both faster (one pipeline setup) and avoids PowerShell pipeline threading issues at high invocation rates. @@ -2394,7 +2417,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Online Version: - https://github.com/Microsoft365DSC/mgx/blob/main/Modules/M365DSC.mgx/help/Invoke-MgxBatchRequest.md + https://github.com/gromedev/mgx/blob/main/module/help/Invoke-MgxBatchRequest.md Invoke-MgxRequest @@ -2466,6 +2489,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Body Request body for write operations (POST, PATCH, PUT). Accepts a hashtable, PSObject or array, which is serialized to JSON, or a string already holding JSON, which is sent verbatim. Ignored (with a warning) on GET. + Enums serialize as camelCase names, byte arrays as base64, TimeSpan as an ISO-8601 duration, and a DateTime without a kind is treated as UTC. A [Flags] combination goes out in the comma-joined form OData expects, with no space: `ignoreCase,multiline`. Each member of a combination is named as it would be alone, so a name set by JsonStringEnumMemberName, or the choice between two members sharing a value, reads the same either way. camelCase lowercases a leading acronym whole, so a member named `IOS` is sent as `ios`; where Graph wants `iOS`, pass the string rather than the enum. A SecureString, PSCredential, script block, certificate, NaN or Infinity anywhere in the body is refused before anything is sent, with an error naming the property. A string that is not JSON is sent verbatim; add a Content-Type to -Headers to declare its real type. Object @@ -2477,7 +2501,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } CheckpointPath - Path to a JSON checkpoint file for resumable pagination. On interruption (Ctrl+C), progress is saved. On re-run with the same checkpoint path, pagination resumes from where it left off. The checkpoint file is deleted on successful completion. + Path to a JSON checkpoint file for resumable pagination. Consulted only by a collection read carrying -All, -Top, -Skip, -Filter, -Search, -Sort or -CountVariable; any other request - a plain collection GET, one with only -PageSize, a single-entity GET, a fan-out over two or more piped IDs, a write - binds it and ignores it. A single piped ID is not a fan-out: it is sent as a direct request and, on a collection read, consults the checkpoint like any other. On interruption (Ctrl+C), progress is saved. On re-run with the same checkpoint path, pagination resumes from where it left off. The checkpoint file is deleted on successful completion. Those writes are gated on their own: -WhatIf reads the file and leaves it exactly as it was, and -Confirm asks about it once per run that consults it, including when several values are piped into a -Uri that carries no placeholder, which runs the request once per value. String @@ -2501,7 +2525,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Confirm - Prompts you for confirmation before running the cmdlet. + Prompts before a write and not before a read. POST, PATCH, PUT and DELETE are confirmed - a direct request one prompt at a time, a fan-out over piped IDs one prompt for all of its items - and a GET is sent without a prompt, the same division -WhatIf makes. With -CheckpointPath there is one prompt more, but only on a collection read carrying -All, -Top, -Skip, -Filter, -Search, -Sort or -CountVariable, asked once per run before any page is fetched, covering the saves and the deletes of the checkpoint file itself - including when several values are piped into a -Uri that carries no placeholder, which runs the request once per value; any other request - a plain collection GET, one with only -PageSize, a single-entity GET, a fan-out over two or more piped IDs, a write - binds the parameter and never reads it, so none of them adds a prompt. A single piped ID is not a fan-out: it is sent as a direct request and, on a collection read, prompts the same as any other. SwitchParameter @@ -2574,9 +2598,9 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Pipeline input for fan-out operations. Each value replaces the {id} placeholder in the URI template. Accepts string values or objects with an Id property. - String + Object - String + Object None @@ -2635,6 +2659,17 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } None + + Envelope + + Return the payload as the service sent it, collection envelope and all, instead of the items inside it. Without it a collection holding one item and a single entity are both one object, which a caller that has to tell them apart cannot do. -All aggregates pages and ignores this. + + + SwitchParameter + + + False + Raw @@ -2719,7 +2754,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } WhatIf - Shows what would happen if the cmdlet runs. The cmdlet is not run. + Describes the write instead of sending it. Only writes are gated: POST, PATCH, PUT and DELETE, single or fanned out over piped IDs, are described and not sent. Reads are sent - a GET under -WhatIf connects, pages, spends resource units, can be throttled, and emits its objects - which is the PowerShell convention and what makes a dry run under $WhatIfPreference useful, since the gated writes are then previewed against real data. The checkpoint at -CheckpointPath is read and then left alone: a preview resumes from the position it finds, so it sends the same reads the run would rather than paging from the start, and it neither saves over that file nor deletes it, so the resume position is exactly where it was. Invoke-MgxBatchRequest makes the other choice and gates reads too. SwitchParameter @@ -2770,6 +2805,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Body Request body for write operations (POST, PATCH, PUT). Accepts a hashtable, PSObject or array, which is serialized to JSON, or a string already holding JSON, which is sent verbatim. Ignored (with a warning) on GET. + Enums serialize as camelCase names, byte arrays as base64, TimeSpan as an ISO-8601 duration, and a DateTime without a kind is treated as UTC. A [Flags] combination goes out in the comma-joined form OData expects, with no space: `ignoreCase,multiline`. Each member of a combination is named as it would be alone, so a name set by JsonStringEnumMemberName, or the choice between two members sharing a value, reads the same either way. camelCase lowercases a leading acronym whole, so a member named `IOS` is sent as `ios`; where Graph wants `iOS`, pass the string rather than the enum. A SecureString, PSCredential, script block, certificate, NaN or Infinity anywhere in the body is refused before anything is sent, with an error naming the property. A string that is not JSON is sent verbatim; add a Content-Type to -Headers to declare its real type. Object @@ -2781,7 +2817,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } CheckpointPath - Path to a JSON checkpoint file for resumable pagination. On interruption (Ctrl+C), progress is saved. On re-run with the same checkpoint path, pagination resumes from where it left off. The checkpoint file is deleted on successful completion. + Path to a JSON checkpoint file for resumable pagination. Consulted only by a collection read carrying -All, -Top, -Skip, -Filter, -Search, -Sort or -CountVariable; any other request - a plain collection GET, one with only -PageSize, a single-entity GET, a fan-out over two or more piped IDs, a write - binds it and ignores it. A single piped ID is not a fan-out: it is sent as a direct request and, on a collection read, consults the checkpoint like any other. On interruption (Ctrl+C), progress is saved. On re-run with the same checkpoint path, pagination resumes from where it left off. The checkpoint file is deleted on successful completion. Those writes are gated on their own: -WhatIf reads the file and leaves it exactly as it was, and -Confirm asks about it once per run that consults it, including when several values are piped into a -Uri that carries no placeholder, which runs the request once per value. String @@ -2805,7 +2841,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Confirm - Prompts you for confirmation before running the cmdlet. + Prompts before a write and not before a read. POST, PATCH, PUT and DELETE are confirmed - a direct request one prompt at a time, a fan-out over piped IDs one prompt for all of its items - and a GET is sent without a prompt, the same division -WhatIf makes. With -CheckpointPath there is one prompt more, but only on a collection read carrying -All, -Top, -Skip, -Filter, -Search, -Sort or -CountVariable, asked once per run before any page is fetched, covering the saves and the deletes of the checkpoint file itself - including when several values are piped into a -Uri that carries no placeholder, which runs the request once per value; any other request - a plain collection GET, one with only -PageSize, a single-entity GET, a fan-out over two or more piped IDs, a write - binds the parameter and never reads it, so none of them adds a prompt. A single piped ID is not a fan-out: it is sent as a direct request and, on a collection read, prompts the same as any other. SwitchParameter @@ -2879,9 +2915,9 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } Pipeline input for fan-out operations. Each value replaces the {id} placeholder in the URI template. Accepts string values or objects with an Id property. - String + Object - String + Object None @@ -2934,6 +2970,18 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } None + + Envelope + + Return the payload as the service sent it, collection envelope and all, instead of the items inside it. Without it a collection holding one item and a single entity are both one object, which a caller that has to tell them apart cannot do. -All aggregates pages and ignores this. + + SwitchParameter + + SwitchParameter + + + False + Raw @@ -3033,7 +3081,7 @@ foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } WhatIf - Shows what would happen if the cmdlet runs. The cmdlet is not run. + Describes the write instead of sending it. Only writes are gated: POST, PATCH, PUT and DELETE, single or fanned out over piped IDs, are described and not sent. Reads are sent - a GET under -WhatIf connects, pages, spends resource units, can be throttled, and emits its objects - which is the PowerShell convention and what makes a dry run under $WhatIfPreference useful, since the gated writes are then previewed against real data. The checkpoint at -CheckpointPath is read and then left alone: a preview resumes from the position it finds, so it sends the same reads the run would rather than paging from the start, and it neither saves over that file nor deletes it, so the resume position is exactly where it was. Invoke-MgxBatchRequest makes the other choice and gates reads too. SwitchParameter @@ -3297,7 +3345,7 @@ Import-Csv ./users.csv | CircuitBreakerMinThroughput - Minimum number of requests in the sampling window before the circuit breaker evaluates failure ratio. Range: 1-1000. Default: 40. + Minimum number of requests in the sampling window before the circuit breaker evaluates failure ratio. Range: 2-1000. Default: 40. Two is the least the breaker will hold; a ratio over a single request is not a measurement. Int32 @@ -3423,6 +3471,28 @@ Import-Csv ./users.csv | 300 + + WhatIf + + Shows what would happen if the cmdlet runs. The cmdlet is not run. Every path that changes an option - -Reset and each individual parameter alike - sits behind this same check, so nothing is changed under -WhatIf. + + + SwitchParameter + + + False + + + Confirm + + Prompts you for confirmation before running the cmdlet. + + + SwitchParameter + + + False + @@ -3489,7 +3559,7 @@ Import-Csv ./users.csv | CircuitBreakerMinThroughput - Minimum number of requests in the sampling window before the circuit breaker evaluates failure ratio. Range: 1-1000. Default: 40. + Minimum number of requests in the sampling window before the circuit breaker evaluates failure ratio. Range: 2-1000. Default: 40. Two is the least the breaker will hold; a ratio over a single request is not a measurement. Int32 @@ -3618,6 +3688,30 @@ Import-Csv ./users.csv | 300 + + WhatIf + + Shows what would happen if the cmdlet runs. The cmdlet is not run. Every path that changes an option - -Reset and each individual parameter alike - sits behind this same check, so nothing is changed under -WhatIf. + + SwitchParameter + + SwitchParameter + + + False + + + Confirm + + Prompts you for confirmation before running the cmdlet. + + SwitchParameter + + SwitchParameter + + + False + @@ -3710,7 +3804,7 @@ Import-Csv ./users.csv | Delta tokens are saved to the file specified by -DeltaPath and persist across successful completions. -CheckpointPath is different: it is an ephemeral mid-run resume position, saved at page boundaries while an enumeration is in flight and deleted the moment it completes. Use both on long enumerations - a killed 3-hour drive sync resumes where it stopped instead of starting over. Drive delta is fully supported: `/me/drive/root/delta`, `/drives/{id}/root/delta`, `/users/{id}/drive/root/delta`, and `/groups/{id}/drive/root/delta` all work, including drive-specific `-Prefer` behaviors and the drive form of `-Latest`. Items that no longer match the query appear with an `@removed` property containing `{"reason": "changed"}` (item moved out of scope or soft-deleted) or `{"reason": "deleted"}` (permanently deleted). Filter these with `Where-Object { -not $_.'@removed' }`. The completion message counts them separately. - Delta tokens expire after approximately 7 days for directory objects (users, groups, applications). When a token expires, Graph returns HTTP 410 Gone. Sync-MgxDelta handles this automatically by deleting the stale token (and any resume checkpoint) and performing a full re-sync with a warning. + Delta tokens expire after approximately 7 days for directory objects (users, groups, applications). When a token expires, Graph returns HTTP 410 Gone. Sync-MgxDelta handles this automatically by deleting the stale token (and this run's own resume checkpoint) and performing a full re-sync with a warning. Resume semantics are at-least-once: in pipeline mode the page in flight at a crash is re-emitted in full on resume; in JSONL mode mid-page checkpoints keep the window to at most 500 items. Deduplicate on `id` downstream if exact-once matters. Source: Use delta query to track changes in Microsoft Graph data (https://learn.microsoft.com/en-us/graph/delta-query-overview) @@ -3748,7 +3842,8 @@ Import-Csv ./users.csv | CheckpointPath - Path for the ephemeral mid-run resume checkpoint. Saved at page boundaries (and every 500 items in JSONL mode) while an enumeration is in flight; deleted on successful completion. Any event that invalidates the enumeration - HTTP 410 Gone, -FullSync, or a -Property/-Filter/-Prefer change - deletes it too. Must differ from -DeltaPath and -OutputFile. + Path for the ephemeral mid-run resume checkpoint. Saved at page boundaries (and every 500 items in JSONL mode) while an enumeration is in flight; deleted on successful completion. Any event that invalidates the enumeration - HTTP 410 Gone, -FullSync, or a -Property/-Filter/-Prefer change - deletes it too. A completion and a 410 delete this run's own checkpoint; one they recognize as another sync's is left as it stands, for the sync that resumes from exactly it. Must differ from -DeltaPath and -OutputFile. + A run stops - with nothing written to the output, the checkpoint, a temp or the delta state - when the temp file the checkpoint names, or the output it would append to or replace, is open in another sync or cannot be opened for writing (a permission, a directory or a pipe at the path, a link that cannot be followed, a path too long for the file system, a socket, a directory above it this account cannot search), or when the copy it stages before promoting a temp fails, in which case both files are left as found: the changes it counts are in that file and this checkpoint is the only thing that counts them, and syncing anyway would write over both and advance the token past those changes. This holds whether the run would append, promote a temp, or re-enumerate from the last saved delta token after its temp vanished. The copy a promotion stages is written beside the output under a name the run creates itself, and the name is cleared first: a link, a pipe, a socket, a copy left by an interrupted promotion or a copy this account cannot open standing there is deleted and the removal reported as a warning, while a directory or a copy another run holds stops the run instead. Wait for that sync to finish, give this run its own -OutputFile and -CheckpointPath, or grant write access to that file - or to the directory holding it, where that is what the run names - and run again. A resumed or promoting run holds the output open for as long as it is appending, which is what stops a second run over the same command line; a run recovering a checkpoint older than 2.1 opens the output as it always has. On Linux and macOS a PowerShell reader of a held file fails with a sharing violation until the run ends, while tail and cat read it as before. String @@ -3893,7 +3988,8 @@ Import-Csv ./users.csv | CheckpointPath - Path for the ephemeral mid-run resume checkpoint. Saved at page boundaries (and every 500 items in JSONL mode) while an enumeration is in flight; deleted on successful completion. Any event that invalidates the enumeration - HTTP 410 Gone, -FullSync, or a -Property/-Filter/-Prefer change - deletes it too. Must differ from -DeltaPath and -OutputFile. + Path for the ephemeral mid-run resume checkpoint. Saved at page boundaries (and every 500 items in JSONL mode) while an enumeration is in flight; deleted on successful completion. Any event that invalidates the enumeration - HTTP 410 Gone, -FullSync, or a -Property/-Filter/-Prefer change - deletes it too. A completion and a 410 delete this run's own checkpoint; one they recognize as another sync's is left as it stands, for the sync that resumes from exactly it. Must differ from -DeltaPath and -OutputFile. + A run stops - with nothing written to the output, the checkpoint, a temp or the delta state - when the temp file the checkpoint names, or the output it would append to or replace, is open in another sync or cannot be opened for writing (a permission, a directory or a pipe at the path, a link that cannot be followed, a path too long for the file system, a socket, a directory above it this account cannot search), or when the copy it stages before promoting a temp fails, in which case both files are left as found: the changes it counts are in that file and this checkpoint is the only thing that counts them, and syncing anyway would write over both and advance the token past those changes. This holds whether the run would append, promote a temp, or re-enumerate from the last saved delta token after its temp vanished. The copy a promotion stages is written beside the output under a name the run creates itself, and the name is cleared first: a link, a pipe, a socket, a copy left by an interrupted promotion or a copy this account cannot open standing there is deleted and the removal reported as a warning, while a directory or a copy another run holds stops the run instead. Wait for that sync to finish, give this run its own -OutputFile and -CheckpointPath, or grant write access to that file - or to the directory holding it, where that is what the run names - and run again. A resumed or promoting run holds the output open for as long as it is appending, which is what stops a second run over the same command line; a run recovering a checkpoint older than 2.1 opens the output as it always has. On Linux and macOS a PowerShell reader of a held file fails with a sharing violation until the run ends, while tail and cat read it as before. String diff --git a/Modules/M365DSC.mgx/en-US/about_Mgx.help.txt b/Modules/M365DSC.mgx/en-US/about_Mgx.help.txt index 127eaa8..66c3839 100644 --- a/Modules/M365DSC.mgx/en-US/about_Mgx.help.txt +++ b/Modules/M365DSC.mgx/en-US/about_Mgx.help.txt @@ -217,6 +217,7 @@ REQUIREMENTS is absent. SEE ALSO + about_Mgx_Errors about_Mgx_Tuning Invoke-MgxRequest Invoke-MgxBatchRequest diff --git a/Modules/M365DSC.mgx/en-US/about_Mgx_Errors.help.txt b/Modules/M365DSC.mgx/en-US/about_Mgx_Errors.help.txt new file mode 100644 index 0000000..60040da --- /dev/null +++ b/Modules/M365DSC.mgx/en-US/about_Mgx_Errors.help.txt @@ -0,0 +1,106 @@ +TOPIC + about_Mgx_Errors + +SHORT DESCRIPTION + What terminates, what writes an error record, and how -ErrorAction and + -ErrorVariable behave across every Mgx cmdlet. + +LONG DESCRIPTION + Mgx follows one rule: only failures that make the whole invocation + meaningless terminate. Everything the service or transport says about an + individual request is a non-terminating error record, so a 10,000-item + fan-out that loses three items reports three errors and delivers 9,997 + results. + +TERMINATING ERRORS + + ThrowTerminatingError is reserved for pre-flight failures - the request + never went out, and no partial work exists: + + - Not connected, or Microsoft.Graph.Authentication missing + (NotConnected, GraphAuthModuleNotLoaded, HttpClientInitFailed) + - Invalid parameters or combinations: an absolute -Uri + (AbsoluteUriNotAllowed), -Search without ConsistencyLevel + (ConsistencyLevelRequired), a missing {id} placeholder, an unwritable + state or output file validated before the first request, a body value + with no JSON meaning (InvalidBodyValue), a second piped item against + one -OutFile (OutFileWithMultipleInputs) - refused before its content + is fetched + + Everything else is non-terminating. + +NON-TERMINATING ERRORS + + Every service and transport failure writes an ErrorRecord and continues: + + ErrorId When + ------- ---- + A Graph error with a code (e.g. + Request_ResourceNotFound); "GraphError" + when the body carried none + HttpError Transport failure with no response + CircuitBroken The circuit breaker is open; the record wraps + the guidance text + BatchItemError A batch item answered >= 400 + BatchItemNotSent A batch item never sent because + another chunk of the batch failed + BatchChunkFailed A chunk's own $batch call failed, or + answered nothing that was sent + FanOutError A per-id GET failed during fan-out + BulkWriteError A per-id write failed during fan-out + BulkWriteInfraError A fan-out write died before an HTTP status + ExpandRelationError A relation fetch failed during expansion + NonJsonResponse The body is not JSON (use -Raw or + Get-MgxContent) + MalformedJsonResponse The body declared JSON and does not parse + IOError / AccessDenied An output, checkpoint, or state file failed + + The ErrorCategory derives from the failure: ObjectNotFound (404), + PermissionDenied (403), AuthenticationError (401), LimitsExceeded (429), + ResourceUnavailable (5xx, open circuit), ConnectionError (transport, + 408), InvalidArgument (400 family), ResourceExists (409, 412). + +-ERRORACTION + + Stop turns the first error record into a terminating error, ending the + pipeline. For batches this stops at the first failed item - after the + dead-letter file is written, so -DeadLetterPath always captures every + failure even under Stop. Continue writes the record and carries on. + SilentlyContinue suppresses the display and continues; records still land + in $Error and -ErrorVariable. + + Ignore discards the record: nothing is displayed, nothing reaches $Error, + and -ErrorVariable stays empty. It is a common parameter only - + $ErrorActionPreference cannot be set to Ignore - so it applies to the one + invocation that carries it. + +-ERRORVARIABLE + + Collects every non-terminating record: one per failed item, URL, or + page. A fan-out over N ids can produce up to N records. + +RETRIES AND ERRORS + + A failure that is retried is not an error until retries are exhausted: + a request that survives three 503s and succeeds writes nothing to the + error stream. The retry history is visible under -Verbose, and the + final record reflects the last failure. + +CANCELLATION + + Ctrl-C produces a warning, not an error record, because the pipeline is + already stopping and error records written during teardown are + unreliable. A cancelled run is therefore indistinguishable from an empty + result by $Error alone; checkpointed cmdlets write their checkpoint, and + -Verbose says the run was cancelled. + +SKIP SWITCHES + + -SkipNotFound and -SkipForbidden (Invoke-MgxRequest, Expand-MgxRelation) + downgrade 404 and 403 per-item failures to a counted warning summary. + Skipped items produce no error records - by design, since the caller + declared them expected. + +SEE ALSO + about_Mgx + about_Mgx_Tuning diff --git a/Modules/M365DSC.mgx/en-US/about_Mgx_Tuning.help.txt b/Modules/M365DSC.mgx/en-US/about_Mgx_Tuning.help.txt index 99e57f3..7143585 100644 --- a/Modules/M365DSC.mgx/en-US/about_Mgx_Tuning.help.txt +++ b/Modules/M365DSC.mgx/en-US/about_Mgx_Tuning.help.txt @@ -74,7 +74,7 @@ CIRCUIT BREAKER Setting Default Hard Max Purpose ------- ------- -------- ------- CircuitBreakerFailureRatio 10% 100% Failure ratio to trip - CircuitBreakerMinThroughput 40 1,000 Min requests before evaluating + CircuitBreakerMinThroughput 40 1,000 Requests before evaluating (2+) CircuitBreakerSamplingDurationSeconds 30s 300s Evaluation window CircuitBreakerDurationSeconds 15s 300s Open-state duration diff --git a/Modules/M365DSC.mgx/help/Disable-MgxResilience.md b/Modules/M365DSC.mgx/help/Disable-MgxResilience.md index b65090b..ffa4747 100644 --- a/Modules/M365DSC.mgx/help/Disable-MgxResilience.md +++ b/Modules/M365DSC.mgx/help/Disable-MgxResilience.md @@ -17,7 +17,7 @@ Disable-MgxResilience [-ProgressAction ] [-WhatIf] [-Confirm] ``` ## DESCRIPTION -Disable-MgxResilience removes the Polly resilience injection from the Microsoft.Graph SDK's HTTP transport, restoring the original SDK HttpClient that was saved by Enable-MgxResilience. +Disable-MgxResilience removes the Polly resilience injection from the Microsoft.Graph SDK's HTTP transport, restoring the original SDK HttpClient that was saved by Enable-MgxResilience. A client the SDK replaced after the injection went on - a Connect-MgGraph to another tenant builds one - is not this module's to restore and is left as it was found, with a warning saying so; the injection's own state comes off either way. After calling this cmdlet, SDK cmdlets revert to their default retry behavior. diff --git a/Modules/M365DSC.mgx/help/Export-MgxCollection.md b/Modules/M365DSC.mgx/help/Export-MgxCollection.md index ae86f5d..6f8cfb0 100644 --- a/Modules/M365DSC.mgx/help/Export-MgxCollection.md +++ b/Modules/M365DSC.mgx/help/Export-MgxCollection.md @@ -86,7 +86,7 @@ Accept wildcard characters: False ``` ### -CheckpointPath -Path to a JSON checkpoint file for resumable exports. Deleted on successful completion. +Path to a JSON checkpoint file for resumable exports. Deleted on successful completion. A run stops - with nothing written to the output, the checkpoint or a temp - when the temp file the checkpoint names, or the output it would append to or replace, is open in another export or cannot be opened for writing (a permission, a directory or a pipe at the path, a link that cannot be followed, a path too long for the file system, a socket, a directory above it this account cannot search), or when the copy it stages before promoting a temp fails, in which case both files are left as found: the rows it counts are in that file and this checkpoint is the only thing that counts them, and exporting anyway would write over both. This holds whether the run would append, promote a temp, or export from the beginning after its temp vanished. The copy a promotion stages is written beside the output under a name the run creates itself, and the name is cleared first: a link, a pipe, a socket, a copy left by an interrupted promotion or a copy this account cannot open standing there is deleted and the removal reported as a warning, while a directory or a copy another run holds stops the run instead. A -WhatIf pass removes nothing: where something stands at that name it says what the run would remove there and leaves it, and where the name is clear it makes the same create at no length before taking it back - so a preview stops where the run would and deletes nothing on the way. Wait for that export to finish, give this run its own -OutputFile and -CheckpointPath, or grant write access to that file - or to the directory holding it, where that is what the run names - and run again. A resumed or promoting run holds the output open for as long as it is appending, which is what stops a second run over the same command line; a run recovering a checkpoint older than 2.1 opens the output as it always has. On Linux and macOS a PowerShell reader of a held file fails with a sharing violation until the run ends, while tail and cat read it as before. ```yaml Type: String diff --git a/Modules/M365DSC.mgx/help/Invoke-MgxBatchRequest.md b/Modules/M365DSC.mgx/help/Invoke-MgxBatchRequest.md index f7df866..68a72b5 100644 --- a/Modules/M365DSC.mgx/help/Invoke-MgxBatchRequest.md +++ b/Modules/M365DSC.mgx/help/Invoke-MgxBatchRequest.md @@ -1,7 +1,7 @@ ---- +--- external help file: Mgx.Cmdlets.dll-Help.xml Module Name: Mgx -online version: https://github.com/Microsoft365DSC/mgx/blob/main/Modules/M365DSC.mgx/help/Invoke-MgxBatchRequest.md +online version: https://github.com/gromedev/mgx/blob/main/module/help/Invoke-MgxBatchRequest.md schema: 2.0.0 --- @@ -14,8 +14,9 @@ Bundle multiple Graph API requests into /$batch calls. ``` Invoke-MgxBatchRequest [-Uri] [-Method ] [-Body ] [-ConsistencyLevel ] - [-Headers ] [-ThrottlePriority ] [-ApiVersion ] [-ProgressAction ] - [-WhatIf] [-Confirm] [] + [-Headers ] [-ThrottlePriority ] [-ApiVersion ] [-DeadLetterPath ] + [-FollowNextLink] [-MaxPage ] + [-ProgressAction ] [-WhatIf] [-Confirm] [] ``` ## DESCRIPTION @@ -25,7 +26,7 @@ Supports GET, POST, PATCH, PUT, and DELETE methods with optional request bodies. Source: [Combine multiple HTTP requests using JSON batching](https://learn.microsoft.com/en-us/graph/json-batching) -Pipeline input can be string URLs (for GET, or combined with -Method/-Body for the same operation on all) or hashtables or PSCustomObjects with Url, Method, and Body members for per-item control. An optional Id member is echoed back on the matching result, so you can correlate on your own key instead of on URL or position. +Pipeline input can be string URLs (for GET, or combined with -Method/-Body for the same operation on all) or hashtables or PSCustomObjects with Url, Method, and Body members for per-item control. Failed items are surfaced as PowerShell ErrorRecords. Use -ErrorAction Stop to halt on the first failure, or inspect $Error after completion. @@ -110,9 +111,12 @@ Batches search queries that require the ConsistencyLevel header. 1..1000 | ForEach-Object { [PSCustomObject]@{ Url = "/users"; Method = "POST"; Body = @{ displayName = "User-$_"; mailNickname = "user$_"; userPrincipalName = "user$_@contoso.com"; passwordProfile = @{ password = "P@ss$(Get-Random -Minimum 10000)!" } } } } | Invoke-MgxBatchRequest -DeadLetterPath ./failed-users.jsonl + +# The redacted password field, as failed-users.jsonl holds it: +# "passwordProfile":"***REDACTED***" ``` -Creates 1000 users via batch. Any failures (status >= 400) are appended to the JSONL dead-letter file with Url, Method, Body (passwords redacted), Status, and Error. Sensitive fields like passwordProfile are automatically replaced with ***REDACTED***. +Creates 1000 users via batch. Anything the server answered with a status >= 400 is appended to the JSONL dead-letter file, and so is anything that was never sent, which carries status 0; each line holds Timestamp, Url, Method, Status and Body, then Error where the item's own response carried an error message - a never-sent item has no response to read one from, and neither does an item of a refused chunk the server did not answer, so those lines end at Body. The Body is redacted by field name, by field value and by URL: a field whose name carries password, secret, credential, key, token, assertion, passphrase or connectionstring is replaced by the marker shown above, which is what passwordProfile gets here; a value that is a pre-authenticated URL is cut after its capability parameter, the way a -Debug trace cuts it; and a request whose own path names the secret it carries - resetPassword, uploadSecret, addKey - has its whole Body replaced by that marker instead, as does an item whose body could not be read for redaction. ### Example 9: Pipeline usage at scale (recommended pattern) ```powershell @@ -120,7 +124,7 @@ Creates 1000 users via batch. Any failures (status >= 400) are appended to the J $urls = 1..10000 | ForEach-Object { "/users/$_" } $urls | Invoke-MgxBatchRequest -# Avoid: calling in a tight loop - many more round-trips than one chunked call +# Avoid: calling in a tight loop - slower and can cause threading errors at 200+ iterations foreach ($url in $urls) { $url | Invoke-MgxBatchRequest } ``` @@ -147,6 +151,8 @@ Accept wildcard characters: False ### -Body Request body for all requests when piping string URLs. Ignored when pipeline input carries its own Body member. +Bodies follow the same serialization contract as Invoke-MgxRequest -Body. An item whose body serialization refuses a value (a SecureString, NaN) fails on its own, like an item whose body is not valid JSON; the rest of the batch is sent. So does an item whose body the redaction check cannot read: a raw-string body carrying an unpaired surrogate escape - `{"displayName":"Jos\ud83d"}` - is a document the JSON parser accepts and the check cannot read a string out of, and the check is what decides whether a body may be sent, so that item is refused naming the reason and the rest of the batch is sent. + ```yaml Type: Object Parameter Sets: (All) @@ -159,38 +165,16 @@ Accept pipeline input: False Accept wildcard characters: False ``` +### -DeadLetterPath +Path to a JSONL file where the batch's failed and never-sent items are appended: anything the server answered with a status >= 400, the items of a chunk whose own POST failed, which carry that failure's status, and the items of chunks that were never POSTed, which carry status 0. Nothing the server confirmed is ever written, so the file holds only work that is still outstanding. Each line holds Timestamp, Url, Method and Status, then Body where the request had one, with sensitive fields redacted, then Error where the item's own response carried an error message: a never-sent item has no response to read one from, and neither does an item of a refused chunk the server did not answer, so their lines end at Body. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Any field whose name carries password, secret, credential, key, token, assertion, passphrase or connectionstring is replaced by a redaction marker, and so is any field whose name carries downloadUrl, because a pre-authenticated URL fetches the file with no token of its own. A value that is a URL carrying a capability parameter - `sig`, `tempauth`, `guestaccesstoken`, `authkey`, `X-Amz-Signature` - is cut after that parameter and the rest replaced by the marker, the way a -Debug trace cuts it, so the host and the parameter name still say which URL failed; an ordinary link is left whole, because `@odata.nextLink` and a `webUrl` are half of what the line is read for. Where the URL itself names the secret - resetPassword, uploadSecret, addKey, synchronization/secrets - the whole Body is that marker, because those bodies hold the credential under a name that says nothing. A body the redaction cannot walk at all - one naming the same property twice is such a body - gets the marker whole for the same reason, and the run writes a warning naming the item whose body was withheld. What stands in the file is: -### -FollowNextLink -Drain `@odata.nextLink` in sub-response bodies, merging the pages into each result. Off by default. - -A batched sub-request against a collection returns only its first page, and `/$batch` does not page. Without this switch each such result is a partial collection, and you cannot tell: the conversion to a hashtable strips `@odata.nextLink` along with the rest of the `@odata.*` metadata, so a truncated read looks exactly like a complete one. - -Follow-up pages are submitted as further batches rather than per-item GETs, so N partial collections drain in ceil(N/20) requests per page. - -If a page cannot be read - an HTTP failure, or a `nextLink` that fails host validation - that one sub-request is marked failed: `Status` becomes the failing page's status, `PagingIncomplete` is set to `$true`, and a non-terminating error is written (`BatchPagingFailed`, `BatchNextLinkRefused`, or `BatchPagingTruncated`). The items already collected stay in `Body`, so you can use them deliberately. One failed link never discards the other results in the batch. - -```yaml -Type: SwitchParameter -Required: False -Position: Named -Default value: False -Accept pipeline input: False -Accept wildcard characters: False +``` +***REDACTED*** ``` -### -MaxPage -Ceiling on the number of pages drained per sub-request when `-FollowNextLink` is used. The default, 0, is unlimited. Hitting the ceiling marks the result `PagingIncomplete` and writes a `BatchPagingTruncated` error, so a bounded read is never mistaken for a complete one. +Read it to decide what to resubmit, and supply the sensitive fields again yourself. Re-piping the file - `Get-Content dead.jsonl | ConvertFrom-Json | Invoke-MgxBatchRequest` - resends the lines that carry no marker, and refuses the ones that do as an error record before the first chunk leaves: a Body that is the marker whole is not JSON, and a Body carrying it in a field parses but is no longer the body that was sent, so that one is refused naming the field. Any body carrying that marker text is refused the same way before the first chunk leaves, whether it came from the file or not. Under -ErrorAction Stop - which this cmdlet's description recommends - the pipeline then ends with nothing sent at all: re-pipe at the default error preference, or filter the marker lines out first. A refused item carries that status whether its POST reached the server and went unanswered or never left at all - an open circuit, a rate limiter with no permit for it - and the status does not separate the two. Read it as the first: the write may have been applied, so resending one is a decision about duplicates rather than a free retry. Only status 0 says nothing was sent. -```yaml -Type: Int32 -Required: False -Position: Named -Default value: 0 -Accept pipeline input: False -Accept wildcard characters: False -``` -### -DeadLetterPath -Path to a JSONL file where failed batch items (status >= 400) are appended, along with any that were never sent because an earlier chunk failed. Each line contains Url, Method, Body (with sensitive fields redacted), Status, Error, Timestamp, and the Id if one was supplied. It is a record of what to retry, not a request you can replay directly: the redaction that keeps passwords out of the file also means a redacted Body is no longer the body that was sent. Read it to decide what to resubmit, and supply the sensitive fields again yourself. +The file is opened before the first request goes out and held open for the run, so a path this run cannot write - a directory standing at it, a parent directory that is not there, a file this account may not write - is refused with nothing sent at all, and the error says so: fix the path and run the same command again. It has to be a file system path with something in it. `Env:\X` resolves to a bare name relative to nothing and an empty path resolves to the working directory, so both are refused rather than written somewhere nobody named. A run with no failed and no never-sent items has no lines for the file, and one it created for them is removed again, so a batch that succeeded whole leaves nothing behind; a file that was already there holds an earlier run's outstanding work and is left as it was, and so is one a second run appending to the same path has put a line in. ```yaml Type: String @@ -222,11 +206,8 @@ Accept wildcard characters: False ### -ConsistencyLevel ConsistencyLevel header added to each individual batch item. Required when any batch item URL contains $search (Graph advanced query capabilities). Takes precedence over the same key in -Headers. Source: [Advanced query capabilities on Microsoft Entra ID objects](https://learn.microsoft.com/en-us/graph/aad-advanced-queries) -### -Headers -Custom headers applied to each individual batch item. Accepts a hashtable of key-value pairs. Merged with -ConsistencyLevel and -ThrottlePriority (dedicated parameters take precedence over matching keys in -Headers). - ```yaml -Type: Hashtable +Type: String Parameter Sets: (All) Aliases: @@ -237,16 +218,13 @@ Accept pipeline input: False Accept wildcard characters: False ``` -### -ThrottlePriority -Throttle priority hint for Graph API. Graph uses this to decide which requests to throttle first under pressure. Valid values: Low, Normal, High. - -Sets `x-ms-throttle-priority` header on each batch item. Use `Low` for background jobs that should yield to interactive workloads. +### -Headers +Custom headers applied to each individual batch item. Accepts a hashtable of key-value pairs. Merged with -ConsistencyLevel and -ThrottlePriority (dedicated parameters take precedence over matching keys in -Headers). ```yaml -Type: String +Type: Hashtable Parameter Sets: (All) Aliases: -Accepted values: Low, Normal, High Required: False Position: Named @@ -255,10 +233,16 @@ Accept pipeline input: False Accept wildcard characters: False ``` +### -ThrottlePriority +Throttle priority hint for Graph API. Graph uses this to decide which requests to throttle first under pressure. Valid values: Low, Normal, High. + +Sets `x-ms-throttle-priority` header on each batch item. Use `Low` for background jobs that should yield to interactive workloads. + ```yaml Type: String Parameter Sets: (All) Aliases: +Accepted values: Low, Normal, High Required: False Position: Named @@ -284,7 +268,7 @@ Accept wildcard characters: False ``` ### -Uri -Graph API URLs to batch. Accepts absolute URLs (https://graph.microsoft.com/v1.0/users/id) or relative URLs (/users/id). Also accepts hashtables or PSCustomObjects with Url, Method, and Body members for per-item control, plus an optional Id that is echoed back on the matching result. +Graph API URLs to batch. Accepts absolute URLs (https://graph.microsoft.com/v1.0/users/id) or relative URLs (/users/id). Also accepts hashtables or PSCustomObjects with Url, Method, and Body members for per-item control. ```yaml Type: Object[] @@ -299,7 +283,9 @@ Accept wildcard characters: False ``` ### -WhatIf -Shows what would happen if the cmdlet runs. The cmdlet is not run. +Shows what would happen if the cmdlet runs. The cmdlet is not run. The gate covers reads as well as writes: an all-GET batch is described and not sent, because even a read batch spends resource units, can be throttled, and emits objects into the pipeline. Invoke-MgxRequest makes the other choice and sends its reads under -WhatIf. + +The count it names is what would be sent. Bodies are validated and checked for the redaction marker before the gate, so an item either of those refuses is left out of that count and named beside it - `PATCH 1 request via $batch; 2 refused` - and each refusal is written as the error record it is, under -WhatIf as without it: a refusal is not an action the gate governs. ```yaml Type: SwitchParameter @@ -313,6 +299,36 @@ Accept pipeline input: False Accept wildcard characters: False ``` +### -FollowNextLink +Drain `@odata.nextLink` in sub-response bodies, merging the pages into each result. Off by default, where a sub-response that carries a next link is returned with the items of its first page and the link beside them. Follow-up pages are submitted as further batches, so N partial collections drain in ceil(N/20) requests per page rather than N. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -MaxPage +Ceiling on the pages drained per sub-request when -FollowNextLink is on. 0, the default, is unlimited. A sub-request stopped at the ceiling with more to read is reported as such rather than returned as though it were whole. + +```yaml +Type: Int32 +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: 0 +Accept pipeline input: False +Accept wildcard characters: False +``` + ### -ProgressAction Determines how the cmdlet responds to progress updates. @@ -339,18 +355,18 @@ String URLs, or hashtables or PSCustomObjects with Url, Method, and Body members ## OUTPUTS ### System.Collections.Hashtable -Per-request results with Url, Method, Status, and Body keys, plus Id when the request supplied one, NotSent on any operation that was never sent because an earlier chunk failed, and PagingIncomplete when -FollowNextLink could not drain the collection. Body here is the RESPONSE body - for a failure that is the error envelope, not the request - so piping results straight back resubmits the wrong thing. Use Url and Method to rebuild the requests you want to retry. +Per-request results with Url, Method, Status, and Body keys. Status says what became of the operation: an item the server answered keeps the status it answered with; an item whose chunk was refused carries the refusal's status (>= 400), which covers both a POST that went out unanswered and one an open circuit or a rate limiter stopped before it left, so read it as a write that may have been applied; only an item in a chunk that was never POSTed gets status 0, and only those carry NotSent - another chunk of the batch failed, not necessarily one before it, since chunks can run in parallel. Body here is the RESPONSE body - for a failure that is the error envelope, not the request - so piping results straight back resubmits the wrong thing. Use Url and Method to rebuild the requests you want to retry. ## NOTES Each batch item is retried individually on 429 (throttled) or 5xx errors (for idempotent methods). POST requests only retry on 429 because POST is non-idempotent - retrying a failed POST on 5xx could create duplicates if the server processed the request before the error. This matches the Kiota SDK retry behavior. Source: [Microsoft Graph error responses and resource types](https://learn.microsoft.com/en-us/graph/errors) -Items that exhaust per-chunk retries get one additional batch-level retry pass. +Items that exhaust per-chunk retries get one additional batch-level retry pass, except after a chunk failure: the run has stopped sending, and that pass is a send. It is skipped for every item it would have picked up, including items in chunks that were POSTed and answered in full, and they are handed back with the status the server gave them rather than resent. Use -Verbose to see retry counts, throttle encounters, and timing. Batching reduces HTTP round-trips (20 operations per request instead of 1), but Graph counts each item inside a batch individually against the server-side write quota (3,000 writes / 2.5 min per app+tenant). Sustained write throughput caps at ~20/sec regardless of batching. See [Set-MgxOption](Set-MgxOption.md) for the full throttle limits table and tuning guidance. Source: [Microsoft Graph service-specific throttling limits](https://learn.microsoft.com/en-us/graph/throttling-limits) -Pipe all items into a single Invoke-MgxBatchRequest call rather than calling it in a loop. One call chunks the items itself, so it makes far fewer round-trips than a loop of single-item calls and reuses one HTTP client, one rate limiter and one circuit breaker. Repeated invocation is safe at any rate the host can drive. +For best performance and stability, always pipe all items into a single Invoke-MgxBatchRequest call rather than calling it in a loop. At 200+ rapid invocations per second, PowerShell's internal pipeline thread safety can race between cmdlet instances. Piping all items into one call avoids this entirely and is significantly faster. ## RELATED LINKS [Invoke-MgxRequest](Invoke-MgxRequest.md) diff --git a/Modules/M365DSC.mgx/help/Invoke-MgxRequest.md b/Modules/M365DSC.mgx/help/Invoke-MgxRequest.md index 0ff53a0..d41de63 100644 --- a/Modules/M365DSC.mgx/help/Invoke-MgxRequest.md +++ b/Modules/M365DSC.mgx/help/Invoke-MgxRequest.md @@ -1,4 +1,4 @@ ---- +--- external help file: Mgx.Cmdlets.dll-Help.xml Module Name: Mgx online version: https://github.com/Microsoft365DSC/mgx/blob/main/Modules/M365DSC.mgx/help/Invoke-MgxRequest.md @@ -16,6 +16,7 @@ General-purpose resilient client for any Microsoft Graph endpoint. ``` Invoke-MgxRequest [-Uri] [-Method ] [-Body ] [-Property ] [-ExpandProperty ] [-ConsistencyLevel ] [-Headers ] [-ApiVersion ] [-Raw] + [-Envelope] [-Filter ] [-Sort ] [-Search ] [-Skip ] [-All] [-Top ] [-PageSize ] [-CountVariable ] [-CheckpointPath ] [-NoPageSize] [-ProgressAction ] [-WhatIf] [-Confirm] [] @@ -25,8 +26,9 @@ Invoke-MgxRequest [-Uri] [-Method ] [-Body ] [-Property ``` Invoke-MgxRequest [-Uri] [-Method ] [-Body ] [-Property ] [-ExpandProperty ] [-ConsistencyLevel ] [-Headers ] [-ApiVersion ] [-Raw] + [-Envelope] [-Filter ] [-Sort ] [-Search ] [-Skip ] [-All] [-Top ] - [-PageSize ] [-CountVariable ] [-CheckpointPath ] [-NoPageSize] [-InputObject ] + [-PageSize ] [-CountVariable ] [-CheckpointPath ] [-NoPageSize] [-InputObject ] [-Concurrency ] [-SkipNotFound] [-SkipForbidden] [-ProgressAction ] [-WhatIf] [-Confirm] [] ``` @@ -183,6 +185,8 @@ Accept wildcard characters: False ### -Body Request body for write operations (POST, PATCH, PUT). Accepts a hashtable, PSObject or array, which is serialized to JSON, or a string already holding JSON, which is sent verbatim. Ignored (with a warning) on GET. +Enums serialize as camelCase names, byte arrays as base64, TimeSpan as an ISO-8601 duration, and a DateTime without a kind is treated as UTC. A [Flags] combination goes out in the comma-joined form OData expects, with no space: `ignoreCase,multiline`. Each member of a combination is named as it would be alone, so a name set by JsonStringEnumMemberName, or the choice between two members sharing a value, reads the same either way. camelCase lowercases a leading acronym whole, so a member named `IOS` is sent as `ios`; where Graph wants `iOS`, pass the string rather than the enum. A SecureString, PSCredential, script block, certificate, NaN or Infinity anywhere in the body is refused before anything is sent, with an error naming the property. A string that is not JSON is sent verbatim; add a Content-Type to -Headers to declare its real type. + ```yaml Type: Object Parameter Sets: (All) @@ -196,7 +200,7 @@ Accept wildcard characters: False ``` ### -CheckpointPath -Path to a JSON checkpoint file for resumable pagination. On interruption (Ctrl+C), progress is saved. On re-run with the same checkpoint path, pagination resumes from where it left off. The checkpoint file is deleted on successful completion. +Path to a JSON checkpoint file for resumable pagination. Consulted only by a collection read carrying -All, -Top, -Skip, -Filter, -Search, -Sort or -CountVariable; any other request - a plain collection GET, one with only -PageSize, a single-entity GET, a fan-out over two or more piped IDs, a write - binds it and ignores it. A single piped ID is not a fan-out: it is sent as a direct request and, on a collection read, consults the checkpoint like any other. On interruption (Ctrl+C), progress is saved. On re-run with the same checkpoint path, pagination resumes from where it left off. The checkpoint file is deleted on successful completion. Those writes are gated on their own: -WhatIf reads the file and leaves it exactly as it was, and -Confirm asks about it once per run that consults it, including when several values are piped into a -Uri that carries no placeholder, which runs the request once per value. ```yaml Type: String @@ -226,7 +230,7 @@ Accept wildcard characters: False ``` ### -Confirm -Prompts you for confirmation before running the cmdlet. +Prompts before a write and not before a read. POST, PATCH, PUT and DELETE are confirmed - a direct request one prompt at a time, a fan-out over piped IDs one prompt for all of its items - and a GET is sent without a prompt, the same division -WhatIf makes. With -CheckpointPath there is one prompt more, but only on a collection read carrying -All, -Top, -Skip, -Filter, -Search, -Sort or -CountVariable, asked once per run before any page is fetched, covering the saves and the deletes of the checkpoint file itself - including when several values are piped into a -Uri that carries no placeholder, which runs the request once per value; any other request - a plain collection GET, one with only -PageSize, a single-entity GET, a fan-out over two or more piped IDs, a write - binds the parameter and never reads it, so none of them adds a prompt. A single piped ID is not a fan-out: it is sent as a direct request and, on a collection read, prompts the same as any other. ```yaml Type: SwitchParameter @@ -319,7 +323,7 @@ Accept wildcard characters: False Pipeline input for fan-out operations. Each value replaces the {id} placeholder in the URI template. Accepts string values or objects with an Id property. ```yaml -Type: String +Type: Object Parameter Sets: Pipeline Aliases: Id @@ -391,6 +395,21 @@ Accept pipeline input: False Accept wildcard characters: False ``` +### -Envelope +Return the payload as the service sent it, collection envelope and all, instead of the items inside it. Without it a collection holding one item and a single entity are both one object, which a caller that has to tell them apart cannot do. -All aggregates pages and ignores this. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + ### -Raw Return raw JSON strings instead of hashtables. Useful for piping to ConvertFrom-Json or writing to files. @@ -512,7 +531,7 @@ Accept wildcard characters: False ``` ### -WhatIf -Shows what would happen if the cmdlet runs. The cmdlet is not run. +Describes the write instead of sending it. Only writes are gated: POST, PATCH, PUT and DELETE, single or fanned out over piped IDs, are described and not sent. Reads are sent - a GET under -WhatIf connects, pages, spends resource units, can be throttled, and emits its objects - which is the PowerShell convention and what makes a dry run under $WhatIfPreference useful, since the gated writes are then previewed against real data. The checkpoint at -CheckpointPath is read and then left alone: a preview resumes from the position it finds, so it sends the same reads the run would rather than paging from the start, and it neither saves over that file nor deletes it, so the resume position is exactly where it was. Invoke-MgxBatchRequest makes the other choice and gates reads too. ```yaml Type: SwitchParameter diff --git a/Modules/M365DSC.mgx/help/Set-MgxOption.md b/Modules/M365DSC.mgx/help/Set-MgxOption.md index 9ffc88d..0af6199 100644 --- a/Modules/M365DSC.mgx/help/Set-MgxOption.md +++ b/Modules/M365DSC.mgx/help/Set-MgxOption.md @@ -17,7 +17,8 @@ Set-MgxOption [-RateLimitBurst ] [-RateLimitPerSecond ] [-NoRateLi [-RateLimitQueueLimit ] [-MaxRetryAfterSeconds ] [-MaxRetryAttempts ] [-TotalTimeoutSeconds ] [-AttemptTimeoutSeconds ] [-CircuitBreakerDurationSeconds ] [-CircuitBreakerFailureRatio ] [-CircuitBreakerMinThroughput ] - [-CircuitBreakerSamplingDurationSeconds ] [-BatchItemsPerSecond ] [-Reset] [] + [-CircuitBreakerSamplingDurationSeconds ] [-BatchChunkConcurrency ] + [-BatchItemsPerSecond ] [-Reset] [-WhatIf] [-Confirm] [] ``` ## DESCRIPTION @@ -133,7 +134,7 @@ Accept wildcard characters: False ``` ### -CircuitBreakerMinThroughput -Minimum number of requests in the sampling window before the circuit breaker evaluates failure ratio. Range: 1-1000. Default: 40. +Minimum number of requests in the sampling window before the circuit breaker evaluates failure ratio. Range: 2-1000. Default: 40. Two is the least the breaker will hold; a ratio over a single request is not a measurement. ```yaml Type: Int32 @@ -297,6 +298,36 @@ Accept pipeline input: False Accept wildcard characters: False ``` +### -WhatIf +Shows what would happen if the cmdlet runs. The cmdlet is not run. Every path that changes an option - -Reset and each individual parameter alike - sits behind this same check, so nothing is changed under -WhatIf. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: wi + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Confirm +Prompts you for confirmation before running the cmdlet. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: cf + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + ### CommonParameters This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see [about_CommonParameters](http://go.microsoft.com/fwlink/?LinkID=113216). diff --git a/Modules/M365DSC.mgx/help/Sync-MgxDelta.md b/Modules/M365DSC.mgx/help/Sync-MgxDelta.md index 6271986..14c841e 100644 --- a/Modules/M365DSC.mgx/help/Sync-MgxDelta.md +++ b/Modules/M365DSC.mgx/help/Sync-MgxDelta.md @@ -28,7 +28,7 @@ Drive delta is fully supported: `/me/drive/root/delta`, `/drives/{id}/root/delta Items that no longer match the query appear with an `@removed` property containing `{"reason": "changed"}` (item moved out of scope or soft-deleted) or `{"reason": "deleted"}` (permanently deleted). Filter these with `Where-Object { -not $_.'@removed' }`. The completion message counts them separately. -Delta tokens expire after approximately 7 days for directory objects (users, groups, applications). When a token expires, Graph returns HTTP 410 Gone. Sync-MgxDelta handles this automatically by deleting the stale token (and any resume checkpoint) and performing a full re-sync with a warning. +Delta tokens expire after approximately 7 days for directory objects (users, groups, applications). When a token expires, Graph returns HTTP 410 Gone. Sync-MgxDelta handles this automatically by deleting the stale token (and this run's own resume checkpoint) and performing a full re-sync with a warning. Resume semantics are at-least-once: in pipeline mode the page in flight at a crash is re-emitted in full on resume; in JSONL mode mid-page checkpoints keep the window to at most 500 items. Deduplicate on `id` downstream if exact-once matters. @@ -123,7 +123,9 @@ Accept wildcard characters: False ``` ### -CheckpointPath -Path for the ephemeral mid-run resume checkpoint. Saved at page boundaries (and every 500 items in JSONL mode) while an enumeration is in flight; deleted on successful completion. Any event that invalidates the enumeration - HTTP 410 Gone, -FullSync, or a -Property/-Filter/-Prefer change - deletes it too. Must differ from -DeltaPath and -OutputFile. +Path for the ephemeral mid-run resume checkpoint. Saved at page boundaries (and every 500 items in JSONL mode) while an enumeration is in flight; deleted on successful completion. Any event that invalidates the enumeration - HTTP 410 Gone, -FullSync, or a -Property/-Filter/-Prefer change - deletes it too. A completion and a 410 delete this run's own checkpoint; one they recognize as another sync's is left as it stands, for the sync that resumes from exactly it. Must differ from -DeltaPath and -OutputFile. + +A run stops - with nothing written to the output, the checkpoint, a temp or the delta state - when the temp file the checkpoint names, or the output it would append to or replace, is open in another sync or cannot be opened for writing (a permission, a directory or a pipe at the path, a link that cannot be followed, a path too long for the file system, a socket, a directory above it this account cannot search), or when the copy it stages before promoting a temp fails, in which case both files are left as found: the changes it counts are in that file and this checkpoint is the only thing that counts them, and syncing anyway would write over both and advance the token past those changes. This holds whether the run would append, promote a temp, or re-enumerate from the last saved delta token after its temp vanished. The copy a promotion stages is written beside the output under a name the run creates itself, and the name is cleared first: a link, a pipe, a socket, a copy left by an interrupted promotion or a copy this account cannot open standing there is deleted and the removal reported as a warning, while a directory or a copy another run holds stops the run instead. Wait for that sync to finish, give this run its own -OutputFile and -CheckpointPath, or grant write access to that file - or to the directory holding it, where that is what the run names - and run again. A resumed or promoting run holds the output open for as long as it is appending, which is what stops a second run over the same command line; a run recovering a checkpoint older than 2.1 opens the output as it always has. On Linux and macOS a PowerShell reader of a held file fails with a sharing violation until the run ends, while tail and cat read it as before. ```yaml Type: String diff --git a/examples/README.md b/examples/README.md index 60aa05f..37be230 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,14 +1,14 @@ # Mgx Examples -Runnable scripts covering common Microsoft Graph scenarios. Each script is self-contained and includes the required scopes in its header comment. +Runnable scripts for things people actually do against Microsoft Graph. Each one is self-contained and names the scopes it needs in its header comment. -Scripts are grouped by purpose - pick the directory that matches what you are trying to do. +Scripts are grouped by the job, not by the cmdlet - pick the directory that matches what you are trying to get done. Within a directory, the shorter script usually comes first and the longer one builds on it. ## Prerequisites ```powershell -Install-Module M365DSC.mgx # once -Import-Module M365DSC.mgx +Install-Module Mgx # once +Import-Module Mgx Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All" # adjust scopes per script ``` @@ -17,81 +17,54 @@ Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All" # adjust scopes per ### Getting Started (`getting-started/`) | Script | Description | Scopes | -| -------- | ------------- | -------- | +|--------|-------------|--------| | [get-all-users.ps1](getting-started/get-all-users.ps1) | Stream all users to the console | `User.Read.All` | | [export-users-to-jsonl.ps1](getting-started/export-users-to-jsonl.ps1) | Export all users to a JSONL file with checkpoint/resume | `User.Read.All` | -| [beta-endpoint.ps1](getting-started/beta-endpoint.ps1) | Access beta endpoints without installing extra modules | `User.Read.All` | - -### Fan-Out & Relations (`fan-out-and-relations/`) - -| Script | Description | Scopes | -| -------- | ------------- | -------- | -| [get-managers-fan-out.ps1](fan-out-and-relations/get-managers-fan-out.ps1) | Fetch every user's manager concurrently | `User.Read.All` | -| [enrich-users-with-manager.ps1](fan-out-and-relations/enrich-users-with-manager.ps1) | Attach manager as a property on each user object | `User.Read.All` | -| [chained-relation-expansion.ps1](fan-out-and-relations/chained-relation-expansion.ps1) | Enrich users with manager + licenses in one pass | `User.Read.All` | -| [group-members-multipage.ps1](fan-out-and-relations/group-members-multipage.ps1) | Stream all members from all groups concurrently | `Group.Read.All`, `User.Read.All` | +| [export-users-to-csv.ps1](getting-started/export-users-to-csv.ps1) | Export all users to CSV, flattening the nested properties | `User.Read.All` | +| [delta-users-sync.ps1](getting-started/delta-users-sync.ps1) | Full sync on first run, changed users only thereafter | `User.Read.All` | ### Reporting & Audit (`reporting-and-audit/`) | Script | Description | Scopes | -| -------- | ------------- | -------- | -| [disabled-accounts-report.ps1](reporting-and-audit/disabled-accounts-report.ps1) | Report all disabled accounts | `User.Read.All` | -| [guest-users-report.ps1](reporting-and-audit/guest-users-report.ps1) | Report all guest (external) accounts | `User.Read.All` | -| [stale-devices-report.ps1](reporting-and-audit/stale-devices-report.ps1) | Report devices inactive for 90+ days | `Device.Read.All` | +|--------|-------------|--------| +| [user-directory-report.ps1](reporting-and-audit/user-directory-report.ps1) | Every enabled user with manager and licenses attached | `User.Read.All` | +| [download-groups-and-members.ps1](reporting-and-audit/download-groups-and-members.ps1) | Download every group and all memberships, nesting resolved locally, with a progress bar | `Group.Read.All`, `User.Read.All` | +| [group-membership-report.ps1](reporting-and-audit/group-membership-report.ps1) | Direct vs effective membership of every group, nesting resolved server-side | `Group.Read.All`, `User.Read.All` | +| [inactive-users-report.ps1](reporting-and-audit/inactive-users-report.ps1) | Users not seen in 90 days, and those who never signed in | `User.Read.All`, `AuditLog.Read.All` | +| [license-report.ps1](reporting-and-audit/license-report.ps1) | Seats per SKU, who holds them, and which are on disabled accounts | `Organization.Read.All`, `User.Read.All` | | [app-secrets-expiry.ps1](reporting-and-audit/app-secrets-expiry.ps1) | Find app secrets and certificates expiring within 30 days | `Application.Read.All` | -| [conditional-access-export.ps1](reporting-and-audit/conditional-access-export.ps1) | Export all Conditional Access policies (beta endpoint) | `Policy.Read.All` | | [export-sign-in-logs.ps1](reporting-and-audit/export-sign-in-logs.ps1) | Export sign-in logs to JSONL with checkpoint/resume | `AuditLog.Read.All` | ### Bulk Writes & Batching (`bulk-writes-and-batching/`) | Script | Description | Scopes | -| -------- | ------------- | -------- | -| [bulk-update.ps1](bulk-writes-and-batching/bulk-update.ps1) | PATCH multiple users via `$batch` (20 per HTTP call) | `User.ReadWrite.All` | -| [dead-letter-retry.ps1](bulk-writes-and-batching/dead-letter-retry.ps1) | Bulk create users with dead-letter tracking for failures | `User.ReadWrite.All` | +|--------|-------------|--------| +| [tenant-snapshot.ps1](bulk-writes-and-batching/tenant-snapshot.ps1) | Users, groups, apps, and licenses in one HTTP round trip | `User.Read.All`, `Group.Read.All`, `Application.Read.All` | +| [bulk-create-users-from-csv.ps1](bulk-writes-and-batching/bulk-create-users-from-csv.ps1) | Create users from a CSV, keeping failures for replay | `User.ReadWrite.All` | +| [assign-licenses.ps1](bulk-writes-and-batching/assign-licenses.ps1) | License everyone who is missing one, seat-checked first | `User.ReadWrite.All`, `Organization.Read.All` | +| [offboard-users.ps1](bulk-writes-and-batching/offboard-users.ps1) | Block sign-in, revoke sessions, and return the licenses | `User.ReadWrite.All`, `Directory.ReadWrite.All` | | [bulk-delete-whatif.ps1](bulk-writes-and-batching/bulk-delete-whatif.ps1) | Bulk delete stale guests with `-WhatIf` preview | `User.ReadWrite.All` | -| [mixed-endpoint-batch.ps1](bulk-writes-and-batching/mixed-endpoint-batch.ps1) | Query users, groups, apps, and SKUs in one HTTP call | `User.Read.All`, `Group.Read.All`, `Application.Read.All` | -### Delta & Sync (`delta-and-sync/`) +### Change Feeds (`change-feeds/`) | Script | Description | Scopes | -| -------- | ------------- | -------- | -| [delta-sync.ps1](delta-and-sync/delta-sync.ps1) | Full sync on first run, incremental changes thereafter | `User.Read.All` | -| [drive-delta-resumable.ps1](delta-and-sync/drive-delta-resumable.ps1) | Resumable OneDrive/SharePoint delta sync with checkpointing | `Files.Read.All` | -| [tenant-change-feed.ps1](delta-and-sync/tenant-change-feed.ps1) | A resumable tenant-wide change feed, baselined with `-Latest` | `Group.Read.All`, `Application.Read.All` | +|--------|-------------|--------| +| [tenant-inventory.ps1](change-feeds/tenant-inventory.ps1) | Snapshot every root collection `$metadata` declares, resumable | whatever read scopes you have | +| [group-change-feed.ps1](change-feeds/group-change-feed.ps1) | Baseline one resource with `-Latest`, then log only what changes | `Group.Read.All` | +| [tenant-change-feed.ps1](change-feeds/tenant-change-feed.ps1) | The same across several resources, resumable, as a scheduled job | `Group.Read.All`, `Application.Read.All` | ### Drive Content (`drive-content/`) -| Script | Description | Scopes | -| ------------------------------------------------------------------ | ------------------------------------------------------------------------ | ---------------- | -| [partial-content-hash.ps1](drive-content/partial-content-hash.ps1) | Identify files by hashing a byte range instead of downloading them whole | `Files.Read.All` | -| [drive-content-triage.ps1](drive-content/drive-content-triage.ps1) | Sniff 4 KB of each file, then download only what survives triage | `Files.Read.All` | +| Script | Description | Scopes | +|--------|-------------|--------| +| [drive-delta-resumable.ps1](drive-content/drive-delta-resumable.ps1) | Resumable OneDrive/SharePoint drive enumeration to JSONL | `Files.Read.All` | +| [download-user-onedrive.ps1](drive-content/download-user-onedrive.ps1) | Download a user's whole OneDrive, preserving the folder tree | `Files.Read.All` | +| [drive-content-triage.ps1](drive-content/drive-content-triage.ps1) | Sniff 4 KB of each file, then download only what survives triage | `Files.Read.All` | ### Resilience & Telemetry (`resilience-and-telemetry/`) | Script | Description | Scopes | -| -------- | ------------- | -------- | -| [resilience-for-existing-scripts.ps1](resilience-and-telemetry/resilience-for-existing-scripts.ps1) | Add retry/circuit breaker to existing SDK scripts (zero code changes) | `User.Read.All` | -| [resilience-status.ps1](resilience-and-telemetry/resilience-status.ps1) | Check, enable, and disable SDK resilience injection | `User.Read.All` | -| [telemetry.ps1](resilience-and-telemetry/telemetry.ps1) | View request counts, retries, and throttle events for the session | `User.Read.All` | -| [tune-rate-limits.ps1](resilience-and-telemetry/tune-rate-limits.ps1) | Tune rate limiter, retry count, and timeouts at runtime | `User.Read.All` | +|--------|-------------|--------| +| [resilience-status.ps1](resilience-and-telemetry/resilience-status.ps1) | Add retry and circuit breaking to existing SDK scripts, then check and remove it | `User.Read.All` | +| [progress-for-long-runs.ps1](resilience-and-telemetry/progress-for-long-runs.ps1) | A live status line for a streamed fan-out: percent, rate, ETA, resource units | `Group.Read.All`, `User.Read.All` | | [resource-unit-budgeting.ps1](resilience-and-telemetry/resource-unit-budgeting.ps1) | Measure what query shapes cost in resource units, and size a fan-out from the measurement | `Group.Read.All`, `User.Read.All` | -| [pacing-observability.ps1](resilience-and-telemetry/pacing-observability.ps1) | Watch the adaptive pacer: slow start, per-workload buckets, and which throttle signals are real | `User.Read.All`, `Files.Read.All` | - -## Cmdlet Coverage - -Every Mgx cmdlet is demonstrated in at least one script: - -| Cmdlet | Scripts | -| -------- | --------- | -| `Invoke-MgxRequest` | get-all-users, beta-endpoint, get-managers-fan-out, enrich-users-with-manager, chained-relation-expansion, group-members-multipage, disabled-accounts-report, guest-users-report, stale-devices-report, app-secrets-expiry, conditional-access-export, bulk-delete-whatif, partial-content-hash, drive-content-triage, telemetry, tune-rate-limits, tenant-resilience-comparison, resource-unit-budgeting, pacing-observability | -| `Invoke-MgxBatchRequest` | bulk-update, dead-letter-retry, mixed-endpoint-batch | -| `Export-MgxCollection` | export-users-to-jsonl, export-sign-in-logs, tenant-resilience-comparison | -| `Expand-MgxRelation` | enrich-users-with-manager, chained-relation-expansion | -| `Sync-MgxDelta` | delta-sync, drive-delta-resumable, tenant-change-feed | -| `Enable-MgxResilience` | resilience-for-existing-scripts, resilience-status, tenant-resilience-comparison | -| `Disable-MgxResilience` | resilience-status, tenant-resilience-comparison | -| `Get-MgxResilience` | resilience-status | -| `Set-MgxOption` | tune-rate-limits, pacing-observability | -| `Get-MgxOption` | tune-rate-limits | -| `Get-MgxTelemetry` | telemetry, tenant-resilience-comparison, resource-unit-budgeting, pacing-observability, partial-content-hash, drive-content-triage | -| `Get-MgxContent` | partial-content-hash, drive-content-triage | diff --git a/examples/bulk-writes-and-batching/assign-licenses.ps1 b/examples/bulk-writes-and-batching/assign-licenses.ps1 new file mode 100644 index 0000000..23374b1 --- /dev/null +++ b/examples/bulk-writes-and-batching/assign-licenses.ps1 @@ -0,0 +1,106 @@ +# Assign a license to every enabled user who does not have one yet. +# +# Two things have to happen and both are batched: a user with no usageLocation +# cannot be licensed at all, so those get a PATCH first, then everyone gets an +# assignLicense POST. Invoke-MgxBatchRequest bundles 20 requests per HTTP call, +# so 400 users are 20 calls rather than 400. +# +# Seats are checked against /subscribedSkus before anything is written - Graph +# will happily refuse item 300 of 400 and leave you working out which ones landed. +# +# Requirements: Connect-MgGraph -Scopes "User.ReadWrite.All", +# "Organization.Read.All", "Directory.ReadWrite.All" + +param( + [string]$SkuPartNumber = 'ENTERPRISEPACK', # Office 365 E3 + [string]$DefaultUsageLocation = 'US', + [switch]$Execute # omit for a preview +) + +Import-Module Mgx + +$sku = Invoke-MgxRequest /subscribedSkus | + Where-Object { $_.skuPartNumber -eq $SkuPartNumber } | + Select-Object -First 1 +if (-not $sku) { throw "No subscription for $SkuPartNumber in this tenant." } + +$available = $sku.prepaidUnits.enabled - $sku.consumedUnits +Write-Host ("{0}: {1} of {2} seats used, {3} free" -f ` + $SkuPartNumber, $sku.consumedUnits, $sku.prepaidUnits.enabled, $available) + +# assignedLicenses is filterable server-side, but only as an advanced query +# ($count=true plus ConsistencyLevel eventual). Checking it here keeps the +# example to one query shape and costs nothing extra on a tenant this size. +$candidates = @(Invoke-MgxRequest /users -All ` + -Filter 'accountEnabled eq true' ` + -Property id,userPrincipalName,usageLocation,assignedLicenses | + Where-Object { $_.assignedLicenses.skuId -notcontains $sku.skuId }) + +Write-Host "$($candidates.Count) user(s) without $SkuPartNumber" +if ($candidates.Count -eq 0) { return } +if ($candidates.Count -gt $available) { + throw "$($candidates.Count) users need a license but only $available seats are free." +} + +$needsLocation = @($candidates | Where-Object { -not $_.usageLocation }) +if (-not $Execute) { + Write-Host "Preview only - re-run with -Execute to write." + Write-Host " would set usageLocation on $($needsLocation.Count) user(s)" + Write-Host " would license $($candidates.Count) user(s)" + $candidates | Select-Object userPrincipalName, usageLocation | Format-Table -AutoSize + return +} + +# Stage 1: usageLocation. Batches run their items concurrently, so anything +# order-dependent has to be a separate call - which is why this is not folded +# into the assignLicense batch below. +if ($needsLocation) { + $patched = $needsLocation | ForEach-Object { + [PSCustomObject]@{ + Url = "/users/$($_.id)" + Method = 'PATCH' + Body = @{ usageLocation = $DefaultUsageLocation } + } + } | Invoke-MgxBatchRequest + Write-Host "usageLocation set: $(@($patched | Where-Object { $_.Status -lt 400 }).Count)" +} + +# Stage 2: the licenses themselves. +$assigned = $candidates | ForEach-Object { + [PSCustomObject]@{ + Url = "/users/$($_.id)/assignLicense" + Method = 'POST' + Body = @{ + addLicenses = @(@{ skuId = $sku.skuId; disabledPlans = @() }) + removeLicenses = @() + } + } +} | Invoke-MgxBatchRequest -DeadLetterPath ./license-failures.jsonl + +$ok = @($assigned | Where-Object { $_.Status -lt 400 }).Count +$fail = @($assigned | Where-Object { $_.Status -ge 400 }).Count +Write-Host "Licensed: $ok Failed: $fail" +if ($fail) { Write-Host "Failed items are in ./license-failures.jsonl" } + +<# +Expected output (preview): + +ENTERPRISEPACK: 22 of 25 seats used, 3 free +3 user(s) without ENTERPRISEPACK +Preview only - re-run with -Execute to write. + would set usageLocation on 1 user(s) + would license 3 user(s) + +userPrincipalName usageLocation +----------------- ------------- +graceb@contoso.com US +henriettam@contoso.com US +irvins@contoso.com + +Expected output (-Execute): + +ENTERPRISEPACK: 22 of 25 seats used, 3 free +3 user(s) without ENTERPRISEPACK +usageLocation set: 1 +Licensed: 3 Failed: 0 +#> diff --git a/examples/bulk-writes-and-batching/bulk-create-users-from-csv.ps1 b/examples/bulk-writes-and-batching/bulk-create-users-from-csv.ps1 new file mode 100644 index 0000000..e6de71b --- /dev/null +++ b/examples/bulk-writes-and-batching/bulk-create-users-from-csv.ps1 @@ -0,0 +1,86 @@ +# Create users from a CSV, keeping the ones that fail. +# +# A batch of 200 new hires will not be 200 clean creates: a duplicate UPN, a +# rejected password, a domain that is not verified. -DeadLetterPath writes those +# items - request and response - to a JSONL file instead of losing them in the +# console, so the run can be fixed and replayed against only what failed. +# +# The CSV is expected to have DisplayName, MailNickname, UserPrincipalName and +# Department columns: +# +# DisplayName,MailNickname,UserPrincipalName,Department +# Grace Blake,graceb,graceb@contoso.com,Retail +# +# Requirements: Connect-MgGraph -Scopes "User.ReadWrite.All" + +param( + [string]$Path = './new-hires.csv', + [string]$DeadLetterPath = './failed-users.jsonl' +) + +Import-Module Mgx + +function New-Password { + # Cryptographically random, unlike Get-Random. A real script delivers this + # out of band rather than leaving it in a variable. + $chars = 'abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789!@#$%^&*' + -join (1..16 | ForEach-Object { + $chars[[System.Security.Cryptography.RandomNumberGenerator]::GetInt32($chars.Length)] + }) +} + +$rows = Import-Csv $Path +Write-Host "$($rows.Count) row(s) in $Path" + +$requests = $rows | ForEach-Object { + [PSCustomObject]@{ + Url = '/users' + Method = 'POST' + Body = @{ + displayName = $_.DisplayName + mailNickname = $_.MailNickname + userPrincipalName = $_.UserPrincipalName + department = $_.Department + accountEnabled = $true + passwordProfile = @{ + password = New-Password + forceChangePasswordNextSignIn = $true + } + } + } +} + +$results = $requests | Invoke-MgxBatchRequest -DeadLetterPath $DeadLetterPath + +$created = @($results | Where-Object { $_.Status -lt 400 }) +$failed = @($results | Where-Object { $_.Status -ge 400 }) +Write-Host "Created: $($created.Count) Failed: $($failed.Count)" + +# What went wrong, grouped - one bad domain looks like 40 unrelated failures +# until they are collapsed like this. +if ($failed) { + $failed | + Group-Object { $_.Body.error.message } | + Select-Object Count, Name | + Format-Table -AutoSize -Wrap + Write-Host "Failed items saved to $DeadLetterPath" +} + +# Replaying: fix the CSV, or edit the dead-letter file and feed it straight back. +# Each line is a complete request, so nothing has to be rebuilt. +# +# Get-Content $DeadLetterPath | ConvertFrom-Json | Invoke-MgxBatchRequest + +<# +Expected output: + +12 row(s) in ./new-hires.csv +WARNING: 2 of 12 batch items failed after all retry attempts. Check $Error for details on each failed item. +Created: 10 Failed: 2 + +Count Name +----- ---- + 2 Another object with the same value for property userPrincipalName already exists. + +Failed items saved to ./failed-users.jsonl +#> diff --git a/examples/bulk-writes-and-batching/offboard-users.ps1 b/examples/bulk-writes-and-batching/offboard-users.ps1 new file mode 100644 index 0000000..a96d1a0 --- /dev/null +++ b/examples/bulk-writes-and-batching/offboard-users.ps1 @@ -0,0 +1,83 @@ +# Offboard departed users: block sign-in, kill live sessions, strip licenses. +# +# Three writes per user, and the order is not optional - blocking sign-in does +# nothing about a session that is already open, so the revoke has to follow it. +# A batch runs its items concurrently, so ordering means three batched stages +# rather than one batch of three requests per user. +# +# Stage 3 reads back each user's licenses first: assignLicense needs the skuIds +# to remove, and the account is already blocked by then, so nothing can change +# underneath it. +# +# Requirements: Connect-MgGraph -Scopes "User.ReadWrite.All", "Directory.ReadWrite.All" + +param( + [string[]]$UserPrincipalName, + [string]$Path, # or a CSV with a UserPrincipalName column + [switch]$Execute +) + +Import-Module Mgx + +$upns = if ($Path) { (Import-Csv $Path).UserPrincipalName } else { $UserPrincipalName } +if (-not $upns) { throw 'Nothing to offboard: pass -UserPrincipalName or -Path.' } + +# Resolve first: a typo in the list should surface here, not halfway through +# stage 2 with some accounts already blocked. +# {id} accepts bare strings as well as objects with an id, so UPNs pipe straight in. +$users = @($upns | Invoke-MgxRequest '/users/{id}' -SkipNotFound ` + -Property id,displayName,userPrincipalName,accountEnabled,assignedLicenses) + +Write-Host "$($users.Count) of $(@($upns).Count) account(s) resolved" +$users | Select-Object userPrincipalName, displayName, accountEnabled | Format-Table -AutoSize + +if (-not $Execute) { + Write-Host "Preview only - re-run with -Execute to write." + return +} + +# Stage 1: block sign-in. +$blocked = $users | ForEach-Object { + [PSCustomObject]@{ Url = "/users/$($_.id)"; Method = 'PATCH'; Body = @{ accountEnabled = $false } } +} | Invoke-MgxBatchRequest +Write-Host "blocked : $(@($blocked | Where-Object { $_.Status -lt 400 }).Count)" + +# Stage 2: revoke refresh tokens, ending sessions that are already open. +$revoked = $users | ForEach-Object { + [PSCustomObject]@{ Url = "/users/$($_.id)/revokeSignInSessions"; Method = 'POST' } +} | Invoke-MgxBatchRequest +Write-Host "revoked : $(@($revoked | Where-Object { $_.Status -lt 400 }).Count)" + +# Stage 3: return the licenses to the pool. Users with none are skipped - an +# empty removeLicenses array is a wasted request, not a no-op. +$licensed = @($users | Where-Object { $_.assignedLicenses.Count -gt 0 }) +if ($licensed) { + $stripped = $licensed | ForEach-Object { + [PSCustomObject]@{ + Url = "/users/$($_.id)/assignLicense" + Method = 'POST' + Body = @{ addLicenses = @(); removeLicenses = @($_.assignedLicenses.skuId) } + } + } | Invoke-MgxBatchRequest -DeadLetterPath ./offboard-failures.jsonl + Write-Host "unlicensed: $(@($stripped | Where-Object { $_.Status -lt 400 }).Count) of $($licensed.Count)" +} + +# Deliberately not done here: deleting the account. A blocked, unlicensed user +# keeps their OneDrive and mailbox reachable for whoever inherits the work - +# download-user-onedrive.ps1 is the other half of that. + +<# +Expected output: + +3 of 3 account(s) resolved + +userPrincipalName displayName accountEnabled +----------------- ----------- -------------- +graceb@contoso.com Grace Blake True +henriettam@contoso.com Henrietta Mueller True +irvins@contoso.com Irvin Sayers True + +blocked : 3 +revoked : 3 +unlicensed: 2 of 2 +#> diff --git a/examples/bulk-writes-and-batching/tenant-snapshot.ps1 b/examples/bulk-writes-and-batching/tenant-snapshot.ps1 new file mode 100644 index 0000000..3e9ebcb --- /dev/null +++ b/examples/bulk-writes-and-batching/tenant-snapshot.ps1 @@ -0,0 +1,53 @@ +# A one-call snapshot of the tenant: users, groups, apps, and licenses. +# +# Four unrelated endpoints in one HTTP round trip instead of four. Useful as the +# opening move of a script that needs a bit of everything, and as the thing to +# run when someone asks how big the tenant is. +# +# Requirements: Connect-MgGraph -Scopes "User.Read.All", "Group.Read.All", "Application.Read.All" + +Import-Module Mgx + +# $count=true returns the total in @odata.count without paging anything, and +# -ConsistencyLevel rides along on each subrequest, which Graph requires for it. +$counts = @( + "/users?`$top=1&`$count=true" + "/groups?`$top=1&`$count=true" + "/applications?`$top=1&`$count=true" + "/servicePrincipals?`$top=1&`$count=true" +) | Invoke-MgxBatchRequest -ConsistencyLevel eventual + +Write-Host "`n=== Tenant size ===" -ForegroundColor Cyan +$counts | ForEach-Object { + " {0,-20} {1,6}" -f ($_.Url -replace '\?.*$', ''), $_.Body.'@odata.count' +} + +$orgUri = "/organization?`$select=displayName,verifiedDomains" +$samples = @($orgUri, '/subscribedSkus') | Invoke-MgxBatchRequest + +# A batch result is Url / Method / Status / Body - the entity itself lives under +# Body, and a collection nests it one deeper under Body.value. Results come back +# keyed by the URL that produced them rather than in the order they were sent. +$byUrl = @{} +foreach ($r in $samples) { $byUrl[$r.Url] = $r.Body } + +$tenant = $byUrl[$orgUri].value[0] +$skus = $byUrl['/subscribedSkus'].value + +Write-Host "`n=== $($tenant.displayName) ===" -ForegroundColor Cyan +" domains : $(($tenant.verifiedDomains.name) -join ', ')" +" licenses: $(($skus | ForEach-Object { '{0} {1}/{2}' -f $_.skuPartNumber, $_.consumedUnits, $_.prepaidUnits.enabled }) -join ' ')" + +<# +Expected output: + +=== Tenant size === + /users 28 + /groups 12 + /applications 9 + /servicePrincipals 47 + +=== Contoso Ltd === + domains : contoso.onmicrosoft.com, contoso.com + licenses: ENTERPRISEPACK 22/25 EMSPREMIUM 4/10 POWER_BI_STANDARD 7/100 +#> diff --git a/examples/change-feeds/group-change-feed.ps1 b/examples/change-feeds/group-change-feed.ps1 new file mode 100644 index 0000000..fef569e --- /dev/null +++ b/examples/change-feeds/group-change-feed.ps1 @@ -0,0 +1,50 @@ +# Watch one resource for changes: baseline now, then get only what changed. +# +# This is the smallest useful change feed, and the thing to read before +# tenant-change-feed.ps1, which does the same across several resources at once. +# +# Two ideas: +# * -Latest baselines the delta token from now, instantly, without enumerating +# the resource first. A change feed does not want the initial dump - it wants +# "what happened since I started watching". +# * Delta state is a file. Point later runs at the same -DeltaPath and they pick +# up exactly where the last one stopped, which makes this a cron job. +# +# Requirements: Connect-MgGraph -Scopes "Group.Read.All" + +Import-Module Mgx + +$deltaFile = "./groups.delta" +$feedFile = "./groups-feed.jsonl" + +# First run only: start the feed from now instead of enumerating every group. +if (-not (Test-Path $deltaFile)) { + Write-Host "First run: baselining from now (no initial enumeration)." + $null = Sync-MgxDelta /groups/delta -DeltaPath $deltaFile -Latest + Write-Host "Baselined. Change a group, then run this again." + return +} + +$changes = Sync-MgxDelta /groups/delta -DeltaPath $deltaFile + +foreach ($group in $changes) { + $group | ConvertTo-Json -Compress -Depth 10 | Add-Content $feedFile +} + +Write-Host "$($changes.Count) change(s) since the last run, appended to $feedFile" +$changes | + Select-Object @{ Name = 'Change'; Expression = { if ($_.'@removed') { 'removed' } else { 'changed' } } }, + id, displayName | + Format-Table -AutoSize + +<# +Expected output (second run): + +3 change(s) since the last run, appended to ./groups-feed.jsonl + +Change id displayName +------ -- ----------- +changed 5cf66b99-8a06-436a-8107-644f1a1c4b3e Mark 8 Project Team +changed 24ffeed6-38b4-4bee-ae29-f0d18b5b7a92 Engineering All Hands +removed 8b1f2c04-19ac-4c0b-9a58-7f5e4c1d2a11 +#> diff --git a/examples/delta-and-sync/tenant-change-feed.ps1 b/examples/change-feeds/tenant-change-feed.ps1 similarity index 90% rename from examples/delta-and-sync/tenant-change-feed.ps1 rename to examples/change-feeds/tenant-change-feed.ps1 index 2a64dba..4b637e7 100644 --- a/examples/delta-and-sync/tenant-change-feed.ps1 +++ b/examples/change-feeds/tenant-change-feed.ps1 @@ -1,5 +1,13 @@ # A tenant change feed: everything that changed since the last run, resumable. # +# group-change-feed.ps1 does this for a single resource and is the shorter read; +# this is the same idea across several resources, with the details that show up +# at tenant scale - checkpointing and duplicate collapsing. +# +# A feed baselined with -Latest deliberately skips what existed before it +# started. If you want that too, tenant-inventory.ps1 is the snapshot half: +# inventory once, then this feed keeps it current. +# # Baseline once with -Latest (instant - no initial enumeration), then each # scheduled run appends only the changes to a JSONL log. Delta state makes # it incremental; -CheckpointPath makes a killed run continue instead of diff --git a/examples/change-feeds/tenant-inventory.ps1 b/examples/change-feeds/tenant-inventory.ps1 new file mode 100644 index 0000000..7c6925d --- /dev/null +++ b/examples/change-feeds/tenant-inventory.ps1 @@ -0,0 +1,123 @@ +# Everything the tenant has: every root collection, each to its own JSONL. +# +# Graph has no /* - but $metadata is the machine-readable list of every root +# collection the API serves (41 of them on v1.0 today), and it needs no token. +# So "grab everything" is: ask the schema for the collections, export each one, +# and treat refusals as answers rather than failures. +# +# Re-running resumes. A set with a pending checkpoint continues mid-enumeration, +# a set that already finished is skipped, so a killed overnight run picks up +# where it died instead of starting over. Delete the output directory to redo. +# +# This is the snapshot half of a pair: run it once, then tenant-change-feed.ps1 +# appends what changes afterwards. Snapshot plus feed is the tenant as of now, +# without ever enumerating it a second time. +# +# Requirements: whatever read scopes you have. Directory.Read.All covers most of +# the directory tier; each extra scope (Sites.Read.All, Policy.Read.All, ...) +# unlocks more sets. A denied set is reported, not fatal. + +param( + [string]$OutputDirectory = './tenant-inventory', + # directoryObjects is the union of users, groups, service principals, and + # devices - exporting it re-transfers what other sets already cover. + [string[]]$Skip = @('directoryObjects') +) + +Import-Module Mgx + +New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null + +# The schema is the wildcard: every EntitySet is a root collection. +$csdl = Invoke-RestMethod 'https://graph.microsoft.com/v1.0/$metadata' +$sets = @($csdl.Edmx.DataServices.Schema.EntityContainer.EntitySet.Name | + Where-Object { $_ -notin $Skip } | Sort-Object) + +Write-Host "$($sets.Count) root collections declared by `$metadata" + +$results = foreach ($set in $sets) { + $outFile = Join-Path $OutputDirectory "$set.jsonl" + $checkpoint = Join-Path $OutputDirectory "$set.checkpoint" + + # Finished on a previous run: output present, no checkpoint pending. + if ((Test-Path $outFile) -and -not (Test-Path $checkpoint)) { + [pscustomobject]@{ Set = $set; Items = $null; Outcome = 'done (previous run)' } + continue + } + + try { + # Empty sets are normal here, so the 0-item warning (which suspects a + # single-entity URI) is suppressed - it does not apply to this workload. + $r = Export-MgxCollection "/$set" -All ` + -OutputFile $outFile -CheckpointPath $checkpoint ` + -WarningAction SilentlyContinue -ErrorAction Stop + [pscustomobject]@{ Set = $set; Items = $r.ItemCount; Outcome = 'exported' } + } + catch { + $msg = ($_.Exception.Message -split "`n")[0] + $outcome = switch -Regex ($msg) { + 'Authorization_RequestDenied|Forbidden' { 'denied - scope not granted' } + 'BadRequest|NotImplemented|not support' { 'not enumerable at the root' } + default { $msg } + } + [pscustomobject]@{ Set = $set; Items = $null; Outcome = $outcome } + } +} + +$results | Format-Table Set, Items, Outcome -AutoSize + +$exported = @($results | Where-Object Outcome -eq 'exported') +$totalItems = ($exported.Items | Measure-Object -Sum).Sum +$ru = (Get-MgxTelemetry).ResourceUnitsConsumed +Write-Host ("{0} of {1} sets exported, {2:N0} objects, {3:N0} resource units" -f ` + $exported.Count, $sets.Count, $totalItems, $ru) + +# Reading the outcomes: an exported set with 0 items is an answer - the +# collection exists and is empty. Only a denial says you cannot know. And the +# refusals are by design, not gaps: chats needs user context, shares needs +# sharing tokens, drives is enumerated per site or user, not tenant-wide. +# +# Properties: each object carries its type's DEFAULT property set. Deliberate - +# naming every declared property makes queries fragile (signInActivity, for +# one, fails the whole request without the license to read it). The full +# per-type property list is in the same $metadata if a set is worth the tuning. +# +# The other half of "everything" is the 31 singletons ($csdl ... .Singleton): +# auditLogs, deviceManagement, reports, security. Those are not collections - +# the interesting data hangs off them as navigation properties, each with its +# own scope, and that is a crawler rather than a loop. + +<# +Expected output: + +40 root collections declared by $metadata + +Set Items Outcome +--- ----- ------- +agreementAcceptances 0 exported +agreements 0 exported +applications 9 exported +applicationTemplates 1743 exported +appRoleAssignments not enumerable at the root +certificateBasedAuthConfiguration 0 exported +chats not enumerable at the root +connections 0 exported +contacts 0 exported +contracts 0 exported +devices 5 exported +directoryRoles 14 exported +domains 2 exported +groups 12 exported +oauth2PermissionGrants 19 exported +organization 1 exported +places 0 exported +servicePrincipals 47 exported +shares not enumerable at the root +sites denied - scope not granted +subscribedSkus 3 exported +teams 12 exported +users 28 exported +... + +34 of 40 sets exported, 1,934 objects, 78 resource units +#> diff --git a/examples/drive-content/download-user-onedrive.ps1 b/examples/drive-content/download-user-onedrive.ps1 new file mode 100644 index 0000000..95eb191 --- /dev/null +++ b/examples/drive-content/download-user-onedrive.ps1 @@ -0,0 +1,97 @@ +# Download a user's whole OneDrive to a local folder, preserving the tree. +# +# The enumeration is a delta call, which is the cheapest way to walk a drive: +# one paged sweep instead of a recursive /children crawl per folder. Each item +# carries parentReference.path, which is what rebuilds the directory structure +# locally. +# +# Downloads go through -OutFile, so bytes stream to disk and never sit in the +# pipeline. Where Graph gave the item a @microsoft.graph.downloadUrl, piping the +# item uses that pre-authenticated URL directly: no Graph round trip for the +# bytes, no resource units, and no bearer token sent to the download host. +# +# Re-running skips files whose local size already matches, so an interrupted run +# continues rather than starting over. +# +# Requirements: Connect-MgGraph -Scopes "Files.Read.All" (delegated: Files.Read +# for your own drive). Reading another user's drive needs the +# application permission Files.Read.All. + +param( + [Parameter(Mandatory)][string]$User, # UPN or object id + [string]$Destination = './onedrive-backup' +) + +Import-Module Mgx + +$root = Join-Path $Destination $User +New-Item -ItemType Directory -Path $root -Force | Out-Null + +$items = Sync-MgxDelta "/users/$User/drive/root/delta" ` + -DeltaPath (Join-Path $root '.delta.json') ` + -CheckpointPath (Join-Path $root '.delta.checkpoint') + +# Folders and deletions drop out here; a removed item arrives as @removed, or as a +# 'deleted' facet if the call asked for -Prefer deltashowremovedasdeleted. +$files = @($items | Where-Object { $_.file -and -not $_.deleted -and -not $_.'@removed' }) +Write-Host "$($files.Count) file(s) to consider under $User's drive" + +$downloaded = 0 +$skipped = 0 +foreach ($item in $files) { + # parentReference.path looks like "/drive/root:/Documents/Reports" - everything + # after the colon is the folder path inside the drive. + $relative = '' + if ($item.parentReference.path -match ':(?

.*)$') { + $relative = $Matches.p.TrimStart('/') + } + $targetDir = if ($relative) { Join-Path $root $relative } else { $root } + New-Item -ItemType Directory -Path $targetDir -Force | Out-Null + $target = Join-Path $targetDir $item.name + + # Size is the cheap check. For a real integrity check compare against + # $item.file.hashes.quickXorHash - Graph reports it in base64 where rclone + # and most other tools print hex, so convert before comparing: + # [System.Convert]::ToHexString([System.Convert]::FromBase64String($h)).ToLowerInvariant() + if ((Test-Path $target) -and (Get-Item $target).Length -eq $item.size) { + $skipped++ + continue + } + + try { + if ($item['@microsoft.graph.downloadUrl']) { + $item | Get-MgxContent -OutFile $target -ErrorAction Stop + } + else { + # No downloadUrl on this item (delta does not always include one) - + # fetch through Graph instead. + Get-MgxContent "/users/$User/drive/items/$($item.id)/content" ` + -OutFile $target -ErrorAction Stop + } + $downloaded++ + } + catch { + Write-Warning "$($item.name): $($_.Exception.Message -replace '\s+', ' ')" + } +} + +$t = Get-MgxTelemetry +Write-Host ("downloaded {0}, already current {1}, {2:N1} MB transferred" -f ` + $downloaded, $skipped, ($t.ContentBytesDownloaded / 1MB)) + +# The delta token is kept next to the files, so the next run enumerates only what +# changed and downloads only that. To back up every mailbox-sized drive in the +# tenant, loop over /users and call this once per user. + +<# +Expected output: + +11 file(s) to consider under adelev@contoso.com's drive +WARNING: Locked-Budget.xlsx: The resource you are attempting to access is locked +downloaded 10, already current 0, 4.7 MB transferred + +onedrive-backup/adelev@contoso.com/ + Documents/Q4-Revenue-Report.pdf + Documents/Reports/Meeting-Notes.txt + Team-Photo.png +#> diff --git a/examples/drive-content/drive-content-triage.ps1 b/examples/drive-content/drive-content-triage.ps1 index 077006d..a529d6b 100644 --- a/examples/drive-content/drive-content-triage.ps1 +++ b/examples/drive-content/drive-content-triage.ps1 @@ -1,16 +1,8 @@ -# Triage a drive by reading 4 KB of each file, then download only what survives triage. +# Triage a drive by reading 4 KB of each file, then download only what survives. # -# -# Two properties of Get-MgxContent make this cheap and safe: -# -# * A ranged read is a Range request. -First 4096 transfers 4 KB, not 4 KB of a 2 GB download -# you abandon. Some endpoints ignore Range and answer 200 with the whole body (profile -# photos do); mgx detects that and truncates client-side, so you never receive more than you -# asked for either way. -# * Piping a DriveItem uses its own @microsoft.graph.downloadUrl - a pre-authenticated URL - -# so there is no Graph round trip at all for the bytes. No resource units, and no bearer -# token sent to the download host. That URL is itself a credential for the file, which is -# why mgx redacts it from -Debug output. +# -First 4096 is a Range request: it transfers 4 KB, not 4 KB of a 2 GB download +# you abandon. Enough to read magic bytes and decide, on a whole drive, for the +# cost of downloading one mid-sized file. # # Requirements: Connect-MgGraph -Scopes "Files.Read.All" # Read-only against Graph. Writes only to the local -OutputDirectory. diff --git a/examples/delta-and-sync/drive-delta-resumable.ps1 b/examples/drive-content/drive-delta-resumable.ps1 similarity index 100% rename from examples/delta-and-sync/drive-delta-resumable.ps1 rename to examples/drive-content/drive-delta-resumable.ps1 diff --git a/examples/delta-and-sync/delta-sync.ps1 b/examples/getting-started/delta-users-sync.ps1 similarity index 100% rename from examples/delta-and-sync/delta-sync.ps1 rename to examples/getting-started/delta-users-sync.ps1 diff --git a/examples/getting-started/export-users-to-csv.ps1 b/examples/getting-started/export-users-to-csv.ps1 new file mode 100644 index 0000000..2c06617 --- /dev/null +++ b/examples/getting-started/export-users-to-csv.ps1 @@ -0,0 +1,44 @@ +# Export all users to a CSV file. +# +# Export-MgxCollection writes JSONL because Graph objects are nested and JSONL +# survives that without losing anything. CSV is flat, so it needs a projection: +# name the columns you want and let Export-Csv stream them to disk as pages +# arrive. Memory stays flat - nothing accumulates in a variable first. +# +# Nested values (assignedLicenses, businessPhones, manager) need flattening by +# hand, as below, or they land in the file as type names. +# +# Requirements: Connect-MgGraph -Scopes "User.Read.All" + +Import-Module Mgx + +$outputFile = "./users.csv" + +$count = 0 +Invoke-MgxRequest /users -All ` + -Property id,displayName,mail,department,jobTitle,accountEnabled,businessPhones | + ForEach-Object { $count++; $_ } | + Select-Object id, displayName, mail, department, jobTitle, accountEnabled, + @{ Name = 'businessPhones'; Expression = { $_.businessPhones -join ';' } } | + Export-Csv -Path $outputFile -NoTypeInformation -Encoding utf8 + +Write-Host "Exported $count users to $outputFile" + +# Excel opens UTF-8 CSV correctly only when it starts with a BOM. If the file is +# for Excel rather than for another script, use -Encoding utf8BOM instead. +# +# For a resumable export over a large tenant, write JSONL with +# Export-MgxCollection -CheckpointPath (see export-users-to-jsonl.ps1) and +# convert afterwards: +# Get-Content ./users.jsonl | ConvertFrom-Json | Export-Csv ./users.csv -NoTypeInformation + +<# +Expected output: + +Exported 28 users to ./users.csv + +users.csv (first 3 lines): +"id","displayName","mail","department","jobTitle","accountEnabled","businessPhones" +"05c76d47-f457-4be9-bf0a-a75b57af73c1","Adele Vance","adelev@contoso.com","Retail","Retail Manager","True","+1 425 555 0109" +"b92d487c-5d51-460b-963a-c395ddc747ba","Alex Wilber","alexw@contoso.com","Marketing","Marketing Assistant","True","+1 858 555 0110" +#> diff --git a/examples/reporting-and-audit/download-groups-and-members.ps1 b/examples/reporting-and-audit/download-groups-and-members.ps1 new file mode 100644 index 0000000..fbda55b --- /dev/null +++ b/examples/reporting-and-audit/download-groups-and-members.ps1 @@ -0,0 +1,153 @@ +# Download every group and ALL its members - nesting resolved - with a +# progress bar. +# +# Three JSONL files: +# groups.jsonl every group +# members-direct.jsonl every direct membership; rows whose @odata.type is a +# group are the nesting edges +# members.jsonl every membership with nesting resolved +# +# One fan-out, not two, and no transitiveMembers. Asking Graph to flatten the +# nesting per group re-transfers every deeply nested member once per ancestor +# group; the direct download already contains the whole graph, so the closure +# is computed locally instead - each membership crosses the wire exactly once, +# at 2 RU per request instead of 3. +# +# Requirements: Connect-MgGraph -Scopes "Group.Read.All", "User.Read.All" + +param( + [string]$OutputDirectory = './groups-export', + # Fan-out width. The default of 5 leaves a big tenant latency-bound; at + # ~2 RU per call, 16 concurrent stays far below the 800 RU/s budget floor + # (resource-unit-budgeting.ps1 measures your tenant's own numbers). + [int]$Concurrency = 16, + [int]$UpdateEvery = 100 +) + +Import-Module Mgx + +New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null +$directFile = Join-Path $OutputDirectory 'members-direct.jsonl' +$resolvedFile = Join-Path $OutputDirectory 'members.jsonl' + +# groups.jsonl is a single enumeration - exactly what Export-MgxCollection is +# for: request loop, JSONL writing, and resume in one cmdlet. The fan-out +# below cannot use it, because Export-MgxCollection drives its own requests +# from a URI rather than accepting piped objects; its rows arrive through the +# pipeline, so those files are written by hand. +$groupsFile = Join-Path $OutputDirectory 'groups.jsonl' +$null = Export-MgxCollection /groups -All ` + -Property id,displayName,description,mail,groupTypes,securityEnabled ` + -OutputFile $groupsFile +$groups = @(Get-Content $groupsFile | ConvertFrom-Json) +Write-Host "$($groups.Count) groups -> groups.jsonl" + +# --- pass 1: one fan-out, every direct membership straight to disk --- +$sw = [System.Diagnostics.Stopwatch]::StartNew() +$started = @{} +$parents = @{} # child group id -> list of parent group ids (built as rows stream by) +$rows = 0 +# One StreamWriter held open for the run - Add-Content would reopen the file +# once per row. +$writer = [System.IO.StreamWriter]::new($directFile, $false, [System.Text.UTF8Encoding]::new($false)) +try { + $groups | Invoke-MgxRequest '/groups/{id}/members?$select=id,displayName' ` + -All -Concurrency $Concurrency -SkipNotFound -SkipForbidden | + ForEach-Object { + $_['groupId'] = $_['_MgxSourceId'] + $_.Remove('_MgxSourceId') + $writer.WriteLine(($_ | ConvertTo-Json -Compress -Depth 10)) + + if ($_['@odata.type'] -eq '#microsoft.graph.group') { + if (-not $parents.ContainsKey($_['id'])) { + $parents[$_['id']] = [System.Collections.Generic.List[string]]::new() + } + $parents[$_['id']].Add($_['groupId']) + } + + $rows++ + $started[$_['groupId']] = $true + if ($rows % $UpdateEvery -eq 0) { + $done = $started.Count / $groups.Count + $t = Get-MgxTelemetry + Write-Progress -Activity 'Downloading direct memberships' ` + -Status ('{0} of {1} groups | {2:N0} rows | {3:N0}/s | {4} requests, {5:N0} RU' -f ` + $started.Count, $groups.Count, $rows, + ($rows / $sw.Elapsed.TotalSeconds), $t.Requests, $t.ResourceUnitsConsumed) ` + -PercentComplete (100 * $done) ` + -SecondsRemaining ([int]($sw.Elapsed.TotalSeconds * (1 - $done) / [Math]::Max($done, 0.01))) + } + } +} +finally { + $writer.Dispose() +} +Write-Progress -Activity 'Downloading direct memberships' -Completed +Write-Host ('{0:N0} direct membership row(s) in {1:mm\:ss} -> members-direct.jsonl' -f $rows, $sw.Elapsed) + +# --- pass 2: resolve the nesting locally - disk to disk, no requests --- +$ancestorCache = @{} +function Get-Ancestors { + # Every group transitively containing $GroupId. The visited set makes a + # membership cycle a non-event rather than a hang. + param([string]$GroupId) + if ($ancestorCache.ContainsKey($GroupId)) { return ,$ancestorCache[$GroupId] } + $seen = [System.Collections.Generic.HashSet[string]]::new() + $queue = [System.Collections.Generic.Queue[string]]::new() + if ($parents.ContainsKey($GroupId)) { foreach ($p in $parents[$GroupId]) { $queue.Enqueue($p) } } + while ($queue.Count -gt 0) { + $g = $queue.Dequeue() + if ($seen.Add($g) -and $parents.ContainsKey($g)) { + foreach ($p in $parents[$g]) { $queue.Enqueue($p) } + } + } + $ancestorCache[$GroupId] = @($seen) + return ,$ancestorCache[$GroupId] +} + +$resolved = 0 +$writer = [System.IO.StreamWriter]::new($resolvedFile, $false, [System.Text.UTF8Encoding]::new($false)) +try { + foreach ($line in [System.IO.File]::ReadLines($directFile)) { + $writer.WriteLine($line) # a member of a group is a member of that group + $resolved++ + $row = $line | ConvertFrom-Json + foreach ($ancestor in (Get-Ancestors $row.groupId)) { + $row.groupId = $ancestor + $writer.WriteLine(($row | ConvertTo-Json -Compress -Depth 10)) + $resolved++ + } + } +} +finally { + $writer.Dispose() +} +Write-Host ('{0:N0} resolved membership row(s) -> members.jsonl' -f $resolved) + +$t = Get-MgxTelemetry +Write-Host ('done in {0:mm\:ss} - {1} requests, {2:N0} resource units' -f ` + $sw.Elapsed, $t.Requests, $t.ResourceUnitsConsumed) + +# Reading it back - everyone in one group, including via nesting. Multiple +# nesting paths collapse to one row, but a member placed DIRECTLY in several +# groups under the same ancestor appears once per placement - dedup on id: +# Get-Content ./groups-export/members.jsonl | ConvertFrom-Json | +# Where-Object { $_.groupId -eq $id -and $_.'@odata.type' -eq '#microsoft.graph.user' } | +# Sort-Object id -Unique +# +# Notes: +# * The nesting edges are the group-typed rows of members-direct.jsonl: +# ... | ConvertFrom-Json | Where-Object '@odata.type' -eq '#microsoft.graph.group' +# * Empty groups contribute no rows, so the bar can top out just short of +# 100 before it clears (see progress-for-long-runs.ps1). +# * A killed run restarts pass 1: checkpoints belong to single enumerations, +# not fan-outs. Pass 2 is local and costs nothing to redo. + +<# +Expected output (final state; the bar redraws in place while running): + +12 groups -> groups.jsonl +1,214 direct membership row(s) in 00:19 -> members-direct.jsonl +1,532 resolved membership row(s) -> members.jsonl +done in 00:20 - 14 requests, 27 RU +#> diff --git a/examples/reporting-and-audit/group-membership-report.ps1 b/examples/reporting-and-audit/group-membership-report.ps1 new file mode 100644 index 0000000..5a3571a --- /dev/null +++ b/examples/reporting-and-audit/group-membership-report.ps1 @@ -0,0 +1,84 @@ +# Membership of every group: direct, and effective through nesting. +# +# A group's real membership is not its member list - groups contain groups. +# /transitiveMembers flattens the nesting server-side, so there is no recursion +# to write and no cycle handling to get wrong. The groups where effective and +# direct counts differ are the ones nesting hides members in. +# +# The memory model is half the point. Tenant-wide membership is too big to +# hold, so no member list is ever kept: the fan-out streams members out as +# pages arrive, each tagged with _MgxSourceId (the group it came from), and +# the loop reduces them to a count on the spot. RAM holds one row per group, +# not the membership of the tenant. +# +# Requirements: Connect-MgGraph -Scopes "Group.Read.All", "User.Read.All" + +Import-Module Mgx + +# One row per group is all that is ever kept. +$groups = @(Invoke-MgxRequest /groups -All -Property id,displayName) +$stats = @{} +foreach ($g in $groups) { + $stats[$g.id] = [pscustomobject]@{ + Group = $g.displayName + Direct = 0 + Effective = 0 + Nested = [System.Collections.Generic.List[string]]::new() + } +} +Write-Host "$($groups.Count) groups" + +# The type casts filter server-side: /members/microsoft.graph.user returns only +# user members, so nothing arrives that has to be discarded, and $select=id +# keeps each streamed object to the two fields the tally needs. +$groups | Invoke-MgxRequest '/groups/{id}/members/microsoft.graph.user?$select=id' ` + -All -SkipNotFound -SkipForbidden | + ForEach-Object { $stats[$_._MgxSourceId].Direct++ } + +$groups | Invoke-MgxRequest '/groups/{id}/members/microsoft.graph.group?$select=id,displayName' ` + -All -SkipNotFound -SkipForbidden | + ForEach-Object { $stats[$_._MgxSourceId].Nested.Add($_.displayName) } + +$groups | Invoke-MgxRequest '/groups/{id}/transitiveMembers/microsoft.graph.user?$select=id' ` + -All -SkipNotFound -SkipForbidden | + ForEach-Object { $stats[$_._MgxSourceId].Effective++ } + +$report = $stats.Values | + Select-Object Group, Direct, Effective, @{ n = 'Nested'; e = { $_.Nested -join ', ' } } | + Sort-Object { $_.Effective - $_.Direct }, Group -Descending + +$report | Format-Table -AutoSize +$hidden = @($report | Where-Object { $_.Effective -gt $_.Direct }) +Write-Host "$($hidden.Count) group(s) have members through nesting alone" + +# Notes: +# * Expand-MgxRelation would attach the member arrays to each group instead - +# right when the enriched objects ARE the deliverable (user-directory-report +# does exactly that), wrong here, where they would only be counted and +# thrown away. Expand holds all input and all fetched relations until its +# fan-out completes; this report only ever holds integers. +# * Nesting is a security-group phenomenon - Microsoft 365 groups do not +# accept groups as members, so their two counts always match. +# * transitiveMembers bills more resource units per call than members; +# resource-unit-budgeting.ps1 measures both shapes. +# * The inverse question - every group a user is effectively in, nesting +# resolved - is one relation: /users/{id}/transitiveMemberOf. +# * On a tenant where this runs for minutes, progress-for-long-runs.ps1 is +# the same fan-out with a live status line. + +<# +Expected output: + +12 groups + +Group Direct Effective Nested +----- ------ --------- ------ +Engineering All Hands 3 11 Engineering Leads, Engineering ICs +Retail 2 9 Retail East, Retail West +U.S. Sales 5 5 +Mark 8 Project Team 8 8 +sg-IT-Helpdesk 4 4 +... + +2 group(s) have members through nesting alone +#> diff --git a/examples/reporting-and-audit/inactive-users-report.ps1 b/examples/reporting-and-audit/inactive-users-report.ps1 new file mode 100644 index 0000000..bca159f --- /dev/null +++ b/examples/reporting-and-audit/inactive-users-report.ps1 @@ -0,0 +1,67 @@ +# Users who have not signed in for 90 days. +# +# accountEnabled only tells you the account is allowed to sign in, not that +# anyone does. signInActivity is the property that answers the question, and it +# is filterable server-side, so the tenant is never enumerated in full. +# +# Two caveats worth knowing before the result is trusted: +# * signInActivity needs a Microsoft Entra ID P1 license and the +# AuditLog.Read.All scope, on top of User.Read.All. +# * An account that has NEVER signed in has no signInActivity at all, so it +# does not match this filter. Those are the second query below. +# +# Requirements: Connect-MgGraph -Scopes "User.Read.All", "AuditLog.Read.All" + +Import-Module Mgx + +$cutoff = (Get-Date).AddDays(-90).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') + +$inactive = Invoke-MgxRequest /users -All ` + -Filter "signInActivity/lastSignInDateTime le $cutoff" ` + -Property id,displayName,userPrincipalName,accountEnabled,signInActivity + +Write-Host "Not seen since $($cutoff.Substring(0,10)): $($inactive.Count)" +$inactive | + Select-Object displayName, userPrincipalName, accountEnabled, + @{ n = 'LastSignIn'; e = { ([datetime]$_.signInActivity.lastSignInDateTime).ToString('yyyy-MM-dd') } } | + Sort-Object LastSignIn | + Format-Table -AutoSize + +# Never signed in at all - created and forgotten, which is the more interesting +# half of the report on most tenants. These have no signInActivity to filter on, +# so this one sweeps and checks locally: one enumeration, same behavior on every +# tenant, and no dependency on how Graph handles a null comparison. +$never = @(Invoke-MgxRequest /users -All ` + -Property id,displayName,userPrincipalName,createdDateTime,accountEnabled,signInActivity | + Where-Object { -not $_.signInActivity.lastSignInDateTime }) + +Write-Host "`nNever signed in: $($never.Count)" +$never | + Select-Object displayName, userPrincipalName, accountEnabled, + @{ n = 'Created'; e = { ([datetime]$_.createdDateTime).ToString('yyyy-MM-dd') } } | + Sort-Object Created | + Format-Table -AutoSize + +# license-report.ps1 answers the follow-up: which of these are holding a seat. +# The same shape reports stale devices: /devices, filtered on +# approximateLastSignInDateTime le $cutoff. + +<# +Expected output: + +Not seen since 2026-05-26: 4 + +displayName userPrincipalName accountEnabled LastSignIn +----------- ----------------- -------------- ---------- +Brian Johnson brianj@contoso.com False 2025-11-02 +Isaiah Langer isaiahl@contoso.com False 2026-01-14 +Pradeep Gupta pradeepg@contoso.com True 2026-02-28 +Raul Razo raul.razo@woodgrovebank.com True 2026-03-19 + +Never signed in: 2 + +displayName userPrincipalName accountEnabled Created +----------- ----------------- -------------- ------- +Sample Vendor vendor@contoso.com True 2025-08-13 +Bianca Pisani bianca.pisani@fabrikam.com True 2025-07-03 +#> diff --git a/examples/reporting-and-audit/license-report.ps1 b/examples/reporting-and-audit/license-report.ps1 new file mode 100644 index 0000000..ab062b1 --- /dev/null +++ b/examples/reporting-and-audit/license-report.ps1 @@ -0,0 +1,80 @@ +# Where the licenses went: seats per SKU, and who is holding them. +# +# /subscribedSkus gives what the tenant bought and how much is consumed; the +# user sweep gives the names behind that number. Both are cheap - assignedLicenses +# comes back with the user object, so no per-user call is needed. +# +# Requirements: Connect-MgGraph -Scopes "Organization.Read.All", "User.Read.All" + +Import-Module Mgx + +$skus = Invoke-MgxRequest /subscribedSkus + +Write-Host "`n=== Seats ===" -ForegroundColor Cyan +$skus | + ForEach-Object { + [pscustomobject]@{ + Sku = $_.skuPartNumber + Purchased = $_.prepaidUnits.enabled + Assigned = $_.consumedUnits + Free = $_.prepaidUnits.enabled - $_.consumedUnits + Warning = $_.prepaidUnits.warning # seats in the 30-day grace period + } + } | + Sort-Object Free | + Format-Table -AutoSize + +# skuId is what a user object carries; skuPartNumber is what a human reads. +$nameById = @{} +foreach ($sku in $skus) { $nameById[[string]$sku.skuId] = $sku.skuPartNumber } + +$users = Invoke-MgxRequest /users -All -Property id,displayName,userPrincipalName,assignedLicenses,accountEnabled + +Write-Host "=== Assignments ===" -ForegroundColor Cyan +$assignments = foreach ($user in $users) { + foreach ($license in $user.assignedLicenses) { + [pscustomobject]@{ + Sku = $nameById[[string]$license.skuId] ?? $license.skuId + User = $user.userPrincipalName + Enabled = $user.accountEnabled + } + } +} + +$assignments | Group-Object Sku | Sort-Object Count -Descending | + Select-Object @{ n = 'Sku'; e = { $_.Name } }, Count | Format-Table -AutoSize + +# The number worth acting on: seats held by accounts that cannot sign in. +$wasted = @($assignments | Where-Object { -not $_.Enabled }) +if ($wasted) { + Write-Host "$($wasted.Count) seat(s) assigned to disabled accounts:" -ForegroundColor Yellow + $wasted | Sort-Object Sku, User | Format-Table Sku, User -AutoSize +} + +<# +Expected output: + +=== Seats === + +Sku Purchased Assigned Free Warning +--- --------- -------- ---- ------- +ENTERPRISEPACK 25 22 3 0 +EMSPREMIUM 10 4 6 0 +POWER_BI_STANDARD 100 7 93 0 + +=== Assignments === + +Sku Count +--- ----- +ENTERPRISEPACK 22 +POWER_BI_STANDARD 7 +EMSPREMIUM 4 + +3 seat(s) assigned to disabled accounts: + +Sku User +--- ---- +EMSPREMIUM brianj@contoso.com +ENTERPRISEPACK brianj@contoso.com +ENTERPRISEPACK isaiahl@contoso.com +#> diff --git a/examples/reporting-and-audit/user-directory-report.ps1 b/examples/reporting-and-audit/user-directory-report.ps1 new file mode 100644 index 0000000..7a3a936 --- /dev/null +++ b/examples/reporting-and-audit/user-directory-report.ps1 @@ -0,0 +1,45 @@ +# A user directory: every enabled user with manager and licenses attached. +# +# Neither relation comes back with the user object, so each needs a call per +# user. Stacked Expand-MgxRelation stages fetch each relation concurrently +# across users, and the second stage receives objects already carrying the +# first - two serial foreach loops collapse into one pass. +# +# Requirements: Connect-MgGraph -Scopes "User.Read.All" + +Import-Module Mgx + +$directory = Invoke-MgxRequest /users -All ` + -Filter 'accountEnabled eq true' ` + -Property id,displayName,department,jobTitle | + Expand-MgxRelation '/users/{id}/manager' -As Manager -Flatten -SkipNotFound | + Expand-MgxRelation '/users/{id}/licenseDetails' -As Licenses -SkipNotFound + +$directory | + Select-Object displayName, department, jobTitle, + @{ n = 'Manager'; e = { $_.Manager.displayName } }, + @{ n = 'Licenses'; e = { (@($_.Licenses.skuPartNumber) | Sort-Object) -join ', ' } } | + Sort-Object department, displayName | + Format-Table -AutoSize + +# The skipped-404 warning is expected: users at the top of the org have no +# manager. Swap Format-Table for Export-Csv to hand the file to whoever asked. + +<# +Expected output: + +WARNING: Skipped 4 entities due to 404 (Not Found) responses. + +displayName department jobTitle Manager Licenses +----------- ---------- -------- ------- -------- +Brian Johnson Engineering Senior Engineer Lee Gu SPE_E3 +Lee Gu Engineering Director SPE_E5 +Christie Cline Human Resources Benefits Specialist Nestor Wilke SPE_E5 +Allan Deyoung IT Corporate Security Officer Nestor Wilke EMSPREMIUM, SPE_E5 +Delia Dennis IT Support Engineer Allan Deyoung EMSPREMIUM, SPE_E5 +Alex Wilber Marketing Marketing Assistant Megan Bowen SPE_E5 +Debra Berger Operations Administrative Assistant Nestor Wilke SPE_E5 +Adele Vance Retail Retail Manager Nestor Wilke SPE_E5 +Cameron White Retail Store Associate Adele Vance SPE_E3 +... +#> diff --git a/examples/resilience-and-telemetry/progress-for-long-runs.ps1 b/examples/resilience-and-telemetry/progress-for-long-runs.ps1 new file mode 100644 index 0000000..6bff533 --- /dev/null +++ b/examples/resilience-and-telemetry/progress-for-long-runs.ps1 @@ -0,0 +1,81 @@ +# A live status line for a long run: how far along, how fast, what it costs. +# +# Streaming is what makes this possible at all. Fan-out results flow through +# the pipeline while the requests are still running, so a ForEach-Object in +# the middle can tally them and draw progress - percent, rate, ETA, and the +# resource units the run has consumed so far. A cmdlet that buffers until +# everything is done has nothing to hang a progress bar on. +# +# The demonstration workload is the transitive-members fan-out from +# group-membership-report.ps1 - the kind of run that takes long enough on a +# real tenant to make you wonder whether it is alive. +# +# Requirements: Connect-MgGraph -Scopes "Group.Read.All", "User.Read.All" + +param( + # Write-Progress is not free; redrawing every item can dominate a fast + # stream. Every 100th is smooth and costs nothing measurable. + [int]$UpdateEvery = 100 +) + +Import-Module Mgx + +$groups = @(Invoke-MgxRequest /groups -All -Property id,displayName) +Write-Host "$($groups.Count) groups to enumerate" + +$sw = [System.Diagnostics.Stopwatch]::StartNew() +$started = @{} # groups whose first member has arrived +$counts = @{} +$members = 0 + +$groups | Invoke-MgxRequest '/groups/{id}/transitiveMembers/microsoft.graph.user?$select=id' ` + -All -SkipNotFound -SkipForbidden | + ForEach-Object { + $members++ + $counts[$_._MgxSourceId] = ($counts[$_._MgxSourceId] ?? 0) + 1 + $started[$_._MgxSourceId] = $true + + if ($members % $UpdateEvery -eq 0) { + # "started", not "finished": a group joins the tally when its first + # member arrives. With the default concurrency of 5, at most 5 + # groups are in flight, so the two differ by at most 5. + $done = $started.Count / $groups.Count + $t = Get-MgxTelemetry + Write-Progress -Activity 'Transitive members, all groups' ` + -Status ('{0} of {1} groups | {2:N0} members | {3:N0}/s | {4} requests, {5:N0} RU' -f ` + $started.Count, $groups.Count, $members, + ($members / $sw.Elapsed.TotalSeconds), $t.Requests, $t.ResourceUnitsConsumed) ` + -PercentComplete (100 * $done) ` + -SecondsRemaining ([int]($sw.Elapsed.TotalSeconds * (1 - $done) / [Math]::Max($done, 0.01))) + } + } + +Write-Progress -Activity 'Transitive members, all groups' -Completed +Write-Host ('{0:N0} members across {1} groups in {2:mm\:ss}' -f $members, $groups.Count, $sw.Elapsed) + +Write-Host "`nLargest groups:" +$counts.GetEnumerator() | Sort-Object Value -Descending | Select-Object -First 5 | + ForEach-Object { + $g = $groups | Where-Object id -eq $_.Key + '{0,8:N0} {1}' -f $_.Value, $g.displayName + } + +# Honest limits of the numbers on the bar: +# * Empty groups never emit a member, so they never join the "started" count - +# the percentage can top out just short of 100 before the bar clears. +# * The ETA assumes groups are similar in size. They are not; one giant group +# at the end makes it optimistic. Treat it as a trend, not a promise. + +<# +Expected output (final state; the bar redraws in place while running): + +12 groups to enumerate +1,178 members across 12 groups in 00:41 + +Largest groups: + 412 Engineering All Hands + 306 All Employees + 144 Retail + 98 U.S. Sales + 67 sg-IT-Helpdesk +#> diff --git a/examples/resilience-and-telemetry/resilience-status.ps1 b/examples/resilience-and-telemetry/resilience-status.ps1 index 1b1b235..6a474ef 100644 --- a/examples/resilience-and-telemetry/resilience-status.ps1 +++ b/examples/resilience-and-telemetry/resilience-status.ps1 @@ -1,7 +1,9 @@ -# Check and toggle SDK resilience injection. +# Add retry, circuit breaking, and rate limiting to existing Microsoft.Graph +# scripts without changing a line of their code - then check it and take it off. # -# Get-MgxResilience reports whether the Polly pipeline is currently -# injected into the Microsoft.Graph SDK HTTP transport. +# Enable-MgxResilience injects a Polly pipeline into the SDK's HTTP transport, so +# unmodified SDK cmdlets (Get-MgUser, Get-MgGroup, ...) retry on 429/5xx and honor +# Retry-After. Get-MgxResilience reports whether it is currently injected; # Disable-MgxResilience removes it and restores the original SDK behavior. # # Requirements: Connect-MgGraph -Scopes "User.Read.All" diff --git a/examples/resilience-and-telemetry/resource-unit-budgeting.ps1 b/examples/resilience-and-telemetry/resource-unit-budgeting.ps1 index ac1340a..01dc8ff 100644 --- a/examples/resilience-and-telemetry/resource-unit-budgeting.ps1 +++ b/examples/resilience-and-telemetry/resource-unit-budgeting.ps1 @@ -1,31 +1,28 @@ # Resource units: what a query actually costs, and how to size a fan-out before running it. # # Graph throttles directory workloads on a resource-unit budget, not on request count or -# bandwidth. Two queries that look alike can differ 5x in cost, and the difference is invisible -# unless you read x-ms-resource-unit. Mgx accumulates it for you: Get-MgxTelemetry reports -# ResourceUnitsConsumed for the session. -# -# This script measures the cost of the query shapes you are choosing between, then uses the -# measured cost - not a guess - to size a tenant-scale fan-out. +# bandwidth. Two queries that look alike can differ several times over in cost, and the +# difference is invisible unless you read x-ms-resource-unit. Mgx accumulates it for you: +# Get-MgxTelemetry reports ResourceUnitsConsumed for the session. # # Requirements: Connect-MgGraph -Scopes "Group.Read.All","User.Read.All" Import-Module M365DSC.mgx function Measure-QueryCost { - <# - Cost of one call, measured rather than assumed. The documented RU table is directionally - right but not exact: transitiveMembers is published at 5 RU and bills 4 here, and the - documented "-1 RU for $select" discount does apply on top of that. - #> + # Cost of one call, measured rather than assumed. The documented RU table is directionally + # right but not exact - transitiveMembers is published at 5 RU and bills 4 here. param([string]$Label, [string]$Uri) $before = (Get-MgxTelemetry).ResourceUnitsConsumed try { $null = Invoke-MgxRequest $Uri -ErrorAction Stop } catch { Write-Warning "$Label : $($_.Exception.Message)"; return } - $cost = (Get-MgxTelemetry).ResourceUnitsConsumed - $before - [pscustomobject]@{ Query = $Label; ResourceUnits = $cost; Uri = $Uri } + [pscustomobject]@{ + Query = $Label + ResourceUnits = (Get-MgxTelemetry).ResourceUnitsConsumed - $before + Uri = $Uri + } } $sample = (Invoke-MgxRequest '/groups?$top=1&$select=id').id @@ -40,8 +37,8 @@ $costs = @( Measure-QueryCost 'single group read' "/groups/$sample" Measure-QueryCost 'users list ($top=25)' "/users?`$top=25" Measure-QueryCost 'users list ($select)' "/users?`$top=25&`$select=id,displayName" -) -$costs | Where-Object { $_ } | Format-Table Query, ResourceUnits -AutoSize +) | Where-Object { $_ } +$costs | Format-Table Query, ResourceUnits -AutoSize # --- size the fan-out from the measured cost --- # @@ -49,7 +46,8 @@ $costs | Where-Object { $_ } | Format-Table Query, ResourceUnits -AutoSize # throttling in one test tenant was appreciably higher (clean at 882 RU/s, 429s from ~1,200 # RU/s), so treat 800 as a conservative floor rather than a hard ceiling - and never as a target. $BudgetRuPerSecond = 800 -$chosen = $costs | Where-Object { $_ -and $_.Query -eq 'transitiveMembers ($select)' } | Select-Object -First 1 +$chosen = $costs | Where-Object { $_.Query -eq 'transitiveMembers ($select)' } | Select-Object -First 1 +$bare = $costs | Where-Object { $_.Query -eq 'transitiveMembers (bare)' } | Select-Object -First 1 if (-not $chosen -or $chosen.ResourceUnits -le 0) { Write-Warning 'No RU header observed - this tenant or endpoint may not be RU-metered. Skipping the projection.' return @@ -65,32 +63,15 @@ Write-Host "`n=== Sizing a per-group fan-out over the whole tenant ===" -Foregro " cost per group : {0} RU ({1})" -f $chosen.ResourceUnits, $chosen.Query " total cost : {0:N0} RU" -f $totalRu " floor at {0} RU/s : {1:N0}s ({2:N1} min) of pure budget consumption" -f $BudgetRuPerSecond, $seconds, ($seconds / 60) -'' -" Dropping `$select would cost {0:N0} RU more ({1:P0} increase)." -f ` - ($groupCount * 1), (1 / $chosen.ResourceUnits) - -# --- what the session actually spent --- -Write-Host "`n=== Session telemetry ===" -ForegroundColor Cyan -$t = Get-MgxTelemetry -" requests : $($t.Requests) ($($t.Succeeded) ok, $($t.Failed) failed)" -" resource units : $($t.ResourceUnitsConsumed)" -" throttle retries : $($t.ThrottleRetries)" -" pacing waits : $($t.AdaptivePacingWaitMs) ms over $($t.AdaptivePacingActivations) activations" -if ($t.Requests -gt 0) { - " average cost : {0:N2} RU per request" -f ($t.ResourceUnitsConsumed / $t.Requests) +if ($bare -and $bare.ResourceUnits -gt $chosen.ResourceUnits) { + $extra = $groupCount * ($bare.ResourceUnits - $chosen.ResourceUnits) + " dropping `$select : {0:N0} RU more ({1:P0} increase)" -f ` + $extra, ($extra / $totalRu) } -Write-Host @" - -Why this matters - * Cost is a property of the query SHAPE, not the object count. Adding `$select reduces it; - `$expand increases it. Measure the shape you are about to run 15,000 times. - * A 403 or 400 costs 0 RU. A fan-out that fails uniformly consumes no budget and triggers no - throttling - it looks fast and healthy while measuring nothing. Check status, not duration. - * x-ms-throttle-limit-percentage is documented as a proximity warning above 0.8 of budget. It - was never observed in testing, even while the tenant was actively returning 429s. Do not - build a control loop that depends on it; rely on 429 + Retry-After, which is reliable. -"@ -ForegroundColor DarkGray +# Cost is a property of the query SHAPE, not the object count: $select reduces it, $expand +# raises it. A 403 or 400 costs 0 RU, so a fan-out that fails uniformly looks fast and healthy +# while measuring nothing - check status, not duration. <# Expected output: @@ -113,22 +94,5 @@ users list ($select) 1 cost per group : 3 RU (transitiveMembers ($select)) total cost : 47,337 RU floor at 800 RU/s : 59s (1.0 min) of pure budget consumption - - Dropping $select would cost 15,779 RU more (33% increase). - -=== Session telemetry === - requests : 9 (9 ok, 0 failed) - resource units : 18 - throttle retries : 0 - pacing waits : 1496 ms over 8 activations - average cost : 2.00 RU per request - -Why this matters - * Cost is a property of the query SHAPE, not the object count. Adding $select reduces it; - $expand increases it. Measure the shape you are about to run 15,000 times. - * A 403 or 400 costs 0 RU. A fan-out that fails uniformly consumes no budget and triggers no - throttling - it looks fast and healthy while measuring nothing. Check status, not duration. - * x-ms-throttle-limit-percentage is documented as a proximity warning above 0.8 of budget. It - was never observed in testing, even while the tenant was actively returning 429s. Do not - build a control loop that depends on it; rely on 429 + Retry-After, which is reliable. + dropping $select : 15,779 RU more (33% increase) #> diff --git a/src/Mgx.Cmdlets/AlcInitializer.cs b/src/Mgx.Cmdlets/AlcInitializer.cs index fb1f5b1..fb01980 100644 --- a/src/Mgx.Cmdlets/AlcInitializer.cs +++ b/src/Mgx.Cmdlets/AlcInitializer.cs @@ -1,6 +1,7 @@ using System.Management.Automation; using System.Reflection; using System.Runtime.Loader; +using Mgx.Engine.Http; namespace Mgx.Cmdlets; @@ -10,6 +11,12 @@ namespace Mgx.Cmdlets; /// msgraph-load-context) to avoid type identity conflicts. Only loads from /// the Dependencies folder for assemblies not found anywhere. /// Pattern adopted from Mge project's ALC coexistence investigation. +/// +/// Dependencies/ is not only mgx's own third-party packages. It also carries assemblies the +/// module never compiles against but a host may not supply - System.IO.Pipelines, which the +/// System.Text.Json of a newer PowerShell needs and net8.0's framework does not include. Those +/// arrive here by the same route: nothing has them loaded, so the fallback below is the only +/// answer, and an empty Dependencies/ is a FileNotFoundException at first use. /// public class AlcInitializer : IModuleAssemblyInitializer, IModuleAssemblyCleanup { @@ -17,9 +24,22 @@ public class AlcInitializer : IModuleAssemblyInitializer, IModuleAssemblyCleanup Path.GetDirectoryName(typeof(AlcInitializer).Assembly.Location)!, "Dependencies"); + // Whether ResolveDependency is subscribed. A process can import the module more than once, + // and += is not idempotent: the second subscription would outlive the -= a removal does and + // go on answering dependency loads out of this module's Dependencies folder for a session + // that no longer has the module. + private static int s_resolverHooked; + public void OnImport() { - AssemblyLoadContext.Default.Resolving += ResolveDependency; + if (Interlocked.Exchange(ref s_resolverHooked, 1) == 0) + AssemblyLoadContext.Default.Resolving += ResolveDependency; + + // Re-arms type-cache invalidation. The hook is attached from a static constructor that + // has long since run by the time a second import happens, and removal detaches it, so + // without this every FindType after the first removal would answer from entries resolved + // before it - including a GraphSession belonging to a module that has been replaced. + Base.MgxCmdletBase.AttachAssemblyLoadHandler(); } internal static Assembly? ResolveDependency(AssemblyLoadContext defaultAlc, AssemblyName name) @@ -71,7 +91,7 @@ public void OnRemove(PSModuleInfo module) // both pieces of static state try { - Base.MgxCmdletBase.ResetHttpClient(); + ReleaseStaticState(); } catch (Exception ex) { @@ -79,10 +99,60 @@ public void OnRemove(PSModuleInfo module) System.Diagnostics.Debug.WriteLine($"[Mgx ALC] Cleanup on remove failed: {ex.Message}"); } - AssemblyLoadContext.Default.Resolving -= ResolveDependency; + // One -= per subscription, and the flag is what says there is one to take off: a removal + // that ran without an import behind it has nothing to detach. + if (Interlocked.Exchange(ref s_resolverHooked, 0) == 1) + AssemblyLoadContext.Default.Resolving -= ResolveDependency; // After the resolver, since this needs no dependency resolution and must not run before // ResetHttpClient, which may trigger loads the type cache should still observe Base.MgxCmdletBase.DetachAssemblyLoadHandler(); } + + ///

+ /// Everything module removal releases except the assembly-load hook, which OnRemove + /// detaches itself. Two reasons, neither of them that the detach cannot be undone - it can, + /// AttachAssemblyLoadHandler subscribes again and an import calls it. It has to run after + /// ResetHttpClient, whose loads the type cache should still observe; and this is the seam + /// tests drive a removal through, which is not the same as asking them to unsubscribe the + /// process's only invalidation hook. + /// + /// ResetHttpClient releases mgx's own client and the pipeline factory. The resilience + /// injection is separate state: it is installed on GraphSession, which belongs to another + /// module and outlives this one, so releasing it means putting the session back on the + /// genuine SDK client and only then letting go. Dropping the references alone would leave + /// the wrapper installed and the SDK sending through a handler belonging to an unloaded + /// module; the bridge-target map survives removal, so a later re-import's + /// Disable-MgxResilience can still take it off - the restore here just makes that + /// recovery unnecessary on the normal path. + /// + /// + /// The Set-MgxOption surface goes with them, and so do the endpoint and the timeout a client + /// build derives, and the telemetry counters a session accumulates. They are the per-process + /// state a removal used to leave set: a fresh import started on the previous import's tuning, + /// with Get-MgxOption reporting it as current - a re-import that looks like a reset was not + /// one - the VersionedBaseUrl its cmdlets composed named the cloud the import before it had + /// connected to until its own first build read the session again, and Get-MgxTelemetry + /// answered for traffic the session had never sent. + /// + /// + internal static void ReleaseStaticState() + { + Base.MgxCmdletBase.ResetHttpClient(); + Base.MgxCmdletBase.SetClientOptions(ResilientGraphClientOptions.Default); + Base.MgxCmdletBase.ReleaseEndpointAndTimeout(); + + // Same shape again: the counters are one process's running total, so a removal that + // leaves them set has Get-MgxTelemetry crediting a fresh import with the requests, + // retries and waits of the import before it. Reset leaves the adaptive pacer's control + // state alone, which is right here too - that describes the tenant's current throttle + // regime rather than anything the departing import accumulated. + MgxTelemetryCollector.Current.Reset(); + + Cmdlets.Configuration.EnableMgxResilience.ReleaseInjection(); + + // Last: the calls above resolve types through this cache, so clearing it earlier would + // only refill it with the entries the removal exists to drop. + Base.MgxCmdletBase.ClearTypeCache(); + } } diff --git a/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs b/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs index 030a875..6f4a6f9 100644 --- a/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs +++ b/src/Mgx.Cmdlets/Base/MgxCmdletBase.cs @@ -6,8 +6,11 @@ using System.Reflection; using System.Security.Cryptography; using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; using Mgx.Engine.Http; using Mgx.Engine.Models; +using Mgx.Engine.Pagination; using Polly.CircuitBreaker; namespace Mgx.Cmdlets.Base; @@ -22,27 +25,60 @@ public abstract class MgxCmdletBase : MgxCmdletCore { private ResilientGraphClient? _client; + // Which test-transport generation _client was built for. Only consulted while the seam + // below is armed; -1 means "built outside a scope", which never matches a live epoch. + private int _clientEpoch = -1; + private static readonly object s_initLock = new(); + + /// + /// The transport every ResilientGraphClient is built on. A replaced one is dropped, never + /// closed: each client captured it as a readonly field in its constructor and goes on + /// sending through it, one page at a time, for as long as its enumeration runs. It used to + /// be disposed TotalTimeoutSeconds after a swap - a per-REQUEST timeout used as a resource + /// lifetime, the same construction the shared rate limiter dropped for the reason recorded + /// in ResiliencePipelineFactory.GetOrCreate. No such bound is correct: an export may hold + /// its transport for hours, and a Connect-MgGraph to another identity, a Set-MgxOption, an + /// Enable-MgxResilience or a Remove-Module on another thread killed it underneath, so the + /// next page threw ObjectDisposedException mid-stream. Letting go is enough: the handler's + /// sockets are closed by its own idle and lifetime timeouts once nothing is sending + /// through them. + /// private static HttpClient? s_graphHttpClient; - private static bool s_ownsHttpClient; // false when using SDK fallback (don't dispose SDK's client) + private static bool s_ownsHttpClient; // false when using SDK fallback (the SDK's own client) + // Identity the cached client was built for private static volatile string? s_cachedAuthFingerprint; - // WeakReference so a disconnected AuthContext, and the certificate it holds, is not kept - // alive by Mgx. A collected target can never be the current context + // WeakReference so a disconnected AuthContext (and the X509Certificate2 it holds) is not + // kept alive by Mgx. A collected target can never be the current context. private static volatile WeakReference? s_cachedAuthContextRef; // TotalTimeoutSeconds the cached client's HttpClient.Timeout was derived from. private static int s_cachedTotalTimeoutSeconds; - internal static volatile string s_graphEndpoint = "https://graph.microsoft.com"; - internal static volatile ResilientGraphClientOptions s_clientOptions = ResilientGraphClientOptions.Default; - /// - /// Test-only transport override. When set, GetClient builds on the supplied HttpClient and - /// skips auth discovery, the Graph SDK reflection path and endpoint detection. + /// The cloud a process answers for before any session has been read, and what a module + /// removal puts the endpoint back to. /// - internal static volatile Func? s_testTransportFactory; + internal const string DefaultGraphEndpoint = "https://graph.microsoft.com"; + + internal static volatile string s_graphEndpoint = DefaultGraphEndpoint; + internal static volatile ResilientGraphClientOptions s_clientOptions = ResilientGraphClientOptions.Default; + + // Test transport seam. The xUnit assembly arms these three through MgxTransportScope, and + // every shipping path leaves s_testTransport null. InternalsVisibleTo names that assembly + // and neither assembly is signed, so the grant is by simple name: it keeps the seam out of + // the public surface and out of the way, not out of reach of code that means to find it. + internal static volatile HttpClient? s_testTransport; + + // Whether an armed test transport presents as mgx-owned. Get-MgxContent fails closed over a + // transport it does not own, so a content test arms this and the rest of the suite does not. + internal static volatile bool s_testTransportOwned; + + // Bumped whenever the armed transport changes. A cmdlet instance that cached a client under + // an earlier scope rebuilds instead of answering from a transport the scope has retired. + internal static int s_testTransportEpoch; /// /// Base URL for Graph API requests (e.g., "https://graph.microsoft.com/v1.0"). @@ -58,27 +94,38 @@ public abstract class MgxCmdletBase : MgxCmdletCore /// protected ResilientGraphClient GetClient() { - if (_client != null) return _client; + // The seam is read before the auth probe below on purpose: a test process has no Graph + // session, so reaching GetCurrentAuthIdentity would take the IsGraphAuthLoaded branch + // and terminate instead of running the cmdlet under the mock wire. + // + // The epoch, then the transport, then the epoch again. A scope arms the transport before + // it bumps the epoch, so a pair read this way either spans no bump - and belongs together + // - or is read again. Taking the transport first can pair one a scope has already retired + // with the epoch that retired it, and this instance then answers from that transport for + // as long as it lives. + int epoch; + HttpClient? testTransport; + do + { + epoch = Volatile.Read(ref s_testTransportEpoch); + testTransport = s_testTransport; + } + while (epoch != Volatile.Read(ref s_testTransportEpoch)); + + // The epoch gates the cached client on both paths, not just the seam's. It never moves + // outside a test process, so this is the plain cache hit in every shipping path - but a + // cmdlet instance that outlives the scope which armed it would otherwise keep answering + // from a transport that scope has already disposed. + if (_client != null && _clientEpoch == epoch) return _client; + _clientEpoch = epoch; - // Ahead of the auth check, since with no Graph SDK in the process the fingerprint is - // empty and the cmdlet would terminate before reaching any HTTP work - var testTransport = s_testTransportFactory; if (testTransport != null) - return _client = ConfigureClient(testTransport(), s_clientOptions); + return ConfigureClient(testTransport, s_clientOptions); var identity = GetCurrentAuthIdentity(WriteVerbose); if (string.IsNullOrEmpty(identity.Fingerprint)) { - // Microsoft.Graph.Authentication is a soft dependency, so an empty fingerprint has two - // causes needing different advice: the module is absent, or it is present but - // disconnected - var (message, errorId) = IsGraphAuthLoaded() - ? ("Not connected to Microsoft Graph. Run Connect-MgGraph first.", - "NotConnected") - : ("Microsoft.Graph.Authentication is not loaded. Install it " - + "(Install-PSResource -Name Microsoft.Graph.Authentication) and run " - + "Connect-MgGraph, or supply your own transport via Enable-MgxResilience.", - "GraphAuthModuleNotLoaded"); + var (message, errorId) = DescribeMissingConnection(IsGraphAuthLoaded()); ThrowTerminatingError(new ErrorRecord( new InvalidOperationException(message), @@ -88,8 +135,9 @@ protected ResilientGraphClient GetClient() return null!; } - // Locals are captured inside the lock, or another runspace could replace - // s_graphHttpClient between lock exit and use + // Lock protects concurrent runspaces from racing on static client/endpoint init. + // Capture locals inside lock to prevent TOCTOU race: another thread could enter + // the lock and replace/dispose s_graphHttpClient between lock exit and usage. HttpClient httpClient; var identityChanged = false; var clientOptions = s_clientOptions; @@ -100,8 +148,8 @@ protected ResilientGraphClient GetClient() previousFingerprint, identity.Fingerprint, StringComparison.Ordinal); var contextReplaced = AuthContextInstanceChanged(identity.AuthContext); - // A borrowed SDK client goes stale when Connect-MgGraph swaps - // GraphSession.GraphHttpClient and the cached instance still carries the old auth + // A client borrowed from the SDK goes stale on its own terms, e.g. after Connect-MgGraph swaps + // GraphSession.GraphHttpClient, and the instance we cached still carries the old auth. var sessionClient = s_ownsHttpClient ? null : TryGetSessionGraphHttpClient(); var borrowedClientStale = sessionClient != null && s_graphHttpClient != null && !ReferenceEquals(s_graphHttpClient, sessionClient); @@ -128,12 +176,15 @@ protected ResilientGraphClient GetClient() + $"{Shorten(identity.Fingerprint)}). Rebuilding the Mgx HTTP client."); } - // Resetting before the build is safe because GetClient falls back to the SDK - // client. Only on a real credential change, so a same-identity reconnect keeps - // its warm rate limiter instead of earning a fresh burst allowance + // Reset-before-Build is intentional here (unlike TryPreInitHttpClient which + // builds first then resets). GetClient() has a fallback path (SDK client), so + // resetting circuit breaker state from the old tenant before attempting to build + // is safe: if BuildCleanHttpClient fails, GetSdkHttpClientFallback provides a + // working client. If both fail, ThrowTerminatingError is the correct response. + // Only on a real credential change: a same-identity reconnect should keep its + // warm rate limiter instead of earning a fresh burst allowance. if (credentialChanged) ResiliencePipelineFactory.Reset(); - // In-flight ResilientGraphClient instances may still reference the old client - ScheduleDelayedHttpClientDispose(s_graphHttpClient, s_ownsHttpClient); + // The old client is let go, not closed - see the note on s_graphHttpClient. s_graphHttpClient = BuildCleanHttpClient(clientOptions.TotalTimeoutSeconds); if (s_graphHttpClient != null) { @@ -162,30 +213,34 @@ protected ResilientGraphClient GetClient() ? null : new WeakReference(identity.AuthContext); s_cachedTotalTimeoutSeconds = clientOptions.TotalTimeoutSeconds; - s_graphEndpoint = GetGraphEndpoint(WriteWarning, WriteVerbose) ?? "https://graph.microsoft.com"; + s_graphEndpoint = GetGraphEndpoint(WriteWarning, WriteVerbose) ?? DefaultGraphEndpoint; } httpClient = s_graphHttpClient!; } - // Outside s_initLock. Enable-MgxResilience takes StateLock then s_initLock, so taking - // StateLock here would invert the lock order and can deadlock + // Outside s_initLock by design. Enable-MgxResilience takes StateLock and then s_initLock + // (via TryPreInitHttpClient), so taking StateLock while holding s_initLock inverts the + // lock order and can deadlock. Never acquire StateLock inside s_initLock. if (identityChanged) Cmdlets.Configuration.EnableMgxResilience.RefreshInjectedClient(WriteWarning, WriteVerbose); - return _client = ConfigureClient(httpClient, clientOptions); + return ConfigureClient(httpClient, clientOptions); } - private ResilientGraphClient ConfigureClient(HttpClient httpClient, ResilientGraphClientOptions options) + /// + /// Wraps a transport in the ResilientGraphClient this cmdlet invocation uses and caches it. + /// Split out of GetClient so the transport seam above reaches the same configuration the + /// auth path produces, rather than a second copy of it that can drift. + /// + private ResilientGraphClient ConfigureClient(HttpClient httpClient, ResilientGraphClientOptions clientOptions) { - var client = new ResilientGraphClient(httpClient, options) - { - BodyReadTimeout = TimeSpan.FromSeconds(options.AttemptTimeoutSeconds), - VerboseWriter = msg => WriteVerbose(msg), - WarningWriter = msg => WriteWarning(msg), - DebugWriter = msg => WriteDebug(msg) - }; - client.DebugEnabled = IsDebugRequested(); - return client; + _client = new ResilientGraphClient(httpClient, clientOptions); + _client.BodyReadTimeout = TimeSpan.FromSeconds(clientOptions.AttemptTimeoutSeconds); + _client.VerboseWriter = msg => WriteVerbose(msg); + _client.WarningWriter = msg => WriteWarning(msg); + _client.DebugWriter = msg => WriteDebug(msg); + _client.DebugEnabled = IsDebugRequested(); + return _client; } /// @@ -214,6 +269,7 @@ private static AuthIdentity GetCurrentAuthIdentity(Action? verbose) verbose?.Invoke($"Failed to read GraphSession.AuthContext: {ex.Message}"); } + // Fallback for SDK internals drift using Get-MgContext try { using var ps = PowerShell.Create(RunspaceMode.CurrentRunspace); @@ -232,6 +288,11 @@ private static AuthIdentity GetCurrentAuthIdentity(Action? verbose) } } + /// + /// True when the live AuthContext is a different object than the one the cached client was + /// built from. Connect-MgGraph replaces the object, so this catches identity changes the + /// value fingerprint cannot see (a rotated ClientSecret above all). + /// private static bool AuthContextInstanceChanged(object? current) { var cachedRef = s_cachedAuthContextRef; @@ -246,7 +307,9 @@ private static bool AuthContextInstanceChanged(object? current) "CertificateThumbprint", "CertificateSubjectName", "SendCertificateChain", "WamEnabled" ]; - /// Builds a comparable fingerprint of the effective Graph identity. + /// + /// Builds a comparable fingerprint of the effective Graph identity. + /// internal static string BuildAuthFingerprint(object? authContext, string? graphEndpoint) { if (authContext == null) return string.Empty; @@ -274,7 +337,9 @@ internal static string BuildAuthFingerprint(object? authContext, string? graphEn private static void AppendField(StringBuilder sb, string value) => sb.Append(FieldSeparator).Append(value.Length).Append(':').Append(value); - /// Reads a named member off an AuthContext-shaped object. + /// + /// Reads a named member off an AuthContext-shaped object. + /// internal static object? ReadAuthMember(object? source, string name) { if (source == null) return null; @@ -303,11 +368,29 @@ private static string Shorten(string? fingerprint) => } /// - /// Whether Microsoft.Graph.Authentication is present in the session. It is a soft dependency, - /// so absent has to be told apart from present but disconnected. + /// Why no token could be obtained, given whether Microsoft.Graph.Authentication is present. + /// It is a soft dependency (see mgx.psd1), so an empty fingerprint has two distinct causes + /// needing different advice: while it sat in RequiredModules the SDK was guaranteed present + /// and "run Connect-MgGraph" was always the right answer, and without it that message sends + /// someone to a cmdlet that does not exist in their session. + /// + internal static (string Message, string ErrorId) DescribeMissingConnection(bool graphAuthLoaded) => + graphAuthLoaded + ? ("Not connected to Microsoft Graph. Run Connect-MgGraph first.", + "NotConnected") + : ("Microsoft.Graph.Authentication is not loaded. Install it " + + "(Install-PSResource -Name Microsoft.Graph.Authentication) and run " + + "Connect-MgGraph, or supply your own transport via Enable-MgxResilience.", + "GraphAuthModuleNotLoaded"); + + /// + /// Whether Microsoft.Graph.Authentication is present in the session. Because mgx does not + /// declare it in RequiredModules, "absent" is a real state that has to be told apart from + /// "present but disconnected" when reporting why a token could not be obtained. /// - /// Checks the type first, then Get-MgContext, since the module can be loaded even when - /// GraphSession is not where we look. + /// Checks the type first, then Get-MgContext: the SDK's internal type layout has moved + /// before, and the module can be loaded even when GraphSession is not where we look. + /// FindType caches only successful lookups, so a module imported later is still seen. /// /// internal static bool IsGraphAuthLoaded() @@ -332,14 +415,14 @@ internal static bool IsGraphAuthLoaded() /// /// The HttpClient the Microsoft.Graph SDK is currently using, or null when it is not - /// initialized. Used to detect that a borrowed SDK client has been replaced underneath us. + /// initialized. Used to detect that a borrowed SDK client has been replaced underneath us, + /// and to borrow from when mgx cannot build a client of its own. /// internal static HttpClient? TryGetSessionGraphHttpClient() { try { - var instance = TryGetGraphSessionInstance(); - return instance?.GetType().GetProperty("GraphHttpClient")?.GetValue(instance) as HttpClient; + return GenuineSessionClient(TryGetGraphSessionInstance()); } catch { @@ -347,6 +430,24 @@ internal static bool IsGraphAuthLoaded() } } + /// + /// GraphSession's client with any wrapper mgx installed there resolved away. + /// Enable-MgxResilience leaves a client of ours on that property, and it already carries a + /// resilience pipeline: borrowing it puts mgx's own pipeline on top of that one, and each + /// layer retries the one beneath it. The configured retry budget then multiplies on the + /// wire, telemetry counts each attempt once per layer, and the pacer halves per layer. + /// + /// The comparison that decides a borrowed client has gone stale reads through here too. It + /// is against the client mgx borrowed, so both sides have to be resolved the same way or a + /// session holding a wrapper reads as a replacement on every invocation. + /// + /// + private static HttpClient? GenuineSessionClient(object? graphSessionInstance) => + graphSessionInstance?.GetType().GetProperty("GraphHttpClient") + ?.GetValue(graphSessionInstance) is HttpClient client + ? Cmdlets.Configuration.EnableMgxResilience.ResolveGenuineSdkClient(client) + : null; + private static string? GetGraphEndpointFrom(object instance) { var env = instance.GetType().GetProperty("Environment")?.GetValue(instance); @@ -369,6 +470,13 @@ internal static bool IsGraphAuthLoaded() } } + /// + /// Builds an auth-only HttpClient using MSAL's AuthenticationHandler from the Graph SDK. + /// Token lifecycle: MSAL's AuthenticationHandler refreshes tokens proactively + /// (5 min before expiry). For operations spanning 2+ hours, token refresh is + /// transparent as long as the Connect-MgGraph session remains valid and the + /// refresh token has not been revoked. + /// private HttpClient? BuildCleanHttpClient(int totalTimeoutSeconds) => BuildCleanHttpClient(WriteWarning, WriteVerbose, totalTimeoutSeconds); @@ -383,14 +491,18 @@ internal static bool IsGraphAuthLoaded() var authContext = instance.GetType().GetProperty("AuthContext")?.GetValue(instance); if (authContext == null) return null; - // A prior SDK call may have replaced GraphSession.Environment with one whose - // AzureADEndpoint is empty, so save and restore it around the MSAL call + // Save AzureADEndpoint before GetAuthenticationProviderAsync. A prior SDK call + // (Connect-MgGraph or Invoke-MgGraphRequest) may have replaced GraphSession.Environment + // with a new object that has empty AzureADEndpoint. Restore it before calling MSAL. var envObj = instance.GetType().GetProperty("Environment")?.GetValue(instance); var savedAadEndpoint = envObj?.GetType().GetProperty("AzureADEndpoint")?.GetValue(envObj)?.ToString(); if (string.IsNullOrEmpty(savedAadEndpoint)) { - // AzureADEndpoint is already empty. Authority is AzureADEndpoint plus the tenant - // id, so it is only usable here when it parses as an absolute URI + // AzureADEndpoint already corrupted (a prior Invoke-MgGraphRequest set it to empty). + // Try to recover the base AAD host from AuthContext.Authority first. + // AuthContext.Authority is computed as AzureADEndpoint + "/" + tenantId - so when + // AzureADEndpoint was already empty at the time of computation, Authority is a + // relative path like "/tenantId" and cannot be parsed as an absolute URI. var aadEndpoint = authContext.GetType().GetProperty("Authority")?.GetValue(authContext)?.ToString(); var aadProp = envObj?.GetType().GetProperty("AzureADEndpoint"); if (aadProp?.CanWrite == true) @@ -405,8 +517,9 @@ internal static bool IsGraphAuthLoaded() } else { - // Neither is usable, so infer the AAD base from GraphEndpoint and fall - // back to global AAD + // Both AzureADEndpoint and AuthContext.Authority are corrupted. + // Infer the correct AAD base from GraphEndpoint (sovereign cloud mapping), + // falling back to global AAD for unknown or missing endpoints. var graphEndpoint = envObj?.GetType().GetProperty("GraphEndpoint")?.GetValue(envObj)?.ToString(); baseAuthority = graphEndpoint switch { @@ -467,9 +580,11 @@ internal static bool IsGraphAuthLoaded() { DefaultRequestVersion = HttpVersion.Version20, DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrLower, - // A safety net above the Polly total timeout, catching connections that bypass - // Polly such as pool exhaustion, a DNS hang or stale TLS. The headroom keeps - // Polly firing first + // Set HttpClient timeout as a safety net above Polly's TotalTimeoutSeconds. + // Polly handles all normal timeout semantics. This outer timeout catches + // edge cases where a connection bypasses Polly (pool exhaustion, DNS hang, + // stale TLS). Set above Polly's TotalTimeoutSeconds so Polly fires first; + // 60s of headroom prevents HttpClient from cancelling before Polly can react. Timeout = TimeSpan.FromSeconds(totalTimeoutSeconds + 60) }; } @@ -481,8 +596,12 @@ internal static bool IsGraphAuthLoaded() } /// - /// Pre-initializes the static HTTP client before any SDK probe runs. Building it while - /// AzureADEndpoint is still intact avoids the save and restore overhead on later calls. + /// Pre-initializes Mgx's static HTTP client before any SDK probe runs. + /// Called by Enable-MgxResilience before ForceInitializeAndGetClient as a + /// performance optimization: builds the clean client while AzureADEndpoint + /// is still intact, avoiding the save/restore overhead on subsequent calls. + /// The root cause fix (RestoreAzureADEndpoint in ForceInitializeAndGetClient) + /// handles the auth poisoning; this method is a belt-and-suspenders optimization. /// internal static void TryPreInitHttpClient(Action warn, Action verbose) { @@ -491,19 +610,23 @@ internal static void TryPreInitHttpClient(Action warn, Action ve var options = s_clientOptions; - // Volatile.Read gives the acquire barrier a non-volatile static read outside a lock - // lacks, which matters on ARM64 + // Quick early exit on the hot path (idempotent Enable calls). + // Volatile.Read ensures ARM64 memory visibility - without it, non-volatile statics + // read outside a lock have no acquire barrier and may return stale values. if (Volatile.Read(ref s_graphHttpClient) != null && !ClientIsStale(identity, options)) return; lock (s_initLock) { if (s_graphHttpClient != null && !ClientIsStale(identity, options)) return; + // Build first. Only dispose/reset after we have a confirmed replacement. + // If BuildCleanHttpClient fails, the existing s_graphHttpClient must remain valid + // and callers fall back via GetClient() on first use. var client = BuildCleanHttpClient(warn, verbose, options.TotalTimeoutSeconds); if (client == null) return; // BuildCleanHttpClient already warned with ex.Message ResiliencePipelineFactory.Reset(); - ScheduleDelayedHttpClientDispose(s_graphHttpClient, s_ownsHttpClient); + // The old client is let go, not closed - see the note on s_graphHttpClient. s_graphHttpClient = client; s_ownsHttpClient = true; s_cachedAuthFingerprint = identity.Fingerprint; @@ -511,10 +634,14 @@ internal static void TryPreInitHttpClient(Action warn, Action ve ? null : new WeakReference(identity.AuthContext); s_cachedTotalTimeoutSeconds = options.TotalTimeoutSeconds; - s_graphEndpoint = GetGraphEndpoint(warn, verbose) ?? "https://graph.microsoft.com"; + s_graphEndpoint = GetGraphEndpoint(warn, verbose) ?? DefaultGraphEndpoint; } } + /// + /// True when the cached client no longer matches the given identity (by either signal) or + /// the timeout it was built with. Callers must hold s_initLock, or accept a benign rebuild. + /// private static bool ClientIsStale(AuthIdentity identity, ResilientGraphClientOptions options) => !string.Equals(s_cachedAuthFingerprint, identity.Fingerprint, StringComparison.Ordinal) || AuthContextInstanceChanged(identity.AuthContext) @@ -527,13 +654,17 @@ private static bool ClientIsStale(AuthIdentity identity, ResilientGraphClientOpt var instance = TryGetGraphSessionInstance(); if (instance == null) return null; - var httpClient = instance.GetType().GetProperty("GraphHttpClient") - ?.GetValue(instance) as HttpClient; + // Resolved, never taken verbatim - see GenuineSessionClient. + var httpClient = GenuineSessionClient(instance); if (httpClient != null) return httpClient; - var endpoint = GetGraphEndpoint(WriteWarning, WriteVerbose) ?? "https://graph.microsoft.com"; + // Use detected endpoint instead of hardcoded graph.microsoft.com + // (supports sovereign clouds: GCC-High, DoD, China) + var endpoint = GetGraphEndpoint(WriteWarning, WriteVerbose) ?? DefaultGraphEndpoint; - // Same AzureADEndpoint save and restore as ForceInitializeAndGetClient + // Save AzureADEndpoint before probe (same issue as ForceInitializeAndGetClient: + // Invoke-MgGraphRequest replaces GraphSession.Environment with a new object + // that has empty AzureADEndpoint, breaking GetAuthenticationProviderAsync). var envObj = instance.GetType().GetProperty("Environment")?.GetValue(instance); var savedAadEndpoint = envObj?.GetType().GetProperty("AzureADEndpoint")?.GetValue(envObj)?.ToString(); @@ -549,8 +680,7 @@ private static bool ClientIsStale(AuthIdentity identity, ResilientGraphClientOpt // Restore AzureADEndpoint on the NEW Environment object the probe created. RestoreAzureADEndpoint(instance, savedAadEndpoint, WriteVerbose); - return instance.GetType().GetProperty("GraphHttpClient") - ?.GetValue(instance) as HttpClient; + return GenuineSessionClient(instance); } catch (Exception ex) { @@ -580,38 +710,62 @@ internal static void RestoreAzureADEndpoint(object graphSessionInstance, string? } } - // Avoids scanning every loaded assembly per call. Only non-null results are cached, since - // a miss now may succeed once the user imports another module - private static readonly ConcurrentDictionary s_typeCache = new(); + // Cache for FindType: avoids scanning all loaded assemblies on every call. + // ConcurrentDictionary is safe for concurrent runspaces. + // Only non-null results are cached: assemblies load lazily in PowerShell, + // so a miss now may succeed after the user imports additional modules. + // Internal on the same terms as the transport seam above: the suite asserts on it. + internal static readonly ConcurrentDictionary s_typeCache = new(); + + // Whether OnAssemblyLoad is subscribed. A process can import the module more than once, and + // += is not idempotent: a second subscription would outlive the -= that removal does, and + // clear the cache once per subscription on every load thereafter. + private static int s_assemblyLoadHooked; static MgxCmdletBase() { - AppDomain.CurrentDomain.AssemblyLoad += OnAssemblyLoad; + AttachAssemblyLoadHandler(); } - // A cached Type carries its defining assembly identity, so re-importing - // Microsoft.Graph.Authentication would leave Mgx pointing at a GraphSession singleton - // nobody else uses. Any assembly load is the signal to re-resolve + // A cached Type carries the identity of the assembly that defined it. Re-importing + // Microsoft.Graph.Authentication - a different version, or into a fresh load context - + // produces a second GraphSession type whose Instance is a different singleton, so a stale + // entry would silently point Mgx at a session nobody else is using. Assemblies only ever + // appear, never disappear, so any load is the signal to re-resolve. Clearing an almost + // always tiny dictionary is cheaper than validating entries on the hot path. private static void OnAssemblyLoad(object? sender, AssemblyLoadEventArgs args) => s_typeCache.Clear(); + /// + /// Subscribes the assembly-load hook, once. Module import calls this: the static constructor + /// runs the first time anything touches this type and never again, so after a removal has + /// detached the hook, nothing would re-arm invalidation for the rest of the process. + /// + internal static void AttachAssemblyLoadHandler() + { + if (Interlocked.Exchange(ref s_assemblyLoadHooked, 1) == 1) return; + AppDomain.CurrentDomain.AssemblyLoad += OnAssemblyLoad; + } + /// /// Detaches the assembly-load hook. Called on module removal so the handler does not /// root this type after the module is gone. /// - internal static void DetachAssemblyLoadHandler() => + internal static void DetachAssemblyLoadHandler() + { + if (Interlocked.Exchange(ref s_assemblyLoadHooked, 0) == 0) return; AppDomain.CurrentDomain.AssemblyLoad -= OnAssemblyLoad; + } /// - /// Stands in for the Graph SDK types the bridge reflects against. Its answers are never cached. + /// Drops every type resolved so far. Module removal does this because the entries carry the + /// identity of the assemblies that defined them: a Microsoft.Graph.Authentication loading + /// after the removal would otherwise keep resolving to the first one's GraphSession, whose + /// Instance is a singleton nobody else is using. /// - internal static volatile Func? s_typeResolverForTests; + internal static void ClearTypeCache() => s_typeCache.Clear(); internal static Type? FindType(string fullName) { - var resolver = s_typeResolverForTests; - if (resolver != null) - return resolver(fullName); - if (s_typeCache.TryGetValue(fullName, out var cached)) return cached; @@ -633,7 +787,7 @@ public static void ResetHttpClient() { lock (s_initLock) { - ScheduleDelayedHttpClientDispose(s_graphHttpClient, s_ownsHttpClient); + // The old client is let go, not closed - see the note on s_graphHttpClient. s_graphHttpClient = null; s_ownsHttpClient = false; s_cachedAuthFingerprint = null; @@ -642,26 +796,14 @@ public static void ResetHttpClient() } } - private static void ScheduleDelayedHttpClientDispose(HttpClient? client, bool owned) - { - // Ownership is passed in, not read off the static, because callers replace that static - // right after this call and disposing a borrowed SDK client would break the session - if (client == null || !owned) return; - var delaySeconds = s_clientOptions.TotalTimeoutSeconds; - _ = Task.Delay(TimeSpan.FromSeconds(delaySeconds)).ContinueWith(_ => - { - try { client.Dispose(); } catch { /* best-effort cleanup */ } - }, TaskScheduler.Default); - } - /// /// Whether the active transport is the mgx-owned clean client (AllowAutoRedirect off) /// rather than the borrowed SDK client. The content path requires ownership: the SDK /// client ships a RedirectHandler that auto-follows a content 302 to a host mgx never /// validated, so Get-MgxContent fails closed when this is false. /// - // A test transport is mgx-built and follows no redirects, so the content path may use it - protected static bool TransportIsOwned => s_ownsHttpClient || s_testTransportFactory != null; + protected static bool TransportIsOwned => + s_testTransport != null ? s_testTransportOwned : s_ownsHttpClient; /// /// Drain buffered verbose messages from the resilience pipeline. @@ -698,243 +840,2552 @@ internal static void SetClientOptions(ResilientGraphClientOptions options) s_clientOptions = options ?? ResilientGraphClientOptions.Default; } - private static bool CanTakeExclusively(string path) + /// + /// Puts back the two values a client build derives rather than a caller sets: the endpoint + /// read off the session it built for, and the TotalTimeoutSeconds the transport was sized to. + /// Both are per-process, so a removal that leaves them set gives the next import the cloud + /// the one before it connected to - in the VersionedBaseUrl every cmdlet composes its request + /// URLs from - until that import's own first build derives one. + /// + /// Beside SetClientOptions in ReleaseStaticState rather than inside ResetHttpClient, for the + /// reason the options release is: a caller that saves and restores these around + /// ResetHttpClient, as the suite's session scopes do, would have its restore released + /// instead of the state. + /// + /// + internal static void ReleaseEndpointAndTimeout() { - try + lock (s_initLock) { - using var _ = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.None); - return true; + s_graphEndpoint = DefaultGraphEndpoint; + s_cachedTotalTimeoutSeconds = 0; } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + } + + /// + /// Why an exclusive claim on a file did not succeed, where it did not. Asked of a temp a + /// checkpoint names, and of the output a checkpoint records its items into: both are files + /// a second run over the same command line can be writing right now, and neither is one + /// this run may take from under it. + /// + private enum FileClaim + { + /// Nothing else holds it, and this account can open it read-write. + Taken, + + /// + /// Something has it open. FileShare.None is honored between .NET processes on both + /// Windows and Unix, so a writer that holds the file makes the claim fail rather than + /// let a sweep take it out from under them. + /// + Held, + + /// + /// It is not a file this run can write at the offset a checkpoint records: mode bits, a + /// denying ACL, a read-only filesystem, a directory standing where the file should be, a + /// symlink loop or an over-long path, or a FIFO that opens and then has no length and no + /// offset to write at. Every failure but the sharing one, in other words, and no second + /// run is implied by any of them - which is the whole reason this is told apart from the + /// case above. + /// + Unopenable, + + /// + /// There is no such file, or no such directory to hold it. Nothing is wrong and no + /// second run is implied: it is the case a caller answers by starting over, and it is + /// told apart from the two above because on a file that is + /// not there raises the same IOException a sharing violation does - so read as one, a + /// missing file reported a second run over a file neither run has. + /// + Absent, + } + + /// + /// Ask for the file, and say what happened. The failures are not one answer: a file another + /// run holds is a file nothing is wrong with, a file this account cannot open read-write has + /// no other run behind it, and a file that is not there is neither - so a caller that + /// collapses them tells the reader to go looking for a run that does not exist. + /// + /// The claim is not finished at the open on Unix. .NET backs the share mode there with a + /// non-blocking flock taken AFTER open(2), so the handle binds to whatever inode the name + /// resolved to and the exclusion over it is granted several syscalls later - and a second + /// run that renamed its own file onto the name in between leaves this one holding an inode + /// with no name on it, with nothing about the handle to say so. So the name is asked again + /// where says it moved, and a name that keeps moving + /// is : another run is mutating it right now, which is what + /// every caller here reads Held as. Windows answers this question at the open itself and + /// keeps the answer it had. + /// + /// + /// + /// What a second opener may still do. for a claim taken and let + /// go in the same breath; the output hold asks for + /// instead, since it keeps the handle for the whole run. + /// + /// + /// The open file where the answer is , and where a claim that + /// was granted could not then be verified - answered with + /// the file still open, since a handle this never hands over is one nothing releases + /// deterministically: only the finalizer does, whenever the GC gets to it. Null for every + /// other answer. The caller owns it wherever it comes with one. + /// + /// + /// Why the file could not be claimed, where the answer is + /// - the open's own refusal, or the failure of the question that finishes a claim the open + /// was granted; null for every other answer. A caller building a stop names this instead of + /// guessing at a cause, and reads to know + /// whether granting write access is advice this run can honestly give. + /// + /// + /// for a claim on a file that has to be there already, which is + /// every caller that reads as an answer. + /// for a claim whose point is to exist and be refused + /// even where the file does not: what it makes is reported in , + /// and it answers Absent for the four names no create can be made through - a link + /// pointing at nothing, which is followed rather than replaced; a parent directory that is + /// not there, which is nothing to make a file in; a path whose parent component is a file + /// rather than a directory, which is nothing to make one in either and reaches the same + /// answer by ENOTDIR; and a name whose entry has gone by the time it is opened, where the + /// attempts left to ask again in have run out or something is standing at the name again by + /// the time the answer is read. + /// + /// + /// Whether the handle is over a file this open made, because nothing stood at the name. True + /// wherever the attempt made the file, including a claim that could not be verified - which + /// is what arms the finally's take-back - and false on every attempt after one the + /// verification found had moved: a claim asked again over a name that has moved is a claim + /// on whatever stands there now, which is not this run's to take away. + /// + /// The name is read a syscall in front of the open, so what this answers is the state the + /// open was asked for in. A file a second mgx run put there in between is one that run's own + /// claim refuses, and no claim of this one's comes away holding it; a file that nothing + /// holds - a shell's touch, an older mgx, any writer that takes no claim - is opened rather + /// than made and reported here as made all the same. What that costs is the one thing this + /// is read for: the empty output a promotion takes back where it stops is, in that window, a + /// file the run found rather than one it made. + /// + /// + /// + /// Whether the attempt this answer came from asked the open to make the file, because + /// nothing stood at the name and no link stood there either. says + /// a file was made; this says one was asked for, which is what tells an open refused over a + /// file that is there from a create the directory would not take. Decided per attempt like + /// the create itself, so it describes the look this answer came out of and no earlier one. + /// + private static FileClaim OpenExclusively(string path, FileShare share, out FileStream? stream, + out UnopenableReason? reason, FileMode mode = FileMode.Open) => + OpenExclusively(path, share, out stream, out reason, mode, out _, out _); + + /// + private static FileClaim OpenExclusively(string path, FileShare share, out FileStream? stream, + out UnopenableReason? reason, FileMode mode, out bool created, out bool askedToMakeIt) + { + for (var attempt = 1; ; attempt++) { - return false; + // A file is made here only where nothing stands at the name and no link does either: + // a create follows a link out of the directory and puts this run's file wherever it + // points, and a name something already stands at is one this claim opens rather than + // makes. Decided per attempt, since a claim is asked again precisely when the name + // has moved - and a second attempt that created a file over what a first attempt + // found would be holding an entry it did not make. + var make = mode == FileMode.OpenOrCreate && NothingStandsAt(path); + var claim = AskForTheFile(path, share, make ? FileMode.OpenOrCreate : FileMode.Open, + out stream, out reason); + created = make && claim == FileClaim.Taken; + askedToMakeIt = make; + + // A file that stood at the name a line ago and does not now: opened as one that has + // to be there, it answers Absent, and a caller that asked for OpenOrCreate is one + // that reads a name with nothing at it as a file it makes. So the name is asked + // again, where it is free to be made now. A link pointing at nothing is not that - + // it stands at the name, a create through it goes elsewhere, and Absent is what its + // caller already reads. + if (claim == FileClaim.Absent && mode == FileMode.OpenOrCreate + && attempt < HeldEntry.ClaimAttempts && NothingStandsAt(path)) + { + continue; + } + + if (claim != FileClaim.Taken) return claim; + + bool standing; + try + { + standing = HeldEntry.StillStandsAt(stream!, path); + } + catch (Exception ex) + { + // The exclusion is granted and the question that finishes it cannot be + // answered: a parent that stopped answering for the name, a share that went, a + // call that failed for a reason this cannot read. A claim that cannot be + // verified is not a claim, so it is refused - and refused as an answer rather + // than as a throw, with the handle and what the attempt made handed back + // exactly as a Taken answer hands them over. Every caller releases what it was + // given on its own path, and already reads Unopenable as a file it does not + // take; one of them releases in a finally, and takes the empty output it made + // back under that same claim. Emptying the out parameters here disarms that + // finally, which then holds nothing and unlinks nothing - and a file with no + // rows in it that no run holds is left standing at the caller's -OutputFile. + // + // The reason is the failure's own sentence, which names the call and the errno + // and says the claim on the file could not be finished. It is not a permission + // on the file: the search bit that went is the parent's, and a stale handle or + // a filesystem error is nobody's grant to give, so advice to open up the file + // would send the reader at the wrong thing. + reason = new UnopenableReason(FirstLine(ex.Message).TrimEnd('.'), + IsPermission: false); + return FileClaim.Unopenable; + } + + if (standing) return claim; + + // Granted over a file that has left the name. Dropped rather than reported, and the + // name asked again from the open: the run that moved it either holds what stands + // there now, which the next claim is refused by, or has let go, which the next claim + // takes honestly. + stream!.Dispose(); + stream = null; + created = false; + if (attempt == HeldEntry.ClaimAttempts) return FileClaim.Held; } } /// - /// Remove leftover "{outputPath}.{guid}.tmp" files. Called only when no resume is pending, - /// where every such file is an orphan by definition - except one a live run still holds, - /// which CanTakeExclusively keeps out of reach. + /// Nothing stands at the name, and no link stands there either - the one state a claim may + /// make its own file in. Read off the directory entry rather than through an open, which is + /// how a dangling link is told from an absent name at all: both answer that nothing is + /// there, and only one of them takes a create somewhere else. /// - protected void DeleteStaleTemps(string outputPath) + private static bool NothingStandsAt(string path) => + !Path.Exists(path) && new FileInfo(path).LinkTarget == null; + + /// + /// The open itself, and which of the four answers it gave. Every caller reaches it through + /// , which is where a claim is finished. + /// + private static FileClaim AskForTheFile(string path, FileShare share, FileMode mode, + out FileStream? stream, out UnopenableReason? reason) { + stream = null; + reason = null; try { - var dir = Path.GetDirectoryName(outputPath); - if (string.IsNullOrEmpty(dir) || !Directory.Exists(dir)) return; - foreach (var stale in Directory.EnumerateFiles(dir, Path.GetFileName(outputPath) + ".*.tmp").ToList()) + stream = new FileStream(path, mode, FileAccess.ReadWrite, share); + if (!stream.CanSeek) { - // A second export running against the same output owns a file matching the same - // glob. Ask for it exclusively first, since Unix would happily unlink a file - // another run is still writing into - if (!CanTakeExclusively(stale)) - { - WriteVerbose($"Left '{Path.GetFileName(stale)}' alone: another run is writing to it."); - continue; - } - try - { - File.Delete(stale); - WriteVerbose($"Deleted an orphaned temp file from an earlier interrupted run: {Path.GetFileName(stale)}"); - } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) - { - WriteWarning($"Could not delete orphaned temp file '{stale}': {ex.Message}. Delete it manually."); - } + // Something opened, and it is not a file with a length and an offset in it. A + // FIFO standing at the output path opens read-write without blocking, and every + // question asked of the handle afterwards - how long is it, cut it back to the + // recorded length, append at the end - is one a stream that cannot seek answers + // with a NotSupportedException, which is neither of the two failures a caller + // here knows how to report and went past both guards with the handle still open. + // So it is decided at the open: nothing this run can resume into, and let go + // before a caller is told to wait for it. Nothing here throws, so the reason is + // this method's own words rather than an exception's - a FIFO is not a + // permission, and granting write access to one would not give it a length. + stream.Dispose(); + stream = null; + reason = new UnopenableReason("the file has no offset to write at", + IsPermission: false); + return FileClaim.Unopenable; } + return FileClaim.Taken; } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + catch (FileNotFoundException) + { + return FileClaim.Absent; + } + catch (DirectoryNotFoundException) + { + // The same answer one level up: ENOENT on a path component. Both are IOExceptions, + // and both have to be caught above the sharing case below rather than reasoned about + // from an HRESULT afterwards. + return FileClaim.Absent; + } + catch (UnauthorizedAccessException ex) + { + reason = ReasonForUnopenable(path, ex); + return FileClaim.Unopenable; + } + catch (IOException ex) { - // Best effort. A sweep failure must never stop the run + // The sharing violation is named, and every other way an open can fail is a file + // this run cannot open. Decided the other way round - anything not on a list of + // access errnos is another run - a symlink loop and an over-long path at the output + // path both stopped the run with "Another export is still writing", sending the + // reader to wait for a second export over a path no run of any kind could open. + if (IsSharingViolation(ex)) return FileClaim.Held; + reason = ReasonForUnopenable(path, ex); + return FileClaim.Unopenable; } } - private static string? ResolveNamedTemp(string outputPath, string tempFileName, long dataLength) + /// + /// Why an open of failed, refined once for every site that has to + /// say it. The claim in is one; the create a promotion stages + /// its copy with is the other, and it built its own words out of the raw exception - so one + /// disk state was two sentences, and a directory at the staging name read "Access to the + /// path '...' is denied" in the applying pass and "a directory stands at the path" in the + /// preview of the same run. + /// + /// carries the runtime's own first line with its + /// sentence-ending period taken off, since every caller quotes it inside a sentence it + /// punctuates itself. The two causes told apart from the rest are the two with a different + /// way out than granting write access on the file: a directory standing at the path, which + /// no grant makes writable at an offset, and a parent directory this account cannot search, + /// where the grant belongs to the directory and not to the file the sentence names. + /// + /// + /// A directory is asked of the filesystem rather than of the errno, since it reaches an + /// access failure as EISDIR or as EACCES depending on the mode on it and both say the same + /// thing about the path - and, from a create, as neither: a create refuses an existing name + /// with "already exists" whatever stands there, so a directory that appears at the staging + /// name between the sweep and the create is asked about on that route too, and the two + /// instructions call one disk state by one name. An unsearchable parent is read off the + /// entry the open failed on, + /// by : a parent that is not there reaches + /// above and never gets here, so what is left to + /// tell apart is a directory this account may not look inside from a file inside one it may. + /// + /// + private static UnopenableReason ReasonForUnopenable(string path, Exception ex, + bool fromCreate = false) { - if (dataLength <= 0) return null; - var dir = Path.GetDirectoryName(outputPath); - if (string.IsNullOrEmpty(dir) || !Directory.Exists(dir)) return null; - if (!string.Equals(tempFileName, Path.GetFileName(tempFileName), StringComparison.Ordinal)) - return null; - if (!IsRunTempName(Path.GetFileName(outputPath), tempFileName)) return null; - var tempPath = Path.Combine(dir, tempFileName); - if (string.Equals(Path.GetFullPath(tempPath), Path.GetFullPath(outputPath), - StringComparison.OrdinalIgnoreCase)) - return null; - if (!File.Exists(tempPath)) return null; - if (new FileInfo(tempPath).Length < dataLength) return null; - return tempPath; + // Only an access failure can be one of the two named causes. Every other way an open + // fails - a symlink loop, an over-long path, a socket, a full volume - is the runtime's + // words and nothing this run adds to them. + if (ex is not UnauthorizedAccessException) + { + // Except the one a create has of its own: it refuses a name that already exists + // with "The file '...' already exists", whatever kind of entry stands there, so a + // directory appearing at the staging name between the sweep and the create read as + // a file in the way while the sweep, one instruction earlier, called the same disk + // state a directory. Asked of the filesystem, the way the access failure below asks + // it, so one entry reads one way wherever the run met it. + if (fromCreate && Directory.Exists(path)) + return new UnopenableReason("a directory stands at the path", IsPermission: false, + IsDirectory: true); + return new UnopenableReason(FirstLine(ex.Message).TrimEnd('.'), IsPermission: false); + } + + if (Directory.Exists(path)) + return new UnopenableReason("a directory stands at the path", IsPermission: false, + IsDirectory: true); + + var parent = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(parent) && Directory.Exists(parent) + && TheParentRefusesToBeSearched(path)) + return new UnopenableReason($"the directory '{parent}' cannot be searched", + IsPermission: true, IsUnsearchableParent: true); + + return new UnopenableReason(FirstLine(ex.Message).TrimEnd('.'), IsPermission: true); } - private static bool IsRunTempName(string outputFileName, string candidate) + /// + /// Whether the entry at cannot even be asked about, which is what a + /// directory missing its search bit refuses - and the one thing that separates a file this + /// account may not write from a file it cannot reach at all. Asked of the entry rather than + /// of the directory, because a directory answers for itself either way: mode 000 or not, a + /// stat of the directory needs the search bit on ITS parent and not on itself, so + /// is true of a directory nothing can be looked up + /// inside. What the missing bit refuses is every name under it, present or absent alike. + /// + /// Told from a name that is simply not there, which is what a create is about to make: a + /// searchable directory answers an absent name with ENOENT, and only an unsearchable one + /// answers a permission. Listing is not the question either - a directory with the search + /// bit and not the read bit refuses to be listed and resolves every name in it - so nothing + /// here enumerates. + /// + /// + private static bool TheParentRefusesToBeSearched(string path) { - var prefix = outputFileName + "."; - const string suffix = ".tmp"; - if (candidate.Length != prefix.Length + 32 + suffix.Length) return false; - if (!candidate.StartsWith(prefix, StringComparison.Ordinal)) return false; - if (!candidate.EndsWith(suffix, StringComparison.Ordinal)) return false; - for (var i = prefix.Length; i < prefix.Length + 32; i++) + try { File.GetAttributes(path); } + catch (UnauthorizedAccessException) { return true; } + catch (Exception ex) when (ex is IOException or ArgumentException + or NotSupportedException) { } + return false; + } + + /// + /// The exception's own explanation, first line only. A caller quotes the runtime's words for + /// why an open failed, and quoting more than the first line risks folding a second sentence, + /// or a path repeated on its own line, into what reads as one clause. + /// + private static string FirstLine(string message) + { + var newline = message.IndexOfAny(['\r', '\n']); + return newline < 0 ? message : message[..newline]; + } + + /// + /// + /// Ask for the file exclusively and let it go again, which is all a caller wants that only + /// has to report on it. The reason a failed claim is unopenable is not asked for here: every + /// caller of this overload already has its own words for the one failure it distinguishes + /// from a hold, or does not report a cause at all. + /// + private static FileClaim ClaimExclusively(string path, out UnopenableReason? reason) + { + var claim = OpenExclusively(path, FileShare.None, out var stream, out reason); + stream?.Dispose(); + return claim; + } + + /// + /// Whether an IOException from opening a file is the sharing violation the share mode exists + /// to produce, rather than one of the other ways an open fails. The engine reads it for the + /// scratch names it sweeps, and one implementation is what keeps the claim here and that + /// sweep from disagreeing about which errno means a second run. + /// + /// + private static bool IsSharingViolation(IOException ex) => + ScratchName.IsSharingViolation(ex); + + /// + /// True when nothing else holds the file and this account can open it read-write. The sweep + /// and the adoption search ask only this much: a file they cannot claim is a file they pass + /// over, whichever of the two reasons it is. + /// + private static bool CanTakeExclusively(string path) => + ClaimExclusively(path, out _) == FileClaim.Taken; + + /// + /// What each directory answered when it was asked whether it keeps two spellings of a name + /// apart. Keyed by full path: a volume does not change its mind, so one answer per directory + /// per process is all of it, and two callers racing for the same key cost a second look and + /// nothing else. An answer read off the directory's own entries is kept here like a probed + /// one; the platform rule a caller falls back to where it may not write is not, so the next + /// caller that may write still asks the volume. + /// + private static readonly ConcurrentDictionary s_caseSensitiveDirectories = + new(StringComparer.Ordinal); + + /// The answer where the directory cannot be asked: the rule the platform applies by default. + protected static bool PlatformCaseRule => OperatingSystem.IsLinux(); + + /// + /// Whether the filesystem under keeps "Users.jsonl" and + /// "users.jsonl" apart. Asked of the directory rather than of the operating system, because + /// the operating system does not answer for it: an APFS volume can be created + /// case-sensitive, a Windows directory can be flagged case-sensitive on its own, and where + /// the guess went the wrong way the temp helpers matched names the filesystem does not - a + /// sweep reaching another export's live temp, and two distinct outputs comparing as one file. + /// + /// It is answered without touching the directory wherever it can be: an entry already there, + /// looked for under its own spelling inverted, settles it - present under the other spelling + /// and the volume folds case, absent and it keeps the two apart. Only a directory with no + /// such entry needs a file written, and says whether this caller + /// may write one. Everything reached above a ShouldProcess gate passes false: -WhatIf + /// reports what a run would do, and creating and removing a file to find out moves the + /// caller's directory's write time. A caller that may not write, in a directory that cannot + /// answer for itself, gets the platform rule - which is what every release before the probe + /// used everywhere - and that answer is not remembered, so the run below still asks the + /// volume. The two therefore differ only in a directory holding nothing, on a volume whose + /// rule is not its platform's default. + /// + /// A directory this run cannot write to answers with the platform rule as well: the probe is + /// a better answer where it can be taken, not a precondition, and a run that cannot write + /// its own output directory has its own error to raise rather than this one. + /// + internal static bool DirectoryNamesAreCaseSensitive(string? directory, bool mayWrite) + { + if (string.IsNullOrEmpty(directory)) return PlatformCaseRule; + string key; + try { key = Path.GetFullPath(directory); } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException + or PathTooLongException or IOException) + { + return PlatformCaseRule; + } + + if (s_caseSensitiveDirectories.TryGetValue(key, out var settled)) return settled; + + // Before this module writes a probe here, take back the one an earlier run left. Only + // on the write path: a caller that may not write deletes nothing either. + if (mayWrite) ReclaimCaseProbeLitter(key); + + if (ReadOnlyCaseAnswer(key) is { } observed) + return s_caseSensitiveDirectories.GetOrAdd(key, observed); + + if (!mayWrite) return PlatformCaseRule; + + return s_caseSensitiveDirectories.GetOrAdd(key, ProbeCaseSensitivity); + } + + /// + /// Whether answering this directory's case rule would still take a write. False once the + /// answer is known, and false where an entry already in the directory gives it - which this + /// keeps, since having asked, it costs nothing more. True only for a directory that exists + /// and has nothing in it to read the answer off, which is the one place a preview and the + /// run it describes can disagree, and the only place worth a line saying so. A directory + /// that is not there is answered no: neither side of a gate can ask one, so they agree. + /// + protected static bool DirectoryCaseRuleNeedsProbe(string? directory) + { + if (string.IsNullOrEmpty(directory)) return false; + string key; + try { key = Path.GetFullPath(directory); } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException + or PathTooLongException or IOException) { - if (candidate[i] is (>= '0' and <= '9') or (>= 'a' and <= 'f')) continue; return false; } - return true; + if (s_caseSensitiveDirectories.ContainsKey(key)) return false; + if (!Directory.Exists(key)) return false; + if (ReadOnlyCaseAnswer(key) is not { } observed) return true; + s_caseSensitiveDirectories.GetOrAdd(key, observed); + return false; } /// - /// Replace the output with the first bytes of a named temp, - /// then remove the temp. Replacing rather than appending keeps a previous run rows out of - /// the output. Taking exactly the recorded file keeps an unrelated leftover from being - /// merged in, and counting bytes rather than lines cannot disagree about a torn final line. - /// Returns false when the temp is absent or shorter than the checkpoint promised, which - /// means the caller must not resume past the items it counted. + /// The answer read off the directory's own entries, or null where they do not give one. An + /// entry that is there under one spelling is there under the inverted spelling too exactly + /// when the volume folds case, so asking for the inverted name settles it with nothing + /// written. + /// + /// Three kinds of candidate are passed over rather than answered from. A name with no ASCII + /// letter inverts to itself, and a name carrying anything outside ASCII inverts through case + /// mappings the filesystem need not share - a sharp s and a dotted capital I are not their + /// own round trip - so a folding volume could be read as a splitting one. A directory that + /// already holds BOTH spellings says nothing either: that is what a case-sensitive volume + /// looks like, not a fold. Neither does an entry unlinked between the listing and the + /// lookup, which is a file that is simply gone rather than a name the volume refused, so the + /// entry is looked for again before its absence is read as an answer. The probe files below + /// are skipped for the same reason: one of them may be another run's, about to vanish. /// - protected static bool TryPromoteNamedTemp(string outputPath, string tempFileName, long dataLength) + private static bool? ReadOnlyCaseAnswer(string directory) { try { - var tempPath = ResolveNamedTemp(outputPath, tempFileName, dataLength); - if (tempPath == null) return false; - - // Staged like the other forms, so the destination is replaced in one Move rather - // than truncated and refilled in place. - var adoptPath = outputPath + ".adopt"; - using (var writer = new FileStream(adoptPath, FileMode.Create, FileAccess.Write)) + var names = Directory.GetFileSystemEntries(directory) + .Select(Path.GetFileName) + .Where(n => !string.IsNullOrEmpty(n)) + .Select(n => n!) + .ToList(); + var present = new HashSet(names, StringComparer.Ordinal); + foreach (var name in names) { - using var temp = new FileStream(tempPath, FileMode.Open, FileAccess.Read); - var buffer = new byte[81920]; - long remaining = dataLength; - while (remaining > 0) - { - var read = temp.Read(buffer, 0, (int)Math.Min(buffer.Length, remaining)); - if (read <= 0) break; - writer.Write(buffer, 0, read); - remaining -= read; - } - if (remaining > 0) - { - writer.Dispose(); - File.Delete(adoptPath); - return false; - } + if (IsCaseProbeName(name)) continue; + if (InvertCase(name) is not { } inverted) continue; + if (present.Contains(inverted)) continue; + var other = Path.Combine(directory, inverted); + if (File.Exists(other) || Directory.Exists(other)) return false; + var mine = Path.Combine(directory, name); + if (File.Exists(mine) || Directory.Exists(mine)) return true; } - File.Move(adoptPath, outputPath, overwrite: true); - try { File.Delete(tempPath); } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { } - return true; - } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) - { - return false; } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { } + return null; } /// - /// Cut the output back to the length a checkpoint recorded for it, dropping anything the - /// interrupted run wrote after its last save. Those items are re-fetched, so dropping them - /// is what keeps a resume from duplicating them. Returns false when the output is shorter - /// than recorded, which means it is no longer the file the checkpoint describes. + /// The name with every ASCII letter's case turned over, or null where that is not a + /// different name this filesystem has to have an opinion about: a name with no ASCII letter + /// in it, or one carrying anything outside ASCII. /// - protected static bool TryTrimOutputToCheckpoint(string outputPath, long dataLength) + private static string? InvertCase(string name) { - try + var chars = name.ToCharArray(); + var sawLetter = false; + for (var i = 0; i < chars.Length; i++) { - // A checkpoint on disk is untrusted input, and SetLength rejects a negative length - // with an exception the catch below does not cover - if (dataLength < 0) return false; - if (!File.Exists(outputPath)) return false; - var actual = new FileInfo(outputPath).Length; - if (actual < dataLength) return false; - if (actual == dataLength) return true; - using var fs = new FileStream(outputPath, FileMode.Open, FileAccess.Write); - fs.SetLength(dataLength); - return true; + var c = chars[i]; + if (c > 0x7F) return null; + if (c is >= 'a' and <= 'z') { chars[i] = (char)(c - 32); sawLetter = true; } + else if (c is >= 'A' and <= 'Z') { chars[i] = (char)(c + 32); sawLetter = true; } } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + return sawLetter ? new string(chars) : null; + } + + /// + /// True for a name the probe below gives its own file: ".mgx-case-", 32 lowercase hex + /// digits, and nothing else. Nothing but this writes one, which is what makes deleting a + /// leftover safe - and holding the exact shape is what keeps that promise about a directory + /// the caller also keeps their own ".mgx-case-notes" in. + /// + private static bool IsCaseProbeName(string candidate) + { + const string prefix = ".mgx-case-"; + if (candidate.Length != prefix.Length + 32) return false; + if (!candidate.StartsWith(prefix, StringComparison.Ordinal)) return false; + for (var i = prefix.Length; i < candidate.Length; i++) { + if (candidate[i] is (>= '0' and <= '9') or (>= 'a' and <= 'f')) continue; return false; } + return true; } /// - /// Recovers a fresh JSONL run interrupted before its temp was promoted, from a checkpoint - /// predating the recorded temp name and length. With only a line count and the newest - /// matching temp to go on, this is safe only when no output exists, since everything the run - /// wrote is then in its temp. Against an existing output there is no way to tell whose items - /// the temp holds and the caller must re-enumerate. Copies exactly - /// lines, then removes the temp. Returns false when nothing - /// usable exists. + /// Delete the probe files an earlier run left in this directory. DeleteOnClose is the + /// kernel's promise on Windows and is kept however the process ends; on Unix the unlink + /// happens where the stream is disposed, so a process killed outright leaves its probe + /// behind, and nothing on disk refers to it again. One is reclaimed the way the stale-temp + /// sweep reclaims a temp - claimed exclusively first - because a probe another run is taking + /// right now is open, and unlinking it under them would answer their question with the + /// wrong half of it. /// - protected static bool TryAdoptOrphanedTemp(string outputPath, long itemCount) + private static void ReclaimCaseProbeLitter(string directory) { try { - if (itemCount <= 0) return false; - if (File.Exists(outputPath)) return false; - var dir = Path.GetDirectoryName(outputPath); - if (string.IsNullOrEmpty(dir) || !Directory.Exists(dir)) return false; - var temp = Directory.EnumerateFiles(dir, Path.GetFileName(outputPath) + ".*.tmp") - .Select(p => new FileInfo(p)) - .OrderByDescending(f => f.LastWriteTimeUtc) - .FirstOrDefault(); - if (temp == null) return false; - - // Staged so the output appears in one Move. A merge that dies halfway leaves only - // the staging file behind, and the caller keeps its checkpoint - the safe direction. - long copied = 0; - var adoptPath = outputPath + ".adopt"; - using (var writer = new StreamWriter(adoptPath, append: false)) + foreach (var entry in Directory.EnumerateFiles(directory, ".mgx-case-*").ToList()) { - using var reader = new StreamReader(temp.FullName); - string? line; - while (copied < itemCount && (line = reader.ReadLine()) != null) - { - writer.WriteLine(line); - copied++; - } + if (!IsCaseProbeName(Path.GetFileName(entry))) continue; + if (!CanTakeExclusively(entry)) continue; + try { File.Delete(entry); } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { } } - if (copied < itemCount) + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { } + } + + /// + /// One probe file, created under a name of its own so nothing else can be reading it, and + /// deleted by hand as well as by the handle. The name is not one a run gives its own temp, + /// so neither the sweep nor adoption can reach it even in the moment it exists. + /// + /// DeleteOnClose is not the same promise on both platforms. Windows unlinks at the last + /// handle close, which the kernel performs however the process ends; Unix unlinks where the + /// stream is disposed, so a process killed outright leaves the file where it is. That is + /// what the reclaim above collects, and it runs before this writes anything. + /// + private static bool ProbeCaseSensitivity(string directory) + { + var name = $".mgx-case-{Guid.NewGuid():N}"; + var probe = Path.Combine(directory, name); + try + { + using (new FileStream(probe, FileMode.CreateNew, FileAccess.Write, FileShare.None, + bufferSize: 1, FileOptions.DeleteOnClose)) { - // Temp holds less than the checkpoint promises - unusable. - File.Delete(adoptPath); - return false; + // It exists under the name it was created with. Whether it also exists under + // the other casing is the whole question. + return !File.Exists(Path.Combine(directory, name.ToUpperInvariant())); } - File.Move(adoptPath, outputPath, overwrite: true); - temp.Delete(); - return true; } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { - return false; + return PlatformCaseRule; + } + finally + { + try { File.Delete(probe); } catch { } } } - #region Shared URL and header builders - - protected static string NormalizePath(string path) + /// + /// Test seam: pins the answer for one directory, so the case-sensitive decision path runs on + /// a host whose own filesystem is not. Null forgets it and the next question probes again. + /// Internal, assigned directly rather than reflected; every shipping path leaves it alone. + /// + internal static void SetDirectoryCaseSensitivity(string directory, bool? caseSensitive) { - if (string.IsNullOrEmpty(path)) return string.Empty; - return path.StartsWith('/') ? path : $"/{path}"; + var key = Path.GetFullPath(directory); + if (caseSensitive is { } value) s_caseSensitiveDirectories[key] = value; + else s_caseSensitiveDirectories.TryRemove(key, out _); } - protected static Dictionary? BuildRequestHeaders( - string? consistencyLevel, System.Collections.Hashtable? extraHeaders) + /// + /// How two file names in one directory are compared. Directory.EnumerateFiles matches its + /// pattern the way the filesystem does - so where "Users.jsonl" answers to "users.jsonl" the + /// glob returns it - and a predicate filtering those results with an ordinal comparison + /// disagreed with the enumeration that produced them: an export whose -OutputFile differed + /// only in case abandoned its own temp and re-enumerated in silence. Both sides are keyed + /// from the one answer above, so they cannot disagree again. + /// + private static StringComparison FileNameComparisonIn(string? directory, bool mayWrite) => + DirectoryNamesAreCaseSensitive(directory, mayWrite) + ? StringComparison.Ordinal + : StringComparison.OrdinalIgnoreCase; + + /// + /// How the "{output}.*.tmp" glob is matched in one directory. The two-argument + /// EnumerateFiles takes its casing from the platform, which is the guess the comparison + /// above no longer makes; the rest of these are that overload's own defaults - Win32 + /// globbing, nothing skipped by attribute, an inaccessible entry raised rather than + /// swallowed - spelled out so that naming the casing does not quietly change them too. + /// + private static EnumerationOptions TempGlobOptions(string directory, bool mayWrite) => new() { - Dictionary? headers = null; + MatchCasing = DirectoryNamesAreCaseSensitive(directory, mayWrite) + ? MatchCasing.CaseSensitive + : MatchCasing.CaseInsensitive, + MatchType = MatchType.Win32, + AttributesToSkip = 0, + IgnoreInaccessible = false, + }; - // Apply extraHeaders first so dedicated parameters can override - if (extraHeaders != null) + /// + /// Whether two paths name the same file, normalized and compared the way the filesystem + /// would answer. A checkpoint records the output it was collecting into so a run can tell + /// its own from someone else's, and a leaf name cannot carry that: "a/users.jsonl" and + /// "b/users.jsonl" are one name and two files. False for anything that is not a path - + /// a checkpoint is untrusted input once it is on disk, and an unusable name is not this + /// run's output. + /// + protected static bool SamePath(string? left, string? right, bool mayWrite) + { + if (left == null || right == null) return false; + try { - headers = new Dictionary(); - foreach (var key in extraHeaders.Keys) - headers[key.ToString()!] = extraHeaders[key]?.ToString() ?? string.Empty; + var recorded = Path.GetFullPath(left); + var mine = Path.GetFullPath(right); + // Keyed on the second path, which is this run's own output at every call site, and + // so names the directory whose rule decides whether the other spelling is that file. + return string.Equals(recorded, mine, + FileNameComparisonIn(Path.GetDirectoryName(mine), mayWrite)); } - - // Dedicated -ConsistencyLevel parameter always wins over -Headers key - if (!string.IsNullOrEmpty(consistencyLevel)) + catch (Exception ex) when (ex is ArgumentException or NotSupportedException + or PathTooLongException or IOException) { - headers ??= new Dictionary(); - headers["ConsistencyLevel"] = consistencyLevel; + return false; + } + } + + /// + /// What a URL addresses, independently of where Graph is reached: the path from its API + /// version onward, plus the query. Everything in front of the version is the endpoint, and + /// a gateway prefix, a trailing slash on the configured endpoint and a sovereign cloud all + /// change that without changing which resource is named - so comparing whole URLs called a + /// checkpoint another run's for reasons that have nothing to do with what it enumerated. + /// Null when the argument is not a URL. + /// + protected static string? ResourceIdentity(string? url) + { + if (!Uri.TryCreate(url, UriKind.Absolute, out var parsed)) return null; + var segments = parsed.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries); + var version = Array.FindIndex(segments, seg => + string.Equals(seg, "v1.0", StringComparison.OrdinalIgnoreCase) + || string.Equals(seg, "beta", StringComparison.OrdinalIgnoreCase)); + // No version segment to anchor on: nothing can be told apart from the endpoint, so the + // whole path stands, which can only refuse where it would have refused before. + var resource = version < 0 ? segments : segments[version..]; + return "/" + string.Join('/', resource) + parsed.Query; + } + + /// + /// Whether two values name the same enumeration. The path is + /// compared without case: Graph answers "/users" and "/Users" from one collection, so an + /// ordinal comparison turned a -Uri typed differently between two runs into a different + /// export, and the resume was refused and the collection enumerated again. The query is + /// compared as written - a $filter value is the service's to interpret, and two spellings of + /// one are not demonstrably the same enumeration. A null identity is one that could not be + /// read as a URL, and matches nothing. + /// + protected static bool SameResourceIdentity(string? recorded, string? current) + { + if (recorded == null || current == null) return false; + var r = recorded.IndexOf('?'); + var c = current.IndexOf('?'); + return string.Equals(r < 0 ? recorded : recorded[..r], + c < 0 ? current : current[..c], StringComparison.OrdinalIgnoreCase) + && string.Equals(r < 0 ? string.Empty : recorded[r..], + c < 0 ? string.Empty : current[c..], StringComparison.Ordinal); + } + + /// + /// Whether the checkpoint records anything for to weigh: + /// a path to compare, or a length to hold a file against. A checkpoint recording neither has + /// nothing there to be weighed, and asking anyway answers no for want of a measurement - so + /// the shape every release before 2.1 wrote was read as another run's, its temp swept by the + /// sweep a refusal holds off over the temp it names, and the position counting those items + /// left standing to refuse the next -Latest. That shape is decided on the evidence it does + /// have - a temp carrying this output's own name, beside an output that is not there - by + /// the reconcile, which refuses it wherever an output exists. Both cmdlets ask this one + /// question in front of the predicate, so one checkpoint reads one way in either. + /// + protected static bool RecordsAFileToWeigh(PaginationCheckpoint checkpoint) => + checkpoint.OutputFile != null || checkpoint.DataLength != null; + + /// + /// Whether the file a checkpoint counted its items into is one this run collects into. A + /// checkpoint that records the output answers directly. One written before that field + /// existed records only where the bytes are - a temp of the output, or the output itself - + /// and reading that silence as "mine" is what let an older release's checkpoint drive a + /// recovery against a file it had never been measured against. It is believed only where + /// the file it points at corroborates it: a named temp has to be one this output's own name + /// produces and hold at least the bytes counted, and with no temp named the output itself + /// has to be there and be at least that long. Nothing to corroborate is not evidence, and a + /// recorded length below zero is not a measurement. + /// + protected static bool RecordedOutputMatches( + string? recordedOutput, string? recordedTemp, long? dataLength, string outputPath, + bool mayWrite) + { + if (recordedOutput != null) return SamePath(recordedOutput, outputPath, mayWrite); + if (dataLength is not { } length || length < 0) return false; + if (recordedTemp != null) + return ResolveNamedTemp(outputPath, recordedTemp, length, mayWrite) != null; + try + { + var info = new FileInfo(outputPath); + return info.Exists && info.Length >= length; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return false; + } + } + + /// + /// Remove leftover "{outputPath}.{guid}.tmp" files, and the copy a promotion stages at + /// "{outputPath}.adopt". Called only when no resume is pending, where every such file is an + /// orphan by definition - except one a live run still holds, which the claim keeps out of + /// reach. "Such a file" is decided by name and not by the glob alone: the glob reaches a + /// longer output's temps too, and those describe work this run knows nothing about. The name + /// has to be one a run gives its own temp, 32 hex digits and all: anything looser is a + /// promise that no file mgx did not write is deleted, made about a directory the caller also + /// keeps their own "users.jsonl.backup.tmp" in. The staging name is exact and reached by + /// name, since no glob of this shape covers it. + /// + protected void DeleteStaleTemps(string outputPath) + { + // "Orphan" is an assumption about a file this run did not create, and a second export + // running against the same output right now owns a file matching the same glob. Windows + // refuses to delete a file someone holds open, so it declined by accident; Unix does + // not, and the other run went on writing into an unlinked inode and lost everything it + // had fetched. Ask for the file exclusively first - if that fails, someone is using it + // and it is not an orphan. + // + // And which of the two failures it was. A file another run holds is a file nothing is + // wrong with; a file this account cannot open read-write has no second run behind it at + // all, and one sentence for both sent the caller looking for a run that does not exist. + // + // Which of the second kind it was is the open's own answer and not this line's to guess: + // "the open was refused on permissions, not on sharing" was said of a directory, of a + // pipe with no length to write at and of a path too long for the volume, and a caller + // who went and granted write access to the file found nothing had changed. + void SweepIfUnheld(string stale, string deleted) + { + var claim = ClaimExclusively(stale, out var reason); + // A file that has gone between the directory listing and the claim is a file this + // sweep has nothing left to do about, and nothing to say either. + if (claim == FileClaim.Absent) return; + if (claim != FileClaim.Taken) + { + WriteVerbose(claim == FileClaim.Held + ? $"Left '{Path.GetFileName(stale)}' alone: another run is writing to it." + : $"Left '{Path.GetFileName(stale)}' alone: this run cannot open it for " + + $"writing - {reason?.Reason ?? "the open was refused"}."); + return; + } + try + { + File.Delete(stale); + WriteVerbose($"{deleted}: {Path.GetFileName(stale)}"); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + WriteWarning($"Could not delete orphaned temp file '{stale}': {ex.Message}. Delete it manually."); + } + } + + try + { + var dir = Path.GetDirectoryName(outputPath); + if (string.IsNullOrEmpty(dir) || !Directory.Exists(dir)) return; + var outputName = Path.GetFileName(outputPath); + foreach (var stale in Directory.EnumerateFiles(dir, outputName + ".*.tmp", + TempGlobOptions(dir, mayWrite: true)) + .Where(p => IsRunTempName(dir, outputName, Path.GetFileName(p), + mayWrite: true)) + .ToList()) + { + SweepIfUnheld(stale, + "Deleted an orphaned temp file from an earlier interrupted run"); + } + + // And the copy a promotion stages beside the output, which the glob above cannot + // reach: no run leaves a file under that name on purpose - it is renamed away where + // the promotion lands and removed where it stops - so one standing here is a run + // killed between those two, holding a partial copy of somebody's rows that nothing + // on disk refers to. + // + // Through the promotion's own sweep of that name, and not the claim-then-delete a + // temp takes. The claim answers for a regular file and for nothing else: a link at + // the name was followed to whatever it pointed at, a pipe and a socket came back + // unopenable and were left standing - reported as a file this account could not + // open - and the next promotion then met exactly the entry that sweep exists to + // clear. One primitive decides the name wherever a run reaches it. + var adoptPath = StagingPathFor(outputPath); + if (SweepStagingName(adoptPath, apply: true, out var adoptFailure, out var adoptSwept)) + { + if (adoptSwept is { } removed) WriteVerbose(removed.Sentence); + } + else + { + WriteVerbose($"Left '{Path.GetFileName(adoptPath)}' alone: " + + $"{adoptFailure!.Value.Reason}."); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // Best effort; a sweep failure must never stop the run. + } + } + + /// + /// The temp a checkpoint names, or null when it cannot be used. A checkpoint is untrusted + /// input once it is on disk, so the recorded name must be one a run could actually have + /// written - "{output}.{32-hex}.tmp" - and must not be the output itself. Anything else + /// (the checkpoint file, the delta state, a crafted path) would be copied into the output + /// as data and then deleted as the spent temp. The file must also be at least as long as + /// the checkpoint promised, since a shorter one means the items it counted are not all + /// there. + /// + private static string? ResolveNamedTemp(string outputPath, string tempFileName, long dataLength, + bool mayWrite) + { + if (dataLength <= 0) return null; + var dir = Path.GetDirectoryName(outputPath); + if (string.IsNullOrEmpty(dir) || !Directory.Exists(dir)) return null; + if (!string.Equals(tempFileName, Path.GetFileName(tempFileName), StringComparison.Ordinal)) + return null; + if (!IsRunTempName(dir, Path.GetFileName(outputPath), tempFileName, mayWrite)) return null; + var tempPath = Path.Combine(dir, tempFileName); + if (string.Equals(Path.GetFullPath(tempPath), Path.GetFullPath(outputPath), + StringComparison.OrdinalIgnoreCase)) + return null; + if (!File.Exists(tempPath)) return null; + if (new FileInfo(tempPath).Length < dataLength) return null; + return tempPath; + } + + /// + /// True when is a name a fresh run gives its temp: + /// the output's own name, a dot, 32 lowercase hex digits (Guid "N"), and ".tmp". Both names + /// live in , which is what decides whether a spelling that + /// differs only in case is the same name. + /// + protected static bool IsRunTempName(string directory, string outputFileName, string candidate, + bool mayWrite) + { + var comparison = FileNameComparisonIn(directory, mayWrite); + var prefix = outputFileName + "."; + const string suffix = ".tmp"; + if (candidate.Length != prefix.Length + 32 + suffix.Length) return false; + if (!candidate.StartsWith(prefix, comparison)) return false; + if (!candidate.EndsWith(suffix, comparison)) return false; + for (var i = prefix.Length; i < prefix.Length + 32; i++) + { + if (candidate[i] is (>= '0' and <= '9') or (>= 'a' and <= 'f')) continue; + return false; + } + return true; + } + + /// + /// Whether the file's last byte ends a line. Every whole row a run writes goes out through + /// WriteLine, which terminates it, so a file that does not end in a newline ends in a row + /// that was cut short - and only the last row can be, since every row before it is followed + /// by its own terminator. + /// + private static bool EndsWithNewline(string path) + { + using var fs = new FileStream(path, FileMode.Open, FileAccess.Read); + if (fs.Length == 0) return false; + fs.Seek(-1, SeekOrigin.End); + return fs.ReadByte() == '\n'; + } + + /// What promoting the temp a checkpoint names answered. + protected enum TempPromotion + { + /// + /// Nothing was promoted: the temp is absent, shorter than the checkpoint promised, or + /// still held by the run writing it. The items it counted are in no file this run can + /// reach, and the caller must not resume past them. + /// + NotPromoted, + + /// + /// The items are in the output now, and the output is this run's until the writing ends. + /// + Taken, + + /// + /// Another run has the output open. Nothing was promoted into it - the claim is asked + /// for before the move - so the temp, and the checkpoint naming it, are as they were + /// found. + /// + OutputHeld, + + /// + /// The output is not a file this run can write at the recorded offset, and nothing was + /// promoted into it either. + /// + OutputUnopenable, + + /// + /// The copy a promotion stages beside the output could not be written, or could not be + /// moved onto the output once it was: a directory or another run's handle at the staging + /// name, a full disk, a directory that turned read-only between the claim and the copy. + /// Nothing was replaced and nothing was removed - the temp holds every item the + /// checkpoint counts, and the checkpoint still names it - so this is not the temp that + /// went missing, which is the one answer a caller may delete a checkpoint over. + /// + StagingFailed, + } + + /// + /// Replace the output with the first bytes of a named temp, + /// take the output for the rest of the run, and then remove the temp. A fresh run that + /// finishes moves its temp over whatever the output held, so recovering an unfinished one + /// has to reach that same file; appending instead would leave the previous run's rows - + /// already consumed - in front of this one's. Unlike the glob-and-newest form, this takes + /// exactly the file the checkpoint recorded, so a leftover from an unrelated run cannot be + /// merged in. Bytes rather than lines: the length was taken from the writer's own position, + /// so it cannot disagree with itself about line endings or a torn final line. + /// + /// What follows a promotion is an append onto the file it landed in, which is the same thing + /// the resumed route does and takes the same hold. The output is asked for BEFORE the move + /// that replaces it, since a rename asks nothing of the lock on either side; + /// is what each platform's lock lets that be. + /// + /// + protected TempPromotion TryPromoteNamedTemp(string outputPath, string tempFileName, + long dataLength, out UnopenableReason? reason, out StagingFailure? staging, + out StagingRemoval? swept) + { + reason = null; + staging = null; + swept = null; + try + { + var tempPath = ResolveNamedTemp(outputPath, tempFileName, dataLength, mayWrite: true); + if (tempPath == null) return TempPromotion.NotPromoted; + + // The claim adoption and the stale-temp sweep take, for the reason they take it: + // this copies the file's rows into the output and then unlinks it, so a run that is + // still writing that temp went on writing into an unlinked inode, lost everything it + // had fetched, and had its rows handed over as this run's. Naming the file changes + // nothing about that - a checkpoint names the temp of the enumeration that wrote it, + // and that enumeration is either over, in which case nothing holds the file, or + // running right now, in which case this is its live output. Taken before anything + // reads the file, so both platforms decide it here rather than Windows failing the + // unlink afterwards with the rows already copied. + // + // And held from here through the move, rather than taken and let go before the copy. + // Two runs over one command line reach this line together, and a claim released + // before the copy is one the second run passes as well: both staged the same bytes, + // one of them lost the staged file's own claim, and the run that lost it read the + // temp as missing, deleted the position counting its items, and exported fresh over + // the output the other was holding and appending to. One claim over the whole + // promotion refuses the second run for as long as the first is promoting - and by + // the time it is let go the temp is gone and the output is this run's, which is what + // refuses it from there on. + FileStream? claimedTemp = null; + FileStream? standing = null; + FileStream? writer = null; + var adoptPath = StagingPathFor(outputPath); + var staged = false; + var createdOutput = false; + var landed = TempPromotion.NotPromoted; + try + { + // The staging name, cleared before either claim below is taken, and the reason + // nothing is ever opened through that name with a mode that follows it or waits + // on it. First, because the sweep asks for what stands at the name exclusively + // and a hard link there is an ALIAS for an inode this run is about to hold: run + // it under those claims and a link to the temp, or to the output, answered its + // own claim Held - the run stopped saying another run had the staged copy open, + // over a name nothing but itself was holding, while the preview holding neither + // deleted the link and answered that the promotion would land. Nothing of this + // promotion's is open here, so an alias for either file answers Taken and is + // unlinked as the alias it is, leaving what it pointed at alone. + // + // A failure here is its own answer, and not the temp's. Read as "the temp is + // missing or incomplete" - which is what the catch around this whole method used + // to make of a directory at the staging name, another run's handle on it or a + // full disk - the caller said the items were in no file at all, deleted the + // position counting them and swept the temp that was holding every one of them. + if (!SweepStagingName(adoptPath, apply: true, out staging, out swept)) + return landed = TempPromotion.StagingFailed; + + if (OpenExclusively(tempPath, FileShare.None, out claimedTemp, out _) + != FileClaim.Taken) + return TempPromotion.NotPromoted; + + // The output the move will replace, asked for before a byte of it is staged. + // Held or unopenable is the whole answer on either platform, and it is reached + // with the directory beside the output untouched: staging first and asking after + // wrote the whole temp out under a name of this run's - 172 ms and a moved + // directory mtime for a 256 MiB temp - to be taken straight back again. + // + // On Unix the handle is kept from here through the move, which is what closes + // the interval a claim released in front of the rename left open: rename(2) + // honors no lock on either side of it, so a second run that took the output in + // between had its rows replaced under it, in an unnamed inode, with no warning + // on any stream. Windows cannot keep it - it refuses to rename over a + // destination another handle has open - so there the claim is let go where it + // is taken and asked for again past the move. + // + // An output that is not there yet is created here in order to be held, and + // removed again by the finally on every answer but Taken: a promotion that + // stops leaves the directory as it found it, and one that lands replaced the + // file with the staged copy in the rename itself. + landed = ClaimStandingOutput(outputPath, out standing, out reason, + out createdOutput); + if (landed != TempPromotion.Taken) return landed; + + // Staged like the other forms, so the destination is replaced in one Move rather + // than truncated and refilled in place. Under the name the sweep above cleared, + // and created by an open that refuses to use anything already standing there: + // what appears at the name in the instants between the two is this run's staging + // failure and never its copy's destination. + try + { + // CreateNew, so the name is this run's own new entry or the open fails: + // nothing at it is followed, truncated or waited on. ReadWrite, because this + // handle is the hold the run keeps past the rename. FileShare.None from the + // instant the entry exists, so the copy is never a file a second promotion + // can claim, and the discard below can only ever unlink a file this handle + // holds. + writer = ScratchName.CreateNew(adoptPath); + staged = true; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // Something appeared at the name in the instants since the sweep, or the + // directory beside the output refuses a new file in it at all - a mode this + // account may not write, a volume with no room. Neither is a temp that has + // gone missing. + staging = StagingFailureAt(adoptPath, ex, fromCreate: true); + return landed = TempPromotion.StagingFailed; + } + try + { + var buffer = new byte[81920]; + long remaining = dataLength; + while (remaining > 0) + { + var read = claimedTemp!.Read(buffer, 0, + (int)Math.Min(buffer.Length, remaining)); + if (read <= 0) break; + writer.Write(buffer, 0, read); + remaining -= read; + } + writer.Flush(); + // Shorter than the length recorded for it, which is the one thing the + // resolve above measured and cannot have measured wrong twice: the temp has + // lost bytes since. Those items are not all in a file this run can reach, + // which is the caller's own route for a temp that is missing or short - the + // only route left to it, now that a staging failure answers for itself. + if (remaining > 0) return landed = TempPromotion.NotPromoted; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + staging = StagingFailureAt(adoptPath, ex); + return landed = TempPromotion.StagingFailed; + } + + landed = MoveOntoTheOutputAndKeepIt(writer, adoptPath, outputPath, dataLength, + out reason, out staging); + // The staged copy's handle is the run's hold on the output past a move that went + // through, and on Windows it was closed before the move. Either way it is no + // longer this frame's to close. + if (landed == TempPromotion.Taken) writer = null; + if (landed != TempPromotion.Taken) return landed; + + // Unlinked after the output was taken, and only where it was taken. A promotion + // that does not land leaves the temp, and the checkpoint still naming it, exactly + // as they were found: the position and the file it counts are the pair they were, + // and the run that comes back promotes the same bytes to the same length over the + // same output, which is the one answer here that costs nothing. + // + // The unlink is the finally below, under the claim this run has held the temp by + // since before it read a byte of it - on Windows past that handle instead, since + // a file this process has open cannot be unlinked there without + // FILE_SHARE_DELETE. Nothing is left open for a second run to pass by then: the + // output is this run's, and that is the claim that stops it. + return TempPromotion.Taken; + } + finally + { + // The empty output the standing claim made where there was none. It exists only + // so that an absent output can be refused to a second run, and a promotion that + // did not land has nothing to leave at the path: the run before it left nothing + // there, and a run that comes back to this checkpoint reaches the same branch. + // Taken back under the claim that made it, in the one order this platform takes + // a name back in - a name goes only while the run unlinking it holds what + // stands there, and a file that is no longer this claim's is one it leaves. + // + // Every other answer lets the handle go and unlinks nothing: past the move what + // it locks is an inode with no name left on it, and the file at the path is the + // one the adopt handle holds; on a stop it is let go with nothing replaced. + if (createdOutput && landed != TempPromotion.Taken) + ScratchName.Discard(outputPath, standing); + else standing?.Dispose(); + + // The temp, taken back under the claim it was read through, and only where the + // output was taken: a promotion that did not land leaves it and the checkpoint + // naming it exactly as it found them, which is what lets the next run promote + // the same bytes to the same length over the same output. + if (landed == TempPromotion.Taken) ScratchName.Discard(tempPath, claimedTemp); + else claimedTemp?.Dispose(); + + // Every answer but Taken leaves the disk as it was found, so the copy staged + // beside the output goes with the promotion that did not land: no glob of a + // later run reaches that name, and no run puts it there to keep. Guarded on + // this run having created it, which is now the only way the name can hold a + // file at all - the create refuses anything already there, and the handle it + // comes away with is held until the move - so what this unlinks is always the + // entry this run made and never a staging file a second promotion has open. + if (landed != TempPromotion.Taken && staged) DiscardStagedFile(adoptPath, writer); + else writer?.Dispose(); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return TempPromotion.NotPromoted; + } + } + + /// + /// The name a promotion stages its copy under: the output's own name with a suffix, in the + /// output's own directory, so the move that puts it at the output path is a rename within one + /// filesystem. No run leaves a file under this name on purpose - it is renamed away where the + /// promotion lands and removed where it stops - so one standing there is a run killed between + /// the two, and takes it with the temps. + /// + protected static string StagingPathFor(string outputPath) => outputPath + ".adopt"; + + /// + /// Clear the staging name, or say why the promotion stops there: + /// decides the entry and this puts its answer into the words a promotion says it in. The + /// kinds and the order they are decided in are that primitive's - the same one the + /// checkpoint and the delta state clear their own scratch names with - and the reason + /// nothing is ever opened through this name with a mode that follows it or waits on it: a + /// promotion that opened it Create/Write staged its copy into whatever stood there, taking + /// a symlink's target away with it and renaming the link onto the output path, and a FIFO + /// with no reader blocked the open forever with this run holding the temp and the output + /// and no way for StopProcessing to reach it. + /// + /// Run first in both passes - before the claim on the temp and the claim on the standing + /// output, and so before anything of this promotion's is open. That place in the order is + /// what lets the sweep's own claim answer at all: run under the promotion's claims, it + /// asked for an entry that may be a hard link to one of the two inodes those claims hold, + /// and a second open file description over a locked inode is refused even from this process + /// - so an alias for the temp, or for the output, read as a staged copy another run had + /// open. + /// + /// + /// + /// Whether the name is cleared or only read. The applying pass takes what stands there off + /// the name; the preview leaves it and reports what a run would take, because a preview + /// removes nothing - its deletions were as real as a run's, so a caller asking what would + /// be touched had a file of theirs deleted to be told that nothing would be. + /// + /// Why the promotion stops here, where it does; null where the name is + /// clear. One wording for one disk state on either pass: a run and a preview of it refuse a + /// directory, or another run's copy, in the same words. + /// What was taken off the name, or - on the preview - what stands there + /// and would be; null where nothing did. The sentence is the pass's own and the kind is the + /// same phrase in both, so a stop that names it and a warning that reports it agree. + private static bool SweepStagingName(string adoptPath, bool apply, + out StagingFailure? failure, out StagingRemoval? found) + { + failure = null; + found = null; + var name = Path.GetFileName(adoptPath); + var entry = apply + ? ScratchName.Sweep(adoptPath, out var stood) + : ScratchName.Inspect(adoptPath, out stood); + switch (entry) + { + case ScratchEntry.Clear: + return true; + case ScratchEntry.Directory: + failure = new StagingFailure(adoptPath, "a directory stands at the path"); + return false; + case ScratchEntry.Held: + failure = new StagingFailure(adoptPath, "another run has the staged copy open"); + return false; + case ScratchEntry.Unreadable: + // The entry could not be read, or could not be unlinked. Refined the way every + // other failed open here is refined, so one disk state is one sentence wherever + // the run met it. + failure = StagingFailureAt(adoptPath, stood!); + return false; + default: + var kind = StagingEntryKind(entry); + found = new StagingRemoval( + apply ? SweptFromStagingName(name, kind) : WouldSweepStagingName(name, kind), + kind); + return true; + } + } + + /// + /// What stood at the staging name, in the phrase every sentence about it uses: the warning a + /// run writes when it took the entry off the name, the -WhatIf line naming what a run would + /// take off it, and the stop that has to say the name was cleared before the staging failed. + /// One phrase per kind, because the same entry read three ways in three sentences is three + /// disk states as far as the reader can tell. + /// + private static string StagingEntryKind(ScratchEntry entry) => entry switch + { + ScratchEntry.Link => "a link, and not what it pointed at", + ScratchEntry.Leftover => "a copy left by an interrupted promotion", + // A regular file, and the one thing that is known about it is that this account could + // not open it. Read into the sentence below, a 0444 or a mode-000 file was reported as a + // pipe or a socket - a kind of entry it is not, and one the caller cannot have put there + // by the means they actually used. + ScratchEntry.Unopenable => "a copy this account cannot open", + _ => "an entry no copy could be staged in, such as a pipe or a socket", + }; + + /// + /// What a preview says about an entry at the staging name: what the run would take off it + /// and what it would put there instead. Said as a WARNING beside the gate's own line, since + /// the gate names one of two actions on -OutputFile and this is neither - and the create + /// test is skipped for that name, because nothing standing at it can be tested without + /// removing it and this pass removes nothing. + /// + private static string WouldSweepStagingName(string name, string kind) => + $"The run would remove what stands at the staging name '{name}' ({kind}) and stage its " + + "copy there."; + + /// + /// What a sweep of the staging name took off it, or what a preview found standing there: the + /// sentence the pass reports and the phrase another sentence names it by. + /// + protected readonly record struct StagingRemoval(string Sentence, string Kind); + + /// + /// What came off the staging name, in the one sentence both cmdlets say it in. Said as a + /// WARNING, since written to the verbose stream it reached nobody who had not asked for + /// verbose - and said once the run's own outcome for the recovery is known, because a + /// warning is a terminating error under -WarningAction Stop: written where the deletion + /// happens, it ended the run with the entry gone, the temp still holding the items and + /// nothing recovered. The verbose line stays at the deletion as the record of when it + /// happened, so -WarningAction SilentlyContinue does not hide it entirely. + /// + private static string SweptFromStagingName(string name, string kind) => + $"Removed what stood at the staging name '{name}': {kind}."; + + /// + /// Why a promotion could not put its copy at the output path, and where it was writing it. + /// The reason is 's, which is the same refinement the claim + /// on a file carries out of a failed open: a caller quotes it inside a sentence it + /// punctuates itself, and the two passes over one disk state have to answer in one wording. + /// + protected readonly record struct StagingFailure(string Path, string Reason); + + /// + /// Whether the open that failed was the create the copy is staged + /// with, whose refusal of an existing name is the runtime's "already exists" for every kind + /// of entry alike - so the refinement has to ask the filesystem what the entry is. + private static StagingFailure StagingFailureAt(string adoptPath, Exception ex, + bool fromCreate = false) => + new(adoptPath, ReasonForUnopenable(adoptPath, ex, fromCreate).Reason); + + /// + /// Put the staged file at the output path and come away holding what is now there, or replace + /// nothing and say why. A promotion is followed by an append onto the file it landed in, so + /// that file has to be this run's from the move onward - and the output standing at the path + /// has to be nobody else's before the move, because a rename asks nothing of the lock on + /// either side of it. + /// + /// On Unix .NET backs with an advisory flock, which belongs to + /// the open file description, and rename(2) moves a directory entry: so the lock the + /// staged copy was created under is still held over the same inode once that inode is the + /// output. The hold therefore travels with the move, and it is the handle the copy was + /// written through - the one the create came away with, never let go and never asked for a + /// second time. Measured on this host: a FileShare.None open of the moved-to path, from this + /// process and from another, is refused while that handle is open, which is also why the + /// output is not asked for again afterwards - flock refuses a second open file description + /// from this very process. + /// + /// + /// The claim on the output the move replaces is 's, taken by + /// the caller before it staged anything and still standing here. Asking for it and letting it + /// go in front of the rename, which is what this did, left the interval between the two + /// instructions open: rename(2) does not honor the lock on the destination either, so a + /// second run that took the output in that interval had it replaced under it - its rows in an + /// unlinked inode, and no warning on any stream. Kept across the move instead, the claim + /// refuses that run for as long as the promotion lasts, and past the move the handle this run + /// has open IS the output. That holds on both branches of the claim: an output that is not + /// there is created by it in order to be held, so a promotion over an absent output refuses + /// a second one the same way a promotion over an existing output does. + /// + /// + /// Windows will not rename over a destination another handle has open, nor rename a source + /// this process has open, so neither hold can travel with the move there. The order this had + /// stands: claim the standing output and let it go, stage, close the copy, move, take the + /// output on the far side. What that leaves is a window two statements wide rather than a + /// page fetch wide, and an output already replaced where the claim on the far side fails. + /// + /// + private TempPromotion MoveOntoTheOutputAndKeepIt(FileStream staged, string adoptPath, + string outputPath, long dataLength, out UnopenableReason? reason, + out StagingFailure? staging) + { + reason = null; + staging = null; + if (OperatingSystem.IsWindows()) + { + // Closed before the move, which is the order that platform allows: it will not + // rename a source this process has open. What that leaves is a name unheld for the + // width of the two instructions, where every other platform holds it from the + // create through the rename. + staged.Dispose(); + if (!TryMoveStagedFile(adoptPath, outputPath, out staging)) + return TempPromotion.StagingFailed; + return TakeOutputForCheckpoint(outputPath, dataLength, out reason) switch + { + CheckpointOutput.Taken => TempPromotion.Taken, + CheckpointOutput.HeldByAnotherRun => TempPromotion.OutputHeld, + CheckpointOutput.CannotBeOpened => TempPromotion.OutputUnopenable, + // The file this run wrote a moment ago is gone, or shorter than it wrote it. + // Something else has replaced it, so those rows are in no file again and the + // caller's own route for a promotion that did not land is the answer. + _ => TempPromotion.NotPromoted, + }; + } + + // The handle the copy was written through, still open, so the lock travels with the + // inode through the rename. Asking for the name again here - which is what this did - + // let go of it between the write and the claim: a second promotion took the copy in + // that interval and had it unlinked under its handle by the run that wrote it. There is + // no interval now, and no second claim to be refused: the name has been this run's + // since the create made it. + if (!TryMoveStagedFile(adoptPath, outputPath, out staging)) + return TempPromotion.StagingFailed; + + // The handle names the output now, and the run keeps it until its writing ends. Its + // length is the length just written, which is what the caller repoints the checkpoint + // at; nothing reads the path it was opened under, which is a name no longer on disk. + ReleaseOutputHold(); + _outputHold = new OutputHold(staged); + return TempPromotion.Taken; + } + + /// + /// The rename itself, and whether it went through. Everything up to it leaves the disk as it + /// was found, so a move that fails has replaced nothing either - and saying so is the answer, + /// where the catch around the whole promotion would have called it a temp that had gone. + /// + private static bool TryMoveStagedFile(string adoptPath, string outputPath, + out StagingFailure? staging) + { + staging = null; + try + { + File.Move(adoptPath, outputPath, overwrite: true); + return true; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + staging = StagingFailureAt(adoptPath, ex); + return false; + } + } + + /// + /// Claim the output a promotion is about to replace and, on Unix, come away holding it. A + /// rename asks nothing of the lock on either side of it, so the interval between letting this + /// claim go and the move is one a second run can take the output in - and it is measured in + /// instructions only until something widens it. The handle is therefore kept from before the + /// staging through the rename, and let go on the far side, where what it locks is an inode + /// with no name left on it. + /// + /// Windows keeps nothing: it refuses to rename over a destination another handle has open, so + /// a claim held here would fail the move itself. There the claim is let go where it is taken + /// and the output is asked for again past the move, which is the order that platform's lock + /// allows. + /// + /// + /// The file's length is no part of the question, which is what tells this from + /// , whose open this otherwise is. No caller of + /// this one can read anything from a length. A promotion replaces the output whatever length + /// it has. And a checkpoint naming a temp that has gone says its items were in that temp, so + /// an output nothing holds is evidence of nothing at all: two rows of one shape are two rows + /// of another shape's length, so a previous run's file measures exactly what a promotion + /// would have left about as often as a promoted one does. + /// + /// + /// FileMode.OpenOrCreate, so the claim is taken on the one branch it used to have nothing to + /// hold. An output that is not there is the state a fresh run's promotion is the ordinary end + /// of, and FileMode.Open answered that Absent with no handle: nothing was held through the + /// staging or the rename, so two runs with different temps over one output path both got + /// past here, both renamed a copy onto that path, and the first one's rows ended in an + /// unlinked inode with its own handle appending into nothing on any disk. The claim makes + /// the file in order to hold it, which refuses the second run exactly as an existing output + /// would - and comes back, so a promotion that does not land can + /// take away again the empty file it made. + /// + /// + /// + /// Whether the file at the path is this claim's own, made by the open because nothing stood + /// there. The claim answers it, since OpenOrCreate never reports an absent file and the + /// claim is the one that knows which of its attempts came away with the handle: a claim + /// asked again over a name that moved is a claim on a file it did not make, and a file a + /// second mgx run put there between the question and the open is one that run's own claim + /// refuses. What is not told apart is a file nothing holds that appeared in that window - a + /// shell's touch, an older mgx, any writer that takes no claim - which this claim opens, + /// reports here as its own, and takes back again where the promotion stops. + /// + /// The claim, still open, for the caller to let go past the move; + /// null on Windows and where the open itself was refused. Open where the claim was granted + /// and could not be verified, for the finally to take back what the attempt made. Where + /// is true this is a handle on an empty file of this run's + /// making. + private static TempPromotion ClaimStandingOutput(string outputPath, out FileStream? standing, + out UnopenableReason? reason, out bool created) + { + if (OperatingSystem.IsWindows()) + { + // Nothing is made here, because nothing is kept: this platform lets the claim go + // where it takes it, so a file created to hold would be released in the same breath + // and refuse no second run - an entry at the caller's output path for no gain. The + // window past the move is that platform's, as it was. + created = false; + var released = OpenExclusively(outputPath, OutputHoldShare, out standing, out reason); + standing?.Dispose(); + standing = null; + return PromotionForStandingClaim(released); + } + + // OpenOrCreate, and the claim makes the file only where nothing stands at the name and + // no link stands there either: a link is followed by a create, so a dangling one would + // put this run's empty file wherever it points - outside the output's own directory - + // and the move that follows replaces the link itself, which leaves that file behind. + // That state is read off the directory entry, without opening it, and read again by + // every attempt the claim makes. + var claim = OpenExclusively(outputPath, OutputHoldShare, out standing, out reason, + FileMode.OpenOrCreate, out created, out var askedToMakeIt); + + // The output is not there and could not be made, so there is nothing held and nothing + // made to take back - and nothing about a second run to report either. What refused it + // is the directory beside it, and what a run does about a directory it cannot put a file + // in is stop on the staging create, in that same directory: the one question both passes + // ask, and the one wording they both answer in. So the answer is left to it rather than + // named here in words the preview has no way to reach. + // + // Read off the attempt that gave the answer, and not off a second look from here. The + // claim decides per attempt whether it is making the file: a name a first look found a + // file at is a create the directory refused, once that file leaves the name in between, + // and a name a create was refused at can have a file standing at it a line later. A + // look from here reads each of those as the other. + // + // And on the create having been refused, which is what leaves nothing held: a create the + // claim was granted and then could not verify is Unopenable off the same attempt, with + // the file made and this run holding it. Left to the staging create, that one would go on + // to rename a copy onto an output no run here can say is still the one it claimed. + if (claim == FileClaim.Unopenable && askedToMakeIt && standing is null) + { + reason = null; + return TempPromotion.Taken; + } + + return PromotionForStandingClaim(claim); + } + + /// + /// + /// The same question, let go again with nothing taken, nothing cut and nothing made, for the + /// callers that only have to report on the file. FileMode.Open here and not the claim's + /// OpenOrCreate: a pass that holds nothing has no second run to refuse, and an output that is + /// not there is one the promotion creates - which is the answer this gives it, and it gives + /// it without leaving a file of its own at the caller's output path. Whether that directory + /// will take a new file at all is a different question, and the staging create asks it in + /// the same directory on both passes. + /// + private static TempPromotion StandingOutputRefusal(string outputPath, + out UnopenableReason? reason) + { + var claim = OpenExclusively(outputPath, OutputHoldShare, out var standing, out reason); + standing?.Dispose(); + return PromotionForStandingClaim(claim); + } + + /// + /// What a claim on the output a promotion would replace means for the promotion. Absent is + /// not among the answers, and for two different reasons: the claim creates the file rather + /// than report it missing, and the report-only form reads a missing output as one the + /// promotion would make. + /// + private static TempPromotion PromotionForStandingClaim(FileClaim claim) => claim switch + { + FileClaim.Held => TempPromotion.OutputHeld, + FileClaim.Unopenable => TempPromotion.OutputUnopenable, + _ => TempPromotion.Taken, + }; + + /// + /// The same answer in the words the two cmdlets build a stop out of. + protected static ClaimRefusal RefusalForStandingOutput(string outputPath, + out UnopenableReason? reason) => + StandingOutputRefusal(outputPath, out reason) switch + { + TempPromotion.OutputHeld => ClaimRefusal.AnotherRunHasIt, + TempPromotion.OutputUnopenable => ClaimRefusal.CannotBeOpened, + _ => ClaimRefusal.None, + }; + + /// + /// Undo the staging. A promotion that stops before its move replaced nothing, and the file it + /// had written beside the output goes with it: no glob of a later run reaches that name, and + /// no run put it there to keep. + /// + /// Unlinked under the handle that holds it, which is what closes the interval between the + /// two: released first, the name was decided and unheld for as long as the discard took, and + /// a second run that swept it in that interval created its own file at the name and had it + /// unlinked from under the handle it was holding it by. Windows keeps dispose-then-delete, + /// since a file this process has open cannot be unlinked there without FILE_SHARE_DELETE. + /// + /// + /// This run's own handle on the entry, disposed here; null where it has + /// none. + private static void DiscardStagedFile(string adoptPath, FileStream? held) => + ScratchName.Discard(adoptPath, held); + + /// + /// Point the checkpoint at the output this run now holds, and save it there. The items are in + /// that file, so a second interruption must not promote the same temp again - and the file it + /// would promote is already gone. + /// + /// The length is the held stream's, read off the handle this run has open rather than stat'd + /// off the name a second time. They are the same number where nothing has moved; where + /// something has, the handle is the file this run wrote and the name is whatever stands there + /// now. + /// + /// + protected void RepointCheckpointAtHeldOutput(PaginationCheckpoint checkpoint, + string checkpointPath, string outputPath, OutputHold hold) + { + try + { + checkpoint.TempFile = null; + checkpoint.OutputFile = Path.GetFullPath(outputPath); + checkpoint.DataLength = hold.Length; + checkpoint.Save(checkpointPath); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + WriteWarning($"Checkpoint save failed after recovery: {ex.Message}"); + } + } + + /// + /// What a second opener of the output may still do while a resumed run holds it, which is + /// not the same question on the two platforms because the lock is not the same lock. + /// + /// .NET on Unix backs FileShare with an advisory flock: + /// takes LOCK_EX, every other mode takes LOCK_SH, and LOCK_SH does not refuse a second + /// writer - measured on this host, where a second FileStream opened Append/Write/Read + /// against a LOCK_SH holder succeeds. LOCK_EX is therefore the only mode that makes a second + /// run's claim fail, and it refuses every .NET opener including a reader; a reader that does + /// not take a lock at all, such as tail or cat, reads the file as before. + /// + /// + /// Windows needs no such trade. The handle is opened for writing, so a second writer is + /// refused by its access rather than by its sharing, and is + /// what lets a reader keep looking at the file for the length of the run. + /// + /// + private static FileShare OutputHoldShare => + OperatingSystem.IsWindows() ? FileShare.Read : FileShare.None; + + /// + /// The output of a resumed run, open, cut back to the length its checkpoint records, and + /// this run's until the writing ends. The claim on that file and the handle the writing goes + /// through are one handle: taken and let go a page-fetch apart, they left a window in which + /// a second run over the same command line passed the same claim, cut the same file back and + /// appended to it - nine lines where seven belonged, and in a sync a delta token advanced + /// past the difference. + /// + protected sealed class OutputHold : IDisposable + { + private readonly FileStream _stream; + + internal OutputHold(FileStream stream) => _stream = stream; + + /// The file's length, read off the open handle rather than the directory entry. + internal long Length => _stream.Length; + + /// + /// Cut the file back to the length the checkpoint recorded, dropping whatever the + /// interrupted run wrote after its last save. Those items are re-fetched, so dropping + /// them is what keeps a resume from writing them twice. + /// + internal void TrimTo(long dataLength) => _stream.SetLength(dataLength); + + /// + /// A writer onto the end of the same handle. Never a second open of the output: on Unix + /// the hold's own LOCK_EX refuses it - flock belongs to the open file description, so + /// even this process is refused - and on Windows the hold's write access does. + /// + internal StreamWriter AppendingWriter() + { + _stream.Position = _stream.Length; + // What StreamWriter(path, append) would have used: UTF-8 with no byte-order mark, + // throwing on unpaired surrogates, over a 1 KiB buffer. leaveOpen keeps the hold + // standing when the writer closes - the run has a checkpoint to delete and a + // summary to write before the file is anyone else's. + return new StreamWriter(_stream, new UTF8Encoding(false, true), 1024, leaveOpen: true); + } + + public void Dispose() => _stream.Dispose(); + } + + /// + /// The hold this run has on the output its own checkpoint records items into, from the + /// reconcile that took it until the writing ends. Null on every other run: a fresh export + /// collects into a temp of its own and promotes it, and a run with nothing to resume has no + /// output to keep. + /// + private OutputHold? _outputHold; + + /// The hold, where this run has one. + protected OutputHold? HeldOutput => _outputHold; + + /// + /// Let the output go. Called where the writing ends, on every way out of the attempt loop, + /// and again from by way of + /// - so a run that ends between the reconcile and the loop, on a + /// missing Graph connection or a pipeline stop, releases it too. Idempotent. + /// + protected void ReleaseOutputHold() + { + var hold = Interlocked.Exchange(ref _outputHold, null); + hold?.Dispose(); + } + + /// + /// Releases the output hold when the enclosing scope ends, whichever way it ends: the + /// summary object, a reported error, a cancellation, or an exception on its way out of the + /// attempt loop. Declared with using above the loop rather than written as a finally + /// around it, which is the same guarantee without moving five hundred lines sideways. + /// + protected readonly struct OutputHoldScope(MgxCmdletBase cmdlet) : IDisposable + { + public void Dispose() => cmdlet.ReleaseOutputHold(); + } + + /// + protected OutputHoldScope ReleasingOutputWhenWritingEnds() => new(this); + + /// + /// What this run may do with the output its own checkpoint records its items into: take it, + /// stop, or start over. One open answers both halves of the question - whether the file is + /// this run's to write and whether it is the file the checkpoint describes - because they + /// are one file and one handle. Asking them separately, with a claim let go in between, + /// answered a directory entry twice and a file's contents never: File.Exists is false + /// for a file in a directory this account cannot search, so a whole unreadable output was + /// reported as no longer holding the items it held and the position counting them deleted. + /// + protected enum CheckpointOutput + { + /// + /// It is the file the checkpoint describes, it now ends exactly at the recorded length, + /// and this run holds it for the rest of the writing. + /// + Taken, + + /// Another run has it open. + HeldByAnotherRun, + + /// + /// It is not a file this run can write at the recorded offset - mode bits, a denying + /// ACL, a read-only filesystem, a directory standing at the path, a symlink loop, or a + /// FIFO with no offset to write at. Nothing here knows whether the items are in it, so + /// the caller stops rather than report a file it never read as a file that has lost its + /// contents. + /// + CannotBeOpened, + + /// + /// It is not the file the checkpoint describes: absent, shorter than the recorded + /// length, or recorded under a length that is not one. The items counted are in no file, + /// which is what the callers delete the checkpoint over and enumerate again from the + /// beginning. + /// + NotTheRecordedFile, + } + + /// + /// Take the output for the rest of this run and cut it back to the length its checkpoint + /// records. The handle is kept in where the answer is + /// , and released by + /// where the run stops appending to it. + /// + protected CheckpointOutput TakeOutputForCheckpoint(string outputPath, long dataLength, + out UnopenableReason? reason) + { + // A checkpoint is untrusted input once it is on disk, and SetLength rejects a negative + // length with an ArgumentOutOfRangeException the catch below does not cover - so a + // hand-edited length escaped as a terminating error naming neither the checkpoint nor + // the file, and left itself on disk to fail the same way next run. ResolveNamedTemp + // already guards its own length; this is the same guard, and it is above the open + // because a length that is not one describes no file worth opening. + reason = null; + if (dataLength < 0) return CheckpointOutput.NotTheRecordedFile; + + var claim = OpenExclusively(outputPath, OutputHoldShare, out var stream, out reason); + if (claim != FileClaim.Taken) + { + // An answer that is not Taken can still carry a handle: a claim granted over the + // file and then refused because the question that finishes it could not be answered + // comes back Unopenable with the file open, so that the caller lets go of it the way + // it lets go of one it kept. Nothing is kept here, so it goes here. + stream?.Dispose(); + return claim switch + { + FileClaim.Held => CheckpointOutput.HeldByAnotherRun, + FileClaim.Unopenable => CheckpointOutput.CannotBeOpened, + _ => CheckpointOutput.NotTheRecordedFile, + }; + } + + var hold = new OutputHold(stream!); + try + { + // Off the handle, not the directory entry: the file this run has is the file it + // measures, and no second stat can disagree with it. + if (hold.Length < dataLength) + { + hold.Dispose(); + return CheckpointOutput.NotTheRecordedFile; + } + if (hold.Length > dataLength) hold.TrimTo(dataLength); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // The cut failed on a handle this run holds, so nothing else is writing the file and + // the account can open it. Whatever the filesystem is refusing, this run has not + // read the file's contents and may not say the items are gone from it. The open + // itself did not fail, so OpenExclusively left no reason above - this is the one + // Unopenable a caught exception names rather than a fixed phrase or ex is + // UnauthorizedAccessException, since a permission that blocks only a truncate and + // not the open that preceded it is not itself the more common shape. + hold.Dispose(); + reason = new UnopenableReason(FirstLine(ex.Message).TrimEnd('.'), + ex is UnauthorizedAccessException); + return CheckpointOutput.CannotBeOpened; + } + + ReleaseOutputHold(); + _outputHold = hold; + return CheckpointOutput.Taken; + } + + /// + /// The same answer without taking anything: the open is made and let go, and nothing is cut. + /// The export's ShouldProcess gate has to name the action a real run would take before the + /// run is allowed to take any, and -WhatIf has to leave every file exactly as it found it - + /// including its length. + /// + protected static CheckpointOutput WouldTakeOutputForCheckpoint(string outputPath, + long dataLength, out UnopenableReason? reason) + { + reason = null; + if (dataLength < 0) return CheckpointOutput.NotTheRecordedFile; + + var claim = OpenExclusively(outputPath, OutputHoldShare, out var stream, out reason); + using (stream) + { + return claim switch + { + FileClaim.Held => CheckpointOutput.HeldByAnotherRun, + FileClaim.Unopenable => CheckpointOutput.CannotBeOpened, + FileClaim.Absent => CheckpointOutput.NotTheRecordedFile, + _ => stream!.Length >= dataLength + ? CheckpointOutput.Taken + : CheckpointOutput.NotTheRecordedFile, + }; + } + } + + /// + /// The refusal half of , for the two callers that have to say + /// which file a stop is about and why. + /// + protected static ClaimRefusal RefusalFrom(CheckpointOutput taken) => taken switch + { + CheckpointOutput.HeldByAnotherRun => ClaimRefusal.AnotherRunHasIt, + CheckpointOutput.CannotBeOpened => ClaimRefusal.CannotBeOpened, + _ => ClaimRefusal.None, + }; + + /// + /// What TryPromoteNamedTemp would answer, without promoting anything. The ShouldProcess gate + /// has to name the action a real run would take before the run is allowed to take any, and + /// -WhatIf has to leave every staged file exactly as it found it. The claim on the temp is + /// asked for here too, and released again: a temp another run holds is one a real run would + /// refuse, so answering from the file's existence alone reported a recovery that would not + /// have happened. Which of the two ways that claim failed is not reported - a temp another + /// run holds and a temp this account cannot open are both temps a real run does not promote, + /// and there is one answer to give the gate either way. The caller that has to tell them + /// apart in a sentence asks . + /// + /// The output the promotion would land in is asked about as well, the way the applying pass + /// asks and with nothing taken and nothing cut. A promotion over an output another run holds + /// stops the run, which is neither of the two actions the gate names, and a preview that + /// never asked reported an append over a file the run would not have reached. + /// + /// + /// The staging name is the one place a real run is not read-only, and the preview reads it + /// anyway: asking only whether something already stands there previewed a recovery over an + /// output directory this account cannot write in at all. Where the name is clear, the + /// preview makes the same create a real run makes, at no length and removed again under the + /// handle that made it, so the two passes fail on the same open for the same reason. Where + /// something already stands at the name, the preview leaves it alone: sweeping it would + /// make deletions as real as a run's, so a caller asking what would be touched would have + /// had a link, a pipe or a leftover copy of theirs deleted just to be told that nothing + /// would be. What stands there comes back instead, for the caller to warn about. What it + /// leaves behind is what it found: the two files the checkpoint stands for are untouched, + /// and the staging name holds what it always held. + /// + /// + protected static TempPromotion CanPromoteNamedTemp(string outputPath, string tempFileName, + long dataLength, out UnopenableReason? reason, out StagingFailure? staging, + out StagingRemoval? swept) + { + reason = null; + staging = null; + swept = null; + try + { + var tempPath = ResolveNamedTemp(outputPath, tempFileName, dataLength, mayWrite: false); + if (tempPath == null) return TempPromotion.NotPromoted; + + // The name the copy is staged under, read in the same place in the order a real run + // sweeps it: first, before either file the checkpoint stands for is asked about, so + // a hard link at the name is an alias nothing of this pass's holds. The gate names + // the action a run would take, and a run that would stop on its own staging was + // previewed as a recovery: a directory there, or another run's copy, is refused in + // the run and refused here in the same words. + // + // Read and not swept. A preview that cleared the name made deletions as real as a + // run's - a caller asking what would be touched had a link, a pipe or a leftover + // copy of theirs deleted in order to be told that nothing would be, and on -WhatIf + // over a checkpoint the run would then have stopped on. What stands there comes back + // instead, for the caller to say what the run would take off the name. + var adoptPath = StagingPathFor(outputPath); + if (!SweepStagingName(adoptPath, apply: false, out staging, out swept)) + return TempPromotion.StagingFailed; + + if (!CanTakeExclusively(tempPath)) return TempPromotion.NotPromoted; + + var standing = StandingOutputRefusal(outputPath, out reason); + if (standing != TempPromotion.Taken) return standing; + + // And the create itself, at no length and taken back again, where the name is clear. + // Claiming the name was not the question a run answers there: the create is what a + // directory this account may not write in refuses, and asking only whether something + // already stands at the name previewed a recovery over an output the run could never + // have staged a copy beside. Made with the mode the run makes it with, so the two + // passes fail on the same open for the same reason. + // + // Skipped where something stands at the name: the run would remove that entry and + // create its own, and there is no way to make the same create without taking the + // entry off the name first. The caller reports what the run would remove instead of + // this pass removing it. + if (swept == null) + { + FileStream probe; + try + { + probe = ScratchName.CreateNew(adoptPath); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + staging = StagingFailureAt(adoptPath, ex, fromCreate: true); + return TempPromotion.StagingFailed; + } + + // The entry this pass made is the only thing in the directory that was not there + // before it, and a preview leaves the directory as it found it. Unlinked under + // the handle that made it, so no second run can take the name in between. + DiscardStagedFile(adoptPath, probe); + } + + return TempPromotion.Taken; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return TempPromotion.NotPromoted; + } + } + + /// + /// Why one of the two files a checkpoint stands for - the temp it names, or the output it + /// records its items into - is not this run's to take, where it is not. + /// + protected enum ClaimRefusal + { + /// + /// Nothing refuses it: there is no such file of this checkpoint's, or nothing stands + /// between this run and it - which, for a temp whose promotion has already failed, + /// means the items it counted are not all in a file this run can reach. + /// + None, + + /// Another run has it open. + AnotherRunHasIt, + + /// This account cannot open it read-write. + CannotBeOpened, + } + + /// Which of the two files a checkpoint stands for a stop is about. + protected enum CheckpointFile + { + /// + /// The temp it names, holding the items it counts while the output still holds an + /// earlier run's. + /// + NamedTemp, + + /// + /// The output it records its items into, which a run that has already resumed writes + /// straight to - so its checkpoint names no temp at all. + /// + Output, + } + + /// + /// How this run reached a checkpoint's output: appending straight into it, promoting a + /// named temp over it, or exporting fresh after the named temp it counted on had already + /// vanished. A stop over the output reads differently on each - where the checkpoint's + /// items are, and what going on would have done to the file - so a caller building the + /// sentence needs to be told which one it is on rather than assuming the appending shape, + /// which used to be the only one that reached a temp-based checkpoint's own claim on the + /// output at all. + /// + protected enum CheckpointStopRoute + { + /// + /// The checkpoint names no temp: a run that had already resumed was appending straight + /// into the output, so the items it counts are in that file, and going on would cut it + /// back to the recorded length under the run still writing past it. + /// + Appending, + + /// + /// The checkpoint names a temp that is still there: the items it counts are in that + /// temp, and going on would promote them, replacing the output outright rather than + /// appending to it. + /// + Promoting, + + /// + /// The checkpoint names a temp that is gone: the items it counted are recoverable from + /// no file this run can reach, and going on would enumerate the collection again from + /// the beginning and replace the output with that result. + /// + FreshAfterVanishedTemp, + } + + /// + /// Why a claim answered , carried out of + /// with the verdict so a caller building a stop can name the + /// open's own reason instead of guessing at one. Built by + /// , which is also what a promotion's staging failure is + /// refined by, so one disk state reads the same whichever site reached it. + /// carries no sentence-ending period of its own - taken off where the + /// runtime's words are read - since a caller quotes it inside a sentence it punctuates + /// itself. is read off which failure produced + /// , not out of its text: the text is the runtime's own words, in + /// whatever language this host reports errors in, and is not this run's to parse for a + /// cause it already knows structurally. + /// + /// and are the two causes worth + /// telling from a permission on the file, because they are the two with a different way out. + /// A directory standing at the output path opens as an access failure on both platforms - + /// EISDIR and a denying ACL arrive in the same catch - and no grant makes a directory + /// writable at an offset: what the caller does about it is remove it or point -OutputFile + /// somewhere else. A file whose parent cannot be searched is an access failure naming the + /// file, and the grant that opens it belongs to the directory - so the advice that named the + /// file sent the caller to change a mode that was never the one refusing them. Both are read + /// off the filesystem rather than the errno, since which failure an open produces depends on + /// the mode standing in the way. + /// + /// + protected readonly record struct UnopenableReason(string Reason, bool IsPermission, + bool IsDirectory = false, bool IsUnsearchableParent = false); + + /// + /// Which of the two files a claim refused, and why, or + /// where neither did. The file and the reason decide what the run can honestly say: the + /// file settles what going on would do to it, and the reason settles whether there is a + /// second run to wait for or a mode to change. and + /// refine that for an output refusal - which of the three ways this + /// run reached the output, and, where the claim failed on a mode rather than a hold, the + /// open's own explanation - and are not read for a temp refusal, which has never had more + /// than one route to it. + /// + protected readonly record struct CheckpointFileRefusal(CheckpointFile File, + ClaimRefusal Refusal, CheckpointStopRoute? Route = null, UnopenableReason? Reason = null) + { + /// Nothing refused either file, and the run goes on. + public static CheckpointFileRefusal None => new(CheckpointFile.NamedTemp, ClaimRefusal.None); + + /// Whether a file was refused at all. + public bool Refuses => Refusal != ClaimRefusal.None; + + /// Whether the reason was another run holding it, rather than the mode on it. + public bool Held => Refusal == ClaimRefusal.AnotherRunHasIt; + } + + /// + /// Why the temp a checkpoint names could not be claimed. Two of the answers leave both + /// files exactly where they are, and they are not the same thing to tell the caller. + /// + /// Another run having it open is the single reason a promotion fails that says nothing is + /// wrong with the files: every row the checkpoint counted is in that temp, and the + /// enumeration that counted them is still collecting into it - so the checkpoint is the + /// position a running export resumes from, and deleting it, or reporting its items as not + /// on disk, describes a run that has not failed and takes away what it would have come + /// back to. + /// + /// A temp this account cannot open read-write reaches the same refusal and has no second + /// run behind it at all. Said in the other's words, it sent the reader looking for an + /// export that was never there rather than at the file's owner and mode. + /// + /// The remaining failures - no temp, a temp shorter than the recorded length - mean those + /// items are in no file, which is what the callers delete the checkpoint over and + /// re-enumerate. + /// + protected static ClaimRefusal RefusalForNamedTemp( + string outputPath, string tempFileName, long dataLength, bool mayWrite) + { + try + { + var tempPath = ResolveNamedTemp(outputPath, tempFileName, dataLength, mayWrite); + if (tempPath == null) return ClaimRefusal.None; + return ClaimExclusively(tempPath, out _) switch + { + FileClaim.Held => ClaimRefusal.AnotherRunHasIt, + FileClaim.Unopenable => ClaimRefusal.CannotBeOpened, + _ => ClaimRefusal.None, + }; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return ClaimRefusal.None; + } + } + + /// + /// Why the checkpoint at -CheckpointPath is not this run's to delete. Every refusal that + /// leaves the file standing records this, and the doors that would otherwise delete it - + /// the completion path, and the sync's 410 - read it and nothing else. A run's own saved + /// position, and a checkpoint refused because the request was rebuilt without a query + /// option the endpoint would not answer, record none: those the doors take with the run. + /// A checkpoint one of whose files - the temp it names, or the output it appends to - could + /// not be claimed reaches no door, because the run that found that out did not go on. + /// + protected enum SparedCheckpoint + { + /// + /// The ownership verdict: the file records another output, or an enumeration nothing + /// beside this run's output corroborates. Whatever wrote it resumes from exactly that. + /// + NotThisRuns, + } + + /// + /// What a door says when it leaves the checkpoint where it was. One reading reaches a door + /// at all: a run that cannot claim a file its own checkpoint stands for stops where it finds + /// out, so no door of a run that got as far as finishing is answering for that file. + /// + protected static string SparedCheckpointVerbose(string occasion) => + $"Left the resume checkpoint alone ({occasion}): it records an enumeration this " + + "run refused as not its own and never wrote over."; + + /// + /// The sentence a stop ends with, and the one a -WhatIf pass over the same condition ends + /// with instead. Both cmdlets refuse a checkpoint whose temp, or whose output, is held or + /// unopenable by ending the run there, and the difference between doing that and reporting + /// that it would be done is a tense. + /// + protected const string RunStopsHere = "This run stops here; nothing was written."; + + /// + protected const string RunWouldStopHere = "This run would stop here; nothing would be written."; + + /// + /// The same for a staging failure, where "nothing was written" is not what the run has to + /// say: the copy it was writing beside the output is gone again, and what the reader needs + /// to hear is that neither of the two files the checkpoint stands for moved. + /// + protected const string NothingWasChanged = "Nothing was changed;"; + + /// + protected const string NothingWouldBeChanged = "Nothing would be changed;"; + + /// + /// The newest "{output}.{32-hex}.tmp" beside the output that nothing else holds open, or + /// null when there is none. Adoption copies the file it picks into the output and then + /// unlinks it, so a temp a second export is writing into right now must not be picked at + /// all: that run went on writing into an unlinked inode, lost everything it had fetched, + /// and its rows were handed over as this export's. The claim is the one DeleteStaleTemps + /// takes before it deletes, and it settles the same question - FileShare.None holds + /// between processes on Windows and Unix alike, so a run with its own temp open makes it + /// fail, and a file nobody can be found holding is an orphan. A candidate that cannot be + /// claimed is passed over rather than ending the search: newest-first reaches a live run's + /// temp before an older one, and the next candidate down may be the orphan recovery exists + /// for. Taking the claim before anything reads the file also keeps Windows declining here, + /// where a live writer already conflicts with the reads below rather than with this. + /// + private static FileInfo? NewestUnheldTemp(string outputPath, bool mayWrite) + { + var dir = Path.GetDirectoryName(outputPath); + if (string.IsNullOrEmpty(dir) || !Directory.Exists(dir)) return null; + var outputName = Path.GetFileName(outputPath); + return Directory.EnumerateFiles(dir, outputName + ".*.tmp", TempGlobOptions(dir, mayWrite)) + .Where(p => IsRunTempName(dir, outputName, Path.GetFileName(p), mayWrite)) + .Select(p => new FileInfo(p)) + .OrderByDescending(f => f.LastWriteTimeUtc) + .FirstOrDefault(f => CanTakeExclusively(f.FullName)); + } + + /// + /// Whether TryAdoptOrphanedTemp would find a temp holding the counted items, without + /// merging it into anything. + /// + /// The staging name is the one place a real run is not read-only, and the preview reads it + /// anyway: asking only whether something already stands there previewed a recovery where a + /// run would stop instead, since an output directory this account may read and not write + /// refuses the create and nothing standing at the name says so. Where the name is clear, + /// the preview makes the same create a real run makes, at no length and removed again under + /// the handle that made it, so the two passes fail on the same open for the same reason. + /// Where something already stands at the name, the preview leaves it alone and reports what + /// stands there instead, for the caller to warn about. What it leaves behind is what it + /// found: the temp is untouched, the output is still not there, and the staging name holds + /// what it always held. + /// + /// + /// The file adoption would take the items from, as soon as it is + /// picked; null where there is none. What a caller names in a staging stop, since a + /// checkpoint on this route records no temp of its own. + /// Why the adoption stops at the staging name, where it does. Its own + /// answer and not a temp that has gone: the items are all in the file + /// names, and a caller that read this as a missing temp deleted + /// the position counting them. + /// What stands at the staging name and what a real run would remove, + /// for the caller to warn about; null where nothing stands there. + protected static bool CanAdoptOrphanedTemp(string outputPath, long itemCount, + out string? tempName, out StagingFailure? staging, out StagingRemoval? swept) + { + tempName = null; + staging = null; + swept = null; + try + { + if (itemCount <= 0) return false; + if (File.Exists(outputPath)) return false; + var temp = NewestUnheldTemp(outputPath, mayWrite: false); + if (temp == null) return false; + tempName = temp.Name; + + // Whole rows only, as below. Read and let go before the name beside the output is + // touched, so nothing of this pass's is open over the sweep. + var endsWhole = EndsWithNewline(temp.FullName); + long counted = 0; + using (var reader = new StreamReader(temp.FullName)) + { + while (counted < itemCount && reader.ReadLine() != null) counted++; + if (counted < itemCount || !(endsWhole || reader.ReadLine() != null)) return false; + } + + // And the name the copy is staged under, read where the run below clears it, so the + // two passes answer one disk state in the same words - and read only, because a + // preview removes nothing. The create test follows it where the name is clear; where + // something stands there the run would remove that entry first, and nothing at the + // name can be tested without doing so. + var adoptPath = StagingPathFor(outputPath); + if (!SweepStagingName(adoptPath, apply: false, out staging, out swept)) return false; + if (swept == null) + { + FileStream probe; + try + { + probe = ScratchName.CreateNew(adoptPath); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + staging = StagingFailureAt(adoptPath, ex, fromCreate: true); + return false; + } + + // The entry this pass made is the only thing in the directory that was not there + // before it, and a preview leaves the directory as it found it. Unlinked under + // the handle that made it, so no second run can take the name in between. + DiscardStagedFile(adoptPath, probe); + } + + return true; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { return false; } + } + + /// + /// Recovers a fresh-run JSONL job that was interrupted before its temp file was promoted, + /// from a checkpoint that predates the recorded temp name and length. The candidates are + /// the names a run gives its own temp - "{output}.{32-hex}.tmp" - and nothing else: the + /// glob that finds them also reaches a caller's own "users.jsonl.backup.tmp", and adoption + /// copies what it takes into the output and then deletes it. With only a line count and the + /// newest of those nothing else holds open to go on, this is safe solely when no output exists: + /// everything the run wrote is then in its temp, and creating the output from it is what a + /// finishing run's own promotion would have done. Against an existing output there is no + /// way to tell whose items the temp holds, so the caller must re-enumerate instead. + /// Copies exactly lines - content beyond the last flush may + /// be absent or torn - then removes the temp. The itemCount-th of them has to be a whole + /// row: ReadLine hands back an unterminated tail as though it were a line, so a temp cut + /// inside a counted row copied the fragment into the output as an item, reported it as + /// recovered, and left the rest of the enumeration appended behind it. A run mgx interrupts + /// cannot leave that state - every checkpoint site flushes the writer before recording the + /// count, so the rows a checkpoint counts are always terminated - but a temp truncated + /// under it by something outside the run can. Returns false when nothing usable exists, + /// leaving the caller to the stale-checkpoint path, which re-enumerates and loses nothing - + /// except where came back with it, which is the one false that + /// route must not be taken for. + /// + /// + protected static bool TryAdoptOrphanedTemp(string outputPath, long itemCount, + out string? tempName, out StagingFailure? staging, out StagingRemoval? swept) + { + tempName = null; + staging = null; + swept = null; + try + { + if (itemCount <= 0) return false; + if (File.Exists(outputPath)) return false; + var temp = NewestUnheldTemp(outputPath, mayWrite: true); + if (temp == null) return false; + tempName = temp.Name; + + // A file that ends mid-row can only end mid-row in its LAST one, so the counted + // rows are whole unless the count reaches that far - which is the case where + // another line still follows the itemCount-th. + var endsWhole = EndsWithNewline(temp.FullName); + var countedRowIsWhole = false; + + // Staged so the output appears in one Move. A merge that dies halfway leaves only + // the staging file behind, and the caller keeps its checkpoint - the safe direction. + // + // Under the same claim on the name a promotion stages by, and for the same reasons: + // the copy went into whatever stood at the name, so a symlink there sent it into the + // link's target and the move then put the link at the output path. The name is + // cleared first and created by an open that refuses to use anything already there, + // and the handle is held from the create through the rename. + long copied = 0; + var adoptPath = StagingPathFor(outputPath); + if (!SweepStagingName(adoptPath, apply: true, out staging, out swept)) return false; + FileStream? staged = null; + try + { + try + { + staged = ScratchName.CreateNew(adoptPath); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // Something appeared at the name in the instants since the sweep, or the + // directory beside the output refuses a new file in it at all. Neither is a + // temp that has gone missing, and the caller's route for one of those + // deletes the position counting items this temp holds every one of. + staging = StagingFailureAt(adoptPath, ex, fromCreate: true); + return false; + } + // What StreamWriter(path, append) wrote through: UTF-8 with no byte-order mark, + // throwing on unpaired surrogates. + using (var writer = new StreamWriter(staged, new UTF8Encoding(false, true), 1024, + leaveOpen: true)) + { + using var reader = new StreamReader(temp.FullName); + string? line; + while (copied < itemCount && (line = reader.ReadLine()) != null) + { + writer.WriteLine(line); + copied++; + } + countedRowIsWhole = endsWhole || reader.ReadLine() != null; + } + if (copied < itemCount || !countedRowIsWhole) + { + // Temp holds less than the checkpoint promises, or ends inside the last row + // it promises - unusable either way. The copy goes with the adoption that + // did not land, unlinked under the handle that holds the name. + DiscardStagedFile(adoptPath, staged); + staged = null; + return false; + } + // Windows will not rename a source this process has open, so there the handle + // goes before the move; everywhere else it holds the name across it. + if (OperatingSystem.IsWindows()) + { + staged.Dispose(); + staged = null; + } + File.Move(adoptPath, outputPath, overwrite: true); + } + finally { staged?.Dispose(); } + temp.Delete(); + return true; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return false; + } + } + + #region Shared URL and header builders + + protected static string NormalizePath(string path) + { + if (string.IsNullOrEmpty(path)) return string.Empty; + // A request URI has no use for a fragment - the server never sees one - so a raw '#' + // is always data (a filename, a filter value). Left alone, System.Uri would treat + // everything after it as a fragment and silently drop it from the wire. + path = path.Replace("#", "%23"); + return path.StartsWith('/') ? path : $"/{path}"; + } + + /// + /// Query-option names already present in a URL's own query string. Graph rejects a URL + /// carrying the same option twice, so a typed parameter defers to what the caller + /// already wrote into -Uri. + /// + protected static HashSet ExistingQueryOptions(string url) + { + var set = new HashSet(StringComparer.OrdinalIgnoreCase); + var q = url.IndexOf('?'); + if (q < 0) return set; + foreach (var part in url[(q + 1)..].Split('&', StringSplitOptions.RemoveEmptyEntries)) + // Decoded: %24top and $top are the same option to the server. + set.Add(Uri.UnescapeDataString(part.Split('=', 2)[0])); + return set; + } + + private static readonly Dictionary s_optionParameterNames = new(StringComparer.OrdinalIgnoreCase) + { + ["$filter"] = "-Filter", ["$select"] = "-Property", ["$orderby"] = "-Sort", + ["$search"] = "-Search", ["$expand"] = "-ExpandProperty", ["$skip"] = "-Skip", + ["$top"] = "-Top", + }; + + /// One warning line for typed parameters that deferred to options in -Uri. + protected static string DescribeDeferredOptions(IReadOnlyList deferred) + { + var names = deferred.Select(o => + s_optionParameterNames.TryGetValue(o, out var param) ? $"{param} ({o})" : o); + return $"{string.Join(", ", names)}: -Uri already carries the option, so the parameter's " + + "value was not added to the query. The -Uri value was sent."; + } + + /// The decoded value of a query option in a built URL, or null when absent. + protected static string? GetQueryOptionValue(string url, string name) + { + var q = url.IndexOf('?'); + if (q < 0) return null; + foreach (var part in url[(q + 1)..].Split('&', StringSplitOptions.RemoveEmptyEntries)) + { + var pieces = part.Split('=', 2); + if (string.Equals(Uri.UnescapeDataString(pieces[0]), name, StringComparison.OrdinalIgnoreCase)) + return pieces.Length > 1 ? Uri.UnescapeDataString(pieces[1]) : string.Empty; + } + return null; + } + + /// + /// Query-degradation retries, shared by the collection paths: some endpoints refuse an + /// auto-added $count=true with a bare 400, and some (e.g. /directoryRoles) refuse $top + /// with Request_UnsupportedQuery. Both recover by dropping the option and retrying. + /// + protected static bool IsCountRejection(GraphServiceException ex, bool countWasAutoAdded) + => countWasAutoAdded && ex.StatusCode == HttpStatusCode.BadRequest; + + protected static bool IsTopRejection(GraphServiceException ex, bool topSuppressed, bool noPageSize) + => !topSuppressed && !noPageSize + && ex.StatusCode == HttpStatusCode.BadRequest + && string.Equals(ex.ErrorCode, "Request_UnsupportedQuery", StringComparison.OrdinalIgnoreCase); + + protected const string CountRejectedVerbose = + "Endpoint rejected $count=true (HTTP 400). Retrying without count parameter."; + protected const string TopRejectedVerbose = + "Endpoint rejected $top (Request_UnsupportedQuery). Retrying without page size."; + + private static readonly Regex s_encodedTriplet = new("%[0-9A-Fa-f]{2}", RegexOptions.Compiled); + + /// + /// Escapes a query-option value, leaving anything already percent-encoded alone - a + /// pre-encoded filter must not be encoded twice (%27 becoming %2527), and a raw + /// apostrophe or space must still be escaped. Any %XX triplet counts as already + /// encoded; a value that means a literal percent-then-hex sequence has to arrive + /// pre-encoded (%25XX) to survive as data. + /// + protected static string EscapeQueryValue(string value) + { + var sb = new StringBuilder(value.Length + 16); + int i = 0; + foreach (Match m in s_encodedTriplet.Matches(value)) + { + sb.Append(Uri.EscapeDataString(value[i..m.Index])); + sb.Append(m.Value); + i = m.Index + m.Length; + } + sb.Append(Uri.EscapeDataString(value[i..])); + return sb.ToString(); + } + + /// + /// A header value from a Hashtable. An array (Prefer takes several values) joins with + /// ", " - HTTP's list form - instead of .NET's array ToString(), which would put the + /// literal text "System.String[]" on the wire. + /// + private static string HeaderValueToString(object? value) + { + if (value == null) return string.Empty; + if (value is string s) return s; + if (value is IEnumerable seq) + return string.Join(", ", seq.Cast().Select(HeaderScalarToString)); + return HeaderScalarToString(value); + } + + /// + /// One header value, rendered the same way on every machine. object.ToString() formats + /// under the thread's culture, so a double or a DateTimeOffset in -Headers put "0,5" on + /// the wire under de-DE where en-US put "0.5" - the same script sending different bytes. + /// CA1305 does not see it: the receiver is typed object, not IFormattable. + /// + private static string HeaderScalarToString(object? value) => value switch + { + null => string.Empty, + string s => s, + IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture), + _ => value.ToString() ?? string.Empty + }; + + protected static Dictionary? BuildRequestHeaders( + string? consistencyLevel, System.Collections.Hashtable? extraHeaders) + { + Dictionary? headers = null; + + // Apply extraHeaders first so dedicated parameters can override + if (extraHeaders != null) + { + // Case-insensitive: HTTP header names are, and a case-variant duplicate + // (If-Match and if-match) must not become two wire headers. + headers = new Dictionary(StringComparer.OrdinalIgnoreCase); + // The name is rendered the same way as the value: a Hashtable key is typed object + // too, and object.ToString() puts a numeric one on the wire under the operator's + // culture. + foreach (var key in extraHeaders.Keys) + headers[HeaderScalarToString(key)] = HeaderValueToString(extraHeaders[key]); + } + + // Dedicated -ConsistencyLevel parameter always wins over -Headers key + if (!string.IsNullOrEmpty(consistencyLevel)) + { + headers ??= new Dictionary(StringComparer.OrdinalIgnoreCase); + headers["ConsistencyLevel"] = consistencyLevel; } return headers; @@ -953,6 +3404,17 @@ protected record ODataListParams( bool IncludeCount = false); protected static string BuildListUrl(string versionedBaseUrl, string relativeUri, ODataListParams p) + => BuildListUrl(versionedBaseUrl, relativeUri, p, out _); + + /// + /// reports typed parameters that were NOT sent because + /// -Uri already carries the option. Callers warn: silently preferring the -Uri value + /// turns "-Property displayName" into a different projection with no sign anything was + /// ignored. Excluded from the report: the automatic page-size $top (only an explicit + /// -Top counts) and the automatic $count. + /// + protected static string BuildListUrl(string versionedBaseUrl, string relativeUri, ODataListParams p, + out List deferredToUri) { var baseUrl = $"{versionedBaseUrl}{NormalizePath(relativeUri)}"; var queryParams = new List(); @@ -964,37 +3426,47 @@ protected static string BuildListUrl(string versionedBaseUrl, string relativeUri } if (!string.IsNullOrEmpty(p.Filter)) - queryParams.Add($"$filter={Uri.EscapeDataString(p.Filter)}"); + queryParams.Add($"$filter={EscapeQueryValue(p.Filter)}"); if (p.Property is { Length: > 0 }) - queryParams.Add($"$select={Uri.EscapeDataString(string.Join(",", p.Property))}"); + queryParams.Add($"$select={EscapeQueryValue(string.Join(",", p.Property))}"); if (p.Sort is { Length: > 0 }) - queryParams.Add($"$orderby={Uri.EscapeDataString(string.Join(",", p.Sort))}"); + queryParams.Add($"$orderby={EscapeQueryValue(string.Join(",", p.Sort))}"); if (!string.IsNullOrEmpty(p.Search)) { - // Graph requires $search values wrapped in double quotes + // Graph API requires $search values wrapped in double quotes: $search="displayName:John" var searchValue = p.Search; if (!searchValue.StartsWith('"') || !searchValue.EndsWith('"')) searchValue = $"\"{searchValue}\""; - queryParams.Add($"$search={Uri.EscapeDataString(searchValue)}"); + queryParams.Add($"$search={EscapeQueryValue(searchValue)}"); } if (p.Skip > 0) queryParams.Add($"$skip={p.Skip}"); if (p.ExpandProperty is { Length: > 0 }) - queryParams.Add($"$expand={Uri.EscapeDataString(string.Join(",", p.ExpandProperty))}"); + queryParams.Add($"$expand={EscapeQueryValue(string.Join(",", p.ExpandProperty))}"); - // Required by -CountVariable, and by Graph advanced queries whenever $search is used + // $count=true: required explicitly via -CountVariable, or implicitly when $search is used + // (Graph advanced query capabilities require $count=true alongside $search) if (p.IncludeCount || !string.IsNullOrEmpty(p.Search)) queryParams.Add("$count=true"); + var existing = ExistingQueryOptions(baseUrl); + var removed = queryParams.Where(qp => existing.Contains(qp.Split('=', 2)[0])) + .Select(qp => qp.Split('=', 2)[0]).ToList(); + queryParams.RemoveAll(qp => existing.Contains(qp.Split('=', 2)[0])); + deferredToUri = removed.Where(name => + !string.Equals(name, "$count", StringComparison.OrdinalIgnoreCase) + && (!string.Equals(name, "$top", StringComparison.OrdinalIgnoreCase) || p.Top > 0)) + .ToList(); + if (queryParams.Count == 0) return baseUrl; - // Append with & when the URI already carries a query + // If URI already contains query parameters, append with & instead of ? var separator = baseUrl.Contains('?') ? "&" : "?"; return $"{baseUrl}{separator}{string.Join("&", queryParams)}"; } @@ -1006,6 +3478,15 @@ protected static string BuildListUrl(string versionedBaseUrl, string relativeUri $"Wait {s_clientOptions.CircuitBreakerDurationSeconds}s or run Get-MgxTelemetry for details. " + $"Tune with Set-MgxOption -CircuitBreakerFailureRatio / -CircuitBreakerMinThroughput."; + /// + /// Codes measured to mean the PATH was fine and the OBJECT was not there, so a beta hint + /// over them sends the caller to re-run a request that fails there too. Only codes with + /// demonstrated semantics belong here. Request_ResourceNotFound does NOT qualify: Graph + /// returns it both for a missing directory object and for a beta-only segment on v1.0 + /// (measured against /users/{id} and /users/{id}/profile), so it cannot be told apart and + /// the hedged hint stays. itemNotFound is the drive service reporting an absent item - an + /// unknown drive segment is a 400, not a 404, so the ambiguity does not arise there. + /// private static readonly HashSet ObjectMissingCodes = new(StringComparer.OrdinalIgnoreCase) { "itemNotFound", @@ -1031,23 +3512,18 @@ protected void WriteBetaHintIfApplicable(HttpStatusCode statusCode, string apiVe WriteWarning("This endpoint may only be available in beta. Retry with -ApiVersion beta."); } - protected static ErrorCategory MapStatusToCategory(HttpStatusCode statusCode) => statusCode switch - { - HttpStatusCode.NotFound => ErrorCategory.ObjectNotFound, - HttpStatusCode.Unauthorized => ErrorCategory.AuthenticationError, - HttpStatusCode.Forbidden => ErrorCategory.PermissionDenied, - HttpStatusCode.BadRequest => ErrorCategory.InvalidArgument, - HttpStatusCode.Conflict => ErrorCategory.ResourceExists, - (HttpStatusCode)429 => ErrorCategory.LimitsExceeded, - _ => ErrorCategory.NotSpecified - }; + // Derived from the failure class, so every status the classifier knows gets a real + // category: a 500 is ResourceUnavailable and a 408 ConnectionError where the old + // six-entry map left both NotSpecified. + protected static ErrorCategory MapStatusToCategory(HttpStatusCode statusCode) + => MgxErrorPresentation.CategoryForStatus(statusCode); /// /// Handles the three common terminal exception types (GraphServiceException, /// BrokenCircuitException, HttpRequestException) that appear in every cmdlet's /// catch cascade. Drains buffered messages, writes beta hint if applicable, /// and writes the error record. - /// Returns true if the exception was handled. False if unrecognized. + /// Returns true if the exception was handled; false if unrecognized. /// protected bool WriteGraphError(Exception ex, object? target, string? apiVersion = null) { @@ -1078,9 +3554,13 @@ protected bool WriteGraphError(Exception ex, object? target, string? apiVersion } } - // Count discrepancy thresholds. Undercount is tolerant, to absorb eventual consistency lag. - // Overcount is much tighter, because the failure mode is a duplicated page from a skiptoken - // overlap that a symmetric tolerance would never catch at scale + // Count discrepancy detection thresholds. + // Not user-configurable (YAGNI). Change these constants if defaults prove problematic. + // Undercount: 10% tolerance prevents noise from eventual consistency lag; + // 100-item floor avoids false alarms on small collections. + // Overcount: much tighter (0.5%, 50-item floor) - the observed failure mode is a + // duplicated page from a service-side skiptoken overlap (~one $top of extras), + // which a symmetric 10% tolerance would never catch at scale. protected const double CountDiscrepancyThreshold = 0.9; protected const long CountDiscrepancyMinItems = 100; protected const double CountOvershootThreshold = 0.005; @@ -1117,13 +3597,87 @@ protected void WriteCountDiscrepancyWarning( + $"were returned ({pct}% shortfall). {cause}."); } + /// What a response body turned out to be, once its declared charset and its + /// byte-order mark are accounted for. + internal enum JsonPayloadKind + { + /// holds the parsed body. + Parsed, + /// No body at all - a 204, or a 200 that sent nothing. + Empty, + /// The body declares a content type that is not JSON. Text is the decoded body. + NotJson, + /// The body declares JSON and does not parse. Text is the decoded body. + Malformed, + } + + /// + /// The read, without the reporting. is the decoded body: as it arrived + /// for a non-JSON one, with any byte-order mark removed for a body meant to parse. + /// + internal readonly record struct JsonPayload( + JsonPayloadKind Kind, JsonElement Json, string Text, string? MediaType); + + /// + /// Decode and parse a response body: the declared charset, the byte-order mark that + /// GetString leaves as U+FEFF and the serializer refuses, and whether the body claims to + /// be JSON at all. Every caller reads a body the same way; what each does about a body it + /// cannot use differs, and that stays with the caller - this writes to no stream, so a + /// fan-out worker can call it too. + /// + internal static JsonPayload ReadJsonPayloadCore( + System.Net.Http.Headers.MediaTypeHeaderValue? declaredType, byte[] bodyBytes) + { + var mediaType = declaredType?.MediaType; + if (bodyBytes.Length == 0) + return new JsonPayload(JsonPayloadKind.Empty, default, string.Empty, mediaType); + + var text = ResolveEncoding(declaredType?.CharSet).GetString(bodyBytes); + if (mediaType != null && !mediaType.EndsWith("json", StringComparison.OrdinalIgnoreCase)) + return new JsonPayload(JsonPayloadKind.NotJson, default, text, mediaType); + + text = text.TrimStart('\uFEFF'); + try + { + return new JsonPayload(JsonPayloadKind.Parsed, + JsonSerializer.Deserialize(text), text, mediaType); + } + catch (JsonException) + { + return new JsonPayload(JsonPayloadKind.Malformed, default, text, mediaType); + } + } + + /// The response's declared charset, defaulting to UTF-8. GetString strips a + /// matching BOM, which the byte-level Deserialize refused. + internal static Encoding ResolveEncoding(string? charset) + { + if (string.IsNullOrEmpty(charset)) return Encoding.UTF8; + // Any charset .NET will not construct falls back to UTF-8. An unknown or malformed name + // is an ArgumentException, but a name .NET knows and refuses to build - utf-7, disabled + // since SYSLIB0001 - is a NotSupportedException, and catching only the first let a + // response header end the pipeline with a .NET deprecation message. + try { return Encoding.GetEncoding(charset.Trim('"')); } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException) + { + return Encoding.UTF8; + } + } + /// - /// Release the Graph client. Invoked by under - /// the Interlocked guard, so this runs exactly once even when StopProcessing - /// (pipeline-stopping thread) races EndProcessing (pipeline thread). + /// Release the Graph client, and the output a resumed run holds. Invoked by + /// under the Interlocked guard, so this runs exactly once + /// even when StopProcessing (pipeline-stopping thread) races EndProcessing (pipeline thread). + /// + /// The hold is released where the writing ends, which is inside the attempt loop's own + /// finally - this is the backstop for every way out that never reaches it: a run that stops + /// between the reconcile and the loop because there is no Graph connection, and a pipeline + /// stopped from outside. + /// /// protected override void DisposeCore() { _client?.Dispose(); + ReleaseOutputHold(); } } diff --git a/src/Mgx.Cmdlets/Base/MgxErrorPresentation.cs b/src/Mgx.Cmdlets/Base/MgxErrorPresentation.cs new file mode 100644 index 0000000..ad60e92 --- /dev/null +++ b/src/Mgx.Cmdlets/Base/MgxErrorPresentation.cs @@ -0,0 +1,77 @@ +using System.Management.Automation; +using System.Net; +using Mgx.Engine.Errors; +using Mgx.Engine.Models; +using Polly.CircuitBreaker; + +namespace Mgx.Cmdlets.Base; + +/// +/// How a classified failure surfaces to PowerShell: the ErrorCategory for a class, the +/// status inside an exception, and the shared presentation of infrastructure failures. +/// Before this existed, the category map covered six statuses and left a 500 in +/// NotSpecified, the status extraction existed verbatim in two cmdlets, and the fan-out +/// error switches disagreed on whether a broken circuit carried its inner status. +/// +internal static class MgxErrorPresentation +{ + /// The PowerShell ErrorCategory for a failure class. + internal static ErrorCategory Category(MgxErrorClass cls) => cls switch + { + MgxErrorClass.Throttle => ErrorCategory.LimitsExceeded, + MgxErrorClass.TransientServer => ErrorCategory.ResourceUnavailable, + MgxErrorClass.TransientTransport => ErrorCategory.ConnectionError, + MgxErrorClass.Authentication => ErrorCategory.AuthenticationError, + MgxErrorClass.Authorization => ErrorCategory.PermissionDenied, + MgxErrorClass.Consistency => ErrorCategory.ResourceUnavailable, + MgxErrorClass.InvalidRequest => ErrorCategory.InvalidArgument, + MgxErrorClass.NotFound => ErrorCategory.ObjectNotFound, + MgxErrorClass.Conflict => ErrorCategory.ResourceExists, + _ => ErrorCategory.NotSpecified, + }; + + /// The category for a status, through the classifier. + internal static ErrorCategory CategoryForStatus(HttpStatusCode statusCode) + => Category(MgxErrorClassifier.Classify((int)statusCode).Class); + + /// The HTTP status a thrown exception carries, when it carries one. + internal static HttpStatusCode? TryGetStatus(Exception ex) + { + if (ex is GraphServiceException gse) return gse.StatusCode; + if (ex is HttpRequestException hre && hre.StatusCode.HasValue) return hre.StatusCode.Value; + return null; + } + + /// Whether -SkipNotFound / -SkipForbidden apply to this status. + internal static bool ShouldSkip(HttpStatusCode? status, bool skipNotFound, bool skipForbidden) + => (skipNotFound && status == HttpStatusCode.NotFound) + || (skipForbidden && status == HttpStatusCode.Forbidden); + + /// + /// One presentation for a per-item failure: id, category, and the exception to report. + /// A broken circuit is rewrapped with the session's guidance text - previously one + /// fan-out path did this and the other did not, so the same failure surfaced + /// differently depending on the cmdlet. Always ResourceUnavailable: the breaker's + /// exception never carries a Graph error (those are constructed above the pipeline), + /// and "stop sending" is the message whatever opened it. + /// + internal static (string ErrorId, ErrorCategory Category, Exception Report) PresentItemFailure( + Exception ex, string fallbackErrorId, string circuitMessage) + { + return ex switch + { + BrokenCircuitException bce => ( + "CircuitBroken", + ErrorCategory.ResourceUnavailable, + new InvalidOperationException(circuitMessage, bce)), + HttpRequestException => ("HttpError", ErrorCategory.ConnectionError, ex), + System.Text.Json.JsonException => ( + "MalformedJsonResponse", ErrorCategory.InvalidData, + new InvalidOperationException($"A response declared JSON but does not parse: {ex.Message}", ex)), + _ => ( + fallbackErrorId, + TryGetStatus(ex) is { } status ? CategoryForStatus(status) : ErrorCategory.NotSpecified, + ex), + }; + } +} diff --git a/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs b/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs index 588e25c..bdd9cac 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Batch/InvokeMgxBatchRequest.cs @@ -1,4 +1,4 @@ -using System.Collections; +using System.Collections; using System.Management.Automation; using System.Net; using System.Text.Json; @@ -26,6 +26,17 @@ namespace Mgx.Cmdlets.Cmdlets.Batch; [OutputType(typeof(Hashtable))] public class InvokeMgxBatchRequest : MgxCmdletBase { + /// Dead-letter lines are read by people; keep non-ASCII readable, matching + /// the request serializer's escaping. + private static readonly JsonSerializerOptions s_deadLetterJsonOptions = new() + { + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + /// + /// Graph API URLs to batch. Accepts absolute URLs (https://graph.microsoft.com/v1.0/users/id) + /// or relative URLs (/users/id). Also accepts Hashtables or PSObjects with Url/Method/Body members. + /// [Parameter(Mandatory = true, Position = 0, ValueFromPipeline = true)] [Alias("Url")] public object[] Uri { get; set; } = []; @@ -64,7 +75,44 @@ public class InvokeMgxBatchRequest : MgxCmdletBase [ArgumentCompleter(typeof(ApiVersionCompleter))] public string ApiVersion { get; set; } = "v1.0"; + + /// + /// Path to a JSONL file where the batch's failed and never-sent items are appended: + /// anything the server answered with a status >= 400, the items of a chunk whose own POST + /// failed, which carry that failure's status, and the items of chunks that were never + /// POSTed, which carry status 0. Nothing the server confirmed is written, so the file holds + /// only work that is still outstanding. + /// + /// Each line holds Timestamp, Url, Method, Status, Body - when the request had one, with + /// credential-named fields and pre-authenticated URLs redacted as a -Debug trace redacts + /// them, or the redaction marker whole when the URL names the secret or the body could not + /// be read for redaction, which is warned about by item - and Error, in that order. Error is + /// there only when the item's own response body carried an error code or message: a + /// never-sent item at status 0 has no response at all, and neither does an item of a refused + /// chunk the server did not answer, so those lines end at Body. + /// + /// It is a record of what to retry rather than a request to replay: a redacted Body is no + /// longer the body that was sent. Re-piping it for retry: + /// Get-Content dead.jsonl | ConvertFrom-Json | Invoke-MgxBatchRequest + /// A line carrying the redaction marker does not go back out that way: it is refused as an + /// error record. A Body that is the marker whole is not JSON, so the record names no field; + /// one carrying the marker in a field parses, and that record names the field - or names + /// the key, since a pre-authenticated URL used as a field name is cut the same way a value + /// is. Under -ErrorAction Stop, the first such refusal ends the pipeline before any chunk of + /// the batch is sent. Rebuild those requests from their Url and Method, supplying the + /// credential again. + /// + /// A status >= 400 does not say the write was not applied - the POST may have reached the + /// server and gone unanswered - so resending one is a decision about duplicates rather than + /// a free retry. Only status 0 says nothing was sent; a body the envelope could not + /// serialize surfaces as a chunk failure at 503 like a refused POST. + /// + /// A file system path, and one with something in it: an empty string resolves to the + /// working directory, and a path on another provider - Env:\X - resolves to a bare name + /// this run would make a file of wherever it happened to be standing. + /// [Parameter] + [ValidateNotNullOrEmpty] public string? DeadLetterPath { get; set; } /// @@ -97,6 +145,19 @@ protected override void ProcessRecord() protected override void EndProcessing() { GraphBatchClient? batchClient = null; + + // The dead-letter file, held open for the run. Declared out here so the finally can + // close it on a run that never reached its own write, and so the three things that + // decide whether an empty file is left behind - was it this run that created it, did + // this run put a line in it, and what did the file hold when this run's own handle last + // looked - are readable from there too. The stream beside the writer is that handle: + // the writer buffers characters, the stream is what can be asked the file's length. + StreamWriter? deadLetterWriter = null; + FileStream? deadLetterStream = null; + string? deadLetterFileThisRunCreated = null; + var deadLetterLines = 0; + long? deadLetterLengthAtClose = null; + try { if (_collected.Count == 0) @@ -105,9 +166,25 @@ protected override void EndProcessing() return; } - string? resolvedDeadLetterPath = DeadLetterPath != null - ? GetUnresolvedProviderPathFromPSPath(DeadLetterPath) - : null; + // Resolved before any network call, and told to be a file system path. Every other + // provider resolves to something this run cannot append to, and resolves it + // silently: "Env:\X" comes back as "X", a name relative to nothing, so the run made + // a file called X in whatever directory it was standing in and reported the dead + // letters written - to a file nobody named, under a name taken from a variable. + string? resolvedDeadLetterPath = null; + if (DeadLetterPath != null) + { + resolvedDeadLetterPath = SessionState.Path.GetUnresolvedProviderPathFromPSPath( + DeadLetterPath, out var provider, out _); + if (provider.ImplementingType.FullName != FileSystemProviderType) + { + ThrowTerminatingError(new ErrorRecord( + new ArgumentException( + $"-DeadLetterPath '{DeadLetterPath}' is not a file system path."), + "DeadLetterPathNotFileSystem", ErrorCategory.InvalidArgument, DeadLetterPath)); + return; + } + } var hasSearch = _collected.Any(c => c.Url.Contains("$search", StringComparison.OrdinalIgnoreCase)); @@ -121,21 +198,116 @@ protected override void EndProcessing() return; } - // -WhatIf is documented as "the cmdlet is not run", without qualification, so the - // gate covers reads as well. A read-only batch changes nothing on the server, but it - // spends resource units, can be throttled, and emits objects into the pipeline - - // none of which is "not run", and none of which the caller asked for. - var writeOps = _collected.Where(c => + // Convert to BatchOperation list. An item whose body is not valid JSON fails on + // its own (non-terminating error) instead of aborting the whole batch; `submitted` + // keeps result indices aligned with the operations actually sent. + var operations = new List(_collected.Count); + var submitted = new List(_collected.Count); + foreach (var input in _collected) + { + JsonElement? body = null; + if (input.Body != null) + { + try + { + var json = InvokeMgxRequest.SerializeBody(input.Body); + body = JsonSerializer.Deserialize(json); + } + catch (Exception ex) when (ex is JsonException or ArgumentException) + { + // ArgumentException: a value serialization refuses (SecureString, NaN), + // named by property path. JsonException: a string body that is not JSON. + WriteError(new ErrorRecord( + new ArgumentException( + $"Body for {input.Method} {input.Url} is not valid JSON: {ex.Message}", ex), + "InvalidBatchItemBody", ErrorCategory.InvalidArgument, input.Url)); + continue; + } + + // A dead-letter line piped back in. The marker stands where something the + // file must not carry used to be, so this is not the body that was sent, + // and sending it writes the literal marker to the tenant - under the field + // name that says what it replaced, or, when the file cut a capability URL + // used as a field name, AS the field name. A whole-body marker is refused + // already, because the marker alone is not JSON; a marker in a field or in + // a key parses, and the recipe in -DeadLetterPath's help ends at the same + // place whichever it is. + // + // The walk reads every string of the body, and a string the JSON parser + // accepted is not always one it will hand over: an unpaired surrogate + // escape, "Jos\ud83d", parses as a string value and throws when read. That + // is one item's body and nothing more, so it is refused like a body that is + // not JSON at all - the guard is the only thing that decides whether this + // body may be sent, and a body it cannot read is one it cannot clear. + string? markerField; + try + { + markerField = FindRedactionMarker(body!.Value); + } + catch (Exception ex) when (ex is InvalidOperationException or JsonException) + { + WriteError(new ErrorRecord( + new ArgumentException( + $"Body for {input.Method} {input.Url} could not be read for the " + + $"redaction check: {Reason(ex.Message)}. Send it as a hashtable " + + "or a string this run can read.", ex), + "InvalidBatchItemBody", ErrorCategory.InvalidArgument, input.Url)); + continue; + } + + if (markerField != null) + { + WriteError(new ErrorRecord( + new ArgumentException( + $"Body for {input.Method} {input.Url} carries the redaction marker " + + $"'{RedactionMarker}' at '{markerField}'. A dead-letter line is not the " + + "body that was sent: rebuild the request from its Url and Method, " + + "supplying the credential again. A body of your own must not carry " + + "that text."), + "InvalidBatchItemBody", ErrorCategory.InvalidArgument, input.Url)); + continue; + } + } + + operations.Add(new BatchOperation(NormalizeToRelativeUrl(input.Url), input.Method, body)); + submitted.Add(input); + } + + if (operations.Count == 0) + return; + + // The gate covers reads as well as writes. A read-only batch changes nothing on the + // server, but it spends resource units, can be throttled, and emits objects into + // the pipeline, none of which a caller who typed -WhatIf asked for. The -WhatIf + // help says so, and says Invoke-MgxRequest makes the other choice. + // + // Over the items that will be sent, not the items that were piped in. The body + // validation and the marker guard have run by now, and a target counting what + // they refused named requests the run was never going to make, at the one + // surface whose whole purpose is saying what is about to happen. The refusals + // are named as their own count beside it: a caller who typed -WhatIf to find + // out what would happen is owed both halves of the answer, and the error + // records for them are already in the stream, since a refusal is not something + // the gate governs. + var writeOps = submitted.Where(c => !string.Equals(c.Method, "GET", StringComparison.OrdinalIgnoreCase)).ToList(); + var refused = _collected.Count - submitted.Count; string target; if (writeOps.Count == 0) { - target = $"GET {_collected.Count} requests via $batch"; + target = $"GET {Requests(submitted.Count)} via $batch"; } - else if (writeOps.All(o => string.Equals(o.Method, writeOps[0].Method, StringComparison.OrdinalIgnoreCase))) + // Only when the batch is that method and nothing else. The count is everything + // that will be sent, reads included, by the reasoning above, and a write verb + // welded onto it described requests that were not in it: one DELETE among nineteen + // GETs read "DELETE 20 requests via $batch". With reads present the writes are + // named as their own count, the way a batch of several write methods already names + // them. + else if (writeOps.Count == submitted.Count + && writeOps.All(o => string.Equals(o.Method, writeOps[0].Method, StringComparison.OrdinalIgnoreCase))) { - target = $"{writeOps[0].Method} {_collected.Count} requests via $batch"; + target = $"{writeOps[0].Method} {Requests(submitted.Count)} via $batch"; } else { @@ -143,9 +315,12 @@ protected override void EndProcessing() .GroupBy(o => o.Method.ToUpperInvariant()) .OrderByDescending(g => g.Count()) .Select(g => $"{g.Count()} {g.Key}"); - target = $"{_collected.Count} requests ({string.Join(", ", breakdown)}) via $batch"; + target = $"{Requests(submitted.Count)} ({string.Join(", ", breakdown)}) via $batch"; } + if (refused > 0) + target += $"; {refused} refused"; + if (!ShouldProcess(target, "Send batch")) return; @@ -165,44 +340,67 @@ protected override void EndProcessing() ItemHeaders = itemHeaders }; - // Convert to BatchOperation list. An item whose body is not valid JSON fails on - // its own (non-terminating error) instead of aborting the whole batch. `submitted` - // keeps result indices aligned with the operations actually sent. - var operations = new List(_collected.Count); - var submitted = new List(_collected.Count); - foreach (var input in _collected) + + // Opened before the first POST, and the handle kept for the run. Opened where the + // lines are written - at the end - an unusable path was discovered with the batch + // already applied: the caller had made every one of those changes and lost the + // record of which to retry, and the only thing that could have made that + // recoverable was learning it before anything went out. A directory standing at the + // path, a parent directory that is not there, a file this account may not write: + // all of them answer here, with nothing sent. + // + // Append, because the parameter's help promises a file a second run adds to rather + // than replaces, and shared the way each platform has to share it for that promise + // to hold: see DeadLetterHoldShare. + if (resolvedDeadLetterPath != null) { - JsonElement? body = null; - if (input.Body != null) + var existed = File.Exists(resolvedDeadLetterPath); + try { - var json = InvokeMgxRequest.SerializeBody(input.Body); - try - { - body = JsonSerializer.Deserialize(json); - } - catch (JsonException ex) - { - WriteError(new ErrorRecord( - new ArgumentException( - $"Body for {input.Method} {input.Url} is not valid JSON: {ex.Message}", ex), - "InvalidBatchItemBody", ErrorCategory.InvalidArgument, input.Url)); - continue; - } + deadLetterStream = new FileStream( + resolvedDeadLetterPath, FileMode.Append, FileAccess.Write, + DeadLetterHoldShare); + deadLetterWriter = new StreamWriter(deadLetterStream); } - - operations.Add(new BatchOperation(NormalizeToRelativeUrl(input.Url), input.Method, body, input.Id)); - submitted.Add(input); + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + WriteError(new ErrorRecord( + new IOException( + $"Failed to write dead-letter file '{resolvedDeadLetterPath}': " + + $"{Reason(ex.Message)} before the batch was sent; nothing was sent.", ex), + "DeadLetterWriteFailed", ErrorCategory.WriteError, resolvedDeadLetterPath)); + return; + } + if (!existed) + deadLetterFileThisRunCreated = resolvedDeadLetterPath; } - if (operations.Count == 0) - return; - var batchResult = batchClient.ExecuteBatchIndexedAsync(operations, CancellationToken) .GetAwaiter().GetResult(); var results = batchResult.Results; var telemetry = batchResult.Telemetry; + // Before anything else this run does with the batch. What the counters hold is + // what the wire did, and it did it whatever becomes of the run afterwards - so the + // session view has to be told here rather than at the end of the reporting, past + // the dead-letter write, the chunk failure and one error record per failed item. + // Every one of those can end the run under -ErrorAction Stop, and a throttled, + // retried batch that ended on one contributed nothing at all to Get-MgxTelemetry: + // the 429s it met and the seconds it spent in Retry-After sleeps were the very + // numbers a caller reads that view to find. Nothing between the call above and + // this can throw, so the record needs no finally to be sure of running. + RecordSessionTelemetry(telemetry); + + // Every line this run prints about itself reads these: the dead-letter line, the + // chunk-failure target, the verbose summary and the warning. Counting for + // themselves, they drifted - the warning said the batch-level retry pass had been + // withheld over runs in which it had gone out, been answered, and been refused on a + // later chunk, while the verbose line beside it credited the run with the retries. + var summary = BatchRunSummary.Of(results, telemetry, batchResult.ChunkFailure != null); + + // Output all results as Hashtables (success and failure) + var drains = FollowNextLink.IsPresent ? DrainNextLinks(batchClient, results) : []; @@ -228,10 +426,9 @@ protected override void EndProcessing() // Only when the caller supplied one, so output is unchanged for callers that did not if (input.Id != null) result["Id"] = input.Id; - - // Status 0 means the operation was never sent, because a chunk before it failed. - // It is not a success and must not read as one - the caller has to be able to - // tell a write that may have landed from one that certainly did not. + // Status 0 means the operation was never sent, because another chunk failed + // first. It is not a success and must not read as one - the caller has to be + // able to tell a write that may have landed from one that certainly did not. if (item.Status == GraphBatchClient.NotSentStatus) result["NotSent"] = true; @@ -239,32 +436,37 @@ protected override void EndProcessing() WriteObject(result); } - // A chunk's POST failed after earlier chunks were applied. Their results are above; - // this says why the rest never went, and NotSent names them. - if (batchResult.ChunkFailure != null) + // Write failed items to the file opened before the batch went out. Two things about + // this write are collected rather than reported immediately. The write failing + // outright is an error record, written first of the errors below - the path was + // usable when the run opened it, so a failure here is the file going out from under + // an applied batch, which is the case a caller most needs the error stream for. A + // body the redactor could not read is a warning per item, written last of everything + // - WriteWarning under -WarningAction Stop throws, and a warning about one line of + // the file must not cost the caller the item errors or the telemetry. + var withheldBodies = new List(); + Exception? deadLetterWriteFailure = null; + if (deadLetterWriter != null) { - var notSent = batchResult.NotSent.Count; - WriteError(new ErrorRecord(batchResult.ChunkFailure, "BatchChunkFailed", - ErrorCategory.NotSpecified, - $"{notSent} of {results.Count} operations were not sent")); - } - - if (resolvedDeadLetterPath != null) - { - var failedCount = 0; try { - using var writer = new StreamWriter(resolvedDeadLetterPath, append: true); for (int i = 0; i < results.Count; i++) { var (_, item) = results[i]; if (item.Status < 400 && item.Status != GraphBatchClient.NotSentStatus) continue; var input = submitted[i]; + // The item's own URL, under the value rule the body's strings are read + // by. A pre-authenticated URL is a request a caller can make as well as + // a value a body can carry - a PUT to an upload session's uploadUrl, a + // GET of a @microsoft.graph.downloadUrl - and the URL written here is + // the one the caller gave, so the line carried the live signature while + // the same URL inside a body beside it was cut. var deadLetter = new JsonObject { ["Timestamp"] = DateTime.UtcNow.ToString("o"), - ["Url"] = input.Url, + ["Url"] = SensitiveNames.RedactCapabilityUrl(input.Url, RedactionMarker) + ?? input.Url, ["Method"] = input.Method, ["Status"] = item.Status, }; @@ -275,30 +477,136 @@ protected override void EndProcessing() if (input.Body != null) { - var bodyJson = InvokeMgxRequest.SerializeBody(input.Body); - var bodyNode = JsonNode.Parse(bodyJson); - RedactSensitiveFields(bodyNode); - deadLetter["Body"] = bodyNode; + // A URL that names a secret takes the whole Body: on resetPassword or + // uploadSecret the credential is the field called `k` or `value`, and + // no name rule reaches it. Url, Method, Status and Error still say + // what failed, which is what the file is read for. + if (UrlNamesSecret(input.Url)) + { + deadLetter["Body"] = RedactionMarker; + } + else + { + // A body the redactor cannot walk takes the marker too. The walk + // is the only thing that decides which of a body's values may be + // written, so a body it cannot read is a body with nothing + // cleared for the file - the same position a URL naming the + // secret puts it in. A duplicate property name is how a body + // gets there: JsonNode.Parse accepts the document and building + // the JsonObject's dictionary to enumerate it throws, and Graph + // itself takes such a body, so the run must not end on one. + try + { + var bodyJson = InvokeMgxRequest.SerializeBody(input.Body); + var bodyNode = JsonNode.Parse(bodyJson); + deadLetter["Body"] = RedactSensitiveFields(bodyNode); + } + catch (Exception ex) + { + deadLetter["Body"] = RedactionMarker; + withheldBodies.Add( + $"Body for {input.Method} {input.Url} was withheld from the " + + $"dead-letter file: it could not be read for redaction ({ex.Message}). " + + "The line carries the redaction marker in its place."); + } + } } var errorMsg = TryExtractBatchErrorMessage(item); if (errorMsg != null) deadLetter["Error"] = errorMsg; - writer.WriteLine(deadLetter.ToJsonString()); - failedCount++; + deadLetterWriter.WriteLine(deadLetter.ToJsonString(s_deadLetterJsonOptions)); + deadLetterLines++; } } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { - WriteWarning($"Failed to write dead-letter file '{resolvedDeadLetterPath}': {ex.Message}"); + // Collected, not written - see the comment above the declaration. The + // exception itself, not just its message: the error record below keeps it + // as the inner exception, so $Error carries what actually refused. + deadLetterWriteFailure = ex; + } + + // Closed here rather than in the finally, so a failure flushing the last line + // is this run's to report and the file is on disk before the lines below say + // what is in it - and before an empty one is removed. The length is read off + // the handle first, while it still answers. + deadLetterLengthAtClose = DeadLetterLength(deadLetterStream); + deadLetterWriter.Dispose(); + deadLetterWriter = null; + deadLetterStream = null; + + // A file this run created and never put a line in says a batch had failures + // when it had none, and a caller watching that path for one would find it after + // every clean run. A file that was already there is an earlier run's record and + // is not this run's to remove; nor is one this run could not finish writing; + // nor is one that is no longer empty, whoever filled it - the share mode admits + // a second run appending to the same path, so "this run created it and wrote + // nothing" says nothing about what stands in it now. + if (deadLetterFileThisRunCreated != null && deadLetterLines == 0 + && deadLetterWriteFailure == null && deadLetterLengthAtClose == 0) + { + TryRemoveEmptyDeadLetterFile(deadLetterFileThisRunCreated); + deadLetterFileThisRunCreated = null; } - if (failedCount > 0) - WriteVerbose($"Wrote {failedCount} failed items to dead-letter file: {resolvedDeadLetterPath}"); + // The file takes both, and they are not the same thing to a caller deciding what + // to re-pipe: a refused write may already have been applied and one that was + // never sent certainly was not. Only after the write finished - a line stating + // what is in a file the run could not finish writing states it of nothing. + if (deadLetterWriteFailure == null && summary.Failed + summary.NotSent > 0) + { + var written = summary.NotSent > 0 + ? $"{summary.Failed} failed and {summary.NotSent} not-sent items" + : $"{summary.Failed} failed items"; + WriteVerbose($"Wrote {written} to dead-letter file: {resolvedDeadLetterPath}"); + } + } + + // The file the caller asked for was opened and then could not be written. An error + // record rather than a warning, because a warning is the one thing a caller can be + // running without: at -WarningAction SilentlyContinue, or Ignore, a run that lost + // the whole record of what to retry said nothing about it at all. + // + // First of the errors this run writes, ahead of the chunk failure and the item + // errors. Under -ErrorAction Stop the run ends here, and that is the intent - the + // item errors it ends before are the very lines the file was going to hold, so a + // caller told about them and not about the missing file would go looking for a + // file that is not there. The withheld-body warnings and the fold in the outcome + // warning still run for a caller who is not stopping. + // + // A run with nothing to write cannot get here: the loop above only touches the + // handle for a failed or not-sent item, so a failure means there were lines and + // some of them are gone. An unusable path is answered before the batch instead, + // where nothing has been applied and there is nothing to be told to retry. + if (deadLetterWriteFailure != null) + { + WriteError(new ErrorRecord( + new IOException( + $"Failed to write dead-letter file '{resolvedDeadLetterPath}': " + + deadLetterWriteFailure.Message, deadLetterWriteFailure), + "DeadLetterWriteFailed", ErrorCategory.WriteError, resolvedDeadLetterPath)); + } + + // A chunk's POST failed while other chunks were being applied. Their results are + // above; this says why the rest never went, and NotSent names them. After the + // dead-letter write, like the item errors below, so -ErrorAction Stop cannot cut + // the file short. + if (batchResult.ChunkFailure != null) + { + // The id stays BatchChunkFailed; the category and wrapping come from the + // failure itself - a throttled chunk is LimitsExceeded, an open circuit + // ResourceUnavailable with the guidance text, not NotSpecified. + var (_, category, report) = MgxErrorPresentation.PresentItemFailure( + batchResult.ChunkFailure, "BatchChunkFailed", CircuitBreakerMessage); + WriteError(new ErrorRecord(report, "BatchChunkFailed", category, + $"{summary.Failed} of {summary.Total} operations failed, " + + $"{summary.NotSent} were not sent")); } - // Per-item errors, so -ErrorAction Stop trips and $Error is populated + // Emit errors for failed items (enables -ErrorAction Stop, populates $Error). + // After the dead-letter write, so -ErrorAction Stop cannot cut the file short. for (int i = 0; i < results.Count; i++) { var (_, item) = results[i]; @@ -307,7 +615,7 @@ protected override void EndProcessing() var skipped = submitted[i]; WriteError(new ErrorRecord( new InvalidOperationException( - $"{skipped.Method} {skipped.Url} was not sent: an earlier chunk failed."), + $"{skipped.Method} {skipped.Url} was not sent: another chunk of the batch failed."), "BatchItemNotSent", ErrorCategory.NotSpecified, skipped.Url)); } else if (item.Status >= 400) @@ -323,7 +631,14 @@ protected override void EndProcessing() } } - WriteBatchTelemetry(telemetry); + WriteBatchTelemetry(telemetry, summary, withheldBodies.Count, deadLetterWriteFailure?.Message); + + // Last of everything this run owes for the batch: WriteWarning throws under + // -WarningAction Stop, and a warning that ends the run here must not be able to + // cost the caller the item errors above or the telemetry WriteBatchTelemetry just + // wrote - both already ran by the time one of these can throw. + foreach (var withheld in withheldBodies) + WriteWarning(withheld); } catch (Exception ex) when (ex is GraphServiceException or BrokenCircuitException or HttpRequestException) { @@ -340,6 +655,25 @@ protected override void EndProcessing() } finally { + // Only reached with the handle still open where the run did not get to its own + // write - a batch that threw, a cancellation, a stop. Guarded, because a throw from + // a finally would replace the failure the run is already carrying. + if (deadLetterWriter != null) + { + deadLetterLengthAtClose = DeadLetterLength(deadLetterStream); + try { deadLetterWriter.Dispose(); } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { } + } + + // Same reasoning as the removal after the write, for the same file on the paths + // that never reach it: an empty file this run made is a record of nothing. Under + // the same three conditions, the length included - a run that came through the + // write and left the file standing because something had been appended to it does + // not get a second, unconditional go at it here. + if (deadLetterFileThisRunCreated != null && deadLetterLines == 0 + && deadLetterLengthAtClose == 0) + TryRemoveEmptyDeadLetterFile(deadLetterFileThisRunCreated); + // Drain verbose messages even on exception so retry/throttle history is visible DrainClientMessages(); batchClient?.DrainVerboseMessages(); @@ -347,6 +681,81 @@ protected override void EndProcessing() } } + /// + /// What the dead-letter file holds, as this run's own handle reports it - so a line a second + /// run appended beside this one's is counted. Null where the handle cannot say, which is not + /// the same answer as zero: the removal below turns on the length, and a length nothing + /// measured leaves the file where it is. + /// + private static long? DeadLetterLength(FileStream? stream) + { + try { return stream?.Length; } + catch (Exception ex) when (ex is IOException or NotSupportedException + or ObjectDisposedException) + { + return null; + } + } + + /// + /// Remove a dead-letter file this run created and put nothing in - while it is still empty + /// and no second run has it open. Silent either way: an empty file holds nothing a caller + /// has to be told about, and a path that will not give it up is a path the next run will + /// simply append to. + /// + /// The claim is what makes the removal honest. The file is opened for append under + /// , which admits a second run appending to the same path - + /// the shape a caller gets from running batches in parallel over one -DeadLetterPath - and + /// unlink honors no lock, so "this run created it and wrote no lines" is not on its own a + /// statement about what stands in the file by the end. FileShare.None asks for the entry + /// exclusively: a sibling still holding it for append refuses this open and the file is + /// left, and under a claim that succeeds the length is read once more, so a sibling that + /// wrote and closed between the two readings is not deleted either. Only then is the path + /// unlinked, under the claim, in the order uses - so no + /// instant exists in which this run has decided on the name and let go of it. + /// + /// + /// + /// What a second opener of the dead-letter file may do while this run holds it, which is not + /// the same question on the two platforms because the lock is not the same lock. The + /// -DeadLetterPath help promises a file a second run adds to rather than replaces, and two + /// batches over one path is what running them in parallel gives, so a sibling appending to + /// it has to get in - and the removal below has to be refused for as long as one is there. + /// + /// .NET on Unix backs FileShare with an advisory flock: FileShare.None takes LOCK_EX and + /// every other mode takes LOCK_SH, and LOCK_SH refuses no second writer. FileShare.Read is + /// therefore what admits the sibling while still refusing the LOCK_EX the removal's claim + /// asks for - FileShare.ReadWrite would take no lock at all, and the claim would succeed + /// over a sibling's open file and delete it on a length read before that sibling had + /// written. + /// + /// + /// Windows checks the sharing both ways round: a second opener's access has to be one this + /// handle's share mode admits, and this handle's access has to be one the second opener's + /// share mode admits. FileShare.Read admits a reader and no appender at all, so a second + /// batch over one -DeadLetterPath could not open the file and wrote none of its dead letters + /// - the shape the help promises, refused on the platform. FileShare.ReadWrite lets the + /// sibling in, and the removal's claim is refused all the same: it asks FileShare.None, + /// which admits none of the writing this handle is already doing. + /// + /// + private static FileShare DeadLetterHoldShare => + OperatingSystem.IsWindows() ? FileShare.ReadWrite : FileShare.Read; + + private static void TryRemoveEmptyDeadLetterFile(string path) + { + FileStream? claim = null; + try + { + claim = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.None); + if (claim.Length != 0) return; + ScratchName.Discard(path, claim); + claim = null; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { } + finally { claim?.Dispose(); } + } + /// /// Parse pipeline input into a BatchInput. Supports: /// - String: use as URL with shared -Method/-Body parameters @@ -416,47 +825,273 @@ private string NormalizeToRelativeUrl(string url) } /// - /// Redact sensitive fields (passwordProfile, credentials, secrets) from a JSON body - /// before writing to the dead-letter file. Modifies the node in-place. + /// "1 request" or "n requests". The gate names a count that can be one - what a batch will + /// send is what the validation and the guard left of it - and "1 requests" is a sentence + /// about a number rather than about the batch. + /// + private static string Requests(int count) => count == 1 ? "1 request" : $"{count} requests"; + + /// What stands in the dead-letter file for a value it must not carry. + internal const string RedactionMarker = "***REDACTED***"; + + /// + /// The one provider -DeadLetterPath may name. Matched on the implementing type rather than + /// on the provider's name, which a module is free to reuse. + /// + private const string FileSystemProviderType = "Microsoft.PowerShell.Commands.FileSystemProvider"; + + /// + /// Where a body carries the redaction marker, as a string value or inside a property name, + /// or null when it carries none. Every depth, and an array's elements as well as an + /// object's properties: the writer puts the marker wherever the walk found a credential, + /// and a keyCredentials array is where it usually finds one. The path is what the caller + /// needs - it says what the file took, and so what has to be supplied again. + /// + /// A name and a value are read the same way, as substrings, because the writer leaves the + /// marker inside text as often as alone. A pre-authenticated URL is cut at its capability + /// parameter and what stands in the file is the URL up to that parameter with the marker + /// after it - whether the URL was a field's value, an element of an array, or the field + /// NAME itself, which is what the cut on a key leaves. Reading a value for equality saw + /// only the cuts that took a whole value, so a line holding a cut uploadUri, + /// azureStorageUri or @microsoft.graph.downloadUrl read as a body that could be sent, and + /// sending it puts the literal marker where the signature belongs. + /// + /// A caller's own text carrying the marker is refused with the rest. The file's cut form is + /// a prefix plus the marker, so nothing in the text separates the two, and refusing a body + /// no file wrote costs the caller a rename; sending one the file did write costs a request + /// the tenant cannot honor and a credential the caller still thinks it supplied. + /// + internal static string? FindRedactionMarker(JsonElement element, string path = "") + { + switch (element.ValueKind) + { + case JsonValueKind.String: + return element.GetString()!.Contains(RedactionMarker, StringComparison.Ordinal) + ? (path.Length == 0 ? "the whole body" : path) + : null; + case JsonValueKind.Object: + foreach (var property in element.EnumerateObject()) + { + var here = path.Length == 0 ? property.Name : $"{path}.{property.Name}"; + + // Before the value, so that a cut key whose own value says nothing is + // still reported as the key: what the cut leaves is the URL up to the + // capability parameter with the marker after it, and a cut that collided + // carries a #2, #3... suffix past that. Either way the name itself is the + // path returned, so the record names the key as written. + if (property.Name.Contains(RedactionMarker, StringComparison.Ordinal)) + return here; + + var found = FindRedactionMarker(property.Value, here); + if (found != null) return found; + } + return null; + case JsonValueKind.Array: + var index = 0; + foreach (var item in element.EnumerateArray()) + { + var found = FindRedactionMarker(item, $"{path}[{index++}]"); + if (found != null) return found; + } + return null; + default: + return null; + } + } + + /// + /// True when a field name or URL path segment carries one of the credential fragments. The + /// fragments are SensitiveNames', and so are the two value rules beside them: a property + /// whose NAME carries one of the fragments, a property whose name carries "downloadurl", and + /// a string value that is a URL carrying one of the capability parameters. That class carries + /// the reasoning for each. + /// + /// This file and a -Debug trace agree on the value rule for the form a body writes a URL in, + /// since a URL is text either way, and on the two name rules only where the value beside the + /// named key is itself a quoted string - a trace regex for a name rule matches a key and its value together, and + /// cannot match past a value that is not a string. This file does not stop there: the value + /// beside a matched key is cleared whatever it holds, so a number, a bool, an array or an + /// object beside a password or a downloadUrl field is missing from the dead-letter line and + /// still stands whole in -Debug output. + /// + /// Two things only the file has, because a trace has nothing to apply them to. The URL rule + /// below reads the request's own path and takes the whole Body; the marker also stands for a + /// body the walk could not read. The tracer sees a batch envelope as text - one item's url + /// is a string beside its body, not a request whose body the tracer knows - so an + /// uploadSecret body still reaches -Debug output whole, and nothing here changes that. /// - internal static void RedactSensitiveFields(JsonNode? node) + private static bool IsSensitiveName(string name) => SensitiveNames.Matches(name); + + /// + /// True when the request's own path names the secret its body carries - resetPassword, + /// changePassword, uploadSecret, synchronization/secrets, addKey, vppTokens. Those bodies + /// hold the credential under a field name that says nothing about it, `k` or `value`, which + /// no name rule can reach; the caller of this decides on the whole Body instead. + /// + /// The path and nothing else. For an http or https URL that is what the parser gives after + /// the authority, so the host is out of it: a tenant that reaches Graph through a host + /// named keyvault.example.com is describing the machine, not the body, and losing every + /// body under it costs the file the half it is read for. Everything else is cut at the + /// first ? or #, a query string carrying filters rather than bodies. + /// + /// The scheme is checked because a parse alone does not say the string was absolute. On + /// Unix a path is a legal file URI, so "/users/u1?$select=keyCredentials" parses as + /// absolute with the query folded into an escaped AbsolutePath - the segment then reads as + /// keyCredentials and the whole body goes, on a $select. On Windows the same string does + /// not parse, and one relative URL would have been judged two ways on two hosts. + /// + /// Every segment is read decoded, because Graph reads the path decoded and the two forms + /// are one request: .../microsoft.graph.reset%50assword is a password reset. The parser + /// unescapes an unreserved character itself, so only the relative form - the one the batch + /// envelope actually carries - could hide a name behind an escape. + /// + internal static bool UrlNamesSecret(string? url) { - if (node is JsonArray rootArr) + if (string.IsNullOrEmpty(url)) return false; + + string path; + // System.Uri, not the cmdlet's -Uri parameter, which shadows the type name here. + if (System.Uri.TryCreate(url, UriKind.Absolute, out var absolute) + && (absolute.Scheme == System.Uri.UriSchemeHttp || absolute.Scheme == System.Uri.UriSchemeHttps)) { - foreach (var item in rootArr) - if (item is JsonObject arrObj) - RedactSensitiveFields(arrObj); - return; + path = absolute.AbsolutePath; } - if (node is not JsonObject obj) return; - foreach (var key in obj.Select(p => p.Key).ToArray()) + else + { + path = url; + var cut = path.IndexOfAny(['?', '#']); + if (cut >= 0) path = path[..cut]; + } + + foreach (var segment in path.Split('/', StringSplitOptions.RemoveEmptyEntries)) + { + if (IsSensitiveName(DecodeSegment(segment))) return true; + } + return false; + } + + /// + /// A path segment as Graph reads it. An unpaired or invalid escape is not a segment anyone + /// meant, so the raw text stands rather than the call failing on it. + /// + private static string DecodeSegment(string segment) + { + if (!segment.Contains('%')) return segment; + try { return System.Uri.UnescapeDataString(segment); } + catch (System.UriFormatException) { return segment; } + } + + /// + /// Redact every field whose name carries a credential token from a JSON body before writing + /// it to the dead-letter file. Modifies the node in-place and returns the body to write, + /// which is the same node except where a rule took the root itself: a node has no way to + /// replace itself in a parent it does not have. + /// + /// Every node, at every depth: an object's properties, an array's elements whatever they + /// are, and the root. A body is whatever the caller built, so descending only into the + /// shapes one was expected to have leaves the rest unread - a password inside an array that + /// is itself inside an array, {"settings":[[{"password":"..."}]]}, reached the file whole, + /// and a body that is one bare string, a pre-authenticated URL and nothing else, has no + /// property and no element to descend into at all and reached it whole too. + /// + /// A name usually decides, and takes the whole value with it. The other thing read rather + /// than matched by name is a pre-authenticated URL, a credential with no name to give it + /// away: an Intune azureStorageUri and a @microsoft.graph.downloadUrl fetch the bytes on the + /// strength of the URL alone. That read applies to a property's value and, separately, to + /// its name - a capability URL used as a key is what a -Debug trace already redacts, since + /// it matches any quoted string in the text, key or value alike. Either way the cut keeps + /// everything up to the capability parameter, so the line still says which host and which + /// kind of URL failed - and where the key was the URL, the value beside it goes whole, the + /// way a name that says credential takes its value. + /// + internal static JsonNode? RedactSensitiveFields(JsonNode? node) + { + // Before the branches below, because neither of them reaches a root that is one value. + // The rule is the value rule, the one the trace and the file agree on outright: a body + // that is a bare capability URL is the same credential in the same text whether it + // stands under a property name or alone. + if (RedactedUrlValue(node) is { } rootValue) + return rootValue; + + if (node is JsonArray arr) { - if (key.Equals("passwordProfile", StringComparison.OrdinalIgnoreCase) - || key.Equals("password", StringComparison.OrdinalIgnoreCase) - || key.Equals("secretText", StringComparison.OrdinalIgnoreCase) - || key.Equals("keyCredentials", StringComparison.OrdinalIgnoreCase) - || key.Equals("passwordCredentials", StringComparison.OrdinalIgnoreCase) - || key.Equals("clientSecret", StringComparison.OrdinalIgnoreCase) - || key.Equals("appPassword", StringComparison.OrdinalIgnoreCase) - || key.Equals("clientAssertion", StringComparison.OrdinalIgnoreCase)) + for (var i = 0; i < arr.Count; i++) { - obj[key] = "***REDACTED***"; + if (RedactedUrlValue(arr[i]) is { } value) + arr[i] = value; + else + RedactSensitiveFields(arr[i]); } - else if (obj[key] is JsonObject child) + return node; + } + if (node is not JsonObject obj) return node; + foreach (var key in obj.Select(p => p.Key).ToArray()) + { + // The capability-URL cut on the key runs before the fragment check below, not + // after: "guestaccesstoken", "authkey" and a passwordreset path all contain a + // TokenList fragment ("token", "key", "password") as plain text, so testing the + // fragment first took that branch and left the key - secret included - untouched. + // JsonObject has no rename and no positional insert, so the cut removes the + // property and re-adds it under the cut name, which moves it to the end: re-adding + // is the only way to change a key. Two keys that cut to the same string - + // "?sig=AAA" and "?sig=BBB" both cut to "?sig=***REDACTED***" - would silently + // collapse to one entry through that re-add, the second overwriting the first, so a + // colliding cut name is suffixed #2, #3... instead. + // + // The value goes with the key, whatever it holds. A pre-authenticated URL used as a + // field name is a credential, so whatever a body files under one is the thing that + // URL was minted for. Deciding that on a fragment in the key's text let the secret + // choose: "?sig=abc" carries none and kept its value, "?sig=abcsecretdef" carries + // "secret" inside the signature and cleared it, and the two keys name the same kind + // of URL. The whole-value rules meet here - a name that says credential takes its + // value, and so does a name that IS one. + if (SensitiveNames.RedactCapabilityUrl(key, RedactionMarker) is { } redactedKey) { - RedactSensitiveFields(child); + obj.Remove(key); + obj[UniqueKey(obj, redactedKey)] = RedactionMarker; + continue; } - else if (obj[key] is JsonArray arr) + + if (IsSensitiveName(key) || SensitiveNames.NamesDownloadUrl(key)) { - foreach (var item in arr) - { - if (item is JsonObject arrObj) - RedactSensitiveFields(arrObj); - } + obj[key] = RedactionMarker; + continue; } + + if (RedactedUrlValue(obj[key]) is { } redactedValue) + obj[key] = redactedValue; + else + RedactSensitiveFields(obj[key]); } + return node; } + /// + /// , or that name suffixed #2, #3... until one names no property + /// already has. The order met: reflects every + /// key this walk has already assigned, so the first collision on a given cut string gets + /// #2 and the next collision on THAT gets #3. + /// + private static string UniqueKey(JsonObject obj, string candidate) + { + if (!obj.ContainsKey(candidate)) return candidate; + for (var n = 2; ; n++) + { + var next = $"{candidate}#{n}"; + if (!obj.ContainsKey(next)) return next; + } + } + + /// + /// The value rule of the two the trace and the file share: a string that is a URL carrying a + /// capability parameter, cut at that parameter. Null for every other node, which is what + /// leaves the rest of a body readable. + /// + private static string? RedactedUrlValue(JsonNode? node) => + node is JsonValue leaf && leaf.TryGetValue(out var text) + ? SensitiveNames.RedactCapabilityUrl(text, RedactionMarker) + : null; + private static string? TryExtractBatchErrorMessage(GraphBatchResponseItem item) { if (!item.Body.HasValue) return null; @@ -476,7 +1111,64 @@ internal static void RedactSensitiveFields(JsonNode? node) return null; } - private void WriteBatchTelemetry(BatchTelemetry telemetry) + /// + /// What the run did, computed once so that every line stating it states the same thing. + /// + /// Operations submitted. + /// Answered with a status in [200,400). + /// Answered with a status of 400 or above, refusals included. + /// Never POSTed, so certainly not applied. + /// Items the batch-level retry pass put on the wire. + /// What became of that pass. + private sealed record BatchRunSummary( + int Total, int Succeeded, int Failed, int NotSent, int BatchLevelRetries, RetryPass Pass) + { + internal static BatchRunSummary Of( + IReadOnlyList<(BatchOperation Operation, GraphBatchResponseItem Response)> results, + BatchTelemetry telemetry, + bool stopped) + { + // The not-sent count comes from the results rather than from telemetry, which + // counts what was answered. A refused item carries a status and an unsent one does + // not, so the results are what say which is which. + var notSent = results.Count(r => r.Response.Status == GraphBatchClient.NotSentStatus); + + // Which of the two ways a stopped run stopped is decided by what the pass put on + // the wire, not by the stop itself. The count is taken as each of the pass's chunks + // is sent, so nonzero means the pass went out; a run stopped by a chunk failure + // before the pass never reaches the pass at all, and leaves it at zero. + var pass = !stopped ? RetryPass.RanClean + : telemetry.BatchLevelRetries > 0 ? RetryPass.RanRefused + : RetryPass.Withheld; + + return new BatchRunSummary(telemetry.TotalRequests, telemetry.Succeeded, + telemetry.Failed, notSent, telemetry.BatchLevelRetries, pass); + } + } + + /// What became of the batch-level retry pass, which decides what the warning + /// may claim about the attempts an item had. + private enum RetryPass + { + /// Nothing stopped the run: every item had the attempts it qualified for, + /// whether or not any of them needed the pass. + RanClean, + + /// The pass went out and its own POST was refused. What it carried before + /// that may have been applied on the server. + RanRefused, + + /// A chunk failed before the pass, so it was skipped for every candidate and + /// nothing of it reached the wire. + Withheld, + } + + /// + /// Fold what the batch met on the wire into the session view Get-MgxTelemetry reads. Called + /// as soon as the batch returns, ahead of everything this run writes about it: none of + /// those numbers depends on how the run ends, and all of them are lost if it ends early. + /// + private static void RecordSessionTelemetry(BatchTelemetry telemetry) { if (telemetry.ThrottleEncounters > 0) MgxTelemetryCollector.Current.RecordBatchItemThrottles(telemetry.ThrottleEncounters); @@ -485,28 +1177,92 @@ private void WriteBatchTelemetry(BatchTelemetry telemetry) // batch retry waits if (telemetry.TotalRetryDelayMs > 0) MgxTelemetryCollector.Current.RecordBatchRetryDelay(telemetry.TotalRetryDelayMs); + } + private void WriteBatchTelemetry( + BatchTelemetry telemetry, BatchRunSummary summary, int withheldBodyCount, string? deadLetterWriteFailure) + { + // Always emit verbose summary with timing breakdown var elapsedSec = telemetry.TotalElapsedMs / 1000.0; - var throughput = telemetry.TotalElapsedMs > 0 ? telemetry.TotalRequests / elapsedSec : 0; - var summary = $"Batch: {telemetry.Succeeded} succeeded, {telemetry.Failed} failed out of {telemetry.TotalRequests} requests in {elapsedSec:F1}s ({throughput:F1}/sec)."; + var throughput = telemetry.TotalElapsedMs > 0 ? summary.Total / elapsedSec : 0; + var notSentPart = summary.NotSent > 0 ? $", {summary.NotSent} not sent" : string.Empty; + var line = $"Batch: {summary.Succeeded} succeeded, {summary.Failed} failed{notSentPart} out of {summary.Total} requests in {elapsedSec:F1}s ({throughput:F1}/sec)."; if (telemetry.ItemRetries > 0) - summary += $" Item retries: {telemetry.ItemRetries}."; + line += $" Item retries: {telemetry.ItemRetries}."; if (telemetry.ThrottleEncounters > 0) - summary += $" Throttle (429) encounters: {telemetry.ThrottleEncounters}."; - if (telemetry.BatchLevelRetries > 0) - summary += $" Batch-level retries: {telemetry.BatchLevelRetries}."; + line += $" Throttle (429) encounters: {telemetry.ThrottleEncounters}."; + if (summary.BatchLevelRetries > 0) + line += $" Batch-level retries: {summary.BatchLevelRetries}."; if (telemetry.TotalRetryDelayMs > 0) - summary += $" Time in retry delays: {telemetry.TotalRetryDelayMs / 1000.0:F1}s."; - WriteVerbose(summary); + line += $" Time in retry delays: {telemetry.TotalRetryDelayMs / 1000.0:F1}s."; + WriteVerbose(line); - if (telemetry.Failed > 0) + // The one line a caller running without -Verbose sees at all, so it says both what + // failed and what never went out - and does not credit the run with retries it withheld, + // nor deny it the ones it sent. + if (summary.Failed > 0 || summary.NotSent > 0) { - WriteWarning( - $"{telemetry.Failed} of {telemetry.TotalRequests} batch items failed after all retry attempts. " - + "Check $Error for details on each failed item."); + // One sentence, so the base is built without its period and the period is added + // once at the end. The clauses below are semicolon-joined continuations of it, and + // a base that had already closed with a period left them reading as ".; 1 body was + // withheld from the dead-letter file" with nothing closing the sentence after them. + var outcome = summary.NotSent > 0 + ? $"{summary.Failed} of {summary.Total} batch items failed and {summary.NotSent} were not sent" + : $"{summary.Failed} of {summary.Total} batch items failed"; + + // Folded in here, ahead of the retry-pass sentence, rather than left to a dedicated + // warning of its own after WriteBatchTelemetry returns: this outcome warning is the + // first WriteWarning call a failed batch makes, so it is the one a run under + // -WarningAction Stop is most likely to reach and stop on, and item failures are + // exactly when a body can go unread or the dead-letter file can fail to open. A + // caller stopped right here still learns both instead of learning neither. + if (withheldBodyCount > 0) + { + outcome += withheldBodyCount == 1 + ? "; 1 body was withheld from the dead-letter file" + : $"; {withheldBodyCount} bodies were withheld from the dead-letter file"; + } + if (deadLetterWriteFailure != null) + outcome += $"; the dead-letter file could not be written: {Reason(deadLetterWriteFailure)}"; + outcome += "."; + + // "after all retry attempts" holds only of a run that ran them, and "withheld" only + // of a pass that never left. Between them is the pass that went out, was answered, + // and was refused on a later chunk: the items it carried may have been applied, and + // a caller told the pass never ran has no reason to check before resubmitting them. + var retries = summary.Pass switch + { + RetryPass.Withheld => + " A chunk failed, so the run stopped sending and the retry pass was withheld.", + RetryPass.RanRefused => + " The retry pass was sent and then refused, so the run stopped sending.", + _ => " They failed after all retry attempts.", + }; + WriteWarning(outcome + retries + " Check $Error for details on each item."); } } + /// + /// A failure's reason as a sentence quoting it carries it: its first line, and one trailing + /// period off that line. The quoting sentence closes with a period of its own, and a + /// platform message ends in one - "...is denied.", "...missing low surrogate." - so keeping + /// both put "..denied.." mid-line. Only one is trimmed: a reason that genuinely ends in an + /// ellipsis keeps what is left of it. + /// + private static string Reason(string text) + { + var line = FirstLine(text); + return line.EndsWith('.') ? line[..^1] : line; + } + + /// The first line of a possibly multi-line exception message, so a reason folded + /// into the one-line outcome warning stays one line. + private static string FirstLine(string text) + { + var end = text.IndexOfAny(['\r', '\n']); + return end < 0 ? text : text[..end]; + } + internal sealed record BatchInput(string Url, string Method, object? Body, string? Id = null); /// diff --git a/src/Mgx.Cmdlets/Cmdlets/Configuration/DisableMgxResilience.cs b/src/Mgx.Cmdlets/Cmdlets/Configuration/DisableMgxResilience.cs index 17fe8e9..047c838 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Configuration/DisableMgxResilience.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Configuration/DisableMgxResilience.cs @@ -6,7 +6,9 @@ namespace Mgx.Cmdlets.Cmdlets.Configuration; /// /// Removes the Polly resilience injection from the Microsoft.Graph SDK's HTTP transport. -/// Restores the original SDK HttpClient that was saved by Enable-MgxResilience. +/// Restores the original SDK HttpClient that was saved by Enable-MgxResilience, when the +/// session is still holding the wrapper this module installed; a client the SDK has replaced +/// since the injection is left as it was found. /// [Cmdlet(VerbsLifecycle.Disable, "MgxResilience", SupportsShouldProcess = true)] public class DisableMgxResilience : PSCmdlet @@ -17,6 +19,23 @@ protected override void ProcessRecord() { if (!EnableMgxResilience.IsEnabled) { + // A wrapper can outlive the state that tracked it. Module removal takes the + // injection off the session before it drops its references, but a removal that + // ran while GraphSession was unreachable cannot, and what it leaves behind is a + // wrapper nothing points at. Reaching it here is the only way back to the + // genuine client: no other reference to it survives. + if (EnableMgxResilience.SessionHoldsInjectedWrapper()) + { + if (!ShouldProcess("Microsoft.Graph SDK HttpClient", + "Restore original SDK HttpClient (remove Polly resilience)")) + return; + + EnableMgxResilience.TryRestoreGenuineSdkClient(); + WriteVerbose("MgxResilience disabled. The wrapper an earlier import left on " + + "the SDK client is gone and SDK cmdlets are restored to original behavior."); + return; + } + WriteWarning("MgxResilience is not currently enabled."); return; } @@ -63,28 +82,63 @@ protected override void ProcessRecord() return; } - if (!ShouldProcess("Microsoft.Graph SDK HttpClient", - "Restore original SDK HttpClient (remove Polly resilience)")) + // Read before the gate, so what a preview reports is decided from state this cmdlet + // has only looked at. Whether the client on the session is one of ours is the same + // question the teardown asks through TryRestoreGenuineSdkClient, asked the same way: + // the bridge-target map, which spans import cycles, rather than the statics a removal + // drops. + var currentClient = clientProp.GetValue(instance) as HttpClient; + var sessionHoldsOurWrapper = EnableMgxResilience.IsInjectedWrapper(currentClient); + + // And it decides the action the gate names, because the two branches below do + // different things. Only one of them restores anything: the other releases the + // injection's own state and leaves the client the session is holding exactly where + // it found it. Naming a restore for both described a run that, in that case, was + // never going to happen - and the name is all a caller running -WhatIf gets. + var action = sessionHoldsOurWrapper + ? "Restore original SDK HttpClient (remove Polly resilience)" + : "Release Mgx resilience state and leave the SDK HttpClient as found"; + + if (!ShouldProcess("Microsoft.Graph SDK HttpClient", action)) return; - var currentClient = clientProp.GetValue(instance) as HttpClient; - if (currentClient != null && !ReferenceEquals(currentClient, EnableMgxResilience.ResilientSdkClient)) + if (!sessionHoldsOurWrapper) { - WriteWarning("The current GraphHttpClient is not the one injected by Enable-MgxResilience. " + - "Another module or Connect-MgGraph may have replaced it. Restoring original anyway."); + // The SDK built a new client after the injection went on - a Connect-MgGraph to + // another tenant does that - so OriginalSdkClient is the previous identity's + // client and installing it would put those credentials back under every SDK + // cmdlet. The injection's own state still has to go, and ReleaseInjection is the + // unwind module removal runs: it restores nothing here, because the client the + // session is holding is not a wrapper of ours to take off. + EnableMgxResilience.ReleaseInjection(); + WriteWarning("The Microsoft.Graph SDK client was replaced after Enable-MgxResilience " + + "ran (by Connect-MgGraph or another module), so it was not this module's to " + + "restore and has been left as it was found. Mgx resilience is now off."); + return; } clientProp.SetValue(instance, originalClient); - // Dispose the resilient client to release the ResilientDelegatingHandler - // and its references to the Polly pipeline. + // These statics are the last references to the ResilientDelegatingHandler and to + // the client wrapped around it, and dropping them is what releases those. Not the + // Polly pipeline: the factory keeps one instance in a static of its own and hands + // the same one to every client it builds, so it outlives this teardown - which is + // the point of it. Releasing it would mean ResiliencePipelineFactory.Reset(), and + // Disable must not call that: the circuit-breaker history, the rate limiter and + // the learned pacing it holds are shared with mgx's own cmdlets, which keep running + // after the SDK injection comes off. var resilientClient = EnableMgxResilience.ResilientSdkClient; EnableMgxResilience.IsEnabled = false; EnableMgxResilience.OriginalSdkClient = null; EnableMgxResilience.ResilientSdkClient = null; + // The handler goes with them. It was left set here, so a disabled session still + // rooted the Polly pipeline and the bridge to the client we just stopped using. + EnableMgxResilience.ActiveHandler = null; // Not disposed. Restoring GraphSession.GraphHttpClient already stops new traffic; // disposing would cancel SDK requests still in flight - a paged read running when the - // user types Disable-MgxResilience would die mid-enumeration rather than finish. + // user types Disable-MgxResilience would die mid-enumeration rather than finish. It + // holds no connections of its own either: the SDK client it bridges to owns those, + // and that client's pool closes its sockets on its own timers. _ = resilientClient; WriteVerbose("MgxResilience disabled. SDK cmdlets restored to original behavior."); diff --git a/src/Mgx.Cmdlets/Cmdlets/Configuration/EnableMgxResilience.cs b/src/Mgx.Cmdlets/Cmdlets/Configuration/EnableMgxResilience.cs index 928bf22..bf71b5b 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Configuration/EnableMgxResilience.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Configuration/EnableMgxResilience.cs @@ -1,6 +1,7 @@ using System.Management.Automation; using System.Net; using System.Reflection; +using System.Runtime.CompilerServices; using Mgx.Engine.Http; using Mgx.Cmdlets.Base; @@ -27,11 +28,108 @@ public class EnableMgxResilience : PSCmdlet // Lock protecting all static state transitions. Used by both Enable and Disable. internal static readonly object StateLock = new(); + // State for Disable-MgxResilience to restore internal static HttpClient? OriginalSdkClient { get; set; } internal static HttpClient? ResilientSdkClient { get; set; } internal static bool IsEnabled { get; set; } internal static ResilientDelegatingHandler? ActiveHandler { get; set; } + // Every wrapper this module has installed, against the genuine SDK client underneath it. The + // keys are weak, so an entry lasts exactly as long as the wrapper somebody still holds and + // the table is not itself a reference the injection has to release. It is how a wrapper left + // by an earlier import is recognized after the statics that tracked it are gone: this + // assembly is never unloaded, so the table spans any number of import cycles. + private static readonly ConditionalWeakTable s_bridgeTargets = new(); + + /// + /// The genuine SDK client under , which is + /// itself unless mgx wrapped it. Wrapping a wrapper multiplies every layer: under a throttle + /// each layer retries the one beneath it, so the attempts reaching the wire go up per layer, + /// telemetry counts each attempt once per layer, and the pacer halves once per layer. + /// + internal static HttpClient ResolveGenuineSdkClient(HttpClient client) + { + while (s_bridgeTargets.TryGetValue(client, out var inner) && !ReferenceEquals(inner, client)) + client = inner; + return client; + } + + /// Whether this client is a wrapper mgx installed, in this import or an earlier one. + internal static bool IsInjectedWrapper(HttpClient? client) => + client != null && s_bridgeTargets.TryGetValue(client, out _); + + /// Whether GraphSession is holding a wrapper of ours right now. + internal static bool SessionHoldsInjectedWrapper() + { + try + { + var instance = MgxCmdletBase.TryGetGraphSessionInstance(); + var current = instance?.GetType().GetProperty("GraphHttpClient")?.GetValue(instance); + return IsInjectedWrapper(current as HttpClient); + } + catch (Exception ex) + { + System.Diagnostics.Debug.WriteLine($"[Mgx] Could not read GraphHttpClient: {ex.Message}"); + return false; + } + } + + /// + /// Puts GraphSession.GraphHttpClient back on the genuine SDK client when the client installed + /// there is a wrapper of ours, and reports whether it did. + /// + /// The wrapper is not disposed: an SDK request already in flight is sending through it, and + /// disposing cancels that request mid-enumeration. Restoring the property stops new traffic, + /// and the wrapper is collected once the requests still using it finish. + /// + /// + internal static bool TryRestoreGenuineSdkClient() + { + try + { + var instance = MgxCmdletBase.TryGetGraphSessionInstance(); + var clientProp = instance?.GetType().GetProperty("GraphHttpClient"); + if (instance == null || clientProp == null) return false; + + if (clientProp.GetValue(instance) is not HttpClient current || !IsInjectedWrapper(current)) + return false; + + clientProp.SetValue(instance, ResolveGenuineSdkClient(current)); + return true; + } + catch (Exception ex) + { + // Module teardown calls this: a reflection failure must not stop the module unloading. + System.Diagnostics.Debug.WriteLine($"[Mgx] Could not restore the SDK client: {ex.Message}"); + return false; + } + } + + /// + /// Unwinds the injection: the session goes back to the genuine SDK client before mgx lets go + /// of the wrapper. Nothing is disposed - see TryRestoreGenuineSdkClient. + /// + /// These statics outlive Remove-Module, so what is left here is what the next import finds. + /// Dropping the references without restoring the session strands the wrapper: it stays + /// installed, and every SDK request the process makes from then on goes through a handler + /// belonging to a module that is no longer loaded, until somebody imports mgx and disables + /// it. What the wrapper bridges to is not lost with these references - s_bridgeTargets is + /// deliberately not one of them - so a later Enable-MgxResilience still wraps the genuine + /// client rather than the wrapper, and Disable-MgxResilience still has it to restore. + /// + /// + internal static void ReleaseInjection() + { + lock (StateLock) + { + TryRestoreGenuineSdkClient(); + IsEnabled = false; + ActiveHandler = null; + ResilientSdkClient = null; + OriginalSdkClient = null; + } + } + protected override void ProcessRecord() { lock (StateLock) @@ -60,12 +158,33 @@ protected override void ProcessRecord() var clientProp = instance.GetType().GetProperty("GraphHttpClient"); var currentClient = clientProp?.GetValue(instance) as HttpClient; + // Already enabled and our client is still the one installed: nothing to do. Reading + // the session is not a change, so this answer is the same with -WhatIf as without. + // The null test earns its place now that the session's client is only forced below + // the gate: a session holding nothing would otherwise compare equal to an injection + // whose build failed, and report itself active. + if (IsEnabled && currentClient != null && ReferenceEquals(currentClient, ResilientSdkClient)) + { + WriteVerbose("MgxResilience is already active."); + return; + } + + // Everything past this line changes state, so all of it is under the gate. The + // pre-initialization resets the pipeline factory and the SDK probe sends a real + // request and leaves a client on the session; dropping the wrapper we installed + // leaves Disable-MgxResilience with no record of what it is taking off. What -WhatIf + // reports is decided above it, from state this cmdlet has only read. + if (!ShouldProcess("Microsoft.Graph SDK HttpClient", + "Replace with Polly resilience pipeline (retry, circuit breaker, rate limiting)")) + return; + // Pre-initialize Mgx's own HTTP client before the SDK probe runs. // The probe calls Invoke-MgGraphRequest which changes Azure Identity internal // state and breaks GetAuthenticationProviderAsync for subsequent callers. // Building Mgx's clean client first ensures it is cached before that happens. MgxCmdletBase.TryPreInitHttpClient(WriteWarning, WriteVerbose); + // Force SDK to initialize its HttpClient if not yet initialized if (currentClient == null) { WriteVerbose("GraphHttpClient not initialized. Triggering initialization..."); @@ -83,33 +202,32 @@ protected override void ProcessRecord() if (IsEnabled) { - if (ReferenceEquals(currentClient, ResilientSdkClient)) - { - WriteVerbose("MgxResilience is already active."); - return; - } // Our client was replaced (e.g., by Connect-MgGraph or Set-MgRequestContext). - // Dispose the old wrapped client to release its handler chain and sockets. WriteVerbose("MgxResilience was reset by SDK. Re-injecting resilience..."); // Not disposed: HttpClient.Dispose cancels its pending-request token source and // the bridge handler forwards that token inward, so SDK requests already in - // flight die. Restoring GraphSession.GraphHttpClient stops new traffic. The old - // client is collected once the requests still using it finish. + // flight die. Restoring GraphSession.GraphHttpClient stops new traffic; the old + // client is collected once the requests still using it finish. It holds no + // connections of its own to release either - it bridges to the SDK's client, + // and that client's pool closes its sockets on its own timers. _ = ResilientSdkClient; ResilientSdkClient = null; // Reset circuit breaker / rate limiter state from the previous tenant ResiliencePipelineFactory.Reset(); } - if (!ShouldProcess("Microsoft.Graph SDK HttpClient", - "Replace with Polly resilience pipeline (retry, circuit breaker, rate limiting)")) - return; + // Wrap the genuine client, never a wrapper. The session can still be holding one from + // an earlier import - its statics went with that import - and a second layer over it + // is what OriginalSdkClient would then be restored to by Disable-MgxResilience. + currentClient = ResolveGenuineSdkClient(currentClient); + // Save the current SDK client AFTER we know build will be attempted OriginalSdkClient = currentClient; var resilientClient = BuildResilientSdkClient(currentClient, WriteWarning); if (resilientClient == null) { + // Rollback: don't leave stale OriginalSdkClient on failure OriginalSdkClient = null; ThrowTerminatingError(new ErrorRecord( new InvalidOperationException( @@ -118,6 +236,7 @@ protected override void ProcessRecord() return; } + // Replace the SDK's HttpClient clientProp!.SetValue(instance, resilientClient); ResilientSdkClient = resilientClient; IsEnabled = true; @@ -129,7 +248,8 @@ protected override void ProcessRecord() private HttpClient? ForceInitializeAndGetClient(object instance, PropertyInfo? clientProp) { - var endpoint = MgxCmdletBase.GetGraphEndpoint(WriteWarning, WriteVerbose) ?? "https://graph.microsoft.com"; + // Use the Graph endpoint from the session (sovereign cloud support) + var endpoint = MgxCmdletBase.GetGraphEndpoint(WriteWarning, WriteVerbose) ?? MgxCmdletBase.DefaultGraphEndpoint; // Save AzureADEndpoint before probe. Invoke-MgGraphRequest replaces // GraphSession.Environment with a new object that has an empty AzureADEndpoint, @@ -193,7 +313,7 @@ internal static void RefreshInjectedClient(Action warn, Action v currentClient = null; } - // Not disposed - see the note above. In-flight SDK requests would be cancelled. + // Not disposed - see the note above; in-flight SDK requests would be cancelled. _ = ResilientSdkClient; ResilientSdkClient = null; ActiveHandler = null; @@ -208,6 +328,8 @@ internal static void RefreshInjectedClient(Action warn, Action v return; } + currentClient = ResolveGenuineSdkClient(currentClient); + var refreshed = BuildResilientSdkClient(currentClient, warn); if (refreshed == null) { @@ -226,17 +348,22 @@ internal static void RefreshInjectedClient(Action warn, Action v } /// - /// Turns off the SDK own retry handler for requests passing through the wrap. + /// Turns off the SDK's own retry handler for requests that pass through the wrap. /// /// That handler sits inside the wrapped chain and answers 429 and 503 itself, so a throttle - /// never reaches the Mgx pipeline. The pacer learns nothing, telemetry books a throttled - /// session as zero retries, and the two retriers compound. + /// never reached Mgx's pipeline: the adaptive pacer stayed in slow start through a live + /// throttle, Get-MgxTelemetry reported no throttle retries while accumulating two minutes of + /// retry delay under another name, and the two retriers compounded - four intended attempts + /// could reach the wire eight times, because the SDK sleeps Retry-After inside the call and + /// Mgx's own attempt timeout then fires. /// - /// Kiota reads this option per request. The type is resolved reflectively, so neither - /// assembly needs a reference to it and a rename leaves the wrap working as before. + /// Kiota reads this option per request. The type is resolved reflectively so the engine and + /// this assembly need no reference to it; if the SDK ever moves or renames it, the wrap keeps + /// working exactly as it did before. /// - /// Throws rather than warns, because it runs on a request thread with no pipeline to write - /// a warning to. The handler catches it and carries on with the inner handler as it was. + /// Throws rather than warning on the paths it cannot satisfy. It runs from the handler's + /// option factory, on a request thread with no pipeline to write a warning to - the handler + /// catches this, notes it, and carries on with the inner retry handler left as it was. /// internal static IReadOnlyDictionary? BuildInnerRetryOverride() { @@ -255,11 +382,13 @@ internal static void RefreshInjectedClient(Action warn, Action v "the Graph SDK's retry option could not be configured, so its own retry handler " + "stays active inside the wrap"); - // MaxRetry is the only lever that removes a retry. ShouldRetry cannot suppress one, - // because the handler ORs it with its own status check. - // The cost is that the handler 503 and 504 retries go too, including on writes, which - // the Mgx pipeline will not take over since it refuses to retry a non-idempotent - // request on a + // MaxRetry is the only lever that removes a retry. The option also exposes ShouldRetry, + // which looks like a way to decline 429 alone and leave the handler's 503 and 504 + // retries intact - it is not: the handler ORs it with its own status check, so + // ShouldRetry can only add retries, never suppress one. Measured against 1.21.1. + // + // The cost is that the handler's 503/504 retries go too, including on writes, and Mgx's + // pipeline will not take those over: it refuses to retry a non-idempotent request on a // 5xx because the write may already have been applied. 429 is unaffected - the pipeline // retries that for every method - so throttled writes still complete. maxRetry.SetValue(option, 0); @@ -267,18 +396,32 @@ internal static void RefreshInjectedClient(Action warn, Action v return new Dictionary { [OptionType] = option }; } - private static HttpClient? BuildResilientSdkClient(HttpClient sdkClient, Action warn) + internal static HttpClient? BuildResilientSdkClient(HttpClient sdkClient, Action warn) { try { + // Bridge to the genuine client whatever the caller hands over: a wrapper wrapping a + // wrapper multiplies wire attempts, telemetry counts and pacer steps per layer. + sdkClient = ResolveGenuineSdkClient(sdkClient); + var (pipeline, rateLimiter) = ResiliencePipelineFactory.GetOrCreate(MgxCmdletBase.s_clientOptions); - // Wrap the existing SDK client, preserving its full handler chain, with the - // resilience layer on top. The chain is ResilientDelegatingHandler, then - // SdkClientBridgeHandler, then the SDK client and its own handlers. - // AdditionalRequestOptionsFactory disarms the SDK retry handler per request. If that - // cannot be arranged it stays armed and the session behaves as before, without the - // measurement. Both paths share one pipeline, rate limiter and circuit breaker + // Wrap the existing SDK client (preserving its full handler chain: + // ODataQueryOptionsHandler, NationalCloudHandler, RedirectHandler, + // AuthenticationHandler, etc.) with our resilience layer on top. + // + // Handler chain: ResilientDelegatingHandler -> SdkClientBridgeHandler -> sdkClient + // The bridge handler delegates SendAsync to the original SDK HttpClient, + // which processes through its complete handler pipeline internally. + // + // The SDK's built-in RetryHandler sits inside this wrap and would otherwise answer + // 429 and 503 itself, before the outer pipeline ever saw them - so the pacer never + // learned from a throttle and telemetry booked a throttled session as zero retries. + // AdditionalRequestOptionsFactory disarms it per request. If that cannot be arranged + // it stays armed and the session behaves as it did before, just without the + // measurement. Both paths share the same pipeline, rate limiter, + // and circuit breaker to prevent cache thrashing and ensure consistent + // failure detection across SDK and direct Mgx cmdlets. var resilientHandler = new ResilientDelegatingHandler(pipeline, rateLimiter) { InnerHandler = new SdkClientBridgeHandler(sdkClient), @@ -290,11 +433,17 @@ internal static void RefreshInjectedClient(Action warn, Action v // against the active client's BaseAddress before any handler runs. Default // request headers are NOT copied - the bridge delegates to sdkClient.SendAsync, // which applies the original client's defaults to each request anyway. - return new HttpClient(resilientHandler) + var wrapper = new HttpClient(resilientHandler) { BaseAddress = sdkClient.BaseAddress, Timeout = sdkClient.Timeout }; + + // What the wrapper bridges to, recorded where it survives module removal. The + // teardown, Disable-MgxResilience and the next Enable-MgxResilience all need the + // genuine client back, and after a removal this is the only thing that still knows it. + s_bridgeTargets.AddOrUpdate(wrapper, sdkClient); + return wrapper; } catch (Exception ex) { @@ -303,6 +452,10 @@ internal static void RefreshInjectedClient(Action warn, Action v } } + /// + /// Bridges from a DelegatingHandler chain to an existing HttpClient, preserving + /// the SDK's full handler pipeline (OData, NationalCloud, Redirect, Auth, etc.). + /// private sealed class SdkClientBridgeHandler : HttpMessageHandler { private readonly HttpClient _sdkClient; diff --git a/src/Mgx.Cmdlets/Cmdlets/Configuration/SetMgxOption.cs b/src/Mgx.Cmdlets/Cmdlets/Configuration/SetMgxOption.cs index 7c2e9ab..b7a86e8 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Configuration/SetMgxOption.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Configuration/SetMgxOption.cs @@ -6,7 +6,7 @@ namespace Mgx.Cmdlets.Cmdlets.Configuration; /// /// Set-MgxOption: Configure resilience options for all Mgx cmdlets. -/// Only parameters explicitly passed are updated. Unspecified values retain their current settings. +/// Only parameters explicitly passed are updated; unspecified values retain their current settings. /// Options take effect on the next cmdlet invocation. /// Use -Reset to restore all options to their defaults. /// @@ -56,7 +56,7 @@ public class SetMgxOption : PSCmdlet public double CircuitBreakerFailureRatio { get; set; } [Parameter] - [ValidateRange(1, 1000)] + [ValidateRange(2, 1000)] public int CircuitBreakerMinThroughput { get; set; } [Parameter] @@ -74,6 +74,10 @@ public class SetMgxOption : PSCmdlet [Parameter] public SwitchParameter Reset { get; set; } + /// + /// Parameters PowerShell adds to every cmdlet. Present in BoundParameters, but none of them + /// is a change to an option. + /// private static readonly HashSet CommonParameterNames = new(StringComparer.OrdinalIgnoreCase) { "Verbose", "Debug", "ErrorAction", "WarningAction", "InformationAction", "ProgressAction", @@ -85,12 +89,51 @@ protected override void ProcessRecord() { var bound = MyInvocation.BoundParameters; + // What the target names is what this run would set, which is not everything bound to it: + // PowerShell binds the common parameters too, so `-Verbose -WhatIf` previewed a change to + // "MgxOptions (Verbose, WhatIf)" - two names that set no option - for a run the guard + // below then reports as unchanged. Filtered through the set that guard counts with, in + // the order they were bound, dropped entirely when it leaves nothing to name, and never + // includes -Reset itself: what it names beside a reset is what the reset ignores, not the + // switch that caused it. + var optionNames = bound.Keys.Cast() + .Where(k => !CommonParameterNames.Contains(k) && k != nameof(Reset)) + .ToArray(); + + // No -Reset and no option: nothing to do (avoids unnecessary pipeline rebuild, which + // would destroy circuit breaker failure history). Ahead of ShouldProcess on purpose - the + // guard used to sit after the gate, so a bare `Set-MgxOption` previewed "Set" on + // "MgxOptions" and prompted for it under -Confirm before landing here and reporting the + // run it had just asked about as unchanged. + // + // Counted without the common parameters, which BoundParameters includes: -Verbose alone + // made this look like a real change, so `Set-MgxOption -Verbose` rebuilt the pipeline and + // discarded exactly the history this guard was written to keep. The same list the target + // below names, so what a preview says and what a run does cannot come apart. + if (!Reset.IsPresent && optionNames.Length == 0) + { + WriteVerbose("No parameters specified. Options unchanged."); + return; + } + + // -Reset restores every default; an option bound beside it does not survive that. Silently + // dropping it would read as the option having taken effect over the defaults it did not + // survive, so the run says what it is ignoring before it asks whether to proceed. + if (Reset.IsPresent && optionNames.Length > 0) + { + WriteWarning("-Reset resets every option; " + + $"{string.Join(", ", optionNames.Select(n => $"-{n}"))} given beside it are ignored."); + } + var target = Reset.IsPresent - ? "MgxOptions (reset to defaults)" - : $"MgxOptions ({string.Join(", ", bound.Keys)})"; + ? optionNames.Length == 0 + ? "MgxOptions (reset to defaults)" + : $"MgxOptions (reset to defaults; ignoring {string.Join(", ", optionNames)})" + : $"MgxOptions ({string.Join(", ", optionNames)})"; if (!ShouldProcess(target, "Set")) return; + // -Reset: restore all defaults and return if (Reset.IsPresent) { MgxCmdletBase.SetClientOptions(ResilientGraphClientOptions.Default); @@ -99,18 +142,7 @@ protected override void ProcessRecord() return; } - // No parameters passed: nothing to do (avoids unnecessary pipeline rebuild - // which would destroy circuit breaker failure history). - // - // Counted without the common parameters, which BoundParameters includes: -Verbose alone - // made this look like a real change, so `Set-MgxOption -Verbose` rebuilt the pipeline and - // discarded exactly the history this guard was written to keep. - if (!bound.Keys.Cast().Any(k => !CommonParameterNames.Contains(k))) - { - WriteVerbose("No parameters specified. Options unchanged."); - return; - } - + // Start from current options, only override values the user actually passed var current = MgxCmdletBase.s_clientOptions; // If user explicitly set rate params but NOT NoRateLimit, diff --git a/src/Mgx.Cmdlets/Cmdlets/Content/GetMgxContent.cs b/src/Mgx.Cmdlets/Cmdlets/Content/GetMgxContent.cs index c7822f9..d824c0c 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Content/GetMgxContent.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Content/GetMgxContent.cs @@ -9,18 +9,25 @@ namespace Mgx.Cmdlets.Cmdlets.Content; /// -/// Fetches content bytes from $value and /content endpoints, whole or as a byte range. -/// The request runs in two hops. Hop one is the authenticated Graph request through the full -/// resilience pipeline. When Graph answers with a 302 to a pre-authenticated download host, -/// hop two fetches it without a token, against an allowlist, so the bearer token never reaches -/// that host. -/// Output is a single byte array to the pipeline, or a file via -OutFile. +/// Get-MgxContent: Fetch content bytes ($value / /content endpoints), whole or as a byte +/// range. Ranged reads are the point: pulling 256 KB headers off 20k files moves ~5 GB +/// instead of 63 GB for a metadata job. +/// +/// Two hops under the hood: the authenticated Graph request (full resilience pipeline, +/// pacing, rate limiting), then - when Graph 302s to a pre-authenticated download host - a +/// token-free fetch whose target is validated against a Microsoft-hosts allowlist. The +/// bearer token never reaches the download host. See GraphContentClient for the mechanism +/// and the transport preconditions. +/// +/// Output is a single byte[] to the pipeline, or a file via -OutFile (temp + atomic move). +/// Piped DriveItems use their @microsoft.graph.downloadUrl directly (validated first) or +/// fall back to /drives/{driveId}/items/{id}/content. /// [Cmdlet(VerbsCommon.Get, "MgxContent", DefaultParameterSetName = "Uri")] [OutputType(typeof(byte[]))] public class GetMgxContent : MgxCmdletBase { - /// A byte array larger than this must go to -OutFile. + /// Pipeline output guard: a byte[] larger than this must go to -OutFile. internal const long MaxPipelineBytes = 100L * 1024 * 1024; [Parameter(Mandatory = true, Position = 0, ParameterSetName = "Uri")] @@ -30,14 +37,17 @@ public class GetMgxContent : MgxCmdletBase [Parameter(Mandatory = true, ValueFromPipeline = true, ParameterSetName = "InputObject")] public object? InputObject { get; set; } + /// First N bytes (Range: bytes=0..N-1). Mutually exclusive with -Offset/-Length. [Parameter] [ValidateRange(1, long.MaxValue)] public long First { get; set; } + /// Range start, used with -Length (Range: bytes=Offset..Offset+Length-1). [Parameter] [ValidateRange(0, long.MaxValue)] public long Offset { get; set; } + /// Range length, from -Offset (default 0). [Parameter] [ValidateRange(1, long.MaxValue)] public long Length { get; set; } @@ -56,8 +66,11 @@ public class GetMgxContent : MgxCmdletBase private string VersionedBaseUrl => $"{s_graphEndpoint}/{ApiVersion}"; private string? _resolvedOutFile; - // -OutFile carries no ParameterSetName, so it binds in the InputObject set too. One - // destination and many piped items would download all of them and keep only the last + // -OutFile carries no ParameterSetName, so it binds in the InputObject set too and the help + // documents that combination. With a single destination and File.Move(overwrite: true) per + // record, piping N items downloaded all N and left one file - the last writer winning, with + // no warning and full transfer cost for the discarded ones. Export-MgxCollection and + // Sync-MgxDelta guard the analogous case; this did not. private bool _wroteOutFile; private bool _transportChecked; @@ -65,8 +78,8 @@ protected override void BeginProcessing() { if (ParameterSetName == "Uri") { - if (Uri.StartsWith("https://", StringComparison.OrdinalIgnoreCase) || - Uri.StartsWith("http://", StringComparison.OrdinalIgnoreCase)) + if (Uri.TrimStart().StartsWith("https://", StringComparison.OrdinalIgnoreCase) || + Uri.TrimStart().StartsWith("http://", StringComparison.OrdinalIgnoreCase)) { ThrowTerminatingError(new ErrorRecord( new ArgumentException( @@ -84,7 +97,7 @@ protected override void BeginProcessing() } } - // -First and -Offset/-Length are two ways to say the same thing and must not disagree + // -First XOR -Offset/-Length: two ways to say the same thing must not disagree. var boundParams = MyInvocation.BoundParameters; var hasFirst = boundParams.ContainsKey(nameof(First)); var hasOffset = boundParams.ContainsKey(nameof(Offset)); @@ -110,30 +123,46 @@ protected override void BeginProcessing() protected override void ProcessRecord() { - if (ParameterSetName == "Uri") + if (ParameterSetName != "Uri" && InputObject == null) { - FetchContent(downloadUrl: null, - relativeUri: Uri, - errorTarget: Uri); + // A null item was never a download; it must not trip the one-OutFile guard. + WriteVerbose("Skipping null pipeline input."); return; } - if (InputObject == null) + if (_wroteOutFile && !string.IsNullOrEmpty(OutFile)) { - WriteVerbose("Skipping null pipeline input."); + // Refuse rather than overwrite - and refuse BEFORE fetching, so the second + // item's content is not downloaded just to be discarded. Checked here, not in + // BeginProcessing, so the legitimate single-piped-item case still works. + ThrowTerminatingError(new ErrorRecord( + new InvalidOperationException( + "-OutFile writes a single file, but more than one item was piped in. " + + "Each item would overwrite the last. Pipe one item, or omit -OutFile " + + "and redirect the byte[] output per item."), + "OutFileWithMultipleInputs", ErrorCategory.InvalidOperation, OutFile)); return; } - var value = UnwrapPSObject(InputObject); + if (ParameterSetName == "Uri") + { + FetchContent(downloadUrl: null, + relativeUri: Uri, + errorTarget: Uri); + return; + } - // The item own pre-authenticated download URL avoids the hop-one round trip and its - // request-budget charge. Validated against the allowlist before any fetch + var value = UnwrapPSObject(InputObject!); // null handled above + + // Prefer the item's own pre-authenticated download URL: no hop-1 round trip, no + // request-budget charge. Validated against the allowlist before any fetch. if (TryGetMember(value, "@microsoft.graph.downloadUrl")?.ToString() is { Length: > 0 } itemDownloadUrl) { FetchContent(downloadUrl: itemDownloadUrl, relativeUri: null, errorTarget: InputObject); return; } + // Fall back to /drives/{driveId}/items/{id}/content from the item's identifiers. var id = Cmdlets.InvokeMgxRequest.ResolvePipelineId(value); var driveId = TryGetMember(TryGetMember(value, "parentReference"), "driveId")?.ToString(); if (string.IsNullOrEmpty(id) || string.IsNullOrEmpty(driveId)) @@ -148,7 +177,7 @@ protected override void ProcessRecord() } FetchContent(downloadUrl: null, - relativeUri: $"/drives/{driveId}/items/{id}/content", + relativeUri: $"/drives/{System.Uri.EscapeDataString(driveId)}/items/{System.Uri.EscapeDataString(id)}/content", errorTarget: InputObject); } @@ -159,6 +188,7 @@ protected override void ProcessRecord() return null; } + /// Bytes the caller actually asked for; null = the whole file. private long? RequestedBytes => First > 0 ? First : Length > 0 ? Length : null; private void FetchContent(string? downloadUrl, string? relativeUri, object? errorTarget) @@ -191,7 +221,8 @@ private void FetchContent(string? downloadUrl, string? relativeUri, object? erro { using var result = downloadUrl != null ? GraphContentClient.GetFromDownloadUrlAsync( - downloadUrl, range, client.BodyReadTimeout, CancellationToken) + downloadUrl, range, client.BodyReadTimeout, CancellationToken, + BuildRequestHeaders(null, Headers)) .GetAwaiter().GetResult() : client.GetContentAsync( $"{VersionedBaseUrl}{NormalizePath(relativeUri!)}", range, @@ -200,15 +231,17 @@ private void FetchContent(string? downloadUrl, string? relativeUri, object? erro DrainClientMessages(); - // The server ignored the range and answered 200 with the full body. Copy only the - // requested bytes, disposing the result aborts the rest of the transfer + // Truncation path: a range was requested but the server ignored it and answered + // 200 with the full body (profile photos do this). Copy only the requested bytes; + // disposing the result aborts the rest of the transfer. long? maxBytes = null; long skipBytes = 0; if (range != null && result.StatusCode == HttpStatusCode.OK) { maxBytes = RequestedBytes; - // The server ignored the offset as well, so discard the head locally or the - // caller receives the wrong bytes and is told it worked + // The server ignored the offset too, not just the length. Discarding the head + // locally is the only way -Offset can mean what it says on this path; without + // it the caller silently receives bytes 0..Length-1 and is told it worked. skipBytes = Offset; WriteVerbose(skipBytes > 0 ? $"Server ignored the Range header (HTTP 200). Discarding the first {skipBytes:N0} bytes and taking {maxBytes:N0}." @@ -217,17 +250,6 @@ private void FetchContent(string? downloadUrl, string? relativeUri, object? erro if (_resolvedOutFile != null) { - if (_wroteOutFile) - { - // Refuse rather than overwrite. Checked here so a single piped item still works - ThrowTerminatingError(new ErrorRecord( - new InvalidOperationException( - "-OutFile writes a single file, but more than one item was piped in. " - + "Each item would overwrite the last. Pipe one item, or omit -OutFile " - + "and redirect the byte[] output per item."), - "OutFileWithMultipleInputs", ErrorCategory.InvalidOperation, OutFile)); - return; - } WriteToFile(result, maxBytes, skipBytes); _wroteOutFile = true; } @@ -241,9 +263,17 @@ private void FetchContent(string? downloadUrl, string? relativeUri, object? erro DrainClientMessages(); WriteWarning("Content download cancelled."); } + catch (GraphServiceException ex) when (ex.StatusCode == HttpStatusCode.NotModified) + { + // The conditional request worked: the caller's If-None-Match/If-Modified-Since + // matched, and there is nothing newer to download. Not an error. + DrainClientMessages(); + WriteVerbose("Not modified; the content matches the caller's condition. Nothing downloaded."); + } catch (InvalidOperationException ex) { - // A validator refusal is a security boundary, not a transient fault + // Validator refusal or transport-redirect detection: a security boundary, not a + // transient fault - surface as such, per item. DrainClientMessages(); WriteError(new ErrorRecord(ex, "ContentDownloadRefused", ErrorCategory.SecurityError, errorTarget)); } @@ -253,17 +283,29 @@ private void FetchContent(string? downloadUrl, string? relativeUri, object? erro } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { - // UnauthorizedAccessException is not an IOException. FileStream throws it for an - // unwritable parent directory, and File.Move throws it for a read-only destination - // on Windows, so both are caught here + // UnauthorizedAccessException is NOT an IOException. FileStream throws it for an + // unwritable parent directory or a destination that is itself a directory, and + // File.Move(overwrite: true) throws it for a read-only destination on Windows - so + // an unwritable -OutFile escaped every filter here, skipped DrainClientMessages, and + // surfaced as an unhandled error naming a temp path the caller never supplied. + // Eleven other sites in this codebase already pair the two; this was the only + // file-writing cmdlet that did not. DrainClientMessages(); WriteError(new ErrorRecord(ex, "IOError", ErrorCategory.WriteError, OutFile)); } + catch (Exception) + { + // Unexpected exception types skip the drains above; the buffered verbose and + // warning messages are the context that explains the failure. + DrainClientMessages(); + throw; + } } private void WriteToFile(GraphContentResult result, long? maxBytes, long skipBytes) { - // Temp plus atomic move, so a failed download never truncates an existing file + // Temp + atomic move, like every other file-writing path in mgx: a failed download + // never truncates an existing file. var tempPath = $"{_resolvedOutFile}.{Guid.NewGuid():N}.tmp"; long copied; try @@ -290,7 +332,7 @@ private void WriteToFile(GraphContentResult result, long? maxBytes, long skipByt private void WriteToPipeline(GraphContentResult result, long? maxBytes, long skipBytes, object? errorTarget) { - // Known oversized before a byte moves, so refuse early + // Known-oversized before a single byte moves: refuse early. var expected = maxBytes ?? result.ContentLength; if (expected > MaxPipelineBytes) { @@ -302,8 +344,8 @@ private void WriteToPipeline(GraphContentResult result, long? maxBytes, long ski return; } - // Unknown length, so enforce during the copy. Cap one byte over the limit to tell a - // capped body from one that is exactly at the limit + // Unknown length (chunked): enforce the guard during the copy - cap one byte over + // the limit so hitting the cap is distinguishable from an exact-limit body. var copyLimit = maxBytes ?? MaxPipelineBytes + 1; using var buffer = new MemoryStream(expected is > 0 and <= int.MaxValue ? (int)expected : 0); var copied = GraphContentClient.CopyWithIdleTimeoutAsync( diff --git a/src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs b/src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs index 7733544..66ff48f 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Delta/SyncMgxDelta.cs @@ -58,9 +58,20 @@ public class SyncMgxDelta : MgxCmdletBase [Parameter] public SwitchParameter FullSync { get; set; } + /// + /// Baseline without enumerating: request only the latest delta token ("sync from now"). + /// Drive resources take ?token=latest; directory and other resources take + /// $deltatoken=latest - the form is chosen automatically from the URI shape. + /// Ignored (with a warning) when usable delta state already exists. + /// [Parameter] public SwitchParameter Latest { get; set; } + /// + /// Path for the ephemeral mid-run resume checkpoint. Deleted on successful completion; + /// any event that invalidates the enumeration (410 Gone, -FullSync, a -Property/-Filter/ + /// -Prefer change) deletes it too, so a stale position can never be resumed. + /// [Parameter] public string? CheckpointPath { get; set; } @@ -74,6 +85,11 @@ public class SyncMgxDelta : MgxCmdletBase private string VersionedBaseUrl => $"{s_graphEndpoint}/{ApiVersion}"; + /// + /// Normalize $select for stable comparison: sort, deduplicate, trim, case-insensitive. + /// Saved to DeltaState.Select so future comparisons are order-independent. + /// Also used for Prefer tokens - the same normalization semantics apply. + /// private static string NormalizeSelect(string? s) => string.IsNullOrEmpty(s) ? "" : string.Join(",", s.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) @@ -82,8 +98,9 @@ private static string NormalizeSelect(string? s) => protected override void BeginProcessing() { - if (Uri.StartsWith("https://", StringComparison.OrdinalIgnoreCase) || - Uri.StartsWith("http://", StringComparison.OrdinalIgnoreCase)) + // Reject absolute URLs (relative paths only) + if (Uri.TrimStart().StartsWith("https://", StringComparison.OrdinalIgnoreCase) || + Uri.TrimStart().StartsWith("http://", StringComparison.OrdinalIgnoreCase)) { ThrowTerminatingError(new ErrorRecord( new ArgumentException( @@ -93,9 +110,11 @@ protected override void BeginProcessing() return; } + // Fail fast: validate delta file is writable before HTTP calls var resolvedDeltaPath = GetUnresolvedProviderPathFromPSPath(DeltaPath); DeltaState.ValidateWriteAccess(resolvedDeltaPath); + // Validate -OutputFile writability before HTTP calls string? resolvedOutputPath = null; if (OutputFile != null) { @@ -107,10 +126,14 @@ protected override void BeginProcessing() "DeltaPathOutputFileCollision", ErrorCategory.InvalidArgument, null)); return; } - DeltaState.ValidateWriteAccess(resolvedOutputPath); + // Named for what it is. Both probes are the same call, and one noun for both told a + // caller whose -OutputFile directory was not writable that they could not write to + // the delta state path - naming a file at -DeltaPath that had just passed its own. + DeltaState.ValidateWriteAccess(resolvedOutputPath, "output path"); } - // The checkpoint must not share a path with either state file + // The checkpoint must not collide with either state file: sharing a path would + // corrupt both the position and the data it describes. if (CheckpointPath != null) { var resolvedCheckpointPath = GetUnresolvedProviderPathFromPSPath(CheckpointPath); @@ -131,7 +154,7 @@ protected override void BeginProcessing() } } - // Warn when the URI does not look like a delta endpoint + // Warn if URI doesn't look like a delta endpoint if (!Uri.Contains("/delta", StringComparison.OrdinalIgnoreCase)) { WriteWarning( @@ -151,8 +174,8 @@ protected override void ProcessRecord() ? GetUnresolvedProviderPathFromPSPath(CheckpointPath) : null; - // -FullSync drops the state and any resume checkpoint, whose position belongs to the - // enumeration being discarded + // Handle -FullSync: delete existing delta state and any resume checkpoint - the + // position it describes belongs to the enumeration being discarded. if (FullSync.IsPresent) { if (File.Exists(resolvedDeltaPath)) @@ -170,44 +193,68 @@ protected override void ProcessRecord() DeleteCheckpoint(resolvedCheckpointPath, "full sync requested"); } - // Normalized so comparison is order-independent - var normalizedSelect = NormalizeSelect(Property != null ? string.Join(",", Property) : null); + // Normalize $select and Prefer for order-independent comparison. The effective + // select is what will go on the wire: a $select already in -Uri wins over + // -Property (the builder defers to it, with a warning), and the state must + // record and compare the wire value or every later run trips the consistency + // check against a select that was never sent. + var normalizedSelect = NormalizeSelect( + GetQueryOptionValue(Uri, "$select") + ?? (Property != null ? string.Join(",", Property) : null)); var normalizedPrefer = NormalizeSelect(Prefer != null ? string.Join(",", Prefer) : null); var currentFilter = Filter; string requestUrl; - // LoadWithResult tells "not found" from "corrupt". The endpoint-independent checks run - // before GetClient so their errors surface without a Graph connection + // LoadWithResult distinguishes "not found" from "corrupt". + // The endpoint-independent state checks run BEFORE GetClient() so their + // errors surface without requiring a Graph connection; the checks that + // compare against the session's endpoint run after it. var (existingState, loadResult) = DeltaState.LoadWithResult(resolvedDeltaPath); - // -Latest baselines from now and returns nothing. That is right for a first run and - // wrong after a state invalidation, where it would drop every change since the last - // successful sync. The resume branch guard never sees an invalidated state, so track it - // here + // -Latest means "baseline from now, return nothing". That is right for a first run and + // catastrophic after a state invalidation: the user is told a full re-sync is starting, + // gets zero items, and a fresh baseline token is persisted - so every change since the + // last successful sync is dropped permanently. The guard that warns "-Latest ignored" + // lives in the resume branch, which an invalidated state never reaches. Track it here // and clear it wherever state is discarded. - var honorLatest = Latest.IsPresent; - - // A live resume checkpoint is not a fresh run either. Honoring -Latest on top of an - // interrupted enumeration strands the items the crashed run collected and still saves a - // from-now token. -FullSync deletes the checkpoint above, so -FullSync -Latest still - // re-baselines, which is what the warning below + var honourLatest = Latest.IsPresent; + + // A live resume checkpoint is not a fresh run either. Without delta state the guards + // below never fire, so -Latest was honored on top of an interrupted enumeration: the + // checkpoint is dropped a moment later as "a different enumeration" (the token=latest + // suffix changes requestUrl), the items the crashed run collected stay stranded in its + // temp, and an empty page still saves a from-now token - so everything before this + // moment is permanently unreachable. -FullSync deletes the checkpoint above, so + // "-FullSync -Latest" still re-baselines from now, which is what the warning below // tells people to use. var hasResumableCheckpoint = resolvedCheckpointPath != null && File.Exists(resolvedCheckpointPath); - if (honorLatest && hasResumableCheckpoint) - honorLatest = false; + if (honourLatest && hasResumableCheckpoint) + honourLatest = false; if (loadResult == DeltaLoadResult.Corrupt) { WriteWarning($"Delta state file '{DeltaPath}' is corrupt. Starting full sync."); - // A corrupt state means the previous position is unknown, which is when baselining - // from now would hide the most - honorLatest = false; + // A corrupt state means the previous position is unknown, which is exactly when + // baselining from now would hide the most: everything since the last good sync. + honourLatest = false; + // And the resume checkpoint goes with the position it counted into, the way it does + // at every other door out of the delta state. What it records is the delta-link + // spelling this run no longer builds, so leaving it standing bought a refusal of the + // run's own earlier position: a warning naming a second sync that does not exist, + // and a stale-temp sweep held off, over a temp that outlives the run by one because + // nothing counts it any more. + DeleteCheckpoint(resolvedCheckpointPath, "delta state corrupt"); + // Nothing downstream may still believe it is there. The -Latest guard below reads + // this to decide which of two sentences to write, and one of them tells the caller + // to delete a checkpoint that has just gone. + hasResumableCheckpoint = false; } if (existingState != null) { - // The deltaLink carries its own version, so a run omitting -ApiVersion would keep - // syncing whichever version built the state. Empty means an older state file, which - // is unknown rather than mismatched + // The deltaLink is absolute and carries its own version, so a run that omits + // -ApiVersion silently keeps syncing whichever version built the state - the + // caller believes they are on the default and are not. Empty means a pre-2.0.1 + // state file: unknown, not mismatched, so upgrades are not broken by this check. if (!string.IsNullOrEmpty(existingState.ApiVersion) && !string.Equals(existingState.ApiVersion, ApiVersion, StringComparison.OrdinalIgnoreCase)) { @@ -232,9 +279,17 @@ protected override void ProcessRecord() return; } - // Order-independent and deduplicated + // Normalized $select comparison (order-independent, deduplicated) var storedSelect = NormalizeSelect(existingState.Select); - if (!string.Equals(storedSelect, normalizedSelect, StringComparison.OrdinalIgnoreCase)) + // State written before the wire-value change recorded the -Property form even + // when -Uri carried the $select. If the stored value matches THAT form for + // the same invocation, nothing actually changed on the wire - accept it, and + // the state is rewritten in the new form on save. + var legacySelect = NormalizeSelect(Property != null ? string.Join(",", Property) : null); + var selectUnchanged = + string.Equals(storedSelect, normalizedSelect, StringComparison.OrdinalIgnoreCase) + || string.Equals(storedSelect, legacySelect, StringComparison.OrdinalIgnoreCase); + if (!selectUnchanged) { WriteWarning( "Property selection changed since last sync " @@ -243,12 +298,13 @@ protected override void ProcessRecord() if (!DeltaState.Delete(resolvedDeltaPath)) WriteVerbose($"Could not delete old delta state at '{DeltaPath}' (file may be locked). It will be overwritten."); DeleteCheckpoint(resolvedCheckpointPath, "property selection changed"); + hasResumableCheckpoint = false; // the -Latest warning below must not name it existingState = null; - honorLatest = false; // a discarded state is not a fresh run + honourLatest = false; // a discarded state is not a fresh run } - // Prefer tokens shape what the enumeration returns, so mixing states across a change - // is unsound + // Detect Prefer change between runs: the tokens shape what the enumeration + // returns (removed facets, sharing annotations), so mixing states is unsound. if (existingState != null) { var storedPrefer = NormalizeSelect(existingState.Prefer); @@ -261,8 +317,9 @@ protected override void ProcessRecord() if (!DeltaState.Delete(resolvedDeltaPath)) WriteVerbose($"Could not delete old delta state at '{DeltaPath}' (file may be locked). It will be overwritten."); DeleteCheckpoint(resolvedCheckpointPath, "Prefer headers changed"); + hasResumableCheckpoint = false; // the -Latest warning below must not name it existingState = null; - honorLatest = false; // a discarded state is not a fresh run + honourLatest = false; // a discarded state is not a fresh run } } @@ -277,14 +334,18 @@ protected override void ProcessRecord() if (!DeltaState.Delete(resolvedDeltaPath)) WriteVerbose($"Could not delete old delta state at '{DeltaPath}' (file may be locked). It will be overwritten."); DeleteCheckpoint(resolvedCheckpointPath, "filter changed"); + hasResumableCheckpoint = false; // the -Latest warning below must not name it existingState = null; - honorLatest = false; // a discarded state is not a fresh run + honourLatest = false; // a discarded state is not a fresh run } } - // GetClient sits between the two validation halves. It runs after the state-file checks - // so their errors need no Graph connection, and before the rest because it is the only - // thing that refreshes s_graphEndpoint from the session + // GetClient() sits between the two validation halves on purpose. It runs after the + // state-file checks above so their errors surface without a Graph connection, and + // before everything below because it is the only thing that refreshes s_graphEndpoint + // from the session: on the first call of a session the endpoint comparison and the + // request URL would otherwise be built against the default endpoint instead of the + // connected one. Invoke-MgxRequest sequences GetClient() first for the same reason. var client = GetClient(); if (existingState != null) @@ -301,7 +362,7 @@ protected override void ProcessRecord() return; } - // The deltaLink comes from a file on disk and is untrusted + // SSRF validation: deltaLink is untrusted (from a file on disk) var deltaUri = new System.Uri(s_graphEndpoint); var validated = NextLinkValidator.Validate(existingState.DeltaLink, deltaUri); if (validated == null) @@ -314,10 +375,13 @@ protected override void ProcessRecord() return; } - // Verify the deltaLink path contains the expected resource, so a tampered state file - // cannot redirect the query to another Graph resource. - // Compare paths to paths. NormalizePath keeps any query while AbsolutePath never has - // one, so a -Uri carrying $select or a trailing slash would always mismatch + // Resource path validation: verify the deltaLink's path contains the expected + // resource. Prevents a tampered delta file from redirecting queries to a different + // Graph resource (e.g., /me/messages instead of /users/delta). + // Compare paths to paths. NormalizePath keeps any query, while AbsolutePath never + // has one, so "/users/delta?$select=id" - the shape Microsoft's delta docs show - + // guaranteed a mismatch: run 1 saved state, run 2 died with a SecurityError accusing + // that state file of tampering. A trailing slash failed identically. var expectedPath = NormalizePath(Uri).Split('?')[0].TrimEnd('/'); if (System.Uri.TryCreate(validated, UriKind.Absolute, out var parsedDelta) && !parsedDelta.AbsolutePath.TrimEnd('/') @@ -344,9 +408,12 @@ protected override void ProcessRecord() else { requestUrl = BuildListUrl(VersionedBaseUrl, Uri, - new ODataListParams(false, Top, Top > 0 ? Top : 999, Filter, Property, null, null, 0, null)); + new ODataListParams(false, Top, Top > 0 ? Top : 999, Filter, Property, null, null, 0, null), + out var deferred); + if (deferred.Count > 0) + WriteWarning(DescribeDeferredOptions(deferred)); - if (Latest.IsPresent && !honorLatest) + if (Latest.IsPresent && !honourLatest) { WriteWarning(hasResumableCheckpoint ? "-Latest ignored: a resume checkpoint exists, so an interrupted enumeration " @@ -357,11 +424,11 @@ protected override void ProcessRecord() + "enumerate to rebuild it. Baselining from now would silently drop every " + "change since the last successful sync."); } - else if (honorLatest) + else if (honourLatest) { - // Sync from now returns an empty page plus a deltaLink, and the empty-page path - // persists the baseline. The token form differs by service: OneDrive and - // SharePoint take token=latest, directory and + // "Sync from now": returns an empty page plus a deltaLink; the existing + // empty-page-still-saves-token path persists the baseline. The token form + // differs by service: OneDrive/SharePoint take token=latest, directory and // everything else $deltatoken=latest. var tokenParam = AdaptivePacing.Classify(Uri) == WorkloadBucket.Drive ? "token=latest" @@ -379,6 +446,10 @@ protected override void ProcessRecord() resolvedCheckpointPath, normalizedSelect, normalizedPrefer, currentFilter, sw); } + /// + /// The Graph API version a deltaLink was issued by, read from the link itself, or null when + /// it cannot be read. + /// private static string? ApiVersionOfLink(string? deltaLink) { if (string.IsNullOrEmpty(deltaLink)) return null; @@ -401,60 +472,652 @@ private void DeleteCheckpoint(string? checkpointPath, string reason) } - private void ReconcileCheckpointWithFiles(string checkpointPath, string outputPath, PaginationCheckpoint checkpoint) + /// + /// Whose the checkpoint on disk is, and where it is not this sync's, why not. Three things + /// have to agree before it is this run's. + /// + /// The output mode: a checkpoint that names an output file, names a temp of one, or records + /// a byte length was written by a sync collecting into a file, and its position counts items + /// that went there rather than down the pipeline. Resuming a pipeline sync from it emits + /// neither those items nor anything before them and then saves a delta token over the lot. + /// The length carries as much of that as the names do: a file-mode run records no temp once + /// it is appending, and none when a cancellation promoted the one it had, so a release that + /// did not yet record the output left file-mode checkpoints naming neither - and every one + /// of them was adopted here. A length is measured on a file, and a pipeline run has none to + /// measure. The reverse is refused on the same grounds. + /// + /// The output itself, on file-mode runs: two syncs of the same -Uri build the same resource, + /// so that comparison says nothing about WHICH file a recorded length was counted in, and + /// applied to another sync's it cut that file to this one's offset, mid-line. + /// + /// The resource, endpoint-independently and however -Uri was typed: what was enumerated, not + /// where Graph was reached, and not which spelling reached it - Graph answers "/users/delta" + /// and "/Users/delta" from one collection, so a case the caller typed differently between + /// two runs is the same enumeration and refusing it costs the run its own position. + /// + /// Nothing here decides differently than the bare predicate this replaced; what a refusal + /// can then say is what changes. A checkpoint recording no output was believed for as long + /// as the files beside it corroborated it, so losing that corroboration - a temp since + /// removed, an output since replaced - is one sync and no second run anywhere, and it is + /// the shape every release before this one wrote. + /// + /// Which is why the corroboration is asked for only where there is something to corroborate: + /// is the export's own guard, and without it + /// a checkpoint recording neither an output nor a length was refused for want of a + /// measurement - the whole pre-2.1 file-mode shape, recovered by the export and refused + /// here, with -Latest suppressed by the file it left standing for good. That shape reaches + /// the resource comparison below and, where the enumeration is this run's, the recovery the + /// export makes of it. + /// + private CheckpointOwnership OwnershipOf(PaginationCheckpoint checkpoint, string? outputPath, + string requestUrl) { + if (outputPath == null) + { + if (checkpoint.OutputFile != null || checkpoint.TempFile != null + || checkpoint.DataLength != null) + { + return CheckpointOwnership.AnotherSyncs; + } + } + else if (RecordsAFileToWeigh(checkpoint) + && !RecordedOutputMatches(checkpoint.OutputFile, checkpoint.TempFile, + checkpoint.DataLength, outputPath, mayWrite: true)) + { + return checkpoint.OutputFile != null + ? CheckpointOwnership.AnotherSyncs + : CheckpointOwnership.Uncorroborated; + } + + return SameResourceIdentity(ResourceIdentity(checkpoint.Resource), + ResourceIdentity(requestUrl)) + ? CheckpointOwnership.Mine + : CheckpointOwnership.AnotherSyncs; + } + + /// Which of the three a checkpoint on disk is. + private enum CheckpointOwnership + { + /// This sync's. + Mine, + + /// + /// Another sync's: it records another output, another enumeration, or the other + /// output mode. + /// + AnotherSyncs, + + /// + /// Records no output file, and the files beside this one no longer stand for it. Which + /// sync wrote it cannot be told from here, and telling the caller it was another one is + /// a diagnosis they can go and check and find nothing behind. + /// + Uncorroborated, + } + + /// + /// Whether the temp a refused checkpoint names is a file the stale-temp sweep would reach. + /// The name comes off a checkpoint, which is untrusted once it is on disk, and nothing here + /// opens it - all it decides is whether the sweep runs at all, so a name of some other shape + /// is answered no rather than refused: the sweep cannot delete it either, and skipping the + /// sweep for it would leave real orphans behind for nothing. + /// + /// "Some other shape" is the sweep's own test of a name and not a looser one kept here. + /// Matching on the prefix and the suffix alone answers yes for "users.jsonl.{guid}.tmp" + /// beside an output named "users" - a file the sweep passes over, because the glob's '*' + /// spans dots and the sweep's own filter does not - so a checkpoint left by the run next + /// door suppressed this output's sweep and its real orphans stayed on disk, which is the + /// case this comment says is answered no. + /// + private static bool RefusedTempIsOnDisk(string? tempFile, string outputPath) + { + if (tempFile == null || tempFile != Path.GetFileName(tempFile)) return false; + var dir = Path.GetDirectoryName(outputPath); + if (string.IsNullOrEmpty(dir)) return false; + if (!IsRunTempName(dir, Path.GetFileName(outputPath), tempFile, mayWrite: true)) + return false; + return File.Exists(Path.Combine(dir, tempFile)); + } + + /// + /// What a caller can act on when the checkpoint on disk records an enumeration this run + /// cannot resume: what is known about the file, what this run does instead, and that a + /// -CheckpointPath belongs to one enumeration. Refusing it is the whole response - the file + /// is left alone, and so is any staging file beside it, because whatever wrote them may + /// still resume from exactly those. + /// + /// Which sync wrote it is not something this can tell, and asserting a second one sent the + /// caller looking for a run that in the commonest shape here does not exist: a checkpoint + /// saved during a delta enumeration, found by the same command line after the state + /// recording that delta was lost - corrupt, deleted by hand, a -DeltaPath since moved - so + /// the run builds a full-sync URL and refuses its own earlier position. There is no second + /// sync to find, and giving each its own -CheckpointPath is what the caller already did. + /// Both readings are offered and neither is asserted; what this run does about it is the + /// same either way. + /// + private static string ForeignCheckpointWarning(string checkpointPath, string? outputPath, string deltaPath) => + $"The resume checkpoint at '{checkpointPath}' records an enumeration this run cannot " + + $"resume from, so it is left as it is and this run enumerates from what '{deltaPath}' " + + "records" + + (outputPath != null ? $", into '{outputPath}'" : "") + + ". It is either another sync's, sharing this -CheckpointPath, or this sync's own from " + + "a pass it no longer makes - a delta enumeration whose state has since been lost " + + "leaves exactly this. Two syncs sharing one -CheckpointPath overwrite each other's " + + "resume position; give each its own."; + + /// + /// What a caller can act on when a checkpoint that records no output file is refused: it + /// was written before the output was recorded, nothing beside this run's output stands for + /// it any more, and this run enumerates from the delta token instead. Naming a different + /// sync there sends the caller looking for a second run over the same -CheckpointPath, and + /// there is none to find - a temp that has since been removed and an output that has since + /// been replaced reach the same refusal. + /// + private static string UncorroboratedCheckpointWarning( + string checkpointPath, string outputPath, string deltaPath) => + $"The resume checkpoint at '{checkpointPath}' records no output file, and the files " + + $"beside '{outputPath}' no longer corroborate it, so it is left as it is and this run " + + $"enumerates from the delta token in '{deltaPath}', into '{outputPath}'; no changes " + + "are lost."; + + /// + /// What this run tells the caller about a checkpoint it will not resume from, or null when + /// it is this sync's and there is nothing to tell. Uncorroborated is reachable only from + /// the branch that has an output to name; a pipeline run refuses on the output mode, which + /// is another sync's checkpoint however little else is known about it. + /// + private string? RefusalFor(PaginationCheckpoint checkpoint, string? outputPath, + string requestUrl, string checkpointPath, string deltaPath) + => OwnershipOf(checkpoint, outputPath, requestUrl) switch + { + CheckpointOwnership.Mine => null, + CheckpointOwnership.Uncorroborated when outputPath != null => + UncorroboratedCheckpointWarning(checkpointPath, outputPath, deltaPath), + _ => ForeignCheckpointWarning(checkpointPath, outputPath, deltaPath), + }; + + /// + /// How the run ends when a file its own checkpoint stands for is held open by another sync, or + /// cannot be opened read-write by this account. Two files reach this, and the reading is one: + /// the changes the checkpoint counts are in that file and in no other, the checkpoint is the + /// only thing on disk that counts them, and this sync has no enumeration to make that leaves + /// the pair intact - it would save its own position over the checkpoint at the first page + /// boundary and move the delta token past changes held in that file alone. So it says what it + /// found, what it did not do, and the two ways out, and every file is exactly where it was. + /// + /// Which file decides what going on would cost, and for an output refusal + /// decides it further: a checkpoint naming a temp + /// that is still there holds the changes in that temp, and going on promotes them, replacing + /// the output outright; one naming a temp that has vanished holds them nowhere this run can + /// reach, and going on re-enumerates from the last saved delta token and replaces the output + /// with the result; and one naming no temp at all was written by a run appending straight + /// into the output, so the changes are in the output itself and going on cuts it back under + /// its writer - whose next write then lands past the hole that offset leaves. + /// + /// Which of the two readings decides only the cause, the advice, and the id and category + /// under them. A held file has a second sync behind it that will either finish, and delete + /// this checkpoint itself, or die and leave it for its own re-run; an unopenable one has no + /// second sync anywhere, only a file whose own open failed for a reason named in + /// - a permission this account can be granted, or a + /// symlink loop, an over-long path or a pipe that granting one does nothing for. + /// + private static string CheckpointFileStopMessage(CheckpointFileRefusal refusal, + string checkpointPath, string outputPath, PaginationCheckpoint checkpoint) + { + // What going on would cost past a held temp, which is the output beside it - named only + // where there is one. A checkpoint naming a temp is reached with nothing at -OutputFile + // at all, and a run that said it would replace that file sent the caller to look at + // something no sync had written yet. + var goingOn = File.Exists(outputPath) + ? $"Going on would replace '{outputPath}', save this run's position over the " + + "checkpoint that counts those items, and advance the delta token past them." + : "Going on would save this run's position over the checkpoint that counts those " + + "items, and advance the delta token past them."; + + // The output is unopenable rather than held. The three routes agree past this point - + // the code beyond this check falls through to the same place a file that never held the + // recorded items reaches, which deletes the checkpoint, replaces the output and advances + // the token at the completion that follows - so only the first clause, naming where the + // items are now, and the recovery word in the advice change per route. + string UnopenableOutputSentence() + { + var reason = refusal.Reason!.Value; + var openFailed = $"'{outputPath}' could not be opened for writing: {reason.Reason}."; + var recovers = refusal.Route switch + { + CheckpointStopRoute.Promoting => " to recover them", + CheckpointStopRoute.FreshAfterVanishedTemp => "", + _ => " to resume from it", + }; + // Four ways out, because the causes do not share one. A permission on the file is + // granted; a permission on the directory holding it is granted there, and the file + // this sentence names is one the caller cannot reach to change or remove; a + // directory at the path is removed or pointed away from, and no grant would have + // made it writable at an offset; and a loop, an over-long path, a socket or a pipe + // is none of those - what this run can honestly say about them is what failed. + var advice = reason.IsDirectory + ? $"Remove it (or point -OutputFile elsewhere) and run again{recovers}, or " + + "remove the checkpoint to sync afresh." + : reason.IsUnsearchableParent + ? $"Grant access to that directory and run again{recovers}, or remove the " + + "checkpoint to sync afresh." + : reason.IsPermission + ? $"Grant write access to that file and run again{recovers}, or remove " + + "it and the checkpoint to sync afresh." + : $"Fix that and run again{recovers}, or remove it and the checkpoint to " + + "sync afresh."; + + return refusal.Route switch + { + CheckpointStopRoute.Promoting => + $"{openFailed} The resume checkpoint at '{checkpointPath}' records " + + $"{checkpoint.ItemsCollected} items into the temp '{checkpoint.TempFile}'; " + + $"this run cannot promote them into '{outputPath}'. Going on would delete " + + "the checkpoint that counts them, replace that file and advance the delta " + + $"token past them. {RunStopsHere} {advice}", + + CheckpointStopRoute.FreshAfterVanishedTemp => + $"{openFailed} The resume checkpoint at '{checkpointPath}' recorded " + + $"{checkpoint.ItemsCollected} items into a temp that is gone; this run " + + $"cannot write a fresh copy into '{outputPath}' either. Going on would " + + "delete the checkpoint that counts them, replace that file and advance the " + + $"delta token past them. {RunStopsHere} {advice}", + + _ => + $"{openFailed} The resume checkpoint at '{checkpointPath}' records " + + $"{checkpoint.ItemsCollected} items into it, so this run can neither " + + "resume into it nor say whether those items are still there. Going on " + + "would delete the checkpoint that counts them, replace that file and " + + $"advance the delta token past them. {RunStopsHere} {advice}", + }; + } + + return (refusal.File, refusal.Held) switch + { + (CheckpointFile.NamedTemp, true) => + "Another sync is still writing the temp file the resume checkpoint at " + + $"'{checkpointPath}' names, so the {checkpoint.ItemsCollected} items it records " + + $"are that run's. {goingOn} {RunStopsHere} Wait for that sync to finish, or " + + "give this one its own -OutputFile and -CheckpointPath.", + + (CheckpointFile.NamedTemp, false) => + $"The temp file '{checkpoint.TempFile}' that the resume checkpoint at " + + $"'{checkpointPath}' names cannot be opened for writing by this account - the " + + "open was refused on permissions, not on sharing - so the " + + $"{checkpoint.ItemsCollected} items it records cannot be recovered here. " + + $"{goingOn} {RunStopsHere} Grant write access to that file and run again to " + + "recover them, or remove it and the checkpoint to sync afresh.", + + (CheckpointFile.Output, true) => refusal.Route switch + { + CheckpointStopRoute.Promoting => + $"Another sync is still writing '{outputPath}'. The resume checkpoint at " + + $"'{checkpointPath}' records {checkpoint.ItemsCollected} items into the " + + $"temp '{checkpoint.TempFile}'; going on would replace '{outputPath}' with " + + "them under the run that holds it, save this run's position over the " + + "checkpoint that counts those items, and advance the delta token past " + + $"them. {RunStopsHere} Wait for that sync to finish, or give this one its " + + "own -OutputFile and -CheckpointPath.", + + CheckpointStopRoute.FreshAfterVanishedTemp => + $"Another sync is still writing '{outputPath}'. The resume checkpoint at " + + $"'{checkpointPath}' recorded {checkpoint.ItemsCollected} items into a " + + "temp that is gone; going on would re-enumerate from the last saved delta " + + $"token and replace '{outputPath}' under the run that holds it, save this " + + "run's position over the checkpoint that counts those items, and advance " + + $"the delta token past them. {RunStopsHere} Wait for that sync to finish, " + + "or give this one its own -OutputFile and -CheckpointPath.", + + _ => + $"Another sync is still writing '{outputPath}', which the resume checkpoint " + + $"at '{checkpointPath}' records {checkpoint.ItemsCollected} items into. " + + "Going on would cut that file back under it, save this run's position over " + + "the checkpoint that counts those items, and advance the delta token past " + + $"them. {RunStopsHere} Wait for that sync to finish, or give this one its " + + "own -OutputFile and -CheckpointPath.", + }, + + _ => UnopenableOutputSentence(), + }; + } + + /// + private static ErrorRecord CheckpointFileStop(CheckpointFileRefusal refusal, + string checkpointPath, string outputPath, PaginationCheckpoint checkpoint) => + new(new InvalidOperationException(CheckpointFileStopMessage(refusal, checkpointPath, + outputPath, checkpoint)), + (refusal.File, refusal.Held) switch + { + (CheckpointFile.NamedTemp, true) => "CheckpointTempHeld", + (CheckpointFile.NamedTemp, false) => "CheckpointTempUnopenable", + (CheckpointFile.Output, true) => "CheckpointOutputHeld", + _ => "CheckpointOutputUnopenable", + }, + // ResourceBusy for a file another run has, which is what a caller waits out. Past + // that the category follows the open's own reason rather than assuming a + // permission: PermissionDenied is a mode this account can be granted, and a symlink + // loop, an over-long path, a socket, a FIFO or a directory at the path is none - + // InvalidOperation says so, and a refusal that carries no reason at all is the + // temp's, whose own sentence names permissions. + refusal.Held + ? ErrorCategory.ResourceBusy + : refusal.Reason is { IsPermission: false } + ? ErrorCategory.InvalidOperation + : ErrorCategory.PermissionDenied, + checkpointPath); + + /// + /// How the run ends when the copy a promotion stages beside the output could not be written, + /// or could not be moved onto the output once it was. Nothing here is a file that has gone + /// missing, and reading it as one was what deleted the checkpoint and moved the delta token + /// past changes held in the temp: every item it counts is in that temp, whole and claimable, + /// and the run that comes back stages the same bytes over the same output. So the sentence + /// says where those changes are, what failed and where, and that both files are as they were + /// found - and the run stops rather than sync past a position it would then save over. + /// + /// The temp those changes are in. The checkpoint's own record of it + /// where it has one, and the file adoption picked where the checkpoint predates that field: + /// either way it is the file the reader has to keep, and a sentence that read it off the + /// checkpoint named nothing at all on the adoption route. + /// What the sweep took off the staging name on the way here, where + /// it took anything: the run deleted an entry and then stopped, and a reader who is told only + /// that the staging failed goes looking for a file that is no longer there. Named in the + /// stop rather than warned about separately, because a warning written before a terminating + /// error is what -WarningAction Stop ends the run on instead of this sentence. + private static string CheckpointStagingStopMessage(StagingFailure staging, + string checkpointPath, string outputPath, PaginationCheckpoint checkpoint, + string tempName, string? removedKind) => + $"The {checkpoint.ItemsCollected} items the resume checkpoint at '{checkpointPath}' " + + $"records are in '{tempName}', whole, but staging them into '{outputPath}' " + + $"failed at '{staging.Path}': {staging.Reason}" + + (removedKind != null + ? $", after removing what stood at the staging name: {removedKind}" + : "") + + $". {NothingWasChanged} fix that and run " + + "again to recover them, or remove the temp and the checkpoint to re-enumerate from the " + + "last saved delta token."; + + /// + private static ErrorRecord CheckpointStagingStop(StagingFailure staging, string checkpointPath, + string outputPath, PaginationCheckpoint checkpoint, string tempName, + string? removedKind) => + new(new InvalidOperationException(CheckpointStagingStopMessage(staging, checkpointPath, + outputPath, checkpoint, tempName, removedKind)), + "CheckpointStagingFailed", ErrorCategory.WriteError, checkpointPath); + + /// + /// What a reconcile leaves for the caller to do. Three of the four say do not resume, and + /// they are not the same answer: one is a checkpoint a live sync is still using, on which + /// the run ends with both files where they were; another is a promotion whose copy could + /// not be written or moved, on which the run also ends with both files where they were; + /// and the last is a checkpoint describing items that are in no file, which is deleted + /// wherever this run is able to delete it. + /// + private enum CheckpointRecovery + { + /// + /// The files hold what the checkpoint says they hold - promoted, trimmed or already + /// so - and the run resumes from the position it records. + /// + Resumable, + + /// + /// Another sync is writing one of the files this checkpoint stands for - the temp it + /// names, or the output it appends to - or this account cannot open it read-write. + /// Nothing here is this run's to recover or to remove, and there is no enumeration it + /// can make that does not destroy what those files hold, so the caller ends the run on + /// it. Which file, and which of the two ways the claim failed, comes back beside this, + /// because between them they decide what the caller can honestly tell the reader to do + /// about it. + /// + Refused, + + /// + /// The copy the promotion stages beside the output could not be written, or could not be + /// moved onto the output once it was. The temp holds every change the checkpoint counts + /// and the checkpoint still names it, so nothing here is recoverable by re-enumerating + /// and nothing is this run's to remove: the caller ends the run on it, naming the write + /// that failed and where. Told from Discarded because that one deletes the position + /// counting those changes and moves the token past them. + /// + StagingFailed, + + /// + /// The items the checkpoint counts are in no file this run can reach, so there is + /// nothing to resume from. The delete that goes with it is attempted and may fail; the + /// answer holds either way. + /// + Discarded, + } + + /// + /// Put the files into the state the checkpoint claims, or delete the checkpoint. A + /// checkpoint records which file its items were written to and how many bytes of that file + /// they occupy, which makes three cases decidable instead of guessed. + /// + /// A temp is named, so the interrupted run was fresh and its items are in that temp while + /// the output still holds the PREVIOUS sync's rows. Those rows were the previous run's + /// result, already replaced by every path that completes - a fresh run that finishes moves + /// its temp over the output, and a cancellation promotes the same way - so recovery + /// promotes too. Appending instead would put rows the caller has already consumed back in + /// front of this sync's. + /// + /// None is named, so the run was appending to the output and its items are already there, + /// past the recorded length only if it wrote more after its last save. Cutting back to that + /// length is what stops those from being written twice. + /// + /// Neither is recorded, so the checkpoint cannot say which file its items are in, and an + /// appending run's checkpoint cannot be told from a fresh one's. When no output exists the + /// ambiguity is harmless - everything the run wrote is in its temp - but against an + /// existing output the only recovery that cannot lose or repeat items is re-enumerating. + /// + /// When the counted items turn out to be in no file, the delta link has not moved, so + /// re-enumerating costs time and loses nothing while resuming past them loses them for good. + /// + /// A file of the checkpoint's that another run has open is none of those: the files are + /// exactly what the checkpoint says they are, and the sync they belong to has not been + /// interrupted. Its items are that run's, and the checkpoint is the position it comes back + /// to, so neither is this run's to touch - and there is nothing left for this run to do + /// either. Resuming would leave those items out of the output while the delta token advanced + /// past them; re-enumerating would replace the output, save this run's position over the + /// checkpoint that counts them, and move the token anyway. Refused comes back, and the caller + /// ends the run there. The named temp is one such file; the output a checkpoint naming no + /// temp records its items into is the other, and there the run that holds it is one that has + /// already resumed - so a second run over the same command line finds nothing in the + /// checkpoint to refuse it by and has to be stopped by the claim instead. + /// + /// Whether the checkpoint may be resumed from is the answer, and not the file left on disk. + /// A bail-out that deletes the checkpoint it has just given up on can fail to delete it - a + /// checkpoint directory this account may read but not unlink from is answered false and + /// leaves the file - so a caller reading the refusal off File.Exists read the one thing a + /// failed delete gets wrong, resumed from the nextLink the warning had promised to + /// re-enumerate past, and advanced the delta token over items that are in no file at all. + /// Discarded binds either way; the file that outlives it is given up on again next run. + /// + private CheckpointRecovery ReconcileCheckpointWithFiles(string checkpointPath, + string outputPath, PaginationCheckpoint checkpoint, out CheckpointFileRefusal refused, + out StagingFailure? staging, out string? stagingTemp, out string? stagingRemoved) + { + refused = CheckpointFileRefusal.None; + staging = null; + stagingTemp = null; + stagingRemoved = null; if (checkpoint.DataLength is not { } dataLength) { if (!File.Exists(outputPath)) { - if (TryAdoptOrphanedTemp(outputPath, checkpoint.ItemsCollected)) - WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted sync's temp file. Resuming from checkpoint."); - else + var adopted = TryAdoptOrphanedTemp(outputPath, checkpoint.ItemsCollected, + out stagingTemp, out staging, out var adoptSwept); + + // What the adoption took off its own staging name before it created one there. + // The name is nobody's to keep - a link, a pipe or a copy left by an interrupted + // promotion standing at it is gone - so the run says which it was. Recorded here + // and warned about once this run's own outcome for the recovery is known: a + // warning is a terminating error under -WarningAction Stop, so written here it + // ended the run with the entry deleted, the changes still in the temp and + // nothing recovered. + stagingRemoved = adoptSwept?.Kind; + if (adoptSwept is { } adoptRemoval) WriteVerbose(adoptRemoval.Sentence); + + if (adopted) { - WriteWarning("Checkpoint found but output file is missing. Deleting stale checkpoint and starting fresh."); - PaginationCheckpoint.Delete(checkpointPath); + WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted sync's temp file. Resuming from checkpoint."); + if (adoptSwept is { } removed) WriteWarning(removed.Sentence); + return CheckpointRecovery.Resumable; } - return; + + // The copy could not be staged beside the output. Every change the checkpoint + // counts is in the temp this names, whole, so the delete below is not the answer: + // it would take the position counting them away and move the token past every + // one. What came off the name goes into the stop the caller writes, since a + // warning in front of a terminating error is what -WarningAction Stop ends on. + if (staging != null) return CheckpointRecovery.StagingFailed; + + WriteWarning("Checkpoint found but output file is missing. Deleting stale checkpoint and starting fresh."); + PaginationCheckpoint.Delete(checkpointPath); + if (adoptSwept is { } removedAnyway) WriteWarning(removedAnyway.Sentence); + return CheckpointRecovery.Discarded; } WriteWarning( "The resume checkpoint does not record which file the interrupted sync's items are in. " + "Re-enumerating from the last saved delta token; no changes are lost."); PaginationCheckpoint.Delete(checkpointPath); - return; + return CheckpointRecovery.Discarded; } if (checkpoint.TempFile != null) { - if (TryPromoteNamedTemp(outputPath, checkpoint.TempFile, dataLength)) + stagingTemp = checkpoint.TempFile; + var promotion = TryPromoteNamedTemp(outputPath, checkpoint.TempFile, dataLength, + out var promotionReason, out staging, out var swept); + + // What the promotion took off its own staging name before it created one there. The + // name is nobody's to keep - a link, a pipe or a copy left by an interrupted + // promotion standing at it is gone - so the run says which it was. Recorded here and + // warned about once this run's own outcome for the recovery is known: a warning is a + // terminating error under -WarningAction Stop, so written here it ended the run with + // the entry deleted, the changes still in the temp and nothing recovered. + stagingRemoved = swept?.Kind; + if (swept is { } promotionRemoval) WriteVerbose(promotionRemoval.Sentence); + + // The copy could not be staged, or could not be moved once it was. Both files are + // where they were found and every change the checkpoint counts is still in the temp + // it names, so this is not the route below for a temp whose changes are in no file: + // that one deletes the position counting them, re-enumerates over the output and + // moves the token past them, and reached from here it did so with the temp holding + // every one. + if (promotion == TempPromotion.StagingFailed) return CheckpointRecovery.StagingFailed; + + // The output the promotion would land in is another run's, or is not a file this + // account can write at the recorded offset. The same stop the branch below reaches, + // about the same file for the same reason: the run that has the output is appending + // to it, and this one would be replacing it under them and moving the token past + // what it wrote. + // + // Asked for before the move, so nothing was replaced - and the temp, with the + // checkpoint still naming it, is exactly as it was found, since the temp is unlinked + // only where the output was kept. So the position and the file it counts are the + // pair they were, and the run that comes back promotes the same changes to the same + // length over the same output. + if (promotion is TempPromotion.OutputHeld or TempPromotion.OutputUnopenable) { + refused = new CheckpointFileRefusal(CheckpointFile.Output, + promotion == TempPromotion.OutputHeld + ? ClaimRefusal.AnotherRunHasIt + : ClaimRefusal.CannotBeOpened, + CheckpointStopRoute.Promoting, promotionReason); + return CheckpointRecovery.Refused; + } + + if (promotion == TempPromotion.Taken) + { + // Those changes are the output now, and the output is this run's - the promotion + // kept it through the move. Repoint the checkpoint at it immediately, so a second + // interruption cannot promote the same temp a second time - and before the + // warnings below, which under -WarningAction Stop are where the run ends: the + // position on disk has to be the one that resumes from what is now in the output. + RepointCheckpointAtHeldOutput(checkpoint, checkpointPath, outputPath, HeldOutput!); WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted sync's temp file. Resuming from checkpoint."); - // Those items are the output now, so repoint the checkpoint at it immediately or a - // second interruption promotes the same temp twice - checkpoint.TempFile = null; - checkpoint.DataLength = new FileInfo(outputPath).Length; - try { checkpoint.Save(checkpointPath); } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) - { - WriteWarning($"Checkpoint save failed after recovery: {ex.Message}"); - } - return; + if (swept is { } removed) WriteWarning(removed.Sentence); + return CheckpointRecovery.Resumable; } + // The temp is there and whole, and another run has it open - so the sync this + // checkpoint belongs to is not interrupted at all, it is collecting into that file + // right now. Its items are that run's, the checkpoint is the position it comes back + // to, and this run has nothing here to recover: deleting the checkpoint below, or + // resuming from it, would be the same mistake as unlinking the temp, one file over. + // + // Or the claim failed because this account cannot open the file read-write, which + // leaves both files in the same place and has no second sync behind it. The refusal + // is the same; only what it can honestly say about the cause differs. + // + // Refused says nothing here, because the sentence a caller acts on is about the run + // and not about the reconcile: this sync cannot go on without replacing the output + // beside these two files, saving its own position over the checkpoint that counts + // their rows, and moving the delta token past them. It ends instead, and the site + // that ends it is the one that can say so before anything has happened. + refused = new CheckpointFileRefusal(CheckpointFile.NamedTemp, + RefusalForNamedTemp(outputPath, checkpoint.TempFile, dataLength, mayWrite: true)); + if (refused.Refuses) return CheckpointRecovery.Refused; + + // The temp is not there, or is shorter than the length recorded for it - and the + // first of those is what a promotion leaves for the instant between its unlink and + // the save that repoints the position at the output. A second run over the same + // command line, released with the first, read that instant and said those changes + // were not on disk, deleted the position counting them, re-enumerated over the + // output the promoter was holding, and moved the delta token past both runs' work. + // So the output is asked about before any of that, and a promoter holding it ends + // this run where the branch below ends it. + // + // Only the refusals are read. An output nothing holds is not evidence that a + // promotion landed in it, whatever it measures: this checkpoint records its changes + // into the temp, and a previous sync's file standing at the path measures exactly + // what a promotion would have left about as often as a promoted one does - two rows + // of one shape being two rows of another shape's length. So that case takes the route + // it always took, which replaces the whole file rather than cutting it back to an + // offset counted in a different one. + refused = new CheckpointFileRefusal(CheckpointFile.Output, + RefusalForStandingOutput(outputPath, out var vanishedTempReason), + CheckpointStopRoute.FreshAfterVanishedTemp, vanishedTempReason); + if (refused.Refuses) return CheckpointRecovery.Refused; + WriteWarning( $"The interrupted sync's temp file is missing or incomplete, so the {checkpoint.ItemsCollected} items it " + "recorded are not on disk. Re-enumerating from the last saved delta token; no changes are lost."); PaginationCheckpoint.Delete(checkpointPath); - return; + // And what the sweep took off the staging name, after what this run did about the + // checkpoint. + if (swept is { } removedAnyway) WriteWarning(removedAnyway.Sentence); + return CheckpointRecovery.Discarded; } - if (!TryTrimOutputToCheckpoint(outputPath, dataLength)) - { - WriteWarning( - $"'{outputPath}' no longer holds the {checkpoint.ItemsCollected} items the resume checkpoint records. " - + "Re-enumerating from the last saved delta token; no changes are lost."); - PaginationCheckpoint.Delete(checkpointPath); - } + // No temp is named, so the run that wrote this checkpoint was appending straight into the + // output and its items are in that file. The claim above, one file over: a sync that has + // resumed once holds -OutputFile open with those changes in it, and leaves a checkpoint + // with no temp for a second run over the same command line to refuse it by - so that run + // judged the checkpoint its own, cut the live file back to the recorded offset, appended, + // and moved the token past changes the output no longer held. + // + // So the claim is not let go. The handle it opens is the handle this run writes through: + // the cut is a SetLength on it, the writer further down appends onto it, and it is + // released where the writing ends. Let go here, a page-fetch before the writer took an + // open of its own, it left a window a second run walked straight through. + var taken = TakeOutputForCheckpoint(outputPath, dataLength, out var takenReason); + if (taken == CheckpointOutput.Taken) return CheckpointRecovery.Resumable; + + refused = new CheckpointFileRefusal(CheckpointFile.Output, RefusalFrom(taken), + CheckpointStopRoute.Appending, takenReason); + if (refused.Refuses) return CheckpointRecovery.Refused; + + // Not the file the checkpoint describes - absent, or shorter than the length recorded - + // so the changes it counts are in no file and there is nothing to append to. + WriteWarning( + $"'{outputPath}' no longer holds the {checkpoint.ItemsCollected} items the resume checkpoint records. " + + "Re-enumerating from the last saved delta token; no changes are lost."); + PaginationCheckpoint.Delete(checkpointPath); + return CheckpointRecovery.Discarded; } private void ExecuteDeltaSync( @@ -468,7 +1131,52 @@ private void ExecuteDeltaSync( string? filter, Stopwatch sw) { + // What the reconcile below takes, on a resumed run, is the output itself, and it is this + // run's until the writing ends. Released on every way out of the loop, and again from + // Dispose, which is what a pipeline stopped from outside reaches. + using var writingEnds = ReleasingOutputWhenWritingEnds(); + bool isFullResync = false; + // Whether this run has given up on the checkpoint on disk, and the temp that checkpoint + // points at. Both outlive the attempt that refused, because the 410 door below sends the + // loop round again: an attempt that has forgotten the refusal resumes from the position + // it was told to leave alone and sweeps away the temp holding its items, seconds after + // warning that neither would be touched. A refusal is still decided again on every load + // - the checkpoint is a file, and the sync that owns it is still going - but only ever + // this way round, and never granted back: the retry rebuilds the request URL, so a + // checkpoint refused against the delta link can compare equal to the full re-sync one, + // and that is not evidence of anything. + var refusedCheckpoint = false; + // The temp that checkpoint named, or null - which is both "nothing refused" and "what + // was refused names no temp". sparedCheckpoint below tells those apart, and the sweep + // reads the pair: a checkpoint written before the name was recorded stands for a temp + // beside this output without saying which one, so its refusal holds the sweep off over + // all of them. + string? refusedTemp = null; + // And which reading refused it, which is what the two doors that would otherwise delete + // the file - the completion path and the 410 - decide on. One reading a sync can carry + // this far: the ownership verdict's, where whatever wrote the file resumes from exactly + // that, so deleting it makes the warning that has just promised both files false. A + // checkpoint whose temp, or whose output, this run could not claim reaches no door, + // because the run that found that out ended there. The cost of sparing is that the file + // suppresses the next -Latest and holds this output's stale-temp sweep off while it + // stands. + SparedCheckpoint? sparedCheckpoint = null; + // The temp an attempt of this run kept when it died, because a checkpoint counting its + // items was on disk. Nothing else records it: the sweep at the top of the next attempt + // is the only thing that reclaims a temp, and a refusal holds that off over every temp + // beside this output - so when the checkpoint stops naming the file, by being replaced + // with the next attempt's or deleted by the run that completed, this is what says which + // one is now a partial copy of the caller's changes that nothing refers to. + string? keptTemp = null; + // Whether an attempt of this run has written a checkpoint of its own over + // -CheckpointPath. The refusal above leaves the file where it is and then the very next + // page boundary saves over the same path, so what the refusal was protecting is gone + // and what is there is this run's - which is the other thing the two doors have to know + // before they spare the file on the refusal's account. Run-scoped, unlike the + // per-attempt flag of the same reading further down: that one answers which attempt + // saved the checkpoint now on disk, and the keep it feeds needs exactly that. + var tookOverCheckpoint = false; for (int attempt = 0; attempt < 2; attempt++) { @@ -477,7 +1185,8 @@ private void ExecuteDeltaSync( var headers = BuildRequestHeaders(null, Headers); if (Prefer is { Length: > 0 }) { - // The dedicated parameter wins over a Prefer key in -Headers + // Dedicated parameter wins over a Prefer key in -Headers (matches the + // ConsistencyLevel convention in BuildRequestHeaders). headers ??= new Dictionary(); headers["Prefer"] = string.Join(",", Prefer); } @@ -488,75 +1197,191 @@ private void ExecuteDeltaSync( var currentFetchUrl = requestUrl; var appendOutput = false; - if (checkpointPath != null && File.Exists(checkpointPath)) + // Refusing is the whole response: the checkpoint is left where it is because + // the sync that wrote it resumes from exactly that, and the items it counted + // are in the temp it names. A refusal already made is not revisited - this run + // has nothing to resume from either way, and the two files stay out of its + // reach for the rest of it. + if (checkpointPath != null && File.Exists(checkpointPath) && !refusedCheckpoint) { - // The checkpoint survived but the output was never promoted from its temp. - // Promote the temp, trimmed to the checkpointed length, so resume appends to - // real data. Otherwise the crashed run items stay in the temp and are never - // emitted, while the delta token - // advances past them on success. - if (outputPath != null) + // Ownership first, in both output modes. A sync writing to the pipeline used + // to skip this block entirely, so a checkpoint from a JSONL sync resumed it + // at that sync's nextLink: the pages before it went to the file and never to + // the pipeline, and the delta token was saved over them on success, which + // puts those changes permanently out of reach. + var orphanCp = PaginationCheckpoint.Load(checkpointPath); + if (orphanCp != null + && RefusalFor(orphanCp, outputPath, requestUrl, checkpointPath, deltaPath) + is { } refusal) { - var orphanCp = PaginationCheckpoint.Load(checkpointPath); - - // The temp glob is shaped from the output path, so a checkpoint from a - // different enumeration must not pull a file into this one. The mismatch - // itself is handled below, here we only decline - var resourceMatches = orphanCp != null - && string.Equals(orphanCp.Resource, requestUrl, StringComparison.Ordinal); - - if (resourceMatches && orphanCp!.NextLink != null) - { - ReconcileCheckpointWithFiles(checkpointPath, outputPath, orphanCp); - } - else if (!File.Exists(outputPath)) - { - WriteWarning("Checkpoint found but output file is missing. Deleting stale checkpoint and starting fresh."); - PaginationCheckpoint.Delete(checkpointPath); - } + // What the refusal leaves has to survive the sweep in this same run, + // the 410 retry that runs it a second time, and the delete either door + // ends with. + refusedCheckpoint = true; + sparedCheckpoint = SparedCheckpoint.NotThisRuns; + refusedTemp = orphanCp.TempFile; + WriteWarning(refusal); } - - if (File.Exists(checkpointPath)) + else { - var checkpoint = PaginationCheckpoint.Load(checkpointPath); - if (checkpoint?.NextLink == null) + // JSONL crash: the checkpoint survives but the output was never promoted + // from its temp file. Promote the temp (trimmed to the checkpointed + // length) so resume appends to real data instead of declaring staleness. + // Without this the resume restarts at checkpoint.NextLink and the crashed + // run's items, sitting only in the temp, are never emitted - while the + // delta token advances past them on success. + // + // A load that answers null reaches none of it. "Checkpoint found but + // output file is missing" is a reading of the checkpoint's contents, and + // a checkpoint nothing could read has none to go on: the delete below is + // for a file that says a sync completed, not for one this run failed to + // open. That case is answered where it is decided, a few lines down. + + // Whether the reconcile left a position worth reloading. Not the same + // question as whether the file is still there: a bail-out deletes the + // checkpoint it gave up on, that delete fails wherever the account cannot + // unlink from the checkpoint's directory, and the file it leaves is what + // File.Exists then answered yes to - so the run reloaded the position it + // had just warned it would re-enumerate past, appended this sync's rows + // onto the previous sync's, and saved a token past items held in no file. + // A checkpoint that outlives being given up on is given up on again next + // run. Resumable where there was nothing to reconcile: a pipeline sync + // has no output to put into any state, and a checkpoint that could not be + // read is answered on its own terms below. + var recovery = CheckpointRecovery.Resumable; + if (outputPath != null && orphanCp != null) { - WriteVerbose("Checkpoint is stale (corrupt or completed). Deleting."); - PaginationCheckpoint.Delete(checkpointPath); - } - else if (!string.Equals(checkpoint.Resource, requestUrl, StringComparison.Ordinal)) - { - // A checkpoint from a different enumeration (different deltaLink, - // different parameters, a completed sync in between). - WriteWarning("Checkpoint belongs to a different enumeration. Deleting checkpoint and starting fresh."); - PaginationCheckpoint.Delete(checkpointPath); + if (orphanCp.NextLink != null) + { + recovery = ReconcileCheckpointWithFiles( + checkpointPath, outputPath, orphanCp, + out var refused, out var staging, out var stagingTemp, + out var stagingRemoved); + + // The write that would have put the checkpoint's changes at the + // output path failed, and both files are where they were found. + // Read as a temp that had gone - which is what the catch around + // the promotion made of it - this run said those changes were on + // no disk, deleted the position counting them, re-enumerated over + // the output and moved the delta token past every one of them, + // with the temp holding them all the while. + if (recovery == CheckpointRecovery.StagingFailed) + { + ThrowTerminatingError(CheckpointStagingStop( + staging!.Value, checkpointPath, outputPath, orphanCp, + stagingTemp!, stagingRemoved)); + return; + } + + // A file of this checkpoint's that this run could not claim ends + // the run, here, before it has enumerated a page, written a byte + // of output, saved a checkpoint, swept a temp or touched the + // delta state. Every other answer this sync could give away the + // two files. + // + // The rows the checkpoint counts are in that file and in no + // other, and the checkpoint is the only thing on disk that + // counts them: a holder still running deletes it itself on the + // way out, a holder that dies keeps its temp for exactly as + // long as a checkpoint counting it is there, and where the + // claim failed on permissions instead it is the position write + // access has to be granted for. Going on takes that away by + // going on: the sweep at the top of this attempt reclaims the + // temp the moment its holder is gone, the first page boundary + // saves this run's own position over the same path, the + // completion door then deletes what it finds there as its own, + // and a page carrying a deltaLink moves the token past changes + // held in that file alone. Resuming is worse again - the items + // are in a file this run cannot have, so they are left out of + // the output while the token advances past them - and where the + // file is the output itself, resuming is a cut taken out from + // under a live writer, whose next write lands past the hole. + // + // Which leaves nothing this sync can do with the -OutputFile + // and -CheckpointPath it was given, and saying so is the whole + // of what it does. + if (recovery == CheckpointRecovery.Refused) + { + ThrowTerminatingError(CheckpointFileStop( + refused, checkpointPath, outputPath, orphanCp)); + return; + } + } + else if (!File.Exists(outputPath)) + { + WriteWarning("Checkpoint found but output file is missing. Deleting stale checkpoint and starting fresh."); + PaginationCheckpoint.Delete(checkpointPath); + } } - else + + // The refusal above is a refusal of this checkpoint, not of one + // read of it: reloading and resuming from the file just left alone is + // the resume it was made to stop. And a checkpoint given up on is not + // resumed from either, however the delete that went with it fared. + if (recovery == CheckpointRecovery.Resumable + && !refusedCheckpoint && File.Exists(checkpointPath)) { - // The checkpoint nextLink comes from a file on disk and is untrusted - var expectedHost = new System.Uri(requestUrl); - var validatedLink = NextLinkValidator.Validate(checkpoint.NextLink, expectedHost); - if (validatedLink != null - && checkpoint.ItemsCollected >= 0 - && checkpoint.PageItemsAlreadyWritten >= 0) + var checkpoint = PaginationCheckpoint.Load(checkpointPath); + if (checkpoint == null) { - resume = new ResumeState( - validatedLink, - checkpoint.PageItemsAlreadyWritten, - checkpoint.ItemsCollected); - currentFetchUrl = validatedLink; - resumedItemCount = checkpoint.ItemsCollected; - appendOutput = outputPath != null && File.Exists(outputPath); - WriteVerbose($"Resuming delta enumeration from checkpoint: {resumedItemCount} items already processed" - + (checkpoint.PageItemsAlreadyWritten > 0 - ? $", skipping {checkpoint.PageItemsAlreadyWritten} items on first page." - : ".")); + // Left where it is. Load answers null for a file torn by a crash + // and for one that is locked or that this account cannot open, + // and this run does the same thing either way - resume stays + // null, so the sync re-enumerates from the delta token. Deleting + // it changed nothing here and destroyed a position the next run, + // or another account, could still have resumed from. + WriteWarning( + "The resume checkpoint could not be read, so it cannot say how far the " + + "interrupted sync got. Re-enumerating from the last saved delta token; " + + "no changes are lost."); } - else + else if (checkpoint.NextLink == null) { - WriteWarning("Checkpoint nextLink failed validation. Deleting checkpoint and starting fresh."); + // A completion marker is this run's to remove, and the reading is + // the file's own: the sync that wrote it finished. Leaving it is + // not free - -Latest is suppressed by the checkpoint file merely + // existing - so a marker that outlives its sync costs the next + // -Latest run the baseline it asked for. + WriteVerbose("Checkpoint indicates the previous sync completed. Deleting stale checkpoint."); PaginationCheckpoint.Delete(checkpointPath); } + else if (RefusalFor(checkpoint, outputPath, requestUrl, + checkpointPath, deltaPath) is { } reloadedRefusal) + { + // Decided again on this load rather than carried over: the + // checkpoint is a file, and the sync that owns it is still going. + refusedCheckpoint = true; + sparedCheckpoint = SparedCheckpoint.NotThisRuns; + refusedTemp = checkpoint.TempFile; + WriteWarning(reloadedRefusal); + } + else + { + // SSRF validation: the checkpoint nextLink is untrusted (a file on disk) + var expectedHost = new System.Uri(requestUrl); + var validatedLink = NextLinkValidator.Validate(checkpoint.NextLink, expectedHost); + if (validatedLink != null + && checkpoint.ItemsCollected >= 0 + && checkpoint.PageItemsAlreadyWritten >= 0) + { + resume = new ResumeState( + validatedLink, + checkpoint.PageItemsAlreadyWritten, + checkpoint.ItemsCollected); + currentFetchUrl = validatedLink; + resumedItemCount = checkpoint.ItemsCollected; + appendOutput = outputPath != null && File.Exists(outputPath); + WriteVerbose($"Resuming delta enumeration from checkpoint: {resumedItemCount} items already processed" + + (checkpoint.PageItemsAlreadyWritten > 0 + ? $", skipping {checkpoint.PageItemsAlreadyWritten} items on first page." + : ".")); + } + else + { + WriteWarning("Checkpoint nextLink failed validation. Deleting checkpoint and starting fresh."); + PaginationCheckpoint.Delete(checkpointPath); + } + } } } } @@ -566,16 +1391,46 @@ private void ExecuteDeltaSync( long itemCount = 0; long removedCount = 0; long totalProcessed = resumedItemCount; - // Seeded from the resume skip, not zero. PageIterator drops skipped items before - // the consumer sees them, so a counter starting at zero records only the newly - // written items of the first resumed page and a mid-page checkpoint there would - // under-count the page + // Seeded from the resume skip, not 0. PageIterator drops the skipped items before + // the consumer ever sees them (PageIterator.cs: "if (isFirstPage && skippedOnPage + // < skipOnFirstPage) continue;"), so a counter starting at 0 records only the + // NEWLY written items of the first resumed page. A mid-page checkpoint there then + // claimed fewer items of that page than the output actually held, and the next + // resume skipped too few and re-emitted the difference - up to a page's worth of + // duplicate lines, which is exactly what the comment below says cannot happen. int pageItemsWritten = resume?.SkipOnFirstPage ?? 0; // What the next two checkpoint sites should say about WHERE the counted items - // are. Set once the writer exists. Null on the pipeline path, which has no file. + // are. Set once the writer exists; null on the pipeline path, which has no file. string? checkpointTempFile = null; long? checkpointDataLength = null; + // Whether the checkpoint on disk is one THIS attempt saved. A checkpoint an + // earlier attempt left names an earlier temp, and the two are not + // interchangeable when it comes to deciding what a file on disk is still for. + var savedOwnCheckpoint = false; + + // A temp a failed attempt of this run kept is kept on one condition: a + // checkpoint counting its items is on disk. Saving a checkpoint here is what + // ends that - the file just written names this attempt's temp, or the output on + // a resumed run, and never the earlier one - so from that moment nothing on + // disk refers to it, no recovery can reach it, and the sweep is still being + // held off on behalf of the refusal that carried it this far. Left where it + // was, it outlived the sync that made it: a partial copy of the caller's + // changes beside the finished output, waiting for some later run's sweep. + // + // Never the temp the checkpoint now names, which is this attempt's own. Best + // effort otherwise: a file something else holds open is not worth failing a + // finished sync over. + void ReleaseKeptTemp() + { + var kept = keptTemp; + if (kept == null + || string.Equals(Path.GetFileName(kept), checkpointTempFile, + StringComparison.OrdinalIgnoreCase)) + return; + keptTemp = null; + try { if (File.Exists(kept)) File.Delete(kept); } catch { } + } void OnPageComplete(PageCompletedInfo info) { @@ -596,33 +1451,101 @@ void SaveBoundaryCheckpoint(PageCompletedInfo info) ItemsCollected = totalProcessed, PageItemsAlreadyWritten = 0, TempFile = checkpointTempFile, + OutputFile = outputPath != null ? Path.GetFullPath(outputPath) : null, DataLength = checkpointDataLength }.Save(checkpointPath); + savedOwnCheckpoint = true; + tookOverCheckpoint = true; + ReleaseKeptTemp(); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { - WriteWarning($"Checkpoint save failed (page boundary): {ex.Message}"); + // Buffered rather than written here. This runs from the page-boundary + // callback, on whichever thread the iterator resumed on, and a page that + // yields nothing resumes on the thread pool - so a WriteWarning from + // here throws PSInvalidOperationException, and the disk problem it was + // reporting ends the sync instead of being reported. A delta enumeration + // meets that shape most: the empty-page limit is raised to 1000 for it + // precisely because delta endpoints return runs of empty pages behind + // nextLinks. The drains write it from the pipeline thread, on the same + // channel the client's own warnings take. + client.EnqueueWarning( + $"Checkpoint save failed (page boundary): {ex.Message}"); } } if (outputPath != null) { // JSONL output mode. Fresh runs write to a temp file and promote on - // success. Checkpointed resumes append to the already-promoted output. + // success; checkpointed resumes append to the already-promoted output. if (!appendOutput) { - // Nothing is being resumed, so any leftover temp is an orphan. Orphans are - // not inert, since TryAdoptOrphanedTemp picks the newest matching file on - // a line count alone and one success makes those rows permanent. Sweeping - // keeps a temp on disk only while a checkpoint describes it - DeleteStaleTemps(outputPath); + // Nothing is being resumed, so no temp on disk describes recoverable + // work. Anything left over was orphaned - by -FullSync, by a -Property + // /-Filter/-Prefer change, by a checkpoint from a different enumeration, + // or by an adoption that declined a torn temp - and orphans are not + // inert: TryAdoptOrphanedTemp picks the NEWEST file matching + // outputPath + ".*.tmp" with nothing but a line count to go on, so a + // survivor from an unrelated enumeration is adoptable by some later + // crash's checkpoint, and one success makes those rows permanent. + // Sweeping here is what keeps "a temp exists only while a checkpoint + // describing it exists" true across runs. + // + // Which is exactly why a refusal is the one case it is wrong in: the + // checkpoint describing that temp IS on disk, left there deliberately a + // moment ago, and sweeping the temp took the one file that made it worth + // keeping - so the sync it belongs to came back to a position pointing at + // nothing and re-enumerated from the start. The sweep is all or nothing + // over this output's temps, so a run that has just refused leaves them to + // the next run that has not. + if (RefusedTempIsOnDisk(refusedTemp, outputPath)) + { + WriteVerbose( + $"Left the temp files beside '{outputPath}' alone: '{refusedTemp}' " + + "holds the items of the checkpoint this run refused."); + } + else if (sparedCheckpoint is SparedCheckpoint.NotThisRuns + && refusedTemp == null) + { + // The refused checkpoint names no temp, which is not the same as + // there being none: a checkpoint written before the name was + // recorded stands for a temp beside this output all the same, and + // nothing here says which of them it is. Keyed on the temp name + // alone, the refusal spared the checkpoint and the sweep took the + // one file it was pointing at, so the sync it belongs to came back + // to a position over nothing - seconds after a warning promising + // both files would be left as they are. The whole sweep goes with + // the refusal instead, and the orphans it would have taken are left + // to the next run over this output that has not refused anything. + WriteVerbose( + $"Left the temp files beside '{outputPath}' alone: one of them " + + "may hold the items of the checkpoint this run refused."); + } + else + { + DeleteStaleTemps(outputPath); + } } + // The output is let go the moment this attempt stops appending to it. Every + // way appendOutput is withdrawn - a checkpoint given up on after the + // reconcile, a nextLink that fails validation, a 410 sending the loop round + // for a full re-sync - ends with this run replacing the output from a temp + // instead, and a handle of its own on the destination is what that move + // fails against. + if (!appendOutput) ReleaseOutputHold(); + var writePath = appendOutput ? outputPath : $"{outputPath}.{Guid.NewGuid():N}.tmp"; // A resumed run appends to the output itself, so there is no temp to name. checkpointTempFile = appendOutput ? null : Path.GetFileName(writePath); try { - using (var writer = new StreamWriter(writePath, appendOutput)) + // The reconcile's own handle on the output, where this run resumed into + // it. Never a second open of that file while the hold stands: on Unix + // the hold's LOCK_EX refuses one from this process as readily as from + // another, and on Windows its write access does. + using (var writer = appendOutput && HeldOutput is { } held + ? held.AppendingWriter() + : new StreamWriter(writePath, appendOutput)) { var enumerable = iterator.StreamAllWithCountAsync( requestUrl, @@ -673,8 +1596,12 @@ void SaveBoundaryCheckpoint(PageCompletedInfo info) ItemsCollected = totalProcessed, PageItemsAlreadyWritten = pageItemsWritten, TempFile = checkpointTempFile, + OutputFile = outputPath != null ? Path.GetFullPath(outputPath) : null, DataLength = checkpointDataLength }.Save(checkpointPath); + savedOwnCheckpoint = true; + tookOverCheckpoint = true; + ReleaseKeptTemp(); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { @@ -722,23 +1649,47 @@ void SaveBoundaryCheckpoint(PageCompletedInfo info) ItemsCollected = totalProcessed, PageItemsAlreadyWritten = pageItemsWritten, TempFile = null, + OutputFile = Path.GetFullPath(outputPath), DataLength = promotedLength }.Save(checkpointPath); + ReleaseKeptTemp(); } catch (Exception promoteEx) when (promoteEx is IOException or UnauthorizedAccessException) { - // Promotion is best-effort. Fall back to the old cleanup. + // Promotion is best-effort; fall back to the old cleanup. } } if (!promoted) { - // A surviving checkpoint describes items that exist only in this - // temp. Deleting it would leave the checkpoint pointing at nothing, - // and the next run would resume in append mode against an output - // that never received these pages. Keep the temp for the next run - // to promote - var resumable = checkpointPath != null && File.Exists(checkpointPath); - if (!resumable) + // A surviving checkpoint describes items that exist ONLY in this + // temp: SaveBoundaryCheckpoint flushes the writer before recording + // the position, so the temp always holds at least ItemsCollected. + // Deleting it leaves the checkpoint pointing past data that is + // nowhere - and the next run then finds a checkpoint, an output + // and no temp, which is the routine "nothing to promote" state, so + // it resumes in APPEND mode against an output that never received + // these pages and the delta token advances past them. + // Keep the temp for the next run to promote. It is deleted by + // promotion, by a later fresh run's own failure once the checkpoint + // is gone, or on the missing-output path below. + // + // The checkpoint counting them has to be one this attempt saved. + // Any other is an earlier attempt's, naming an earlier temp, and + // this attempt's items are then counted by nothing: keeping the + // file left a page no recovery can reach, and the newest temp + // beside an output is what the pre-length adoption path picks up + // on a line count alone. + var resumable = checkpointPath != null && File.Exists(checkpointPath) + && savedOwnCheckpoint; + if (resumable) + { + // Named, because nothing else on disk will be: the sweep the + // next attempt reaches is held off by the refusal, not by + // this, and the checkpoint that makes the file worth keeping + // is replaced or deleted without a word about it. + keptTemp = writePath; + } + else { try { if (File.Exists(writePath)) File.Delete(writePath); } catch { } } @@ -791,7 +1742,43 @@ void SaveBoundaryCheckpoint(PageCompletedInfo info) // Success: the checkpoint's job is done - delete it BEFORE saving delta // state, so a crash between the two leaves a fresh incremental (correct) // rather than a resumable position into a completed enumeration (wrong). - DeleteCheckpoint(checkpointPath, "sync completed"); + // + // "Its job" is this run's job, and the file on disk is only this run's once no + // refusal left it standing and a save of its own has landed on the path. A run + // that refused the checkpoint and then finished without ever saving one - + // everything it had fitting in a single page, or a 410 arriving before the + // first boundary - was deleting the file it had warned it would leave alone, + // orphaning the temp it had just spared the sweep and sending the sync that + // needs both back to page one. + // + // One reading spares it: the ownership verdict's, where whatever wrote the + // file resumes from exactly it. It costs the next -Latest its baseline and + // holds this output's stale-temp sweep off, and that is cheaper than deleting + // a position another run is coming back to. A checkpoint whose temp, or whose + // output, this run could not claim never reaches here at all - that run stopped + // at the reconcile, having written nothing - so this door is only ever answering + // for a run that had an enumeration of its own to finish. + // + // The 410 door reads the same thing for the same reason. savedOwnCheckpoint + // answers only for the attempt that is finishing here and is not asked: every + // site that sets it sets the run-scoped flag beside it, which also covers the + // attempt a query the endpoint refused sent round again with a boundary + // checkpoint of this run's already over the path. + if (sparedCheckpoint is SparedCheckpoint.NotThisRuns && !tookOverCheckpoint) + { + WriteVerbose(SparedCheckpointVerbose("sync completed")); + } + else + { + DeleteCheckpoint(checkpointPath, "sync completed"); + } + + // And with it goes the last thing that could have named a temp an earlier + // attempt kept - the attempt that finished need never have saved a checkpoint of + // its own to release it, if everything it had left fitted in one page. Only + // here, on the way out of a run that completed: an attempt that dies leaves the + // checkpoint and the temp it names for the next run to resume from. + ReleaseKeptTemp(); // Save delta state ONLY after successful completion (Architect P0). // Zero-item responses still save the token (Adversarial P0). @@ -839,13 +1826,48 @@ void SaveBoundaryCheckpoint(PageCompletedInfo info) && ex.StatusCode == HttpStatusCode.Gone) { // 410 Gone: delta token expired (>7 days for directory objects). - // Delete delta state AND any checkpoint - both describe the dead + // Delete delta state AND any checkpoint of this run's - both describe the dead // enumeration - and restart with full sync. // Second attempt builds fresh URL (no delta token), so 410 won't recur. DrainClientMessages(); if (!DeltaState.Delete(deltaPath)) WriteVerbose("Could not delete expired delta state (file may be locked). It will be overwritten."); - DeleteCheckpoint(checkpointPath, "delta token expired (410 Gone)"); + // The output this run took at the reconcile goes with the position it was taken + // for, which is the position the 410 has just declared dead. The retry starts + // over: it collects into a temp of its own and promotes that at the end, so it + // has no use for a handle opened at an offset counted in the enumeration behind + // it - and carried into the retry, that handle is one the retry's own reconcile + // asks about and is refused by. Measured: where the checkpoint delete below + // fails and the retry reads the same position again, the claim answered held and + // the run ended telling the caller to wait for a second sync that was this one. + ReleaseOutputHold(); + // The refusal has to still hold over the file that is actually there. A refused + // checkpoint is spared here because some run comes back to exactly it - but the + // first page boundary of this run saves over the same path, and from then on + // the refused file is gone and the position on disk is this run's own. Sparing + // that one keeps a position into the enumeration the 410 has just declared + // dead: the retry re-enumerates in full, so the resource it records matches + // nothing this command line builds again, and every later run refuses it in + // turn - warning about a sync that was never there and holding off its + // stale-temp sweep on that account. It goes the way any other position into a + // dead enumeration goes. + // + // A checkpoint whose temp, or whose output, this run could not claim does not + // reach this door either: a run that finds that out at the reconcile has not + // sent a request yet, so it cannot be the run a 410 answered. + if (sparedCheckpoint is SparedCheckpoint.NotThisRuns && !tookOverCheckpoint) + { + // The token that expired is this run's; what the refusal left standing is + // not. It was left there moments ago for a sync that resumes from exactly + // it, and an expired token of this one's is no reason to take it away right + // after the warning said both files would be left where they were. + WriteVerbose( + SparedCheckpointVerbose("delta token expired (410 Gone)")); + } + else + { + DeleteCheckpoint(checkpointPath, "delta token expired (410 Gone)"); + } isFullResync = true; requestUrl = BuildListUrl(VersionedBaseUrl, Uri, new ODataListParams(false, Top, Top > 0 ? Top : 999, Filter, Property, null, null, 0, null)); diff --git a/src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs b/src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs index 5ff25a2..c666c18 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs @@ -24,6 +24,10 @@ namespace Mgx.Cmdlets.Cmdlets.Expand; [OutputType(typeof(Hashtable))] public class ExpandMgxRelation : MgxCmdletBase { + /// + /// Object to enrich. Accepts a Hashtable (what the Mgx cmdlets emit) or a PSCustomObject; + /// the relation is attached in the same shape the object arrived in. + /// [Parameter(Mandatory = true, ValueFromPipeline = true)] public object InputObject { get; set; } = null!; @@ -76,6 +80,18 @@ protected override void BeginProcessing() { base.BeginProcessing(); + // Reject absolute URLs (relative paths only); concatenation onto the versioned + // base URL would otherwise silently produce /v1.0/https:/... on the wire. + if (Uri.TrimStart().StartsWith("https://", StringComparison.OrdinalIgnoreCase) || + Uri.TrimStart().StartsWith("http://", StringComparison.OrdinalIgnoreCase)) + { + ThrowTerminatingError(new ErrorRecord( + new ArgumentException( + $"-Uri must be a relative path (e.g., /groups/{{id}}/members), not an absolute URL. Got: '{Uri}'"), + "AbsoluteUriNotAllowed", ErrorCategory.InvalidArgument, null)); + return; + } + if (!Uri.Contains("{id}", StringComparison.OrdinalIgnoreCase)) { ThrowTerminatingError(new ErrorRecord( @@ -125,6 +141,13 @@ protected override void EndProcessing() { WriteWarning("Expand-MgxRelation cancelled by user."); } + catch (Exception) + { + // Unexpected exception types skip the drains above; the buffered verbose and + // warning messages are the context that explains the failure. + DrainClientMessages(); + throw; + } base.EndProcessing(); } @@ -188,13 +211,17 @@ private void ExecuteFanOut() if (!response.IsSuccessStatusCode) { - var errorBody = await response.Content.ReadAsStringAsync(ct); + var errorBody = await client.ReadBodyAsStringAsync(response, ct); throw new GraphServiceException(response.StatusCode, errorBody); } - using var stream = await response.Content.ReadAsStreamAsync(ct); - var json = await JsonSerializer.DeserializeAsync(stream, cancellationToken: ct); - var root = json.Clone(); + // The same read the rest of the module does: the declared charset, and the + // byte-order mark the byte-level deserialize refuses - which lost every + // relation to a per-URL error. A body this cannot use is this URL's failure, + // which is what the fan-out reports and where the streams are written from. + var bodyBytes = await client.ReadBodyAsBytesAsync(response, ct); + var payload = ReadJsonPayloadCore(response.Content.Headers.ContentType, bodyBytes); + var root = ReadRelationBody(payload); if (root.ValueKind == JsonValueKind.Object && root.TryGetProperty("value", out var valueEl) @@ -271,6 +298,11 @@ private void ExecuteFanOut() } + /// + /// Output all buffered objects with the relation property attached. + /// Objects missing IdProperty or with errored IDs get null for the relation. + /// Preserves original pipeline order. + /// private void OutputBufferedObjects(Dictionary resultsById) { // Cache converted results per ID to avoid redundant JsonToHashtable calls @@ -285,8 +317,10 @@ private void OutputBufferedObjects(Dictionary resultsById if (id != null && resultsById.ContainsKey(id)) { - // Converted per input object rather than once per id, so two inputs sharing an id - // get their own hashtables and writing to one relation cannot rewrite the other + // Converted per input object rather than once per id. Two inputs carrying the + // same id used to receive the SAME hashtables, so writing to one output's + // relation silently rewrote the others'. The conversion only repeats when an id + // actually appears twice, which is the same case that made the sharing visible. var items = resultsById[id]; var converted = items.Select(JsonToHashtable).ToArray(); @@ -341,6 +375,42 @@ private void OutputBufferedObjects(Dictionary resultsById } } + /// + /// The relation carried by a body, whatever that body declares itself to be. A relation is + /// not always an entity: /$count answers text/plain with the number in the body, and this + /// cmdlet has no -Raw to fall back on, so the declared type decides how the bytes are + /// decoded and not whether they are read. A body that is no JSON value at all carries no + /// relation to attach, which is this URL's failure. + /// + private static JsonElement ReadRelationBody(JsonPayload payload) + { + if (payload.Kind == JsonPayloadKind.Parsed) + return payload.Json.Clone(); + + if (payload.Kind == JsonPayloadKind.NotJson) + { + try + { + // Decoded by the declared charset above; the mark GetString leaves behind is + // still the serializer's invalid start character. + return JsonSerializer.Deserialize(payload.Text.TrimStart('\uFEFF')); + } + catch (JsonException) { } + } + + throw new InvalidOperationException(DescribeUnusableBody(payload)); + } + + /// Why a body carries no relation to attach, for the per-URL error record. + private static string DescribeUnusableBody(JsonPayload payload) => payload.Kind switch + { + JsonPayloadKind.Empty => "The response has no content, so there is no relation to attach.", + JsonPayloadKind.NotJson => $"The response is {payload.MediaType} and does not parse as JSON. " + + "Body starts: " + payload.Text[..Math.Min(payload.Text.Length, 200)], + _ => "The response declared JSON but does not parse. Body starts: " + + payload.Text[..Math.Min(payload.Text.Length, 200)], + }; + private string BuildUrl(string id) { var resolved = IdPlaceholder.Replace(Uri, System.Uri.EscapeDataString(id)); @@ -351,7 +421,7 @@ private string BuildUrl(string id) // -Uri: Graph rejects a URL carrying $top twice, which turned -Top into a 400 for // every input object. // The client-side truncation in the lambda enforces -Top either way. - if (Top > 0 && !HasTopQueryOption(url)) + if (Top > 0 && !ExistingQueryOptions(url).Contains("$top")) { var separator = url.Contains('?') ? "&" : "?"; url = $"{url}{separator}$top={Top}"; @@ -360,18 +430,6 @@ private string BuildUrl(string id) return url; } - private static bool HasTopQueryOption(string url) - { - var q = url.IndexOf('?'); - if (q < 0) return false; - foreach (var part in url[(q + 1)..].Split('&')) - { - var name = part.Split('=', 2)[0]; - if (string.Equals(name, "$top", StringComparison.OrdinalIgnoreCase)) return true; - } - return false; - } - private void HandleFanOutErrors( Dictionary errors, Dictionary urlToId) @@ -382,7 +440,7 @@ private void HandleFanOutErrors( foreach (var (url, ex) in errors) { var id = urlToId.GetValueOrDefault(url, url); - var statusCode = GetStatusCodeFromException(ex); + var statusCode = MgxErrorPresentation.TryGetStatus(ex); if (SkipNotFound.IsPresent && statusCode == HttpStatusCode.NotFound) { @@ -395,19 +453,9 @@ private void HandleFanOutErrors( continue; } - var (errorId, category) = ex switch - { - BrokenCircuitException bce => ("CircuitBroken", bce.InnerException is GraphServiceException inner - ? MapStatusToCategory(inner.StatusCode) - : ErrorCategory.ResourceUnavailable), - HttpRequestException => ("HttpError", ErrorCategory.ConnectionError), - _ => ("ExpandRelationError", statusCode.HasValue - ? MapStatusToCategory(statusCode.Value) - : ErrorCategory.NotSpecified) - }; - Exception reportEx = ex is BrokenCircuitException - ? new InvalidOperationException(CircuitBreakerMessage, ex) : ex; - WriteError(new ErrorRecord(reportEx, errorId, category, id)); + var (errorId, category, report) = + MgxErrorPresentation.PresentItemFailure(ex, "ExpandRelationError", CircuitBreakerMessage); + WriteError(new ErrorRecord(report, errorId, category, id)); } int skippedTotal = skipped404 + skipped403; @@ -420,10 +468,4 @@ private void HandleFanOutErrors( } } - private static HttpStatusCode? GetStatusCodeFromException(Exception ex) - { - if (ex is GraphServiceException gse) return gse.StatusCode; - if (ex is HttpRequestException hre && hre.StatusCode.HasValue) return hre.StatusCode.Value; - return null; - } } diff --git a/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs b/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs index a67ca59..50f06bb 100644 --- a/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs +++ b/src/Mgx.Cmdlets/Cmdlets/Export/ExportMgxCollection.cs @@ -2,7 +2,6 @@ using System.Management.Automation; using System.Net; using Mgx.Cmdlets.Base; -using Mgx.Cmdlets.Models; using Mgx.Engine.Http; using Mgx.Engine.Models; using Mgx.Engine.Pagination; @@ -12,13 +11,13 @@ namespace Mgx.Cmdlets.Cmdlets.Export; /// /// Export-MgxCollection: Stream paginated Graph API results directly to a JSONL file. -/// One JSON object per line. No PSObject conversion, minimal memory pressure. +/// One JSON object per line; no PSObject conversion, minimal memory pressure. /// Supports checkpoint/resume for interrupted exports. -/// The consumer owns the checkpoint lifecycle, saving at page boundaries and mid-page flushes -/// so a crash resume cannot duplicate items. +/// Consumer owns checkpoint lifecycle: saves at page boundaries and mid-page flushes +/// to prevent duplicate items on crash resume (H6 dedup fix). /// [Cmdlet(VerbsData.Export, "MgxCollection", SupportsShouldProcess = true)] -[OutputType(typeof(MgxExportResult))] +[OutputType(typeof(Models.MgxExportResult))] public class ExportMgxCollection : MgxCmdletBase { [Parameter(Mandatory = true, Position = 0)] @@ -83,6 +82,18 @@ public class ExportMgxCollection : MgxCmdletBase protected override void BeginProcessing() { + // Reject absolute URLs (relative paths only); concatenation onto the versioned + // base URL would otherwise silently produce /v1.0/https:/... on the wire. + if (Uri.TrimStart().StartsWith("https://", StringComparison.OrdinalIgnoreCase) || + Uri.TrimStart().StartsWith("http://", StringComparison.OrdinalIgnoreCase)) + { + ThrowTerminatingError(new ErrorRecord( + new ArgumentException( + $"-Uri must be a relative path (e.g., /users), not an absolute URL. Got: '{Uri}'"), + "AbsoluteUriNotAllowed", ErrorCategory.InvalidArgument, null)); + return; + } + // $search requires ConsistencyLevel: eventual. Error if missing (data loss otherwise) if (!string.IsNullOrEmpty(Search) && string.IsNullOrEmpty(ConsistencyLevel)) { @@ -124,7 +135,7 @@ protected override void ProcessRecord() // -All says how far to page, -Top says how much to return. They answer different // questions, so -All no longer discards the cap: -Top is documented as the total - // maximum, and Invoke-MgxRequest already honors it either way. Neither, and a single + // maximum, and Invoke-MgxRequest already honours it either way. Neither, and a single // page is the limit. int maxItems; bool defaultedToPageSize = false; @@ -138,56 +149,105 @@ protected override void ProcessRecord() defaultedToPageSize = true; } - // Checkpoint safety: a checkpoint without its output file usually means a - // first-run export died mid-flight. The data lives in an orphaned temp file - // (fresh runs write to ..tmp and rename on success), so try to - // adopt it: trim to exactly the item count the checkpoint recorded (content - // past the last flush may be missing or torn) and promote it to the output - // path. Only when no usable temp exists is the checkpoint truly stale. - if (cpPath != null && File.Exists(cpPath)) + // Track whether $count=true was auto-added (not user-requested). + // If the endpoint rejects it with 400, retry without. Settled before the checkpoint is + // looked at: which URL this run fetches is half of whether the checkpoint on disk is + // about it, and none of this needs a Graph connection. + bool countAutoAdded = !string.IsNullOrEmpty(Filter) + && !ExistingQueryOptions(Uri).Contains("$count"); + bool includeAutoCount = countAutoAdded; + bool suppressTop = false; + + // Which form of that URL the checkpoint on disk was written under, when it is this + // export's. The loop below drops the auto-added $count on a bare 400 and the automatic + // $top on a Request_UnsupportedQuery, so an export interrupted after a rejection + // recorded a URL this run does not build until its own retry. Comparing the first form + // alone called that checkpoint another export's: the reconcile recovered nothing, the + // sweep below deleted the temp holding its items, and the retry - which does build that + // form - matched with the append already decided from the output merely existing, so + // the remainder of the enumeration was appended onto a previous export's file. Every + // form goes to the same comparison, and it settles which URL this run resumes before + // anything is promoted, trimmed or deleted. + // + // And decided without writing anything, like everything else above the gate: which + // spellings of a file name the output's directory keeps apart is a question this run + // would otherwise answer by creating a file there, and -WhatIf reports what a run would + // do without moving the caller's directory's write time to do it. An entry already in + // the directory answers it for free; only a directory holding nothing cannot, and there + // the platform's own rule stands for this side of the gate while the run below asks the + // volume itself. Said out loud, because that is the one case where the two can differ. + if (cpPath != null && File.Exists(cpPath) + && DirectoryCaseRuleNeedsProbe(Path.GetDirectoryName(outputPath))) { - var orphanCp = PaginationCheckpoint.Load(cpPath); - if (orphanCp?.NextLink != null) - { - ReconcileCheckpointWithFiles(cpPath, outputPath, orphanCp); - } - else if (!File.Exists(outputPath)) - { - WriteWarning("Checkpoint found but output file is missing. Deleting stale checkpoint and starting fresh."); - PaginationCheckpoint.Delete(cpPath); - } + WriteVerbose( + $"Nothing in '{Path.GetDirectoryName(outputPath)}' says whether it keeps two " + + "spellings of a name apart, and nothing above the ShouldProcess gate writes a " + + "file to ask it, so the checkpoint's file names are compared here under the " + + $"{(PlatformCaseRule ? "case-sensitive" : "case-insensitive")} rule this " + + "platform applies by default. A run that goes past the gate asks the directory " + + "itself, so the two differ only on a volume whose rule is not that default."); + } + + if (cpPath != null && File.Exists(cpPath) + && AttemptFormOf(PaginationCheckpoint.Load(cpPath), outputPath, countAutoAdded, + mayWrite: false) is { } form) + { + includeAutoCount = form.IncludeCount; + suppressTop = form.SuppressTop; } - // ShouldProcess check (before requiring Graph connection, so -WhatIf works without auth) - var initialAppend = cpPath != null && File.Exists(cpPath) && File.Exists(outputPath); - var action = initialAppend ? "Append JSONL data" : "Export JSONL data"; - if (!ShouldProcess(outputPath, action)) + // ShouldProcess check (before requiring Graph connection, so -WhatIf works without auth). + // The action it names is what recovery below would do, worked out without doing any of + // it: recovery promotes, trims and deletes, and it used to run above this gate, so + // -WhatIf rewrote the output and removed checkpoints on the way to reporting that it + // would not. It stays above GetClient either way - "what would this do" is not a + // question that should need a Graph connection. + var wouldAppend = ReconcileCheckpoint(cpPath, outputPath, includeAutoCount, suppressTop, apply: false); + if (!ShouldProcess(outputPath, wouldAppend ? "Append JSONL data" : "Export JSONL data")) + { + // A preview that ends here is the only one that has to say this. The pass below + // reaches the same condition and ends the run on it, so a caller who is not + // previewing reads it as the error it is. + if (_wouldRemoveAtStagingName is { } wouldRemove) WriteWarning(wouldRemove); + if (_wouldStopOnCheckpointFile is { } wouldStop) WriteWarning(wouldStop); return; + } - // Init client after ShouldProcess (populates s_graphEndpoint for sovereign clouds) - var client = GetClient(); + // Whether this run appends to the output rather than exporting into a fresh temp, + // decided once, against every form of the URL the attempts below can build, and only + // ever withdrawn afterwards. Recomputed from a checkpoint and an output merely + // EXISTING, it came back true on the attempt after the reconcile had refused - and the + // rest of the enumeration went onto whatever file was sitting at -OutputFile. A + // refusal has to outlast the attempt that made it. + var appendToOutput = ReconcileCheckpoint(cpPath, outputPath, includeAutoCount, suppressTop, apply: true); - // Track whether $count=true was auto-added (not user-requested). - // If the endpoint rejects it with 400, retry without. - bool countAutoAdded = !string.IsNullOrEmpty(Filter); - bool includeAutoCount = countAutoAdded; - bool suppressTop = false; + // What the reconcile just took, on a resumed run, is the output itself, and it is this + // run's until the writing ends. Released here on every way out of the loop below, + // including the one that never reaches it - GetClient on a session that is not connected + // - and again from Dispose, which is what a pipeline stopped from outside reaches. + using var writingEnds = ReleasingOutputWhenWritingEnds(); - // If checkpoint was saved during a previous retry (URL without $count=true), - // match the checkpoint's URL to avoid mismatch and data loss on resume - if (countAutoAdded && cpPath != null && File.Exists(cpPath) && File.Exists(outputPath)) - { - var existingCp = PaginationCheckpoint.Load(cpPath); - if (existingCp?.Resource != null && !existingCp.Resource.Contains("$count=true")) - includeAutoCount = false; - } + // Init client after the gate (populates s_graphEndpoint for sovereign clouds) + var client = GetClient(); + + // Whether an attempt of this run has written a checkpoint of its own over + // -CheckpointPath. A refusal leaves the file where it is and the very next page boundary + // saves over the same path, so what the refusal was protecting is gone and what is there + // is this run's - which is what the completion path has to know before it spares the + // file on the refusal's account. Run-scoped, unlike the per-attempt flag of the same + // reading inside the loop: that one answers which attempt saved the checkpoint now on + // disk, and the temp it decides to keep needs exactly that. + var tookOverCheckpoint = false; for (int attempt = 0; attempt < 3; attempt++) { try { - // Recalculate append (checkpoint may have been invalidated on previous attempt) - var append = cpPath != null && File.Exists(cpPath) && File.Exists(outputPath); + // The checkpoint can go between attempts - a previous one deleted it as stale, + // or finished with it - so its existence is re-checked. Whose it is, is not + // re-decided: that answer is above, and it can only be taken away here. + appendToOutput = appendToOutput + && cpPath != null && File.Exists(cpPath) && File.Exists(outputPath); var url = BuildUrl(includeAutoCount, suppressTop); var headers = BuildHeaders(); @@ -197,19 +257,81 @@ protected override void ProcessRecord() long resumedItemCount = 0; string currentFetchUrl = url; - if (cpPath != null && append) + if (cpPath != null && appendToOutput) { var checkpoint = PaginationCheckpoint.Load(cpPath); if (checkpoint != null) { - if (checkpoint.NextLink == null) + if (!DescribesThisExport(checkpoint, outputPath, includeAutoCount, + suppressTop, mayWrite: true)) + { + // Ownership is re-decided on every load, not carried over from the + // check above: the checkpoint is a file, the run that owns it is + // still going, and it can be replaced between the two reads. A + // checkpoint that is not this export's is left where it is - the + // export it does belong to needs it - and this run exports fresh. + // Only ever this way round: a refusal stands for the rest of the + // run, and no later attempt grants it back from the two files + // being where they were. + // + // And said out loud. This is the only refusal that lands on a + // checkpoint already accepted, so a resume has been reported and is + // being taken back: the collection is enumerated from the first page + // again, and the pages collected under the resume are collected a + // second time. Withdrawn in silence, the recovery message was the + // last thing on any stream, and a run that had quietly started over + // was indistinguishable from one that resumed. + // + // Two readings reach here and only one of them is about whose the + // file is, which is where they part company: the run does the same + // thing either way - drops the resume and exports fresh - and what + // it leaves on disk afterwards is the opposite thing. Every form of + // the request this run can build goes to the same comparison the + // adoption above the loop makes, so a checkpoint that matches one + // of them is this export's own, saved under the form an endpoint + // that refused $top or the auto-added $count has since sent the + // loop round without. The verdict is about the ones that match none. + var ownership = + AttemptFormOf(checkpoint, outputPath, countAutoAdded, + mayWrite: true) != null + ? CheckpointOwnership.Mine + : OwnershipOf(checkpoint, outputPath, includeAutoCount, + suppressTop, mayWrite: true); + appendToOutput = false; + if (ownership == CheckpointOwnership.Mine) + { + // This export's own position, into the enumeration this run is + // about to make again and replace the output with. Nothing is + // held back on its account: the completion path deletes it as + // it deletes any checkpoint of this run's, and a temp it still + // names is this run's own for the sweep below to reclaim. Left + // standing instead, it is a position nothing comes back to, and + // the next run over the same command line reads it as its own - + // the finished output cut back to the byte count of an + // enumeration that has since been replaced, and the remainder + // appended from a nextLink counting items it no longer holds. + WriteWarning(RebuiltRequestWarning(cpPath, outputPath)); + } + else + { + // Another export's, or one nothing beside this output + // corroborates. The export it belongs to resumes from exactly + // that, so for the rest of the run the sweep below leaves this + // checkpoint's temp where it is, and the completion path leaves + // the checkpoint itself. + _sparedCheckpoint = SparedCheckpoint.NotThisRuns; + _refusedCheckpointTemp = checkpoint.TempFile; + WriteWarning(ReplacedCheckpointWarning(cpPath, outputPath)); + } + } + else if (checkpoint.NextLink == null) { // Completion marker: previous export finished, checkpoint is stale WriteVerbose("Checkpoint indicates previous export completed. Deleting stale checkpoint."); PaginationCheckpoint.Delete(cpPath); - append = false; + appendToOutput = false; } - else if (string.Equals(checkpoint.Resource, url, StringComparison.Ordinal)) + else { // Validate NextLink (SSRF protection) var expectedHost = new System.Uri(url); @@ -230,15 +352,17 @@ protected override void ProcessRecord() { WriteWarning("Checkpoint nextLink failed validation. Deleting checkpoint and starting fresh."); PaginationCheckpoint.Delete(cpPath); - append = false; + appendToOutput = false; } } - else - { - WriteWarning("Checkpoint resource mismatch. Deleting checkpoint and starting fresh."); - PaginationCheckpoint.Delete(cpPath); - append = false; - } + } + else + { + // The unreadable checkpoint above could not be deleted either - it is + // locked, or the account cannot touch it. Appending to the output on + // the strength of a file nothing can read is what has to stop, not the + // export: write a fresh run to a temp and let it replace the output. + appendToOutput = false; } } @@ -246,18 +370,83 @@ protected override void ProcessRecord() // This protects any pre-existing output file from truncation if // the Graph request fails on the first page. // Use GUID to prevent collision when multiple exports target the same file. - if (!append) + if (!appendToOutput) { // No resume is pending, so every "{output}.{guid}.tmp" on disk is an orphan. // Leaving them is not inert: the pre-length adoption path picks the NEWEST // match with only a line count to go on, so a survivor of an unrelated run is // adoptable by some later crash's checkpoint. - DeleteStaleTemps(outputPath); + // + // Except after a refusal, where one of them is not an orphan at all: a + // checkpoint describing it is on disk, left there deliberately a moment ago, + // and the items it counts are in that temp. Sweeping it took the one file + // that made the checkpoint worth keeping, so the export it belongs to came + // back to a position pointing at nothing and re-enumerated from the first + // page - the cost the refusal was written to avoid. The sweep is all or + // nothing over this output's temps, so a run that has just refused leaves + // them to the next run that has not. A run that could not claim the temp + // its checkpoint names never gets here to decide: it stops above the client. + // + // And once more on a retry, where the checkpoint standing over the temp is + // this run's own. An endpoint that refuses the auto-added $count, or $top, + // sends the attempt loop round again after pages have already been + // collected and checkpointed, and the attempt that died kept its temp for + // exactly that checkpoint to resume from. Sweeping here deleted it while + // the checkpoint naming it was still on disk - the one combination neither + // this policy nor the retry's intends - and the next invocation found a + // position pointing at nothing and exported from the first page. + if (RefusedTempIsOnDisk(_refusedCheckpointTemp, outputPath)) + { + WriteVerbose( + $"Left the temp files beside '{outputPath}' alone: '{_refusedCheckpointTemp}' " + + "holds the items of the checkpoint this run refused."); + } + else if (_sparedCheckpoint is SparedCheckpoint.NotThisRuns + && _refusedCheckpointTemp == null) + { + // The refused checkpoint names no temp, which is not the same as there + // being none: a checkpoint written before the name was recorded stands + // for a temp beside this output all the same, and there is nothing here + // that says which of them it is. Keyed on the temp name alone, the + // refusal spared the checkpoint and the sweep took the one file it was + // pointing at - so the export it belongs to came back to a position over + // nothing and re-enumerated from the first page, the exact cost the + // refusal is written to avoid, under a warning that had just promised + // the files would be left as they are. So the whole sweep goes with the + // refusal, and the orphans it would have taken are left to the next run + // over this output that has not refused anything. + WriteVerbose( + $"Left the temp files beside '{outputPath}' alone: one of them may " + + "hold the items of the checkpoint this run refused."); + } + else if (_keptTempPath != null && File.Exists(_keptTempPath)) + { + WriteVerbose( + $"Left the temp files beside '{outputPath}' alone: " + + $"'{Path.GetFileName(_keptTempPath)}' holds the items of this run's own " + + "checkpoint, which a previous attempt left on disk."); + } + else + { + DeleteStaleTemps(outputPath); + } } - var writePath = append ? outputPath : $"{outputPath}.{Guid.NewGuid():N}.tmp"; + // The output is let go the moment this attempt stops appending to it. Every way + // appendToOutput is withdrawn - the checkpoint or the output gone between + // attempts, an ownership verdict landing on a reload, a 410 sending the loop + // round for a full pass - ends with this run replacing the output from a temp + // instead, and a handle of its own on the destination is what that move fails + // against. An attempt that still appends keeps the hold it was given. + if (!appendToOutput) ReleaseOutputHold(); + + var writePath = appendToOutput ? outputPath : $"{outputPath}.{Guid.NewGuid():N}.tmp"; // What the checkpoint sites below should say about WHERE the counted items are. // A resumed run appends to the output itself, so it has no temp to name. - string? checkpointTempFile = append ? null : Path.GetFileName(writePath); + string? checkpointTempFile = appendToOutput ? null : Path.GetFileName(writePath); + // Whether the checkpoint on disk is one THIS attempt saved. A checkpoint an + // earlier attempt left names an earlier temp, and the two are not + // interchangeable when it comes to deciding what a file on disk is still for. + var savedOwnCheckpoint = false; long? checkpointDataLength = null; long itemCount = 0; // Seeded from the resume skip, not 0. PageIterator drops the skipped items before @@ -269,9 +458,38 @@ protected override void ProcessRecord() long totalWritten = resumedItemCount; long? reportedODataCount = null; + // A temp a failed attempt of this run kept is kept on one condition: a + // checkpoint counting its items is on disk. Saving a checkpoint here is what + // ends that - the file just written describes this attempt's temp, or the + // output on a resumed run, and never the earlier one - so from that moment + // nothing on disk refers to it, no recovery can reach it, and the sweep at the + // top of the next attempt is still being held off on its behalf. Left where it + // was, it outlived the export that made it: a partial copy of the caller's + // data sitting beside the finished file, waiting for some later run's sweep. + // + // Never the file this attempt is writing into, which on a resumed run is the + // output itself. Best effort otherwise: a temp something else holds open is + // not worth failing an export over. + void ReleaseKeptTemp() + { + var kept = _keptTempPath; + if (kept == null + || string.Equals(kept, writePath, StringComparison.OrdinalIgnoreCase)) + return; + _keptTempPath = null; + try { if (File.Exists(kept)) File.Delete(kept); } catch { } + } + try { - using (var writer = new StreamWriter(writePath, append)) + // The reconcile's own handle on the output, where this run resumed into it. + // Never a second open of that file while the hold stands: on Unix the hold's + // LOCK_EX refuses one from this process as readily as from another, and on + // Windows its write access does. The pre-length adoption route appends + // without a hold and opens its own, as it always has. + using (var writer = appendToOutput && HeldOutput is { } held + ? held.AppendingWriter() + : new StreamWriter(writePath, appendToOutput)) { var iterator = new PageIterator(client); @@ -297,12 +515,26 @@ protected override void ProcessRecord() ItemsCollected = totalWritten, PageItemsAlreadyWritten = 0, TempFile = checkpointTempFile, + OutputFile = Path.GetFullPath(outputPath), DataLength = checkpointDataLength }.Save(cpPath); + savedOwnCheckpoint = true; + tookOverCheckpoint = true; + ReleaseKeptTemp(); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { - WriteWarning($"Checkpoint save failed (page boundary): {ex.Message}"); + // Buffered rather than written here. This callback runs + // on whichever thread the iterator resumed on, and a + // page that yields nothing resumes on the thread pool - + // so a WriteWarning from here throws + // PSInvalidOperationException, and the disk problem it + // was reporting ends the export instead of being + // reported. The drains write it from the pipeline + // thread, on the same channel the client's own warnings + // take. + client.EnqueueWarning( + $"Checkpoint save failed (page boundary): {ex.Message}"); } } if (info.NextPageUrl != null) @@ -340,8 +572,12 @@ protected override void ProcessRecord() ItemsCollected = totalWritten, PageItemsAlreadyWritten = pageItemsWritten, TempFile = checkpointTempFile, + OutputFile = Path.GetFullPath(outputPath), DataLength = checkpointDataLength }.Save(cpPath); + savedOwnCheckpoint = true; + tookOverCheckpoint = true; + ReleaseKeptTemp(); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { @@ -357,23 +593,74 @@ protected override void ProcessRecord() } } - // Writer closed. Delete checkpoint before moving temp file. - // If crash after delete but before move: temp file orphaned, user re-runs fresh. - if (cpPath != null) PaginationCheckpoint.Delete(cpPath); - - // Move temp file to final path on success. - if (!append) + // Writer closed. Promote the temp, and only then let go of the checkpoint. + // + // The move is the one step that can still fail with every page already + // fetched: the output held open by another process, a read-only + // destination, a share that dropped between the last write and the rename. + // Deleting the checkpoint first made that failure destroy the run - the + // catch below read "no checkpoint" as "nothing to resume", deleted the temp + // holding every item, and the next invocation enumerated from the first + // page. This way round a failed move leaves the checkpoint and the temp on + // disk exactly as an interruption does, and the next run promotes the temp + // and resumes from the last page the checkpoint recorded. + if (!appendToOutput) { File.Move(writePath, outputPath, overwrite: true); } + + // The items are in the output now, so the position describing them is + // spent. A crash in the gap costs a re-enumeration: the checkpoint names a + // temp that is no longer there, which recovery reads as unusable. + // + // Spent for this export, and the file on disk is only this export's once no + // refusal left it standing and a save of its own has landed on the path. A + // run that refused what it found and then finished without ever saving one, + // everything it had fitting in a single page, deleted the file it had warned + // it would leave alone, orphaning the temp it had just spared the sweep and + // sending the export that needs both back to page one. + // + // One of the three readings spares it: the ownership verdict's, where + // whatever wrote the file resumes from exactly it. Not a checkpoint refused + // for a request the endpoint made this run rebuild - that one records a + // position into an enumeration this run has just replaced and nothing more, + // and left standing beside the finished output it is what the next run over + // the same command line reads as its own and promotes a superseded temp + // over a completed file from. And not a checkpoint whose temp, or whose + // output, this run could not claim, which reaches no door at all: that run + // stopped at the reconcile, above the client, having written nothing. + // + // tookOverCheckpoint is the flag, and savedOwnCheckpoint has nothing left + // to add: it answers for the attempt finishing here, and every site that + // sets it sets the run-scoped one beside it - which also covers the attempt + // a query the endpoint refused sent round again with a boundary checkpoint + // of this run's already over the path. + if (_sparedCheckpoint is SparedCheckpoint.NotThisRuns && !tookOverCheckpoint) + { + WriteVerbose(SparedCheckpointVerbose("export completed")); + } + else if (cpPath != null) + { + PaginationCheckpoint.Delete(cpPath); + } + + // And with it goes the last thing that could have named a temp an earlier + // attempt kept - the attempt that finished need never have saved a + // checkpoint of its own to release it, if everything it had left fitted in + // one page. Only here, on the way out of a run that promoted: a promotion + // that failed leaves the checkpoint and the temp for the next run to resume + // from, and both are still exactly what that run needs. + ReleaseKeptTemp(); } catch (Exception attemptEx) { - if (!append) + if (!appendToOutput) { - // Cancelling a checkpointed fresh run promotes the temp, already flushed - // on unwind, and saves a checkpoint matching its content, so the printed - // resume hint holds for first runs too + // User cancellation of a checkpointed fresh run: promote the temp + // file (the using block already flushed it on unwind) and save a + // checkpoint matching its exact content, so the printed resume + // hint is true for first runs too. Previously the temp was + // deleted here and the next run declared the checkpoint stale. var cancelled = attemptEx is OperationCanceledException && CancellationToken.IsCancellationRequested; var promoted = false; @@ -396,12 +683,14 @@ protected override void ProcessRecord() ItemsCollected = totalWritten, PageItemsAlreadyWritten = pageItemsWritten, TempFile = null, + OutputFile = Path.GetFullPath(outputPath), DataLength = promotedLength }.Save(cpPath); + ReleaseKeptTemp(); } catch (Exception promoteEx) when (promoteEx is IOException or UnauthorizedAccessException) { - // Promotion is best-effort. Fall back to the old cleanup. + // Promotion is best-effort; fall back to the old cleanup. } } if (!promoted) @@ -413,11 +702,26 @@ protected override void ProcessRecord() // naming a missing file and start the export over - resume worked // after a kill or a Ctrl-C but never after a handled error, which // is the common way a long export dies. Keep the temp for the next - // run to promote. It is deleted by promotion or by the stale-temp + // run to promote; it is deleted by promotion or by the stale-temp // sweep once the checkpoint is gone. - var resumable = cpPath != null && File.Exists(cpPath); - if (!resumable) + // + // The checkpoint counting them has to be one this attempt saved. + // Any other is an earlier attempt's, naming an earlier temp, and + // this attempt's items are then counted by nothing: keeping the + // file left a partial page no recovery can reach, and the newest + // temp beside an output is what the pre-length adoption path picks + // up on a line count alone. + var resumable = cpPath != null && File.Exists(cpPath) && savedOwnCheckpoint; + if (resumable) + { + // Named for the sweep above, which the next attempt reaches + // before this checkpoint has been resumed from or replaced. + _keptTempPath = writePath; + } + else { + // Whatever is named stays named: the checkpoint on disk is + // still the earlier attempt's, and so is the temp it counts. try { if (File.Exists(writePath)) File.Delete(writePath); } catch { } } } @@ -471,22 +775,27 @@ protected override void ProcessRecord() WriteWarning($"Export cancelled. {resumeHint}"); return; } - catch (GraphServiceException ex) when (includeAutoCount && countAutoAdded && ex.StatusCode == HttpStatusCode.BadRequest) + catch (System.Text.Json.JsonException ex) { - // Auto-added $count=true rejected by this endpoint. Retry without it + // A page body that does not parse. Items already exported stay in the file; + // report the failure instead of ending the pipeline with a raw exception. DrainClientMessages(); - WriteVerbose("Endpoint rejected $count=true (HTTP 400). Retrying without count parameter."); + WriteError(new ErrorRecord( + new InvalidOperationException($"A response page declared JSON but does not parse: {ex.Message}", ex), + "MalformedJsonResponse", ErrorCategory.InvalidData, Uri)); + return; + } + catch (GraphServiceException ex) when (IsCountRejection(ex, includeAutoCount && countAutoAdded)) + { + DrainClientMessages(); + WriteVerbose(CountRejectedVerbose); includeAutoCount = false; continue; } - catch (GraphServiceException ex) when ( - !suppressTop - && !NoPageSize.IsPresent - && ex.StatusCode == HttpStatusCode.BadRequest - && string.Equals(ex.ErrorCode, "Request_UnsupportedQuery", StringComparison.OrdinalIgnoreCase)) + catch (GraphServiceException ex) when (IsTopRejection(ex, suppressTop, NoPageSize.IsPresent)) { DrainClientMessages(); - WriteVerbose("Endpoint rejected $top (Request_UnsupportedQuery). Retrying without page size."); + WriteVerbose(TopRejectedVerbose); suppressTop = true; continue; } @@ -509,34 +818,642 @@ protected override void ProcessRecord() ErrorCategory.PermissionDenied, OutputFile)); return; } + catch (Exception) + { + // Unexpected exception types skip the drains above; the buffered verbose + // and warning messages are the context that explains the failure. + DrainClientMessages(); + throw; + } } } - private string BuildUrl(bool includeCount, bool suppressTop = false) => BuildListUrl( - VersionedBaseUrl, Uri, - new ODataListParams(NoPageSize.IsPresent || suppressTop, Top, PageSize, Filter, - Property, Sort, Search, Skip, ExpandProperty, - IncludeCount: includeCount)); + private bool _warnedDeferredOptions; + + private string BuildUrl(bool includeCount, bool suppressTop = false) + { + var url = BuildListUrl( + VersionedBaseUrl, Uri, + new ODataListParams(NoPageSize.IsPresent || suppressTop, Top, PageSize, Filter, + Property, Sort, Search, Skip, ExpandProperty, + IncludeCount: includeCount), + out var deferred); + if (deferred.Count > 0 && !_warnedDeferredOptions) + { + _warnedDeferredOptions = true; + WriteWarning(DescribeDeferredOptions(deferred)); + } + return url; + } private Dictionary? BuildHeaders() => BuildRequestHeaders(ConsistencyLevel, Headers); - private void ReconcileCheckpointWithFiles(string checkpointPath, string outputPath, PaginationCheckpoint checkpoint) + /// + /// What a caller can act on when the checkpoint on disk turns out to be another export's: + /// which two files disagree, what this run does instead, and that a -CheckpointPath belongs + /// to one export. Refusing it is the whole response - the file is left alone, and so is any + /// staging file beside it, because the export that wrote it resumes from exactly those. + /// + private static string ForeignCheckpointWarning(string checkpointPath, string outputPath) => + $"The resume checkpoint at '{checkpointPath}' belongs to a different export, so it is " + + $"left as it is and '{outputPath}' is exported from the beginning. Two exports sharing " + + "one -CheckpointPath overwrite each other's resume position; give each its own."; + + /// + /// What a caller can act on when the request was rebuilt under this export's own feet: the + /// endpoint refused a query option part-way through, the loop went round without it, and + /// the position on disk counts a URL this run has stopped building. It is this export's + /// own, and the resume it was announced for is being taken back, so the sentence has to say + /// both - the collection is enumerated from the first page again, and the file goes the way + /// any of this run's own positions go once the output is complete. + /// + private static string RebuiltRequestWarning(string checkpointPath, string outputPath) => + $"The resume checkpoint at '{checkpointPath}' records this export under a form of the " + + "request the endpoint has since refused, so the resume is dropped and " + + $"'{outputPath}' is exported from the beginning; the checkpoint is this run's own and " + + "goes with the completed export."; + + /// + /// And what they can act on when a resume already announced turns out to be another run's + /// position: the file at -CheckpointPath was replaced between the reconcile's read and this + /// one, so what is there now describes an output or an enumeration that is not this one. + /// Both files are left where they are, because whatever wrote them resumes from exactly + /// that. + /// + private static string ReplacedCheckpointWarning(string checkpointPath, string outputPath) => + $"The resume checkpoint at '{checkpointPath}' no longer describes this export - another " + + $"run replaced the file - so the resume is dropped and '{outputPath}' is exported from " + + "the beginning; the checkpoint and the temp it names are left as they are."; + + /// + /// What a caller can act on when a checkpoint that records no output file is refused: it + /// was written before the output was recorded, nothing beside this one's output stands for + /// it any more, and this run exports from the beginning. Naming a different export there + /// sends the caller looking for a second run over the same -CheckpointPath, and there is + /// none to find - a temp that has since been removed and an output that has since been + /// replaced reach the same refusal. + /// + private static string UncorroboratedCheckpointWarning(string checkpointPath, string outputPath) => + $"The resume checkpoint at '{checkpointPath}' records no output file, and the files " + + $"beside '{outputPath}' no longer corroborate it, so it is left as it is and " + + $"'{outputPath}' is exported from the beginning; nothing is lost."; + + /// + /// How the run ends when a file its own checkpoint stands for is open in another export, or + /// cannot be opened read-write by this account. Two files reach this, and the reading is one: + /// the rows the checkpoint counts are in that file and in no other, the checkpoint is the + /// only thing on disk that counts them, and this export has no pass to make that leaves the + /// pair intact - it saves its own position over the checkpoint at the first page boundary, + /// after which the completion door deletes what it finds there as its own. So it says what + /// it found, what it did not do, and the two ways out, and every file is exactly where it was. + /// + /// Which file decides what going on would cost, and for an output refusal + /// decides it further: a checkpoint naming a temp + /// that is still there holds the rows in that temp, and going on promotes them, replacing + /// the output outright; one naming a temp that has vanished holds them nowhere this run can + /// reach, and going on enumerates the collection again and replaces the output with the + /// result; and one naming no temp at all was written by a run appending straight into the + /// output, so the rows are in the output itself and going on cuts it back under its writer - + /// whose next write then lands past the hole that offset leaves. All three reach the same + /// claim on the same file, because a rename and a cut are both closed to a run that does not + /// hold it. + /// + /// Not phrased as a fault in the held case: nothing has gone wrong with either file, two + /// exports were pointed at one -OutputFile, and the caller has only to decide which of them + /// they meant. The unopenable case has no second export to go and find at all, and the + /// sentence written for the other one pointed the reader at a run that was never there + /// instead of at the open's own reason - which is named now, in , rather than assumed to be a permission: a symlink loop and an over-long path + /// reach it too, and granting write access fixes neither. + /// + /// The closing sentence is the caller's: a run that is stopping says so, and a -WhatIf pass + /// over the same two files says what would happen instead. + /// + private static string CheckpointFileStopMessage(CheckpointFileRefusal refusal, + string checkpointPath, string outputPath, PaginationCheckpoint checkpoint, string closing) + { + // What going on would cost past a held temp, which is the output beside it - named only + // where there is one. A checkpoint naming a temp is reached with nothing at -OutputFile + // at all, and a run that said it would replace that file sent the caller to look at + // something no export had written yet. + var goingOn = File.Exists(outputPath) + ? $"Going on would replace '{outputPath}' and save this run's position over the " + + "checkpoint that counts those items." + : "Going on would save this run's position over the checkpoint that counts those " + + "items."; + + // The output is unopenable rather than held. The three routes agree past this point - + // the code beyond this check falls through to the same place a file that never held the + // recorded items reaches, which deletes the checkpoint and replaces the output at the + // completion that follows - so only the first clause, naming where the items are now, + // and the recovery word in the advice change per route. + string UnopenableOutputSentence() + { + var reason = refusal.Reason!.Value; + var openFailed = $"'{outputPath}' could not be opened for writing: {reason.Reason}."; + var recovers = refusal.Route switch + { + CheckpointStopRoute.Promoting => " to recover them", + CheckpointStopRoute.FreshAfterVanishedTemp => "", + _ => " to resume from it", + }; + // Four ways out, because the causes do not share one. A permission on the file is + // granted; a permission on the directory holding it is granted there, and the file + // this sentence names is one the caller cannot reach to change or remove; a + // directory at the path is removed or pointed away from, and no grant would have + // made it writable at an offset; and a loop, an over-long path, a socket or a pipe + // is none of those - what this run can honestly say about them is what failed. + var advice = reason.IsDirectory + ? $"Remove it (or point -OutputFile elsewhere) and run again{recovers}, or " + + "remove the checkpoint to export afresh." + : reason.IsUnsearchableParent + ? $"Grant access to that directory and run again{recovers}, or remove the " + + "checkpoint to export afresh." + : reason.IsPermission + ? $"Grant write access to that file and run again{recovers}, or remove " + + "it and the checkpoint to export afresh." + : $"Fix that and run again{recovers}, or remove it and the checkpoint to " + + "export afresh."; + + return refusal.Route switch + { + CheckpointStopRoute.Promoting => + $"{openFailed} The resume checkpoint at '{checkpointPath}' records " + + $"{checkpoint.ItemsCollected} items into the temp '{checkpoint.TempFile}'; " + + $"this run cannot promote them into '{outputPath}'. Going on would delete " + + $"the checkpoint that counts them and replace that file. {closing} {advice}", + + CheckpointStopRoute.FreshAfterVanishedTemp => + $"{openFailed} The resume checkpoint at '{checkpointPath}' recorded " + + $"{checkpoint.ItemsCollected} items into a temp that is gone; this run " + + $"cannot export a fresh copy into '{outputPath}' either. Going on would " + + $"delete the checkpoint that counts them and replace that file. {closing} " + + $"{advice}", + + _ => + $"{openFailed} The resume checkpoint at '{checkpointPath}' records " + + $"{checkpoint.ItemsCollected} items into it, so this run can neither " + + "resume into it nor say whether those items are still there. Going on " + + "would delete the checkpoint that counts them and replace that file. " + + $"{closing} {advice}", + }; + } + + return (refusal.File, refusal.Held) switch + { + (CheckpointFile.NamedTemp, true) => + "Another export is still writing the temp file the resume checkpoint at " + + $"'{checkpointPath}' names, so the {checkpoint.ItemsCollected} items it records " + + $"are that run's. {goingOn} {closing} Wait for that export to finish, or give " + + "this one its own -OutputFile and -CheckpointPath.", + + (CheckpointFile.NamedTemp, false) => + $"The temp file '{checkpoint.TempFile}' that the resume checkpoint at " + + $"'{checkpointPath}' names cannot be opened for writing by this account - the " + + "open was refused on permissions, not on sharing - so the " + + $"{checkpoint.ItemsCollected} items it records cannot be recovered here. " + + $"{goingOn} {closing} Grant write access to that file and run again to recover " + + "them, or remove it and the checkpoint to export afresh.", + + (CheckpointFile.Output, true) => refusal.Route switch + { + CheckpointStopRoute.Promoting => + $"Another export is still writing '{outputPath}'. The resume checkpoint at " + + $"'{checkpointPath}' records {checkpoint.ItemsCollected} items into the " + + $"temp '{checkpoint.TempFile}'; going on would replace '{outputPath}' with " + + "them under the run that holds it, and save this run's position over the " + + $"checkpoint that counts those items. {closing} Wait for that export to " + + "finish, or give this one its own -OutputFile and -CheckpointPath.", + + CheckpointStopRoute.FreshAfterVanishedTemp => + $"Another export is still writing '{outputPath}'. The resume checkpoint at " + + $"'{checkpointPath}' recorded {checkpoint.ItemsCollected} items into a " + + "temp that is gone; going on would export from the beginning and replace " + + $"'{outputPath}' under the run that holds it, and save this run's position " + + $"over the checkpoint that counts those items. {closing} Wait for that " + + "export to finish, or give this one its own -OutputFile and " + + "-CheckpointPath.", + + _ => + $"Another export is still writing '{outputPath}', which the resume " + + $"checkpoint at '{checkpointPath}' records {checkpoint.ItemsCollected} " + + "items into. Going on would cut that file back under it and save this " + + "run's position over the checkpoint that counts those items. " + + $"{closing} Wait for that export to finish, or give this one its own " + + "-OutputFile and -CheckpointPath.", + }, + + _ => UnopenableOutputSentence(), + }; + } + + /// + /// The record the run ends on, which is that sentence in the present tense under an id and a + /// category naming which file the claim failed on and which of the two ways it failed. + /// + private static ErrorRecord CheckpointFileStop(CheckpointFileRefusal refusal, + string checkpointPath, string outputPath, PaginationCheckpoint checkpoint) => + new(new InvalidOperationException(CheckpointFileStopMessage(refusal, checkpointPath, + outputPath, checkpoint, RunStopsHere)), + (refusal.File, refusal.Held) switch + { + (CheckpointFile.NamedTemp, true) => "CheckpointTempHeld", + (CheckpointFile.NamedTemp, false) => "CheckpointTempUnopenable", + (CheckpointFile.Output, true) => "CheckpointOutputHeld", + _ => "CheckpointOutputUnopenable", + }, + // ResourceBusy for a file another run has, which is what a caller waits out. Past + // that the category follows the open's own reason rather than assuming a + // permission: PermissionDenied is a mode this account can be granted, and a symlink + // loop, an over-long path, a socket, a FIFO or a directory at the path is none - + // InvalidOperation says so, and a refusal that carries no reason at all is the + // temp's, whose own sentence names permissions. + refusal.Held + ? ErrorCategory.ResourceBusy + : refusal.Reason is { IsPermission: false } + ? ErrorCategory.InvalidOperation + : ErrorCategory.PermissionDenied, + checkpointPath); + + /// + /// How the run ends when the copy a promotion stages beside the output could not be written, + /// or could not be moved onto the output once it was. Nothing here is a file that has gone + /// missing, and reading it as one was what deleted the checkpoint: every item it counts is in + /// the temp it names, whole and claimable, and the run that comes back stages the same bytes + /// over the same output. So the sentence says where those items are, what failed and where, + /// and that both files are as they were found - and the run stops rather than export past a + /// position it would then save over. + /// + /// The temp those items are in. The checkpoint's own record of it + /// where it has one, and the file adoption picked where the checkpoint predates that field: + /// either way it is the file the reader has to keep, and a sentence that read it off the + /// checkpoint named nothing at all on the adoption route. + /// What the sweep took off the staging name on the way here, where + /// it took anything: the run deleted an entry and then stopped, and a reader who is told only + /// that the staging failed goes looking for a file that is no longer there. Named in the + /// stop rather than warned about separately, because a warning written before a terminating + /// error is what -WarningAction Stop ends the run on instead of this sentence. + private static string CheckpointStagingStopMessage(StagingFailure staging, + string checkpointPath, string outputPath, PaginationCheckpoint checkpoint, string tempName, + string? removedKind, string closing) => + $"The {checkpoint.ItemsCollected} items the resume checkpoint at '{checkpointPath}' " + + $"records are in '{tempName}', whole, but staging them into '{outputPath}' " + + $"failed at '{staging.Path}': {staging.Reason}" + + (removedKind != null + ? $", after removing what stood at the staging name: {removedKind}" + : "") + + $". {closing} fix that and run again to " + + "recover them, or remove the temp and the checkpoint to export afresh."; + + /// + /// + /// The record the run ends on. A write that did not happen, under the checkpoint whose items + /// it was for: not a claim refused on either of the two files, which is what the ids and + /// categories beside this one are about. + /// + private static ErrorRecord CheckpointStagingStop(StagingFailure staging, string checkpointPath, + string outputPath, PaginationCheckpoint checkpoint, string tempName, + string? removedKind) => + new(new InvalidOperationException(CheckpointStagingStopMessage(staging, checkpointPath, + outputPath, checkpoint, tempName, removedKind, NothingWasChanged)), + "CheckpointStagingFailed", ErrorCategory.WriteError, checkpointPath); + + /// + /// The form of this export's URL a checkpoint on disk was written under - whether the + /// auto-added $count was on it and whether the automatic $top was - or null when no form of + /// it is this export's. The attempt loop reaches each of these in turn, dropping $count on a + /// bare 400 and $top on a Request_UnsupportedQuery, and a checkpoint records whichever URL + /// was current when it was saved. Every form is put to the same comparison the loop makes, + /// so a URL the loop can legitimately build is recognized as this export's and one it cannot + /// is still refused. Ordered as the loop reaches them, so an unambiguous match is the one + /// this run would have built first. + /// + private (bool IncludeCount, bool SuppressTop)? AttemptFormOf( + PaginationCheckpoint? checkpoint, string outputPath, bool countAutoAdded, bool mayWrite) + { + if (checkpoint == null) return null; + bool[] counts = countAutoAdded ? [true, false] : [false]; + bool[] tops = [false, true]; + foreach (var includeCount in counts) + { + foreach (var suppressTop in tops) + { + if (DescribesThisExport(checkpoint, outputPath, includeCount, suppressTop, mayWrite)) + return (includeCount, suppressTop); + } + } + return null; + } + + /// + /// True when a checkpoint on disk is about the export this run is making. The reconcile + /// below promotes and trims files before any other part of the checkpoint is looked at, and + /// a byte offset applies to whatever file it is handed - so a -CheckpointPath shared with a + /// second export cut a file that checkpoint knows nothing about, mid-line, and the resumed + /// pages were appended onto the torn byte. Two things have to agree: the output the writing + /// run named - the whole path, since two exports to "users.jsonl" in different directories + /// agree on the name and on nothing else, and on checkpoints too old to record one, the + /// files on disk standing in for it - and the resource, path and query both. The query is + /// half of what a resource is: -Top, -Filter and -Select change which items come back and + /// in what order, so a checkpoint recorded under one of them counts a different enumeration + /// than the one this run is making. Leaving it out put the exact comparison after the + /// promoting and trimming instead of before it, which is not a comparison that can refuse. + /// + private bool DescribesThisExport(PaginationCheckpoint checkpoint, string outputPath, + bool includeCount, bool suppressTop, bool mayWrite) + => OwnershipOf(checkpoint, outputPath, includeCount, suppressTop, mayWrite) + == CheckpointOwnership.Mine; + + /// + /// Which reading refused the checkpoint this run found at -CheckpointPath, or null where + /// nothing refused it. Set wherever a refusal that leaves the file standing is reported and + /// never cleared - an attempt rebuilds the request URL, so a checkpoint refused under one + /// form of it can compare equal under another, and that is not evidence of anything. + /// + /// Read on the way out of a run that completed, where the delete would otherwise take a + /// position this run had warned it would leave alone. One of the three refusals records a + /// reading here: the ownership verdict, where whatever wrote the file resumes from exactly + /// it. A checkpoint refused because the endpoint made this run rebuild the request records + /// none - it is a position into an enumeration this run replaces and nothing else, so it + /// goes with the completed export and the temp it names is this run's own to sweep. And a + /// checkpoint whose temp, or whose output, this run could not claim never reaches a door: + /// that run stops at the reconcile, with nothing enumerated and nothing written. + /// + /// Not something can answer either: a checkpoint + /// written by a run that was appending to its output names no temp at all, and refusing + /// that one counts the same. + /// + private SparedCheckpoint? _sparedCheckpoint; + + /// + /// The temp file a checkpoint this run refused still points at, or null. Refusing is the + /// whole response: the checkpoint is left where it is because the export that wrote it + /// resumes from exactly that, and the items it counted are in the temp it names. + /// + /// Null is two readings, and tells them apart: nothing was + /// refused, or what was refused records no temp name. The second stands for a temp all the + /// same - a checkpoint written before the name was recorded says only that the items are + /// somewhere beside this output - so the sweep reads the pair rather than this alone. + /// + /// + private string? _refusedCheckpointTemp; + + /// + /// What a run would stop on - a temp it could not claim, or the output it would append to - + /// worked out by the preview pass above the ShouldProcess gate, or null where nothing would + /// stop it. The gate names one of two actions on -OutputFile and "nothing at all" is + /// neither, so the sentence goes out beside it - and only on the -WhatIf side, where a real + /// run has already ended on the same condition by the time the gate is past. + /// + private string? _wouldStopOnCheckpointFile; + + /// + /// What a run would take off the staging name before it staged its copy there, worked out by + /// the preview pass and reported beside the gate; null where the name is clear. The pass + /// used to clear the name itself, which made its deletions as real as a run's: a caller who + /// ran -WhatIf to find out what would be touched had the link, the pipe or the leftover copy + /// at that name deleted in order to be told that nothing would be. It is named now and left + /// where it is, and the create test that follows a clear name is skipped for it - nothing + /// standing at the name can be tested without taking it off. + /// + private string? _wouldRemoveAtStagingName; + + /// + /// The temp a failed attempt of this run left on disk for a checkpoint of its own, or null. + /// The attempt loop comes back round on a query the endpoint refused, and the sweep it + /// reaches on the way is the one thing that can take that temp away while the checkpoint + /// naming it is still there. + /// + /// It says nothing once that checkpoint has been replaced by one naming another file, or + /// deleted by the run that finished: the temp is then a partial copy of the caller's data + /// that nothing refers to, and it is deleted where the checkpoint changes rather than left + /// beside the output for a later run's sweep. + /// + private string? _keptTempPath; + + /// + /// Whether that temp is a file the stale-temp sweep would reach. The name comes off a + /// checkpoint, which is untrusted once it is on disk, and nothing here opens it - all it + /// decides is whether the sweep runs at all, so a name of some other shape is answered no + /// rather than refused: the sweep cannot delete it either, and skipping the sweep for it + /// would leave real orphans behind for nothing. + /// + /// "Some other shape" is the sweep's own test of a name and not a looser one kept here. + /// Matching on the prefix and the suffix alone answers yes for "users.jsonl.{guid}.tmp" + /// beside an output named "users" - a file the sweep passes over, because the glob's '*' + /// spans dots and the sweep's own filter does not - so a checkpoint left by the run next + /// door suppressed this output's sweep and its real orphans stayed on disk, which is the + /// case this comment says is answered no. + /// + private static bool RefusedTempIsOnDisk(string? tempFile, string outputPath) { + if (tempFile == null || tempFile != Path.GetFileName(tempFile)) return false; + var dir = Path.GetDirectoryName(outputPath); + if (string.IsNullOrEmpty(dir)) return false; + if (!IsRunTempName(dir, Path.GetFileName(outputPath), tempFile, mayWrite: true)) + return false; + return File.Exists(Path.Combine(dir, tempFile)); + } + + /// Whose the checkpoint on disk is, and where it is not this export's, why not. + private enum CheckpointOwnership + { + /// This export's. + Mine, + + /// Another export's: it records another output, or another enumeration. + AnotherExports, + + /// + /// Records no output file, and the files beside this one no longer stand for it. Which + /// export wrote it cannot be told from here, and telling the caller it was another one + /// is a diagnosis they can go and check and find nothing behind. + /// + Uncorroborated, + } + + /// + /// Which of the three the checkpoint is. A checkpoint recording neither an output nor a + /// length has nothing here to be weighed: RecordedOutputMatches needs a length to hold a + /// file against, and answering "not yours" for want of one put the whole pre-2.1.0 shape + /// behind a refusal, where the stale-temp sweep deleted the items it was pointing at. That + /// shape is decided on the evidence it does have - a temp carrying this output's own name, + /// beside an output that is not there - by the reconcile, which refuses it wherever an + /// output exists. A recorded resource that is not a URL says nothing that can be matched + /// and is refused with the enumerations that do not match. + /// + private CheckpointOwnership OwnershipOf(PaginationCheckpoint checkpoint, string outputPath, + bool includeCount, bool suppressTop, bool mayWrite) + { + if (RecordsAFileToWeigh(checkpoint) + && !RecordedOutputMatches(checkpoint.OutputFile, checkpoint.TempFile, + checkpoint.DataLength, outputPath, mayWrite)) + { + return checkpoint.OutputFile != null + ? CheckpointOwnership.AnotherExports + : CheckpointOwnership.Uncorroborated; + } + + return SameResourceIdentity(ResourceIdentity(checkpoint.Resource), + ResourceIdentity(BuildUrl(includeCount, suppressTop))) + ? CheckpointOwnership.Mine + : CheckpointOwnership.AnotherExports; + } + + /// + /// Put the files into the state the checkpoint claims, or leave them alone, and answer + /// whether the run that follows appends to the output rather than exporting into a fresh + /// temp. With false nothing is written, deleted or warned about: + /// the ShouldProcess gate has to name the action before the run is allowed to take any, and + /// -WhatIf has to leave every file it found exactly as it found it - and leave the directory + /// holding them as it found it too, which is why the flag also decides whether the case rule + /// of that directory may be probed or has to be assumed. + /// + /// A checkpoint records which file its items were written to and how many bytes of that + /// file they occupy, which makes three cases decidable instead of guessed. + /// + /// A temp is named, so the interrupted run was fresh and its items are in that temp while + /// the output still holds a PREVIOUS export. An export is a snapshot and a fresh run that + /// finishes moves its temp over the output, so recovery promotes the temp the same way - + /// appending would leave the previous export's rows in front of this one's. + /// + /// None is named, so the run was appending to the output and its items are already there, + /// past the recorded length only if it wrote more after its last save. Cutting back to that + /// length is what stops those from being written twice. + /// + /// Neither is recorded, so the checkpoint predates this and is handled as it was before. + /// + /// When the counted items turn out to be in no file, nothing has been promoted and no token + /// has moved, so starting over costs a pass and loses nothing, while resuming past them + /// would drop them from the output for good. + /// + private bool ReconcileCheckpoint(string? checkpointPath, string outputPath, + bool includeCount, bool suppressTop, bool apply) + { + if (checkpointPath == null || !File.Exists(checkpointPath)) return false; + + var checkpoint = PaginationCheckpoint.Load(checkpointPath); + if (checkpoint == null) + { + // Load answers null for a checkpoint that does not deserialize and for one that + // cannot be read at all. Either way it says nothing about how much of the output is + // already there, and the run below decides to append from the file merely EXISTING - + // which appended a second complete export onto the first. A checkpoint that cannot + // be read is no checkpoint. + // + // It is left where it is. The loop below forces a fresh export whenever the load + // answers null, so deleting the file changes nothing this run does - and "cannot be + // read" covers a lock and a denying ACL as well as a torn file, so it threw away a + // position that the next run, or another account, could still have resumed from. + if (apply) + { + WriteWarning( + "The resume checkpoint could not be read, so it cannot say what the output already holds. " + + "Exporting again from the beginning; nothing is lost."); + } + return false; + } + + var ownership = OwnershipOf(checkpoint, outputPath, includeCount, suppressTop, apply); + if (ownership != CheckpointOwnership.Mine) + { + if (apply) + { + // What the refusal leaves has to survive this same run - the sweep a few + // lines on, and the delete the completion path ends with. + _sparedCheckpoint = SparedCheckpoint.NotThisRuns; + _refusedCheckpointTemp = checkpoint.TempFile; + WriteWarning(ownership == CheckpointOwnership.AnotherExports + ? ForeignCheckpointWarning(checkpointPath, outputPath) + : UncorroboratedCheckpointWarning(checkpointPath, outputPath)); + } + return false; + } + + if (checkpoint.NextLink == null) + { + // Completion marker. With the output beside it the loop below deletes it and + // exports fresh; without one it describes nothing at all. + if (!File.Exists(outputPath) && apply) + { + WriteWarning("Checkpoint found but output file is missing. Deleting stale checkpoint and starting fresh."); + PaginationCheckpoint.Delete(checkpointPath); + } + return false; + } + if (checkpoint.DataLength is not { } dataLength) { - // An older checkpoint records neither field, so adoption has only a line count and - // the newest matching temp, which is safe only when there is no output to merge into + // Written before any of this was recorded. Adoption then has only a line count and + // the newest matching temp to go on, which is safe to attempt only when there is no + // output it could be merged into - exactly the case this path used to be limited to. if (!File.Exists(outputPath)) { - if (TryAdoptOrphanedTemp(outputPath, checkpoint.ItemsCollected)) - WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted export's temp file. Resuming from checkpoint."); - else + string? adoptedTemp; + StagingFailure? adoptStaging; + StagingRemoval? adoptSwept; + var adopted = apply + ? TryAdoptOrphanedTemp(outputPath, checkpoint.ItemsCollected, + out adoptedTemp, out adoptStaging, out adoptSwept) + : CanAdoptOrphanedTemp(outputPath, checkpoint.ItemsCollected, + out adoptedTemp, out adoptStaging, out adoptSwept); + + // What the adoption took off its own staging name before it created one there, + // or - on the preview, which takes nothing off it - what a run would. The + // applying pass records the deletion here, where it happened, and warns about it + // below once this run's own outcome for the recovery is known: a warning is a + // terminating error under -WarningAction Stop, so written here it ended the run + // with the entry deleted, the items still in the temp and nothing recovered. + if (adoptSwept is { } adoptRemoval) + { + if (apply) WriteVerbose(adoptRemoval.Sentence); + else _wouldRemoveAtStagingName = adoptRemoval.Sentence; + } + + // The copy could not be staged beside the output. Every item the checkpoint + // counts is in the temp this names, whole, so this is not the route below for a + // checkpoint whose output has gone: that one deletes the position counting them + // and exports afresh, and reached from here it did so with the temp holding + // every one. + if (adoptStaging is { } adoptFailure) + { + if (apply) + { + ThrowTerminatingError(CheckpointStagingStop(adoptFailure, checkpointPath, + outputPath, checkpoint, adoptedTemp!, adoptSwept?.Kind)); + return false; + } + + // The preview pass reaches the same name and may not throw. What a real run + // would do there is stop, which is not one of the two actions the gate names. + _wouldStopOnCheckpointFile = CheckpointStagingStopMessage(adoptFailure, + checkpointPath, outputPath, checkpoint, adoptedTemp!, null, + NothingWouldBeChanged); + return false; + } + + if (adopted) + { + if (apply) + { + WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted export's temp file. Resuming from checkpoint."); + // And what the sweep took off the staging name, now that what this run + // did about the checkpoint is on the stream in front of it. Under + // -WarningAction Stop the run ends here, with the items recovered into + // the output and the position they resume from still on disk. + if (adoptSwept is { } removed) WriteWarning(removed.Sentence); + } + return true; + } + + if (apply) { WriteWarning("Checkpoint found but output file is missing. Deleting stale checkpoint and starting fresh."); PaginationCheckpoint.Delete(checkpointPath); + if (adoptSwept is { } removedAnyway) WriteWarning(removedAnyway.Sentence); } - return; + return false; } // The output exists and the checkpoint cannot say whether its items are in it. Both @@ -547,43 +1464,261 @@ private void ReconcileCheckpointWithFiles(string checkpointPath, string outputPa // enumeration onto the earlier export - a 100,000-row file coming back with 163,037. // Undecidable means start over: an export re-runs from the first page and replaces // the output, which costs a pass and cannot leave a wrong file behind. - WriteWarning( - "The resume checkpoint does not record which file the interrupted export's items are in. " - + "Exporting again from the beginning; nothing is lost."); - PaginationCheckpoint.Delete(checkpointPath); - return; + if (apply) + { + WriteWarning( + "The resume checkpoint does not record which file the interrupted export's items are in. " + + "Exporting again from the beginning; nothing is lost."); + PaginationCheckpoint.Delete(checkpointPath); + } + return false; } if (checkpoint.TempFile != null) { - if (TryPromoteNamedTemp(outputPath, checkpoint.TempFile, dataLength)) + TempPromotion promotion; + UnopenableReason? promotionReason; + StagingFailure? promotionStaging = null; + StagingRemoval? promotionSwept; + if (apply) + promotion = TryPromoteNamedTemp(outputPath, checkpoint.TempFile, dataLength, + out promotionReason, out promotionStaging, out promotionSwept); + else + promotion = CanPromoteNamedTemp(outputPath, checkpoint.TempFile, dataLength, + out promotionReason, out promotionStaging, out promotionSwept); + + // What the promotion took off its own staging name before it created one there, or - + // on the preview, which takes nothing off it - what a run would. The applying pass + // records the deletion here, where it happened, and warns about it below once this + // run's own outcome for the recovery is known: a warning is a terminating error + // under -WarningAction Stop, so written here it ended the run with the entry + // deleted, the items still in the temp and nothing recovered. + if (promotionSwept is { } promotionRemoval) + { + if (apply) WriteVerbose(promotionRemoval.Sentence); + else _wouldRemoveAtStagingName = promotionRemoval.Sentence; + } + + // The copy could not be staged, or could not be moved once it was. Both files are + // where they were found and every item the checkpoint counts is still in the temp it + // names, so this is not the route below for a temp whose items are in no file: that + // one deletes the position counting them and exports afresh over the output, and + // reached from here it did so with the temp holding all of them. + if (promotion == TempPromotion.StagingFailed) { - WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted export's temp file. Resuming from checkpoint."); - // Those items are the output now. Repoint the checkpoint at it before anything - // else can fail, so a second interruption cannot promote the same temp again. - checkpoint.TempFile = null; - checkpoint.DataLength = new FileInfo(outputPath).Length; - try { checkpoint.Save(checkpointPath); } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + if (apply) { - WriteWarning($"Checkpoint save failed after recovery: {ex.Message}"); + ThrowTerminatingError(CheckpointStagingStop(promotionStaging!.Value, + checkpointPath, outputPath, checkpoint, checkpoint.TempFile, + promotionSwept?.Kind)); + return false; } - return; + + // The preview pass reaches the same name and may not throw. What a real run + // would do there is stop, which is not one of the two actions the gate names. + _wouldStopOnCheckpointFile = CheckpointStagingStopMessage(promotionStaging!.Value, + checkpointPath, outputPath, checkpoint, checkpoint.TempFile, null, + NothingWouldBeChanged); + return false; } - WriteWarning( - $"The interrupted export's temp file is missing or incomplete, so the {checkpoint.ItemsCollected} items it " - + "recorded are not on disk. Exporting again from the beginning; nothing is lost."); - PaginationCheckpoint.Delete(checkpointPath); - return; + // The output the promotion would land in is another run's, or is not a file this + // account can write at the recorded offset. The same stop the branch below reaches, + // about the same file for the same reason: the run that has the output is appending + // to it, and this one would be replacing it under them. + // + // Asked for before the move, so nothing was replaced - and the temp, with the + // checkpoint still naming it, is exactly as it was found, since the temp is unlinked + // only where the output was kept. So the position and the file it counts are the + // pair they were, and the run that comes back promotes the same rows to the same + // length over the same output. + if (promotion is TempPromotion.OutputHeld or TempPromotion.OutputUnopenable) + { + var promotedOutputRefusal = new CheckpointFileRefusal(CheckpointFile.Output, + promotion == TempPromotion.OutputHeld + ? ClaimRefusal.AnotherRunHasIt + : ClaimRefusal.CannotBeOpened, + CheckpointStopRoute.Promoting, promotionReason); + if (apply) + { + ThrowTerminatingError(CheckpointFileStop(promotedOutputRefusal, checkpointPath, + outputPath, checkpoint)); + return false; + } + + // The preview pass asks the same question of the same file and may not throw: + // it reports what a run would do and does none of it. What it would do is stop, + // which is not one of the two actions the ShouldProcess gate names, so the gate + // writes this sentence beside the one it does name. + _wouldStopOnCheckpointFile = CheckpointFileStopMessage(promotedOutputRefusal, + checkpointPath, outputPath, checkpoint, RunWouldStopHere); + return false; + } + + if (promotion == TempPromotion.Taken) + { + if (apply) + { + // Those items are the output now, and the output is this run's - the + // promotion kept it through the move. Repoint the checkpoint at it before + // anything else can fail, so a second interruption cannot promote the same + // temp again - and before the warning below, which under -WarningAction Stop + // is where the run ends: the position on disk has to be the one that + // resumes from what is now in the output. + RepointCheckpointAtHeldOutput(checkpoint, checkpointPath, outputPath, + HeldOutput!); + WriteWarning($"Recovered {checkpoint.ItemsCollected} items from an interrupted export's temp file. Resuming from checkpoint."); + // And what the sweep took off the staging name, now that what this run did + // about the checkpoint is on the stream in front of it. + if (promotionSwept is { } removed) WriteWarning(removed.Sentence); + } + return true; + } + + // The temp is there and whole, and another run has it open - so the export this + // checkpoint belongs to is not interrupted at all, it is collecting into that file + // right now. Nothing here is a recovery: the items are that run's, the checkpoint is + // the position it comes back to, and this run exports from the beginning into a temp + // of its own. Deleting the checkpoint below would be the same mistake as unlinking + // the temp, one file over. + // + // Or the claim failed because this account cannot open the file read-write, which + // leaves both files in the same place and has no second export behind it. The + // refusal is the same; only what it can honestly say about the cause differs. + var namedTempRefusal = new CheckpointFileRefusal(CheckpointFile.NamedTemp, + RefusalForNamedTemp(outputPath, checkpoint.TempFile, dataLength, apply)); + if (namedTempRefusal.Refuses) + { + // The run ends here, before the client is asked for, before the sweep a few + // lines on, before a temp of this run's own exists and before anything is + // written to -OutputFile or -CheckpointPath. + // + // Leaving the two files alone and exporting anyway was not enough, because + // exporting IS what takes them: the first page boundary saves this run's own + // position over the same path, the completion door then deletes what it finds + // there as its own, and the holder's temp is an orphan the next sweep takes + // with the rows still in it. The property the caller wants - this run proceeds + // and the other run's work survives - is not available at any price, since + // proceeding means replacing the output the checkpoint records and saving over + // the checkpoint itself. So this run does not proceed. + // + // The rows it is protecting are in that temp and in no other file, and the + // checkpoint is the only thing on disk that counts them: a holder still running + // deletes it itself on the way out, a holder that dies keeps its temp for + // exactly as long as a checkpoint counting it is there, and where the claim + // failed on permissions instead it is the position write access has to be + // granted for. + if (apply) + { + ThrowTerminatingError(CheckpointFileStop(namedTempRefusal, checkpointPath, + outputPath, checkpoint)); + return false; + } + + // The preview pass reaches the same two files and may not throw: -WhatIf reports + // what a run would do and does none of it. What it would do is stop, which is + // not one of the two actions the ShouldProcess gate below names, so the gate + // writes this sentence beside the one it does name. + _wouldStopOnCheckpointFile = CheckpointFileStopMessage(namedTempRefusal, + checkpointPath, outputPath, checkpoint, RunWouldStopHere); + return false; + } + + // The temp is not there, or is shorter than the length recorded for it - and the + // first of those is what a promotion leaves for the instant between its unlink and + // the save that repoints the position at the output. A second run over the same + // command line, released with the first, read that instant and said those items were + // not on disk, deleted the position counting them, exported from the beginning and + // moved its own temp over the output at the end - which was the promoting run's + // output, held and being appended to. So the output is asked about before any of + // that, and a promoter holding it ends this run where the branch below ends it. + // + // Only the refusals are read. An output nothing holds is not evidence that a + // promotion landed in it, whatever it measures: this checkpoint records its items + // into the temp, and a previous export's file standing at the path measures exactly + // what a promotion would have left about as often as a promoted one does - two rows + // of one shape being two rows of another shape's length. So that case takes the route + // it always took, which replaces the whole file rather than cutting it back to an + // offset counted in a different one. + var vanishedTempRefusal = new CheckpointFileRefusal(CheckpointFile.Output, + RefusalForStandingOutput(outputPath, out var vanishedTempReason), + CheckpointStopRoute.FreshAfterVanishedTemp, vanishedTempReason); + if (vanishedTempRefusal.Refuses) + { + if (apply) + { + ThrowTerminatingError(CheckpointFileStop(vanishedTempRefusal, checkpointPath, + outputPath, checkpoint)); + return false; + } + + _wouldStopOnCheckpointFile = CheckpointFileStopMessage(vanishedTempRefusal, + checkpointPath, outputPath, checkpoint, RunWouldStopHere); + return false; + } + + if (apply) + { + WriteWarning( + $"The interrupted export's temp file is missing or incomplete, so the {checkpoint.ItemsCollected} items it " + + "recorded are not on disk. Exporting again from the beginning; nothing is lost."); + PaginationCheckpoint.Delete(checkpointPath); + // And what the sweep took off the staging name, after what this run did about + // the checkpoint. + if (promotionSwept is { } removedAnyway) WriteWarning(removedAnyway.Sentence); + } + return false; } - if (!TryTrimOutputToCheckpoint(outputPath, dataLength)) + // The checkpoint names no temp, so the run that wrote it was appending straight into the + // output: the items it counts are in that file, and cutting it back to the recorded + // length is what stops the ones written after the last save from being written twice. + // + // Which is the same claim on the output that the branch above makes on a temp, and for + // the same reason. A run that has resumed once holds -OutputFile open with the rows in + // it, and its checkpoint names no temp for a second run over the same command line to + // refuse it by - so that run judged the checkpoint its own, cut the live file back to + // its own offset and appended, and the holder's next write landed past the hole. + // + // So the claim is not let go. The handle it opens is the handle this run writes through: + // the cut below is a SetLength on it, the writer downstream appends onto it, and it is + // released where the writing ends. Asked for and released here, a page-fetch before the + // writer took an open of its own, it left a window a second run walked straight through + // - both runs past the same claim, both cutting, both appending. + CheckpointOutput taken; + UnopenableReason? takenReason; + if (apply) taken = TakeOutputForCheckpoint(outputPath, dataLength, out takenReason); + else taken = WouldTakeOutputForCheckpoint(outputPath, dataLength, out takenReason); + if (taken == CheckpointOutput.Taken) return true; + + var outputRefusal = new CheckpointFileRefusal(CheckpointFile.Output, RefusalFrom(taken), + CheckpointStopRoute.Appending, takenReason); + if (outputRefusal.Refuses) + { + // Where the temp branch ends the run, before the client, the sweep and the first + // page - and on the preview pass, where it says the same thing in the future tense. + if (apply) + { + ThrowTerminatingError(CheckpointFileStop(outputRefusal, checkpointPath, + outputPath, checkpoint)); + return false; + } + + _wouldStopOnCheckpointFile = CheckpointFileStopMessage(outputRefusal, checkpointPath, + outputPath, checkpoint, RunWouldStopHere); + return false; + } + + // Not the file the checkpoint describes - absent, or shorter than the length recorded - + // so the items it counts are in no file and there is nothing to append to. + if (apply) { WriteWarning( $"'{outputPath}' no longer holds the {checkpoint.ItemsCollected} items the resume checkpoint records. " + "Exporting again from the beginning; nothing is lost."); PaginationCheckpoint.Delete(checkpointPath); } + + return false; } } diff --git a/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs b/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs index b708c2c..98120a4 100644 --- a/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs +++ b/src/Mgx.Cmdlets/Cmdlets/InvokeMgxRequest.cs @@ -1,8 +1,10 @@ -using System.Collections; +using System.Collections; using System.Management.Automation; using System.Net; +using System.Net.Http.Headers; using System.Text; using System.Text.Json; +using System.Text.Json.Serialization; using Mgx.Cmdlets.Base; using Mgx.Engine.Http; using Mgx.Engine.Models; @@ -14,8 +16,8 @@ namespace Mgx.Cmdlets.Cmdlets; /// /// Invoke-MgxRequest: General-purpose resilient client for any Microsoft Graph endpoint. /// Supports streaming pagination, fan-out concurrency, write operations, and checkpoint/resume. -/// For bulk writes, Invoke-MgxBatchRequest is faster than fan-out: fewer round-trips and -/// server-side pacing. +/// For bulk writes (>10 items), consider Invoke-MgxBatchRequest (measured ~1.5x faster +/// than fan-out for PATCH at 1k scale; fewer HTTP round-trips and server-side pacing). /// [Cmdlet(VerbsLifecycle.Invoke, "MgxRequest", DefaultParameterSetName = "Direct", SupportsShouldProcess = true)] @@ -58,6 +60,15 @@ public class InvokeMgxRequest : MgxCmdletBase [Parameter] public SwitchParameter Raw { get; set; } + /// + /// Emit the payload as the service sent it, collection envelope and all, instead of the items + /// inside it. A caller that has to tell one item in a collection from a single entity cannot do + /// it from the unwrapped output: both are one object. -All keeps aggregating pages and ignores + /// this. + /// + [Parameter] + public SwitchParameter Envelope { get; set; } + #endregion #region List parameters @@ -101,6 +112,10 @@ public class InvokeMgxRequest : MgxCmdletBase #region Fan-out parameters + /// + /// Entity ID, or an object carrying one. Accepts a plain string, a Hashtable (what the + /// Mgx cmdlets emit), or a PSCustomObject; the 'id' member is extracted in ProcessRecord. + /// [Parameter(ValueFromPipeline = true, ParameterSetName = "Pipeline")] [Alias("Id")] public object? InputObject { get; set; } @@ -120,8 +135,26 @@ public class InvokeMgxRequest : MgxCmdletBase private readonly List _pipelineIds = []; private bool _isFanOut; + /// + /// The checkpoint gate's answer for this invocation, asked once and reused. -CheckpointPath, + /// -Confirm and -WhatIf are all bound once for the whole run, but a -Uri without a {id} + /// placeholder is not a fan-out: ProcessRecord runs the direct branch below once per piped + /// item, and each of those calls reaches ExecuteList's checkpoint gate on its own. Without + /// this, three piped values asked ShouldProcess three times over the same file - three + /// prompts, and three "What if" lines under -WhatIf, for one run. Set on whichever item + /// reaches the gate first; every later item on that route reuses the answer instead of + /// asking again, a refusal included. + /// + private bool? _mayWriteCheckpoint; + + /// + /// Full base URL including API version (e.g., "https://graph.microsoft.com/v1.0"). + /// private string VersionedBaseUrl => $"{s_graphEndpoint}/{ApiVersion}"; + /// + /// Whether the current invocation is a collection/list operation. + /// private bool IsCollectionMode => All.IsPresent || Top > 0 || !string.IsNullOrEmpty(Filter) || !string.IsNullOrEmpty(Search) || Sort is { Length: > 0 } || @@ -129,9 +162,25 @@ public class InvokeMgxRequest : MgxCmdletBase protected override void BeginProcessing() { + // A fresh cmdlet instance answers this fresh each run; reset explicitly rather than + // relying on that, so the field's own comment is the one place its lifetime is stated. + _mayWriteCheckpoint = null; + + // Reject absolute URLs (relative paths only); concatenation onto the versioned + // base URL would otherwise silently produce /v1.0/https:/... on the wire. + if (Uri.TrimStart().StartsWith("https://", StringComparison.OrdinalIgnoreCase) || + Uri.TrimStart().StartsWith("http://", StringComparison.OrdinalIgnoreCase)) + { + ThrowTerminatingError(new ErrorRecord( + new ArgumentException( + $"-Uri must be a relative path (e.g., /users), not an absolute URL. Got: '{Uri}'"), + "AbsoluteUriNotAllowed", ErrorCategory.InvalidArgument, null)); + return; + } + _isFanOut = Uri.Contains("{id}", StringComparison.OrdinalIgnoreCase); - // $search requires ConsistencyLevel eventual, and omitting it loses data silently + // $search requires ConsistencyLevel: eventual. Error if missing (data loss otherwise) if (!string.IsNullOrEmpty(Search) && string.IsNullOrEmpty(ConsistencyLevel)) { ThrowTerminatingError(new ErrorRecord( @@ -141,8 +190,8 @@ protected override void BeginProcessing() return; } - // $count=true needs ConsistencyLevel eventual on directory endpoints, added automatically - // for -CountVariable and -Filter so discrepancy detection works + // $count=true requires ConsistencyLevel: eventual on directory endpoints; + // auto-add when -CountVariable or -Filter is used (enables count discrepancy detection) if ((!string.IsNullOrEmpty(CountVariable) || !string.IsNullOrEmpty(Filter)) && string.IsNullOrEmpty(ConsistencyLevel)) { @@ -150,7 +199,7 @@ protected override void BeginProcessing() WriteVerbose("Auto-adding ConsistencyLevel:eventual header (required by -Filter/-CountVariable for $count=true)."); } - // Most Graph directory endpoints ignore $skip silently + // $skip is not supported by most Graph directory endpoints (silently ignored) if (Skip > 0) WriteWarning("-Skip ($skip) is not supported by many Graph API endpoints (e.g., /users, /groups). The parameter may be silently ignored."); } @@ -180,7 +229,17 @@ protected override void ProcessRecord() } // Direct mode (no fan-out): execute immediately - ExecuteRequest(Uri, sourceId: null); + try + { + ExecuteRequest(Uri, sourceId: null); + } + catch (Exception) + { + // Unexpected exception types skip the drains inside; the buffered verbose and + // warning messages are the context that explains the failure. + DrainClientMessages(); + throw; + } } /// @@ -220,6 +279,13 @@ protected override void EndProcessing() { WriteWarning("Request cancelled by user."); } + catch (Exception) + { + // Unexpected exception types skip the drains above; the buffered verbose and + // warning messages are the context that explains the failure. + DrainClientMessages(); + throw; + } finally { base.EndProcessing(); @@ -255,17 +321,46 @@ private void ExecuteRequest(string relativeUri, string? sourceId) private void ExecuteList(string relativeUri, string? sourceId) { - // Whether $count=true was added automatically rather than requested - // If the endpoint rejects it with 400, retry without. - bool countAutoAdded = !string.IsNullOrEmpty(Filter) && string.IsNullOrEmpty(CountVariable); + // Track whether $count=true was auto-added (not user-requested via -CountVariable, + // and not already written into -Uri, where dropping it changes nothing and the + // "retry without count" rebuild would re-send a byte-identical request). + bool countAutoAdded = !string.IsNullOrEmpty(Filter) && string.IsNullOrEmpty(CountVariable) + && !ExistingQueryOptions(Uri).Contains("$count"); bool includeAutoCount = countAutoAdded; bool suppressTop = false; - // Resolve the consumer-owned checkpoint path once, before the retry loop + // Consumer-owned checkpoint: resolve path once before the retry loop var cpPath = CheckpointPath != null ? GetUnresolvedProviderPathFromPSPath(CheckpointPath) : null; + // -CheckpointPath is the one file this read path writes, and it is the caller's own + // resume position: every page boundary saves the next link over it, completion deletes + // it, and a checkpoint describing another enumeration is deleted where it is found. + // Reads go out under -WhatIf, by the convention above ExecuteWrite, but a preview that + // overwrites or removes the position a later run resumes from has changed the thing it + // claimed only to describe. So the file gets a gate of its own and the reads keep the + // convention: a refusal skips the saves and the deletes, and the enumeration goes on to + // page and emit. + // + // One decision, taken here rather than at each site. The page-boundary callback runs on + // whichever thread the iterator resumes on - a page that yields nothing resumes on the + // thread pool - and a cmdlet API call from there throws. -Confirm therefore asks once + // per run about one file, which is the question it has to answer anyway - and "once per + // run" has to hold across ProcessRecord calls too, not just within this one: a -Uri with + // no {id} placeholder is not a fan-out, so several piped values still reach ExecuteList + // once each through the direct branch in ProcessRecord, each with its own attempt loop + // and its own call here. _mayWriteCheckpoint (reset once in BeginProcessing) is what + // makes the second and later calls consult the first item's answer instead of asking + // ShouldProcess again - a refusal on the first item refuses the rest. + // + // Reading the checkpoint is not gated: resuming is what the run being previewed would + // do, and a preview that ignored it would page from the start and describe a different + // set of requests. The branches that refuse a checkpoint leave resume null and enumerate + // from the beginning either way - the delete is all the gate withholds. + var mayWriteCheckpoint = cpPath == null + || (_mayWriteCheckpoint ??= ShouldProcess(cpPath, "Save resume checkpoint")); + // If checkpoint was saved during a previous retry (URL without $count=true), // match the checkpoint's URL to avoid mismatch on resume if (countAutoAdded && cpPath != null) @@ -283,10 +378,12 @@ private void ExecuteList(string relativeUri, string? sourceId) var url = BuildCollectionUrl(relativeUri, includeCount: !string.IsNullOrEmpty(CountVariable) || includeAutoCount, noPageSize: suppressTop); - var iterator = new PageIterator(GetClient()); - // -All says how far to page and -Top how much to return, which are different - // questions. -All must not zero the cap, or a bounded slice walks the whole - // collection at the slice page size + var client = GetClient(); + var iterator = new PageIterator(client); + // -All says how far to page, -Top says how much to return, and they are not the + // same question: -All used to zero the cap, so asking for a bounded slice of a + // large collection walked all of it. Worse, -Top also sets the page size, so the + // walk ran at the slice's page size - 150 rows at a time across the whole tenant. var maxItems = Top > 0 ? Top : 0; var headers = BuildHeaders(); long itemCount = 0; @@ -301,7 +398,7 @@ private void ExecuteList(string relativeUri, string? sourceId) if (checkpoint.NextLink == null) { // Completion marker: previous run finished - PaginationCheckpoint.Delete(cpPath); + if (mayWriteCheckpoint) PaginationCheckpoint.Delete(cpPath); } else if (string.Equals(checkpoint.Resource, url, StringComparison.Ordinal)) { @@ -309,19 +406,20 @@ private void ExecuteList(string relativeUri, string? sourceId) var validated = NextLinkValidator.Validate(checkpoint.NextLink, expectedHost); if (validated != null && checkpoint.ItemsCollected >= 0) { - // Page-boundary only. Pipeline items are ephemeral with no file to - // dedup against, so a resumed page may re-emit items already sent - // and the consumer owns deduplication + // Page-boundary only: skipOnFirstPage = 0 because pipeline items are + // ephemeral (no file to dedup against). On resume, the interrupted page + // may re-emit items already sent downstream. Downstream consumers + // (e.g., Export-Csv -Append) are responsible for their own dedup. resume = new ResumeState(validated, 0, checkpoint.ItemsCollected); } else { - PaginationCheckpoint.Delete(cpPath); + if (mayWriteCheckpoint) PaginationCheckpoint.Delete(cpPath); } } else { - PaginationCheckpoint.Delete(cpPath); + if (mayWriteCheckpoint) PaginationCheckpoint.Delete(cpPath); } } } @@ -340,7 +438,7 @@ private void ExecuteList(string relativeUri, string? sourceId) onPageComplete: info => { // Save page-boundary checkpoint - if (cpPath != null && info.NextPageUrl != null) + if (mayWriteCheckpoint && cpPath != null && info.NextPageUrl != null) { try { @@ -353,7 +451,14 @@ private void ExecuteList(string relativeUri, string? sourceId) } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { - WriteWarning($"Checkpoint save failed: {ex.Message}"); + // Buffered rather than written here. This callback runs on + // whichever thread the iterator resumed on, and a page that + // yields nothing resumes on the thread pool - so a WriteWarning + // from here throws PSInvalidOperationException, and the disk + // problem it was reporting ends the enumeration instead of + // being reported. The drains below write it from the pipeline + // thread, on the same channel the client's own warnings take. + client.EnqueueWarning($"Checkpoint save failed: {ex.Message}"); } } }, @@ -371,7 +476,7 @@ private void ExecuteList(string relativeUri, string? sourceId) } catch (PipelineStoppedException) { - // The pipeline consumer is done, such as Select-Object -First, so stop + // Pipeline consumer is done (e.g., Select-Object -First N); stop gracefully throw; } finally @@ -384,7 +489,7 @@ private void ExecuteList(string relativeUri, string? sourceId) WriteCountDiscrepancyWarning(relativeUri, reportedODataCount.Value, itemCount, Filter); // Delete checkpoint on successful completion - if (cpPath != null) PaginationCheckpoint.Delete(cpPath); + if (mayWriteCheckpoint && cpPath != null) PaginationCheckpoint.Delete(cpPath); return; // Success, exit the retry loop } catch (PipelineStoppedException) @@ -396,21 +501,24 @@ private void ExecuteList(string relativeUri, string? sourceId) WriteWarning("Request cancelled by user."); return; } - catch (GraphServiceException ex) when (includeAutoCount && countAutoAdded && ex.StatusCode == HttpStatusCode.BadRequest) + catch (JsonException ex) + { + // A page body that does not parse. Items from earlier pages have already been + // emitted; report the failure instead of crashing the pipeline. + WriteError(new ErrorRecord( + new InvalidOperationException($"A response page declared JSON but does not parse: {ex.Message}", ex), + "MalformedJsonResponse", ErrorCategory.InvalidData, relativeUri)); + return; + } + catch (GraphServiceException ex) when (IsCountRejection(ex, includeAutoCount && countAutoAdded)) { - // This endpoint rejects the automatic $count=true, so retry without it - WriteVerbose("Endpoint rejected $count=true (HTTP 400). Retrying without count parameter."); + WriteVerbose(CountRejectedVerbose); includeAutoCount = false; continue; } - catch (GraphServiceException ex) when ( - !suppressTop - && !NoPageSize.IsPresent - && ex.StatusCode == HttpStatusCode.BadRequest - && string.Equals(ex.ErrorCode, "Request_UnsupportedQuery", StringComparison.OrdinalIgnoreCase)) + catch (GraphServiceException ex) when (IsTopRejection(ex, suppressTop, NoPageSize.IsPresent)) { - // This endpoint does not support $top, so retry without a page size - WriteVerbose("Endpoint rejected $top (Request_UnsupportedQuery). Retrying without page size."); + WriteVerbose(TopRejectedVerbose); suppressTop = true; continue; } @@ -423,6 +531,16 @@ private void ExecuteList(string relativeUri, string? sourceId) WriteGraphError(ex, relativeUri, ApiVersion); return; } + finally + { + // Every way out of the attempt above - the success path, a retry via continue, + // or any catch - drains here once, from the pipeline thread. The last catch + // already drains inside WriteGraphError before it writes its error record, so + // that exit reaches this a second time; TryDequeue found the queue already + // empty from the first drain, so the second call writes nothing and nothing + // doubles. + DrainClientMessages(); + } } } @@ -440,15 +558,14 @@ private void ExecuteGet(string relativeUri, string? sourceId) if (!response.IsSuccessStatusCode) { - var body = response.Content.ReadAsStringAsync(CancellationToken).GetAwaiter().GetResult(); + var body = client.ReadBodyAsStringAsync(response, CancellationToken).GetAwaiter().GetResult(); throw new GraphServiceException(response.StatusCode, body); } - using var stream = response.Content.ReadAsStreamAsync(CancellationToken).GetAwaiter().GetResult(); - var json = JsonSerializer.DeserializeAsync(stream, cancellationToken: CancellationToken) - .AsTask().GetAwaiter().GetResult(); - - OutputPayload(json, sourceId); + var bodyBytes = client.ReadBodyAsBytesAsync(response, CancellationToken).GetAwaiter().GetResult(); + var json = ReadJsonPayload(response, bodyBytes, relativeUri); + if (json != null) + OutputPayload(json.Value, sourceId); } catch (OperationCanceledException) when (CancellationToken.IsCancellationRequested) { @@ -466,16 +583,34 @@ private void ExecuteGet(string relativeUri, string? sourceId) private void ExecuteWrite(HttpMethod method, string relativeUri, string? sourceId) { + // The gate sits on the write and on the bulk write below, and on nothing else: reads go + // out under -WhatIf, per the PowerShell convention, so that a dry run under + // $WhatIfPreference has real data for the gated writes to be previewed against. The help + // says so, because "the cmdlet is not run" would not be true of a GET that connects, + // pages, spends resource units and emits objects. Invoke-MgxBatchRequest gates its reads + // too, for the reason stated above its own gate; the two contracts are documented apart. if (!ShouldProcess(relativeUri, method.Method)) return; + string? serializedBody; + try + { + serializedBody = ResolveRequestBody(method); + } + catch (ArgumentException ex) + { + // Pre-flight: nothing was sent. Terminating, per about_Mgx_Errors. + ThrowTerminatingError(new ErrorRecord(ex, "InvalidBodyValue", + ErrorCategory.InvalidArgument, relativeUri)); + return; + } + NoteNonJsonStringBody(); + try { var url = $"{VersionedBaseUrl}{NormalizePath(relativeUri)}"; var client = GetClient(); var headers = BuildHeaders(); - - var serializedBody = ResolveRequestBody(method); HttpContent? content = serializedBody != null ? new StringContent(serializedBody, Encoding.UTF8, "application/json") : null; @@ -488,7 +623,7 @@ private void ExecuteWrite(HttpMethod method, string relativeUri, string? sourceI if (!response.IsSuccessStatusCode) { - var body = response.Content.ReadAsStringAsync(CancellationToken).GetAwaiter().GetResult(); + var body = client.ReadBodyAsStringAsync(response, CancellationToken).GetAwaiter().GetResult(); throw new GraphServiceException(response.StatusCode, body); } @@ -496,14 +631,12 @@ private void ExecuteWrite(HttpMethod method, string relativeUri, string? sourceI if (response.StatusCode == HttpStatusCode.NoContent) return; - // stream.Length throws on network and decompression streams, so read bytes to - // handle a null ContentLength and empty bodies - var bodyBytes = response.Content.ReadAsByteArrayAsync(CancellationToken).GetAwaiter().GetResult(); - if (bodyBytes.Length > 0) - { - var jsonEl = JsonSerializer.Deserialize(bodyBytes); - OutputPayload(jsonEl, sourceId); - } + // stream.Length throws NotSupportedException on network/decompression streams. + // Read as bytes to safely handle null ContentLength (chunked transfer) and empty bodies. + var bodyBytes = client.ReadBodyAsBytesAsync(response, CancellationToken).GetAwaiter().GetResult(); + var jsonEl = ReadJsonPayload(response, bodyBytes, relativeUri); + if (jsonEl != null) + OutputPayload(jsonEl.Value, sourceId); } finally { @@ -542,18 +675,18 @@ private void ExecuteFanOut() if (_pipelineIds.Count == 1) { - // A single id runs directly, without the fan-out machinery + // Single ID: direct execution, no ConcurrentFanOut overhead var resolved = ResolveTemplate(_pipelineIds[0]); ExecuteRequest(resolved, _pipelineIds[0]); return; } - // Deduplicated to avoid key collisions and repeat requests + // Deduplicate pipeline IDs to avoid dict key collision and redundant HTTP calls var uniqueIds = _pipelineIds.Distinct(StringComparer.OrdinalIgnoreCase).ToList(); if (uniqueIds.Count < _pipelineIds.Count) WriteVerbose($"Deduplicated {_pipelineIds.Count} pipeline IDs to {uniqueIds.Count} unique IDs."); - // Initializes the client, which populates s_graphEndpoint for sovereign clouds + // Ensure client is initialized (populates s_graphEndpoint for sovereign clouds) var client = GetClient(); var fanOut = new ConcurrentFanOut(client, Concurrency); var headers = BuildHeaders(); @@ -577,6 +710,10 @@ private void ExecuteFanOut() } } + /// + /// Collection fan-out: each ID resolves to a collection endpoint (e.g., /groups/{id}/members). + /// Uses FetchAllAsync which calls GetCollectionPageAsync (expects "value" array). + /// private void ExecuteCollectionFanOut(ConcurrentFanOut fanOut, List uniqueIds, Dictionary? headers) { var urls = uniqueIds.Select(id => BuildCollectionUrl(ResolveTemplate(id), includeCount: false)).ToList(); @@ -608,6 +745,10 @@ private void ExecuteCollectionFanOut(ConcurrentFanOut fanOut, List uniqu HandleFanOutErrors(fanOutResult.Errors); } + /// + /// Entity fan-out: each ID resolves to a single entity endpoint (e.g., /users/{id}). + /// Uses ForEachAsync with GetAsync per entity since the response is a flat object, not a collection. + /// private void ExecuteEntityFanOut(ConcurrentFanOut fanOut, List uniqueIds, Dictionary? headers) { // Clear results from any previous invocation @@ -628,33 +769,33 @@ private void ExecuteEntityFanOut(ConcurrentFanOut fanOut, List uniqueIds if (!response.IsSuccessStatusCode) { - var body = await response.Content.ReadAsStringAsync(ct); + var body = await client.ReadBodyAsStringAsync(response, ct); throw new GraphServiceException(response.StatusCode, body); } - using var stream = await response.Content.ReadAsStreamAsync(ct); - var json = await JsonSerializer.DeserializeAsync(stream, cancellationToken: ct); - - // Clone detaches from the parent JsonDocument buffer, which becomes invalid - // once the response stream is disposed - var cloned = json.Clone(); + // The body crosses to the cmdlet thread undecoded. ReadJsonPayload answers + // the charset, the empty body, the non-JSON body and the parse failure, and + // every one of those answers is a verbose, output or error record - none of + // which a worker thread may write. + var bodyBytes = await client.ReadBodyAsBytesAsync(response, ct); + var contentType = response.Content.Headers.ContentType; - // WriteObject must run on the cmdlet thread, so ConcurrentFanOut collects - // results and they are written afterwards - // Store in a thread-safe structure lock (_entityFanOutResults) { - _entityFanOutResults.Add((id, cloned)); + _entityFanOutResults.Add((id, response.StatusCode, contentType, bodyBytes)); } }, CancellationToken).GetAwaiter().GetResult(); DrainClientMessages(); // Output results on the cmdlet thread - foreach (var (sourceId, json) in _entityFanOutResults) + foreach (var (sourceId, status, contentType, bodyBytes) in _entityFanOutResults) { + var json = ReadJsonPayload(status, contentType, bodyBytes, ResolveTemplate(sourceId)); + if (json == null) + continue; totalItems++; - OutputItem(json, sourceId); + OutputItem(json.Value, sourceId); } HandleFanOutErrors(errors); @@ -665,17 +806,34 @@ private void ExecuteEntityFanOut(ConcurrentFanOut fanOut, List uniqueIds } } - private readonly List<(string sourceId, JsonElement json)> _entityFanOutResults = []; + private readonly List<(string sourceId, HttpStatusCode status, + MediaTypeHeaderValue? contentType, byte[] bodyBytes)> _entityFanOutResults = []; + /// + /// Write fan-out: execute POST/PATCH/PUT/DELETE for each piped ID concurrently. + /// Same body is applied to all operations. URIs are resolved via {id} template. + /// private void ExecuteWriteFanOut(ConcurrentFanOut fanOut, List uniqueIds, Dictionary? headers, HttpMethod httpMethod) { if (!ShouldProcess($"{httpMethod.Method} {uniqueIds.Count} items via {Uri}", "Bulk write")) return; + string? serializedBody; try { // Serialize body once (shared across all operations) - string? serializedBody = ResolveRequestBody(httpMethod); + serializedBody = ResolveRequestBody(httpMethod); + } + catch (ArgumentException ex) + { + ThrowTerminatingError(new ErrorRecord(ex, "InvalidBodyValue", + ErrorCategory.InvalidArgument, Uri)); + return; + } + NoteNonJsonStringBody(); + + try + { // Build operations list: (id, resolved URL) var operations = uniqueIds.Select(id => @@ -762,7 +920,10 @@ private void HandleBulkWriteErrors(IReadOnlyList errors) } var ex = new InvalidOperationException($"HTTP {error.StatusCode} for '{error.Id}': {error.Message}"); - // StatusCode 0 = infrastructure exception (network, circuit breaker, deserialization) + // StatusCode 0 = a write that reached no HTTP status at all: the network, an open + // circuit. Not a body that stalls or does not parse - the server answered those, and + // they carry the status it answered with, so they take the branch below and read as + // what they are rather than as a request that never arrived. var (errorId, category) = error.StatusCode == 0 ? ("BulkWriteInfraError", ErrorCategory.ConnectionError) : ("BulkWriteError", MapStatusToCategory(statusCode)); @@ -782,7 +943,7 @@ private void HandleFanOutErrors(Dictionary errors) bool has404 = false; foreach (var (key, ex) in errors) { - var statusCode = GetStatusCodeFromException(ex); + var statusCode = MgxErrorPresentation.TryGetStatus(ex); // Only a 404 that might mean "no such endpoint" is worth a beta hint. A 404 naming // a missing object says the path was fine, and hinting over it sends the caller to @@ -801,14 +962,9 @@ private void HandleFanOutErrors(Dictionary errors) continue; } - // Preserve diagnostic specificity for infrastructure exceptions - var (errorId, category) = ex switch - { - Polly.CircuitBreaker.BrokenCircuitException => ("CircuitBroken", ErrorCategory.ResourceUnavailable), - HttpRequestException => ("HttpError", ErrorCategory.ConnectionError), - _ => ("FanOutError", statusCode.HasValue ? MapStatusToCategory(statusCode.Value) : ErrorCategory.NotSpecified) - }; - WriteError(new ErrorRecord(ex, errorId, category, key)); + var (errorId, category, report) = + MgxErrorPresentation.PresentItemFailure(ex, "FanOutError", CircuitBreakerMessage); + WriteError(new ErrorRecord(report, errorId, category, key)); } if (has404) @@ -835,13 +991,12 @@ private string ResolveTemplate(string id) System.Text.RegularExpressions.RegexOptions.IgnoreCase); } - private static HttpStatusCode? GetStatusCodeFromException(Exception ex) - { - if (ex is GraphServiceException gse) return gse.StatusCode; - if (ex is HttpRequestException hre && hre.StatusCode.HasValue) return hre.StatusCode.Value; - return null; - } - + /// + /// Check if a GraphServiceException should be silently skipped based on + /// -SkipNotFound / -SkipForbidden switches. Used by single-request paths + /// (ExecuteGet, ExecuteWrite, ExecuteList) so that these switches work + /// consistently regardless of whether the pipeline has 1 or N items. + /// private bool ShouldSkipGraphError(GraphServiceException ex) { if (SkipNotFound.IsPresent && ex.StatusCode == HttpStatusCode.NotFound) @@ -864,11 +1019,27 @@ private bool ShouldSkipGraphError(GraphServiceException ex) private string BuildCollectionUrl(string relativeUri) => BuildCollectionUrl(relativeUri, includeCount: !string.IsNullOrEmpty(CountVariable) || !string.IsNullOrEmpty(Filter)); - private string BuildCollectionUrl(string relativeUri, bool includeCount, bool noPageSize = false) => BuildListUrl( - VersionedBaseUrl, relativeUri, - new ODataListParams(NoPageSize.IsPresent || noPageSize, Top, PageSize, Filter, - Property, Sort, Search, Skip, ExpandProperty, - IncludeCount: includeCount)); + private int _warnedDeferredOptions; + + private void WarnDeferredOptions(List deferred) + { + // Interlocked: URL building runs inside fan-out lambdas that can resume off the + // pipeline thread; the warning must fire exactly once and from one caller. + if (deferred.Count == 0 || Interlocked.Exchange(ref _warnedDeferredOptions, 1) == 1) return; + WriteWarning(DescribeDeferredOptions(deferred)); + } + + private string BuildCollectionUrl(string relativeUri, bool includeCount, bool noPageSize = false) + { + var url = BuildListUrl( + VersionedBaseUrl, relativeUri, + new ODataListParams(NoPageSize.IsPresent || noPageSize, Top, PageSize, Filter, + Property, Sort, Search, Skip, ExpandProperty, + IncludeCount: includeCount), + out var deferred); + WarnDeferredOptions(deferred); + return url; + } private string BuildGetUrl(string relativeUri) { @@ -876,10 +1047,15 @@ private string BuildGetUrl(string relativeUri) var queryParams = new List(); if (Property is { Length: > 0 }) - queryParams.Add($"$select={System.Uri.EscapeDataString(string.Join(",", Property))}"); + queryParams.Add($"$select={EscapeQueryValue(string.Join(",", Property))}"); if (ExpandProperty is { Length: > 0 }) - queryParams.Add($"$expand={System.Uri.EscapeDataString(string.Join(",", ExpandProperty))}"); + queryParams.Add($"$expand={EscapeQueryValue(string.Join(",", ExpandProperty))}"); + + var existing = ExistingQueryOptions(baseUrl); + WarnDeferredOptions(queryParams.Where(qp => existing.Contains(qp.Split('=', 2)[0])) + .Select(qp => qp.Split('=', 2)[0]).ToList()); + queryParams.RemoveAll(qp => existing.Contains(qp.Split('=', 2)[0])); if (queryParams.Count == 0) return baseUrl; @@ -896,8 +1072,89 @@ private string BuildGetUrl(string relativeUri) private Dictionary? BuildHeaders() => BuildRequestHeaders(ConsistencyLevel, Headers); + /// + /// Emit a Graph response payload. A collection envelope ({"value":[...]}, returned by GET and by + /// action endpoints such as /directoryObjects/getByIds) is unwrapped into one item per element; + /// anything else is emitted whole. + /// + /// + /// A string -Body travels verbatim under Content-Type: application/json. When it is not + /// JSON that is usually intentional (a raw upload) - but the endpoint sees the wrong + /// content type unless -Headers overrides it, and the resulting 400 names neither. + /// A verbose note, not a warning: raw string bodies are a supported path. + /// + private void NoteNonJsonStringBody() + { + if (Body is null || UnwrapPSObject(Body) is not string s || string.IsNullOrWhiteSpace(s)) return; + if (Headers != null && Headers.Keys.Cast() + .Any(k => string.Equals(k.ToString(), "Content-Type", StringComparison.OrdinalIgnoreCase))) + return; + try + { + using var _ = JsonDocument.Parse(s); + } + catch (JsonException) + { + WriteVerbose("-Body is a string that is not JSON; it is sent verbatim with " + + "Content-Type: application/json. Add a Content-Type to -Headers to declare its real type."); + } + } + + /// + /// A response body as JSON for output, or null when nothing further should be emitted: + /// an empty body (204, or 200 with no content), a non-JSON body (emitted as text under + /// -Raw, otherwise an error), or a body that declares JSON and does not parse. + /// + private JsonElement? ReadJsonPayload(HttpResponseMessage response, byte[] bodyBytes, string relativeUri) + => ReadJsonPayload(response.StatusCode, response.Content.Headers.ContentType, bodyBytes, relativeUri); + + /// + /// The same read for a caller that no longer holds the response: the entity fan-out reads + /// the body on a worker thread and decodes it here, where writing to the streams is legal. + /// + private JsonElement? ReadJsonPayload(HttpStatusCode statusCode, MediaTypeHeaderValue? declaredType, + byte[] bodyBytes, string relativeUri) + { + var payload = ReadJsonPayloadCore(declaredType, bodyBytes); + switch (payload.Kind) + { + case JsonPayloadKind.Empty: + WriteVerbose($"HTTP {(int)statusCode}: response has no content; nothing to emit."); + return null; + + case JsonPayloadKind.NotJson: + if (Raw.IsPresent) + { + WriteObject(payload.Text); + return null; + } + WriteError(new ErrorRecord( + new InvalidOperationException( + $"The response is {payload.MediaType}, not JSON. Use -Raw to receive it as text, or Get-MgxContent for file and media content."), + "NonJsonResponse", ErrorCategory.InvalidData, relativeUri)); + return null; + + case JsonPayloadKind.Malformed: + var snippet = payload.Text[..Math.Min(payload.Text.Length, 200)]; + WriteError(new ErrorRecord( + new InvalidOperationException( + $"The response declared JSON but does not parse. Body starts: {snippet}"), + "MalformedJsonResponse", ErrorCategory.InvalidData, relativeUri)); + return null; + + default: + return payload.Json; + } + } + private void OutputPayload(JsonElement json, string? sourceId) { + if (Envelope.IsPresent) + { + OutputItem(json, sourceId); + return; + } + var items = TryUnwrapCollection(json, out var truncated); if (items == null) { @@ -954,6 +1211,11 @@ private void OutputItem(JsonElement element, string? sourceId) WriteObject(ht); } + /// + /// Serialized request body for a write method, or null when no content should be sent. + /// Graph requires Content-Type: application/json on POST/PATCH/PUT even with an empty body, + /// so those default to "{}". DELETE sends no content. + /// private string? ResolveRequestBody(HttpMethod method) { var serialized = Body != null ? SerializeBody(Body) : null; @@ -963,6 +1225,153 @@ private void OutputItem(JsonElement element, string? sourceId) return method != HttpMethod.Delete ? "{}" : null; } + /// + /// Options for -Body serialization, chosen for what Graph accepts rather than STJ's + /// defaults: enums as camelCase names (Graph never takes them numerically), TimeSpan as + /// an Edm.Duration string, a Kind-less DateTime pinned to UTC (Graph rejects a bare + /// timestamp), and readable non-ASCII instead of \uXXXX - both forms decode the same, + /// but dead-letter files and -Debug traces are read by people. + /// + internal static readonly JsonSerializerOptions BodyJsonOptions = new() + { + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + Converters = + { + // Claims [Flags] enums only; every other enum falls through to the converter below. + new GraphFlagsEnumConverter(), + new JsonStringEnumConverter(JsonNamingPolicy.CamelCase), + new GraphDurationConverter(), + new GraphDateTimeConverter(), + }, + }; + + /// + /// OData spells a flags combination "ignoreCase,multiline"; JsonStringEnumConverter writes + /// ", " between the names, which Graph will not parse back into a flags-typed enum. + /// + /// Only that multi-name form is rewritten, and only its separator. Each name in it is + /// written by the converter STJ would otherwise have used, one member at a time, so a + /// combination spells its members exactly as they spell themselves alone. Resolving them + /// here from Enum.ToString did not: it picks the other name when two members share a value, + /// and it cannot see [JsonStringEnumMemberName], so a member sent as "read-only" alone went + /// out as "read" in a combination, which is not a name the service was ever given. + /// + /// Everything else - a single member, an alias, a combination with a name of its own, a + /// value with bits no member covers - is handed to that converter whole, so name resolution + /// and the numeric fallback stay byte-for-byte what they were. Formatting the number here + /// instead would carry the current culture's negative sign, which sv-SE writes as U+2212. + /// + /// Components are taken largest first, so a member covering several bits wins over the bits + /// themselves, and are written in ascending value order: {A=1,B=2,C=4,BC=6} at 7 is "a,bc". + /// + private sealed class GraphFlagsEnumConverter : JsonConverterFactory + { + public override bool CanConvert(Type typeToConvert) + => typeToConvert.IsEnum && typeToConvert.IsDefined(typeof(FlagsAttribute), inherit: false); + + public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options) + => (JsonConverter)Activator.CreateInstance( + typeof(FlagsConverter<>).MakeGenericType(typeToConvert), + new JsonStringEnumConverter(JsonNamingPolicy.CamelCase).CreateConverter(typeToConvert, options))!; + + private sealed class FlagsConverter : JsonConverter where T : struct, Enum + { + /// Every declared value with bits of its own, largest first. + private static readonly ulong[] s_members = Enum.GetValues() + .Select(ToBits).Where(bits => bits != 0).Distinct().OrderByDescending(bits => bits).ToArray(); + + private readonly JsonConverter _inner; + + public FlagsConverter(JsonConverter inner) => _inner = inner; + + public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + => _inner.Read(ref reader, typeToConvert, options); + + public override T ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + => _inner.ReadAsPropertyName(ref reader, typeToConvert, options); + + public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) + { + if (JoinNames(value, options) is { } joined) writer.WriteStringValue(joined); + else _inner.Write(writer, value, options); + } + + public override void WriteAsPropertyName(Utf8JsonWriter writer, T value, JsonSerializerOptions options) + { + if (JoinNames(value, options) is { } joined) writer.WritePropertyName(joined); + else _inner.WriteAsPropertyName(writer, value, options); + } + + /// The comma-joined form, or null when the value is not a multi-name one. + private string? JoinNames(T value, JsonSerializerOptions options) + { + var remaining = ToBits(value); + var names = new List(); + + foreach (var member in s_members) + { + if (remaining == 0) break; + if ((remaining & member) != member) continue; + if (NameOf(member, options) is not { } name) return null; + + // Chosen largest first, written smallest first. + names.Insert(0, name); + remaining &= ~member; + } + + // One name is not this converter's business - a lone member, an alias, a + // combination named in its own right, zero - and neither is a value with bits + // left over, which has no names at all and belongs in the numeric fallback. + return remaining == 0 && names.Count > 1 ? string.Join(",", names) : null; + } + + /// What the inner converter writes for one member alone, unquoted. + private string? NameOf(ulong member, JsonSerializerOptions options) + { + var buffer = new System.Buffers.ArrayBufferWriter(initialCapacity: 32); + using (var writer = new Utf8JsonWriter(buffer)) + _inner.Write(writer, (T)Enum.ToObject(typeof(T), member), options); + + var reader = new Utf8JsonReader(buffer.WrittenSpan); + return reader.Read() && reader.TokenType == JsonTokenType.String ? reader.GetString() : null; + } + + /// The value's bits, sign-extended rather than refused for a signed enum. + private static ulong ToBits(T value) => ((IConvertible)value).GetTypeCode() switch + { + TypeCode.Byte or TypeCode.UInt16 or TypeCode.UInt32 or TypeCode.UInt64 + => ((IConvertible)value).ToUInt64(provider: null), + _ => unchecked((ulong)((IConvertible)value).ToInt64(provider: null)), + }; + } + } + + /// Edm.Duration is ISO-8601 ("PT1H"); STJ's default "01:00:00" is refused. + private sealed class GraphDurationConverter : JsonConverter + { + public override TimeSpan Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + => System.Xml.XmlConvert.ToTimeSpan(reader.GetString()!); + + public override void Write(Utf8JsonWriter writer, TimeSpan value, JsonSerializerOptions options) + => writer.WriteStringValue(System.Xml.XmlConvert.ToString(value)); + } + + /// + /// A DateTime with Kind=Unspecified would serialize with no offset, which Graph rejects. + /// Assume UTC - the read side already resolves timestamps to UtcDateTime, so a value that + /// round-trips through mgx keeps its meaning. + /// + private sealed class GraphDateTimeConverter : JsonConverter + { + public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + => reader.GetDateTime(); + + public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options) + => writer.WriteStringValue(value.Kind == DateTimeKind.Unspecified + ? DateTime.SpecifyKind(value, DateTimeKind.Utc) + : value); + } + /// /// Serialize a -Body argument to the JSON that goes on the wire. A raw JSON string is passed /// through verbatim, everything else is serialized. @@ -970,22 +1379,29 @@ private void OutputItem(JsonElement element, string? sourceId) internal static string SerializeBody(object body) { var value = UnwrapPSObject(body); + RefuseUnserializable(value, "-Body"); if (value is string s) return s; // Still a PSObject after unwrapping: a PSCustomObject, whose members live on the wrapper - if (value is PSObject pso) return JsonSerializer.Serialize(PSOToDict(pso)); - if (value is IDictionary dict) return JsonSerializer.Serialize(DictionaryToDict(dict)); + if (value is PSObject pso) return JsonSerializer.Serialize(PSOToDict(pso), BodyJsonOptions); + if (value is IDictionary dict) return JsonSerializer.Serialize(DictionaryToDict(dict), BodyJsonOptions); // Handle array body (object[], ArrayList, List, ... from PowerShell) - if (value is IEnumerable seq) return JsonSerializer.Serialize(EnumerableToArray(seq)); - return JsonSerializer.Serialize(value); + if (value is IEnumerable seq and not byte[]) return JsonSerializer.Serialize(EnumerableToArray(seq), BodyJsonOptions); + return JsonSerializer.Serialize(value, BodyJsonOptions); } - internal static Dictionary PSOToDict(PSObject pso) + internal static Dictionary PSOToDict(PSObject pso, string path = "-Body") { + // Every property kind, matching what ConvertTo-Json reads - so -Body $obj and + // -Body ($obj | ConvertTo-Json) produce the same members. ScriptProperties are + // evaluated; one whose getter throws serializes as null rather than failing the + // whole body. var dict = new Dictionary(); foreach (var prop in pso.Properties) { - if (prop.MemberType == PSMemberTypes.NoteProperty) - dict[prop.Name] = UnwrapValue(prop.Value); + object? raw; + try { raw = prop.Value; } + catch (GetValueException) { raw = null; } + dict[prop.Name] = UnwrapValue(raw, $"{path}.{prop.Name}"); } return dict; } @@ -994,11 +1410,11 @@ internal static string SerializeBody(object body) /// Flatten any IDictionary (Hashtable or ordered dictionary) into a serializable /// dictionary, unwrapping nested PowerShell values. /// - internal static Dictionary DictionaryToDict(IDictionary source) + internal static Dictionary DictionaryToDict(IDictionary source, string path = "-Body") { var dict = new Dictionary(); foreach (DictionaryEntry entry in source) - dict[entry.Key.ToString()!] = UnwrapValue(entry.Value); + dict[entry.Key.ToString()!] = UnwrapValue(entry.Value, $"{path}.{entry.Key}"); return dict; } @@ -1007,22 +1423,66 @@ internal static string SerializeBody(object body) /// A PSCustomObject keeps its members on the PSObject (its BaseObject is an empty /// marker), so it must be read through PSOToDict rather than its BaseObject. /// - internal static object? UnwrapValue(object? value) + private const int MaxBodyDepth = 64; + + internal static object? UnwrapValue(object? value, string path = "-Body") { + // Depth from the path: each nesting level appends a segment. A self-referencing + // hashtable ($h.self = $h) recursed to a StackOverflowException, which no catch + // can stop - the process died. 64 levels is far beyond any real Graph body. + if (path.Length > MaxBodyDepth * 8) + { + var depth = path.Count(c => c is '.' or '['); + if (depth > MaxBodyDepth) + throw new ArgumentException( + $"The value at '{path[..64]}...' nests deeper than {MaxBodyDepth} levels. " + + "Is the body self-referencing?"); + } if (value is PSObject pso) { var unwrapped = UnwrapPSObject(pso); - return ReferenceEquals(unwrapped, pso) ? PSOToDict(pso) : UnwrapValue(unwrapped); + return ReferenceEquals(unwrapped, pso) ? PSOToDict(pso, path) : UnwrapValue(unwrapped, path); } + RefuseUnserializable(value, path); if (value is IDictionary dict) - return DictionaryToDict(dict); - if (value is IEnumerable seq and not string) - return EnumerableToArray(seq); + return DictionaryToDict(dict, path); + // byte[] is Edm.Binary and serializes as base64; flattening it into object?[] + // would emit a JSON array of integers instead. + if (value is IEnumerable seq and not (string or byte[])) + return EnumerableToArray(seq, path); return value; } - private static object?[] EnumerableToArray(IEnumerable source) => - source.Cast().Select(UnwrapValue).ToArray(); + /// + /// Values that would serialize to something other than what they mean. A SecureString + /// reflects to {"Length":8} - the request "succeeds" carrying garbage instead of the + /// secret, or worse, would carry the secret if it round-tripped. NaN and Infinity have + /// no JSON representation, and STJ's own refusal does not say which property. + /// + private static void RefuseUnserializable(object? value, string path) + { + switch (value) + { + case System.Security.SecureString: + case PSCredential: + case ScriptBlock: + case System.Security.Cryptography.X509Certificates.X509Certificate: + throw new ArgumentException( + $"The value at '{path}' is a {value.GetType().Name}, which does not serialize " + + "to JSON meaningfully. Convert it to what the endpoint expects before sending."); + case double d when double.IsNaN(d) || double.IsInfinity(d): + throw new ArgumentException($"The value at '{path}' is {d}; JSON has no representation for it."); + case float f when float.IsNaN(f) || float.IsInfinity(f): + throw new ArgumentException($"The value at '{path}' is {f}; JSON has no representation for it."); + } + } + + /// + /// Flatten any non-string sequence (object[], ArrayList, List<object>, ...) into an array of + /// unwrapped values. + /// + private static object?[] EnumerableToArray(IEnumerable source, string path = "-Body") => + source.Cast().Select((v, i) => UnwrapValue(v, $"{path}[{i}]")).ToArray(); #endregion diff --git a/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj b/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj index d0da489..d6030ff 100644 --- a/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj +++ b/src/Mgx.Cmdlets/Mgx.Cmdlets.csproj @@ -1,4 +1,4 @@ - + net10.0 @@ -6,6 +6,10 @@ enable Mgx.Cmdlets true + false @@ -18,25 +22,45 @@ - - + + + + + - - - $(MSBuildThisFileDirectory)..\..\Modules\M365DSC.mgx - $(ModuleRoot)\Dependencies + $(MSBuildThisFileDirectory)../../Modules/M365DSC.mgx + $(ModuleRoot)/Dependencies @@ -48,7 +72,7 @@ - + diff --git a/src/Mgx.Engine/Errors/MgxErrorClass.cs b/src/Mgx.Engine/Errors/MgxErrorClass.cs new file mode 100644 index 0000000..64a22a6 --- /dev/null +++ b/src/Mgx.Engine/Errors/MgxErrorClass.cs @@ -0,0 +1,48 @@ +namespace Mgx.Engine.Errors; + +/// +/// What went wrong, as one vocabulary shared by retry, circuit breaking, pacing, +/// telemetry, and the errors cmdlets surface. Classification happens in +/// ; what to do about a class happens in +/// . Nothing else in the codebase decides either question. +/// +public enum MgxErrorClass +{ + /// 429: the service is pacing this workload. Retryable for every method. + Throttle, + + /// 500/502/503/504: the service failed and may not have processed the request. + TransientServer, + + /// + /// The request may never have arrived: a connection failure, a timeout (408 or a + /// client-side attempt timeout), a reset mid-body. + /// + TransientTransport, + + /// 401: the token is missing, expired, or not valid here. + Authentication, + + /// 403: authenticated, but not permitted. + Authorization, + + /// + /// Declared for the read-after-write window (a 404 immediately following a successful + /// write). Nothing produces it yet; the consistency policy that will is future work, + /// and telemetry and category mappings are complete for it now so adding the producer + /// does not change any schema. + /// + Consistency, + + /// 400 and kin: the request is malformed and will fail identically every time. + InvalidRequest, + + /// 404: the path is valid and the object is not there. + NotFound, + + /// 409/412: the object changed underneath the request. + Conflict, + + /// Everything else, including cancellation: retrying cannot help. + Permanent, +} diff --git a/src/Mgx.Engine/Errors/MgxErrorClassifier.cs b/src/Mgx.Engine/Errors/MgxErrorClassifier.cs new file mode 100644 index 0000000..2bc018f --- /dev/null +++ b/src/Mgx.Engine/Errors/MgxErrorClassifier.cs @@ -0,0 +1,70 @@ +using System.Net.Http.Headers; +using Polly.CircuitBreaker; +using Polly.Timeout; + +namespace Mgx.Engine.Errors; + +/// +/// The one place a status code or exception becomes an . +/// Before this existed the same decision lived in the retry predicate, the circuit-breaker +/// predicate, the batch item loop, the content download pipeline, and the pacer - each with +/// its own list, and the lists disagreed in ways none of them documented. +/// +public static class MgxErrorClassifier +{ + /// Classify an HTTP status. For non-success codes; a 2xx/3xx is not a failure + /// and lands in only because nothing retries it. + public static MgxErrorInfo Classify(int statusCode, string? errorCode = null, TimeSpan? serverRetryAfter = null) + { + var cls = statusCode switch + { + 429 => MgxErrorClass.Throttle, + 500 or 502 or 503 or 504 => MgxErrorClass.TransientServer, + // A 408 is a timeout the request may not have survived - transport, not server: + // the server did not fail, it never saw the whole request. + 408 => MgxErrorClass.TransientTransport, + 401 => MgxErrorClass.Authentication, + 403 => MgxErrorClass.Authorization, + 404 => MgxErrorClass.NotFound, + 409 or 412 => MgxErrorClass.Conflict, + 400 or 405 or 411 or 413 or 414 or 415 or 416 or 422 or 501 => MgxErrorClass.InvalidRequest, + // Unknown 5xx (505, 507, ...) deliberately Permanent: mgx has never retried + // them, and classifying them TransientServer would silently start. + _ => MgxErrorClass.Permanent, + }; + return new MgxErrorInfo(cls, statusCode, errorCode, serverRetryAfter); + } + + /// Classify a response. Reads status and Retry-After; never the body - + /// retry decisions run at ResponseHeadersRead, so the body is not available. + public static MgxErrorInfo Classify(HttpResponseMessage response, string? errorCode = null) + => Classify((int)response.StatusCode, errorCode, + RetryAfterPolicy.ServerRequested(response.Headers.RetryAfter)); + + /// + /// Classify an exception. is the caller's own + /// token state: a cancellation-shaped exception after the caller cancelled is + /// Permanent - nothing may act on it - while the same exception without a cancelled + /// token is a timeout, which is transport. + /// + public static MgxErrorInfo Classify(Exception exception, bool cancellationRequested) + { + if (cancellationRequested && exception + is OperationCanceledException or TimeoutRejectedException) + return new MgxErrorInfo(MgxErrorClass.Permanent, 0); + + var cls = exception switch + { + HttpRequestException => MgxErrorClass.TransientTransport, + System.Net.Sockets.SocketException => MgxErrorClass.TransientTransport, + IOException => MgxErrorClass.TransientTransport, + // TaskCanceledException without user cancellation is HttpClient's own timeout. + OperationCanceledException => MgxErrorClass.TransientTransport, + TimeoutRejectedException => MgxErrorClass.TransientTransport, + // The breaker is open; retrying into it is the thing it exists to stop. + BrokenCircuitException => MgxErrorClass.Permanent, + _ => MgxErrorClass.Permanent, + }; + return new MgxErrorInfo(cls, 0); + } +} diff --git a/src/Mgx.Engine/Errors/MgxErrorInfo.cs b/src/Mgx.Engine/Errors/MgxErrorInfo.cs new file mode 100644 index 0000000..8c698b0 --- /dev/null +++ b/src/Mgx.Engine/Errors/MgxErrorInfo.cs @@ -0,0 +1,16 @@ +namespace Mgx.Engine.Errors; + +/// +/// The classified facts of a failure. Facts only - what to do with them +/// (retry, count against the breaker, surface) belongs to . +/// +/// The failure class. +/// HTTP status, or 0 when there was no response (transport, timeout, circuit). +/// Graph's error.code, when the body has been read. Retry-time +/// classification runs at ResponseHeadersRead and never has it. +/// The server's Retry-After, unclamped. Policy clamps. +public readonly record struct MgxErrorInfo( + MgxErrorClass Class, + int StatusCode, + string? ErrorCode = null, + TimeSpan? ServerRetryAfter = null); diff --git a/src/Mgx.Engine/Errors/MgxErrorPolicy.cs b/src/Mgx.Engine/Errors/MgxErrorPolicy.cs new file mode 100644 index 0000000..506874b --- /dev/null +++ b/src/Mgx.Engine/Errors/MgxErrorPolicy.cs @@ -0,0 +1,42 @@ +namespace Mgx.Engine.Errors; + +/// +/// What to do about a classified failure. Retry and circuit breaking legitimately +/// disagree - that disagreement lives here, in one visible place, instead of being +/// implied by two lists a hundred lines apart. +/// +public static class MgxErrorPolicy +{ + /// + /// Whether another attempt may help. Throttles retry for every method including POST + /// (matches the Kiota SDK); server and transport failures retry only when the method is + /// idempotent, because a 5xx or a dead connection may mean a write was already applied. + /// + public static bool ShouldRetry(MgxErrorClass cls, bool isIdempotent) => cls switch + { + MgxErrorClass.Throttle => true, + MgxErrorClass.TransientServer or MgxErrorClass.TransientTransport => isIdempotent, + _ => false, + }; + + /// + /// Whether the failure counts toward opening the circuit breaker. + /// 429 and 408 are excluded deliberately: 429 is the service pacing us, not failing - + /// Retry-After handles it, and counting it would open the circuit exactly when the + /// correct response is to slow down and keep going. 408 is a client-perceived timeout, + /// not a server-side failure indicator. + /// + public static bool CountsAsCircuitFailure(in MgxErrorInfo info) => + info.Class == MgxErrorClass.TransientServer + || (info.Class == MgxErrorClass.TransientTransport && info.StatusCode != 408); + + /// + /// The download-host retry filter, narrower than on purpose: + /// no 408 and no cancellation-shaped timeouts, because the content path handles stalls + /// with its own idle timeout and rebuilds hop 1 on an expired URL. Throttles, 5xx, and + /// connection failures retry; everything else surfaces. + /// + public static bool ShouldRetryDownload(in MgxErrorInfo info, Exception? exception) => + info.Class is MgxErrorClass.Throttle or MgxErrorClass.TransientServer + || exception is HttpRequestException; +} diff --git a/src/Mgx.Engine/Errors/RetryAfterPolicy.cs b/src/Mgx.Engine/Errors/RetryAfterPolicy.cs new file mode 100644 index 0000000..1ef12b8 --- /dev/null +++ b/src/Mgx.Engine/Errors/RetryAfterPolicy.cs @@ -0,0 +1,57 @@ +using System.Net.Http.Headers; + +namespace Mgx.Engine.Errors; + +/// +/// One reading of Retry-After. The header has two forms - delta-seconds and an HTTP date - +/// and before this existed the pipeline, the batch client, the content client, and the +/// pacer each parsed it themselves, two of them with their own caps. +/// +public static class RetryAfterPolicy +{ + /// The server's requested delay, unclamped: delta-seconds verbatim, or + /// date minus now (which may be negative for a past date). Null when absent. + public static TimeSpan? ServerRequested(RetryConditionHeaderValue? retryAfter) + => retryAfter?.Delta + ?? (retryAfter?.Date is { } date ? date - DateTimeOffset.UtcNow : null); + + /// + /// The delay to actually sleep: the server's request clamped to , + /// or null when the header is absent or asks for the past - callers fall back to their + /// exponential backoff. A delta is honored even at zero; a date is honored only in the + /// future. This is exactly what the pipeline and the content client each did alone. + /// + public static TimeSpan? Resolve(RetryConditionHeaderValue? retryAfter, TimeSpan cap) + { + if (retryAfter?.Delta is { } delta) + return delta > cap ? cap : delta; + if (retryAfter?.Date is { } date) + { + var delay = date - DateTimeOffset.UtcNow; + if (delay > TimeSpan.Zero) + return delay > cap ? cap : delay; + } + return null; + } + + /// + /// The batch form: whole seconds from a raw header string, rounded up, floored at zero, + /// clamped to . carries + /// what the server asked for so the caller can say when it clamped. + /// + public static bool TryResolveSeconds(string? headerValue, int capSeconds, + out int requestedSeconds, out int clampedSeconds) + { + requestedSeconds = 0; + clampedSeconds = 0; + if (headerValue == null || !RetryConditionHeaderValue.TryParse(headerValue, out var ra)) + return false; + + var delay = ra.Delta ?? (ra.Date.HasValue ? ra.Date.Value - DateTimeOffset.UtcNow : (TimeSpan?)null); + if (!delay.HasValue) return false; + + requestedSeconds = (int)Math.Ceiling(delay.Value.TotalSeconds); + clampedSeconds = Math.Min(Math.Max(requestedSeconds, 0), capSeconds); + return true; + } +} diff --git a/src/Mgx.Engine/Http/AdaptiveRequestPacer.cs b/src/Mgx.Engine/Http/AdaptiveRequestPacer.cs index 3a5b9bd..0f97795 100644 --- a/src/Mgx.Engine/Http/AdaptiveRequestPacer.cs +++ b/src/Mgx.Engine/Http/AdaptiveRequestPacer.cs @@ -1,3 +1,4 @@ +using Mgx.Engine.Errors; using System.Diagnostics; using System.Globalization; @@ -328,12 +329,9 @@ internal static void RecordResponse(WorkloadBucket bucket, HttpResponseMessage r Volatile.Write(ref s_lastPercentageTicks[b], Stopwatch.GetTimestamp()); } - if ((int)response.StatusCode == 429) - { - TimeSpan? retryAfter = response.Headers.RetryAfter?.Delta - ?? (response.Headers.RetryAfter?.Date is { } d ? d - DateTimeOffset.UtcNow : null); - RecordThrottle(bucket, retryAfter); - } + var info = MgxErrorClassifier.Classify(response); + if (info.Class == MgxErrorClass.Throttle) + RecordThrottle(bucket, info.ServerRetryAfter); } /// @@ -404,7 +402,9 @@ internal static double LastThrottlePercentage if (s_latencyBaselineMs[i] > 0) { var ratio = (double)s_lastLatencyMs[i] / s_latencyBaselineMs[i]; - (facts ??= []).Add($"latency {s_lastLatencyMs[i]}ms ({ratio:0.0}x of {s_latencyBaselineMs[i]}ms baseline)"); + // Parsed by the benchmark harness, so the separator must not follow the ambient culture. + var ratioText = ratio.ToString("0.0", CultureInfo.InvariantCulture); + (facts ??= []).Add($"latency {s_lastLatencyMs[i]}ms ({ratioText}x of {s_latencyBaselineMs[i]}ms baseline)"); } } if (facts != null) diff --git a/src/Mgx.Engine/Http/GraphBatchClient.cs b/src/Mgx.Engine/Http/GraphBatchClient.cs index 313c81e..0de62cb 100644 --- a/src/Mgx.Engine/Http/GraphBatchClient.cs +++ b/src/Mgx.Engine/Http/GraphBatchClient.cs @@ -1,3 +1,4 @@ +using Mgx.Engine.Errors; using System.Collections.Concurrent; using System.Diagnostics; using System.Text; @@ -9,7 +10,7 @@ namespace Mgx.Engine.Http; /// /// Sends batched requests to Graph /$batch endpoint (up to 20 per call). /// Uses a two-layer retry design: Polly (via ResilientGraphClient) handles -/// transport-level retries on the outer $batch POST. This class handles +/// transport-level retries on the outer $batch POST; this class handles /// per-item retries within the 200-OK batch response body (429, 5xx for /// idempotent methods). Graph always returns HTTP 200 for $batch, so Polly /// never sees per-item errors. @@ -51,20 +52,41 @@ public sealed class GraphBatchClient /// public Dictionary? ItemHeaders { get; set; } + /// + /// Runs once a parallel run has stopped sending: a chunk's own POST failed, the failure is + /// recorded and the stop its siblings read is canceled. Null on every shipping path: the + /// suite holds a wire answer here so that what a sibling does next happens on the far side + /// of the stop, rather than a moment either way of it. Per client rather than static, so a + /// test arms only the run it built. + /// + internal Action? SendingStopped { get; set; } + + /// + /// Runs as a chunk enters the wait between two of its own attempts, with the stop already + /// linked into that wait. Null on every shipping path: the suite answers a peer chunk here + /// so a refusal lands on a backoff that is running, which is the only state the question of + /// whether the stop ends it can be asked from. Per client rather than static, so a test arms + /// only the run it built. + /// + internal Action? RetryWaitStarted { get; set; } + private readonly int _batchChunkConcurrency; private readonly int _batchItemsPerSecond; - // When the last batch completed and how many items it carried, so successive small-batch - // calls are paced and not only the chunks inside one call + // Cross-call pacing state: tracks when the last batch completed and how many items + // it processed so that successive small-batch calls (e.g., 20 items = 1 chunk) still + // get paced. Without this, pacing only fires between chunks within a single call. private static long s_lastBatchCompletedTicks; private static int s_lastBatchItemCount; - // The adapted rate persists across calls, so a later call starts reduced rather than back - // at the configured BatchItemsPerSecond + // Cross-call adaptive rate: when 429s trigger rate halving, persist the adapted rate + // across Invoke-MgxBatchRequest calls so subsequent calls start at the reduced rate + // instead of resetting to the configured BatchItemsPerSecond. private static int s_adaptedItemsPerSecond; - // When the last 429 was seen, so the adapted rate can expire rather than holding every - // later batch down for the life of the session + // When the last 429 was seen, so the adapted rate can expire. Without an expiry a single + // throttling episode kept every later batch in the process at the reduced rate for the + // lifetime of the session, with no way to recover short of restarting PowerShell. private static long s_lastThrottleTicks; // Chunk-loop AIMD tuning. The shared math (halve on throttle, additive recovery, expiry @@ -107,7 +129,7 @@ public async Task> ExecuteBatchAsync( var operations = urls.Select(u => new BatchOperation(u)).ToList(); var result = await ExecuteBatchIndexedAsync(operations, cancellationToken); - // URL-keyed for the legacy overload + // Convert to URL-keyed dictionary (backward compatible) var results = new Dictionary(result.Results.Count); foreach (var (op, response) in result.Results) results[op.Url] = response; @@ -115,16 +137,138 @@ public async Task> ExecuteBatchAsync( } /// - /// Execute multiple requests of any HTTP method as batches, chunked to the Graph limit of 20 - /// and allowing duplicate URLs. Returns per-item results and operational telemetry. Items - /// that exhausted per-chunk retries on a retryable status get one follow-up batch. + /// Execute multiple requests (any HTTP method) as batches. + /// Returns BatchExecutionResult with per-item results and operational telemetry. + /// Auto-chunks into 20-request batches per Graph API limit. + /// Supports duplicate URLs (e.g., multiple POSTs to /users). + /// After all chunks complete, items that exhausted per-chunk retries but have + /// retryable status codes are retried once more as a follow-up batch. /// /// - /// Status recorded for an operation never sent because an earlier chunk failed. Distinct from - /// any HTTP status, so a caller can tell a write that may have landed from one that did not. + /// Status recorded for an operation that was never sent, because another chunk failed + /// first - not necessarily one before it, since chunks can run in parallel. Distinct from + /// any HTTP status so a caller can tell "not attempted" from "attempted and refused" - the + /// difference between a write that may have landed and one that certainly did not. /// public const int NotSentStatus = 0; + /// + /// Status recorded for the items of a chunk whose own POST failed without a status of its + /// own. Two refusals land on it and it cannot separate them: a POST that went out and whose + /// answer never completed - a stalled response body, a transport failure mid-flight - and a + /// POST that never left, refused by an open circuit or by a rate limiter with no permit for + /// it. It is read as the first, because that is the reading a caller can act on safely: the + /// writes may have been applied, so resending them is a decision about duplicates. + /// NotSentStatus is kept for the refusals nothing went out for and the code knows it - a + /// chunk the run's stop reached before its first attempt, and the chunks never POSTed at + /// all - where claiming the opposite would be the more expensive error. + /// + private const int RefusedWithoutStatus = 503; + + /// + /// What a chunk spent on its way to whatever happened to it. Held by the caller of the + /// attempt loop rather than returned from it, so a chunk that fails still reports what it + /// cost - the throttles it met are the reason it failed, not a detail lost with it. + /// + private sealed class ChunkAttemptCounts + { + public int ItemRetries; + public int ThrottleEncounters; + public long RetryDelayMs; + } + + /// + /// A chunk's own $batch call failed. Carries the per-item results the earlier attempts of + /// that same chunk already collected - the server answered those items, and after the + /// failure they are the only record of what it applied - and what those attempts cost. + /// + private sealed class ChunkSendFailedException : Exception + { + public ChunkSendFailedException( + Exception cause, + (BatchOperation Operation, GraphBatchResponseItem Response)?[] partial, + int refusedStatus, + ChunkAttemptCounts counts) + : base(cause.Message, cause) + { + Cause = cause; + Partial = partial; + Counts = counts; + // Below 400 is not a status a refused item may carry: a success is counted in + // [200,400) and the dead-letter file keeps nothing under 400, so a 304 from a proxy + // or a captive portal reported twenty refused writes as twenty applied ones. Graph + // answers $batch with 200 or an error; anything else came from something between. + RefusedStatus = refusedStatus >= 400 ? refusedStatus : RefusedWithoutStatus; + } + + /// The failure itself, as the caller is to see it. + public Exception Cause { get; } + + /// By chunk-relative index; null where no sub-response arrived. + public (BatchOperation Operation, GraphBatchResponseItem Response)?[] Partial { get; } + + /// Status for the items of this chunk that have no sub-response. + public int RefusedStatus { get; } + + /// The retries, throttles and delay this chunk had spent when it failed. + public ChunkAttemptCounts Counts { get; } + } + + /// + /// Writes a refused chunk's own slots: an item the server answered keeps that answer, the + /// rest take the refusal status, covering both a POST that went out unanswered and one an + /// open circuit or a rate limiter stopped before it left - read as the first, so none of + /// them may be reported as never sent. + /// + private static void FillRefusedChunk( + (BatchOperation Operation, GraphBatchResponseItem Response)?[] results, + BatchOperation[] chunk, int offset, ChunkSendFailedException failure) + { + for (int i = 0; i < chunk.Length; i++) + { + var answered = i < failure.Partial.Length ? failure.Partial[i] : null; + results[offset + i] = answered ?? (chunk[i], new GraphBatchResponseItem + { + Id = (offset + i).ToString(), + Status = failure.RefusedStatus + }); + } + } + + /// + /// Writes back what the batch-level retry pass got answers for before its own POST was + /// refused. Every item the pass carries already holds a status the first pass gave it, so + /// unlike a refused first-pass chunk there is nothing to fill in for the rest: an item this + /// pass got no answer of its own for keeps the one it came in with. Left out, an item the + /// pass confirmed is reported with the failure that sent it here - for a write the server + /// applied, that is an instruction to apply it a second time. + /// + private static void FillAnsweredRetryChunk( + (BatchOperation Operation, GraphBatchResponseItem Response)?[] results, + IReadOnlyList<(int OriginalIndex, BatchOperation Op)> candidates, int offset, + ChunkSendFailedException failure) + { + for (int i = 0; i < failure.Partial.Length; i++) + { + var answered = failure.Partial[i]; + if (answered != null) + results[candidates[offset + i].OriginalIndex] = answered; + } + } + + /// + /// A failed chunk's retries, throttles and delay. They are the run's, not the chunk's: the + /// chunk that met the throttles is often the one that then failed, and reading them only + /// off the chunks that succeeded reported a batch throttled into refusal as an untroubled + /// one that happened to end badly. + /// + private static void RecordChunkCosts(BatchTelemetry telemetry, ChunkSendFailedException failure) + { + telemetry.AddItemRetries(failure.Counts.ItemRetries); + telemetry.AddThrottleEncounters(failure.Counts.ThrottleEncounters); + telemetry.AddRetryDelayMs(failure.Counts.RetryDelayMs); + } + public async Task ExecuteBatchIndexedAsync( IReadOnlyList operations, CancellationToken cancellationToken = default) @@ -142,9 +286,10 @@ public async Task ExecuteBatchIndexedAsync( Exception? chunkFailure = null; var notSent = new List(); - // A recent write batch delays this one to hold target throughput. Writes only, since a - // GET batch does not hit the write throttle. Capped to one chunk worth, to smooth the - // gap between calls rather than re-pace the whole + // Cross-call pacing: if a previous write batch completed recently, delay to maintain + // target throughput. Only applies to batches containing writes (POST/PATCH/DELETE) - + // GET-only batches don't hit Graph's write throttle. Delay is capped to one chunk's + // worth (MaxBatchSize items) to smooth the gap between calls, not re-pace the whole // previous batch. var hasWrites = HasWriteOperations(operations); if (_batchItemsPerSecond > 0 && hasWrites) @@ -165,17 +310,20 @@ public async Task ExecuteBatchIndexedAsync( } } + // Process in chunks of MaxBatchSize var chunks = operations.Chunk(MaxBatchSize).ToArray(); if (_batchChunkConcurrency <= 1) { - // Sequential mode applies backpressure between chunks + // Sequential mode (default): cross-chunk backpressure delays between chunks int globalOffset = 0; int crossChunkDelaySeconds = 0; long prevChunkElapsedMs = 0; int chunkIndex = 0; - // Starts at the persisted adapted rate, or the configured one, and halves on each - // 429. Persisted so a loop of calls does not reset to full rate and re-throttle + // Adaptive pacing: starts at the persisted adapted rate (if any), otherwise + // the configured rate. Halves on each 429 encounter. Persisted across calls + // so that successive Invoke-MgxBatchRequest invocations in a loop don't reset + // to the full rate and immediately get re-throttled. var adapted = Volatile.Read(ref s_adaptedItemsPerSecond); if (adapted > 0 && AdaptivePacing.AdaptedRateHasExpired( Interlocked.Read(ref s_lastThrottleTicks), Stopwatch.GetTimestamp())) @@ -205,7 +353,7 @@ public async Task ExecuteBatchIndexedAsync( } else if (globalOffset > 0 && effectiveItemsPerSecond > 0 && hasWrites) { - // Smooths write throughput to avoid burst and stall + // Inter-chunk pacing: smooth write throughput to avoid burst-and-stall. var targetMs = (int)(chunk.Length / (double)effectiveItemsPerSecond * 1000); var pacingMs = targetMs - (int)prevChunkElapsedMs; if (pacingMs > 0) @@ -227,20 +375,23 @@ public async Task ExecuteBatchIndexedAsync( (chunkResults, throttleDelay, chunkRetries, chunkThrottles, chunkRetryDelayMs) = await SendBatchWithRetryAsync(chunk, cancellationToken); } - // Only an HTTP-level failure of the chunk itself. A malformed envelope is a - // protocol violation rather than a chunk that did not land, and still throws - catch (GraphServiceException ex) + // Any way this chunk's own call can fail, an envelope that answers nothing that + // was sent included. Each of them is a failure of this chunk and is reported as + // one; none of them is a reason to lose the chunks applied before it. + catch (ChunkSendFailedException ex) { // The chunks before this one were applied on the server. Letting this // exception out discards their results, which is the only record of what // landed - and for writes that is the thing the caller most needs. Stop // here, keep them, and hand the failure back alongside. - // - // The unsent slots are filled rather than dropped: callers line results up - // against their own input list by position, so a short list would misattribute - // every error after the gap. - chunkFailure = ex; - for (int i = globalOffset; i < operations.Count; i++) + chunkFailure = ex.Cause; + RecordChunkCosts(telemetry, ex); + FillRefusedChunk(results, chunk, globalOffset, ex); + + // Only the chunks after this one were never sent. The unsent slots are filled + // rather than dropped: callers line results up against their own input list + // by position, so a short list would misattribute every error after the gap. + for (int i = globalOffset + chunk.Length; i < operations.Count; i++) { notSent.Add(operations[i]); results[i] = (operations[i], new GraphBatchResponseItem @@ -298,7 +449,7 @@ public async Task ExecuteBatchIndexedAsync( else { // Parallel mode: SemaphoreSlim-bounded concurrent chunk execution - // No cross-chunk backpressure. Chunks run independently + // No cross-chunk backpressure; chunks run independently var chunkOffsets = new int[chunks.Length]; int runningOffset = 0; for (int j = 0; j < chunks.Length; j++) @@ -308,16 +459,30 @@ public async Task ExecuteBatchIndexedAsync( } using var semaphore = new SemaphoreSlim(_batchChunkConcurrency); + Exception? parallelFailure = null; + // Cancelled when a chunk's own POST fails. The chunks still running read it to stop + // before their next attempt, and wait on it alongside their backoff - a chunk part + // way through a Retry-After has stopped sending, and holding the call open for the + // rest of a delay it will not act on serves nobody. + using var stopSource = new CancellationTokenSource(); var tasks = chunks.Select(async (chunk, chunkIndex) => { bool acquired = false; try { + // A chunk has already failed its own POST. The sequential path stops there + // instead of sending the rest, and so does this one: what is in flight + // finishes and is kept, nothing further goes out. Read before queueing as + // well as after acquiring - a chunk that waited out the queue must not send + // on the strength of a check it made before the refusal happened. + if (Volatile.Read(ref parallelFailure) != null) return; await semaphore.WaitAsync(cancellationToken); acquired = true; + if (Volatile.Read(ref parallelFailure) != null) return; var (chunkResults, _, chunkRetries, chunkThrottles, chunkRetryDelayMs) = - await SendBatchWithRetryAsync(chunk, cancellationToken); + await SendBatchWithRetryAsync(chunk, cancellationToken, + stopRequested: stopSource.Token); telemetry.AddItemRetries(chunkRetries); telemetry.AddThrottleEncounters(chunkThrottles); telemetry.AddRetryDelayMs(chunkRetryDelayMs); @@ -326,6 +491,22 @@ public async Task ExecuteBatchIndexedAsync( for (int i = 0; i < chunkResults.Count; i++) results[offset + i] = chunkResults[i]; } + // A failure of the chunk's own call, as in the sequential branch. Task.WhenAll + // would rethrow it and take the populated results array with it - the results of + // the chunks that DID land, which for writes is the record the caller most needs. + catch (ChunkSendFailedException ex) + { + // Chunks fail concurrently, and only the first can be the one failure the + // caller is handed. The others are still things that happened to the run, + // and dropping them leaves a chunk whose items all failed with no statement + // anywhere of why. + if (Interlocked.CompareExchange(ref parallelFailure, ex.Cause, null) != null) + _pendingVerbose.Enqueue($"Batch chunk {chunkIndex} also failed: {ex.Cause.Message}"); + stopSource.Cancel(); + SendingStopped?.Invoke(); + RecordChunkCosts(telemetry, ex); + FillRefusedChunk(results, chunk, chunkOffsets[chunkIndex], ex); + } finally { if (acquired) semaphore.Release(); @@ -333,25 +514,52 @@ public async Task ExecuteBatchIndexedAsync( }).ToArray(); await Task.WhenAll(tasks); + + if (parallelFailure != null) + { + chunkFailure = parallelFailure; + // A failed chunk filled its own slots above, so every slot still empty belongs + // to a chunk that was never sent. Filled rather than dropped, for the same reason + // as above: callers line results up against their own input list by position. + for (int i = 0; i < results.Length; i++) + { + if (results[i] != null) continue; + notSent.Add(operations[i]); + results[i] = (operations[i], new GraphBatchResponseItem + { + Id = i.ToString(), + Status = NotSentStatus + }); + } + } } // Batch-level retry for items that exhausted per-chunk retries // but have retryable status codes. Throttle pressure may have subsided. + // + // Not after a chunk failure. The run stopped sending and this pass is a send, so it is + // skipped for every candidate rather than for the refused chunk's own. For a candidate + // a refusal carried, resending is the duplicate write the stop exists to prevent; for + // one a chunk that was POSTed and answered in full left failing, it is an extra attempt + // the run no longer has. Both are handed back as they stand, for the caller to resubmit + // knowing what it is resubmitting. var failedRetryable = new List<(int OriginalIndex, BatchOperation Op)>(); - for (int i = 0; i < results.Length; i++) + if (chunkFailure is null) { - if (results[i] == null) continue; - var (op, response) = results[i]!.Value; - if (IsRetryable(response.Status, op.Method)) - failedRetryable.Add((i, op)); + for (int i = 0; i < results.Length; i++) + { + if (results[i] == null) continue; + var (op, response) = results[i]!.Value; + if (IsRetryable(response.Status, op.Method)) + failedRetryable.Add((i, op)); + } } if (failedRetryable.Count > 0) { - telemetry.BatchLevelRetries = failedRetryable.Count; - - // A pause before the batch-level retry pass lets throttle pressure subside - var backpressureDelay = Math.Min(2, _maxRetryAfterSeconds); + // Backpressure delay before the batch-level retry pass. + // Minimum 2s pause to let throttle pressure subside before retrying failed items. + var backpressureDelay = 2; var backpressureJitter = backpressureDelay * Random.Shared.NextDouble() * 0.5; _pendingVerbose.Enqueue( $"Batch-level retry: {failedRetryable.Count} items exhausted per-chunk retries. " @@ -387,8 +595,35 @@ public async Task ExecuteBatchIndexedAsync( } var p2ChunkSw = Stopwatch.StartNew(); - var (chunkResults, chunkThrottleDelay, chunkRetries, chunkThrottles, chunkRetryDelayMs) = - await SendBatchWithRetryAsync(chunk, cancellationToken); + IReadOnlyList<(BatchOperation Operation, GraphBatchResponseItem Response)> chunkResults; + int chunkThrottleDelay, chunkRetries, chunkThrottles; + long chunkRetryDelayMs; + // Counted as each chunk goes out, for the same reason item retries are: the + // pass can be refused on its first chunk, and the candidates queued behind that + // chunk are then never POSTed. Read off the candidate list before the pass ran, + // the count reported forty batch-level retries over a wire that carried twenty. + // Here rather than after the call, so a chunk the server refused still counts - + // its POST went out, and what it carried may have been applied. + telemetry.BatchLevelRetries += chunk.Length; + try + { + (chunkResults, chunkThrottleDelay, chunkRetries, chunkThrottles, chunkRetryDelayMs) = + await SendBatchWithRetryAsync(chunk, cancellationToken); + } + // This pass exists to improve results that are already in hand. Its own POST + // failing does not unmake them: the items it carried keep the statuses the + // server gave them, which is what the caller has to act on. Thrown, it takes + // the whole array with it - every chunk that landed included - and the run ends + // with nothing said about writes the server applied. What its earlier attempts + // did get answers for is written back before it stops: those items were answered + // in this pass, and a later attempt of it being refused does not unmake that. + catch (ChunkSendFailedException ex) + { + chunkFailure = ex.Cause; + RecordChunkCosts(telemetry, ex); + FillAnsweredRetryChunk(results, failedRetryable, retryOffset, ex); + break; + } phase2PrevChunkMs = p2ChunkSw.ElapsedMilliseconds; phase2ThrottleDelay = chunkThrottleDelay; telemetry.AddItemRetries(chunkRetries); @@ -415,8 +650,10 @@ public async Task ExecuteBatchIndexedAsync( var finalResults = results.Select(r => r!.Value).ToList(); - // Compute success/failure counts and total elapsed - telemetry.Succeeded = finalResults.Count(r => r.Response.Status < 400); + // Compute success/failure counts and total elapsed. An operation that was never sent has + // no status and is neither: counting it as a success reported a run that stopped after + // one chunk as one in which everything worked. + telemetry.Succeeded = finalResults.Count(r => r.Response.Status is >= 200 and < 400); telemetry.Failed = finalResults.Count(r => r.Response.Status >= 400); telemetry.TotalElapsedMs = batchSw.ElapsedMilliseconds; @@ -436,26 +673,94 @@ public async Task ExecuteBatchIndexedAsync( }; } + /// + /// Returns (Results, ThrottleDelaySeconds, ItemRetries, ThrottleEncounters, RetryDelayMs). + /// ThrottleDelaySeconds: highest Retry-After seen, for cross-chunk backpressure. + /// ItemRetries: total individual item retries in this chunk. + /// ThrottleEncounters: number of 429 responses seen across all attempts. + /// RetryDelayMs: total milliseconds spent in retry delays within this chunk. + /// , once cancelled, ends the chunk before its next attempt + /// goes out and ends any backoff it is already waiting out. The items still pending keep + /// the last status the server gave them. + /// private async Task<(IReadOnlyList<(BatchOperation Operation, GraphBatchResponseItem Response)> Results, int ThrottleDelaySeconds, int ItemRetries, int ThrottleEncounters, long RetryDelayMs)> SendBatchWithRetryAsync( BatchOperation[] operations, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + CancellationToken stopRequested = default) { - // Track results by original index position + // Results by original index position, held out here so that however the send fails it + // fails with them: they are the record of what the server applied, and no shape of + // failure is a reason to discard it. var results = new (BatchOperation Operation, GraphBatchResponseItem Response)?[operations.Length]; + // Held out here for the same reason as the results: a chunk that fails has still spent + // its retries and met its throttles, and the run's telemetry is where that is reported. + var counts = new ChunkAttemptCounts(); + try + { + return await SendChunkAttemptsAsync(operations, results, counts, cancellationToken, stopRequested); + } + // The caller cancelled. Not a chunk failure, and reporting it as one would bury it. + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (ChunkSendFailedException) + { + throw; + } + // Every other shape - a stalled response body, an open circuit, an envelope that does + // not answer what was sent - is this chunk's failure and not the whole run's. Letting + // it out would take the results array with it, and every other chunk's outcome in it. + catch (Exception ex) + { + throw new ChunkSendFailedException(ex, results, RefusalStatus(ex), counts); + } + } + + /// + /// The status the items of a failed chunk take when the failure carries none of its own. + /// + private static int RefusalStatus(Exception ex) => ex switch + { + GraphServiceException graph => (int)graph.StatusCode, + HttpRequestException { StatusCode: { } status } => (int)status, + _ => RefusedWithoutStatus, + }; + + private async Task<(IReadOnlyList<(BatchOperation Operation, GraphBatchResponseItem Response)> Results, int ThrottleDelaySeconds, int ItemRetries, int ThrottleEncounters, long RetryDelayMs)> SendChunkAttemptsAsync( + BatchOperation[] operations, + (BatchOperation Operation, GraphBatchResponseItem Response)?[] results, + ChunkAttemptCounts counts, + CancellationToken cancellationToken, + CancellationToken stopRequested) + { // Track which indices are still pending var pendingIndices = Enumerable.Range(0, operations.Length).ToList(); // Track last-seen throttle delay for cross-chunk backpressure int lastThrottleDelaySeconds = 0; - int totalItemRetries = 0; - int totalThrottleEncounters = 0; - long totalRetryDelayMs = 0; for (int attempt = 0; attempt <= MaxPerRequestRetries; attempt++) { if (pendingIndices.Count == 0) break; + // A chunk that was already sending when another chunk was refused must not send + // again: after the refusal the run is stopping, and a POST into it is a write + // nothing will report on. Read at the first attempt as well as the later ones - + // the refusal can land in the window between the permit and the first POST. + if (stopRequested.IsCancellationRequested) + { + // At the first attempt nothing has gone out at all. Returning no results leaves + // the chunk's slots to the run's never-sent handling, which is what happened to + // it; a status here would claim its writes were attempted. + if (attempt == 0) return ([], 0, 0, 0, 0); + break; + } var batchRequest = new GraphBatchRequest(); var idToIndex = new Dictionary(); + // What the previous attempt got for the items this one is about to re-send. Put back + // if this attempt is stopped before it reaches the wire: an attempt that never went + // out does not unmake the answer before it. + var withheld = new (BatchOperation Operation, GraphBatchResponseItem Response)?[pendingIndices.Count]; for (int i = 0; i < pendingIndices.Count; i++) { @@ -464,6 +769,12 @@ public async Task ExecuteBatchIndexedAsync( var id = (i + 1).ToString(); idToIndex[id] = originalIndex; + // In flight again, so an earlier attempt's answer is no longer the last thing + // that happened to it. If this attempt fails the item is attempted-unknown, + // not whatever it was before. + withheld[i] = results[originalIndex]; + results[originalIndex] = null; + var item = new GraphBatchRequestItem { Id = id, @@ -484,13 +795,46 @@ public async Task ExecuteBatchIndexedAsync( batchRequest.Requests.Add(item); } + // Counted here rather than when the retry was decided on: between the two the run + // can stop, and a retry counted in front of a backoff that the stop then ends is a + // request the summary claims went out over one that never did. + if (attempt > 0) counts.ItemRetries += pendingIndices.Count; + var json = JsonSerializer.Serialize(batchRequest, JsonOptions); var content = new StringContent(json, Encoding.UTF8, "application/json"); var httpSw = Stopwatch.StartNew(); // paceGate: false - this class owns batch throughput (cross-call pacing, item-level // AIMD). Running the request pacer's gate on top would stack two independent AIMD // controllers on one workload. Outer responses still feed pacer signal state. - using var response = await _client.PostAsync(_batchUrl, content, cancellationToken, paceGate: false); + // + // stopRetries carries the same stop the loop above reads. The transport retries this + // POST on its own for a throttled envelope, and it did so after the run had stopped + // sending: the chunk waited out the Retry-After the server asked for and then put + // its writes back on the wire. It reaches only the decision to send again, never the + // attempt in flight - that one finishes and keeps its answer. + HttpResponseMessage response; + try + { + response = await _client.PostAsync(_batchUrl, content, cancellationToken, + paceGate: false, stopRetries: stopRequested); + } + // The stop reached this attempt while it was still waiting for its turn to send - a + // permit from the token bucket - so nothing of it went out. The same situation as + // the check at the top of the loop, a moment later, and answered the same way. + catch (RequestNotSentException) + { + // At the first attempt nothing of the chunk has gone out at all, so its slots + // belong to the run's never-sent handling rather than to a status here. + if (attempt == 0) return ([], 0, 0, 0, 0); + + // A later attempt was counted above in front of a POST that then never left. The + // run reports the requests that went out, not the ones it had ready. + counts.ItemRetries -= pendingIndices.Count; + for (int i = 0; i < pendingIndices.Count; i++) + results[pendingIndices[i]] = withheld[i]; + break; + } + using var responseScope = response; var httpMs = httpSw.ElapsedMilliseconds; if (httpMs > 5000) _pendingVerbose.Enqueue($"Batch HTTP slow: {httpMs}ms for {pendingIndices.Count} items (attempt {attempt + 1})"); @@ -498,15 +842,20 @@ public async Task ExecuteBatchIndexedAsync( if (!response.IsSuccessStatusCode) { using var errCts = _client.CreateBodyReadCts(cancellationToken); + Exception refusal; try { var errorBody = await response.Content.ReadAsStringAsync(errCts.Token); - throw new GraphServiceException(response.StatusCode, errorBody); + refusal = new GraphServiceException(response.StatusCode, errorBody); } catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) { - throw new HttpRequestException($"Response body read timed out for error response (HTTP {(int)response.StatusCode})."); + refusal = new HttpRequestException(ResilientGraphClient.BodyReadTimedOutMessage); } + // A refusal on a later attempt does not unmake the sub-responses the earlier + // attempts got: those items were answered, and for a write that answer is the + // record of what the server did. They travel with the failure. + throw new ChunkSendFailedException(refusal, results, (int)response.StatusCode, counts); } GraphBatchResponse? batchResponse; @@ -519,7 +868,7 @@ public async Task ExecuteBatchIndexedAsync( catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) { // Body read timed out but user didn't cancel - treat as transient network error - throw new HttpRequestException("Response body read timed out. The server sent headers but the body stream stalled."); + throw new HttpRequestException(ResilientGraphClient.BodyReadTimedOutMessage); } // RD-H3: Validate batch response is not null/empty @@ -542,16 +891,31 @@ public async Task ExecuteBatchIndexedAsync( var retryIndices = new List(); int maxRetryAfterSeconds = 0; + // Sub-responses the envelope gave no status of its own. Collected rather than + // recorded, and answered for below. + List? statusless = null; foreach (var item in batchResponse.Responses) { if (!idToIndex.TryGetValue(item.Id, out var originalIndex)) continue; var op = operations[originalIndex]; + // The deserializer cannot tell a "status" member the envelope left out from one + // set to 0, and neither of those is a status a server answers with - while 0 is + // exactly NotSentStatus. A sub-response exists only for a request that went out, + // so recording it would hand back an answered write as one that certainly never + // reached the server, with a dead-letter line inviting the caller to send it + // again. Left out of the results, for the refusal below to fill. + if (item.Status <= 0) + { + (statusless ??= []).Add(item.Id); + continue; + } + // Count all 429s regardless of retry budget if (item.Status == 429) { - totalThrottleEncounters++; + counts.ThrottleEncounters++; // Parse Retry-After for delay calculation. Handles both formats: // - delta-seconds: "120" (integer seconds) @@ -561,15 +925,10 @@ public async Task ExecuteBatchIndexedAsync( var retryAfterValue = item.Headers .FirstOrDefault(h => string.Equals(h.Key, "Retry-After", StringComparison.OrdinalIgnoreCase)) .Value; - if (retryAfterValue != null && - System.Net.Http.Headers.RetryConditionHeaderValue.TryParse(retryAfterValue, out var ra)) { - var delay = ra.Delta - ?? (ra.Date.HasValue ? ra.Date.Value - DateTimeOffset.UtcNow : (TimeSpan?)null); - if (delay.HasValue) + if (RetryAfterPolicy.TryResolveSeconds(retryAfterValue, _maxRetryAfterSeconds, + out var seconds, out var clamped)) { - var seconds = (int)Math.Ceiling(delay.Value.TotalSeconds); - var clamped = Math.Min(Math.Max(seconds, 0), _maxRetryAfterSeconds); if (clamped < seconds && attempt < MaxPerRequestRetries) { _pendingVerbose.Enqueue( @@ -581,31 +940,65 @@ public async Task ExecuteBatchIndexedAsync( } } + // Recorded even when it is about to be retried: the next attempt overwrites it, + // and if there is no next attempt this is what the server last said about the + // item - which is not the same thing as never having sent it. + results[originalIndex] = (op, item); if (IsRetryable(item.Status, op.Method) && attempt < MaxPerRequestRetries) - { retryIndices.Add(originalIndex); - } - else - { - results[originalIndex] = (op, item); - } + } + + // An envelope that answers a request without saying what happened to it has not + // answered it, and is reported the way the count and id checks above report an + // envelope that answers the wrong requests: as this chunk's failure. The items it + // did answer keep those answers, and the rest take the refusal status - which says + // the write may have landed, not that it certainly did not. + if (statusless != null) + { + throw new InvalidOperationException( + $"Graph $batch answered {statusless.Count} of {batchResponse.Responses.Count} requests " + + $"without a status (id {string.Join(", ", statusless)}) " + + $"(attempt {attempt + 1}/{MaxPerRequestRetries + 1}). " + + "The response may have been truncated or rewritten in transit."); } pendingIndices = retryIndices; - totalItemRetries += retryIndices.Count; lastThrottleDelaySeconds = Math.Max(lastThrottleDelaySeconds, maxRetryAfterSeconds); if (retryIndices.Count > 0 && attempt < MaxPerRequestRetries) { - var baseDelaySeconds = Math.Min( - maxRetryAfterSeconds > 0 ? maxRetryAfterSeconds : (int)Math.Pow(2, attempt), - _maxRetryAfterSeconds); - // 0-50% jitter prevents a thundering herd on batch retries + // Waiting out a backoff for an attempt that will not be sent holds the whole + // batch open for a Retry-After the run has already stopped honoring. + if (stopRequested.IsCancellationRequested) break; + + // The exponential fallback answers to the same ceiling as a server-sent + // Retry-After: a cap on how long one attempt may hold the batch open is not + // a cap on only the delays the server asked for. + var baseDelaySeconds = maxRetryAfterSeconds > 0 + ? maxRetryAfterSeconds + : Math.Min((int)Math.Pow(2, attempt), _maxRetryAfterSeconds); + // C4: Add 0-50% jitter to prevent thundering herd on batch retries var jitter = baseDelaySeconds * Random.Shared.NextDouble() * 0.5; var retrySw = Stopwatch.StartNew(); - await Task.Delay(TimeSpan.FromSeconds(baseDelaySeconds + jitter), cancellationToken); - totalRetryDelayMs += retrySw.ElapsedMilliseconds; + // A refusal that lands mid-wait ends the wait. Checked only before it started, + // a chunk that stopped sending 300ms into a peer's Retry-After still held the + // call open for the rest of it - a minute, and up to the cap plus jitter. + using (var delayCts = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, stopRequested)) + { + try + { + RetryWaitStarted?.Invoke(); + await Task.Delay(TimeSpan.FromSeconds(baseDelaySeconds + jitter), delayCts.Token); + } + // The stop, not the caller's cancellation. The loop continues to the checks + // at the top of the next attempt, which end the chunk where it stands. + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + } + } + counts.RetryDelayMs += retrySw.ElapsedMilliseconds; } } @@ -619,7 +1012,8 @@ public async Task ExecuteBatchIndexedAsync( $"Batch response missing result for request at index {nullIndex} (URL: {missingOp.Url}). " + "Graph returned the expected number of responses but with non-matching IDs."); } - return (results.Select(r => r!.Value).ToList(), lastThrottleDelaySeconds, totalItemRetries, totalThrottleEncounters, totalRetryDelayMs); + return (results.Select(r => r!.Value).ToList(), lastThrottleDelaySeconds, + counts.ItemRetries, counts.ThrottleEncounters, counts.RetryDelayMs); } /// @@ -638,10 +1032,13 @@ public void DrainVerboseMessages() VerboseWriter(msg); } + /// + /// Whether a batch response item should be retried: the same classifier and policy the + /// request pipeline uses, so the two cannot drift apart again. Batch items carry a + /// status and inline headers, never an exception - that is the whole difference. + /// private static bool IsRetryable(int statusCode, string method) - { - if (statusCode == 429) return true; - if (string.Equals(method, "POST", StringComparison.OrdinalIgnoreCase)) return false; - return statusCode is 408 or 500 or 502 or 503 or 504; - } + => MgxErrorPolicy.ShouldRetry( + MgxErrorClassifier.Classify(statusCode).Class, + isIdempotent: !string.Equals(method, "POST", StringComparison.OrdinalIgnoreCase)); } diff --git a/src/Mgx.Engine/Http/GraphContentClient.cs b/src/Mgx.Engine/Http/GraphContentClient.cs index eec7ad7..f762712 100644 --- a/src/Mgx.Engine/Http/GraphContentClient.cs +++ b/src/Mgx.Engine/Http/GraphContentClient.cs @@ -1,3 +1,4 @@ +using Mgx.Engine.Errors; using System.Net; using System.Net.Http.Headers; using Polly; @@ -5,7 +6,9 @@ namespace Mgx.Engine.Http; -/// Result of a content fetch. Owns the response: dispose it to release the connection. +/// +/// Result of a content fetch. Owns the response: dispose it to release the connection. +/// public sealed class GraphContentResult : IDisposable { public required Stream Content { get; init; } @@ -31,28 +34,31 @@ public void Dispose() /// /// The two-hop content path behind Get-MgxContent. /// -/// Hop one is the authenticated Graph request through ResilientGraphClient, with the full -/// pipeline, pacer and bucket lease. A 2xx with a body is content Graph served directly. A -/// redirect is the drive-item case, and its Location is validated against DownloadUrlValidator -/// before hop two touches it. +/// Hop 1 (Graph, authenticated) goes through ResilientGraphClient.SendAsync - full pipeline, +/// pacer, bucket lease. A 2xx with a body is content served directly by Graph (attachments, +/// photos). A redirect is the drive-item case: the Location is a pre-authenticated URL on a +/// download host, validated against DownloadUrlValidator before hop 2 touches it. /// -/// Hop two fetches from the download host through a singleton HttpClient with no auth handler, -/// so the bearer token never reaches that host. AllowAutoRedirect is off and every Location is -/// re-validated over at most three manual redirects, since an open redirect on an allowlisted -/// host must not bypass the validator. Its own small retry pipeline handles 429 and 5xx with -/// Retry-After. It has no circuit breaker, so a CDN outage cannot poison the Graph circuit, and -/// no bucket charge, since the request budget is Graph-side. On a 401 or 403 the caller re-runs -/// hop one once for a fresh URL. +/// Hop 2 (download host, token-free) uses a static singleton HttpClient with NO auth handler: +/// the bearer must never reach the download host. AllowAutoRedirect is off and redirects are +/// followed manually (max 3), re-validating every Location - an open redirect on an +/// allowlisted host must not silently bypass the validator. Its own small retry pipeline +/// handles 429/5xx with Retry-After; deliberately NO circuit breaker (a CDN outage must not +/// poison the Graph circuit) and NO bucket charge (the request budget is Graph-side). +/// On 401/403 (expired pre-auth URL) the caller re-runs hop 1 once for a fresh URL. /// -/// The 302 is only observable because the owned clean HttpClient sets AllowAutoRedirect to -/// false. On a transport that auto-follows, the bytes would arrive as a 2xx from an unvalidated -/// host, so any 2xx whose request URI left the Graph host is rejected here and the cmdlet layer -/// refuses non-owned transports before calling in. +/// Transport precondition (fail closed): the 302 is only observable because the owned clean +/// HttpClient sets AllowAutoRedirect=false. On a transport that auto-follows (the SDK +/// fallback client ships Kiota's RedirectHandler; injected transports are unknown), the +/// bytes would arrive as a 2xx from a host mgx never validated - so any 2xx whose request +/// URI left the Graph host is rejected here, and the cmdlet layer refuses non-owned +/// transports outright before calling in. /// public static class GraphContentClient { private const int MaxManualRedirects = 3; + /// Statuses hop 1 treats as "go fetch from the download host". private static bool IsRedirect(HttpStatusCode status) => status is HttpStatusCode.Moved or HttpStatusCode.Found or HttpStatusCode.SeeOther or HttpStatusCode.TemporaryRedirect @@ -62,7 +68,7 @@ or HttpStatusCode.SeeOther or HttpStatusCode.TemporaryRedirect /// (redirects, 429/5xx, auth-expiry) can be mocked. Never set from production code. internal static HttpClient? DownloadClientForTests; - // Token-free singleton for hop 2. No auth handler by construction. Decompression stays + // Token-free singleton for hop 2. No auth handler by construction; decompression stays // off so ranged reads and Content-Length are byte-exact. private static readonly HttpClient s_downloadClient = new(new SocketsHttpHandler { @@ -72,34 +78,32 @@ or HttpStatusCode.SeeOther or HttpStatusCode.TemporaryRedirect ConnectTimeout = TransportDefaults.ConnectTimeout }) { - // Covers until response headers (ResponseHeadersRead). The body copy is bounded by + // Covers until response headers (ResponseHeadersRead); the body copy is bounded by // the caller's idle timeout, not this. Timeout = TimeSpan.FromSeconds(100) }; - // Hop two retries three times with exponential backoff and jitter on 429, 5xx and transport - // errors, honoring Retry-After clamped to two minutes + // Small reactive pipeline for hop 2: retry 3 with exponential backoff + jitter on + // 429/5xx/transport errors, honoring Retry-After clamped to two minutes. /// - /// Delay for one download retry, from Retry-After, which has two legal forms: delta-seconds - /// and an HTTP-date. A date already in the past yields no delay rather than a negative one, - /// and both forms are capped at 120 seconds. Polly MaxDelay does not clamp a value returned - /// from a DelayGenerator, so that cap is the only bound on the sleep. - /// Internal rather than inlined so a test can reach it without driving a real 429 through - /// GetContentAsync. + /// Delay for one download retry, from Retry-After. The header has two legal forms: + /// delta-seconds and an HTTP-date. Honoring only Delta silently fell back to plain + /// exponential backoff whenever a download host chose the date form - and the main pipeline + /// already handles both, so this path was the odd one out. A date already in the past yields + /// no delay rather than a negative one, and both forms are capped at 120s. + /// + /// Internal rather than inlined into the DelayGenerator lambda because the pipeline is a + /// private static field with no seam: a test could only reach the lambda by driving a real + /// 429 through GetContentAsync. The test that covers this used to hold its own copy of the + /// logic instead, which passed with the whole feature deleted from the product. + /// Polly's MaxDelay does NOT clamp a value returned from a DelayGenerator, so the cap here + /// is the only bound on the sleep. /// internal static TimeSpan? ResolveRetryDelay(RetryConditionHeaderValue? retryAfter) - { - var cap = TimeSpan.FromSeconds(120); - if (retryAfter?.Delta is { } delta) - return delta > cap ? cap : delta; - if (retryAfter?.Date is { } date) - { - var delay = date - DateTimeOffset.UtcNow; - if (delay > TimeSpan.Zero) - return delay > cap ? cap : delay; - } - return null; - } + // The 120s cap is hardcoded because this pipeline is a type-initialized static and + // cannot see ResilientGraphClientOptions; making it options-driven is a redesign of + // this class, not a parameter change. + => RetryAfterPolicy.Resolve(retryAfter, TimeSpan.FromSeconds(120)); private static readonly ResiliencePipeline s_downloadPipeline = new ResiliencePipelineBuilder() @@ -112,13 +116,13 @@ or HttpStatusCode.SeeOther or HttpStatusCode.TemporaryRedirect MaxDelay = TimeSpan.FromSeconds(120), ShouldHandle = args => { - if (args.Outcome.Result?.StatusCode is (HttpStatusCode)429 - or HttpStatusCode.InternalServerError - or HttpStatusCode.BadGateway - or HttpStatusCode.ServiceUnavailable - or HttpStatusCode.GatewayTimeout) - return ValueTask.FromResult(true); - return ValueTask.FromResult(args.Outcome.Exception is HttpRequestException); + var info = args.Outcome.Result is { } response + ? MgxErrorClassifier.Classify(response) + : args.Outcome.Exception is { } ex + ? MgxErrorClassifier.Classify(ex, cancellationRequested: false) + : new MgxErrorInfo(MgxErrorClass.Permanent, 0); + return ValueTask.FromResult( + MgxErrorPolicy.ShouldRetryDownload(info, args.Outcome.Exception)); }, DelayGenerator = args => ValueTask.FromResult(ResolveRetryDelay(args.Outcome.Result?.Headers.RetryAfter)), @@ -142,9 +146,12 @@ public static async Task GetContentAsync( CancellationToken cancellationToken) { var graphHost = new Uri(requestUri).Host; + // Preserve case-insensitivity: Dictionary(IDictionary) silently swaps in the + // ordinal comparer, which made the hop-2 conditional forward and the + // client-request-id suppression casing-sensitive on this path only. var requestHeaders = headers != null - ? new Dictionary(headers) - : new Dictionary(); + ? new Dictionary(headers, StringComparer.OrdinalIgnoreCase) + : new Dictionary(StringComparer.OrdinalIgnoreCase); if (range != null) requestHeaders["Range"] = range.ToString(); @@ -190,7 +197,8 @@ public static async Task GetContentAsync( $"Download host '{TryGetHost(absolute)}' is not on the allowed list " + "(SharePoint/OneDrive download hosts only). Refusing to fetch content from it."); - var hop2 = await FetchFromDownloadHostAsync(validated, range, cancellationToken); + var hop2 = await FetchFromDownloadHostAsync(validated, range, cancellationToken, + FilterDownloadHeaders(requestHeaders)); if (hop2.StatusCode is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden && authAttempt == 0) @@ -220,7 +228,7 @@ public static async Task GetContentAsync( /// /// Fetch directly from a pre-authenticated download URL (a piped driveItem's /// @microsoft.graph.downloadUrl), skipping hop 1. The caller MUST have validated the URL - /// through DownloadUrlValidator first. This method validates again and throws otherwise. + /// through DownloadUrlValidator first; this method validates again and throws otherwise. /// No auth refresh is possible on this path - the URL is short-lived, and a 401/403 means /// the item must be re-fetched for a fresh one. /// @@ -228,14 +236,16 @@ public static async Task GetFromDownloadUrlAsync( string downloadUrl, RangeHeaderValue? range, TimeSpan bodyReadTimeout, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + Dictionary? headers = null) { var validated = DownloadUrlValidator.Validate(downloadUrl) ?? throw new InvalidOperationException( $"Download host '{TryGetHost(downloadUrl)}' is not on the allowed list " + "(SharePoint/OneDrive download hosts only). Refusing to fetch content from it."); - var response = await FetchFromDownloadHostAsync(validated, range, cancellationToken); + var response = await FetchFromDownloadHostAsync(validated, range, cancellationToken, + FilterDownloadHeaders(headers)); if (response.IsSuccessStatusCode) return await WrapAsync(response, fromDownloadHost: true, bodyReadTimeout, cancellationToken); @@ -245,8 +255,38 @@ public static async Task GetFromDownloadUrlAsync( throw new Models.GraphServiceException(status, body); } + /// + /// The caller headers that cross to the download host: the READ-conditional family + /// plus Accept. Everything else stays on hop 1 - the download host is token-free by + /// construction, and correlation or auth headers must never reach it. If-Match and + /// If-Unmodified-Since stay behind too: they are write validators, and a Graph etag + /// forwarded to a SharePoint blob host cannot match - it would turn a shared + /// -Headers splat into a 412 on every download. + /// + private static readonly string[] s_forwardableDownloadHeaders = + ["If-None-Match", "If-Modified-Since", "If-Range", "Accept"]; + + private static Dictionary? FilterDownloadHeaders(Dictionary? headers) + { + if (headers == null) return null; + Dictionary? filtered = null; + foreach (var (key, value) in headers) + { + // Compare by name, not by the source dictionary's comparer - a caller-built + // dictionary may be case-sensitive, and header names are not. + if (s_forwardableDownloadHeaders.Any(n => string.Equals(n, key, StringComparison.OrdinalIgnoreCase))) + (filtered ??= new Dictionary(StringComparer.OrdinalIgnoreCase))[key] = value; + } + return filtered; + } + + /// + /// Token-free fetch with manual, re-validated redirects. Every hop must pass the + /// allowlist: an open redirect on an allowlisted host is not a pass. + /// private static async Task FetchFromDownloadHostAsync( - string url, RangeHeaderValue? range, CancellationToken cancellationToken) + string url, RangeHeaderValue? range, CancellationToken cancellationToken, + Dictionary? conditionalHeaders = null) { var current = url; for (var redirects = 0; ; redirects++) @@ -262,6 +302,9 @@ private static async Task FetchFromDownloadHostAsync( var request = new HttpRequestMessage(HttpMethod.Get, target); if (range != null) request.Headers.Range = range; + if (conditionalHeaders != null) + foreach (var (name, value) in conditionalHeaders) + request.Headers.TryAddWithoutValidation(name, value); return await (DownloadClientForTests ?? s_downloadClient).SendAsync( request, HttpCompletionOption.ResponseHeadersRead, ctx.CancellationToken); }, @@ -297,7 +340,7 @@ private static async Task WrapAsync( HttpResponseMessage response, bool fromDownloadHost, TimeSpan bodyReadTimeout, CancellationToken cancellationToken) { - // ResponseHeadersRead: opening the stream is cheap. The caller copies with an idle + // ResponseHeadersRead: opening the stream is cheap; the caller copies with an idle // timeout (CopyWithIdleTimeoutAsync) so a stalled body cannot hang forever. using var bodyCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); bodyCts.CancelAfter(bodyReadTimeout); @@ -360,7 +403,7 @@ public static async Task CopyWithIdleTimeoutAsync( // Discard the bytes before the requested offset. Only reachable when a ranged request // was answered with 200 and the whole body: the server ignored the offset, so it has to - // be honored here instead. Without this, -Offset 1MB -Length 64KB returned bytes + // be honoured here instead. Without this, -Offset 1MB -Length 64KB returned bytes // 0..65535 and reported success - the wrong bytes, silently, which is worse than an // error because nothing downstream can tell. long skipped = 0; @@ -402,6 +445,11 @@ public static async Task CopyWithIdleTimeoutAsync( return total; } + /// + /// One read, bounded by the idle timeout. A stalled body must not hang forever, and the + /// timeout applies per read rather than to the transfer as a whole so a slow-but-progressing + /// download is not killed. + /// private static async Task ReadWithIdleTimeoutAsync( Stream source, byte[] buffer, int count, TimeSpan idleTimeout, CancellationToken cancellationToken) diff --git a/src/Mgx.Engine/Http/GraphRequestTracer.cs b/src/Mgx.Engine/Http/GraphRequestTracer.cs index 7c8b9ea..2538c7e 100644 --- a/src/Mgx.Engine/Http/GraphRequestTracer.cs +++ b/src/Mgx.Engine/Http/GraphRequestTracer.cs @@ -12,6 +12,10 @@ internal static partial class GraphRequestTracer /// Bodies longer than this are cut, with the omitted length noted. internal const int MaxBodyChars = 4096; + /// + /// Response headers worth tracing. Everything else (caching, CORS, transport) is noise. + /// Prefix entries match any header starting with the value. + /// private static readonly string[] ResponseHeaderNames = [ "request-id", "client-request-id", "x-ms-ags-diagnostic", "Retry-After", @@ -20,28 +24,80 @@ internal static partial class GraphRequestTracer private static readonly string[] ResponseHeaderPrefixes = ["RateLimit-", "x-ms-throttle-"]; - [GeneratedRegex("\"([^\"]*(?:password|secret|credential|token|key)[^\"]*)\"\\s*:\\s*\"[^\"]*\"", - RegexOptions.IgnoreCase)] - private static partial Regex SensitiveJsonValue(); + /// + /// Properties whose NAME says credential. The fragments come from SensitiveNames, the same + /// list the dead-letter writer reads, so a clientAssertion or a connectionString is missing + /// from a trace and from the file alike. Not source-generated, because the pattern is that + /// list: written out here it would be a second copy of it, free to drift. + /// + /// Compiled, because the other two patterns are source-generated and this one runs over the + /// same bodies: interpreted, it is the slowest thing in a -Debug trace of a large response. + private static readonly Regex SensitiveJsonValue = new( + $"\"([^\"]*(?:{SensitiveNames.RegexAlternation})[^\"]*)\"\\s*:\\s*\"[^\"]*\"", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled); /// - /// A pre-authenticated URL is a credential whose property name does not say so, so match on - /// the value instead. Matching every URL would redact @odata.nextLink, which paging bugs are - /// diagnosed with. + /// A pre-authenticated URL is a credential whose property name says nothing about it: + /// "@microsoft.graph.downloadUrl" carries none of the SensitiveNames fragments, so + /// SensitiveJsonValue leaves it whole and the tempauth JWT reaches -Debug output. Match on + /// the VALUE instead - a URL carrying one of the capability parameters Graph, SharePoint and + /// the download hosts actually use. Those parameters are SensitiveNames', the definition the + /// dead-letter writer reads too, so a SAS URL missing from a trace is missing from the file; + /// that class says why the list is not simply every URL. /// - [GeneratedRegex("\"(https?://[^\"]*?(?:tempauth|guestaccesstoken|authkey|X-Amz-Signature|(? - /// Property names carrying a pre-authenticated URL whose capability sits in the path rather - /// than in a recognizable parameter. + /// A capability URL used as a property NAME, which takes the value beside it with it. A + /// pre-authenticated URL as a field name is itself a credential, so whatever a body files + /// under one is the thing that URL was minted for, and the dead-letter writer clears it. + /// CapabilityUrlValue() below finds such a key too - a quoted string is a quoted string, + /// key or value - but it stops at that string's closing quote and leaves the value standing. + /// The colon is what says the string was a key: a JSON value is followed by a comma or a + /// closing brace or bracket, never by one. + /// + /// That trailing `"[^"]*"` only reaches the value when it is itself a quoted string. The + /// dead-letter writer clears the value beside a cut key whatever it holds, so a number, a + /// bool, an array or an object beside a capability-URL key is missing from the dead-letter + /// line and still stands whole in -Debug output - this pattern has nothing to match there. /// - [GeneratedRegex("\"([^\"]*downloadurl[^\"]*)\"\\s*:\\s*\"[^\"]*\"", + [GeneratedRegex($"\"(https?://[^\"]*?(?:{SensitiveNames.CapabilityUrlParameters})=)[^\"]*\"\\s*:\\s*\"[^\"]*\"", + RegexOptions.IgnoreCase)] + private static partial Regex CapabilityUrlProperty(); + + /// + /// The value rule again on a URL that has lost its scheme and host. A $batch envelope + /// carries each item's url relative - Invoke-MgxBatchRequest normalizes it before the POST, + /// which is what Graph's batch format takes - so a pre-authenticated URL a caller passed as + /// an item URL reaches a trace as "/x?tempauth=..." and the two patterns above, anchored on + /// the scheme, have nothing in it to match. The leading slash stands in for the scheme: it + /// says the quoted string is a path rather than prose that happens to carry a parameter. + /// + /// A value only, with no property counterpart. The dead-letter writer reads a body's keys + /// and values through SensitiveNames.RedactCapabilityUrl, which reads the same two forms, so + /// a relative path is cut on both sides: in the file, where a value written whole is a live + /// credential on disk, and here. Where the two are read side by side is the item URL, which + /// is absolute in the file where a caller gave it absolute, relative on the wire, and cut + /// either way. + /// + [GeneratedRegex($"\"(/[^\"]*?(?:{SensitiveNames.CapabilityUrlParameters})=)[^\"]*\"", + RegexOptions.IgnoreCase)] + private static partial Regex RelativeCapabilityUrlValue(); + + /// + /// Property names that carry a pre-authenticated URL even when the value does not expose a + /// recognizable parameter - the short-form download hosts put the capability in the path. + /// The fragment is SensitiveNames', which the writer reads one property name at a time. + /// + [GeneratedRegex($"\"([^\"]*{SensitiveNames.DownloadUrlFragment}[^\"]*)\"\\s*:\\s*\"[^\"]*\"", RegexOptions.IgnoreCase)] private static partial Regex DownloadUrlProperty(); - /// Trace line for an outgoing request. is 1-based so retries are visible. + /// + /// Trace line for an outgoing request. is 1-based so retries are visible. + /// internal static string FormatRequest(HttpRequestMessage request, byte[]? body, int attempt) { var sb = new StringBuilder(); @@ -50,8 +106,8 @@ internal static string FormatRequest(HttpRequestMessage request, byte[]? body, i .Append(request.Method.Method).Append(' ').Append(request.RequestUri); sb.AppendLine().Append(" Headers:"); - // The auth handler attaches the bearer token further down the pipeline, so it is not on - // this message. Shown for completeness, never its value + // The bearer token is attached further down the pipeline by the auth handler, + // so it is not on this HttpRequestMessage. Show it for completeness, never its value. sb.AppendLine().Append(" Authorization: Bearer "); foreach (var header in request.Headers) sb.AppendLine().Append(" ").Append(header.Key).Append(": ").Append(Join(header.Value)); @@ -70,7 +126,9 @@ internal static string FormatRequest(HttpRequestMessage request, byte[]? body, i return sb.ToString(); } - /// Trace line for a response. is null when the body was not buffered. + /// + /// Trace line for a response. is null when the body was not buffered. + /// internal static string FormatResponse(HttpResponseMessage response, long elapsedMs, string? body) { var sb = new StringBuilder(); @@ -105,6 +163,13 @@ private static bool IsTraced(string name) => private static string Join(IEnumerable values) => string.Join(", ", values); + /// + /// A content 302's Location is a pre-authenticated URL: it grants the file bytes without a + /// bearer token, so tracing it verbatim writes a live credential into -Debug output. The + /// capability is not always in the query - some download hosts carry it in the path - so + /// keep only scheme and host, which is the whole diagnostic value of the header anyway. + /// The JSON-body redaction cannot reach header values, hence this hook. + /// private static string RedactHeaderValue(string name, string value) { if (!name.Equals("Location", StringComparison.OrdinalIgnoreCase)) return value; @@ -113,14 +178,21 @@ private static string RedactHeaderValue(string name, string value) : ""; } + /// Redact credential-looking JSON properties and pre-authenticated URLs, then truncate. private static string Sanitize(string body) { - var redacted = SensitiveJsonValue().Replace(body, "\"$1\": \"\""); - // Property-name match first, covering a downloadUrl whose capability sits in the path + var redacted = SensitiveJsonValue.Replace(body, "\"$1\": \"\""); + // Property-name match first: covers a downloadUrl whose capability sits in the path. redacted = DownloadUrlProperty().Replace(redacted, "\"$1\": \"\""); - // Then any URL value carrying a capability parameter, keeping the parameter name visible - // so the trace still shows which kind of URL was redacted + // A capability URL used as a key, and the value beside it, before the pass below cuts + // such a key on its own and leaves that value standing. + redacted = CapabilityUrlProperty().Replace(redacted, "\"$1\": \"\""); + // Then any remaining URL value carrying a capability parameter, keeping the parameter + // name visible so a trace still shows WHICH kind of URL was redacted. redacted = CapabilityUrlValue().Replace(redacted, "\"$1\""); + // And the same value with its scheme and host gone, which is the form a $batch item's + // own url has by the time it is on the wire. + redacted = RelativeCapabilityUrlValue().Replace(redacted, "\"$1\""); return redacted.Length <= MaxBodyChars ? redacted : redacted[..MaxBodyChars] + $"... [truncated, {redacted.Length - MaxBodyChars} more chars]"; diff --git a/src/Mgx.Engine/Http/ResiliencePipelineFactory.cs b/src/Mgx.Engine/Http/ResiliencePipelineFactory.cs index a0bcf2e..e6ea951 100644 --- a/src/Mgx.Engine/Http/ResiliencePipelineFactory.cs +++ b/src/Mgx.Engine/Http/ResiliencePipelineFactory.cs @@ -1,3 +1,4 @@ +using Mgx.Engine.Errors; using System.Net; using System.Net.Http.Headers; using System.Threading.RateLimiting; @@ -6,6 +7,8 @@ using Polly.Retry; using Polly.Timeout; + + namespace Mgx.Engine.Http; /// @@ -23,7 +26,7 @@ public static class ResiliencePipelineFactory /// /// Property key for passing idempotency info into the Polly retry predicate. - /// POST is the only non-idempotent Graph method. It only retries on 429. + /// POST is the only non-idempotent Graph method; it only retries on 429. /// internal static readonly ResiliencePropertyKey IsIdempotentKey = new("IsIdempotent"); @@ -35,8 +38,18 @@ public static class ResiliencePipelineFactory internal static readonly ResiliencePropertyKey?> VerboseWriterKey = new("VerboseWriter"); /// - /// Get or create a shared resilience pipeline and rate limiter, rebuilding whenever the - /// options instance changes. + /// Property key for a caller's stop signal. Once canceled, no further attempt is decided + /// on for this request. It deliberately does not cancel the request: the attempt in flight + /// finishes and its answer is returned, because that answer is the only record of what the + /// server did with it. What stops is the decision to send another one. + /// + internal static readonly ResiliencePropertyKey StopRetriesKey = new("StopRetries"); + + /// + /// Get or create a shared resilience pipeline and rate limiter. + /// Rebuilds when options change (detected by reference equality, since + /// Set-MgxOption creates a new ResilientGraphClientOptions each time). + /// Old rate limiters are disposed after a delay to avoid racing with in-flight clients. /// public static (ResiliencePipeline Pipeline, TokenBucketRateLimiter? RateLimiter) GetOrCreate(ResilientGraphClientOptions options) @@ -46,8 +59,21 @@ public static (ResiliencePipeline Pipeline, TokenBucketRate if (s_pipeline != null && ReferenceEquals(s_cachedOptions, options)) return (s_pipeline, s_rateLimiter); - // The retired limiter is never disposed because in-flight clients hold it as a - // readonly field for as long as their request runs + // Schedule delayed disposal of the old rate limiter. It may still be + // referenced by in-flight ResilientGraphClient instances, so we wait + // TotalTimeoutSeconds to ensure all in-flight requests have completed. + // Not disposed. The limiter is captured as a readonly field by every + // ResilientGraphClient already built from it, and by the handler + // Enable-MgxResilience injects into the SDK - and that handler is not rebuilt when + // options change. Disposing on a timer therefore broke live sessions: any + // Set-MgxOption call left every SDK cmdlet throwing "Cannot access a disposed + // object. Object name: 'TokenBucketRateLimiter'" once the delay elapsed, recoverable + // only by Disable-MgxResilience; Enable-MgxResilience. + // + // The delay was TotalTimeoutSeconds - a per-REQUEST timeout used as a resource + // lifetime. No such bound is correct: a client may hold its limiter for a multi-hour + // export. Dropping the reference is enough; the limiter owns no unmanaged handle and + // its replenishment timer is rooted only by the limiter itself. TokenBucketRateLimiter? rateLimiter = null; if (!options.NoRateLimit) @@ -67,8 +93,8 @@ public static (ResiliencePipeline Pipeline, TokenBucketRate s_rateLimiter = rateLimiter; s_cachedOptions = options; - // Every client build passes through here, so pacing configuration applied here - // reaches the pacer on the next invocation + // The chokepoint every client build passes through: apply pacing configuration + // here so Set-MgxOption changes reach the pacer on the next invocation. AdaptiveRequestPacer.Configure(options); return (s_pipeline, rateLimiter); @@ -84,21 +110,34 @@ public static void Reset() lock (s_lock) { s_pipeline = null; + // Dispose after delay: in-flight clients may still reference the old limiter. + // Default 300s covers the maximum total timeout window. + // Not disposed, for the reason documented in GetOrCreate. s_rateLimiter = null; s_cachedOptions = null; - // Learned pacing state describes the old tenant, so clear it with the breaker history + // Learned pacing state describes the old tenant; clear it with the CB history. AdaptiveRequestPacer.Reset(); - // GraphBatchClient keeps its own AIMD state in separate statics, so it has to be - // cleared too or a fresh credential starts at the previous tenant learned rate + // GraphBatchClient keeps its own AIMD state in separate statics, so clearing the + // request pacer alone left the batch pacer describing the previous tenant: a fresh + // credential started at the throttled item rate learned elsewhere, and its first + // batch was additionally delayed by the old tenant's completion timestamp. GraphBatchClient.ResetPacingState(); } } + /// + /// Disposes a rate limiter after a delay. TokenBucketRateLimiter holds an internal + /// Timer (via AutoReplenishment) that acts as a GC root. Immediate disposal would + /// cause ObjectDisposedException in in-flight clients, so we wait for the total + /// timeout window to expire before disposing. + /// + private static ResiliencePipeline BuildPipeline(ResilientGraphClientOptions options) { var maxRetryAfterCap = options.MaxRetryAfterSeconds; return new ResiliencePipelineBuilder() + // Total timeout .AddTimeout(TimeSpan.FromSeconds(options.TotalTimeoutSeconds)) // Retry .AddRetry(new RetryStrategyOptions @@ -110,63 +149,32 @@ private static ResiliencePipeline BuildPipeline(ResilientGr MaxDelay = TimeSpan.FromSeconds(maxRetryAfterCap), ShouldHandle = args => { - // 429 is safe to retry for every method, including POST - if (args.Outcome.Result?.StatusCode == (HttpStatusCode)429) - return ValueTask.FromResult(true); - - // Non-idempotent methods retry only on 429, since a 5xx may mean the request - // was partially processed - var isIdempotent = args.Context.Properties.GetValue(IsIdempotentKey, true); - if (!isIdempotent) + // The caller has stopped sending. Asked before the classification because + // it is not about this outcome: whatever the server said, the run this + // request belongs to is not sending anything else. + if (args.Context.Properties.GetValue(StopRetriesKey, default).IsCancellationRequested) return ValueTask.FromResult(false); - if (args.Outcome.Result?.StatusCode is HttpStatusCode.InternalServerError - or HttpStatusCode.BadGateway - or HttpStatusCode.ServiceUnavailable - or HttpStatusCode.GatewayTimeout - or HttpStatusCode.RequestTimeout) - return ValueTask.FromResult(true); - - if (args.Outcome.Exception is HttpRequestException) - return ValueTask.FromResult(true); - - // The Polly attempt timeout throws TimeoutRejectedException, so a - // TaskCanceledException here is either the user or the HttpClient timeout - if (args.Outcome.Exception is TaskCanceledException && - !args.Context.CancellationToken.IsCancellationRequested) - return ValueTask.FromResult(true); - - // Idempotent methods retry the per-attempt timeout, which matters under - // Enable-MgxResilience where the SDK retry handler may sleep a Retry-After - // longer than AttemptTimeoutSeconds. TotalTimeout still bounds the operation - if (args.Outcome.Exception is TimeoutRejectedException) - return ValueTask.FromResult(isIdempotent); - - return ValueTask.FromResult(false); + // Classification and the retry decision live in Errors/, shared with the + // batch client and the download pipeline; ErrorPolicyParityTests holds + // them to the decisions this predicate used to make inline. Per-attempt + // timeouts (TimeoutRejectedException) retry so the Enable-MgxResilience + // path survives the SDK honoring a Retry-After longer than + // AttemptTimeoutSeconds; the outer TotalTimeout still bounds the whole. + var isIdempotent = args.Context.Properties.GetValue(IsIdempotentKey, true); + var info = args.Outcome.Result is { } response + ? MgxErrorClassifier.Classify(response) + : args.Outcome.Exception is { } ex + ? MgxErrorClassifier.Classify(ex, args.Context.CancellationToken.IsCancellationRequested) + : new MgxErrorInfo(MgxErrorClass.Permanent, 0); + return ValueTask.FromResult(MgxErrorPolicy.ShouldRetry(info.Class, isIdempotent)); }, DelayGenerator = args => - { - // Honor the Retry-After header - if (args.Outcome.Result?.Headers.RetryAfter is RetryConditionHeaderValue retryAfter) - { - if (retryAfter.Delta.HasValue) - { - var delay = retryAfter.Delta.Value; - var cap = TimeSpan.FromSeconds(maxRetryAfterCap); - return ValueTask.FromResult(delay > cap ? cap : delay); - } - if (retryAfter.Date.HasValue) - { - var delay = retryAfter.Date.Value - DateTimeOffset.UtcNow; - if (delay > TimeSpan.Zero) - { - var cap = TimeSpan.FromSeconds(maxRetryAfterCap); - return ValueTask.FromResult(delay > cap ? cap : delay); - } - } - } - return ValueTask.FromResult(null); - }, + // Respect Retry-After from the service; null falls back to the + // exponential backoff configured above. + ValueTask.FromResult(RetryAfterPolicy.Resolve( + args.Outcome.Result?.Headers.RetryAfter, + TimeSpan.FromSeconds(maxRetryAfterCap))), OnRetry = args => { var response = args.Outcome.Result; @@ -223,28 +231,25 @@ or HttpStatusCode.GatewayTimeout return default; } }) - // Circuit breaker (5xx, not 429 or 408) - // 408 excluded: it's a client-perceived timeout, not a server-side failure indicator. - // Including it would cause proxy timeouts to trip the circuit breaker incorrectly. + // Circuit breaker. What counts as a circuit failure - and why 429 and 408 + // deliberately do not - lives in MgxErrorPolicy.CountsAsCircuitFailure, + // beside the retry policy it diverges from. .AddCircuitBreaker(new CircuitBreakerStrategyOptions { FailureRatio = options.CircuitBreakerFailureRatio, SamplingDuration = TimeSpan.FromSeconds(options.CircuitBreakerSamplingDurationSeconds), MinimumThroughput = options.CircuitBreakerMinThroughput, BreakDuration = TimeSpan.FromSeconds(options.CircuitBreakerDurationSeconds), - ShouldHandle = new PredicateBuilder() - .HandleResult(r => r.StatusCode == HttpStatusCode.InternalServerError) - .HandleResult(r => r.StatusCode == HttpStatusCode.BadGateway) - .HandleResult(r => r.StatusCode == HttpStatusCode.ServiceUnavailable) - .HandleResult(r => r.StatusCode == HttpStatusCode.GatewayTimeout) - .Handle() - // Exclude user cancellation (Ctrl+C) from circuit breaker failure counting. - // Only count non-user TaskCanceledException (e.g., HttpClient timeout). - .Handle(e => !e.CancellationToken.IsCancellationRequested) - // Count per-attempt timeouts as failures. Without this, repeated - // timeouts (e.g., downstream hung) never trip the circuit breaker, - // wasting MaxRetryAttempts * AttemptTimeoutSeconds before giving up. - .Handle(), + ShouldHandle = args => + { + var info = args.Outcome.Result is { } response + ? MgxErrorClassifier.Classify(response) + : args.Outcome.Exception is { } ex and not BrokenCircuitException + ? MgxErrorClassifier.Classify(ex, + (ex as OperationCanceledException)?.CancellationToken.IsCancellationRequested ?? false) + : new MgxErrorInfo(MgxErrorClass.Permanent, 0); + return ValueTask.FromResult(MgxErrorPolicy.CountsAsCircuitFailure(info)); + }, OnOpened = _ => { MgxTelemetryCollector.Current.RecordCircuitBreakerTrip(); diff --git a/src/Mgx.Engine/Http/ResilientDelegatingHandler.cs b/src/Mgx.Engine/Http/ResilientDelegatingHandler.cs index 7656dec..d43f0d6 100644 --- a/src/Mgx.Engine/Http/ResilientDelegatingHandler.cs +++ b/src/Mgx.Engine/Http/ResilientDelegatingHandler.cs @@ -85,6 +85,29 @@ protected override async Task SendAsync( // Buffer content upfront so retries can reconstruct a fresh request body. // Also snapshot all content headers (not just ContentType) to preserve // Content-Encoding, Content-Disposition, etc. on retry. + // What the pipeline cannot replay, it does not manage. A body whose declared + // length exceeds the replay-buffer cap (a drive-content upload runs to 250MB) + // passes to the SDK chain untouched: no clone, no retry-option stamp, no + // per-attempt timeout, no circuit counting - exactly what the SDK does without + // the wrap, which is the only promise the wrap can keep for a stream it cannot + // rewind. A body with no declared length is buffered whatever its size - the + // read is unavoidable to know, and once buffered it replays like any other. + if (request.Content?.Headers.ContentLength > ResilientGraphClient.MaxRequestBodyBytes) + { + var passSw = System.Diagnostics.Stopwatch.StartNew(); + var passSucceeded = false; + try + { + var passthrough = await base.SendAsync(request, cancellationToken); + passSucceeded = passthrough.IsSuccessStatusCode; + return passthrough; + } + finally + { + MgxTelemetryCollector.Current.RecordRequest(passSucceeded, passSw.ElapsedMilliseconds); + } + } + byte[]? contentBytes = null; List>>? contentHeaders = null; if (request.Content != null) @@ -92,6 +115,7 @@ protected override async Task SendAsync( contentBytes = await request.Content.ReadAsByteArrayAsync(cancellationToken); contentHeaders = request.Content.Headers.ToList(); } + var clientRequestId = Guid.NewGuid().ToString(); RateLimitLease? lease = null; var context = ResilienceContextPool.Shared.Get(cancellationToken); @@ -143,6 +167,10 @@ protected override async Task SendAsync( foreach (var header in request.Headers) clone.Headers.TryAddWithoutValidation(header.Key, header.Value); clone.Headers.TryAddWithoutValidation("SdkVersion", MgxSdkVersion.Value); + // Parity with the owned client: one correlation id per logical request, + // shared across attempts - unless the SDK already stamped its own. + if (!clone.Headers.Contains("client-request-id")) + clone.Headers.TryAddWithoutValidation("client-request-id", clientRequestId); // Copy request options (used by SDK handlers for per-request metadata) #pragma warning disable CS8714 // nullability mismatch in IDictionary generic diff --git a/src/Mgx.Engine/Http/ResilientGraphClient.cs b/src/Mgx.Engine/Http/ResilientGraphClient.cs index b8125c9..6e5ef23 100644 --- a/src/Mgx.Engine/Http/ResilientGraphClient.cs +++ b/src/Mgx.Engine/Http/ResilientGraphClient.cs @@ -24,7 +24,12 @@ public sealed class ResilientGraphClient : IDisposable private readonly ConcurrentQueue _pendingWarnings = new(); private readonly ConcurrentQueue _pendingDebug = new(); - /// Maximum request body size (4MB). Graph API rejects larger bodies on most endpoints. + /// + /// Maximum request body size (4MB). Bodies on this path are JSON to Graph endpoints, + /// where 4MB is the service's own cap, so refusing early is a favor. The resilience + /// wrap deliberately differs: it carries SDK requests (content uploads run to 250MB) + /// and passes an over-cap body to the SDK untouched instead of refusing it. + /// internal const int MaxRequestBodyBytes = 4 * 1024 * 1024; /// @@ -35,6 +40,27 @@ public sealed class ResilientGraphClient : IDisposable /// public TimeSpan BodyReadTimeout { get; set; } = TimeSpan.FromSeconds(30); + /// + /// Where one request is, as a single word. Entering an attempt and the caller's stop each + /// take it with one exchange, so the two orderings are the only ones that exist: the stop + /// arrives while the request is between attempts and ends the wait, or it finds the attempt + /// already on the wire and leaves it - and its answer - alone. + /// + private static class SendState + { + /// Nothing has gone out. The first attempt is the caller's own to guard. + public const int BeforeFirstAttempt = 0; + + /// An attempt is on the wire, or its answer is still being handed back. + public const int InAttempt = 1; + + /// An attempt is behind it and another may be waiting out a backoff. + public const int BetweenAttempts = 2; + + /// The stop took the request between attempts. Nothing further goes out. + public const int Stopped = 3; + } + private static readonly JsonSerializerOptions JsonOptions = new() { PropertyNameCaseInsensitive = true, @@ -84,6 +110,15 @@ public ResilientGraphClient(HttpClient httpClient, ResilientGraphClientOptions? /// public bool DebugEnabled { get; set; } + /// + /// Runs at the head of each attempt, before it claims the request, and is given the attempt + /// number. Null on every shipping path: the suite holds an attempt here to sit inside the + /// stretch between the retry decision and the claim, where a stop and the attempt about to + /// go out contend for the same word - a window too narrow to enter any other way. Per client + /// rather than static, so a test arms only the transport it built. + /// + internal Func? AttemptEntryGate { get; set; } + /// Drain buffered verbose messages. Must be called on the pipeline thread. public void DrainVerboseMessages() { @@ -97,9 +132,11 @@ public void DrainVerboseMessages() VerboseWriter(msg); } - // For engine components that already report through this client's buffered channel - // (PageIterator runs on the enumeration thread, where a cmdlet cannot WriteWarning). - internal void EnqueueWarning(string message) => _pendingWarnings.Enqueue(message); + // For anything that reports from the enumeration thread, where a cmdlet cannot + // WriteWarning: PageIterator itself, and the page-boundary callbacks the iterator invokes + // on whichever thread it resumed on. Public for the latter - they live in the cmdlets + // assembly - and paired with DrainWarningMessages, which is where the message is written. + public void EnqueueWarning(string message) => _pendingWarnings.Enqueue(message); // Same threading contract as DrainVerboseMessages. public void DrainWarningMessages() @@ -125,6 +162,11 @@ public void DrainDebugMessages() DebugWriter(msg); } + /// + /// POST is the only non-idempotent method in Graph API. + /// GET/PUT/DELETE are idempotent by HTTP spec. + /// PATCH in Graph is always absolute property assignment (not incremental), so it's idempotent. + /// private static bool IsIdempotent(HttpMethod method) => method != HttpMethod.Post; @@ -142,7 +184,8 @@ public async Task SendAsync( int permitCount = 1, bool paceGate = true, bool traceResponseBody = true, - bool redirectIsSuccess = false) + bool redirectIsSuccess = false, + CancellationToken stopRetries = default) { // Buffer content bytes before pipeline so retries reconstruct fresh HttpContent. // Snapshot ALL content headers (not just ContentType) to preserve @@ -160,7 +203,58 @@ public async Task SendAsync( } RateLimitLease? lease = null; - var context = ResilienceContextPool.Shared.Get(cancellationToken); + + // A caller that has stopped sending - a batch chunk refused while its siblings are still + // going. It ends the retry loop without touching the attempt on the wire: the pipeline + // asks the stop before deciding on another attempt, and the token below ends a backoff + // that is already running, so the call comes back instead of waiting out a Retry-After + // it will not act on. Canceling the request itself would take the answer to the attempt + // in flight with it, and that answer is the record of what the server applied - the very + // thing the caller stops in order to keep. So the link is canceled only between + // attempts: never before the first, which has not gone out and is the caller's own to + // guard, and never during one. + // + // Where the request is lives in one word, and the stop and the attempt about to go out + // contend for it with an exchange each. Two words read one after the other cannot say + // where the request is: the pair can be read in a state neither of them was ever in. + var sendState = SendState.BeforeFirstAttempt; + CancellationTokenSource? stopLink = null; + var stopReg = default(CancellationTokenRegistration); + var pipelineToken = cancellationToken; + if (stopRetries.CanBeCanceled) + { + stopLink = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + pipelineToken = stopLink.Token; + stopReg = stopRetries.Register(() => + { + // One exchange decides it. Between attempts the stop takes the request and ends + // the backoff; from any other state the exchange fails and the stop is left to + // the retry predicate, which refuses the next attempt without reaching this one. + if (Interlocked.CompareExchange( + ref sendState, SendState.Stopped, SendState.BetweenAttempts) + == SendState.BetweenAttempts) + { + stopLink.Cancel(); + } + }); + } + + // The waits ahead of the first attempt: the pacer's gate and a permit from the token + // bucket. Either can park a request for seconds, and a caller that stopped sending + // during one of them cannot make the check again - the one it made before calling is + // stale by the time the wait ends, and the request would go out on the strength of it. + // The stop reaches these two waits and no others: past them the request is going out, + // and its answer is the thing the caller stopped in order to keep. + CancellationTokenSource? preSendLink = null; + var preSendToken = cancellationToken; + if (stopRetries.CanBeCanceled) + { + preSendLink = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, stopRetries); + preSendToken = preSendLink.Token; + } + + var context = ResilienceContextPool.Shared.Get(pipelineToken); + context.Properties.Set(ResiliencePipelineFactory.StopRetriesKey, stopRetries); context.Properties.Set(ResiliencePipelineFactory.IsIdempotentKey, IsIdempotent(method)); // Always set VerboseWriterKey to buffer messages (prevents stale writers from pooled contexts) context.Properties.Set(ResiliencePipelineFactory.VerboseWriterKey, @@ -171,6 +265,9 @@ public async Task SendAsync( var totalSw = Stopwatch.StartNew(); bool succeeded = false; + // The last status an attempt got, for a stop that ends the request between attempts: + // the answer is gone by then, but what it said is not. + HttpStatusCode? lastStatus = null; var bucket = AdaptivePacing.Classify(requestUri); try { @@ -179,7 +276,7 @@ public async Task SendAsync( // GraphBatchClient owns batch throughput - but their responses still feed signals. if (paceGate) { - var pacedMs = await AdaptiveRequestPacer.WaitAsync(bucket, cancellationToken); + var pacedMs = await AdaptiveRequestPacer.WaitAsync(bucket, preSendToken); if (pacedMs > 0) { MgxTelemetryCollector.Current.RecordPacingWait(pacedMs); @@ -190,7 +287,7 @@ public async Task SendAsync( if (_rateLimiter != null) { var limiterSw = Stopwatch.StartNew(); - lease = await _rateLimiter.AcquireAsync(permitCount, cancellationToken); + lease = await _rateLimiter.AcquireAsync(permitCount, preSendToken); MgxTelemetryCollector.Current.RecordRateLimiterWait(limiterSw.ElapsedMilliseconds); if (!lease.IsAcquired) throw new InvalidOperationException("Rate limit exceeded. Too many concurrent requests. Reduce -Concurrency on fan-out cmdlets, increase the queue with Set-MgxOption -RateLimitQueueLimit, or disable with Set-MgxOption -NoRateLimit."); @@ -201,6 +298,7 @@ public async Task SendAsync( async ctx => { attempt++; + if (AttemptEntryGate is { } gate) await gate(attempt); var request = new HttpRequestMessage(method, requestUri); if (contentBytes != null) { @@ -213,37 +311,106 @@ public async Task SendAsync( if (headers != null) { foreach (var (key, value) in headers) - request.Headers.TryAddWithoutValidation(key, value); + { + // Content-Length is computed from the actual body; a caller value + // can only disagree with it, and the transport fails the request + // on either direction of the mismatch. + if (string.Equals(key, "Content-Length", StringComparison.OrdinalIgnoreCase)) + { + if (attempt == 1) + _pendingWarnings.Enqueue( + "Header 'Content-Length' is computed from the request body and was not overridden."); + continue; + } + if (request.Headers.TryAddWithoutValidation(key, value)) continue; + // The request collection refuses content headers (Content-Type, + // Content-Disposition, ...); they belong on the content, replacing + // any default the buffered content carried. Remove throws on a + // malformed name where TryAdd merely returns false - a name both + // collections refuse must warn, not crash the request. + if (request.Content != null) + { + try + { + request.Content.Headers.Remove(key); + if (request.Content.Headers.TryAddWithoutValidation(key, value)) continue; + } + catch (Exception ex) when (ex is FormatException or ArgumentException) + { + } + } + if (attempt == 1) + _pendingWarnings.Enqueue($"Header '{key}' is not valid on this request and was not sent."); + } } - request.Headers.TryAddWithoutValidation("SdkVersion", MgxSdkVersion.Value); - request.Headers.TryAddWithoutValidation("client-request-id", clientRequestId); + // A caller-supplied value wins: appending a second one would put two + // values on the wire and defeat the correlation id the caller is logging. + if (headers == null || !headers.ContainsKey("SdkVersion")) + request.Headers.TryAddWithoutValidation("SdkVersion", MgxSdkVersion.Value); + if (headers == null || !headers.ContainsKey("client-request-id")) + request.Headers.TryAddWithoutValidation("client-request-id", clientRequestId); if (DebugEnabled) _pendingDebug.Enqueue(GraphRequestTracer.FormatRequest(request, contentBytes, attempt)); var httpSw = Stopwatch.StartNew(); - // Stream response headers immediately instead of buffering entire body - var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ctx.CancellationToken); - MgxTelemetryCollector.Current.RecordHttpTime(httpSw.ElapsedMilliseconds); - // Per-attempt network time feeds the latency baseline (telemetry-only: - // makes the SPO soft-clamp visible. Not a pacing input in 2.1). - AdaptiveRequestPacer.RecordLatency(bucket, httpSw.ElapsedMilliseconds); - - if (DebugEnabled) + // From here to the answer is the window a stop must not touch: the request + // is going out, and canceling it would lose what the server did with it. + // The window is claimed rather than announced - a stop that took the request + // first keeps it, and this attempt ends here instead of going out on the + // strength of having overwritten the stop. + int seen; + do { - if (traceResponseBody) - { - await TraceResponseAsync(response, httpSw.ElapsedMilliseconds, ctx.CancellationToken); - } - else + seen = Volatile.Read(ref sendState); + // The stop reached the request before this attempt claimed it, so this + // one does not go out. Ended here rather than left to meet the canceled + // token below: the stop takes the word and cancels the link as two + // steps, and between them lies every instruction from here to the send. + // An attempt that read the word and went anyway is on the transport + // before the token arrives - the transport has the request and the + // cancel lands on a send already made - and the caller is then told + // that nothing further was sent over a request that was. + if (seen == SendState.Stopped) + throw new OperationCanceledException(stopRetries); + } + while (Interlocked.CompareExchange(ref sendState, SendState.InAttempt, seen) != seen); + try + { + // Stream response headers immediately instead of buffering entire body + var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ctx.CancellationToken); + lastStatus = response.StatusCode; + MgxTelemetryCollector.Current.RecordHttpTime(httpSw.ElapsedMilliseconds); + // Per-attempt network time feeds the latency baseline (telemetry-only: + // makes the SPO soft-clamp visible; not a pacing input in 2.1). + AdaptiveRequestPacer.RecordLatency(bucket, httpSw.ElapsedMilliseconds); + + if (DebugEnabled) { - // Content path: buffering a multi-megabyte download to trace it - // would defeat the streaming read. Headers-only line instead. - _pendingDebug.Enqueue(GraphRequestTracer.FormatResponse(response, httpSw.ElapsedMilliseconds, null)); + if (traceResponseBody) + { + await TraceResponseAsync(response, httpSw.ElapsedMilliseconds, ctx.CancellationToken); + } + else + { + // Content path: buffering a multi-megabyte download to trace it + // would defeat the streaming read. Headers-only line instead. + _pendingDebug.Enqueue(GraphRequestTracer.FormatResponse(response, httpSw.ElapsedMilliseconds, null)); + } } - } - return response; + return response; + } + finally + { + // One transition out, so there is no instant at which the state says the + // attempt is over while its response is still being handed back unread. + // A stop that landed during the attempt is not resumed here: the retry it + // would have stopped is already refused by the retry predicate, and + // canceling now would take the answer the attempt was kept alive for. + Interlocked.CompareExchange( + ref sendState, SendState.BetweenAttempts, SendState.InAttempt); + } }, context); // A redirect counts as success only for the caller that expects one. Graph answers @@ -275,30 +442,53 @@ public async Task SendAsync( return result; } + // The caller stopped this request's retries while it was waiting to be retried, and the + // wait ended with them. What the server last said is what the request is reported as: a + // retry that never went out does not turn a throttle into a call that got no answer. + catch (OperationCanceledException) when (stopRetries.IsCancellationRequested + && !cancellationToken.IsCancellationRequested) + { + // Nothing of this request went out: the stop reached it while it was still waiting + // for its turn to send. Said in its own words rather than as a refusal - a refusal + // means the server may have applied the writes, and whether the caller sends them + // again turns on exactly that difference. + if (Volatile.Read(ref sendState) == SendState.BeforeFirstAttempt) + throw new RequestNotSentException(); + throw new HttpRequestException(RetriesStoppedMessage, null, lastStatus); + } finally { MgxTelemetryCollector.Current.RecordRequest(succeeded, totalSw.ElapsedMilliseconds); + stopReg.Dispose(); + stopLink?.Dispose(); + preSendLink?.Dispose(); ResilienceContextPool.Shared.Return(context); lease?.Dispose(); } } - /// Send a GET request through the resilience pipeline. + /// + /// Send a GET request through the resilience pipeline. + /// public Task GetAsync( string requestUri, CancellationToken cancellationToken = default, Dictionary? headers = null) => SendAsync(HttpMethod.Get, requestUri, headers: headers, cancellationToken: cancellationToken); - /// Send a POST request through the resilience pipeline. + /// + /// Send a POST request through the resilience pipeline. + /// public Task PostAsync( string requestUri, HttpContent content, CancellationToken cancellationToken = default, Dictionary? headers = null, int permitCount = 1, - bool paceGate = true) - => SendAsync(HttpMethod.Post, requestUri, content, headers, cancellationToken, permitCount, paceGate); + bool paceGate = true, + CancellationToken stopRetries = default) + => SendAsync(HttpMethod.Post, requestUri, content, headers, cancellationToken, permitCount, paceGate, + stopRetries: stopRetries); /// /// Fetch content bytes ($value / /content endpoints), optionally a byte range. @@ -313,7 +503,9 @@ public Task GetContentAsync( CancellationToken cancellationToken = default) => GraphContentClient.GetContentAsync(this, requestUri, range, headers, cancellationToken); - /// Fetch a collection page and deserialize. + /// + /// Fetch a collection page and deserialize. + /// public async Task GetCollectionPageAsync( string requestUri, CancellationToken cancellationToken = default, @@ -330,7 +522,60 @@ public async Task GetCollectionPageAsync( } catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) { - throw new HttpRequestException("Response body read timed out. The server sent headers but the body stream stalled."); + throw new HttpRequestException(BodyReadTimedOutMessage); + } + } + + /// + /// What a caller sees when it stopped this request's retries while it was waiting to be + /// retried. Nothing further was sent; the status the exception carries is the last one the + /// server gave, so a stopped retry does not read as a request that never got an answer. + /// + internal const string RetriesStoppedMessage = + "The caller stopped retrying this request. No further attempt was sent."; + + /// + /// What a caller sees when a server sends headers and then stops sending the body. + /// + internal const string BodyReadTimedOutMessage = + "Response body read timed out. The server sent headers but the body stream stalled."; + + /// + /// Read a response body as bytes under . Requests are sent + /// with HttpCompletionOption.ResponseHeadersRead, and under that option HttpClient.Timeout + /// does not bound the content read at all - nor does the pipeline's attempt timeout, which + /// ends when the headers arrive. A read on the caller's token alone therefore never returns + /// against a stream that stalls without closing. + /// + public async Task ReadBodyAsBytesAsync( + HttpResponseMessage response, CancellationToken cancellationToken = default) + { + using var bodyCts = CreateBodyReadCts(cancellationToken); + try + { + return await response.Content.ReadAsByteArrayAsync(bodyCts.Token); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new HttpRequestException(BodyReadTimedOutMessage); + } + } + + /// + /// Read a response body as text under , for the same reason as + /// . An error body stalls exactly as a success body does. + /// + public async Task ReadBodyAsStringAsync( + HttpResponseMessage response, CancellationToken cancellationToken = default) + { + using var bodyCts = CreateBodyReadCts(cancellationToken); + try + { + return await response.Content.ReadAsStringAsync(bodyCts.Token); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { + throw new HttpRequestException(BodyReadTimedOutMessage); } } @@ -345,13 +590,22 @@ internal CancellationTokenSource CreateBodyReadCts(CancellationToken cancellatio return cts; } + /// + /// Buffer the response body and queue a trace line for it. Buffering is what makes the body + /// readable twice: the caller still reads it normally afterwards. + /// A trace must never break a request, so a failed read degrades to a headers-only line. + /// private async Task TraceResponseAsync(HttpResponseMessage response, long elapsedMs, CancellationToken ct) { string? body = null; try { using var bodyCts = CreateBodyReadCts(ct); - await response.Content.LoadIntoBufferAsync(bodyCts.Token); + // WaitAsync rather than the LoadIntoBufferAsync(CancellationToken) overload: that + // overload is .NET 9+, and taking it here would raise the module's floor from + // PowerShell 7.4 to 7.5 for the sake of one debug-only trace. The buffering itself + // is uncancellable this way, but BodyReadTimeout still bounds the wait. + await response.Content.LoadIntoBufferAsync().WaitAsync(bodyCts.Token); body = await response.Content.ReadAsStringAsync(bodyCts.Token); } catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested) @@ -366,10 +620,30 @@ private async Task ThrowIfGraphErrorAsync(HttpResponseMessage response, Cancella { if (response.IsSuccessStatusCode) return; using var bodyCts = CreateBodyReadCts(ct); - var body = await response.Content.ReadAsStringAsync(bodyCts.Token); + string body; + try + { + body = await response.Content.ReadAsStringAsync(bodyCts.Token); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + // An error body stalls as a success body does, and the caller's contract has no + // room for the raw cancellation: it catches HttpRequestException. + throw new HttpRequestException(BodyReadTimedOutMessage); + } throw new GraphServiceException(response.StatusCode, body); } + /// + /// Log Graph throttle proximity headers to verbose output. + /// These headers are officially documented but conditionally sent by Graph: + /// - x-ms-throttle-limit-percentage: only appears when >80% of throttle budget consumed + /// - x-ms-throttle-scope: typically only on 429 responses (format: Scope/Limit/AppId/TenantId) + /// - x-ms-throttle-information: diagnostic reason on 429 (e.g., CPULimitExceeded, ResourceUnitLimitExceeded) + /// Reliability varies by Graph endpoint. Some workloads never send these headers. + /// Tested against live tenant: headers do not appear at low request volumes (50 req). + /// Only logs when headers are present and VerboseWriter is set. + /// private void LogThrottleHeaders(HttpResponseMessage response) { if (VerboseWriter == null && WarningWriter == null) return; @@ -385,7 +659,7 @@ private void LogThrottleHeaders(HttpResponseMessage response) : null; // Header value is a ratio (0.8 = 80%, 1.2 = 120%). Scale: 0.8-1.8. - // Display as percentage for clarity. Fall back to raw value if unparseable. + // Display as percentage for clarity; fall back to raw value if unparseable. string msg; if (double.TryParse(pctStr, System.Globalization.NumberStyles.Any, System.Globalization.CultureInfo.InvariantCulture, out var pct)) @@ -417,3 +691,18 @@ public void Dispose() // Don't dispose _httpClient either: it's owned by the caller (MgxCmdletBase). } } + +/// +/// A request the caller's stop ended before any attempt of it went out. Distinct from +/// , which ends a request that had +/// already been sent at least once and carries the status the server last gave it: this one +/// carries no status because there is nothing to report, and it is the only shape of stop +/// after which a caller can resubmit without asking whether the write already landed. +/// +internal sealed class RequestNotSentException : Exception +{ + public RequestNotSentException() + : base("The caller stopped sending before this request went out. Nothing was sent.") + { + } +} diff --git a/src/Mgx.Engine/Http/ResilientGraphClientOptions.cs b/src/Mgx.Engine/Http/ResilientGraphClientOptions.cs index b5bf51f..0e41b88 100644 --- a/src/Mgx.Engine/Http/ResilientGraphClientOptions.cs +++ b/src/Mgx.Engine/Http/ResilientGraphClientOptions.cs @@ -126,13 +126,18 @@ public double CircuitBreakerFailureRatio : throw new ArgumentOutOfRangeException(nameof(CircuitBreakerFailureRatio), value, "Must be between 0.01 and 1.0."); } - /// Minimum requests before circuit breaker evaluates. Range: 1-1,000. Default: 40. + /// + /// Minimum requests before circuit breaker evaluates. Range: 2-1,000. Default: 40. + /// The floor is 2 because that is the least Polly's breaker will hold: a lower value is + /// refused when the pipeline is built, one call after the option was set and in Polly's + /// words, so it is refused here instead. + /// public int CircuitBreakerMinThroughput { get => _circuitBreakerMinThroughput; - init => _circuitBreakerMinThroughput = value is > 0 and <= 1000 + init => _circuitBreakerMinThroughput = value is >= 2 and <= 1000 ? value - : throw new ArgumentOutOfRangeException(nameof(CircuitBreakerMinThroughput), value, "Must be between 1 and 1,000."); + : throw new ArgumentOutOfRangeException(nameof(CircuitBreakerMinThroughput), value, "Must be between 2 and 1,000."); } /// Circuit breaker sampling window in seconds. Range: 5-300. Default: 30. diff --git a/src/Mgx.Engine/Http/SensitiveNames.cs b/src/Mgx.Engine/Http/SensitiveNames.cs new file mode 100644 index 0000000..b637539 --- /dev/null +++ b/src/Mgx.Engine/Http/SensitiveNames.cs @@ -0,0 +1,105 @@ +using System.Text.RegularExpressions; + +namespace Mgx.Engine.Http; + +/// +/// The name fragments that mark a value as a credential, matched case-insensitively as +/// substrings. Graph spells its secrets differently on nearly every resource - newPassword on +/// an administrative reset, preSharedKey and sharedSecret on Wi-Fi and VPN profiles, +/// uploadSecret on a trust framework key set - so a list of exact names is behind the surface +/// the day it is written. Substrings over-redact, and that is the cheaper error: a redacted +/// diagnostic field costs a line of context, an unredacted PSK is a credential sitting in a +/// file. +/// +/// One definition, two readers - and three rules, not one. The fragments are the first. A +/// property whose name carries "downloadurl" and a string value that is a URL carrying one of +/// the capability parameters are the other two, and they are here for the same reason: they say +/// credential without saying password, and each reader needs the same answer from them. +/// GraphRequestTracer applies all three to -Debug output as text; Invoke-MgxBatchRequest applies +/// all three to a dead-letter body as parsed JSON, and the fragments again to a request path. +/// +/// The value rule agrees on the form it is written in, since a URL is text either way, and on +/// both forms it comes in: a $batch item's own url is on the wire with its scheme and host +/// stripped, and a caller who was handed a relative URL by Graph passes it on as one, so a path +/// carrying a capability parameter is read as such by GraphRequestTracer and by +/// RedactCapabilityUrl below - which the file reads a body's keys and values through - alike. +/// The two name rules agree only where the value beside the named key is itself a quoted string +/// - all a trace regex can match. +/// Invoke-MgxBatchRequest clears that value whatever it holds, so a number, a bool, an array or +/// an object beside a password or a downloadUrl field is missing from the file and still stands +/// whole in a trace. +/// +public static class SensitiveNames +{ + private static readonly string[] TokenList = + [ + "password", "secret", "credential", "key", "token", "assertion", "passphrase", + "connectionstring" + ]; + + /// The fragments, lowercase. Compare case-insensitively. + public static IReadOnlyList Tokens => TokenList; + + /// True when a field name or URL path segment carries one of the fragments. + public static bool Matches(string name) + { + foreach (var token in TokenList) + { + if (name.Contains(token, StringComparison.OrdinalIgnoreCase)) + return true; + } + return false; + } + + /// + /// The fragments as a regex alternation, for a caller matching names inside text rather + /// than one name at a time. + /// + internal static string RegexAlternation { get; } = + string.Join('|', TokenList.Select(Regex.Escape)); + + /// + /// The name fragment that says a property's value is a pre-authenticated URL even when the + /// value exposes no parameter to recognize it by - the short-form download hosts put the + /// capability in the path, so "@microsoft.graph.downloadUrl" and its older spelling + /// "@content.downloadUrl" are credentials under a name no fragment above reaches. + /// + internal const string DownloadUrlFragment = "downloadurl"; + + /// + /// The query parameters that carry a pre-authenticated URL's capability, as a regex + /// alternation rather than a list of names because `sig` needs a lookbehind to be read as a + /// parameter of its own. Deliberately not every URL: @odata.nextLink is a URL too, and + /// losing it would remove the thing paging bugs are diagnosed with. + /// + internal const string CapabilityUrlParameters = + "tempauth|guestaccesstoken|authkey|X-Amz-Signature|(?True when a property name says its value is a pre-authenticated URL. + public static bool NamesDownloadUrl(string name) => + name.Contains(DownloadUrlFragment, StringComparison.OrdinalIgnoreCase); + + /// + /// The capability-parameter rule on one unescaped value, for the reader holding parsed JSON + /// rather than text. Returns the value cut after the first capability parameter's `=` with + /// in place of the rest - the shape the tracer's replacement + /// makes, which keeps the parameter name readable so it is still clear which kind of URL + /// went - or null when the value carries no capability parameter and may be kept whole. + /// + /// The pattern differs from the tracer's only in what bounds the value: a quote ends a + /// string in the JSON text the tracer reads, and bounds nothing in a value that has already + /// been unescaped. The parameters both of them test are the definition above, and so are the + /// two forms a URL arrives in - with its scheme, or with a leading slash standing in for one, + /// which is what says the string is a path rather than prose that happens to carry a + /// parameter. + /// + public static string? RedactCapabilityUrl(string value, string marker) + { + var match = CapabilityUrl.Match(value); + return match.Success ? match.Groups[1].Value + marker : null; + } + + private static readonly Regex CapabilityUrl = new( + $"^((?:https?://|/).*?(?:{CapabilityUrlParameters})=)", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled); +} diff --git a/src/Mgx.Engine/Mgx.Engine.csproj b/src/Mgx.Engine/Mgx.Engine.csproj index 48fc256..56c733e 100644 --- a/src/Mgx.Engine/Mgx.Engine.csproj +++ b/src/Mgx.Engine/Mgx.Engine.csproj @@ -8,7 +8,7 @@ + into a build failure --> $(WarningsAsErrors);CA1416 diff --git a/src/Mgx.Engine/Models/GraphResponse.cs b/src/Mgx.Engine/Models/GraphResponse.cs index d2762a7..d99eaf8 100644 --- a/src/Mgx.Engine/Models/GraphResponse.cs +++ b/src/Mgx.Engine/Models/GraphResponse.cs @@ -80,6 +80,13 @@ public sealed class GraphBatchResponseItem [JsonPropertyName("id")] public string Id { get; set; } = string.Empty; + /// + /// The status the server answered this item with. The deserializer leaves it at 0 for a + /// sub-response carrying no "status" member at all, and 0 is + /// - so a value here is a status only after + /// has checked the envelope actually supplied one. Nothing + /// else may read a 0 on a deserialized item as "the request was never sent". + /// [JsonPropertyName("status")] public int Status { get; set; } diff --git a/src/Mgx.Engine/Models/GraphServiceException.cs b/src/Mgx.Engine/Models/GraphServiceException.cs index 22304b3..0da2e4c 100644 --- a/src/Mgx.Engine/Models/GraphServiceException.cs +++ b/src/Mgx.Engine/Models/GraphServiceException.cs @@ -12,16 +12,20 @@ public class GraphServiceException : Exception public HttpStatusCode StatusCode { get; } public string? ErrorCode { get; } + public string? InnerErrorMessage { get; } + public GraphServiceException(HttpStatusCode statusCode, string responseBody) - : base(FormatAndExtract(statusCode, responseBody, out var code)) + : base(FormatAndExtract(statusCode, responseBody, out var code, out var innerMessage)) { StatusCode = statusCode; ErrorCode = code; + InnerErrorMessage = innerMessage; } - private static string FormatAndExtract(HttpStatusCode statusCode, string responseBody, out string? errorCode) + private static string FormatAndExtract(HttpStatusCode statusCode, string responseBody, out string? errorCode, out string? innerErrorMessage) { errorCode = null; + innerErrorMessage = null; if (string.IsNullOrEmpty(responseBody)) return $"HTTP {(int)statusCode}: {statusCode}"; @@ -44,6 +48,7 @@ private static string FormatAndExtract(HttpStatusCode statusCode, string respons var code = AsString(errorObj, "code"); var message = AsString(errorObj, "message"); errorCode = code; + innerErrorMessage = ExtractInnerErrorMessage(errorObj); // Build formatted message from whatever Graph provided var formatted = !string.IsNullOrEmpty(code) @@ -52,6 +57,12 @@ private static string FormatAndExtract(HttpStatusCode statusCode, string respons ? message : $"HTTP {(int)statusCode}: {statusCode}"; + if (!string.IsNullOrEmpty(innerErrorMessage) + && !string.Equals(innerErrorMessage, message, StringComparison.Ordinal)) + { + formatted += $"\n{innerErrorMessage}"; + } + var guidance = GetGuidanceForCode(code); if (guidance != null) formatted += $"\nHint: {guidance}"; @@ -66,6 +77,30 @@ private static string FormatAndExtract(HttpStatusCode statusCode, string respons return $"HTTP {(int)statusCode}: {statusCode}"; } + private static string? ExtractInnerErrorMessage(JsonElement errorObj) + { + string? message = null; + var current = errorObj; + + for (var depth = 0; depth < 8; depth++) + { + if (!current.TryGetProperty("innerError", out var inner) || inner.ValueKind != JsonValueKind.Object) + { + break; + } + + var innerMessage = AsString(inner, "message"); + if (!string.IsNullOrWhiteSpace(innerMessage)) + { + message = innerMessage.Trim(); + } + + current = inner; + } + + return message; + } + private static string? AsString(JsonElement obj, string name) { if (!obj.TryGetProperty(name, out var v)) return null; diff --git a/src/Mgx.Engine/Pagination/AtomicFile.cs b/src/Mgx.Engine/Pagination/AtomicFile.cs deleted file mode 100644 index 8c4675d..0000000 --- a/src/Mgx.Engine/Pagination/AtomicFile.cs +++ /dev/null @@ -1,31 +0,0 @@ -namespace Mgx.Engine.Pagination; - -/// Shared file primitives for the state and checkpoint writers. -internal static class AtomicFile -{ - private const int MoveAttempts = 5; - private const int MoveRetryDelayMs = 20; - - /// - /// Replace with , retrying briefly on - /// a sharing violation. On Windows the replace fails while anything else holds the - /// destination open, including an indexer or a scanner that opened the file this process - /// just wrote, so a first failure says nothing about whether the move can succeed. - /// - internal static void Replace(string source, string destination) - { - for (var attempt = 1; ; attempt++) - { - try - { - File.Move(source, destination, overwrite: true); - return; - } - catch (Exception ex) when (ex is IOException or UnauthorizedAccessException - && attempt < MoveAttempts) - { - Thread.Sleep(MoveRetryDelayMs); - } - } - } -} diff --git a/src/Mgx.Engine/Pagination/ConcurrentFanOut.cs b/src/Mgx.Engine/Pagination/ConcurrentFanOut.cs index 25c8a7d..1237277 100644 --- a/src/Mgx.Engine/Pagination/ConcurrentFanOut.cs +++ b/src/Mgx.Engine/Pagination/ConcurrentFanOut.cs @@ -197,6 +197,10 @@ public async Task BulkWriteAsync( var tasks = operations.Select(async op => { bool acquired = false; + // The status this operation was answered with, for a failure that happens after + // the answer arrived. 0 until the server answers: the code for a write that never + // reached one. + int answered = 0; try { await semaphore.WaitAsync(cancellationToken); @@ -209,20 +213,23 @@ public async Task BulkWriteAsync( try { using var response = await _client.SendAsync(method, op.url, content, headers, cancellationToken); + answered = (int)response.StatusCode; if (!response.IsSuccessStatusCode) { - string body; + string errorMessage; using var errCts = _client.CreateBodyReadCts(cancellationToken); try { - body = await response.Content.ReadAsStringAsync(errCts.Token); + var body = await response.Content.ReadAsStringAsync(errCts.Token); + errorMessage = TryExtractGraphError(body) ?? $"HTTP {(int)response.StatusCode}"; } catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) { - body = $"Response body read timed out (HTTP {(int)response.StatusCode})"; + // Reported as it stands: a stall is not a Graph error document, and + // running it through TryExtractGraphError leaves a bare status line. + errorMessage = ResilientGraphClient.BodyReadTimedOutMessage; } - var errorMessage = TryExtractGraphError(body) ?? $"HTTP {(int)response.StatusCode}"; errorBag.Add(new BulkWriteError(op.id, (int)response.StatusCode, errorMessage)); var errCurrent = Volatile.Read(ref succeeded) + errorBag.Count; @@ -233,10 +240,16 @@ public async Task BulkWriteAsync( // Capture response body for POST/PATCH (created/updated entities) if (response.StatusCode != HttpStatusCode.NoContent) { - var bodyBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken); + var bodyBytes = await _client.ReadBodyAsBytesAsync(response, cancellationToken); if (bodyBytes.Length > 0) { - var json = JsonSerializer.Deserialize(bodyBytes); + // Read as a stream, which skips a UTF-8 byte-order mark - the way + // the batch client already reads its bodies. The byte-level overload + // refuses one, and refused it as a failure of the write: a 201 the + // server had applied came back as a status-0 transport error. + using var bodyStream = new MemoryStream(bodyBytes); + var json = await JsonSerializer.DeserializeAsync( + bodyStream, cancellationToken: cancellationToken); responseBag.Add((op.id, json)); } } @@ -259,7 +272,12 @@ public async Task BulkWriteAsync( } catch (Exception ex) { - var statusCode = ex is GraphServiceException gse ? (int)gse.StatusCode : 0; + // A body that stalls or does not parse is a failure of the read, not of the + // write: the server answered, and on a POST it may have applied it. The record + // carries the status it answered with and names the body problem, so a caller + // cannot mistake an applied write for a request that never reached the service + // and repeat it. Only a write with no answer at all is a 0. + var statusCode = ex is GraphServiceException gse ? (int)gse.StatusCode : answered; errorBag.Add(new BulkWriteError(op.id, statusCode, ex.Message)); var current = Volatile.Read(ref succeeded) + errorBag.Count; diff --git a/src/Mgx.Engine/Pagination/DeltaState.cs b/src/Mgx.Engine/Pagination/DeltaState.cs index b129ebc..0ca4f25 100644 --- a/src/Mgx.Engine/Pagination/DeltaState.cs +++ b/src/Mgx.Engine/Pagination/DeltaState.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using System.Text; using System.Text.Json; using System.Text.Json.Serialization; @@ -84,7 +85,9 @@ public static (DeltaState? State, DeltaLoadResult Result) LoadWithResult(string } } - /// Backward-compatible Load. Returns null for both "not found" and "corrupt". + /// + /// Backward-compatible Load. Returns null for both "not found" and "corrupt". + /// public static DeltaState? Load(string path) => LoadWithResult(path).State; /// @@ -102,12 +105,22 @@ public void Save(string path) var tmpPath = normalizedPath + ".tmp"; try { - File.WriteAllText(tmpPath, json); - AtomicFile.Replace(tmpPath, normalizedPath); + // Cleared and then created, rather than written into whatever stands at the + // name: an open that creates or truncates follows a symlink there and fills or + // truncates the link's target with this state, and a FIFO with no reader blocks + // the write forever - past a cancellation, since nothing on the pipeline thread + // can reach a blocked open. + using (var scratch = ScratchName.Create(tmpPath, "the delta state's staging file")) + using (var writer = new StreamWriter(scratch, new UTF8Encoding(false, true), + 1024, leaveOpen: true)) + { + writer.Write(json); + } + File.Move(tmpPath, normalizedPath, overwrite: true); } catch { - // The staging file is not the state. Leaving it behind only invites a later run + // The staging file is not the state; leaving it behind only invites a later run // to wonder what it is. try { if (File.Exists(tmpPath)) File.Delete(tmpPath); } catch { } throw; @@ -119,7 +132,9 @@ public void Save(string path) /// Delete delta state file and temp file. Acquires per-path lock to avoid /// racing with concurrent Save operations. /// - /// Returns true if the file was deleted (or didn't exist), false if deletion failed. + /// + /// Returns true if the file was deleted (or didn't exist), false if deletion failed. + /// public static bool Delete(string path) { var normalizedPath = Path.GetFullPath(path); @@ -141,10 +156,16 @@ public static bool Delete(string path) } /// - /// Validates write access to the delta file path. Call in BeginProcessing - /// to fail fast before any HTTP calls. + /// Validates write access to a path the run will write to. Call in BeginProcessing to fail + /// fast before any HTTP calls. /// - public static void ValidateWriteAccess(string path) + /// The file whose directory has to be writable. + /// + /// What that file is, for the failure to name. The sync probes two of them, and one noun for + /// both reported an -OutputFile directory it could not write to as the delta state path - + /// sending the caller to a file that had just passed the same probe. + /// + public static void ValidateWriteAccess(string path, string what = "delta state path") { // NOTE: a directory passes the probe below, because the probe writes ".probe" // NEXT to the target rather than to it, so the fail-fast check succeeds and the real @@ -157,16 +178,62 @@ public static void ValidateWriteAccess(string path) if (dir != null && !Directory.Exists(dir)) Directory.CreateDirectory(dir); + // The probe name is cleared before it is created, and created by an open that refuses + // to use anything already standing there. Written with Create/Write, the probe followed + // a symlink at that name and truncated the link's target to nothing - a file outside + // this directory, taken away by a check whose whole purpose is to touch nothing - and a + // FIFO left there blocked the open in BeginProcessing, before the run had anything a + // cancellation could unwind. var probe = path + ".probe"; + var swept = ScratchName.Sweep(probe, out var stood); + + // A file another run holds at that name is that run's own probe, taken out a moment ago + // in this very directory - which is the question this one is asking. Two runs over one + // -DeltaPath, or one -OutputFile, reach here together as a matter of course, and + // refusing either of them for the other's sake would end a run over nothing that is + // wrong with the path. The two refusals left say something about the path itself: a + // directory at the name, or an entry the directory will not let go of. + if (swept == ScratchEntry.Held) return; + if (swept.Refused()) + { + throw new InvalidOperationException( + $"Cannot write to {what} '{path}': {swept.WhatStood(probe, stood)}.", stood); + } + + FileStream created; try { - File.WriteAllText(probe, ""); - File.Delete(probe); + created = ScratchName.CreateNew(probe); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { + // Another run reached the name first, which on this name is its probe: a create + // refuses a name that already exists, and a create that meets the claim the other + // run is holding that name by is refused as a sharing violation. Either way it is + // the answer the sweep above gives a held probe, for the reason it gives it. + // + // Read off the refusal as well as off the name, because the entry that refuses this + // create is one the run that made it is about to take away: asked of the name alone, + // the question arrives after the answer has gone. Two syncs released together over + // one -DeltaPath had the second create refused by the first one's probe, found the + // name clear an instruction later, and ended in BeginProcessing naming a path + // nothing was wrong with. + // + // A directory is not that. It is the path's own refusal, it is not this run's to + // remove, and it is still standing to be asked about - which is asked of the + // filesystem, since a create meets one as an access failure on Windows and as a + // name that already exists on Unix. Neither is a create the directory refused + // outright, which is the failure this whole probe exists to reach. + var reachedFirst = File.Exists(probe) + || (ex is IOException refused + && (ScratchName.IsAlreadyExists(refused) + || ScratchName.IsSharingViolation(refused))); + if (reachedFirst && !Directory.Exists(probe)) return; throw new InvalidOperationException( - $"Cannot write to delta state path '{path}': {ex.Message}", ex); + $"Cannot write to {what} '{path}': {ex.Message}", ex); } + + // Unlinked under the handle that holds it, so the name is never decided and unheld. + ScratchName.Discard(probe, created); } } diff --git a/src/Mgx.Engine/Pagination/HeldEntry.cs b/src/Mgx.Engine/Pagination/HeldEntry.cs new file mode 100644 index 0000000..e13bc62 --- /dev/null +++ b/src/Mgx.Engine/Pagination/HeldEntry.cs @@ -0,0 +1,301 @@ +using System.Runtime.InteropServices; + +namespace Mgx.Engine.Pagination; + +/// +/// Whether the file a run holds is still the file standing at the name it asked for. A claim +/// taken on a name is finished here, and nowhere else. +/// +/// .NET backs on Unix with a non-blocking flock taken AFTER +/// open(2), so the two halves of a claim are not one instruction: the handle binds to +/// whatever inode the name resolved to at the open, and the exclusion over it is granted several +/// syscalls later. A run descheduled between them comes back holding an inode a second run +/// renamed its own file over in the interval - rename(2) honors no lock on either side - +/// and the lock is then granted over a file with no name left on it. Nothing about that handle +/// says so: it reads its own length, it takes appends, and the claim it stands for was over a +/// name that is now somebody else's. Two runs promoting over one output both answered Taken that +/// way, and the second renamed its copy over the first one's rows. +/// +/// +/// So the entry now at the name has to be the inode the handle holds, and that inode has to +/// still have a name. Both are asked of the platform after the lock is granted, and a caller +/// that cannot answer yes drops the handle and asks the name again rather than acting on it. +/// +/// +/// What that reaches is the writers that take a claim: a promotion's hold on the output it +/// renames onto and on the staging copy it renames from, the sweep of a scratch name, the hold +/// a resumed run keeps on the output it appends to, and the files a promotion takes back when it +/// is done - the temp it spent, which goes only where the promotion landed, and the staged copy +/// it wrote and the empty output it made in order to hold one, which go only where it stopped. +/// Between those, no run renames onto or unlinks a name whose current inode another run holds. +/// +/// +/// A writer that takes no claim is refused by none of it. This is a rule between claims, and +/// rename and unlink honor no lock in the first place: a plain export's final move onto its +/// output, the atomic saves of the checkpoint and the delta state - which let the staging claim +/// go before the rename that spends it - an older mgx, another tool, a shell. Each of those +/// replaces a name while a run holds what stands at it, and the run holding it goes on writing +/// into an inode that has left the directory. What the rule holds apart is two claims; it does +/// not hold the name still. +/// +/// +/// Windows needs none of it. The open there IS the exclusion, and the kernel refuses a rename +/// onto a name another handle has open and a rename of one too, so the question cannot come up +/// and every caller keeps the answer it had. +/// +/// +public static class HeldEntry +{ + /// + /// How many times a claim is asked again when the name moved under it before the caller is + /// told the name is held. Each attempt is a fresh open, so a re-ask over a name a second run + /// has taken meets that run's own lock and answers Held on its own; what the attempts are + /// for is the other outcome, a name whose entry was replaced by a run that has since let go, + /// where the next open is the one that succeeds. Three, because a run has to be descheduled + /// between an open and a lock to lose one at all, and nothing waits between them: a name + /// that moves under three consecutive claims is a name being mutated right now, which is + /// what a hold means to every caller here. + /// + public const int ClaimAttempts = 3; + + /// + /// Test seam: run in the instant between a claim being granted and the check that the name + /// still resolves to it, with the path that was claimed. That instant is the whole defect + /// and no test can hit it by racing for it, so it is fired here instead. Null on every + /// shipping path; internal, assigned directly rather than reflected, and reset by the test + /// that set it. + /// + internal static Action? BeforeTheClaimIsVerified; + + /// + /// Whether is the entry standing at : the + /// same inode on the same device, with a name still on it. False says the name moved under + /// the handle - unlinked, renamed away, or replaced by another run's file - and a caller + /// that acts on the name anyway acts on a file that is not the one it claimed. + /// + /// The fd is read off , and reading it hands the file whatever that + /// stream still has buffered to write. So a claim is verified with the buffer written out, + /// and a caller that reaches this with rows waiting in one pays for that write here and + /// meets its failure here rather than where it writes. One route does: a promotion whose + /// copy into the staged file fails part way through returns without reaching the flush that + /// follows the loop, and the discard in its finally verifies that writer - so the rows still + /// in the buffer are written out here, and a failure of that write is met here, in a discard + /// whose caller has already answered. The discard swallows that failure; the dispose that + /// follows it re-attempts the same write and lets the failure out, to the promotion's own + /// catch. Every other caller reaches this with nothing waiting, or with the buffer flushed an + /// instruction earlier. + /// + /// + /// + /// The question could not be answered: an entry point that is not there on this host, or a + /// failure from the call that is not the name resolving to nothing. Raised naming the call + /// and the errno rather than answered "still there", because a claim that cannot be verified + /// is not a claim, and every caller here reads a raised IOException as a file it has to + /// leave alone. + /// + public static bool StillStandsAt(FileStream held, string path) + { + BeforeTheClaimIsVerified?.Invoke(path); + + // The open is the exclusion on Windows and the kernel keeps the name still under it. + if (OperatingSystem.IsWindows()) return true; + + // The handle first, so its link count is read before the name is looked at: an entry + // that answers for the name afterwards is one this handle held while it still had a + // name, and a rename landing between the two readings takes this inode's last name with + // it and is read here. + var mine = OfTheHandle(held, path); + if (mine.Links == 0) return false; + + var there = AtTheName(path); + return there is { } entry && entry.Device == mine.Device && entry.Inode == mine.Inode; + } + + /// What a stat answered: the file's identity, and how many names it has. + private readonly record struct Identity(long Device, ulong Inode, ulong Links); + + private static Identity OfTheHandle(FileStream held, string path) + { + var handle = held.SafeFileHandle; + var counted = false; + try + { + // The fd, held for as long as the call takes: a handle disposed on another thread in + // the middle of it would otherwise be a stat of whatever fd the runtime handed that + // number out to next. + handle.DangerousAddRef(ref counted); + var fd = (int)handle.DangerousGetHandle(); + var buffer = new byte[StatBufferBytes]; + int answer; + try + { + answer = OperatingSystem.IsMacOS() + ? (MacPlainNamesAreIno64 ? MacFstat(fd, buffer) : MacFstatIno64(fd, buffer)) + : LinuxStatxOfFd(fd, string.Empty, AtEmptyPath, StatxWanted, buffer); + } + catch (EntryPointNotFoundException ex) { throw NotOnThisHost(FdCall, path, ex); } + + if (answer != 0) throw Failed(FdCall, path, Marshal.GetLastPInvokeError()); + return OperatingSystem.IsMacOS() ? MacIdentity(buffer) : LinuxIdentity(buffer, path); + } + finally + { + if (counted) handle.DangerousRelease(); + } + } + + /// + /// What stands at the name, or null where the name resolves to nothing. Symlinks are + /// followed, because the open that took the claim followed them too: the question is whether + /// this handle is what that same name reaches now. + /// + private static Identity? AtTheName(string path) + { + var buffer = new byte[StatBufferBytes]; + int answer; + try + { + answer = OperatingSystem.IsMacOS() + ? (MacPlainNamesAreIno64 ? MacStat(path, buffer) : MacStatIno64(path, buffer)) + : LinuxStatx(AtFdCwd, path, 0, StatxWanted, buffer); + } + catch (EntryPointNotFoundException ex) { throw NotOnThisHost(NameCall, path, ex); } + + if (answer == 0) + { + return OperatingSystem.IsMacOS() ? MacIdentity(buffer) : LinuxIdentity(buffer, path); + } + + var errno = Marshal.GetLastPInvokeError(); + if (NameResolvesToNothing(errno)) return null; + throw Failed(NameCall, path, errno); + } + + /// + /// The call each half is asked with, for the sentence a failure is raised in. + /// + private static string FdCall => OperatingSystem.IsMacOS() ? "fstat(2)" : "statx(2)"; + + /// + private static string NameCall => OperatingSystem.IsMacOS() ? "stat(2)" : "statx(2)"; + + /// + /// ENOENT and ENOTDIR, the two failures that say the name reaches no entry rather than that + /// the question failed - a name that has gone, and a name whose parent is no longer a + /// directory. Both numbers are the same on macOS and on Linux. Every other failure is + /// raised: an answer this cannot read is not an answer that the file is still there. + /// + private static bool NameResolvesToNothing(int errno) => errno is 2 or 20; + + /// + /// A libc without the symbol is a host this cannot verify a claim on. Raised rather than + /// answered "still there", which is the silence this whole file exists to remove. + /// + private static IOException NotOnThisHost(string named, string path, Exception ex) => + new($"{named} is not available on this host, so the claim on '{path}' cannot be " + + "verified.", ex); + + /// + private static IOException Failed(string named, string path, int errno) => + new($"{named} on '{path}' failed with errno {errno}, so the claim on it cannot be " + + "verified."); + + // --- macOS ------------------------------------------------------------------------------- + // + // struct stat with 64-bit inodes, which is the only shape either supported architecture + // reaches: st_dev int32 at 0, st_mode uint16 at 4, st_nlink uint16 at 6, st_ino uint64 at 8. + // The entry point differs by architecture and the struct does not. arm64 has never had the + // 32-bit-inode form, so the plain names ARE the 64-bit ones there; x86_64 carries both, and + // the plain names there are the old shape - a different struct, whose st_ino is 32 bits at + // offset 4 - so the "$INODE64" symbols are the ones asked for. This is what + // does to every stat call at compile time (__DARWIN_INODE64, suffixed on x86_64 and empty + // on arm64); nothing here can be compiled through that header, so the choice is made once, + // at runtime, off the architecture the process is actually running as - which is x64 under + // Rosetta, where the x64 libSystem is the one loaded. + + private static bool MacPlainNamesAreIno64 => + RuntimeInformation.ProcessArchitecture != Architecture.X64; + + private static Identity MacIdentity(byte[] buffer) => new( + BitConverter.ToInt32(buffer, 0), + BitConverter.ToUInt64(buffer, 8), + BitConverter.ToUInt16(buffer, 6)); + + [DllImport(Libc, EntryPoint = "fstat", SetLastError = true)] + private static extern int MacFstat(int fd, byte[] buffer); + + [DllImport(Libc, EntryPoint = "stat", SetLastError = true)] + private static extern int MacStat([MarshalAs(UnmanagedType.LPUTF8Str)] string path, + byte[] buffer); + + [DllImport(Libc, EntryPoint = "fstat$INODE64", SetLastError = true)] + private static extern int MacFstatIno64(int fd, byte[] buffer); + + [DllImport(Libc, EntryPoint = "stat$INODE64", SetLastError = true)] + private static extern int MacStatIno64([MarshalAs(UnmanagedType.LPUTF8Str)] string path, + byte[] buffer); + + // --- Linux ------------------------------------------------------------------------------- + // + // statx, and not stat: glibc exported no plain "stat" symbol before 2.33 - the header turned + // it into __xstat with a struct version argument - and struct stat's own layout is per + // architecture, so a fixed set of offsets would be a different struct on x86_64, arm64 and + // arm. struct statx is one shape everywhere and its offsets are the kernel's ABI: + // stx_nlink uint32 at 16, stx_ino uint64 at 32, and stx_dev_major / stx_dev_minor uint32 at + // 136 / 140, past four 16-byte timestamps and the two rdev halves at 128 / 132. The wrapper + // has been in glibc since 2.28 and in musl since 1.2, which is under every distribution + // .NET 8 supports; a host without it raises rather than answering. + + private const int AtFdCwd = -100; + private const int AtEmptyPath = 0x1000; + + /// STATX_NLINK (0x4) and STATX_INO (0x100), the two fields read out of it. + private const uint StatxWanted = 0x4 | 0x100; + + private static Identity LinuxIdentity(byte[] buffer, string path) + { + // What the kernel actually filled in. Asking for a field is not being given it - a + // filesystem that cannot answer clears its bit - and reading a field that was never + // written would compare this run's claim against a zero. + var answered = BitConverter.ToUInt32(buffer, 0); + if ((answered & StatxWanted) != StatxWanted) + { + throw new IOException( + $"statx(2) on '{path}' answered without the inode or the link count, so the " + + "claim on it cannot be verified."); + } + var major = BitConverter.ToUInt32(buffer, 136); + var minor = BitConverter.ToUInt32(buffer, 140); + return new Identity( + ((long)major << 32) | minor, + BitConverter.ToUInt64(buffer, 32), + BitConverter.ToUInt32(buffer, 16)); + } + + [DllImport(Libc, EntryPoint = "statx", SetLastError = true)] + private static extern int LinuxStatx(int dirFd, + [MarshalAs(UnmanagedType.LPUTF8Str)] string path, int flags, uint mask, byte[] buffer); + + /// + /// + /// The same call over an open file: the fd in the directory slot, an empty path, and + /// AT_EMPTY_PATH to say the fd is the file rather than the directory to resolve in. + /// + [DllImport(Libc, EntryPoint = "statx", SetLastError = true)] + private static extern int LinuxStatxOfFd(int fd, + [MarshalAs(UnmanagedType.LPUTF8Str)] string path, int flags, uint mask, byte[] buffer); + + /// + /// libSystem on macOS and the C library on Linux, under the name .NET resolves on both. + /// + private const string Libc = "libc"; + + /// + /// Room for either struct: 144 bytes for macOS's stat with 64-bit inodes, and 256 for + /// Linux's statx, which is 256 bytes by the kernel's ABI - the spare fields at its tail are + /// part of that size, and a field a later kernel adds is one of those spares being given a + /// name. One size for both, since the fields read out of each are at the offsets above and + /// nothing here walks off the end of what the call wrote. + /// + private const int StatBufferBytes = 256; +} diff --git a/src/Mgx.Engine/Pagination/PaginationCheckpoint.cs b/src/Mgx.Engine/Pagination/PaginationCheckpoint.cs index 0a68d29..53653e3 100644 --- a/src/Mgx.Engine/Pagination/PaginationCheckpoint.cs +++ b/src/Mgx.Engine/Pagination/PaginationCheckpoint.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using System.Text; using System.Text.Json; using System.Text.Json.Serialization; @@ -6,7 +7,7 @@ namespace Mgx.Engine.Pagination; /// /// Checkpoint state for resumable pagination. -/// Saved as JSON after each page. Auto-deleted on successful completion. +/// Saved as JSON after each page; auto-deleted on successful completion. /// Uses atomic write (write to .tmp, then rename) to prevent corruption. /// public sealed class PaginationCheckpoint @@ -37,6 +38,17 @@ public sealed class PaginationCheckpoint [JsonPropertyName("tempFile")] public string? TempFile { get; set; } + /// + /// Path of the output the writing run was collecting into. DataLength is a byte offset into + /// "the output" and nothing else here says WHICH one, so a checkpoint path used by two runs + /// let one run's offset cut the other run's file. The whole path and not the name: two + /// exports to "users.jsonl" in different directories share a name and share nothing else. + /// Null on checkpoints written before this was recorded, which say nothing about ownership + /// and are read as evidence of it only where the files on disk corroborate them. + /// + [JsonPropertyName("outputFile")] + public string? OutputFile { get; set; } + /// /// Byte length of that file at the moment this checkpoint was saved, captured after the /// writer was flushed. Null on checkpoints written before this was recorded, which are @@ -56,7 +68,9 @@ public sealed class PaginationCheckpoint // Per-path lock prevents concurrent runspaces from corrupting the same checkpoint (RD-H7) private static readonly ConcurrentDictionary s_pathLocks = new(StringComparer.OrdinalIgnoreCase); - /// Load a checkpoint from disk. Returns null if the file doesn't exist or is corrupt. + /// + /// Load a checkpoint from disk. Returns null if the file doesn't exist or is corrupt. + /// public static PaginationCheckpoint? Load(string path) { if (!File.Exists(path)) return null; @@ -68,12 +82,12 @@ public sealed class PaginationCheckpoint catch (JsonException) { // Corrupt checkpoint file (e.g., partial write from crash). - // Treat as no checkpoint. Caller will start fresh. + // Treat as no checkpoint; caller will start fresh. return null; } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { - // Locked, or the account cannot read it. Treat as no checkpoint either way. Windows + // Locked, or the account cannot read it; treat as no checkpoint either way. Windows // reports a denying ACL as UnauthorizedAccessException, which is not an IOException, // so catching only the latter made an unreadable checkpoint throw on Windows and // resume cleanly everywhere else. @@ -97,8 +111,18 @@ public void Save(string path) var tmpPath = normalizedPath + ".tmp"; try { - File.WriteAllText(tmpPath, json); - AtomicFile.Replace(tmpPath, normalizedPath); + // Cleared and then created, rather than written into whatever stands at the + // name: an open that creates or truncates follows a symlink there and fills or + // truncates the link's target with this checkpoint, and a FIFO with no reader + // blocks the write forever - past a cancellation, since nothing on the pipeline + // thread can reach a blocked open. + using (var scratch = ScratchName.Create(tmpPath, "the checkpoint's staging file")) + using (var writer = new StreamWriter(scratch, new UTF8Encoding(false, true), + 1024, leaveOpen: true)) + { + writer.Write(json); + } + File.Move(tmpPath, normalizedPath, overwrite: true); } catch { @@ -110,7 +134,9 @@ public void Save(string path) } } - /// Returns true if deleted (or didn't exist), false if deletion failed. + /// + /// Returns true if deleted (or didn't exist), false if deletion failed. + /// public static bool Delete(string path) { try diff --git a/src/Mgx.Engine/Pagination/ScratchName.cs b/src/Mgx.Engine/Pagination/ScratchName.cs new file mode 100644 index 0000000..f31e6b9 --- /dev/null +++ b/src/Mgx.Engine/Pagination/ScratchName.cs @@ -0,0 +1,502 @@ +namespace Mgx.Engine.Pagination; + +/// +/// What stands at a scratch name. The same reading either way round: what a sweep took off the +/// name, or what an inspection found standing there and left. +/// +public enum ScratchEntry +{ + /// Nothing stands at the name. + Clear, + + /// A link. Swept, the link goes and what it pointed at is left alone. + Link, + + /// A regular file nothing holds - a leftover from a run that was killed. + Leftover, + + /// + /// A regular file this account cannot open read-write. Swept, since the name is nobody's to + /// keep, but nothing here could look inside it to say more than that. + /// + Unopenable, + + /// + /// Something that is not a file with a length and an offset in it - a pipe, a socket, a + /// device. Swept: nothing can be written at the name while it stands there. + /// + NotAFile, + + /// + /// A directory. Not this caller's to delete, and it could hold anything. + /// + Directory, + + /// A file another run has open. + Held, + + /// + /// The entry could not be read, or could not be unlinked. The exception says why. + /// + Unreadable, +} + +/// +/// The scratch names a run makes beside a file it is writing - the write probe, the atomic +/// saves' staging file, the copy a promotion stages - cleared before anything is opened +/// through them. +/// +/// No run leaves a file under one of those names on purpose: each is created, written and +/// renamed away or removed inside one operation, so an entry standing at one is a run that was +/// killed between two of those steps, or something a caller put there. Opening the name to +/// find out is what has to stop: an open that creates or truncates follows a symlink and +/// truncates the link's target, a FIFO with no reader blocks the open forever with no way for +/// a cancellation to reach it, and a create that refuses an existing name cannot say what kind +/// of entry refused it. So the entry is decided off the directory rather than through a +/// handle, and the kinds divide by what a run can honestly do about them: a link goes - the +/// link and never its target - and so does anything at the name that is not a regular file, +/// and a regular file no run holds. A directory and a file another run has open stop the +/// caller with the disk as it was found. +/// +/// +/// It lives here rather than beside the promotion that first needed it because the engine +/// cannot see the cmdlet base: the checkpoint and the delta state write scratch names of their +/// own, and one primitive is what keeps a pipe at a checkpoint's staging name from wedging a +/// run that a pipe at an output's staging name is refused at. +/// +/// +public static class ScratchName +{ + /// + /// Clear , or say why the caller stops there. Nothing of the + /// caller's may be open over the name when this runs: the claim below asks for the entry + /// exclusively, and a hard link at the name is an alias for whatever inode it points at - + /// so run under a claim on that inode, the sweep reads its own caller's lock as a second + /// run holding the entry. + /// + /// + /// The exception behind , for a caller that words + /// its own refusal from it; null for every other answer. + /// + public static ScratchEntry Sweep(string path, out Exception? failure) => + Decide(path, remove: true, out failure); + + /// + /// What stands at , with the name left exactly as it was found. The + /// same reading gives, so a pass that reports what a run would take off + /// the name reports the kind that run would name - and a preview that swept in order to find + /// out made deletions of its own, which is the one thing it is asked not to do. + /// + /// + public static ScratchEntry Inspect(string path, out Exception? failure) => + Decide(path, remove: false, out failure); + + private static ScratchEntry Decide(string path, bool remove, out Exception? failure) + { + // Asked again where the name moved under the claim below, and answered Held where it + // keeps moving: a name being mutated right now is a name this caller leaves alone, which + // is what every caller here already does with a name a second run holds. Each attempt is + // a fresh open, so the run that took the name answers the next one with its own lock. + for (var attempt = 1; ; attempt++) + { + if (DecideOnce(path, remove, out failure) is { } entry) return entry; + if (attempt == HeldEntry.ClaimAttempts) return ScratchEntry.Held; + } + } + + /// + /// One attempt at the name. Null where the claim below was granted over an entry that had + /// left the name - the one answer that is neither what stands there nor a refusal, and the + /// one asking again can change. + /// + private static ScratchEntry? DecideOnce(string path, bool remove, out Exception? failure) + { + failure = null; + try + { + // Read off the directory entry rather than through an open, which is what lets a + // dangling link and a symlink loop be answered at all: both are links and neither + // opens. File.Delete unlinks the link and leaves what it pointed at alone. + if (new FileInfo(path).LinkTarget != null) + { + if (remove) File.Delete(path); + return ScratchEntry.Link; + } + + if (Directory.Exists(path)) return ScratchEntry.Directory; + + FileStream? claim = null; + try + { + bool seekable; + try + { + // FileAccess.ReadWrite, because it is the one access a FIFO answers without + // blocking: a read-only or write-only open of a pipe with nothing on the + // other end waits for a peer forever, and a run parked there holds + // everything it had claimed with no way for a cancellation to reach it. + // FileMode.Open, so nothing is created and nothing is truncated. And + // FileShare.None, so what comes back a leftover is an entry no second run + // is using. + claim = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, + FileShare.None); + // Something opened, and it is not a file with a length and an offset in it: + // a FIFO opens read-write without blocking and answers every question after + // that with a NotSupportedException. Told apart here rather than at the + // first write, where a caller has already committed to the name. + seekable = claim.CanSeek; + } + catch (FileNotFoundException) { return ScratchEntry.Clear; } + catch (DirectoryNotFoundException) + { + // The same answer one level up: ENOENT on a path component. + return ScratchEntry.Clear; + } + catch (UnauthorizedAccessException) + { + // A file this account may not open. The name is still nobody's to keep, and + // the unlink asks the directory rather than the file. + if (remove) Unlink(path); + return ScratchEntry.Unopenable; + } + catch (IOException ex) + { + // The sharing violation is the one failure with a second run behind it. + // Everything else - a socket, a symlink loop, an over-long path - is an + // entry nothing can be written at. + if (IsSharingViolation(ex)) return ScratchEntry.Held; + if (remove) Unlink(path); + return ScratchEntry.NotAFile; + } + + // And the claim is not finished at the open. .NET takes the lock behind + // FileShare.None after open(2) on Unix, so this handle can have come away over + // an inode a second run renamed its own file over in between - the lock granted + // on a file with no name left on it, and the unlink below then taking that run's + // file off the name. What is held has to be what stands at the name. + if (!HeldEntry.StillStandsAt(claim, path)) return null; + + var entry = seekable ? ScratchEntry.Leftover : ScratchEntry.NotAFile; + if (!remove) return entry; + + // Held across the unlink rather than let go in front of it, so there is no + // instant in which the name is decided and unheld: a second run sweeping the + // same name in that interval created its own file at it and had it deleted from + // under the handle it was holding it by. The entry unlinked is the entry + // claimed - the line above is what makes that true - and unlink(2) honors no + // lock, so holding it costs the delete nothing. + // + // Windows keeps the order it had: a file this process has open cannot be + // unlinked there without FILE_SHARE_DELETE, so the handle goes in front of the + // delete and the interval is that platform's. + if (!OperatingSystem.IsWindows()) + { + File.Delete(path); + return entry; + } + + claim.Dispose(); + claim = null; + + // And on the far side of that interval the delete can meet a second run instead + // of the entry this sweep read: the name was let go, so a run sweeping it in + // that instant created its own file at it and is holding it, and the delete + // arrives at that run's claim. What stands there is not a leftover this sweep + // can answer for - it is the answer the claim above gives a name another run + // has, which every caller here already leaves the name alone on. Read as an + // entry nothing could be made of, it stopped runs over a name that was simply + // in use. + var taken = DeleteWaitingOutADeleteInFlight(path); + if (taken == null) return entry; + failure = taken; + return ScratchEntry.Held; + } + finally { claim?.Dispose(); } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + failure = ex; + return ScratchEntry.Unreadable; + } + } + + /// + /// Whether the entry stops the caller rather than being one a sweep clears. + /// + public static bool Refused(this ScratchEntry entry) => + entry is ScratchEntry.Directory or ScratchEntry.Held or ScratchEntry.Unreadable; + + /// Whether a sweep takes the entry off the name. + public static bool Removable(this ScratchEntry entry) => + entry is ScratchEntry.Link or ScratchEntry.Leftover or ScratchEntry.Unopenable + or ScratchEntry.NotAFile; + + /// + /// What stood at the name, for a caller that stops on it to name inside its own sentence. + /// Punctuated by the caller. + /// + public static string WhatStood(this ScratchEntry entry, string path, Exception? failure) + { + var name = Path.GetFileName(path); + return entry switch + { + ScratchEntry.Directory => $"a directory stands at '{name}'", + ScratchEntry.Held => $"another run has '{name}' open", + _ => $"'{name}' could not be cleared: " + + FirstLine(failure?.Message ?? "the reason is not known").TrimEnd('.'), + }; + } + + /// + /// The name, cleared and then made by this caller: the handle that comes back is over an + /// entry this run created, since the create refuses to use anything already standing there. + /// + /// + /// The name could not be cleared - a directory at it, another run holding it - or the + /// create was refused. A caller with a sentence of its own to say asks + /// and creates the file itself. + /// + public static FileStream Create(string path, string what) + { + var swept = Sweep(path, out var failure); + if (swept.Refused()) + { + throw new IOException( + $"Cannot write {what} '{path}': {swept.WhatStood(path, failure)}.", failure); + } + return CreateNew(path); + } + + /// + /// How many times a scratch name is asked again when Windows answers that this account may + /// not touch it, and how long each wait is. A delete of a name that has been started and has + /// not finished leaves the entry in the directory with every open of it refused, and both a + /// create and an unlink meet that as ERROR_ACCESS_DENIED - the same code a directory this + /// account cannot write answers with, so the two cannot be told apart from the exception. + /// + /// What tells them apart is time. The pending state ends when the last handle on the entry + /// goes, which is the next few milliseconds; a directory this account cannot write is still + /// refusing an hour later. So the name is asked again a few times over a few tens of + /// milliseconds, and only a refusal that outlasts that is the path's own - the one this + /// probe exists to reach, and it costs that path 40 ms it was going to stop on anyway. + /// + /// + /// Measured on Windows Server 2022, four runs creating and unlinking one name together: over + /// 40 runs of that shape, 154 creates and 64 unlinks were refused this way and not one of + /// them was anything about the directory. + /// + /// + /// Unix has a delete in flight of its own, and the same wait answers it. A create there is + /// two steps - the entry is made, and the lock over it is taken after - and a second run + /// sweeping the same name in between opens the entry this one just made, is granted the lock + /// first, reads a file nothing holds and takes it off the name. What comes back to the + /// create is the refused lock, over an entry that is on its way out of the directory; the + /// "already exists" a step later is the same instant with that unlink not yet landed. Both + /// are waited out, and only after a refused lock - a name a second run simply has its own + /// copy standing at is answered at once, as it always was. + /// + /// + private const int DeleteInFlightAttempts = 6; + + /// + private const int DeleteInFlightWaitMs = 8; + + /// + /// The create every scratch name is made by: FileMode.CreateNew refuses to use anything + /// already standing at the name, so the handle that comes back is over an entry this run + /// made and no other - which is what lets a caller unlink it afterwards without asking whose + /// it is. FileShare.None from the instant the entry exists, for the same reason. + /// + /// + public static FileStream CreateNew(string path) + { + var takenInTheGap = false; + for (var attempt = 1; ; attempt++) + { + try + { + return new FileStream(path, FileMode.CreateNew, FileAccess.ReadWrite, + FileShare.None); + } + catch (UnauthorizedAccessException) when (OperatingSystem.IsWindows() + && attempt < DeleteInFlightAttempts + && !Directory.Exists(path)) + { + Thread.Sleep(DeleteInFlightWaitMs); + } + catch (IOException ex) when (!OperatingSystem.IsWindows() + && attempt < DeleteInFlightAttempts + && (IsSharingViolation(ex) + || (takenInTheGap && IsAlreadyExists(ex)))) + { + // The entry this create made, claimed by a second run in the instant before the + // lock over it was taken - and read by that run as the one thing it could not + // be, a file nothing holds. It is being unlinked as this waits, and the name is + // this run's again on the other side of that. + takenInTheGap = true; + Thread.Sleep(DeleteInFlightWaitMs); + } + } + } + + /// + /// The unlink a Windows sweep takes the name back with, which answers for itself: a name + /// that cannot be cleared is one the caller stops on, so unlike this + /// does not swallow the failure. Two refusals are not that. A delete another run has already + /// started on the name is the name going, which is all the sweep wanted, and it is waited + /// out. A second run holding its own file at the name is that run's to answer for, and it is + /// reported as the hold it is. + /// + /// + /// Null where the name is clear. The sharing violation where a second run has taken the name + /// since this sweep let go of it. Any other refusal is raised, for the caller to report as + /// an entry it could not read. + /// + /// + private static IOException? DeleteWaitingOutADeleteInFlight(string path) + { + for (var attempt = 1; ; attempt++) + { + try + { + File.Delete(path); + return null; + } + catch (IOException ex) when (IsSharingViolation(ex)) + { + return ex; + } + catch (UnauthorizedAccessException) when (attempt < DeleteInFlightAttempts + && !Directory.Exists(path)) + { + Thread.Sleep(DeleteInFlightWaitMs); + } + } + } + + /// + /// Take the name back, under the handle that holds it. On Unix the entry is unlinked while + /// this caller still holds it, so the name is never decided and unheld: let go in front of + /// the delete, it is a name a second run can sweep, create its own file at and have deleted + /// from under it. Windows refuses to unlink a file this process has open without + /// FILE_SHARE_DELETE, so there the handle goes first and the interval is that platform's. + /// + /// Holding the entry is not on its own knowing that the name still reaches it - the lock + /// behind FileShare.None is granted after the open that resolved the name, and a handle can + /// come away over a file another run has since renamed its own copy over. So the name is + /// taken back only while is what stands at it, and left exactly as + /// it is otherwise: what is there belongs to the run that put it there. A discard with no + /// handle has nothing standing in this caller's name and removes nothing. + /// + /// + /// And a claim that cannot be verified is not a claim: where the question above fails rather + /// than answers - a host without the call, a parent directory that has stopped answering for + /// the name, a filesystem that fails it for a reason this cannot read - nothing is removed + /// and the name is left for the next sweep, whatever this caller is holding. So the temp a + /// promotion spends, the copy it staged and the write probe are taken back under the claim + /// where the claim can be finished, and left standing where it cannot. + /// + /// + /// Best effort, both ways round: a name that cannot be unlinked is a leftover the next + /// sweep takes, and a caller that has already answered must not fail on it. + /// + /// + public static void Discard(string path, FileStream? held) + { + if (!OperatingSystem.IsWindows()) + { + if (held != null && StillHolds(held, path)) Unlink(path); + held?.Dispose(); + return; + } + held?.Dispose(); + Unlink(path); + } + + /// + /// + /// Whether the handle is still what stands at the name, for a caller that must not fail. A + /// question that cannot be answered is answered no: the entry is left, which costs a leftover + /// the next sweep takes - where removing a file this run turns out not to hold costs the run + /// that does hold it its rows. + /// + private static bool StillHolds(FileStream held, string path) + { + try { return HeldEntry.StillStandsAt(held, path); } + catch (IOException) { return false; } + } + + private static void Unlink(string path) + { + try { File.Delete(path); } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { } + } + + /// + /// Whether an IOException from opening a file is the sharing violation the share mode exists + /// to produce, rather than one of the other ways an open fails. Named positively, because + /// the sharing case is the only one that means nothing is wrong with the file and another + /// run has it, and the only one a caller answers by waiting: a failure nothing here + /// recognizes is reported as a file this run cannot open, which is true of it whatever the + /// cause, instead of as a second run the caller can go and look for and not find. + /// + /// Windows carries the win32 code in the low half of the HRESULT. Unix puts the raw errno + /// there and nothing beside it, so the two forms cannot be read for each other: .NET backs + /// on Unix with a non-blocking flock, and a refused LOCK_EX + /// arrives as EWOULDBLOCK - 35 on macOS, measured for a second opener both inside the + /// process and outside it, and 11 on Linux, where the name EAGAIN shares that value. The + /// subclasses .NET maps by hand carry a win32-shaped HRESULT even on Unix - an over-long + /// path is a PathTooLongException with 0x800700CE in it - and are one more failure this + /// cannot mistake for a second run. + /// + /// + /// Each of those numbers is read on the platform that uses it and nowhere else, because the + /// two are a pair either way round: 11 is EDEADLK on macOS and 35 is EDEADLK on Linux. Taken + /// as a union, whichever platform this ran on read a deadlock the kernel had just refused to + /// enter as another run holding the file, and told the caller to wait for it. + /// + /// + public static bool IsSharingViolation(IOException ex) => ex.HResult switch + { + unchecked((int)0x80070020) => true, // ERROR_SHARING_VIOLATION + unchecked((int)0x80070021) => true, // ERROR_LOCK_VIOLATION + 35 => OperatingSystem.IsMacOS(), // EWOULDBLOCK (EDEADLK on Linux) + 11 => OperatingSystem.IsLinux(), // EAGAIN (EDEADLK on macOS) + _ => false, + }; + + /// + /// Whether an IOException from a create is the runtime's refusal of a name that already + /// exists, rather than one of the other ways a create fails. Read off the exception and not + /// off the name, which is the one thing a second look cannot answer: a create refused by + /// another run's scratch file is refused by an entry that run is about to take away, so by + /// the time the name can be asked about it is clear again - and a caller that decided from + /// that answer called a directory it could write in a directory it could not. + /// + /// What WEARS the name is a different question, and this cannot answer it: a file, a + /// directory and a link all refuse a create alike. A caller that has to tell them apart asks + /// the filesystem, which is honest about the entries that are still there to be asked about. + /// + /// + /// Windows carries the win32 code in the low half of the HRESULT and Unix puts the raw errno + /// there, the same pair reads. EEXIST is 17 on both Unixes + /// this runs on, so unlike the sharing errnos there is nothing here to tell macOS and Linux + /// apart; the platform is asked anyway, so that a win32 HRESULT can never be read as an + /// errno on the one host where the two are shaped alike. + /// + /// + public static bool IsAlreadyExists(IOException ex) => ex.HResult switch + { + unchecked((int)0x80070050) => true, // ERROR_FILE_EXISTS + unchecked((int)0x800700B7) => true, // ERROR_ALREADY_EXISTS + 17 => !OperatingSystem.IsWindows(), // EEXIST + _ => false, + }; + + /// The runtime's own first line of a message a caller quotes inside a sentence. + private static string FirstLine(string message) + { + var end = message.IndexOfAny(['\r', '\n']); + return end < 0 ? message : message[..end]; + } +} diff --git a/tests/Ecosystem/Mgx.Ecosystem.Tests.ps1 b/tests/Ecosystem/Mgx.Ecosystem.Tests.ps1 new file mode 100644 index 0000000..9758a6e --- /dev/null +++ b/tests/Ecosystem/Mgx.Ecosystem.Tests.ps1 @@ -0,0 +1,100 @@ +<# + Module load and assembly-load-context resolution are irreversible within a process, so + every permutation here runs in its own child pwsh. What a permutation can assert with no + tenant: the import works, the cmdlets resolve, the engine types JIT against whatever + assemblies the neighboring modules already loaded, and teardown leaves nothing armed. + A real Graph call under load order stays with tests/Live and the benchmarks. + + Provenance: GraphSDK-2148 (module isolation) - see tmp/CORPUS.md (private). Az.Accounts ships its + own System.Text.Json; the interesting failure is a TypeLoadException the first time an + engine type JITs, which is why the option round-trip and telemetry calls are the probe. + + A missing module is a failure, not a skip: a suite that skips itself when unconfigured + silently stops running. CI installs all three in the ecosystem job. +#> + +BeforeDiscovery { + $script:RequiredModules = 'Az.Accounts', 'PnP.PowerShell', 'ExchangeOnlineManagement' + foreach ($m in $script:RequiredModules) { + if (-not (Get-Module -ListAvailable $m)) { + throw "Ecosystem tests need the $m module installed. Install-PSResource $m, or run the ecosystem CI job." + } + } +} + +BeforeAll { + $repo = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent + $script:Manifest = Join-Path $repo 'Modules/M365DSC.mgx/M365DSC.mgx.psd1' + if (-not (Test-Path $script:Manifest)) { throw "Built module not found at $script:Manifest - run ./build.ps1 first." } + + # The battery runs after the permutation's imports. Order inside matters: the side-load + # check must precede the Get-MgContext stub, and resilience and fingerprint run after the + # remove/re-import so they exercise the re-imported module, not the first import. + $script:Battery = @' +$results = [ordered]@{} +function Check([string]$name, [scriptblock]$body) { + try { $results[$name] = @{ ok = [bool](& $body); detail = '' } } + catch { $results[$name] = @{ ok = $false; detail = $_.Exception.Message } } +} +Check 'cmdletsResolve' { (Get-Command -Module M365DSC.mgx -CommandType Cmdlet).Count -eq 12 } +Check 'noGraphSideLoad' { @(Get-Module 'Microsoft.Graph*').Count -eq 0 } +Check 'optionRoundTrip' { Set-MgxOption -MaxRetryAttempts 4; (Get-MgxOption).MaxRetryAttempts -eq 4 } +Check 'telemetry' { $null -ne (Get-MgxTelemetry) } +Check 'depsResolve' { + # Polly and the rate limiter load lazily; Assembly.Load drives the module's own + # Resolving hook, which is the mechanism whose neighbors could break it. + $polly = [Reflection.Assembly]::Load('Polly.Core') + $rate = [Reflection.Assembly]::Load('System.Threading.RateLimiting') + $stj = [Reflection.Assembly]::Load('System.Text.Json') + ($null -ne $polly) -and ($null -ne $rate) -and ($null -ne $stj) +} +Check 'removeReimport' { + Remove-Module M365DSC.mgx -Force + Import-Module '__MANIFEST__' -ErrorAction Stop + (Get-Command -Module M365DSC.mgx -CommandType Cmdlet).Count -eq 12 -and $null -ne (Get-MgxOption) +} +Check 'resilienceNamesTheSession' { + # ThrowTerminatingError is statement-terminating; no -ErrorAction downgrades it, + # so the record arrives as a caught exception carrying the id. + try { Enable-MgxResilience; $false } + catch { $_.FullyQualifiedErrorId -like 'GraphSessionNotFound*' } +} +Check 'fingerprintFromStub' { + function global:Get-MgContext { [PSCustomObject]@{ TenantId = 'eco-tenant'; ClientId = 'eco-client'; AuthType = 'AppOnly'; Environment = 'Global' } } + $base = (Get-Command Invoke-MgxRequest).ImplementingType.BaseType + $m = $base.GetMethod('BuildAuthFingerprint', [Reflection.BindingFlags]'Static,NonPublic,Public') + $fp = $m.Invoke($null, @((Get-MgContext), $null)) + -not [string]::IsNullOrEmpty($fp) +} +[pscustomobject]@{ battery = $results } | ConvertTo-Json -Depth 4 -Compress +'@ -replace '__MANIFEST__', $script:Manifest + + $script:Permutations = [ordered]@{ + 'az-before-mgx' = @('Import-Module Az.Accounts', 'Import-Module __MANIFEST__') + 'az-after-mgx' = @('Import-Module __MANIFEST__', 'Import-Module Az.Accounts') + 'pnp-before-mgx' = @('Import-Module PnP.PowerShell', 'Import-Module __MANIFEST__') + 'pnp-after-mgx' = @('Import-Module __MANIFEST__', 'Import-Module PnP.PowerShell') + 'exo-before-mgx' = @('Import-Module ExchangeOnlineManagement', 'Import-Module __MANIFEST__') + 'exo-after-mgx' = @('Import-Module __MANIFEST__', 'Import-Module ExchangeOnlineManagement') + 'stack-mgx-last' = @('Import-Module Az.Accounts', 'Import-Module PnP.PowerShell', 'Import-Module ExchangeOnlineManagement', 'Import-Module __MANIFEST__') + 'stack-mgx-first' = @('Import-Module __MANIFEST__', 'Import-Module Az.Accounts', 'Import-Module PnP.PowerShell', 'Import-Module ExchangeOnlineManagement') + 'interleaved' = @('Import-Module Az.Accounts', 'Import-Module __MANIFEST__', 'Set-MgxOption -MaxRetryAttempts 3', 'Import-Module PnP.PowerShell', 'Import-Module ExchangeOnlineManagement') + } + + function Invoke-Permutation([string[]]$imports) { + $lines = ($imports -replace '__MANIFEST__', $script:Manifest) + $script:Battery + $out = & pwsh -NoProfile -NonInteractive -Command ($lines -join "`n") 2>&1 + $json = @($out) | Where-Object { "$_" -match '^\{' } | Select-Object -Last 1 + if (-not $json) { throw "Permutation produced no result. Output: $(($out | Select-Object -First 5) -join ' | ')" } + ($json | ConvertFrom-Json).battery + } +} + +Describe 'Ecosystem isolation' { + It 'holds under <_>' -ForEach @('az-before-mgx', 'az-after-mgx', 'pnp-before-mgx', 'pnp-after-mgx', 'exo-before-mgx', 'exo-after-mgx', 'stack-mgx-last', 'stack-mgx-first', 'interleaved') { + $battery = Invoke-Permutation $script:Permutations[$_] + $failed = $battery.PSObject.Properties | Where-Object { -not $_.Value.ok } | + ForEach-Object { "$($_.Name): $($_.Value.detail)" } + $failed -join '; ' | Should -BeNullOrEmpty + } +} diff --git a/tests/Live/Mgx.Live.Tests.ps1 b/tests/Live/Mgx.Live.Tests.ps1 new file mode 100644 index 0000000..629dd30 --- /dev/null +++ b/tests/Live/Mgx.Live.Tests.ps1 @@ -0,0 +1,193 @@ +<# + These exist because the rest of the suite mocks HTTP. A mocked test proves the code does + what it was written to do; it cannot prove the request that leaves the process is one Graph + accepts. Enable-MgxResilience shipped a wrapper client with no BaseAddress and a green + suite: every test passed an absolute URI, and every relative-URI call would have thrown. + + So: relative URIs wherever a caller would use one, real responses, no mocks in this file. + + Read-only. Nothing here writes to a tenant. + + Run: Invoke-Pester -Path ./tests/Live + Needs AZURE_TENANT_ID / AZURE_CLIENT_ID / AZURE_CLIENT_CERTIFICATE_PATH for a directory + tenant, and optionally MGX_LIVE_CONTENT_* for the content tests. +#> + +BeforeAll { + $repo = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent + if (-not $env:AZURE_TENANT_ID -or -not $env:AZURE_CLIENT_ID -or + (-not $env:AZURE_CLIENT_CERTIFICATE_PATH -and -not $env:AZURE_CLIENT_SECRET)) { + throw "Live tests need AZURE_TENANT_ID, AZURE_CLIENT_ID, and either " + + "AZURE_CLIENT_CERTIFICATE_PATH or AZURE_CLIENT_SECRET. " + + "They are not skipped on purpose: a live suite that skips itself is a live suite nobody runs." + } + . "$repo/tests/benchmarks/common.ps1" + Import-MgxLocal + Connect-MgxBenchmark | Out-Null + + # Sized from the tenant rather than assumed. These tests have to hold on a 19-user tenant + # and a 100,000-user one: a live suite that only passes against one directory is one that + # gets switched off the first time it meets another. + $tok = Get-BenchAppToken + $script:UserCount = [int](Invoke-RestMethod -Uri 'https://graph.microsoft.com/v1.0/users/$count' ` + -Headers @{ Authorization = "Bearer $tok"; ConsistencyLevel = 'eventual' }) + # Enough to need more than one page where the tenant allows it, never more than it holds. + $script:Slice = [Math]::Min(150, $script:UserCount) + $script:SmallSlice = [Math]::Min(5, $script:UserCount) +} + +Describe 'Invoke-MgxRequest' { + It 'returns objects for a relative URI' { + (Invoke-MgxRequest '/users?$top=2' -WarningAction SilentlyContinue).Count | Should -Be 2 + } + It 'honors -Property' { + (Invoke-MgxRequest '/users?$top=1' -Property id -WarningAction SilentlyContinue).id | Should -Not -BeNullOrEmpty + } + It 'honors -Filter' { + Invoke-MgxRequest /users -Filter "startsWith(displayName,'b')" -Top 1 | Should -Not -BeNullOrEmpty + } + It 'caps the total at -Top' { + (Invoke-MgxRequest /users -Top $script:Slice -Property id | Measure-Object).Count | + Should -Be $script:Slice + } + It 'caps the total at -Top even with -All' { + (Invoke-MgxRequest /users -All -Top $script:Slice -Property id | Measure-Object).Count | + Should -Be $script:Slice + } + It 'reaches the beta endpoint under -ApiVersion beta' { + (Invoke-MgxRequest /users -ApiVersion beta -Top $script:SmallSlice -Property id | Measure-Object).Count | + Should -Be $script:SmallSlice + } + It 'stamps each entity of an {id} fan-out with the id it came from' { + # The only thing that says which piped id produced which object once the fan-out has + # interleaved them. Nothing else in the suite asserts the property exists. + $ids = @(Invoke-MgxRequest /users -Top $script:SmallSlice -Property id | ForEach-Object { $_.id }) + $seen = @($ids | Invoke-MgxRequest '/users/{id}' -Property id | ForEach-Object { $_['_MgxSourceId'] }) + Compare-Object ($ids | Sort-Object) ($seen | Sort-Object) | Should -BeNullOrEmpty + } + It 'refuses an {id} Uri with nothing piped to it' { + { Invoke-MgxRequest '/users/{id}' } | Should -Throw '*pipeline*' + } + It 'returns the user the SDK returns for the same id' { + # Read against the SDK rather than against itself: same tenant, same id, same three + # fields. A mock can agree with whatever the test author wrote; the directory cannot. + $id = (Invoke-MgxRequest '/users?$top=1' -Property id -WarningAction SilentlyContinue).id + $mgx = Invoke-MgxRequest "/users/$id" -Property id,displayName,userPrincipalName + $mg = Get-MgUser -UserId $id -Property Id,DisplayName,UserPrincipalName + $mgx.id | Should -Be $mg.Id + $mgx.displayName | Should -Be $mg.DisplayName + $mgx.userPrincipalName | Should -Be $mg.UserPrincipalName + } +} + +Describe 'Invoke-MgxBatchRequest' { + It 'executes a batch of relative URIs' { + $ids = Invoke-MgxRequest /users -Top $script:SmallSlice -Property id | ForEach-Object { $_.id } + $res = $ids | ForEach-Object { "/users/$_" } | Invoke-MgxBatchRequest -Method GET + ($res | Measure-Object).Count | Should -Be $script:SmallSlice + } +} + +Describe 'Expand-MgxRelation' { + It 'fans out over a relation' { + $users = Invoke-MgxRequest /users -Top $script:SmallSlice -Property id,displayName + $r = $users | Expand-MgxRelation -Uri '/users/{id}/manager' -As manager -SkipNotFound -SkipForbidden + $r | Should -Not -BeNull + } +} + +Describe 'Export-MgxCollection' { + It 'writes one JSONL line per object' { + $f = Join-Path ([IO.Path]::GetTempPath()) "mgxlive-$([Guid]::NewGuid().ToString('N')).jsonl" + try { + Export-MgxCollection /users -OutputFile $f -Top $script:Slice -Property id | Out-Null + (Get-Content $f).Count | Should -Be $script:Slice + } finally { Remove-Item $f -ErrorAction SilentlyContinue } + } +} + +Describe 'Sync-MgxDelta' { + It 'baselines with -Latest and writes state' { + $d = Join-Path ([IO.Path]::GetTempPath()) "mgxlive-$([Guid]::NewGuid().ToString('N')).json" + try { + Sync-MgxDelta /users/delta -DeltaPath $d -Latest | Out-Null + Test-Path $d | Should -BeTrue + } finally { Remove-Item $d -ErrorAction SilentlyContinue } + } +} + +Describe 'Get-MgxOption' { + It 'returns the current settings' { Get-MgxOption | Should -Not -BeNull } +} + +Describe 'Set-MgxOption' { + It 'changes a setting and reports it back' { + $before = (Get-MgxOption).RateLimitPerSecond + try { + Set-MgxOption -RateLimitPerSecond 42 + (Get-MgxOption).RateLimitPerSecond | Should -Be 42 + } finally { if ($before) { Set-MgxOption -RateLimitPerSecond $before } } + } +} + +Describe 'Get-MgxTelemetry' { + It 'counts requests the session made' { + Invoke-MgxRequest '/users?$top=1' -WarningAction SilentlyContinue | Out-Null + (Get-MgxTelemetry).Requests | Should -BeGreaterThan 0 + } + It 'zeroes on -Reset' { + Get-MgxTelemetry -Reset | Out-Null + (Get-MgxTelemetry).Requests | Should -Be 0 + } +} + +Describe 'Enable-MgxResilience' { + # The regression this file was written for: the wrapper dropped BaseAddress, so a RELATIVE + # -Uri threw from PrepareUri before any handler ran. Absolute URIs were unaffected, which is + # why a full green suite said nothing. + AfterEach { Disable-MgxResilience -ErrorAction SilentlyContinue | Out-Null } + + It 'leaves a relative-URI SDK call working' { + Enable-MgxResilience | Out-Null + Invoke-MgGraphRequest -Method GET -Uri '/v1.0/users?$top=1' | Should -Not -BeNull + } + It 'leaves SDK cmdlets working' { + Enable-MgxResilience | Out-Null + (Get-MgUser -Top ([Math]::Min(2, $script:UserCount)) -Property id | Measure-Object).Count | + Should -Be ([Math]::Min(2, $script:UserCount)) + } +} + +Describe 'Disable-MgxResilience' { + It 'restores plain SDK calls' { + Enable-MgxResilience | Out-Null + Disable-MgxResilience | Out-Null + Invoke-MgGraphRequest -Method GET -Uri '/v1.0/users?$top=1' | Should -Not -BeNull + } +} + +Describe 'Get-MgxResilience' { + It 'reports state either side of a wrap' { + Get-MgxResilience | Should -Not -BeNull + Enable-MgxResilience | Out-Null + try { Get-MgxResilience | Should -Not -BeNull } finally { Disable-MgxResilience | Out-Null } + } +} + +Describe 'Get-MgxContent' { + # Content needs Files.Read.All / Sites.Read.All, which the directory tenant does not carry. + # Point MGX_LIVE_CONTENT_URI at a drive item on a tenant that does. + BeforeAll { $script:contentUri = $env:MGX_LIVE_CONTENT_URI } + + It 'downloads a byte range' -Skip:(-not $env:MGX_LIVE_CONTENT_URI) { + $bytes = Get-MgxContent $script:contentUri -First 256 + $bytes.Length | Should -Be 256 + } + It 'downloads a whole file to -OutFile' -Skip:(-not $env:MGX_LIVE_CONTENT_URI) { + $f = Join-Path ([IO.Path]::GetTempPath()) "mgxlive-$([Guid]::NewGuid().ToString('N')).bin" + try { + Get-MgxContent $script:contentUri -OutFile $f | Out-Null + (Get-Item $f).Length | Should -BeGreaterThan 0 + } finally { Remove-Item $f -ErrorAction SilentlyContinue } + } +} diff --git a/tests/Live/Mgx.Throttle.Tests.ps1 b/tests/Live/Mgx.Throttle.Tests.ps1 new file mode 100644 index 0000000..bd61299 --- /dev/null +++ b/tests/Live/Mgx.Throttle.Tests.ps1 @@ -0,0 +1,137 @@ +<# + The one thing the rest of the suite cannot reach: what Enable-MgxResilience does when Graph + is actually refusing requests. + + It matters because the failure is silent. The Graph SDK's own retry handler sits inside the + wrap and answers 429 itself, so a throttled session looks identical to a clean one - the + pacer never slows down, and telemetry reports zero retries while the SDK sleeps out of sight + inside the call. Mocks cannot catch that: a mock 429 goes wherever the test author points it. + + This needs a second machine. One client cannot drive a tenant's budget from a single NAT'd + connection - measured 587 req/s from a laptop against 4,641 from a VM on the same network, + same test, same minute - and below roughly 700 req/s the tenant never refuses anything, so + the run proves nothing. Point MGX_LIVE_THROTTLE_SSH at a host with real egress. + + Run: MGX_LIVE_THROTTLE_SSH='user@host' Invoke-Pester -Path ./tests/Live/Mgx.Throttle.Tests.ps1 + + NOT read-only in effect: it deliberately exhausts the tenant's request budget and leaves it + refusing requests for several minutes afterwards. Run it last, or on its own. It is skipped + unless MGX_LIVE_THROTTLE_SSH is set, because most people running this suite do not have a + second machine to hand. +#> + +BeforeAll { + $repo = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent + if (-not $env:AZURE_TENANT_ID -or -not $env:AZURE_CLIENT_ID -or + (-not $env:AZURE_CLIENT_CERTIFICATE_PATH -and -not $env:AZURE_CLIENT_SECRET)) { + throw "Live tests need AZURE_TENANT_ID, AZURE_CLIENT_ID, and either " + + "AZURE_CLIENT_CERTIFICATE_PATH or AZURE_CLIENT_SECRET." + } + $script:Configured = [bool]$env:MGX_LIVE_THROTTLE_SSH + . "$repo/tests/benchmarks/common.ps1" + Import-MgxLocal + Connect-MgxBenchmark | Out-Null + + if ($script:Configured) { + $script:Target = $env:MGX_LIVE_THROTTLE_SSH + $script:SshArgs = @('-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8') + if ($env:MGX_LIVE_THROTTLE_KEY) { $script:SshArgs = @('-i', $env:MGX_LIVE_THROTTLE_KEY) + $script:SshArgs } + $script:Remote = '/tmp/mgx-throttle-drain.py' + + # Reads the token on stdin. Never as an argument - argv is world-readable in ps - and + # never written to disk on the remote host. The certificate itself never leaves here: + # the assertion is signed locally and only the resulting bearer token crosses. + $drainer = @' +import http.client, ssl, sys, threading, time +token = sys.stdin.readline().strip() +workers, seconds = int(sys.argv[1]), int(sys.argv[2]) +ctx = ssl.create_default_context(); hdrs = {"Authorization": "Bearer " + token} +stop = time.time() + seconds +def run(): + c = None + while time.time() < stop: + try: + if c is None: + c = http.client.HTTPSConnection("graph.microsoft.com", 443, context=ctx, timeout=20) + c.request("GET", "/v1.0/users?$top=1&$select=id", headers=hdrs) + r = c.getresponse(); r.read() + except Exception: + try: c.close() + except Exception: pass + c = None +ts = [threading.Thread(target=run, daemon=True) for _ in range(workers)] +for t in ts: t.start() +for t in ts: t.join() +'@ + $drainer | & ssh @script:SshArgs $script:Target "cat > $script:Remote" + if ($LASTEXITCODE -ne 0) { throw "Could not stage the load generator on $script:Target." } + + $script:Token = Get-BenchAppToken + $script:Drain = Start-Job -ScriptBlock { + param($sshArgs, $target, $remote, $token) + $token | & ssh @sshArgs $target "python3 $remote 200 300" + } -ArgumentList $script:SshArgs, $script:Target, $script:Remote, $script:Token + + # Confirm the tenant is refusing requests before asserting anything about the refusal. + # Without this the test can only distinguish "the bridge saw no throttle" from "there + # was no throttle to see" by guessing, and a green result would mean nothing. + $probe = 'https://graph.microsoft.com/v1.0/users?$top=1&$select=id' + $deadline = (Get-Date).AddSeconds(120) + $script:ThrottleConfirmedAt = $null + while ((Get-Date) -lt $deadline -and -not $script:ThrottleConfirmedAt) { + $r = Invoke-WebRequest -Uri $probe -Headers @{ Authorization = "Bearer $script:Token" } ` + -SkipHttpErrorCheck -ErrorAction SilentlyContinue + if ($r.StatusCode -eq 429) { $script:ThrottleConfirmedAt = Get-Date } + else { Start-Sleep -Seconds 3 } + } + } +} + +AfterAll { + if ($script:Configured) { + if ($script:Drain) { Stop-Job $script:Drain -ErrorAction SilentlyContinue; Remove-Job $script:Drain -Force -ErrorAction SilentlyContinue } + # Stopping the job kills the local ssh client, not the process on the far side - that + # keeps running to its own deadline, hammering the tenant long after the test reports + # green. Kill it where it lives. The bracket makes the pattern not match the command + # carrying it, which would otherwise kill this shell before the rm ran. + & ssh @script:SshArgs $script:Target "rm -f $script:Remote; pkill -f '[m]gx-throttle-drain'" 2>$null + Disable-MgxResilience -ErrorAction SilentlyContinue | Out-Null + } +} + +Describe 'Enable-MgxResilience' { + Context 'under a real throttle' { + + It 'drove the tenant into refusing requests' -Skip:(-not $env:MGX_LIVE_THROTTLE_SSH) { + # Fails rather than skips: the run was asked for, so not reaching a throttle is a + # result about the setup, not a reason to report a pass. + $script:ThrottleConfirmedAt | Should -Not -BeNullOrEmpty -Because ` + 'the load generator never pushed the tenant to 429 - check the host has real egress, and that 200 workers is enough for it' + } + + It 'sees the throttle the SDK would otherwise absorb' -Skip:(-not $env:MGX_LIVE_THROTTLE_SSH) { + Enable-MgxResilience | Out-Null + Get-MgxTelemetry -Reset | Out-Null + foreach ($i in 1..12) { Get-MgUser -Top 1 -ErrorAction SilentlyContinue | Out-Null } + $t = Get-MgxTelemetry + + # The whole point of the wrap. Zero here with the tenant refusing requests means the + # SDK's own retry handler answered them first and Mgx never learned a thing. + $t.ThrottleRetries | Should -BeGreaterThan 0 -Because ` + 'the SDK retry handler inside the wrap must be disarmed, or a throttled session is indistinguishable from a clean one' + } + + It 'lets the pacer learn from it' -Skip:(-not $env:MGX_LIVE_THROTTLE_SSH) { + # Retries alone are not the win - slowing down is. A pacer still in slow-start after a + # throttle is one that never saw it. + (Get-MgxTelemetry).PacingState | Should -Match 'capped' -Because ` + 'the pacer should cap its rate once it has met a 429, not stay in slow-start' + } + + It 'still completes the caller''s requests' -Skip:(-not $env:MGX_LIVE_THROTTLE_SSH) { + # Surfacing throttling must not mean surfacing failures: the retries exist so the + # caller's call still returns. + (Get-MgxTelemetry).Failed | Should -Be 0 + } + } +} diff --git a/tests/Mgx.E2ETests/Infrastructure/MgxCmdletHost.cs b/tests/Mgx.E2ETests/Infrastructure/MgxCmdletHost.cs index f70ca7a..430e3d8 100644 --- a/tests/Mgx.E2ETests/Infrastructure/MgxCmdletHost.cs +++ b/tests/Mgx.E2ETests/Infrastructure/MgxCmdletHost.cs @@ -1,4 +1,5 @@ -using System.Management.Automation; +using System.Management.Automation; +using System.Threading; using System.Management.Automation.Runspaces; using System.Reflection; using Mgx.Cmdlets.Base; @@ -72,7 +73,9 @@ public MgxCmdletHost(HttpClient transport, string graphEndpoint, GraphBatchClient.ResetPacingState(); MgxCmdletBase.SetClientOptions(options ?? FastOptions); MgxCmdletBase.s_graphEndpoint = graphEndpoint; - MgxCmdletBase.s_testTransportFactory = () => transport; + MgxCmdletBase.s_testTransportOwned = true; + MgxCmdletBase.s_testTransport = transport; + Interlocked.Increment(ref MgxCmdletBase.s_testTransportEpoch); } public MgxResult Run(Action build) @@ -102,7 +105,9 @@ [.. ps.Streams.Verbose.Select(v => v.Message)], public void Dispose() { - MgxCmdletBase.s_testTransportFactory = null; + MgxCmdletBase.s_testTransportOwned = false; + MgxCmdletBase.s_testTransport = null; + Interlocked.Increment(ref MgxCmdletBase.s_testTransportEpoch); MgxCmdletBase.s_graphEndpoint = "https://graph.microsoft.com"; MgxCmdletBase.SetClientOptions(ResilientGraphClientOptions.Default); ResiliencePipelineFactory.Reset(); diff --git a/tests/Mgx.IntegrationTests/Cmdlets/Base/AuthFingerprintTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/Base/AuthFingerprintTests.cs index b86d9a4..b7ab00c 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/Base/AuthFingerprintTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/Base/AuthFingerprintTests.cs @@ -7,25 +7,56 @@ namespace Mgx.IntegrationTests.Cmdlets; /// The cached Graph HttpClient is keyed on this fingerprint. Keying it on tenant id alone /// meant that reconnecting the same tenant with a different app registration silently reused /// the previous application's token, so every one of these cases is a stale-credential bug. +/// (Corpus: M365DSC-4426, stale credentials.) /// +[Collection("Pipeline")] public class AuthFingerprintTests { [Fact] - public void Reports_the_graph_sdk_as_absent_when_it_is_not_loaded() + public void A_session_with_no_context_reads_as_disconnected_not_as_a_missing_module() { - // Microsoft.Graph.Authentication is a soft dependency whose absence selects - // GraphAuthModuleNotLoaded over "run Connect-MgGraph", which would name a cmdlet the - // session does not have. Asserted on the type rather than on IsGraphAuthLoaded, whose - // Get-MgContext fallback finds an installed-but-unloaded module on a developer machine - Assert.Null(MgxCmdletBase.FindType("Microsoft.Graph.PowerShell.Authentication.GraphSession")); + // Microsoft.Graph.Authentication is a soft dependency, so "absent" and "present but + // disconnected" are different states needing different advice: naming Connect-MgGraph + // to someone who does not have the module sends them to a cmdlet that does not exist. + // + // Only the disconnected half is reachable through the cmdlet in this process: the suite + // declares a stand-in GraphSession for the resilience-injection scenarios and FindType + // resolves a type by full name, so IsGraphAuthLoaded answers true from the type alone + // whether or not a test has armed a session. The absent half is held by the test below, + // which asks the choice itself rather than the state it is made from. + using var scope = GraphSessionScope.Arm(); + + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1"); + var thrown = Record.Exception(() => ps.Invoke()); + + var ids = ps.Streams.Error.Select(e => e.FullyQualifiedErrorId).ToList(); + if (thrown is CmdletInvocationException invocation) + ids.Add(invocation.ErrorRecord.FullyQualifiedErrorId); + + Assert.Contains(ids, id => id.StartsWith("NotConnected", StringComparison.Ordinal)); + Assert.DoesNotContain(ids, id => id.StartsWith("GraphAuthModuleNotLoaded", StringComparison.Ordinal)); } [Fact] - public void Reports_the_graph_sdk_as_present_once_its_session_type_resolves() + public void An_absent_module_and_a_disconnected_session_give_different_advice() { - using var stub = new Fakes.GraphSdkStub(); - - Assert.True(MgxCmdletBase.IsGraphAuthLoaded()); + // Naming Connect-MgGraph to someone without the module is the failure this splits, so + // the two messages have to differ in the instruction, not only in the error id. + var disconnected = MgxCmdletBase.DescribeMissingConnection(graphAuthLoaded: true); + var absent = MgxCmdletBase.DescribeMissingConnection(graphAuthLoaded: false); + + Assert.Equal("NotConnected", disconnected.ErrorId); + Assert.Contains("Connect-MgGraph", disconnected.Message, StringComparison.Ordinal); + Assert.DoesNotContain("Install-PSResource", disconnected.Message, StringComparison.Ordinal); + + Assert.Equal("GraphAuthModuleNotLoaded", absent.ErrorId); + Assert.Contains("Install-PSResource -Name Microsoft.Graph.Authentication", + absent.Message, StringComparison.Ordinal); } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/Base/BetaHintTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/Base/BetaHintTests.cs index 017eaa2..91c46a0 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/Base/BetaHintTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/Base/BetaHintTests.cs @@ -1,6 +1,5 @@ using System.Management.Automation; using System.Net; -using System.Reflection; using System.Text; using Mgx.Cmdlets.Base; using Mgx.Engine.Http; @@ -34,49 +33,11 @@ protected override Task SendAsync( } } - private static void InjectMock(HttpMessageHandler handler) - { - ResiliencePipelineFactory.Reset(); - var t = typeof(MgxCmdletBase); - t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, new HttpClient(handler)); - t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - // Cleared with the fingerprint. A stale reference from an earlier test reads as a - // credential change and the injected client is rebuilt away - t.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - // Get-MgxContent refuses to run over a transport mgx does not own (the SDK client - // auto-follows redirects, defeating download-host validation), so the mock must claim - // ownership or the cmdlet errors before any HTTP happens and the test is vacuous. - t.GetField("s_ownsHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, true); - t.GetField("s_graphEndpoint", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! - .SetValue(null, "https://graph.microsoft.com"); - var options = new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }; - t.GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! - .SetValue(null, options); - // Owning the client makes GetClient() check the cached timeout; a mismatch forces a - // rebuild that cannot succeed under a mock. Cache the value the options carry. - t.GetField("s_cachedTotalTimeoutSeconds", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, options.TotalTimeoutSeconds); - ResiliencePipelineFactory.Reset(); - } - - private static void CleanupMock() - { - var t = typeof(MgxCmdletBase); - t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - ResiliencePipelineFactory.Reset(); - } - private static (List Warnings, List Errors) Run( string cmdlet, string uri, HttpStatusCode status, string body) { var handler = new SingleResponseHandler(status, body); - InjectMock(handler); - try + using (MgxTransportScope.Inject(handler, owned: true)) { using var ps = PowerShell.Create(); ps.AddCommand("Import-Module") @@ -101,7 +62,6 @@ private static (List Warnings, List Errors) Run( errors.Add($"(silent; {handler.Requests.Count} HTTP requests: {string.Join(", ", handler.Requests)})"); return (warnings, errors); } - finally { CleanupMock(); } } private const string ItemNotFound = diff --git a/tests/Mgx.IntegrationTests/Cmdlets/Base/RecoveryAndUrlHelperTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/Base/RecoveryAndUrlHelperTests.cs index 6b826af..cd06794 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/Base/RecoveryAndUrlHelperTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/Base/RecoveryAndUrlHelperTests.cs @@ -1,4 +1,4 @@ -using System.Management.Automation; +using System.Management.Automation; using System.Net; using Mgx.Cmdlets.Base; @@ -22,14 +22,8 @@ public static string Url(string relativeUri, public static ErrorCategory Category(HttpStatusCode status) => MapStatusToCategory(status); - public static bool Trim(string outputPath, long dataLength) => - TryTrimOutputToCheckpoint(outputPath, dataLength); - - public static bool Promote(string outputPath, string tempFileName, long dataLength) => - TryPromoteNamedTemp(outputPath, tempFileName, dataLength); - public static bool Adopt(string outputPath, long itemCount) => - TryAdoptOrphanedTemp(outputPath, itemCount); + TryAdoptOrphanedTemp(outputPath, itemCount, out _, out _, out _); } private readonly string _workDir = Directory.CreateDirectory( @@ -119,7 +113,7 @@ public void Options_append_to_a_uri_that_already_carries_a_query() [InlineData(HttpStatusCode.BadRequest, ErrorCategory.InvalidArgument)] [InlineData(HttpStatusCode.Conflict, ErrorCategory.ResourceExists)] [InlineData((HttpStatusCode)429, ErrorCategory.LimitsExceeded)] - [InlineData(HttpStatusCode.InternalServerError, ErrorCategory.NotSpecified)] + [InlineData(HttpStatusCode.InternalServerError, ErrorCategory.ResourceUnavailable)] public void A_status_maps_to_the_category_powershell_reports(HttpStatusCode status, ErrorCategory expected) { Assert.Equal(expected, Probe.Category(status)); @@ -127,128 +121,6 @@ public void A_status_maps_to_the_category_powershell_reports(HttpStatusCode stat // --- trimming an output back to its checkpoint --- - [Fact] - public void Trimming_cuts_off_what_was_written_after_the_last_save() - { - var path = InWorkDir("out.jsonl"); - File.WriteAllText(path, "0123456789"); - - Assert.True(Probe.Trim(path, 4)); - Assert.Equal("0123", File.ReadAllText(path)); - } - - [Fact] - public void Trimming_an_output_that_already_matches_leaves_it_alone() - { - var path = InWorkDir("exact.jsonl"); - File.WriteAllText(path, "0123"); - - Assert.True(Probe.Trim(path, 4)); - Assert.Equal("0123", File.ReadAllText(path)); - } - - [Fact] - public void An_output_shorter_than_the_checkpoint_is_no_longer_the_file_it_describes() - { - var path = InWorkDir("short.jsonl"); - File.WriteAllText(path, "012"); - - Assert.False(Probe.Trim(path, 10)); - } - - [Fact] - public void A_missing_output_cannot_be_trimmed() - { - Assert.False(Probe.Trim(InWorkDir("absent.jsonl"), 4)); - } - - [Fact] - public void A_negative_length_from_a_tampered_checkpoint_is_refused() - { - var path = InWorkDir("negative.jsonl"); - File.WriteAllText(path, "0123"); - - Assert.False(Probe.Trim(path, -1)); - Assert.Equal("0123", File.ReadAllText(path)); - } - - // --- promoting the temp a checkpoint named --- - - [Fact] - public void The_named_temp_becomes_the_output_and_is_removed() - { - var path = InWorkDir("promote.jsonl"); - var tempName = TempNameFor("promote.jsonl"); - File.WriteAllText(InWorkDir(tempName), "0123456789"); - - Assert.True(Probe.Promote(path, tempName, 10)); - Assert.Equal("0123456789", File.ReadAllText(path)); - Assert.False(File.Exists(InWorkDir(tempName))); - } - - [Fact] - public void Only_the_bytes_the_checkpoint_recorded_are_promoted() - { - var path = InWorkDir("partial.jsonl"); - var tempName = TempNameFor("partial.jsonl"); - File.WriteAllText(InWorkDir(tempName), "0123456789"); - - Assert.True(Probe.Promote(path, tempName, 4)); - Assert.Equal("0123", File.ReadAllText(path)); - } - - [Fact] - public void A_temp_shorter_than_the_checkpoint_promised_is_refused() - { - var path = InWorkDir("truncated.jsonl"); - var tempName = TempNameFor("truncated.jsonl"); - File.WriteAllText(InWorkDir(tempName), "012"); - - Assert.False(Probe.Promote(path, tempName, 10)); - Assert.False(File.Exists(path)); - } - - [Fact] - public void A_temp_name_from_another_export_is_refused() - { - var path = InWorkDir("mine.jsonl"); - var foreignName = TempNameFor("theirs.jsonl"); - File.WriteAllText(InWorkDir(foreignName), "0123456789"); - - Assert.False(Probe.Promote(path, foreignName, 10)); - } - - [Fact] - public void A_temp_name_carrying_a_path_is_refused() - { - var path = InWorkDir("mine.jsonl"); - var tempName = TempNameFor("mine.jsonl"); - File.WriteAllText(InWorkDir(tempName), "0123456789"); - - Assert.False(Probe.Promote(path, Path.Combine("..", tempName), 10)); - } - - [Fact] - public void A_temp_name_that_is_not_the_run_shape_is_refused() - { - var path = InWorkDir("mine.jsonl"); - File.WriteAllText(InWorkDir("mine.jsonl.notaguid.tmp"), "0123456789"); - - Assert.False(Probe.Promote(path, "mine.jsonl.notaguid.tmp", 10)); - } - - [Fact] - public void A_zero_length_promotion_is_refused() - { - var path = InWorkDir("empty.jsonl"); - var tempName = TempNameFor("empty.jsonl"); - File.WriteAllText(InWorkDir(tempName), "0123456789"); - - Assert.False(Probe.Promote(path, tempName, 0)); - } - - // --- adopting an orphaned temp from an older checkpoint --- - [Fact] public void The_newest_matching_temp_is_adopted_line_for_line() { diff --git a/tests/Mgx.IntegrationTests/Cmdlets/Base/TransportSeamOrderTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/Base/TransportSeamOrderTests.cs new file mode 100644 index 0000000..7fc37cf --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/Base/TransportSeamOrderTests.cs @@ -0,0 +1,52 @@ +using Mgx.Cmdlets.Base; +using Mgx.IntegrationTests.Fakes; + +namespace Mgx.IntegrationTests; + +/// +/// The transport seam is a pair. GetClient takes the epoch, then the transport, then the epoch +/// again and reads both again on a change; a scope arms the transport before it bumps the epoch. +/// Either half on its own is wrong: a reader that takes the transport first can pair one the +/// scope has already retired with the epoch that retired it, and then answers from that +/// transport for as long as the cmdlet instance lives. +/// +/// The interleaving cannot be observed without racing the two, so what is pinned here is the +/// write order the reader depends on, seen from between the two writes. +/// +[Collection("Pipeline")] +public class TransportSeamOrderTests +{ + [Fact] + public void A_scope_arms_its_transport_before_it_bumps_the_epoch() + { + using var outer = MgxTransportScope.Inject(new StubHttpMessageHandler()); + var outerTransport = MgxCmdletBase.s_testTransport; + Assert.NotNull(outerTransport); + var epochBefore = Volatile.Read(ref MgxCmdletBase.s_testTransportEpoch); + + HttpClient? transportInWindow = null; + var epochInWindow = 0; + var windows = 0; + + using (var inner = MgxTransportScope.Inject(new StubHttpMessageHandler())) + { + // Arming happened in the constructor: the transport is the inner scope's and the + // epoch has moved once, so a client cached over the outer one no longer matches. + Assert.NotSame(outerTransport, MgxCmdletBase.s_testTransport); + Assert.Equal(epochBefore + 1, Volatile.Read(ref MgxCmdletBase.s_testTransportEpoch)); + + inner.BetweenPublishWrites = () => + { + windows++; + transportInWindow = MgxCmdletBase.s_testTransport; + epochInWindow = Volatile.Read(ref MgxCmdletBase.s_testTransportEpoch); + }; + } + + Assert.Equal(1, windows); + Assert.Same(outerTransport, transportInWindow); // the transport goes back first + Assert.Equal(epochBefore + 1, epochInWindow); // and only then does the epoch move + Assert.Equal(epochBefore + 2, Volatile.Read(ref MgxCmdletBase.s_testTransportEpoch)); + Assert.Same(outerTransport, MgxCmdletBase.s_testTransport); + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/Base/TypeCacheInvalidationTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/Base/TypeCacheInvalidationTests.cs new file mode 100644 index 0000000..66101e0 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/Base/TypeCacheInvalidationTests.cs @@ -0,0 +1,168 @@ +using System.Reflection; +using System.Reflection.Emit; +using System.Runtime.Loader; +using Mgx.Cmdlets; +using Mgx.Cmdlets.Base; + +namespace Mgx.IntegrationTests; + +/// +/// FindType caches by full name, and a cached Type carries the identity of the assembly that +/// defined it. Invalidation is an AssemblyLoad hook that module removal detaches, so what has to +/// hold across a removal is that an import arms it again - once - and that the removal itself +/// leaves nothing resolved behind. +/// +/// In the Pipeline collection because every test that imports the module assembly is, and +/// importing it runs the initializer these tests drive directly. +/// +[Collection("Pipeline")] +public class TypeCacheInvalidationTests +{ + private const string GraphSessionTypeName = "Microsoft.Graph.PowerShell.Authentication.GraphSession"; + + /// + /// An entry only this test writes and only this test reads. The cache is process-global and + /// the collections running alongside this one both resolve types into it and load assemblies + /// that clear it, so neither "empty" nor "not empty" is this test's own doing - but an entry + /// put here and gone afterwards can only have been cleared. + /// + private const string Sentinel = "Mgx.IntegrationTests.TypeCacheInvalidationTests+Sentinel"; + + /// + /// What this test resolved, or put there itself, before a removal it expects to drop both. + /// The releases a removal makes resolve types of their own on the way through, so the entry + /// this test wrote is not enough on its own: it is cleared by any load, including one those + /// releases trigger, while the entries they then resolve are exactly what the removal's own + /// clear has to take out afterwards. + /// + private static readonly string[] Resolved = [GraphSessionTypeName, Sentinel]; + + [Fact] + public void An_assembly_loading_after_a_remove_and_re_import_clears_the_type_cache() + { + // Resolve something first: "cleared afterwards" has to mean the removal cleared the + // cache, not that nothing had resolved a type yet. + Assert.NotNull(MgxCmdletBase.FindType(GraphSessionTypeName)); + MgxCmdletBase.s_typeCache[Sentinel] = typeof(TypeCacheInvalidationTests); + + try + { + // A removal is ReleaseStaticState and then the detach; an import is OnImport. + AlcInitializer.ReleaseStaticState(); + MgxCmdletBase.DetachAssemblyLoadHandler(); + Assert.False(AnyStillCached(), "module removal left resolved types cached"); + + new AlcInitializer().OnImport(); + + // A load from a collection running alongside this one clears the cache too, so the + // resolved entry only has to end up there, not be there on the first attempt. + var primed = false; + for (var attempt = 0; attempt < 10 && !primed; attempt++) + { + Assert.NotNull(MgxCmdletBase.FindType(GraphSessionTypeName)); + primed = MgxCmdletBase.s_typeCache.ContainsKey(GraphSessionTypeName); + } + Assert.True(primed, "resolving a type after the import cached nothing"); + + MgxCmdletBase.s_typeCache[Sentinel] = typeof(TypeCacheInvalidationTests); + LoadAnAssembly("MgxTypeCacheProbe"); + + Assert.False(AnyStillCached(), + "a type resolved before the load is still cached, so a Microsoft.Graph.Authentication " + + "loading now would keep resolving to the session of the one it replaced"); + } + finally + { + MgxCmdletBase.s_typeCache.TryRemove(Sentinel, out _); + MgxCmdletBase.AttachAssemblyLoadHandler(); + } + } + + [Fact] + public void A_second_import_leaves_one_subscription_for_a_removal_to_detach() + { + try + { + new AlcInitializer().OnImport(); + new AlcInitializer().OnImport(); + Assert.Equal(1, InvalidationSubscriptions()); + + // One removal has to be enough. A second subscription would survive this and go on + // clearing the cache from a module that is gone. + MgxCmdletBase.DetachAssemblyLoadHandler(); + Assert.Equal(0, InvalidationSubscriptions()); + } + finally + { + MgxCmdletBase.AttachAssemblyLoadHandler(); + } + } + + /// + /// The dependency resolver is the other subscription an import makes, and += is no more + /// idempotent there. A second one survives the -= a removal does and goes on answering + /// dependency loads out of a Dependencies folder belonging to a module the session removed. + /// + [Fact] + public void A_second_import_leaves_one_dependency_resolver_for_a_removal_to_detach() + { + try + { + new AlcInitializer().OnImport(); + new AlcInitializer().OnImport(); + Assert.Equal(1, ResolverSubscriptions()); + + // The removal itself, module argument and all - OnRemove never reads it. + new AlcInitializer().OnRemove(null!); + Assert.Equal(0, ResolverSubscriptions()); + } + finally + { + new AlcInitializer().OnImport(); + } + } + + /// Whether anything this test resolved, or wrote itself, is still cached. + private static bool AnyStillCached() => Resolved.Any(MgxCmdletBase.s_typeCache.ContainsKey); + + /// + /// How many times mgx's invalidation hook is subscribed. AppDomain.AssemblyLoad forwards to a + /// static event on AssemblyLoadContext, and its invocation list is the only place the count + /// shows: an extra subscription is silent until a removal detaches one and the other stays. + /// + private static int InvalidationSubscriptions() + { + var field = typeof(AssemblyLoadContext).GetField("AssemblyLoad", BindingFlags.Static | BindingFlags.NonPublic); + Assert.True(field != null, "AssemblyLoadContext no longer keeps its subscribers in a static field"); + var subscribers = (Delegate?)field!.GetValue(null); + return subscribers?.GetInvocationList().Count(d => d.Method.DeclaringType == typeof(MgxCmdletBase)) ?? 0; + } + + /// + /// How many times mgx's dependency resolver is subscribed to the default load context. + /// AssemblyLoadContext keeps its Resolving subscribers in a private instance field, and that + /// invocation list is the only place the count shows: a second subscription is silent until a + /// removal detaches one and the other stays. + /// + private static int ResolverSubscriptions() + { + var field = typeof(AssemblyLoadContext).GetField( + "_resolving", BindingFlags.Instance | BindingFlags.NonPublic); + Assert.True(field != null, "AssemblyLoadContext no longer keeps its Resolving subscribers in _resolving"); + var subscribers = (Delegate?)field!.GetValue(AssemblyLoadContext.Default); + return subscribers?.GetInvocationList().Count(d => d.Method.DeclaringType == typeof(AlcInitializer)) ?? 0; + } + + /// + /// Loads an assembly, which is what invalidation keys off. A dynamic one raises the event + /// exactly as a file-backed load does, and leaves nothing behind that the rest of the suite + /// has to be able to resolve. + /// + private static void LoadAnAssembly(string name) + { + var assembly = AssemblyBuilder.DefineDynamicAssembly(new AssemblyName(name), AssemblyBuilderAccess.Run); + assembly.DefineDynamicModule(name) + .DefineType($"{name}.Marker", TypeAttributes.Public) + .CreateType(); + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/BatchErrorSurfacingTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/BatchErrorSurfacingTests.cs new file mode 100644 index 0000000..f38df48 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/BatchErrorSurfacingTests.cs @@ -0,0 +1,208 @@ +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// A failed batch item is an error the pipeline can see: it lands in $Error, counts for +/// -ErrorVariable, and stops the pipeline under -ErrorAction Stop - with or without a +/// dead-letter file. (Corpus: M365DSC-7198, batch semantics.) +/// +[Collection("Pipeline")] +public class BatchErrorSurfacingTests +{ + private static System.Management.Automation.PowerShell CreateShell() + { + var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + return ps; + } + + private const string OneOkOneNotFound = """ + { "responses": [ + { "id": "1", "status": 200, "body": { "id": "u1" } }, + { "id": "2", "status": 404, "body": { "error": { "code": "Request_ResourceNotFound", "message": "u2 does not exist" } } } + ] } + """; + + [Fact] + public void A_failed_item_writes_an_error_record_without_a_dead_letter_path() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, OneOkOneNotFound); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new[] { "/users/u1", "/users/u2" }) + .AddParameter("Method", "GET"); + var output = ps.Invoke(); + + Assert.Equal(2, output.Count); + var error = Assert.Single(ps.Streams.Error); + Assert.StartsWith("BatchItemError", error.FullyQualifiedErrorId); + Assert.Contains("u2", error.Exception.Message); + Assert.Equal(System.Management.Automation.ErrorCategory.ObjectNotFound, + error.CategoryInfo.Category); + } + } + + [Fact] + public void ErrorAction_Stop_stops_on_a_failed_item() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, OneOkOneNotFound); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new[] { "/users/u1", "/users/u2" }) + .AddParameter("Method", "GET") + .AddParameter("ErrorAction", "Stop"); + var ex = Assert.ThrowsAny(() => ps.Invoke()); + Assert.Contains("u2", ex.Message); + } + } + + /// First POST answers its chunk in full; every later POST is refused outright. + private sealed class FirstChunkOnlyHandler : HttpMessageHandler + { + private int _posts; + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + _posts++; + if (_posts == 1) + { + var body = "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, 20).Select(i => + $"{{ \"id\": \"{i}\", \"status\": 200, \"body\": {{ \"id\": \"u{i}\" }} }}")) + "] }"; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") + }); + } + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent( + "{\"error\":{\"code\":\"BadRequest\",\"message\":\"batch rejected\"}}", + System.Text.Encoding.UTF8, "application/json") + }); + } + } + + [Fact] + public void Stop_preference_cannot_cut_the_dead_letter_file_short() + { + using var transport = MgxTransportScope.Inject(new FirstChunkOnlyHandler()); + var deadLetter = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", Enumerable.Range(1, 25).Select(i => $"/users/u{i}").ToArray()) + .AddParameter("Method", "GET") + .AddParameter("DeadLetterPath", deadLetter) + .AddParameter("ErrorAction", "Stop"); + try { ps.Invoke(); } catch { /* Stop terminates - expected */ } + + // First chunk answered 20 items (all 200 in this fixture), second chunk's POST was + // refused, so its 5 items failed. Every failure must be on disk even though Stop + // terminated at the first error record. + Assert.True(File.Exists(deadLetter), "dead-letter file was not written"); + var lines = File.ReadAllLines(deadLetter); + Assert.Equal(5, lines.Length); + } + finally + { + File.Delete(deadLetter); + } + } + + [Fact] + public void Items_of_a_refused_chunk_each_write_an_error_record() + { + using (MgxTransportScope.Inject(new FirstChunkOnlyHandler())) + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", Enumerable.Range(1, 25).Select(i => $"/users/u{i}").ToArray()) + .AddParameter("Method", "GET"); + ps.Invoke(); + + var errors = ps.Streams.Error.ToList(); + // The second chunk's five items went out and were refused, so each reads as a failed + // request. None of them is "not sent": that is reserved for a chunk never POSTed. + Assert.Equal(5, errors.Count(e => e.FullyQualifiedErrorId.StartsWith("BatchItemError"))); + Assert.DoesNotContain(errors, e => e.FullyQualifiedErrorId.StartsWith("BatchItemNotSent")); + var chunk = Assert.Single(errors, e => e.FullyQualifiedErrorId.StartsWith("BatchChunkFailed")); + // The chunk died on a 400; the record carries the classified category, not NotSpecified. + Assert.Equal(System.Management.Automation.ErrorCategory.InvalidArgument, + chunk.CategoryInfo.Category); + // Nothing was left unsent here - the refused chunk was the last one. The record has + // to say what did happen to its five operations, not only what did not. + Assert.Equal("5 of 25 operations failed, 0 were not sent", chunk.TargetObject); + } + } + + /// One chunk, both items answered, the second sub-response missing its status. + private const string OneOkOneWithoutAStatus = """ + { "responses": [ + { "id": "1", "status": 204, "body": null }, + { "id": "2", "body": null } + ] } + """; + + /// + /// "Not sent" is the cmdlet's word for a write that certainly did not reach the server, and + /// it carries a NotSent flag on the output, an error record telling the caller another chunk + /// failed, and a dead-letter line the help documents as re-pipeable. A PATCH the server + /// answered must reach none of those. + /// + [Fact] + public void A_write_answered_without_a_status_is_not_offered_back_for_resubmission() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, OneOkOneWithoutAStatus); + var deadLetter = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + try + { + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new[] { "/users/u1", "/users/u2" }) + .AddParameter("Method", "PATCH") + .AddParameter("Body", new System.Collections.Hashtable { ["x"] = 1 }) + .AddParameter("DeadLetterPath", deadLetter); + var output = ps.Invoke(); + + var second = Assert.IsType(output[1].BaseObject); + Assert.Equal("/users/u2", second["Url"]); + Assert.NotEqual(0, Assert.IsType(second["Status"])); + Assert.False(second.ContainsKey("NotSent"), + "a write the server answered was flagged as one that never went out"); + + Assert.DoesNotContain(ps.Streams.Error, + e => e.FullyQualifiedErrorId.StartsWith("BatchItemNotSent", StringComparison.Ordinal)); + } + + var line = Assert.Single(File.ReadAllLines(deadLetter), l => l.Contains("/users/u2", StringComparison.Ordinal)); + // Compact, as the writer emits it: the dead-letter serializer does not indent, so a + // needle with a space after the colon matches nothing the file can ever hold. + Assert.DoesNotContain("\"Status\":0", line, StringComparison.Ordinal); + } + finally + { + if (File.Exists(deadLetter)) File.Delete(deadLetter); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/BulkWriteErrorSurfacingTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/BulkWriteErrorSurfacingTests.cs new file mode 100644 index 0000000..e8f8177 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/BulkWriteErrorSurfacingTests.cs @@ -0,0 +1,86 @@ +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// The two error ids a write fan-out can report are a statement about what reached the server. +/// BulkWriteInfraError is for a write that got no HTTP status at all - the network, an open +/// circuit - and it is the one a caller may repeat. Everything else carries the status the +/// server answered with, a body that failed to read after that answer included: the write may +/// have been applied, and repeating it is a decision about duplicates rather than a free retry. +/// +[Collection("Pipeline")] +public class BulkWriteErrorSurfacingTests +{ + private static (List Errors, int Output) Run(string script) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript(script); + var output = ps.Invoke(); + return (ps.Streams.Error.ToList(), output.Count); + } + + /// + /// A 201 whose body does not parse. The POST reached the server and the server applied it; + /// only the read of what it created failed. Reported under the id reserved for a write that + /// got no status, it would read as the one failure a caller can safely repeat - and each + /// repeat creates the entity a second time. + /// + [Fact] + public void A_body_that_does_not_parse_after_a_201_keeps_the_status_the_server_answered() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.Created, "THIS IS NOT JSON"); + using (MgxTransportScope.Inject(wire)) + { + var (errors, _) = Run( + "'u1','u2' | Invoke-MgxRequest -Uri '/users/{id}' -Method POST -Body @{ x = 1 } -Confirm:$false"); + + Assert.Equal(2, errors.Count); + Assert.DoesNotContain(errors, + e => e.FullyQualifiedErrorId.StartsWith("BulkWriteInfraError", StringComparison.Ordinal)); + Assert.All(errors, e => + Assert.StartsWith("BulkWriteError", e.FullyQualifiedErrorId, StringComparison.Ordinal)); + Assert.All(errors, e => + Assert.Contains("HTTP 201", e.Exception.Message, StringComparison.Ordinal)); + } + } + + /// Fails every request before a status exists, the way a dead route does. + private sealed class NeverConnectsHandler : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + => throw new HttpRequestException("no route to host"); + } + + /// + /// The other half of the same statement: a write that never reached a status is the one that + /// takes the infrastructure branch, and it still does. + /// + [Fact] + public void A_write_that_reached_no_status_is_the_one_reported_as_infrastructure() + { + using (MgxTransportScope.Inject(new NeverConnectsHandler(), options: new ResilientGraphClientOptions + { + NoRateLimit = true, + MaxRetryAttempts = 1 + })) + { + var (errors, _) = Run( + "'u1','u2' | Invoke-MgxRequest -Uri '/users/{id}' -Method POST -Body @{ x = 1 } -Confirm:$false"); + + Assert.NotEmpty(errors); + Assert.All(errors, e => + Assert.StartsWith("BulkWriteInfraError", e.FullyQualifiedErrorId, StringComparison.Ordinal)); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DeltaCheckpointIntegrityTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DeltaCheckpointIntegrityTests.cs index 7ae119a..dac7056 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/DeltaCheckpointIntegrityTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/DeltaCheckpointIntegrityTests.cs @@ -1,6 +1,5 @@ using System.Management.Automation; using System.Net; -using System.Reflection; using Mgx.Cmdlets.Base; using Mgx.Engine.Http; using Mgx.Engine.Pagination; @@ -32,34 +31,15 @@ public class DeltaCheckpointIntegrityTests """; private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; - private static void InjectMock(MockHttpHandler handler) - { - ResiliencePipelineFactory.Reset(); - var t = typeof(MgxCmdletBase); - t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, new HttpClient(handler)); - t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - // Cleared with the fingerprint. A stale reference from an earlier test reads as a - // credential change and the injected client is rebuilt away - t.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_ownsHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, false); - t.GetField("s_graphEndpoint", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! - .SetValue(null, "https://graph.microsoft.com"); - t.GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! - .SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); - } - - private static void CleanupMock() - { - var t = typeof(MgxCmdletBase); - t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - ResiliencePipelineFactory.Reset(); - } + private const string EmptyChangesPage = """ + {"value":[],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B3"} + """; + private const string ChangesPage3Linked = """ + {"value":[{"id":"b4"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B4"} + """; + private const string EmptyFinalPage = """ + {"value":[],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D2"} + """; private static PowerShell Shell() { @@ -74,6 +54,38 @@ private static PowerShell Shell() return ps; } + /// + /// The shape a delta enumeration is full of: pages behind nextLinks with nothing in them, + /// including the last one before the deltaLink. PageIterator raises its empty-page limit to + /// 1000 for exactly this, so the sync walks them rather than giving up. Every answer takes + /// a real asynchronous hop - a socket read always yields and Task.FromResult never does, + /// and the thread the iterator resumes on is what these pages are here to settle. + /// + private sealed class EmptyMiddlePageHandler : HttpMessageHandler + { + private readonly object _lock = new(); + public List Urls { get; } = []; + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + lock (_lock) { Urls.Add(url); } + + await Task.Yield(); + + var body = url.Contains("skiptoken=B4", StringComparison.Ordinal) ? EmptyFinalPage + : url.Contains("skiptoken=B3", StringComparison.Ordinal) ? ChangesPage3Linked + : url.Contains("skiptoken=B2", StringComparison.Ordinal) ? EmptyChangesPage + : ChangesPage1; + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") + }; + } + } + private sealed record Env(string Dir, string DeltaPath, string CheckpointPath, string OutputPath); private static Env NewEnv(string checkpointName = "run.checkpoint") @@ -111,6 +123,37 @@ private static void Sync(Env env) private static string[] Ids(string outputPath) => File.Exists(outputPath) ? File.ReadAllLines(outputPath) : []; + private static void SyncToPipeline(Env env) + { + using var ps = Shell(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", env.DeltaPath) + .AddParameter("CheckpointPath", env.CheckpointPath); + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + } + + /// + /// A sync whose warnings and error records the caller reads. Nothing is swallowed here: a + /// run that dies off the pipeline thread has to fail the test that says it does not. + /// + private static (List Warnings, int Errors) SyncReporting(Env env) + { + using var ps = Shell(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", env.DeltaPath) + .AddParameter("CheckpointPath", env.CheckpointPath) + .AddParameter("OutputFile", env.OutputPath); + ps.Invoke(); + return ([.. ps.Streams.Warning.Select(w => w.Message)], ps.Streams.Error.Count); + } + + /// A checkpoint torn between the write and the rename: no position in it at all. + private const string TornCheckpoint = + """{"resource":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D1","nextLi"""; + // ---------------------------------------------------------------- D5 /// @@ -130,7 +173,7 @@ public void A_torn_temp_does_not_let_the_token_advance_past_what_it_lost() handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 2 re-enumerates handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); Sync(env); var temp = Directory.GetFiles(env.Dir, "out.jsonl.*.tmp").Single(); @@ -149,7 +192,7 @@ public void A_torn_temp_does_not_let_the_token_advance_past_what_it_lost() ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], Ids(env.OutputPath)); } - finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + finally { try { Directory.Delete(env.Dir, true); } catch { } } } // ---------------------------------------------------------------- D6 @@ -174,7 +217,7 @@ public void The_checkpoints_own_staging_file_is_never_adopted_as_data() handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); Sync(env); var temp = Directory.GetFiles(env.Dir, "out.jsonl.*.tmp") @@ -192,7 +235,7 @@ public void The_checkpoints_own_staging_file_is_never_adopted_as_data() ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], Ids(env.OutputPath)); } - finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + finally { try { Directory.Delete(env.Dir, true); } catch { } } } // ---------------------------------------------------------------- D7 @@ -218,7 +261,7 @@ public void A_truncated_output_is_not_mistaken_for_the_baseline_it_describes() handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 3 re-enumerates handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); handler.QueueResponse(HttpStatusCode.OK, ChangesPage3); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); Sync(env); Sync(env); @@ -237,7 +280,7 @@ public void A_truncated_output_is_not_mistaken_for_the_baseline_it_describes() ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}", "{\"id\":\"b4\"}"], Ids(env.OutputPath)); } - finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + finally { try { Directory.Delete(env.Dir, true); } catch { } } } // ---------------------------------------------------------------- D8 @@ -261,7 +304,7 @@ public void Items_written_after_the_last_checkpoint_are_not_duplicated_on_resume handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.OK, ChangesPage3); // run 3: b4 - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); Sync(env); Sync(env); @@ -279,7 +322,7 @@ public void Items_written_after_the_last_checkpoint_are_not_duplicated_on_resume ids); Assert.Equal(ids.Length, ids.Distinct().Count()); } - finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + finally { try { Directory.Delete(env.Dir, true); } catch { } } } // ---------------------------------------------------------------- D9 @@ -303,7 +346,7 @@ public void A_temp_that_survives_its_own_adoption_is_not_merged_twice() handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // run 2: adopts, then dies handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); // run 3: b3 - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); Sync(env); var temp = Directory.GetFiles(env.Dir, "out.jsonl.*.tmp").Single(); @@ -322,7 +365,7 @@ public void A_temp_that_survives_its_own_adoption_is_not_merged_twice() ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], Ids(env.OutputPath)); } - finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + finally { try { Directory.Delete(env.Dir, true); } catch { } } } /// @@ -348,7 +391,7 @@ public void A_checkpoint_naming_the_output_as_its_own_temp_does_not_consume_it() var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); Sync(env); @@ -357,7 +400,7 @@ public void A_checkpoint_naming_the_output_as_its_own_temp_does_not_consume_it() ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], Ids(env.OutputPath)); } - finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + finally { try { Directory.Delete(env.Dir, true); } catch { } } } /// @@ -385,7 +428,7 @@ public void A_checkpoint_naming_a_file_that_is_not_a_temp_does_not_consume_it() var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); Sync(env); @@ -394,7 +437,7 @@ public void A_checkpoint_naming_a_file_that_is_not_a_temp_does_not_consume_it() ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], Ids(env.OutputPath)); } - finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + finally { try { Directory.Delete(env.Dir, true); } catch { } } } // ---------------------------------------------------------------- D10 @@ -414,7 +457,7 @@ public void Adoption_takes_the_temp_its_checkpoint_describes_not_the_newest_one( handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); Sync(env); // leaves its own temp with b1,b2 var own = Directory.GetFiles(env.Dir, "out.jsonl.*.tmp").Single(); @@ -431,7 +474,7 @@ public void Adoption_takes_the_temp_its_checkpoint_describes_not_the_newest_one( Assert.Contains("{\"id\":\"b1\"}", ids); Assert.Contains("{\"id\":\"b2\"}", ids); } - finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + finally { try { Directory.Delete(env.Dir, true); } catch { } } } // ---------------------------------------------------------------- D11 @@ -450,7 +493,7 @@ public void A_non_object_item_in_a_page_does_not_terminate_the_run() var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, """{"value":[{"id":"b1"},"not-an-object",{"id":"b2"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D2"}"""); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); Exception? escaped = null; using (var ps = Shell()) @@ -469,7 +512,7 @@ public void A_non_object_item_in_a_page_does_not_terminate_the_run() Assert.Contains("{\"id\":\"b1\"}", ids); Assert.Contains("{\"id\":\"b2\"}", ids); } - finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + finally { try { Directory.Delete(env.Dir, true); } catch { } } } // ---------------------------------------------------------------- D12 @@ -517,7 +560,7 @@ static string Page(string nextLink) => handler.QueueResponse(HttpStatusCode.OK, Page(Page2Link)); handler.QueueResponse(HttpStatusCode.OK, """{"value":[{"id":"tail"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D2"}"""); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); Sync(env); @@ -534,6 +577,195 @@ static string Page(string nextLink) => Assert.Equal(1001, ids.Length); Assert.Equal("{\"id\":\"tail\"}", ids[^1]); } - finally { CleanupMock(); try { Directory.Delete(env.Dir, true); } catch { } } + finally { try { Directory.Delete(env.Dir, true); } catch { } } + } + + /// + /// PaginationCheckpoint.Load answers null for a file torn by a crash and for one that is + /// locked or that this account cannot open, and the sync does the same thing either way: + /// resume stays null, so it re-enumerates from the delta token. Deleting the file changed + /// nothing about the run and destroyed a position the next run, or another account, could + /// still have resumed from. In file mode the delete came out of the branch that reads + /// "output file is missing" - a reading of contents, off a checkpoint with none to read. + /// + [Fact] + public void A_checkpoint_this_sync_cannot_read_is_left_for_a_sync_that_can() + { + var env = NewEnv(); + try + { + File.WriteAllText(env.CheckpointPath, TornCheckpoint); + var before = File.ReadAllBytes(env.CheckpointPath); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + + // The interrupted run never promoted its output, so this is the branch that used to + // call the checkpoint stale for it. The sync then fails on its first page, so + // nothing on the success path can be what removes the file. + Assert.False(File.Exists(env.OutputPath)); + Sync(env); + + Assert.True(File.Exists(env.CheckpointPath), + "a position this sync could not read was deleted anyway"); + Assert.Equal(before, File.ReadAllBytes(env.CheckpointPath)); + } + finally { try { Directory.Delete(env.Dir, true); } catch { } } + } + + /// + /// The same checkpoint against a sync writing to the pipeline, which reaches the second of + /// the two deletes and took the file with no warning on any stream. + /// + [Fact] + public void A_checkpoint_a_pipeline_sync_cannot_read_is_left_for_a_sync_that_can() + { + var env = NewEnv(); + try + { + File.WriteAllText(env.CheckpointPath, TornCheckpoint); + var before = File.ReadAllBytes(env.CheckpointPath); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + + SyncToPipeline(env); + + Assert.True(File.Exists(env.CheckpointPath), + "a position this sync could not read was deleted anyway"); + Assert.Equal(before, File.ReadAllBytes(env.CheckpointPath)); + } + finally { try { Directory.Delete(env.Dir, true); } catch { } } + } + + /// + /// A checkpoint recording no nextLink says something an unreadable one cannot: the sync + /// that wrote it got to the end. That one is still deleted by a run that gets nowhere, + /// because -Latest is suppressed by the checkpoint file merely existing - a marker left + /// lying there costs the next -Latest run the baseline it asked for. + /// + [Fact] + public void A_checkpoint_that_says_the_previous_sync_finished_is_deleted() + { + var env = NewEnv(); + try + { + new PaginationCheckpoint + { + Resource = DeltaLink1, + NextLink = null, + ItemsCollected = 3, + PageItemsAlreadyWritten = 0, + TempFile = null, + DataLength = null, + }.Save(env.CheckpointPath); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + + SyncToPipeline(env); + + Assert.False(File.Exists(env.CheckpointPath)); + } + finally { try { Directory.Delete(env.Dir, true); } catch { } } + } + + /// + /// The last step of a completed sync is the move that puts its temp over the output, and it + /// can fail with every page already fetched: a destination another process holds open, a + /// read-only file, a share that dropped. The checkpoint goes only once that move has + /// landed, so a failure leaves both files exactly as an interruption leaves them and the + /// next run resumes from them instead of re-enumerating from the delta token. + /// + [Fact] + public void A_failed_promotion_keeps_the_checkpoint_and_the_temp_holding_the_sync() + { + var env = NewEnv(); + try + { + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 1: b1,b2 + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); // run 1: b3, then the deltaLink + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2Final); // run 2, resumed + using var transport = MgxTransportScope.Inject(handler); + + // Something at the output path the promotion cannot replace, so both pages are + // fetched and written and the move at the end of the run is what fails. + Directory.CreateDirectory(env.OutputPath); + + Sync(env); + + Assert.True(File.Exists(env.CheckpointPath), + "the failed promotion deleted the position the next run resumes from"); + var cp = PaginationCheckpoint.Load(env.CheckpointPath); + Assert.NotNull(cp); + Assert.NotNull(cp!.TempFile); + var temp = Path.Combine(env.Dir, cp.TempFile!); + Assert.True(File.Exists(temp), + "the failed promotion deleted the temp holding every item the sync fetched"); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(temp)); + + // With the occupier gone: the temp is promoted, one page is fetched - the one the + // checkpoint recorded - and the enumeration does not start over. + Directory.Delete(env.OutputPath); + var before = handler.RequestCount; + Sync(env); + + Assert.Equal(1, handler.RequestCount - before); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], Ids(env.OutputPath)); + } + finally { try { Directory.Delete(env.Dir, true); } catch { } } + } + + /// + /// A checkpoint the sync cannot write, over pages that are empty behind nextLinks. The + /// page-boundary callback runs on whichever thread the iterator resumed on, and a page that + /// writes nothing resumes on the thread pool - so warning about the disk from there ended + /// the sync at that page: the items behind it were never fetched, and the delta token never + /// advanced, leaving the caller to re-enumerate from the old one. The message is buffered + /// on the client's warning channel and written by a drain on the pipeline thread: one per + /// failed save, and the enumeration runs to the deltaLink. + /// + /// The last save fails at the boundary before the empty final page, so no item follows it + /// and the per-item drain never comes round again. The drain on the way out is what carries + /// that one. + /// + /// + [Fact] + public void A_failed_boundary_save_warns_and_the_sync_goes_on() + { + var env = NewEnv(); + try + { + // Every save into this path fails: Save stages its bytes under this name first, and + // no account writes a file over a directory. The directory holding it is not sealed + // instead, the way a read-only disk would seal it, because the sync writes its + // output and its delta state there too and neither write is what is under test. + Directory.CreateDirectory(env.CheckpointPath + ".tmp"); + + var handler = new EmptyMiddlePageHandler(); + using var transport = MgxTransportScope.Inject(handler); + var (warnings, errors) = SyncReporting(env); + + Assert.Equal(0, errors); + + // Each of the three boundaries carrying a nextLink tries to save, and each fails. + Assert.Equal(3, warnings.Count( + w => w.StartsWith("Checkpoint save failed (page boundary)", StringComparison.Ordinal))); + + // The empty page sits behind the second request, so a sync that died reporting its + // failed save never asks for the rest - where the items and the token both are. + Assert.Equal(4, handler.Urls.Count); + Assert.Contains("skiptoken=B4", handler.Urls[3]); + Assert.Equal( + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b4\"}"], + Ids(env.OutputPath)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(env.DeltaPath)!.DeltaLink); + } + finally { try { Directory.Delete(env.Dir, true); } catch { } } } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DeltaQueryTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DeltaQueryTests.cs index 7b9cc5f..5d31c39 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/DeltaQueryTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/DeltaQueryTests.cs @@ -194,6 +194,7 @@ public async Task PageIterator_PreservesRemovedAnnotation() Assert.Equal("deleted", removed.GetProperty("reason").GetString()); // @removed does NOT start with @odata. so it survives JsonToHashtable's // @odata.* stripping filter + Assert.False("@removed".StartsWith("@odata.", StringComparison.OrdinalIgnoreCase)); } // --- DeltaState persistence --- @@ -559,12 +560,204 @@ public async Task Delta_OutputFile_WritesJsonlFormat() // --- #4: OutputFile temp file cleanup on error --- + [Fact] + public async Task Delta_OutputFile_TempFileCleanedUpOnError() + { + ResiliencePipelineFactory.Reset(); + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, DeltaPage1); + // Queue enough 500s to exhaust Polly retries (1 initial + 1 retry = 2 attempts) + handler.QueueResponse(HttpStatusCode.InternalServerError); + handler.QueueResponse(HttpStatusCode.InternalServerError); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions + { + NoRateLimit = true, + MaxRetryAttempts = 1 + }); + + var outputPath = Path.Combine(Path.GetTempPath(), $"delta-cleanup-{Guid.NewGuid()}.jsonl"); + var tmpPath = $"{outputPath}.tmp"; + try + { + var iterator = new PageIterator(client); + var writePath = $"{outputPath}.{Guid.NewGuid():N}.tmp"; + try + { + using (var writer = new StreamWriter(writePath, append: false)) + { + await foreach (var item in iterator.StreamAllWithCountAsync( + "https://graph.microsoft.com/v1.0/users/delta", + 0, + null, + onDeltaLink: _ => { })) + { + writer.WriteLine(item.GetRawText()); + } + } + File.Move(writePath, outputPath, overwrite: true); + } + catch + { + // Simulate the cmdlet's cleanup behavior + try { if (File.Exists(writePath)) File.Delete(writePath); } catch { } + throw; + } + + Assert.Fail("Should have thrown"); + } + catch (GraphServiceException) + { + // Expected: 500 error on page 2 + Assert.False(File.Exists(outputPath), "Final output file should not exist after error"); + } + finally + { + if (File.Exists(outputPath)) File.Delete(outputPath); + } + } + // --- #5: OutputFile atomic rename on success --- + [Fact] + public async Task Delta_OutputFile_AtomicRenameOnSuccess() + { + ResiliencePipelineFactory.Reset(); + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, DeltaPage2WithToken); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + + var outputPath = Path.Combine(Path.GetTempPath(), $"delta-atomic-{Guid.NewGuid()}.jsonl"); + try + { + var iterator = new PageIterator(client); + var writePath = $"{outputPath}.{Guid.NewGuid():N}.tmp"; + using (var writer = new StreamWriter(writePath, append: false)) + { + await foreach (var item in iterator.StreamAllWithCountAsync( + "https://graph.microsoft.com/v1.0/users/delta", + 0, + null, + onDeltaLink: _ => { })) + { + writer.WriteLine(item.GetRawText()); + } + } + // Atomic rename + File.Move(writePath, outputPath, overwrite: true); + + Assert.True(File.Exists(outputPath)); + Assert.False(File.Exists(writePath), "Temp file should be gone after rename"); + Assert.Single(File.ReadAllLines(outputPath)); + } + finally + { + if (File.Exists(outputPath)) File.Delete(outputPath); + } + } + // --- #6: Cancellation does NOT save delta state --- + [Fact] + public async Task Delta_Cancellation_DoesNotSaveDeltaState() + { + ResiliencePipelineFactory.Reset(); + var handler = new MockHttpHandler(); + // Page 1 returns items, page 2 will be cancelled + handler.QueueResponse(HttpStatusCode.OK, DeltaPage1); + handler.SetDefaultResponse(HttpStatusCode.OK, DeltaPage2WithToken); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + + var deltaPath = Path.Combine(Path.GetTempPath(), $"delta-cancel-{Guid.NewGuid()}.json"); + try + { + var cts = new CancellationTokenSource(); + var iterator = new PageIterator(client); + string? capturedDeltaLink = null; + int itemCount = 0; + + await Assert.ThrowsAsync(async () => + { + await foreach (var item in iterator.StreamAllWithCountAsync( + "https://graph.microsoft.com/v1.0/users/delta", + 0, + null, + onDeltaLink: dl => capturedDeltaLink = dl, + cancellationToken: cts.Token)) + { + itemCount++; + if (itemCount >= 1) cts.Cancel(); // Cancel after first item + } + }); + + // Delta state should NOT be saved (cmdlet only saves after successful completion) + Assert.False(File.Exists(deltaPath), "Delta state should not be saved on cancellation"); + } + finally + { + DeltaState.Delete(deltaPath); + } + } + // --- #7: Cancellation cleans up temp file --- + [Fact] + public async Task Delta_Cancellation_CleansUpTempFile() + { + ResiliencePipelineFactory.Reset(); + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, DeltaPage1); + handler.SetDefaultResponse(HttpStatusCode.OK, DeltaPage2WithToken); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + + var outputPath = Path.Combine(Path.GetTempPath(), $"delta-cancel-out-{Guid.NewGuid()}.jsonl"); + var cts = new CancellationTokenSource(); + string? writePath = null; + + try + { + writePath = $"{outputPath}.{Guid.NewGuid():N}.tmp"; + var iterator = new PageIterator(client); + + try + { + using (var writer = new StreamWriter(writePath, append: false)) + { + await foreach (var item in iterator.StreamAllWithCountAsync( + "https://graph.microsoft.com/v1.0/users/delta", + 0, + null, + onDeltaLink: _ => { }, + cancellationToken: cts.Token)) + { + writer.WriteLine(item.GetRawText()); + cts.Cancel(); + } + } + } + catch (OperationCanceledException) + { + // Simulate cmdlet cleanup + try { if (File.Exists(writePath)) File.Delete(writePath); } catch { } + } + + Assert.False(File.Exists(writePath), "Temp file should be cleaned up on cancellation"); + Assert.False(File.Exists(outputPath), "Output file should not exist after cancellation"); + } + finally + { + if (writePath != null && File.Exists(writePath)) File.Delete(writePath); + if (File.Exists(outputPath)) File.Delete(outputPath); + } + } + // --- #8: Delta state NOT saved on GraphServiceException --- [Fact] @@ -694,6 +887,14 @@ await Assert.ThrowsAsync( // --- #12: IOException from file operations --- + [Fact] + public void Delta_OutputFile_IOException_WhenPathInvalid() + { + // Verify that writing to an invalid path throws IOException + var invalidPath = Path.Combine(Path.GetTempPath(), new string('x', 300), "output.jsonl"); + Assert.ThrowsAny(() => new StreamWriter(invalidPath)); + } + // --- #13: No deltaLink received warning scenario --- [Fact] @@ -725,46 +926,6 @@ public async Task Delta_NoDeltaLink_WhenEndpointReturnsNone() // --- Cmdlet-level tests via PowerShell.Create() --- - /// - /// Injects a mock HttpClient into MgxCmdletBase's static fields so Sync-MgxDelta - /// can run without a real Graph connection. Uses reflection because the fields - /// are private static. - /// - private static void InjectMockHttpClient(MockHttpHandler handler) - { - ResiliencePipelineFactory.Reset(); - var baseType = typeof(MgxCmdletBase); - var httpClientField = baseType.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!; - var fingerprintField = baseType.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)!; - var ownsClientField = baseType.GetField("s_ownsHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!; - - var httpClient = new HttpClient(handler); - var endpointField = baseType.GetField("s_graphEndpoint", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)!; - var optionsField = baseType.GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)!; - - httpClientField.SetValue(null, httpClient); - // The cmdlet recomputes the identity from the mocked Get-MgContext (a PSCustomObject - // carrying only TenantId), so the cached fingerprint must be built from an equivalent - // context or GetClient() sees a credential change and discards the injected mock. - fingerprintField.SetValue(null, MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - // Cleared with the fingerprint. A stale reference from an earlier test reads as a - // credential change and the injected client is rebuilt away - baseType.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - ownsClientField.SetValue(null, false); - endpointField.SetValue(null, "https://graph.microsoft.com"); - optionsField.SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true }); - } - - private static void CleanupMockHttpClient() - { - var baseType = typeof(MgxCmdletBase); - baseType.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - baseType.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - baseType.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - ResiliencePipelineFactory.Reset(); - } - private static PowerShell CreateTestShell() { var ps = PowerShell.Create(); @@ -795,7 +956,8 @@ public void Cmdlet_410Gone_DeletesStateAndResyncs() // Second attempt (fresh URL) succeeds with items + deltaLink handler.QueueResponse(HttpStatusCode.OK, DeltaPage2WithToken); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-410-{Guid.NewGuid()}.json"); try @@ -828,7 +990,6 @@ public void Cmdlet_410Gone_DeletesStateAndResyncs() finally { DeltaState.Delete(deltaPath); - CleanupMockHttpClient(); } } @@ -842,11 +1003,9 @@ public void Cmdlet_OutputFile_TempFileCleanedUpOnError() handler.QueueResponse(HttpStatusCode.InternalServerError, errorBody); // Page 2 fails handler.QueueResponse(HttpStatusCode.InternalServerError, errorBody); // Polly retry also fails - InjectMockHttpClient(handler); - // Override options to limit retries (default 7 would need 8 error responses) - var optField = typeof(MgxCmdletBase).GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)!; - optField.SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); + // Limit retries: the default would need eight error responses queued. + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-tempclean-{Guid.NewGuid()}.json"); var outputPath = Path.Combine(Path.GetTempPath(), $"cmdlet-tempclean-{Guid.NewGuid()}.jsonl"); @@ -861,9 +1020,13 @@ public void Cmdlet_OutputFile_TempFileCleanedUpOnError() // Output file should NOT exist (error before atomic rename) Assert.False(File.Exists(outputPath), "Output file should not exist after error"); - // No .tmp files should remain + // No .tmp of this run's should remain. Named for the output this test generated, + // not for the prefix every process running the suite shares: the temps a sync + // leaves are "{output}.{guid}.tmp", so that name picks out this invocation's and + // nothing else's, and a sibling run's in-flight temp in the shared directory is + // another test's business. var dir = Path.GetDirectoryName(outputPath)!; - var tmpFiles = Directory.GetFiles(dir, "*.tmp").Where(f => f.Contains("cmdlet-tempclean")).ToArray(); + var tmpFiles = Directory.GetFiles(dir, Path.GetFileName(outputPath) + ".*.tmp"); Assert.Empty(tmpFiles); // Should have errors Assert.True(ps.HadErrors, "Should have errors from 500 response"); @@ -872,7 +1035,6 @@ public void Cmdlet_OutputFile_TempFileCleanedUpOnError() { DeltaState.Delete(deltaPath); if (File.Exists(outputPath)) File.Delete(outputPath); - CleanupMockHttpClient(); } } @@ -886,19 +1048,16 @@ public void Cmdlet_BrokenCircuit_WritesErrorRecord() for (int i = 0; i < 20; i++) handler.QueueResponse(HttpStatusCode.InternalServerError); - InjectMockHttpClient(handler); - // Use aggressive CB settings so it trips quickly - var optionsField = typeof(MgxCmdletBase).GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)!; - optionsField.SetValue(null, new ResilientGraphClientOptions - { - NoRateLimit = true, - CircuitBreakerMinThroughput = 2, - CircuitBreakerFailureRatio = 0.5, - CircuitBreakerDurationSeconds = 30, - MaxRetryAttempts = 1, - MaxRetryAfterSeconds = 1 - }); - ResiliencePipelineFactory.Reset(); + // Aggressive circuit-breaker settings so it trips quickly. + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions + { + NoRateLimit = true, + CircuitBreakerMinThroughput = 2, + CircuitBreakerFailureRatio = 0.5, + CircuitBreakerDurationSeconds = 30, + MaxRetryAttempts = 1 + }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-cb-{Guid.NewGuid()}.json"); @@ -925,7 +1084,6 @@ public void Cmdlet_BrokenCircuit_WritesErrorRecord() finally { DeltaState.Delete(deltaPath); - CleanupMockHttpClient(); } } @@ -937,14 +1095,8 @@ public void Cmdlet_HttpRequestException_WritesErrorRecord() handler.QueueException(new HttpRequestException("Connection refused")); handler.QueueException(new HttpRequestException("Connection refused")); // For Polly retry - InjectMockHttpClient(handler); - var optionsField2 = typeof(MgxCmdletBase).GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)!; - optionsField2.SetValue(null, new ResilientGraphClientOptions - { - NoRateLimit = true, - MaxRetryAttempts = 1 - }); - ResiliencePipelineFactory.Reset(); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-http-{Guid.NewGuid()}.json"); @@ -967,7 +1119,6 @@ e.Exception is HttpRequestException || finally { DeltaState.Delete(deltaPath); - CleanupMockHttpClient(); } } @@ -1034,7 +1185,8 @@ public void Cmdlet_Prefer_SentOnEveryPage_AndPersistedNormalized() handler.QueueResponse(HttpStatusCode.OK, DeltaPage1); handler.QueueResponse(HttpStatusCode.OK, DeltaPage2WithToken); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-prefer-{Guid.NewGuid()}.json"); try @@ -1062,7 +1214,6 @@ public void Cmdlet_Prefer_SentOnEveryPage_AndPersistedNormalized() finally { DeltaState.Delete(deltaPath); - CleanupMockHttpClient(); } } @@ -1072,7 +1223,8 @@ public void Cmdlet_PreferDrift_ForcesFullResync_AndDeletesCheckpoint() var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, DeltaPage2WithToken); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-preferdrift-{Guid.NewGuid()}.json"); var cpPath = Path.Combine(Path.GetTempPath(), $"cmdlet-preferdrift-{Guid.NewGuid()}.checkpoint"); @@ -1113,7 +1265,6 @@ public void Cmdlet_PreferDrift_ForcesFullResync_AndDeletesCheckpoint() { DeltaState.Delete(deltaPath); PaginationCheckpoint.Delete(cpPath); - CleanupMockHttpClient(); } } @@ -1123,7 +1274,8 @@ public void Cmdlet_Latest_UsesDeltatokenForm_ForDirectoryResources() var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, EmptyDeltaWithToken); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-latest-dir-{Guid.NewGuid()}.json"); try @@ -1145,7 +1297,6 @@ public void Cmdlet_Latest_UsesDeltatokenForm_ForDirectoryResources() finally { DeltaState.Delete(deltaPath); - CleanupMockHttpClient(); } } @@ -1162,7 +1313,8 @@ public void Cmdlet_Latest_UsesTokenForm_ForDrives() } """); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-latest-drive-{Guid.NewGuid()}.json"); try @@ -1183,7 +1335,6 @@ public void Cmdlet_Latest_UsesTokenForm_ForDrives() finally { DeltaState.Delete(deltaPath); - CleanupMockHttpClient(); } } @@ -1193,7 +1344,8 @@ public void Cmdlet_Latest_IgnoredWhenStateExists() var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, EmptyDeltaWithToken); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-latest-ignored-{Guid.NewGuid()}.json"); try @@ -1220,7 +1372,6 @@ public void Cmdlet_Latest_IgnoredWhenStateExists() finally { DeltaState.Delete(deltaPath); - CleanupMockHttpClient(); } } @@ -1236,10 +1387,8 @@ public void Cmdlet_Checkpoint_SurvivesCrash_ThenResumesAndDeletesOnSuccess() handler.QueueResponse(HttpStatusCode.InternalServerError, errorBody); handler.QueueResponse(HttpStatusCode.InternalServerError, errorBody); - InjectMockHttpClient(handler); - var optField = typeof(MgxCmdletBase).GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)!; - optField.SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-cp-resume-{Guid.NewGuid()}.json"); var cpPath = Path.Combine(Path.GetTempPath(), $"cmdlet-cp-resume-{Guid.NewGuid()}.checkpoint"); @@ -1289,7 +1438,6 @@ public void Cmdlet_Checkpoint_SurvivesCrash_ThenResumesAndDeletesOnSuccess() { DeltaState.Delete(deltaPath); PaginationCheckpoint.Delete(cpPath); - CleanupMockHttpClient(); } } @@ -1302,7 +1450,8 @@ public void Cmdlet_Checkpoint_JsonlOrphanedTemp_IsAdopted() var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, DeltaPage2WithToken); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var stem = $"cmdlet-cp-adopt-{Guid.NewGuid()}"; var deltaPath = Path.Combine(Path.GetTempPath(), $"{stem}.json"); var cpPath = Path.Combine(Path.GetTempPath(), $"{stem}.checkpoint"); @@ -1318,7 +1467,8 @@ public void Cmdlet_Checkpoint_JsonlOrphanedTemp_IsAdopted() Resource = "https://graph.microsoft.com/v1.0/users/delta?$top=999", NextLink = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=page2", ItemsCollected = 2, - PageItemsAlreadyWritten = 0 + PageItemsAlreadyWritten = 0, + OutputFile = outputPath }.Save(cpPath); File.WriteAllLines(tempPath, [ @@ -1348,7 +1498,6 @@ public void Cmdlet_Checkpoint_JsonlOrphanedTemp_IsAdopted() PaginationCheckpoint.Delete(cpPath); if (File.Exists(outputPath)) File.Delete(outputPath); if (File.Exists(tempPath)) File.Delete(tempPath); - CleanupMockHttpClient(); } } @@ -1362,7 +1511,8 @@ public void Cmdlet_410Gone_DeletesCheckpointToo() """{"error":{"code":"deltaTokenExpired","message":"Delta token has expired"}}"""); handler.QueueResponse(HttpStatusCode.OK, DeltaPage2WithToken); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-410cp-{Guid.NewGuid()}.json"); var cpPath = Path.Combine(Path.GetTempPath(), $"cmdlet-410cp-{Guid.NewGuid()}.checkpoint"); @@ -1398,7 +1548,6 @@ public void Cmdlet_410Gone_DeletesCheckpointToo() { DeltaState.Delete(deltaPath); PaginationCheckpoint.Delete(cpPath); - CleanupMockHttpClient(); } } @@ -1408,7 +1557,8 @@ public void Cmdlet_RemovedItems_AreCountedAndPassedThrough() var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, DeltaPageWithRemoved); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var deltaPath = Path.Combine(Path.GetTempPath(), $"cmdlet-removed-{Guid.NewGuid()}.json"); try @@ -1431,7 +1581,6 @@ public void Cmdlet_RemovedItems_AreCountedAndPassedThrough() finally { DeltaState.Delete(deltaPath); - CleanupMockHttpClient(); } } @@ -1470,7 +1619,8 @@ public void Cmdlet_Checkpoint_MidPage_BoundsProgressLostWhenARunDiesInsideAPage( var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, BuildSingleDeltaPage(ItemsInPage)); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var stem = $"cmdlet-cp-midpage-{Guid.NewGuid()}"; var deltaPath = Path.Combine(Path.GetTempPath(), $"{stem}.json"); var cpPath = Path.Combine(Path.GetTempPath(), $"{stem}.checkpoint"); @@ -1513,7 +1663,6 @@ public void Cmdlet_Checkpoint_MidPage_BoundsProgressLostWhenARunDiesInsideAPage( { try { File.Delete(leftover); } catch { } } - CleanupMockHttpClient(); } } [Fact] @@ -1527,7 +1676,8 @@ public void Latest_is_refused_when_the_previous_state_was_discarded() // triggers it. var handler = new MockHttpHandler(); handler.SetDefaultResponse(HttpStatusCode.OK, DeltaPage2WithToken); - InjectMockHttpClient(handler); + using var transport = MgxTransportScope.Inject(handler, + options: new ResilientGraphClientOptions { NoRateLimit = true }); var deltaPath = Path.Combine(Path.GetTempPath(), $"latest-discard-{Guid.NewGuid()}.json"); try @@ -1557,7 +1707,6 @@ public void Latest_is_refused_when_the_previous_state_was_discarded() finally { DeltaState.Delete(deltaPath); - CleanupMockHttpClient(); } } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DeltaRecoveryTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DeltaRecoveryTests.cs index bee9859..9437d1a 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/DeltaRecoveryTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/DeltaRecoveryTests.cs @@ -1,6 +1,5 @@ using System.Management.Automation; using System.Net; -using System.Reflection; using Mgx.Cmdlets.Base; using Mgx.Engine.Http; using Mgx.Engine.Pagination; @@ -32,35 +31,6 @@ public class DeltaRecoveryTests """; private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; - private static void InjectMock(MockHttpHandler handler) - { - ResiliencePipelineFactory.Reset(); - var t = typeof(MgxCmdletBase); - t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, new HttpClient(handler)); - t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - // Cleared with the fingerprint. A stale reference from an earlier test reads as a - // credential change and the injected client is rebuilt away - t.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_ownsHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, false); - t.GetField("s_graphEndpoint", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! - .SetValue(null, "https://graph.microsoft.com"); - t.GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! - .SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); - } - - private static void CleanupMock() - { - var t = typeof(MgxCmdletBase); - t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - ResiliencePipelineFactory.Reset(); - } - private static PowerShell Shell() { var ps = PowerShell.Create(); @@ -112,7 +82,7 @@ public void A_transient_error_midrun_does_not_strand_the_pages_the_checkpoint_co handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // ... and its retry handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); // run 3 resumes - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); try { Sync(deltaPath, checkpointPath, outputPath); @@ -139,7 +109,7 @@ public void A_transient_error_midrun_does_not_strand_the_pages_the_checkpoint_co Assert.False(File.Exists(checkpointPath)); Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); } - finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + finally { try { Directory.Delete(dir, true); } catch { } } } /// @@ -161,7 +131,7 @@ public void A_completed_sync_replaces_the_previous_syncs_output() handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 2: b1,b2 handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); // run 2: b3 - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); try { Sync(deltaPath, checkpointPath, outputPath); @@ -172,7 +142,7 @@ public void A_completed_sync_replaces_the_previous_syncs_output() ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], File.ReadAllLines(outputPath)); } - finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + finally { try { Directory.Delete(dir, true); } catch { } } } /// @@ -198,7 +168,7 @@ public void A_checkpoint_that_does_not_name_its_file_reenumerates_rather_than_gu handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // run 3 re-enumerates handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); try { Sync(deltaPath, checkpointPath, outputPath); @@ -219,7 +189,7 @@ public void A_checkpoint_that_does_not_name_its_file_reenumerates_rather_than_gu File.ReadAllLines(outputPath)); Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); } - finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + finally { try { Directory.Delete(dir, true); } catch { } } } /// @@ -242,7 +212,7 @@ public void A_first_run_that_dies_midway_still_recovers_its_pages() handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); try { Sync(deltaPath, checkpointPath, outputPath); @@ -256,7 +226,53 @@ public void A_first_run_that_dies_midway_still_recovers_its_pages() Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); } - finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The same guard through the sync's own adoption call. A checkpoint too old to record a + /// length leaves adoption a line count and nothing else, and a temp cut inside the row that + /// count reaches hands the fragment back as a line like any other - so it went into the + /// JSONL output as a change, was reported as a recovery, and the rest of the enumeration + /// was appended behind it. Nothing is lost by refusing: the delta token has not moved. + /// + [Fact] + public void A_sync_does_not_adopt_a_temp_cut_inside_the_row_the_checkpoint_counts() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-recovery-torn-{Guid.NewGuid():N}")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var outputPath = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // A first sync collects page one into its temp and dies on page two. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, outputPath); + var checkpoint = PaginationCheckpoint.Load(checkpointPath)!; + var temp = Path.Combine(dir, checkpoint.TempFile!); + + // The same position as a release that recorded no length left it, and a temp that + // lost its tail after the two rows the checkpoint counts had been written. + File.WriteAllText(temp, "{\"id\":\"b1\"}\n{\"id\":\"b"); + checkpoint.TempFile = null; + checkpoint.DataLength = null; + checkpoint.Save(checkpointPath); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + Sync(deltaPath, checkpointPath, outputPath); + + var lines = File.ReadAllLines(outputPath); + foreach (var line in lines) + System.Text.Json.JsonDocument.Parse(line).Dispose(); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], lines); + } + finally { try { Directory.Delete(dir, true); } catch { } } } /// @@ -281,7 +297,7 @@ public void Latest_is_ignored_while_a_resume_checkpoint_exists() handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // ... and its retry handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); // run 2 resumes - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); try { Sync(deltaPath, checkpointPath, outputPath); @@ -305,15 +321,16 @@ public void Latest_is_ignored_while_a_resume_checkpoint_exists() Assert.Contains("$deltatoken=D2", token); Assert.DoesNotContain("token=latest", token); } - finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + finally { try { Directory.Delete(dir, true); } catch { } } } /// - /// A resume opens the output itself for append, and a denying ACL or read-only bit there - /// raises UnauthorizedAccessException - which does not derive from IOException, so a - /// handler catching only the latter turns an ordinary permission failure into an unhandled - /// error. It must surface the same way an unwritable path does everywhere else: as an - /// AccessDenied error record. + /// A resume appends into the output itself, so a denying ACL or read-only bit there is this + /// run's problem before any request is made: the claim on the file the checkpoint counts its + /// items into fails on permissions, and the run ends on an error record naming that file and + /// the two ways out of it. What it may never be is an unhandled failure - + /// UnauthorizedAccessException does not derive from IOException, and a handler catching only + /// the latter let an ordinary permission problem out of the cmdlet as one. /// [Fact] public void A_denied_output_ends_the_resume_as_an_error_record_not_an_unhandled_failure() @@ -331,7 +348,7 @@ public void A_denied_output_ends_the_resume_as_an_error_record_not_an_unhandled_ handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); // run 3 resumes - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); try { Sync(deltaPath, checkpointPath, outputPath); @@ -356,15 +373,26 @@ public void A_denied_output_ends_the_resume_as_an_error_record_not_an_unhandled_ .AddParameter("DeltaPath", deltaPath) .AddParameter("CheckpointPath", checkpointPath) .AddParameter("OutputFile", outputPath); - var escaped = Record.Exception(() => ps.Invoke()); - - Assert.Null(escaped); - Assert.Contains(ps.Streams.Error, - e => e.FullyQualifiedErrorId.StartsWith("AccessDenied", StringComparison.Ordinal)); + // Only the wrapper a terminating error arrives in is caught here. Any other type + // escaping this call is the unhandled failure this is about, and fails the test. + List errors = []; + try { ps.Invoke(); } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.PermissionDenied, stop.CategoryInfo.Category); + Assert.Contains($"'{outputPath}' could not be opened for writing:", + stop.Exception.Message); + Assert.Contains( + "Grant write access to that file and run again to resume from it, or remove it " + + "and the checkpoint to sync afresh.", + stop.Exception.Message); } finally { - CleanupMock(); try { if (OperatingSystem.IsWindows()) @@ -395,7 +423,7 @@ public void A_run_that_dies_with_no_checkpoint_leaves_no_temp() handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); try { Sync(deltaPath, checkpointPath, outputPath); @@ -403,7 +431,7 @@ public void A_run_that_dies_with_no_checkpoint_leaves_no_temp() Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); Assert.False(File.Exists(outputPath)); } - finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + finally { try { Directory.Delete(dir, true); } catch { } } } /// @@ -427,7 +455,7 @@ public void FullSync_with_Latest_still_baselines_from_now_despite_a_checkpoint() handler.QueueResponse(HttpStatusCode.OK, """{"value":[],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=FROMNOW"}"""); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); try { Sync(deltaPath, checkpointPath, outputPath); @@ -446,7 +474,7 @@ public void FullSync_with_Latest_still_baselines_from_now_despite_a_checkpoint() Assert.Contains("$deltatoken=FROMNOW", DeltaState.Load(deltaPath)!.DeltaLink); Assert.Contains("deltatoken=latest", handler.Requests[^1].RequestUri!.ToString()); } - finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + finally { try { Directory.Delete(dir, true); } catch { } } } /// @@ -466,15 +494,15 @@ public void A_fresh_run_clears_temps_orphaned_by_an_earlier_enumeration() var checkpointPath = Path.Combine(dir, "run.checkpoint"); var outputPath = Path.Combine(dir, "out.jsonl"); - // An orphan from an enumeration that no longer exists, long enough to satisfy any - // later checkpoint's line count. - var foreign = $"{outputPath}.deadbeef.tmp"; + // An orphan from an enumeration that no longer exists, named the way a run names its + // own temp, and long enough to satisfy any later checkpoint's line count. + var foreign = $"{outputPath}.deadbeefdeadbeefdeadbeefdeadbeef.tmp"; File.WriteAllLines(foreign, Enumerable.Range(0, 50).Select(i => $"{{\"id\":\"foreign{i}\"}}")); var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, Baseline); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); try { Sync(deltaPath, checkpointPath, outputPath); @@ -485,7 +513,7 @@ public void A_fresh_run_clears_temps_orphaned_by_an_earlier_enumeration() Assert.Equal(["{\"id\":\"a1\"}", "{\"id\":\"a2\"}"], lines); Assert.DoesNotContain(lines, l => l.Contains("foreign")); } - finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + finally { try { Directory.Delete(dir, true); } catch { } } } /// @@ -524,7 +552,7 @@ public void Resuming_from_a_midpage_checkpoint_repeats_no_items() var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, page2); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); try { Sync(deltaPath, checkpointPath, outputPath); @@ -535,6 +563,6 @@ public void Resuming_from_a_midpage_checkpoint_repeats_no_items() Assert.Equal("{\"id\":\"q0\"}", lines[999]); Assert.Equal("{\"id\":\"q1\"}", lines[1000]); } - finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + finally { try { Directory.Delete(dir, true); } catch { } } } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DeltaSharedCheckpointTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DeltaSharedCheckpointTests.cs new file mode 100644 index 0000000..29e86e2 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/DeltaSharedCheckpointTests.cs @@ -0,0 +1,3365 @@ +using System.Reflection; +using System.Management.Automation; +using System.Net; +using System.Text; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// Two syncs of the same -Uri build the same resource string, so the resource comparison that +/// guards recovery cannot tell them apart. What separates them is the file they collect into, +/// and a checkpoint that records a byte length used to record nothing about which file that +/// length was measured in - so a shared -CheckpointPath let one sync cut the other's output +/// back to its own offset, mid-line, and append the resumed page onto the torn byte. +/// +[Collection("Pipeline")] +public class DeltaSharedCheckpointTests +{ + private const string ChangesPage1 = """ + {"value":[{"id":"b1"},{"id":"b2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2"} + """; + private const string ChangesPage2 = """ + {"value":[{"id":"b3"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D2"} + """; + // One page and no delta token, so a run that reaches it finishes without saving any state - + // and the run after it builds exactly the request URL this one did. + private const string ChangesPageNoToken = """ + {"value":[{"id":"b9"}]} + """; + // What "$deltatoken=latest" answers: nothing to return now, and a token to return from next + // time. + private const string LatestBaseline = """ + {"value":[],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D9"} + """; + private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + + private const string GroupsPage = """ + {"value":[{"id":"g1"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/groups/delta?$deltatoken=G1"} + """; + // A first page with a continuation, so an attempt can reach a page boundary - and save a + // checkpoint of its own naming its own temp - before the request after it answers 410. + private const string GroupsPageOne = """ + {"value":[{"id":"g1"},{"id":"g2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/groups/delta?$skiptoken=G2"} + """; + private const string GroupsResync = """ + {"value":[{"id":"g9"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/groups/delta?$deltatoken=G1"} + """; + private const string TokenExpired = + """{"error":{"code":"deltaTokenExpired","message":"Delta token has expired"}}"""; + + private static readonly string[] OldSync = + ["{\"id\":\"old-0000001\"}", "{\"id\":\"old-0000002\"}", "{\"id\":\"old-0000003\"}"]; + + /// A groups sync holding a token older than Graph keeps them. + private static void StaleState(string deltaPath) => + new DeltaState + { + DeltaLink = "https://graph.microsoft.com/v1.0/groups/delta?$deltatoken=stale", + Resource = "/groups/delta", + GraphEndpoint = "https://graph.microsoft.com", + Select = "", + ApiVersion = "v1.0", + }.Save(deltaPath); + + /// A users sync partway through, holding the token its last run was issued. + private static void UsersState(string deltaPath) => + new DeltaState + { + DeltaLink = "https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D0", + Resource = "/users/delta", + GraphEndpoint = "https://graph.microsoft.com", + Select = "", + ApiVersion = "v1.0", + }.Save(deltaPath); + + /// + /// Runs a sync on this thread and parks it inside the window the hold exists to close: past + /// the reconcile, which has claimed the output and cut it back, and short of the writer that + /// used to open a handle of its own a page fetch later. The verbose the resume announces + /// itself with is written from exactly there, and a subscriber to that stream blocks the + /// pipeline where it is raised. + /// + private static (string[] Warnings, ErrorRecord[] Errors) SyncParkedAtResume( + string deltaPath, string checkpointPath, string outputPath, + ManualResetEventSlim arrived, ManualResetEventSlim released) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + ps.Streams.Verbose.DataAdding += (_, e) => + { + if (e.ItemAdded is VerboseRecord v + && v.Message.Contains("Resuming delta enumeration", StringComparison.Ordinal) + && !arrived.IsSet) + { + arrived.Set(); + released.Wait(); + } + }; + + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("OutputFile", outputPath) + .AddParameter("Verbose"); + List errors = []; + try { ps.Invoke(); } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + return ([.. ps.Streams.Warning.Select(w => w.Message)], [.. errors]); + } + + /// + /// A sync built and waiting to be let go: the runspace is up and the module imported before + /// is set, so what releases is two runs + /// arriving at the reconcile together rather than two runspaces starting up. + /// + private static (string[] Warnings, ErrorRecord[] Errors) SyncReleasedWith(string deltaPath, + string checkpointPath, string outputPath, + ManualResetEventSlim ready, ManualResetEventSlim go) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("OutputFile", outputPath); + + ready.Set(); + go.Wait(); + List errors = []; + try { ps.Invoke(); } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + return ([.. ps.Streams.Warning.Select(w => w.Message)], [.. errors]); + } + + /// + /// Answers by URL rather than from a queue, and holds the first request that arrives until it + /// is let go. Two runs racing over one command line each get the page they ask for however + /// they interleave, and the one that reaches the wire - which is the one that got past the + /// reconcile - keeps the output for as long as the test needs it to. + /// + private sealed class GatedChangesByUrlHandler : HttpMessageHandler + { + private int _served; + + /// How many requests this handler has answered or is holding. + internal int Served => Volatile.Read(ref _served); + + /// Set the moment the first request arrives. + internal ManualResetEventSlim Arrived { get; } = new(false); + + /// Set by the test to let the parked request answer. + internal ManualResetEventSlim Released { get; } = new(false); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + if (Interlocked.Increment(ref _served) == 1) + { + Arrived.Set(); + Released.Wait(cancellationToken); + } + var body = request.RequestUri!.ToString() + .Contains("skiptoken=B2", StringComparison.Ordinal) ? ChangesPage2 : ChangesPage1; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } + + private static void Sync(string deltaPath, string checkpointPath, string outputPath, + string uri = "/users/delta") + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", uri) + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("OutputFile", outputPath); + // A run that dies is part of the scenario, so a terminating error is expected. + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + } + + /// + /// The same sync with the error stream captured too. A run that will not touch the two files + /// the checkpoint stands for ends on a terminating error, and the id and the sentence are + /// the whole of what the caller has to act on. Both places PowerShell can put that record + /// are read, since which one it uses depends on how the command was added. + /// + private static (string[] Warnings, ErrorRecord[] Errors) SyncRun(string deltaPath, + string checkpointPath, string outputPath, bool latest = false) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("OutputFile", outputPath); + if (latest) ps.AddParameter("Latest"); + List errors = []; + try { ps.Invoke(); } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + return ([.. ps.Streams.Warning.Select(w => w.Message)], [.. errors]); + } + + /// + /// Everything a run that stops may not have moved: both state files, the output if there is + /// one, the temp the checkpoint names, and the directory's own entry list - which is what + /// says no temp of the stopped run's, and no probe of its own, outlived it. + /// + private sealed class DiskState + { + public byte[] Checkpoint = []; + public byte[] Temp = []; + public byte[]? Output; + public byte[]? Delta; + public string[] Entries = []; + } + + private static DiskState ReadDisk(string dir, string checkpointPath, string temp, + string outputPath, string deltaPath) => new() + { + Checkpoint = File.ReadAllBytes(checkpointPath), + Temp = ReadShared(temp), + Output = File.Exists(outputPath) ? ReadShared(outputPath) : null, + Delta = File.Exists(deltaPath) ? File.ReadAllBytes(deltaPath) : null, + Entries = [.. Directory.GetFiles(dir).Select(f => Path.GetFileName(f)) + .OrderBy(n => n, StringComparer.Ordinal)], + }; + + /// + /// The same, for the shape a resumed sync leaves: its changes are in the output itself and + /// the checkpoint names no temp, so there is no third file to read. + /// + private static DiskState ReadDisk(string dir, string checkpointPath, string outputPath, + string deltaPath) => new() + { + Checkpoint = File.ReadAllBytes(checkpointPath), + Output = ReadShared(outputPath), + Delta = File.Exists(deltaPath) ? File.ReadAllBytes(deltaPath) : null, + Entries = [.. Directory.GetFiles(dir).Select(f => Path.GetFileName(f)) + .OrderBy(n => n, StringComparer.Ordinal)], + }; + + private static void AssertUnmoved(DiskState before, DiskState after) + { + Assert.Equal(before.Entries, after.Entries); + Assert.Equal(before.Checkpoint, after.Checkpoint); + Assert.Equal(before.Temp, after.Temp); + Assert.Equal(before.Output, after.Output); + Assert.Equal(before.Delta, after.Delta); + } + + /// + /// Reads a file a live run still holds open, offering the sharing that holder asked for. + /// Windows checks the sharing both ways round - what the reader asks of the file, and what + /// it will let a handle already on it go on doing - so File.ReadAllBytes, which asks for + /// FileShare.Read and nothing more, is refused by a run holding the file to write it. + /// + private static byte[] ReadShared(string path) + { + using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + using var ms = new MemoryStream(); + fs.CopyTo(ms); + return ms.ToArray(); + } + + private static string[] SyncWarnings(string deltaPath, string checkpointPath, string outputPath, + string uri = "/users/delta", bool latest = false, + string[]? property = null, string[]? prefer = null, string? filter = null) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", uri) + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("OutputFile", outputPath); + if (latest) ps.AddParameter("Latest"); + if (property != null) ps.AddParameter("Property", property); + if (prefer != null) ps.AddParameter("Prefer", prefer); + if (filter != null) ps.AddParameter("Filter", filter); + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + return [.. ps.Streams.Warning.Select(w => w.Message)]; + } + + /// + /// Sync A is interrupted twice, so the checkpoint it leaves records a length with no temp + /// to explain it. Sync B, its own delta state and its own output, must not have that + /// length applied to its file. + /// + [Fact] + public void A_checkpoint_from_another_sync_does_not_cut_this_ones_output() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-shared-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var outA = Path.Combine(dir, "outA.jsonl"); + var outB = Path.Combine(dir, "outB.jsonl"); + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // A run 1, page 1 + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // A run 1, page 2 + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // A run 2 promotes, then dies + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // B, page 1 + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); // B, page 2 + using var transport = MgxTransportScope.Inject(handler); + try + { + Sync(deltaA, checkpointPath, outA); + Sync(deltaA, checkpointPath, outA); + + var cp = PaginationCheckpoint.Load(checkpointPath)!; + Assert.Null(cp.TempFile); + Assert.Equal(new FileInfo(outA).Length, cp.DataLength); + + // B has its own delta state and its own output, which already holds an earlier sync. + File.WriteAllLines(outB, OldSync); + Sync(deltaB, checkpointPath, outB); + + var lines = File.ReadAllLines(outB); + Assert.All(lines, l => Assert.StartsWith("{\"id\":\"", l)); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], lines); + + // A's own output is untouched by B. + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(outA)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same collision between two directories rather than two names. What distinguishes the + /// syncs is which file they collect into, and a leaf comparison cannot see that: sync B + /// adopted A's position and cut B's own output to A's byte count. + /// + [Fact] + public void A_checkpoint_for_the_same_name_in_another_directory_does_not_cut_this_ones_output() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-shared-dir-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var outA = Path.Combine(Directory.CreateDirectory(Path.Combine(dir, "a")).FullName, "out.jsonl"); + var outB = Path.Combine(Directory.CreateDirectory(Path.Combine(dir, "b")).FullName, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // A run 1, page 1 + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // A run 1, page 2 + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); // A run 2 promotes, then dies + handler.QueueResponse(HttpStatusCode.InternalServerError, ServerError); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); // B, page 1 + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); // B, page 2 + using var transport = MgxTransportScope.Inject(handler); + try + { + Sync(deltaA, checkpointPath, outA); + Sync(deltaA, checkpointPath, outA); + + var cp = PaginationCheckpoint.Load(checkpointPath)!; + Assert.Null(cp.TempFile); + Assert.Equal(new FileInfo(outA).Length, cp.DataLength); + + File.WriteAllLines(outB, OldSync); + Sync(deltaB, checkpointPath, outB); + + var lines = File.ReadAllLines(outB); + Assert.All(lines, l => Assert.StartsWith("{\"id\":\"", l)); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], lines); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(outA)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same sync, resumed with -Uri typed differently. Graph answers "/users/delta" and + /// "/Users/delta" from one collection, so both runs enumerate the same thing - but the + /// recorded resource was compared ordinally, which made the second run a different sync: + /// its own position was refused as another's, the temp holding the changes the first run + /// had already collected was swept, and the enumeration started over. + /// + [Fact] + public void A_checkpoint_written_under_another_spelling_of_the_resource_still_resumes() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-spelling-{Guid.NewGuid():N}")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // Page one reaches the temp; page two fails, so the position survives. + Sync(deltaPath, checkpointPath, output, "/users/delta"); + Assert.NotNull(PaginationCheckpoint.Load(checkpointPath)); + Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + + // The resume, typed the other way, with only the page the dead run never got. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var first = handler.CapturedRequests.Count; + Sync(deltaPath, checkpointPath, output, "/Users/delta"); + + Assert.Contains("skiptoken=B2", handler.CapturedRequests[first].Uri); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + var state = DeltaState.Load(deltaPath); + Assert.NotNull(state); + Assert.Contains("$deltatoken=D2", state!.DeltaLink); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A sync writing to the pipeline was outside the ownership check entirely, so a checkpoint + /// left by a sync collecting into a file resumed it at that sync's nextLink. Every page + /// before that one went to the file and never to the pipeline, and the delta token was + /// saved over them on success - which is the one failure a delta sync cannot recover from, + /// since there is no re-fetch once the token moves. + /// + [Fact] + public void A_pipeline_sync_does_not_resume_from_a_checkpoint_that_collected_into_a_file() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-mode-{Guid.NewGuid():N}")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + try + { + // The other sync's position: page 1 is already in its file, page 2 is what remains. + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users/delta?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = Path.Combine(dir, "elsewhere.jsonl"), + DataLength = 24, + }.Save(checkpointPath); + + using var transport = MgxTransportScope.Inject(new ByUrlHandler()); + + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpointPath); + var emitted = ps.Invoke() + .Select(r => ((System.Collections.Hashtable)r.BaseObject)["id"]!.ToString()!) + .ToArray(); + + // Every change, not just the ones after the other sync's position. + Assert.Equal(["b1", "b2", "b3"], emitted); + // Only now may a token stand for this enumeration. + var state = DeltaState.Load(deltaPath); + Assert.NotNull(state); + Assert.Contains("$deltatoken=D2", state!.DeltaLink); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same checkpoint from a release that did not record the output file yet. A file-mode + /// run names no temp while it is appending, and none once a cancellation has promoted the + /// one it had, so all such a checkpoint says about where its items went is the length it + /// counted. A pipeline sync adopted every one of those: it emitted the tail of the + /// enumeration, saved the delta token over everything before it, and there is no re-fetch + /// once the token has moved. A pipeline run measures no file, so a length is the file mode. + /// + [Fact] + public void A_pipeline_sync_does_not_resume_from_a_checkpoint_that_records_only_a_length() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-legacy-{Guid.NewGuid():N}")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + try + { + // The other sync's position, in the shape 2.1.3 and before wrote it. + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users/delta?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = null, + DataLength = 24, + }.Save(checkpointPath); + + using var transport = MgxTransportScope.Inject(new ByUrlHandler()); + + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpointPath); + var emitted = ps.Invoke() + .Select(r => ((System.Collections.Hashtable)r.BaseObject)["id"]!.ToString()!) + .ToArray(); + + // Every change, not just the ones after the other sync's position. + Assert.Equal(["b1", "b2", "b3"], emitted); + // Only now may a token stand for this enumeration. + var state = DeltaState.Load(deltaPath); + Assert.NotNull(state); + Assert.Contains("$deltatoken=D2", state!.DeltaLink); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// How far "it is left as it is" has to reach. The refused checkpoint's items are in the + /// temp it names, and the stale-temp sweep a few lines later deleted every temp beside the + /// output - including that one. The sync it belongs to then came back to a position + /// pointing at a file that is gone and re-enumerated from the delta token, which is the + /// whole cost the refusal was written to avoid. + /// + [Fact] + public void A_refused_checkpoints_temp_survives_the_run_that_refused_it() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-refused-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // Sync A dies on page 2: its two items are in its temp, the output was never + // promoted, and the checkpoint names both. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + + var cp = PaginationCheckpoint.Load(checkpointPath)!; + Assert.NotNull(cp.TempFile); + var temp = Path.Combine(dir, cp.TempFile!); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(temp)); + var before = File.ReadAllBytes(checkpointPath); + + // Sync B: another collection, its own delta state, the same -OutputFile and + // -CheckpointPath, failing before its first page boundary. + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.Equal(before, File.ReadAllBytes(checkpointPath)); + Assert.True(File.Exists(temp), + "the refused checkpoint's items were swept away by the run that refused it"); + + // A, back where it left off, rather than at the delta token. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + Sync(deltaA, checkpointPath, output); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The refusal's promise where the run that made it goes on to finish. A sync that refused + /// the checkpoint it found and then completed without ever saving one of its own - one page + /// was all it had - deleted the file it had warned a moment earlier it would leave alone, + /// so the temp it had spared the sweep was orphaned and the sync that owns both came back + /// to a position that was no longer there and re-enumerated from the beginning. + /// + [Fact] + public void A_completed_sync_that_saved_no_checkpoint_leaves_the_refused_one_standing() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-donerefused-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // Sync A dies on page 2: its two items are in its temp, and the checkpoint names it. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + + var cp = PaginationCheckpoint.Load(checkpointPath)!; + var temp = Path.Combine(dir, cp.TempFile!); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(temp)); + var before = File.ReadAllBytes(checkpointPath); + + // Sync B: another collection over the same -CheckpointPath and -OutputFile. It + // refuses what it finds and then finishes in one page, so no boundary of its own + // is ever reached and nothing of this run's is written over that path. + handler.QueueResponse(HttpStatusCode.OK, GroupsPage); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.Equal(["{\"id\":\"g1\"}"], File.ReadAllLines(output)); + Assert.True(File.Exists(checkpointPath), + "the run that refused the checkpoint deleted it on the way out"); + Assert.Equal(before, File.ReadAllBytes(checkpointPath)); + Assert.True(File.Exists(temp), + "the refused checkpoint's items outlived the checkpoint that counted them"); + + // A, back where it left off, rather than at page one. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + Sync(deltaA, checkpointPath, output); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And where the refusal has already been voided. This run reached a page boundary, which + /// saved its own position over the same path - so the file the success delete reaches is + /// this run's own, describing an enumeration that has just finished, and leaving it costs + /// the next -Latest run the baseline it asks for. Taking the path over is what a second + /// sync sharing it gets; the delete is not what makes that so. + /// + [Fact] + public void A_completed_sync_that_wrote_over_the_refused_position_still_deletes_it() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-donetookover-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + + // Two pages, so the first boundary saves this run's position over the refused one. + handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); + handler.QueueResponse(HttpStatusCode.OK, GroupsResync); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.Equal(["{\"id\":\"g1\"}", "{\"id\":\"g2\"}", "{\"id\":\"g9\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpointPath), + "a completed sync left its own position into a finished enumeration behind"); + Assert.True(File.Exists(Path.Combine(dir, theirs)), + "the refused checkpoint's staging file was swept by the run that refused it"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The refusal that is not an ownership verdict at all, and what a sync can do about it: + /// nothing. A checkpoint refused because the temp it names could not be claimed counts rows + /// that are in that temp and in no other file, and it is the only thing on disk that counts + /// them. Every way of going on takes them: the sweep at the top of the attempt reclaims the + /// temp the moment its holder is gone, the first page boundary saves this run's position + /// over the checkpoint, the completion door then deletes what it finds there as its own, and + /// a page carrying a deltaLink moves the token past changes held in that temp alone. So the + /// run ends where it finds out, with one page's worth of work in front of it and none of it + /// done. + /// + [Fact] + public void A_held_temp_stops_the_sync_before_anything_is_written() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-heldtemp-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // A sync dies on page 2: its two items are in its temp, and the checkpoint names + // both files. No token was saved, so every run below builds the same request URL + // and the checkpoint is this command line's own by every test there is. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + + var temp = Path.Combine(dir, PaginationCheckpoint.Load(checkpointPath)!.TempFile!); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(temp)); + + // The next run finds that temp held open - by another process, not necessarily + // another sync - with one page of changes waiting for it on the wire. + handler.QueueResponse(HttpStatusCode.OK, ChangesPageNoToken); + string[] warnings; + ErrorRecord[] errors; + DiskState before, after; + int requestsBefore; + using (new FileStream(temp, FileMode.Open, FileAccess.ReadWrite, FileShare.Read)) + { + before = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + requestsBefore = handler.RequestCount; + (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + after = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + } + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointTempHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Contains("Another sync is still writing the temp file", + stop.Exception.Message); + Assert.Contains("the 2 items it records are that run's", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Wait for that sync to finish, or give this one its own -OutputFile and " + + "-CheckpointPath.", + stop.Exception.Message); + + // Nothing that promises to leave the files alone and then goes on to replace them. + Assert.DoesNotContain(warnings, w => w.Contains("re-enumerates")); + Assert.DoesNotContain(warnings, w => w.Contains("Both files are left as they are")); + + // And nothing ran: no request, no output, no token, and both files exactly as the + // holder left them - down to the directory's own entry list, so no temp of this + // run's and no probe survives it either. + Assert.Equal(requestsBefore, handler.RequestCount); + AssertUnmoved(before, after); + Assert.False(File.Exists(output), "the stopped run wrote an output"); + Assert.Null(DeltaState.Load(deltaPath)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same disk with two pages on the wire instead of one, which is the shape that made + /// sparing the file pointless: a refusing run that reaches one page boundary saves its own + /// position over the very path it has just promised to leave, and the door it walks out of + /// deletes that as its own. A run that stops has no page boundary to reach. + /// + [Fact] + public void A_held_temp_stops_the_sync_that_would_have_paged_twice() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-heldtemp2p-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + + var temp = Path.Combine(dir, PaginationCheckpoint.Load(checkpointPath)!.TempFile!); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(temp)); + + // Two pages: a first with a continuation, so the run would save a boundary + // checkpoint of its own, and a second that completes it. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + handler.QueueResponse(HttpStatusCode.OK, ChangesPageNoToken); + ErrorRecord[] errors; + DiskState before, after; + int requestsBefore; + using (new FileStream(temp, FileMode.Open, FileAccess.ReadWrite, FileShare.Read)) + { + before = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + requestsBefore = handler.RequestCount; + (_, errors) = SyncRun(deltaPath, checkpointPath, output); + after = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + } + + Assert.StartsWith("CheckpointTempHeld", Assert.Single(errors).FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(requestsBefore, handler.RequestCount); + AssertUnmoved(before, after); + Assert.Equal(2, PaginationCheckpoint.Load(checkpointPath)!.ItemsCollected); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The shape the delta token turns into a permanent loss. A page carrying a deltaLink is a + /// completed enumeration, so a run that reaches one saves a token over the position the + /// held checkpoint records - and every later run of the same command line then builds a + /// request that checkpoint no longer matches and refuses it as another sync's, for good. A + /// run that stops saves no token, because it fetches no page. + /// + [Fact] + public void A_stopped_sync_saves_no_token_from_the_page_it_never_fetched() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-heldtemptoken-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + var temp = Path.Combine(dir, PaginationCheckpoint.Load(checkpointPath)!.TempFile!); + + // One page, and it finishes the enumeration: had the run taken it, the token would + // have moved past the changes in the temp beside it. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + ErrorRecord[] errors; + DiskState before, after; + using (new FileStream(temp, FileMode.Open, FileAccess.ReadWrite, FileShare.Read)) + { + before = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + (_, errors) = SyncRun(deltaPath, checkpointPath, output); + after = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + } + + Assert.StartsWith("CheckpointTempHeld", Assert.Single(errors).FullyQualifiedErrorId, + StringComparison.Ordinal); + AssertUnmoved(before, after); + Assert.False(File.Exists(deltaPath), "a token was saved for a page nothing fetched"); + + // Which leaves the pair reachable. The holder dies, and the next run of the same + // command line is the one it was kept for. + var recovered = SyncWarnings(deltaPath, checkpointPath, output); + + Assert.Contains(recovered, w => w.Contains("Recovered 2 items")); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The whole cost of getting that wrong, on one disk. Sync A holds its temp with two rows + /// in it; sync B over the same command line stops; A then dies, leaving the two files B + /// left alone. A's re-run is what recovers those rows - it promotes the temp over the + /// output and appends the rest of the enumeration - and it can only do that while the + /// checkpoint counting them is there. + /// + [Fact] + public void The_rows_a_held_temp_holds_come_back_after_the_sync_that_stopped() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-heldtempback-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // A collects page one into its temp and is still going. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + var temp = Path.Combine(dir, PaginationCheckpoint.Load(checkpointPath)!.TempFile!); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(temp)); + + // B stops. Nothing is queued for it, and it asks for nothing. + ErrorRecord[] errors; + using (new FileStream(temp, FileMode.Open, FileAccess.ReadWrite, FileShare.Read)) + { + (_, errors) = SyncRun(deltaPath, checkpointPath, output); + Assert.StartsWith("CheckpointTempHeld", + Assert.Single(errors).FullyQualifiedErrorId, StringComparison.Ordinal); + } + + // A dies, leaving exactly the two files B left alone. + Assert.True(File.Exists(checkpointPath), "B took A's position with it"); + Assert.True(File.Exists(temp), "B took A's temp with it"); + + // A's re-run. It promotes the two rows and resumes from the page its own + // checkpoint recorded, so the row below is appended to them rather than replacing + // them - which is what tells a recovery from a fresh enumeration here. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var warnings = SyncWarnings(deltaPath, checkpointPath, output); + + Assert.Contains(warnings, w => w.Contains("Recovered 2 items")); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// What -Latest is told. A from-now baseline over these two files is a run like any other: + /// the checkpoint is a position an enumeration is still in progress at, so the baseline is + /// refused first, and then the temp it names stops the enumeration that was going to happen + /// instead. Nothing is written either way, and the token the caller asked for is not saved + /// over the changes the temp holds. + /// + [Fact] + public void A_held_temp_stops_a_Latest_baseline_too() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-heldtemplatest-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + var temp = Path.Combine(dir, PaginationCheckpoint.Load(checkpointPath)!.TempFile!); + + handler.QueueResponse(HttpStatusCode.OK, LatestBaseline); + string[] warnings; + ErrorRecord[] errors; + DiskState before, after; + using (new FileStream(temp, FileMode.Open, FileAccess.ReadWrite, FileShare.Read)) + { + before = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + (warnings, errors) = SyncRun(deltaPath, checkpointPath, output, latest: true); + after = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + } + + var refused = Assert.Single(warnings, w => w.Contains("-Latest ignored")); + Assert.Contains("a resume checkpoint exists", refused); + Assert.StartsWith("CheckpointTempHeld", Assert.Single(errors).FullyQualifiedErrorId, + StringComparison.Ordinal); + AssertUnmoved(before, after); + Assert.False(File.Exists(deltaPath), "a from-now token was saved over the temp's rows"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other half of the same refusal, where there is no second sync at all: a temp this + /// account cannot open read-write. The advice - grant write access, run again - is the only + /// thing that recovers those rows, so the run giving it must still leave the position they + /// are counted by. It stops rather than enumerate over it. This follows the advice and + /// measures what comes back. + /// + [Fact] + public void An_unopenable_temp_stops_the_sync_and_says_how_to_recover_it() + { + if (OperatingSystem.IsWindows()) return; + + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-unopenabletemp-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + var temp = Path.Combine(dir, PaginationCheckpoint.Load(checkpointPath)!.TempFile!); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(temp)); + + // 0444: the owner cannot open it read-write either, which is the claim failing on + // permissions rather than on another handle. + File.SetUnixFileMode(temp, UnixFileMode.UserRead); + + // Nothing queued: the run under test asks for nothing, and the page below belongs + // to the run that follows its advice. + var before = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + var requestsBefore = handler.RequestCount; + var (_, errors) = SyncRun(deltaPath, checkpointPath, output); + var after = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointTempUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Contains("cannot be opened for writing by this account", stop.Exception.Message); + Assert.Contains("not on sharing", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Grant write access to that file and run again to recover them, or remove it " + + "and the checkpoint to sync afresh.", + stop.Exception.Message); + Assert.Equal(requestsBefore, handler.RequestCount); + AssertUnmoved(before, after); + + // Follow it. + File.SetUnixFileMode(temp, UnixFileMode.UserRead | UnixFileMode.UserWrite); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var recovered = SyncWarnings(deltaPath, checkpointPath, output); + + Assert.Contains(recovered, w => w.Contains("Recovered 2 items")); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + } + finally + { + try + { + foreach (var f in Directory.EnumerateFiles(dir)) + File.SetUnixFileMode(f, UnixFileMode.UserRead | UnixFileMode.UserWrite); + Directory.Delete(dir, true); + } + catch { } + } + } + + /// + /// And the 410 door, which used to have a reading of its own to make here. It no longer + /// sees this case at all: a run that stops at the reconcile has not sent a request, so an + /// expired token is not something it can have been told. The door's own sentence said the + /// checkpoint was left alone and the same run then deleted it, which is the second way the + /// spared pair went. + /// + [Fact] + public void A_stopped_sync_never_reaches_the_410_door() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-heldtemp410-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // No delta token is ever saved, so every run below builds the same request URL and + // the checkpoint this one leaves is that command line's own by every test there is. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + var temp = Path.Combine(dir, PaginationCheckpoint.Load(checkpointPath)!.TempFile!); + + // An expired token and the full re-sync behind it, both waiting on the wire. + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + handler.QueueResponse(HttpStatusCode.OK, ChangesPageNoToken); + string[] warnings; + ErrorRecord[] errors; + DiskState before, after; + int requestsBefore; + using (new FileStream(temp, FileMode.Open, FileAccess.ReadWrite, FileShare.Read)) + { + before = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + requestsBefore = handler.RequestCount; + (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + after = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + } + + Assert.StartsWith("CheckpointTempHeld", Assert.Single(errors).FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.DoesNotContain(warnings, w => w.Contains("410 Gone")); + Assert.Equal(requestsBefore, handler.RequestCount); + AssertUnmoved(before, after); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// How far "spare the temps a refused checkpoint names" reaches. The sweep only ever + /// deletes names a run gives its own temp - the output's name, a dot, 32 hex digits, + /// ".tmp" - so a refused checkpoint naming "out.jsonl.{guid}.tmp" beside an output called + /// "out" names a file this sweep could not touch either way. Skipping on it bought that + /// file nothing and cost this output its own orphans, which the pre-length adoption path + /// then picks up on a line count alone. + /// + [Fact] + public void A_refused_temp_this_sweep_could_never_reach_does_not_spare_this_outputs_orphans() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-sweep-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var mine = Path.Combine(dir, "out"); + var theirs = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // The sync into "out" collects page one into its temp and dies on page two. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, mine); + var orphan = Assert.Single(Directory.GetFiles(dir, "out.*.tmp")); + + // The sync next door takes the shared checkpoint over: it refuses what it finds, + // saves its own position at its first page boundary, and dies on page two. The + // first sync's temp is now an orphan - nothing on disk describes it. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaB, checkpointPath, theirs); + var theirTemp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + + // "out" again. It refuses a checkpoint naming a temp beside a different output, + // and that name must not stand in the way of its own sweep. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + Sync(deltaA, checkpointPath, mine); + + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(mine)); + Assert.False(File.Exists(orphan), + "a temp name the sweep could never reach suppressed this output's sweep"); + Assert.True(File.Exists(theirTemp), + "the refused checkpoint's own staging file was deleted"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// What each refusal says. A checkpoint recording another sync's output is a second sync + /// over one -CheckpointPath, and naming that is what the caller acts on. One recording no + /// output at all is not: it was believed while the files beside it corroborated it, and a + /// single sync whose temp has since gone, or whose output has been replaced, reaches the + /// same refusal. Every release before this one recorded no output, so that is the ordinary + /// upgrade path - and it was told there was a second sync somewhere and to give each its + /// own -CheckpointPath, which fixes nothing it has. + /// + [Fact] + public void A_refusal_reports_the_cause_it_can_show() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-refusal-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + File.WriteAllLines(output, OldSync); + + // The shape an interrupted fresh sync left before outputs were recorded, whose temp + // is no longer beside it. + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users/delta?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = $"out.jsonl.{Guid.NewGuid():N}.tmp", + OutputFile = null, + DataLength = 24, + }.Save(checkpointPath); + + var uncorroborated = SyncWarnings(deltaPath, checkpointPath, output); + Assert.Contains(uncorroborated, w => w.Contains("no longer corroborate")); + Assert.DoesNotContain(uncorroborated, + w => w.Contains("records an enumeration this run cannot resume from")); + + // The same position, recording an output this run is not collecting into. + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users/delta?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = Path.Combine(dir, "elsewhere.jsonl"), + DataLength = 24, + }.Save(checkpointPath); + + Assert.Contains(SyncWarnings(deltaPath, checkpointPath, output), + w => w.Contains("records an enumeration this run cannot resume from")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And the cause it cannot show. A run interrupted partway through an incremental sync + /// leaves a checkpoint recording the delta URL it was enumerating. Lose the state under it - + /// corrupt, deleted by hand, a -DeltaPath moved - and the same command line comes back + /// building a full-sync URL, which is not that one, so it refuses its own earlier position. + /// The refusal is right; the reading was not. It named a different sync, over a + /// -CheckpointPath no second sync has ever touched, and told the caller to give each of them + /// its own - which is the arrangement they were already running. + /// + [Fact] + public void A_refusal_does_not_name_a_second_sync_it_cannot_see() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-selfrefusal-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // One sync, one -CheckpointPath: interrupted on page 2 of its incremental pass, so + // what it leaves records the delta URL it was reading. + UsersState(deltaPath); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + Assert.Contains("$deltatoken=D0", PaginationCheckpoint.Load(checkpointPath)!.Resource); + + // The state goes, and the same command line enumerates in full. + File.Delete(deltaPath); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var refusal = Assert.Single( + SyncWarnings(deltaPath, checkpointPath, output), + w => w.Contains(checkpointPath)); + + Assert.Contains("records an enumeration this run cannot resume from", refusal); + Assert.Contains("either another sync's, sharing this -CheckpointPath", refusal); + Assert.Contains("this sync's own from a pass it no longer makes", refusal); + Assert.DoesNotContain("belongs to a different sync", refusal); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A 410 says the token this run holds is dead. It says nothing about a checkpoint this + /// same run has already decided belongs to another sync, and deleting that one took the + /// other sync's position away seconds after warning that it would be left alone - while the + /// retry, running the stale-temp sweep a second time with the refusal forgotten, took the + /// temp holding its items too. Both halves of the other sync's progress, on the one path + /// that promised to touch neither. + /// + [Fact] + public void An_expired_token_does_not_delete_the_checkpoint_this_run_refused() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410refused-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // Sync A dies on page 2: its two items are in its temp, and the checkpoint names + // both files. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + + var cp = PaginationCheckpoint.Load(checkpointPath)!; + var temp = Path.Combine(dir, cp.TempFile!); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(temp)); + var before = File.ReadAllBytes(checkpointPath); + + // Sync B: another collection over the same -CheckpointPath and -OutputFile, holding + // a token Graph answers 410 for. It refuses A's checkpoint, re-syncs in full, and + // dies before a page boundary of its own. + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.True(File.Exists(checkpointPath), + "the expired token deleted a checkpoint this run had refused as another sync's"); + Assert.Equal(before, File.ReadAllBytes(checkpointPath)); + Assert.True(File.Exists(temp), + "the retry swept away the temp the refusal a moment earlier had spared"); + + // A, back where it left off, rather than at the delta token. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + Sync(deltaA, checkpointPath, output); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And the run that refused still gets its own sync done: the 410 retry enumerates in full, + /// collects into the output and saves the token it was issued, with the other sync's temp + /// still where the refusal left it. + /// + [Fact] + public void A_full_resync_after_a_refusal_still_completes_its_own_sync() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410done-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var temp = Path.Combine(dir, PaginationCheckpoint.Load(checkpointPath)!.TempFile!); + + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + handler.QueueResponse(HttpStatusCode.OK, GroupsPage); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.Equal(["{\"id\":\"g1\"}"], File.ReadAllLines(output)); + Assert.Contains("$deltatoken=G1", DeltaState.Load(deltaB)!.DeltaLink); + Assert.True(File.Exists(temp), + "the retry swept away the temp the refusal a moment earlier had spared"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// What the run that refused leaves beside its own output. The attempt a 410 sends round the + /// loop again keeps its temp - a checkpoint counting those items is on disk - and the retry + /// spares it, because the sweep is held off over every temp beside this output for as long + /// as the refusal stands. What ends the holding off was never written: the checkpoint is + /// replaced by the retry's, or deleted by the run that completed, without a word about the + /// file it used to name. So a sync that promoted its output and reported success left a + /// partial copy of the changes it had already collected beside the finished file, for some + /// later sync's sweep to reclaim. + /// + [Fact] + public void A_full_resync_after_a_refusal_leaves_no_temp_of_its_own() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410kept-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // Sync A dies on page 2: the checkpoint names the temp holding its two items. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + + // Sync B refuses that checkpoint, collects a page of its own into a temp, and loses + // its token to a 410 on the request after it. The retry re-syncs in full. + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + handler.QueueResponse(HttpStatusCode.OK, GroupsResync); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.Equal(["{\"id\":\"g9\"}"], File.ReadAllLines(output)); + Assert.Contains("$deltatoken=G1", DeltaState.Load(deltaB)!.DeltaLink); + + // The one temp beside the output is the refused checkpoint's, which this run had no + // business touching either way. + var left = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(theirs, Path.GetFileName(left)); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(left)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same compound with the 410 on the first request of all, so the attempt that dies has + /// saved no checkpoint and its temp is empty. The keep asked only whether a checkpoint was + /// on disk, and the one there is the refused sync's: it counts nothing this run wrote and + /// names a temp of its own, so an empty file was kept on a foreign file's account and left + /// beside the output the retry went on to finish. + /// + [Fact] + public void An_attempt_that_saved_no_checkpoint_keeps_no_temp_on_a_refused_ones_account() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410empty-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + handler.QueueResponse(HttpStatusCode.OK, GroupsResync); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.Equal(["{\"id\":\"g9\"}"], File.ReadAllLines(output)); + var left = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(theirs, Path.GetFileName(left)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And the temp goes at the moment it stops being anything, not at the end of the run. The + /// retry saves its own first page-boundary checkpoint, which names its own temp - so the + /// one the attempt before it left is from then on counted by nothing - and then dies. What + /// is beside the output is the temp the checkpoint names, and the refused sync's. + /// + [Fact] + public void A_retry_deletes_the_temp_its_own_checkpoint_has_stopped_naming() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410release-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + + // The retry reaches a page boundary of its own and dies on the page after it. + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + var cp = PaginationCheckpoint.Load(checkpointPath); + Assert.NotNull(cp); + Assert.NotNull(cp!.TempFile); + Assert.NotEqual(theirs, cp.TempFile); + + var temps = Directory.GetFiles(dir, "out.jsonl.*.tmp") + .Select(Path.GetFileName).Order().ToArray(); + Assert.Equal(new[] { theirs, cp.TempFile }.Order().ToArray(), temps); + Assert.Equal(["{\"id\":\"g1\"}", "{\"id\":\"g2\"}"], + File.ReadAllLines(Path.Combine(dir, cp.TempFile!))); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other end of it: the retry dies before a page boundary of its own. The temp the + /// attempt before it filled is what an interrupted run leaves and stays where it is, while + /// the retry's own, which nothing counts, is the one that goes. The position that counted + /// the first goes with the token it belongs to - the 410 door deletes it, because the + /// attempt that took the path over wrote it, and it enumerates what Graph has just declared + /// gone. + /// + [Fact] + public void A_retry_that_dies_leaves_the_temp_its_own_attempt_filled() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410dead-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.False(File.Exists(checkpointPath), + "the expired token spared a position of this run's own into the dead enumeration"); + + var temps = Directory.GetFiles(dir, "out.jsonl.*.tmp") + .Select(Path.GetFileName).Order().ToArray(); + Assert.Equal(2, temps.Length); + var mine = temps.Single(t => t != theirs); + Assert.Equal(["{\"id\":\"g1\"}", "{\"id\":\"g2\"}"], + File.ReadAllLines(Path.Combine(dir, mine!))); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And what the run after it finds. The refusal is decided against the file that was on + /// disk when the run started, and the very next page boundary saves this run's own position + /// over that path - so by the time the 410 door reads the refusal, the file it spares is + /// this run's, recording the enumeration Graph has just declared gone. The retry then dies + /// before saving a position of its own, and the door has already deleted the delta state, + /// so the next invocation of the same command line started with nothing to resume from and + /// found that checkpoint waiting. It refused it - correctly; it enumerates something this + /// command line can no longer build - and warned about a sync that was never there, while + /// holding off its stale-temp sweep for a second run over files nothing counted any more. + /// + [Fact] + public void A_refused_position_this_run_wrote_over_goes_with_the_expired_token() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410tookover-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // Sync A dies on page 2: the checkpoint names the temp holding its two items. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + + // Sync B refuses that checkpoint, reaches a page boundary of its own - which saves + // over the same path - loses its token to a 410, and dies again in the re-sync + // before it can record a position of the enumeration it is now making. + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.False(File.Exists(checkpointPath), + "the 410 door spared a position this run had written over the refused one"); + Assert.True(File.Exists(Path.Combine(dir, theirs)), + "the retry swept away the temp the refusal a moment earlier had spared"); + Assert.False(File.Exists(deltaB), "the expired delta state outlived the 410"); + + // The next invocation of B's command line, with nothing of its own left to resume + // from: no checkpoint to refuse, so no warning about a sync that was never there, + // and the sweep it no longer holds off reclaims what the interrupted run left. + handler.QueueResponse(HttpStatusCode.OK, GroupsResync); + var warnings = SyncWarnings(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.DoesNotContain(warnings, w => w.Contains("resume checkpoint")); + Assert.Equal(["{\"id\":\"g9\"}"], File.ReadAllLines(output)); + Assert.Contains("$deltatoken=G1", DeltaState.Load(deltaB)!.DeltaLink); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other branch of the same door, which is the refusal's promise where it still means + /// something: the 410 arrives on the first request of all, so no attempt of this run has + /// saved anything over the path and what is there is still the position that was refused. + /// It is left exactly as it was found, byte for byte, and it is still the other sync's + /// enumeration rather than this one's. + /// + [Fact] + public void An_expired_token_spares_the_refused_position_no_attempt_wrote_over() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410intact-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaA = Path.Combine(dir, "stateA.json"); + var deltaB = Path.Combine(dir, "stateB.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaA, checkpointPath, output); + var theirs = PaginationCheckpoint.Load(checkpointPath)!.TempFile!; + var before = File.ReadAllBytes(checkpointPath); + + // The same compound as above, differing only in reaching no page boundary before + // the 410 - so the door decides over a path this run has not written to. + StaleState(deltaB); + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + Sync(deltaB, checkpointPath, output, "/groups/delta"); + + Assert.Equal(before, File.ReadAllBytes(checkpointPath)); + var cp = PaginationCheckpoint.Load(checkpointPath); + Assert.NotNull(cp); + Assert.Contains("/users/delta", cp!.Resource); + Assert.Equal(theirs, cp.TempFile); + Assert.True(File.Exists(Path.Combine(dir, theirs)), + "the retry swept away the temp the refusal a moment earlier had spared"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A state file that cannot be read is a door out of the delta position like -FullSync and + /// the query-shape changes, and every one of those takes the resume checkpoint with the + /// position it discards. This one did not, and what it left describes the delta-link + /// spelling the fresh sync no longer builds - so the run refused its own earlier position, + /// warned about a second sync that was never there, and held its stale-temp sweep off over + /// a temp nothing counted any more. + /// + [Fact] + public void A_corrupt_delta_state_takes_the_checkpoint_of_the_position_it_abandons() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-corrupt-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // A sync resuming from a token, interrupted on page two: the checkpoint records the + // delta link it was enumerating, and its temp holds the two items it collected. + UsersState(deltaPath); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + var temp = Path.Combine(dir, PaginationCheckpoint.Load(checkpointPath)!.TempFile!); + Assert.True(File.Exists(temp)); + + // And the state file is unreadable by the time the next run starts. + File.WriteAllText(deltaPath, "{ this is not a delta state"); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var warnings = SyncWarnings(deltaPath, checkpointPath, output); + + Assert.Contains(warnings, w => w.Contains("is corrupt. Starting full sync.")); + Assert.DoesNotContain(warnings, + w => w.Contains("records an enumeration this run cannot resume from")); + Assert.False(File.Exists(checkpointPath), + "the abandoned position outlived the state it described"); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(["{\"id\":\"b3\"}"], File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And what the caller is told on the way past. -Latest is suppressed by a resume checkpoint + /// merely existing, and the sentence saying so tells the caller to delete that file - which + /// this door has just done. The advice has to describe the disk as it is. + /// + [Fact] + public void A_corrupt_state_does_not_send_the_caller_after_a_checkpoint_it_deleted() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-corruptlatest-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + UsersState(deltaPath); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + Assert.True(File.Exists(checkpointPath)); + + File.WriteAllText(deltaPath, "{ this is not a delta state"); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var warnings = SyncWarnings(deltaPath, checkpointPath, output, latest: true); + + Assert.Contains(warnings, w => w.Contains("the previous delta state was discarded")); + Assert.DoesNotContain(warnings, w => w.Contains("a resume checkpoint exists")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same sentence, at the three doors a changed query shape opens. A $select, a Prefer + /// header or a filter that differs from the state's discards that state and the resume + /// checkpoint counted into it - and the -Latest guard downstream still read a checkpoint as + /// standing, so the advice it wrote told the caller to go and delete a file the door had + /// just deleted, and said nothing about the state that had actually been discarded. + /// + [Theory] + [InlineData("select")] + [InlineData("prefer")] + [InlineData("filter")] + public void A_query_shape_door_does_not_send_the_caller_after_a_checkpoint_it_deleted(string door) + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-shapelatest-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // A sync resuming from a token, interrupted on page two: the checkpoint it leaves + // records the delta link it was enumerating. + UsersState(deltaPath); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + Assert.True(File.Exists(checkpointPath)); + + // The next run asks for a different shape of the same query, with -Latest on. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var warnings = SyncWarnings(deltaPath, checkpointPath, output, latest: true, + property: door == "select" ? ["id", "displayName"] : null, + prefer: door == "prefer" ? ["return=minimal"] : null, + filter: door == "filter" ? "startswith(displayName,'a')" : null); + + Assert.Contains(warnings, w => w.Contains(door switch + { + "select" => "Property selection changed since last sync", + "prefer" => "Prefer headers changed since last sync", + _ => "Filter changed since last sync", + })); + Assert.Contains(warnings, w => w.Contains("the previous delta state was discarded")); + Assert.DoesNotContain(warnings, w => w.Contains("a resume checkpoint exists")); + Assert.False(File.Exists(checkpointPath), + "the position counted into the discarded state outlived it"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// Answers by URL rather than in order, so what the sync asks for first is what decides + /// which items it receives. + /// + + /// + /// The disk a resumed sync leaves at a page boundary. One run dies with its changes in a + /// temp; the next promotes that temp over the output, appends the page it resumed onto, + /// saves its position against the output itself - there is no temp left to name - and dies + /// too. That last position is the one shape a shared -CheckpointPath offers no temp to + /// refuse a second run by. No page carried a deltaLink, so the token is still the one the + /// state file held when this started. + /// + private static void ResumedAndAppending(MockHttpHandler handler, string deltaPath, + string checkpointPath, string output) + { + UsersState(deltaPath); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + } + + /// + /// The refusal one file over from the temp claim, and the one it could not reach. A sync that + /// has resumed once writes straight into -OutputFile and saves a position naming no temp at + /// all, so a second run over the same command line found nothing in the checkpoint to refuse + /// it by: it judged the file its own, cut it back to the recorded offset under a live writer, + /// appended, and moved the delta token past changes the output no longer held. The holder's + /// next write landed past the hole the cut left. There is no pass this run can make that + /// leaves any of that standing, so it makes none. + /// + [Fact] + public void A_held_output_stops_the_sync_before_anything_is_written() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-heldout-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + ResumedAndAppending(handler, deltaPath, checkpointPath, output); + + // The premise, off the file: this position counts its changes into the output and + // names no temp, which is the shape the claim on the temp has nothing to say about. + var cp = PaginationCheckpoint.Load(checkpointPath)!; + Assert.Null(cp.TempFile); + Assert.Equal(Path.GetFullPath(output), cp.OutputFile); + Assert.Equal(new FileInfo(output).Length, cp.DataLength); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + + // Sync A is not dead. It holds the output the way a resumed run does - appending, + // sharing reads - and has written one more change since its last save, so the file + // is longer than the checkpoint records and cutting it back leaves a hole under it. + // And the page waiting on the wire carries a deltaLink, so going on would move the + // token as well. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + DiskState before, after; + int requestsBefore; + string[] warnings; + ErrorRecord[] errors; + using (var live = new StreamWriter( + new FileStream(output, FileMode.Append, FileAccess.Write, FileShare.Read))) + { + live.WriteLine("{\"id\":\"b9-live\"}"); + live.Flush(); + Assert.True(new FileInfo(output).Length > cp.DataLength, + "the holder wrote nothing past the recorded length, so no cut would show"); + + before = ReadDisk(dir, checkpointPath, output, deltaPath); + requestsBefore = handler.RequestCount; + (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.ResourceBusy, stop.CategoryInfo.Category); + Assert.Contains($"Another sync is still writing '{output}'", + stop.Exception.Message); + Assert.Contains(checkpointPath, stop.Exception.Message); + Assert.Contains($"records {cp.ItemsCollected} items into", stop.Exception.Message); + Assert.Contains("would cut that file back under it", stop.Exception.Message); + Assert.Contains("advance the delta token past them", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Wait for that sync to finish, or give this one its own -OutputFile and " + + "-CheckpointPath.", + stop.Exception.Message); + + // And not the sentence for a file whose contents this run never looked at. + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + Assert.DoesNotContain(warnings, w => w.Contains("Re-enumerating")); + + // Nothing ran: no request, and the output, the position, the delta state and the + // directory's entry list exactly as the holder left them. Read after the record + // and not before it, so a run that took the checkpoint with it fails on the + // answer it did not give. + after = ReadDisk(dir, checkpointPath, output, deltaPath); + Assert.Equal(requestsBefore, handler.RequestCount); + AssertUnmoved(before, after); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + + // A goes on, and the change it writes next lands where its last one ended. + live.WriteLine("{\"id\":\"b10-live\"}"); + live.Flush(); + } + + // Ordinal, both of them: the culture-aware comparisons treat a NUL as ignorable, + // which is the one byte this is looking for. + var published = File.ReadAllText(output); + Assert.False(published.Contains('\0'), + "the cut left a hole for the holder's next write to land past"); + Assert.EndsWith("{\"id\":\"b9-live\"}" + Environment.NewLine + + "{\"id\":\"b10-live\"}" + Environment.NewLine, published, + StringComparison.Ordinal); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other half of the same claim, where there is no second sync at all: the output the + /// checkpoint counts its changes into cannot be opened for writing by this account. It was + /// reported as a file that no longer holds those changes - a reading of the file's contents, + /// given about a file this run had not been able to open - and the position counting them was + /// deleted on the strength of it, with the token then advancing past them. The advice is the + /// only thing that recovers them, so the run giving it leaves all of that standing. This + /// follows the advice and measures what comes back. + /// + [Fact] + public void An_unopenable_output_stops_the_sync_and_says_how_to_recover_it() + { + if (OperatingSystem.IsWindows()) return; + + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-unopenout-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + ResumedAndAppending(handler, deltaPath, checkpointPath, output); + var cp = PaginationCheckpoint.Load(checkpointPath)!; + Assert.Null(cp.TempFile); + var recorded = cp.DataLength!.Value; + + // The changes the checkpoint counts are there, and one written after its last save + // with them - so a run that can open the file cuts back to the recorded length. + File.AppendAllText(output, "{\"id\":\"b9-past-the-save\"}" + Environment.NewLine); + Assert.True(new FileInfo(output).Length > recorded); + + // 0444: the owner cannot open it read-write either, which is the claim failing on + // permissions rather than on another handle. + File.SetUnixFileMode(output, UnixFileMode.UserRead); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var before = ReadDisk(dir, checkpointPath, output, deltaPath); + var requestsBefore = handler.RequestCount; + var (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.PermissionDenied, stop.CategoryInfo.Category); + Assert.Contains($"'{output}' could not be opened for writing:", stop.Exception.Message); + Assert.Contains("is denied", stop.Exception.Message); + // The reason's own trailing period is trimmed before it is quoted, so the sentence + // this joins into is punctuated once, not twice. + Assert.DoesNotContain(".. ", stop.Exception.Message); + Assert.Contains("is denied. The resume checkpoint", stop.Exception.Message); + Assert.Contains($"The resume checkpoint at '{checkpointPath}' records", + stop.Exception.Message); + Assert.Contains("advance the delta token past them", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Grant write access to that file and run again to resume from it, or remove it " + + "and the checkpoint to sync afresh.", + stop.Exception.Message); + + // Not said about a file whose contents this run could not read, and not acted on. + // The disk is read after the record and not before it, so a run that deleted the + // position fails on the answer it did not give. + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + Assert.Equal(requestsBefore, handler.RequestCount); + AssertUnmoved(before, ReadDisk(dir, checkpointPath, output, deltaPath)); + Assert.True(File.Exists(checkpointPath), + "the position counting the changes the advice recovers was deleted"); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + + // Follow it. The run that comes back cuts the output to the recorded length and + // appends the page it resumes onto, so the change written past the last save goes, + // nothing is written twice, and the token moves only now. + File.SetUnixFileMode(output, UnixFileMode.UserRead | UnixFileMode.UserWrite); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var resumed = SyncWarnings(deltaPath, checkpointPath, output); + + Assert.DoesNotContain(resumed, w => w.Contains("no longer holds")); + Assert.Equal( + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b1\"}", "{\"id\":\"b2\"}", + "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try + { + foreach (var f in Directory.EnumerateFiles(dir)) + File.SetUnixFileMode(f, UnixFileMode.UserRead | UnixFileMode.UserWrite); + Directory.Delete(dir, true); + } + catch { } + } + } + + /// + /// The answer the claim must not swallow. An output shorter than the length the checkpoint + /// records is not the file that checkpoint describes - the changes it counts are in no file + /// at all - and nothing holds it and nothing refuses to open it. That still takes today's + /// route: say so, delete the position, and re-enumerate from the last saved token. + /// + [Fact] + public void A_stale_length_output_still_reenumerates_from_the_token() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-staleout-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + ResumedAndAppending(handler, deltaPath, checkpointPath, output); + var cp = PaginationCheckpoint.Load(checkpointPath)!; + Assert.Null(cp.TempFile); + + // The output is replaced by one that does not hold those changes. + File.WriteAllText(output, "{\"id\":\"short\"}" + Environment.NewLine); + Assert.True(new FileInfo(output).Length < cp.DataLength); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + Assert.Empty(errors); + Assert.Contains(warnings, + w => w.Contains($"'{output}' no longer holds the {cp.ItemsCollected} items")); + Assert.False(File.Exists(checkpointPath), "the stale position was left on disk"); + Assert.Equal(["{\"id\":\"b3\"}"], File.ReadAllLines(output)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + private sealed class ByUrlHandler : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var body = request.RequestUri!.ToString().Contains("skiptoken=B2", StringComparison.Ordinal) + ? ChangesPage2 + : ChangesPage1; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } + + /// + /// The disk a resumed sync leaves at a page boundary, with every change in the output + /// exactly once. One run dies with its changes in a temp; the next promotes that temp over + /// the output, saves its position against the output itself - there is no temp left to name + /// - and dies at its first request. No page carried a deltaLink, so the token is still the + /// one the state file held when this started. + /// + private static void PromotedAndAppending(MockHttpHandler handler, string deltaPath, + string checkpointPath, string output) + { + UsersState(deltaPath); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + Sync(deltaPath, checkpointPath, output); + } + + /// + /// Two syncs over one command line, both past the stat the reconcile makes. The claim on the + /// output was asked for and let go inside the reconcile and the writer opened its own handle + /// a page fetch later, so for that window neither run held the file: both passed the claim, + /// both cut it back to the recorded offset, both appended, and the token moved past changes + /// the file no longer held. The claim is the handle now, so the run that takes it keeps it + /// across that window, and the second run meets a file the first one has. + /// + /// A is parked in the window itself. The verbose announcing the resume is written after the + /// reconcile - the claim taken, the change past the last save cut away - and before the + /// writer is opened, which is exactly where the two runs used to overlap. + /// + [Fact] + public void Two_syncs_released_together_leave_one_with_the_output_and_the_token() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-interleave-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + // A real second thread, not a scheduled task: the two runs have to be able to be inside + // the cmdlet at the same time, and joining one is not a blocking wait on work the test + // framework might have scheduled behind it. + (string[] Warnings, ErrorRecord[] Errors) aResult = default; + Exception? aFailed = null; + Thread? a = null; + using var arrived = new ManualResetEventSlim(false); + using var released = new ManualResetEventSlim(false); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + PromotedAndAppending(handler, deltaPath, checkpointPath, output); + + var cp = PaginationCheckpoint.Load(checkpointPath)!; + Assert.Null(cp.TempFile); + Assert.Equal(Path.GetFullPath(output), cp.OutputFile); + var recorded = cp.DataLength!.Value; + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(output)); + Assert.Equal(new FileInfo(output).Length, recorded); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + + // A change written after the last save, so the cut A makes is visible from outside. + File.AppendAllText(output, "{\"id\":\"b9-past-the-save\"}" + Environment.NewLine); + Assert.True(new FileInfo(output).Length > recorded); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var requestsBefore = handler.RequestCount; + + a = new Thread(() => + { + try + { + aResult = SyncParkedAtResume(deltaPath, checkpointPath, output, + arrived, released); + } + catch (Exception ex) { aFailed = ex; } + }); + a.Start(); + Assert.True(arrived.Wait(TimeSpan.FromSeconds(60)), + "sync A never reached the window between its reconcile and its writer"); + + // Past the reconcile: the change written after the last save has been cut away. And + // short of the writer: no page has been asked for, so nothing has been appended. + Assert.Equal(recorded, new FileInfo(output).Length); + Assert.Equal(requestsBefore, handler.RequestCount); + + var (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.ResourceBusy, stop.CategoryInfo.Category); + Assert.Contains($"Another sync is still writing '{output}'", stop.Exception.Message); + Assert.Contains($"records {cp.ItemsCollected} items into", stop.Exception.Message); + Assert.Contains("advance the delta token past them", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + + // B sent no request and took nothing with it: the position A resumes from is still + // on disk, the file is what A cut it back to, and the token has not moved. + Assert.Equal(requestsBefore, handler.RequestCount); + Assert.True(File.Exists(checkpointPath), "B deleted the position A is resuming from"); + Assert.Equal(recorded, new FileInfo(output).Length); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + + released.Set(); + Assert.True(a.Join(TimeSpan.FromSeconds(60)), "sync A never finished"); + Assert.Null(aFailed); + Assert.Empty(aResult.Errors); + + // One run had the file. Every change once, in order, the position spent, and the + // token advanced exactly once - to the link the page A fetched carried. + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpointPath), + "the completed sync left its position behind"); + var state = DeltaState.Load(deltaPath)!; + Assert.Contains("$deltatoken=D2", state.DeltaLink); + Assert.Equal(3, state.ItemCount); + } + finally + { + released.Set(); + a?.Join(TimeSpan.FromSeconds(30)); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A directory standing where the output should be. It is not a file, so File.Exists said no + /// and the run took the route it takes for an output that is not there at all - warning that + /// the file no longer held the changes, deleting the position, and re-enumerating from a + /// token that is about to advance past them. Opening a directory read-write is refused on + /// both platforms, and that is an answer. The write probe above this passes: it writes + /// beside the path, not to it. + /// + [Fact] + public void A_directory_standing_at_the_output_path_stops_the_sync() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-outdir-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + PromotedAndAppending(handler, deltaPath, checkpointPath, output); + var cp = PaginationCheckpoint.Load(checkpointPath)!; + Assert.Null(cp.TempFile); + var position = File.ReadAllBytes(checkpointPath); + var state = File.ReadAllBytes(deltaPath); + + File.Delete(output); + Directory.CreateDirectory(output); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var requestsBefore = handler.RequestCount; + var (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + // A directory is not a permission, whatever errno the open came back with: no grant + // makes one writable at an offset, and the way out is to take it off the path. + Assert.Equal(ErrorCategory.InvalidOperation, stop.CategoryInfo.Category); + Assert.Contains($"'{output}' could not be opened for writing: a directory stands at " + + "the path.", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Remove it (or point -OutputFile elsewhere) and run again to resume from it, or " + + "remove the checkpoint to sync afresh.", + stop.Exception.Message); + Assert.DoesNotContain("Grant write access", stop.Exception.Message); + + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + Assert.DoesNotContain(warnings, w => w.Contains("Re-enumerating")); + Assert.Equal(requestsBefore, handler.RequestCount); + Assert.True(File.Exists(checkpointPath), "the position was deleted over a directory"); + Assert.Equal(position, File.ReadAllBytes(checkpointPath)); + Assert.Equal(state, File.ReadAllBytes(deltaPath)); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + Assert.True(Directory.Exists(output)); + Assert.Empty(Directory.GetFileSystemEntries(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The two write probes the sync makes before it fetches anything, and what each of them + /// calls the file it could not write. Both are the same call, and one noun for both told a + /// caller whose -OutputFile directory was not writable that they could not write to the + /// delta state path - naming a file that had just passed its own probe. + /// + [Fact] + public void The_write_probe_names_the_file_it_could_not_write() + { + if (OperatingSystem.IsWindows()) return; + + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-probe-{Guid.NewGuid():N}")).FullName; + var outDir = Directory.CreateDirectory(Path.Combine(dir, "out")).FullName; + var stateDir = Directory.CreateDirectory(Path.Combine(dir, "state")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // r-x: the directory can be searched and not written, which is what the probe + // fails on. Where the write below succeeds the account is root, a mode says + // nothing, and there is nothing here to measure. + File.SetUnixFileMode(outDir, UnixFileMode.UserRead | UnixFileMode.UserExecute); + var output = Path.Combine(outDir, "out.jsonl"); + var writable = true; + try { File.WriteAllText(output, ""); } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + writable = false; + } + if (writable) return; + + var deltaPath = Path.Combine(dir, "state.json"); + var (_, errors) = SyncRun(deltaPath, checkpointPath, output); + var stop = Assert.Single(errors); + Assert.Contains($"Cannot write to output path '{output}'", stop.Exception.Message); + Assert.DoesNotContain("delta state path", stop.Exception.Message); + + // And the probe on -DeltaPath keeps its own noun. + File.SetUnixFileMode(stateDir, UnixFileMode.UserRead | UnixFileMode.UserExecute); + var unwritableState = Path.Combine(stateDir, "state.json"); + var (_, stateErrors) = SyncRun(unwritableState, checkpointPath, + Path.Combine(dir, "fine.jsonl")); + var stateStop = Assert.Single(stateErrors); + Assert.Contains($"Cannot write to delta state path '{unwritableState}'", + stateStop.Exception.Message); + } + finally + { + try + { + foreach (var d in new[] { outDir, stateDir }) + File.SetUnixFileMode(d, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + Directory.Delete(dir, true); + } + catch { } + } + } + + /// + /// The disk an interrupted fresh sync leaves: its changes are in a temp beside the output, + /// the checkpoint names that temp, and there is no output at all yet. The run after it + /// promotes the temp over the output and appends onto what it promoted. + /// + private static void InterruptedIntoATemp(MockHttpHandler handler, string deltaPath, + string checkpointPath, string output) + { + UsersState(deltaPath); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + } + + /// + /// The other route that ends in an append, one file over. A sync whose checkpoint names a + /// temp promotes it over the output and then appends onto it - and used to hold nothing + /// between the move that put the changes there and the writer's own open a page fetch later. + /// The checkpoint it repoints in that gap names no temp, which is the shape a second run + /// over the same command line reads as an output it may cut back, append to, and move the + /// token past. So the promotion takes the file it promoted into. + /// + [Fact] + public void A_promoting_sync_holds_the_output_it_promoted_into() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-promotehold-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + (string[] Warnings, ErrorRecord[] Errors) aResult = default; + Exception? aFailed = null; + Thread? a = null; + using var arrived = new ManualResetEventSlim(false); + using var released = new ManualResetEventSlim(false); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + InterruptedIntoATemp(handler, deltaPath, checkpointPath, output); + + // The premise: the changes are in a temp, the checkpoint names it, and nothing is at + // -OutputFile for anyone to be holding. + var before = PaginationCheckpoint.Load(checkpointPath)!; + Assert.NotNull(before.TempFile); + Assert.False(File.Exists(output)); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var requestsBefore = handler.RequestCount; + + a = new Thread(() => + { + try + { + aResult = SyncParkedAtResume(deltaPath, checkpointPath, output, + arrived, released); + } + catch (Exception ex) { aFailed = ex; } + }); + a.Start(); + Assert.True(arrived.Wait(TimeSpan.FromSeconds(60)), + "sync A never reached the window between its promotion and its writer"); + + // Past the promotion: the changes are in the output, the temp is gone, and the + // position on disk names no temp - which is exactly what a second run reads as an + // output it may take. And short of the writer: no page has been asked for. + // + // Measured off the directory entry, since the run holds the file and a reader of it + // is refused until the run ends. What it holds is read at the end, when it does not. + var promotedBytes = ("{\"id\":\"b1\"}" + Environment.NewLine + + "{\"id\":\"b2\"}" + Environment.NewLine).Length; + Assert.Equal(promotedBytes, new FileInfo(output).Length); + var repointed = PaginationCheckpoint.Load(checkpointPath)!; + Assert.Null(repointed.TempFile); + Assert.Equal(promotedBytes, repointed.DataLength); + Assert.Equal(["out.jsonl", "run.checkpoint", "state.json"], + Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal)); + Assert.Equal(requestsBefore, handler.RequestCount); + + var (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.ResourceBusy, stop.CategoryInfo.Category); + Assert.Contains($"Another sync is still writing '{output}'", stop.Exception.Message); + Assert.Contains("advance the delta token past them", stop.Exception.Message); + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + + // B sent no request, took nothing with it, and moved no token. + Assert.Equal(requestsBefore, handler.RequestCount); + Assert.True(File.Exists(checkpointPath), "B deleted the position A is resuming from"); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + + released.Set(); + Assert.True(a.Join(TimeSpan.FromSeconds(60)), "sync A never finished"); + Assert.Null(aFailed); + Assert.Empty(aResult.Errors); + + // One run had the file. Every change once, in order, the position spent, and the + // token advanced exactly once. + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpointPath), + "the completed sync left its position behind"); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + released.Set(); + a?.Join(TimeSpan.FromSeconds(30)); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The promotion route's own claim on the file it replaces. A sync whose checkpoint names a + /// temp puts those changes over the output and appends onto what it put there, and it asked + /// for the file only after the move - which a rename asks nothing about, on either side of + /// it. So the move went through over an output another run was holding and appending to, that + /// run's next write landed in an inode with no name left on it, and this one would have gone + /// on to move the delta token past changes the file at the path no longer held. + /// + [Fact] + public void A_promotion_over_a_held_output_stops_the_sync_and_replaces_nothing() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-promoteheld-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + InterruptedIntoATemp(handler, deltaPath, checkpointPath, output); + var cp = PaginationCheckpoint.Load(checkpointPath)!; + var temp = Path.Combine(dir, cp.TempFile!); + Assert.True(File.Exists(temp), "the interrupted run left no temp to promote"); + Assert.False(File.Exists(output)); + + // A previous sync's changes at -OutputFile, and the run that has them: appending, + // sharing reads, the way a resumed run holds its own output. + File.WriteAllLines(output, OldSync); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + using (var live = new StreamWriter( + new FileStream(output, FileMode.Append, FileAccess.Write, FileShare.Read)) + { AutoFlush = true }) + { + var before = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + var requestsBefore = handler.RequestCount; + var (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.ResourceBusy, stop.CategoryInfo.Category); + Assert.Contains($"Another sync is still writing '{output}'", + stop.Exception.Message); + // The temp, not the output, is where these items are counted on the promotion + // route, and going on replaces the output rather than cutting it back. + Assert.Contains($"The resume checkpoint at '{checkpointPath}' records " + + $"{cp.ItemsCollected} items into the temp '{cp.TempFile}'", + stop.Exception.Message); + Assert.Contains($"going on would replace '{output}' with them under the run " + + "that holds it", stop.Exception.Message); + Assert.DoesNotContain("cut that file back", stop.Exception.Message); + Assert.Contains("advance the delta token past them", stop.Exception.Message); + + // No request, no recovery, no token: the temp still holds the changes the + // checkpoint counts, the checkpoint still names it, and the staged file the move + // would have come from was taken back. + Assert.Equal(requestsBefore, handler.RequestCount); + Assert.DoesNotContain(warnings, w => w.Contains("Recovered")); + AssertUnmoved(before, ReadDisk(dir, checkpointPath, temp, output, deltaPath)); + Assert.False(File.Exists($"{output}.adopt")); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + + // And the same inode, not a new file wearing the name: the holder's next write + // is read back through the path. + live.WriteLine("{\"id\":\"old-0000004\"}"); + } + + Assert.Equal([.. OldSync, "{\"id\":\"old-0000004\"}"], File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The window the second run walked through, one cmdlet over. A checkpoint naming a temp that + /// is not on disk was read as changes in no file at all: the sync said so, deleted the + /// position counting them, re-enumerated from the last saved token and moved its own temp over + /// the output - which was the promoting run's output, held and being appended to - and then + /// advanced the token past both runs' work. The output is asked about before any of that now. + /// + [Fact] + public void A_vanished_temp_over_a_held_output_stops_the_sync() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-vanished-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + InterruptedIntoATemp(handler, deltaPath, checkpointPath, output); + + // The instant a promoting run leaves between its temp's unlink and the save that + // repoints the position: the changes are in the output at the length recorded for + // them, the temp is gone, and the checkpoint still names it. + var cp = PaginationCheckpoint.Load(checkpointPath)!; + var temp = Path.Combine(dir, cp.TempFile!); + var recorded = cp.DataLength!.Value; + File.WriteAllBytes(output, File.ReadAllBytes(temp)[..(int)recorded]); + File.Delete(temp); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + using var live = new StreamWriter( + new FileStream(output, FileMode.Append, FileAccess.Write, FileShare.Read)); + + var before = ReadDisk(dir, checkpointPath, output, deltaPath); + var requestsBefore = handler.RequestCount; + var (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Contains($"Another sync is still writing '{output}'", stop.Exception.Message); + // The temp this checkpoint named is gone, not held - so the items are recorded + // (past tense) into a temp the sentence does not go on to name, and going on means + // re-enumerating fresh rather than cutting the output back. + Assert.Contains($"The resume checkpoint at '{checkpointPath}' recorded " + + $"{cp.ItemsCollected} items into a temp that is gone", stop.Exception.Message); + Assert.Contains("going on would re-enumerate from the last saved delta token and " + + $"replace '{output}' under the run that holds it", stop.Exception.Message); + Assert.DoesNotContain("cut that file back", stop.Exception.Message); + Assert.Contains("advance the delta token past them", stop.Exception.Message); + Assert.DoesNotContain(warnings, w => w.Contains("missing or incomplete")); + Assert.Equal(requestsBefore, handler.RequestCount); + AssertUnmoved(before, ReadDisk(dir, checkpointPath, output, deltaPath)); + Assert.True(File.Exists(checkpointPath), + "the position the promoting run is resuming from was deleted"); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// Two syncs released together on the promotion route. Both completed: whichever lost the + /// promotion found the temp gone, warned that the changes it counted were not on disk, deleted + /// the position counting them, re-enumerated over the output the other one was holding, and + /// moved the delta token past work that was no longer in any file. A run that loses a file + /// ends there instead now - whichever of the files it lost, and whether one of them lost one + /// or both did - and the token moves once, or not at all. + /// + [Fact] + public void Two_syncs_released_together_on_the_promotion_route_advance_the_token_once() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-together-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + var gate = new GatedChangesByUrlHandler(); + var results = new (string[] Warnings, ErrorRecord[] Errors)[2]; + var failures = new Exception?[2]; + var threads = new Thread[2]; + var ready = new[] { new ManualResetEventSlim(false), new ManualResetEventSlim(false) }; + using var go = new ManualResetEventSlim(false); + try + { + var setup = new MockHttpHandler(); + setup.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using (MgxTransportScope.Inject(setup)) + { + InterruptedIntoATemp(setup, deltaPath, checkpointPath, output); + } + Assert.NotNull(PaginationCheckpoint.Load(checkpointPath)!.TempFile); + Assert.False(File.Exists(output)); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + + using var transport = MgxTransportScope.Inject(gate); + for (var i = 0; i < threads.Length; i++) + { + var slot = i; + threads[slot] = new Thread(() => + { + try + { + results[slot] = SyncReleasedWith(deltaPath, checkpointPath, output, + ready[slot], go); + } + catch (Exception ex) { failures[slot] = ex; } + }); + threads[slot].Start(); + } + + foreach (var r in ready) + Assert.True(r.Wait(TimeSpan.FromSeconds(60)), "a sync never came up"); + go.Set(); + + // A run that got past the reconcile holds the output and is parked on its first + // request. It stays there until the other has finished, which is what a run that + // stops does with nothing on the wire at all - and both of them losing a file is an + // outcome of the race too: then no sync has the output and none of them asks for a + // page. + Assert.True( + SpinWait.SpinUntil(() => gate.Arrived.IsSet || threads.All(t => !t.IsAlive), + TimeSpan.FromSeconds(60)), + "neither sync reached the wire or finished"); + if (gate.Arrived.IsSet) + { + Assert.True( + SpinWait.SpinUntil(() => threads.Any(t => !t.IsAlive), + TimeSpan.FromSeconds(60)), + "both syncs were still running with one of them holding the output"); + } + + gate.Released.Set(); + foreach (var t in threads) + Assert.True(t.Join(TimeSpan.FromSeconds(60)), "a sync never finished"); + Assert.All(failures, Assert.Null); + + // Exactly one run stopped, on one of the files this checkpoint stands for, and it + // neither asked for a page nor said the changes it was protecting were lost. + // + // Which file it stopped on is the race's to decide, and both answers are the same + // run: the stop that loses the temp, or the output the promoter is holding, and the + // stop that reaches the staging name first and finds the promoter's copy standing at + // it. The second is the narrow one - it needs this run to arrive between the other's + // create and its rename - and it is not one platform's: the staging name is swept + // before either claim on both. So the sentence asserted is the one that belongs to + // whichever route the run took, and what is asserted of the run itself is the same + // either way, below: one winner, every change once, and nothing of this one's left + // on the disk. + var stops = results.SelectMany(r => r.Errors).ToArray(); + Assert.All(stops, s => Assert.StartsWith("Checkpoint", s.FullyQualifiedErrorId, + StringComparison.Ordinal)); + foreach (var stop in stops) + { + if (stop.FullyQualifiedErrorId.StartsWith("CheckpointStagingFailed", + StringComparison.Ordinal)) + { + // The promoter's own staged copy, reached between its create and its rename. + // Named as a staging that failed and not as an entry nothing could be made + // of, and the sentence says the changes are still whole in the temp the + // checkpoint names. Which refusal the open came back with is the platform's + // and the moment's, so the reason it quotes is not asserted. + Assert.Contains("whole, but staging them into", stop.Exception.Message); + Assert.Contains("Nothing was changed;", stop.Exception.Message); + } + else + { + Assert.Contains("This run stops here; nothing was written.", + stop.Exception.Message); + } + } + Assert.DoesNotContain(results.SelectMany(r => r.Warnings), + w => w.Contains("missing or incomplete")); + + if (gate.Arrived.IsSet) + { + // One run had the file: every change once, in order, the position spent by the + // run that finished, and the token moved exactly once. + Assert.Single(stops); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpointPath)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + else + { + // Neither had it: the files they were released on are the files they leave, and + // the token stands where the interrupted run left it. + Assert.Equal(2, stops.Length); + Assert.False(File.Exists(output)); + var left = PaginationCheckpoint.Load(checkpointPath); + Assert.NotNull(left!.TempFile); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + } + + Assert.Empty(Directory.GetFiles(dir, "*.adopt")); + } + finally + { + gate.Released.Set(); + foreach (var t in threads) t?.Join(TimeSpan.FromSeconds(30)); + foreach (var r in ready) r.Dispose(); + gate.Arrived.Dispose(); + gate.Released.Dispose(); + gate.Dispose(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The write a promotion makes before it replaces anything, refused. A directory standing + /// where the staged copy goes is one of the ways it fails, and the answer used to come out of + /// the catch around the whole promotion as "nothing was promoted" - which the caller reads as + /// the temp having gone: it said the changes it counted were not on disk, deleted the position + /// counting them, re-enumerated over the output and moved the delta token past every one of + /// them. The staging says so for itself now, and both files and the token are where they were. + /// + [Fact] + public void A_directory_at_the_staging_name_stops_the_sync_and_changes_nothing() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-staging-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + InterruptedIntoATemp(handler, deltaPath, checkpointPath, output); + var cp = PaginationCheckpoint.Load(checkpointPath)!; + var temp = Path.Combine(dir, cp.TempFile!); + Assert.True(File.Exists(temp), "the interrupted run left no temp to promote"); + + var adopt = $"{output}.adopt"; + Directory.CreateDirectory(adopt); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var before = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + var requestsBefore = handler.RequestCount; + var (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointStagingFailed", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.WriteError, stop.CategoryInfo.Category); + Assert.Contains($"The {cp.ItemsCollected} items the resume checkpoint at " + + $"'{checkpointPath}' records are in '{cp.TempFile}', whole, but staging them " + + $"into '{output}' failed at '{adopt}': ", stop.Exception.Message); + Assert.Contains("Nothing was changed; fix that and run again to recover them, or " + + "remove the temp and the checkpoint to re-enumerate from the last saved delta " + + "token.", stop.Exception.Message); + + // The runtime's own words, punctuated once: the reason is quoted inside a sentence + // that ends it, and a message already ending in a period put two there. + Assert.DoesNotContain(".. Nothing was changed;", stop.Exception.Message); + + Assert.DoesNotContain(warnings, w => w.Contains("missing or incomplete")); + Assert.DoesNotContain(warnings, w => w.Contains("Recovered")); + Assert.Equal(requestsBefore, handler.RequestCount); + AssertUnmoved(before, ReadDisk(dir, checkpointPath, temp, output, deltaPath)); + Assert.True(Directory.Exists(adopt), "the run removed the caller's directory"); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same write refused the other way: another run holding the staging name, which is what + /// two syncs over one command line reach a page apart. Nothing this run may take, and nothing + /// it may report as a temp that has gone. + /// + [Fact] + public void A_held_staging_name_stops_the_sync_and_leaves_the_holders_file() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-staging-held-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + InterruptedIntoATemp(handler, deltaPath, checkpointPath, output); + var cp = PaginationCheckpoint.Load(checkpointPath)!; + var temp = Path.Combine(dir, cp.TempFile!); + var adopt = $"{output}.adopt"; + + using var holder = new FileStream(adopt, FileMode.Create, FileAccess.ReadWrite, + FileShare.None); + holder.Write("{\"id\":\"theirs\"}\n"u8); + holder.Flush(); + var holderBytes = holder.Length; + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var before = ReadDisk(dir, checkpointPath, temp, output, deltaPath); + var requestsBefore = handler.RequestCount; + var (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointStagingFailed", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.WriteError, stop.CategoryInfo.Category); + Assert.Contains($"staging them into '{output}' failed at '{adopt}': ", + stop.Exception.Message); + Assert.Contains("Nothing was changed;", stop.Exception.Message); + Assert.DoesNotContain(warnings, w => w.Contains("missing or incomplete")); + Assert.Equal(requestsBefore, handler.RequestCount); + + AssertUnmoved(before, ReadDisk(dir, checkpointPath, temp, output, deltaPath)); + Assert.Equal(holderBytes, holder.Length); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A pipe at the staging name, which is one of the kinds the sweep removes. What came off + /// the name went to the verbose stream, so the deletion reached nobody who had not asked for + /// verbose - and a sync that removes an entry beside the caller's output has to say it did. + /// It is a warning now, in the one sentence both cmdlets say it in. + /// + [Fact] + public void A_pipe_at_the_staging_name_is_swept_and_the_sync_warns_what_it_removed() + { + if (OperatingSystem.IsWindows()) return; + + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-staging-pipe-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + var adopt = $"{output}.adopt"; + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + InterruptedIntoATemp(handler, deltaPath, checkpointPath, output); + Mkfifo(adopt); + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + + // On its own thread with a watchdog: an open of a pipe with no reader never comes + // back, and a test that called the sync inline would hang the suite. + (string[] Warnings, ErrorRecord[] Errors) result = ([], []); + var running = new Thread(() => result = SyncRun(deltaPath, checkpointPath, output)) + { IsBackground = true }; + running.Start(); + Assert.True(running.Join(TimeSpan.FromSeconds(30)), + "the sync never returned: the staging open is waiting for a reader on the pipe"); + + Assert.Empty(result.Errors); + Assert.Contains(result.Warnings, w => w.Contains( + $"Removed what stood at the staging name '{Path.GetFileName(adopt)}': " + + "an entry no copy could be staged in, such as a pipe or a socket.")); + Assert.Contains(result.Warnings, w => w.Contains("Recovered 2 items")); + + // The pipe is gone, the promotion landed in a regular file at the output path, and + // the resumed page is behind the promoted changes. + Assert.False(File.Exists(adopt)); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { File.Delete(adopt); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + private static void Mkfifo(string path) + { + using var mkfifo = System.Diagnostics.Process.Start("/usr/bin/mkfifo", [path]); + Assert.NotNull(mkfifo); + mkfifo.WaitForExit(); + Assert.Equal(0, mkfifo.ExitCode); + } + + /// + /// The two output paths a grant does nothing for. A symlink loop and a FIFO both refuse the + /// open, and both used to end under ErrorCategory.PermissionDenied - the category a caller + /// filtering on it reads as a mode to change, over paths where there is no mode to change + /// and the sentence beside it already said as much. The open's own reason decides the + /// category now, and a 0444 output above still ends under PermissionDenied. + /// + [Fact] + public void An_output_no_grant_would_open_stops_the_sync_under_the_right_category() + { + if (OperatingSystem.IsWindows()) return; + + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-outcategory-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + PromotedAndAppending(handler, deltaPath, checkpointPath, output); + Assert.Null(PaginationCheckpoint.Load(checkpointPath)!.TempFile); + var position = File.ReadAllBytes(checkpointPath); + + // A link that points at itself: the open comes back ELOOP, which is neither a hold + // nor a mode. + File.Delete(output); + using (var ln = System.Diagnostics.Process.Start("/bin/ln", ["-s", output, output])) + { + ln!.WaitForExit(); + Assert.Equal(0, ln.ExitCode); + } + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var requestsBefore = handler.RequestCount; + var loop = Assert.Single(SyncRun(deltaPath, checkpointPath, output).Errors); + + Assert.StartsWith("CheckpointOutputUnopenable", loop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.InvalidOperation, loop.CategoryInfo.Category); + Assert.Contains("Too many levels of symbolic links", loop.Exception.Message); + // No trailing period on this reason, so trimming one off it is a no-op: the single + // period the sentence joins on is the one it always had. + Assert.Contains($"Too many levels of symbolic links : '{output}'. The resume " + + "checkpoint", loop.Exception.Message); + Assert.DoesNotContain("Grant write access", loop.Exception.Message); + + // A pipe: it opens, and then has no length and no offset to write at. + File.Delete(output); + using (var mkfifo = System.Diagnostics.Process.Start("/usr/bin/mkfifo", [output])) + { + mkfifo!.WaitForExit(); + Assert.Equal(0, mkfifo.ExitCode); + } + + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var fifo = Assert.Single(SyncRun(deltaPath, checkpointPath, output).Errors); + + Assert.StartsWith("CheckpointOutputUnopenable", fifo.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.InvalidOperation, fifo.CategoryInfo.Category); + Assert.Contains($"'{output}' could not be opened for writing: the file has no offset " + + "to write at.", fifo.Exception.Message); + Assert.DoesNotContain("Grant write access", fifo.Exception.Message); + + // Neither run asked for a page, and neither took the position counting the changes + // the advice tells the caller to come back for. + Assert.Equal(requestsBefore, handler.RequestCount); + Assert.Equal(position, File.ReadAllBytes(checkpointPath)); + Assert.Contains("$deltatoken=D0", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { File.Delete(output); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The 410 door sends the attempt loop round again with whatever the attempt before it was + /// holding, and a run that resumed from a checkpoint holds -OutputFile. The door deletes the + /// checkpoint, so the retry normally finds nothing to reconcile - but that delete can fail, + /// on a checkpoint directory this account may read and not unlink from, and then the retry + /// reads the same position again with the output still this run's. The claim answered held, + /// about this run's own handle, and the run ended telling the caller to wait for a second + /// sync and to give it its own -OutputFile: there was no second sync, and both were already + /// this one's. The hold goes with the position it was taken for, which is the position the + /// 410 has just declared dead. + /// + [Fact] + public void An_expired_token_does_not_leave_the_retry_refused_by_this_runs_own_hold() + { + if (OperatingSystem.IsWindows()) return; + + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-410hold-{Guid.NewGuid():N}")).FullName; + var cpDir = Directory.CreateDirectory(Path.Combine(dir, "cp")).FullName; + var checkpointPath = Path.Combine(cpDir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // An interrupted first pass, resumed: its changes are in the output, the checkpoint + // records that file and no temp, and there is no delta state yet - a token comes + // with the deltaLink at the end. So the URL this run builds is the URL it rebuilds + // for the retry, and one position is this run's on both attempts. + File.WriteAllText(output, "{\"id\":\"a1\"}\n{\"id\":\"a2\"}\n"); + var recorded = new FileInfo(output).Length; + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users/delta?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = output, + DataLength = recorded, + }.Save(checkpointPath); + + // The delete the 410 door makes, refused: 0555 on the checkpoint's own directory. + using (var chmod = System.Diagnostics.Process.Start("/bin/chmod", ["555", cpDir])) + { + chmod!.WaitForExit(); + Assert.Equal(0, chmod.ExitCode); + } + + handler.QueueResponse(HttpStatusCode.Gone, TokenExpired); + handler.QueueResponse(HttpStatusCode.OK, ChangesPage2); + var (warnings, errors) = SyncRun(deltaPath, checkpointPath, output); + + // The premise: the door could not take the position, so the retry read it again. + Assert.Contains(warnings, w => w.Contains("Could not delete resume checkpoint")); + Assert.Contains(warnings, w => w.Contains("Delta token expired (HTTP 410 Gone)")); + + // And the retry was not refused by the handle the attempt before it left standing. + Assert.DoesNotContain(errors, + e => e.FullyQualifiedErrorId.StartsWith("CheckpointOutputHeld", + StringComparison.Ordinal)); + Assert.DoesNotContain(errors, + e => e.Exception.Message.Contains("Another sync is still writing")); + Assert.Empty(errors); + } + finally + { + using (var chmod = System.Diagnostics.Process.Start("/bin/chmod", ["755", cpDir])) + chmod!.WaitForExit(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The sync's twin of the export's unsearchable-parent advice. The open comes back as an + /// access failure naming the file, and the sentence read it as a mode on the file and told + /// the caller to grant write access to it - a file they cannot stat, let alone chmod, + /// through a parent that refuses to be looked into. The grant is on the directory, and the + /// sentence says which directory. + /// + /// Built from the refusal rather than run into: this sync asks whether it can write beside + /// -OutputFile before it reconciles anything, and a directory that refuses to be searched + /// refuses that probe first - so the sentence is reached here the way the run would reach + /// it, and the words it produces are what this pins. + /// + [Fact] + public void The_unsearchable_parent_stop_says_where_the_grant_belongs() + { + var baseType = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); + var reasonType = baseType.GetNestedType("UnopenableReason", + BindingFlags.NonPublic | BindingFlags.Public)!; + var refusalType = baseType.GetNestedType("CheckpointFileRefusal", + BindingFlags.NonPublic | BindingFlags.Public)!; + var fileType = baseType.GetNestedType("CheckpointFile", + BindingFlags.NonPublic | BindingFlags.Public)!; + var claimType = baseType.GetNestedType("ClaimRefusal", + BindingFlags.NonPublic | BindingFlags.Public)!; + var routeType = baseType.GetNestedType("CheckpointStopRoute", + BindingFlags.NonPublic | BindingFlags.Public)!; + var message = typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).GetMethod( + "CheckpointFileStopMessage", BindingFlags.Static | BindingFlags.NonPublic)!; + + const string vault = "/somewhere/vault"; + const string output = $"{vault}/out.jsonl"; + var reason = Activator.CreateInstance(reasonType, + [$"the directory '{vault}' cannot be searched", true, false, true]); + var refusal = Activator.CreateInstance(refusalType, + [ + Enum.Parse(fileType, "Output"), + Enum.Parse(claimType, "CannotBeOpened"), + Enum.Parse(routeType, "Appending"), + reason, + ]); + var checkpoint = new PaginationCheckpoint { ItemsCollected = 7, DataLength = 42 }; + + var sentence = (string)message.Invoke(null, + [refusal, "/somewhere/run.checkpoint", output, checkpoint])!; + + Assert.Contains($"'{output}' could not be opened for writing: the directory " + + $"'{vault}' cannot be searched.", sentence); + Assert.Contains( + "Grant access to that directory and run again to resume from it, or remove the " + + "checkpoint to sync afresh.", + sentence); + Assert.DoesNotContain("Grant write access to that file", sentence); + } + + /// + /// Both streams of a sync, for the scenarios whose evidence is on the verbose one: whether + /// the run swept the temps beside its output is not something a warning says. + /// + private static (string[] Warnings, string[] Verbose) SyncStreams(string deltaPath, + string checkpointPath, string outputPath, string uri = "/users/delta", + bool latest = false) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", uri) + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("OutputFile", outputPath) + .AddParameter("Verbose"); + if (latest) ps.AddParameter("Latest"); + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + return ([.. ps.Streams.Warning.Select(w => w.Message)], + [.. ps.Streams.Verbose.Select(v => v.Message)]); + } + + /// + /// The shape every release before 2.1 wrote: the position, the count, and nothing about + /// which files the changes are in. Written by rewriting a real checkpoint, so the resource + /// and the nextLink are the ones this run built rather than ones a test guessed at. + /// + private static void StripToPre21Shape(string checkpointPath) + { + var cp = PaginationCheckpoint.Load(checkpointPath)!; + Assert.NotNull(cp.TempFile); + Assert.NotNull(cp.DataLength); + cp.TempFile = null; + cp.OutputFile = null; + cp.DataLength = null; + cp.Save(checkpointPath); + } + + /// + /// A refusal leaves the checkpoint standing because the sync that wrote it resumes from + /// exactly that, and the changes it counted are in a temp beside this output. A checkpoint + /// written before the temp's name was recorded names none, so keyed on that name the sweep + /// in the same run found nothing to spare and deleted the file the spared position was + /// pointing at - seconds after a warning promising both files would be left as they are. + /// The refusal takes the whole sweep with it instead. + /// + [Fact] + public void A_refusal_of_a_checkpoint_naming_no_temp_holds_the_whole_sweep_off() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-sweepoff-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var groupsState = Path.Combine(dir, "groups-state.json"); + var usersState = Path.Combine(dir, "users-state.json"); + var output = Path.Combine(dir, "out.jsonl"); + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // A groups sync killed after its first page: two changes in a temp beside this + // output, no output yet, and a position counting them. + handler.QueueResponse(HttpStatusCode.OK, GroupsPageOne); + Sync(groupsState, checkpointPath, output, "/groups/delta"); + var orphan = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + var counted = File.ReadAllBytes(orphan); + StripToPre21Shape(checkpointPath); + Assert.False(File.Exists(output)); + + // A users sync over the same -CheckpointPath and -OutputFile. It refuses that + // position, and dies on its own first request without an output or a temp of its + // own to leave behind. + var (warnings, verbose) = SyncStreams(usersState, checkpointPath, output); + + Assert.Contains(warnings, w => w.Contains( + "records an enumeration this run cannot resume from", StringComparison.Ordinal)); + Assert.Contains(verbose, v => v.Contains( + $"Left the temp files beside '{output}' alone: one of them may hold the items " + + "of the checkpoint this run refused.", StringComparison.Ordinal)); + Assert.True(File.Exists(orphan), + "the sweep took the temp the refused checkpoint stands for"); + Assert.Equal(counted, File.ReadAllBytes(orphan)); + Assert.True(File.Exists(checkpointPath), "the refused position was deleted"); + Assert.False(File.Exists(output), "the run that stopped left an output behind"); + + // And the sync that owns both comes back to find them where it left them. + handler.QueueResponse(HttpStatusCode.OK, GroupsResync); + var (recovered, _) = SyncStreams(groupsState, checkpointPath, output, "/groups/delta"); + + Assert.Contains(recovered, w => w.Contains( + "Recovered 2 items from an interrupted sync's temp file.", + StringComparison.Ordinal)); + Assert.Equal(["{\"id\":\"g1\"}", "{\"id\":\"g2\"}", "{\"id\":\"g9\"}"], + File.ReadAllLines(output)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The corroboration a checkpoint recording no output is believed on is asked for only + /// where there is something to corroborate. Applied to a checkpoint recording neither an + /// output nor a length, the predicate answered no for want of a measurement - so the whole + /// pre-2.1 file-mode shape was refused here while the export recovered it, this cmdlet's + /// own adoption branch was reachable from no shape any release ever wrote, and the position + /// left standing refused every -Latest that followed for as long as it was there. + /// + [Fact] + public void A_checkpoint_recording_neither_field_recovers_and_latest_follows_it() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-delta-neither-{Guid.NewGuid():N}")).FullName; + var checkpointPath = Path.Combine(dir, "run.checkpoint"); + var deltaPath = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerError); + using var transport = MgxTransportScope.Inject(handler); + try + { + // A first sync, with no state to resume from, killed after its first page: two + // changes in a temp, no output, and no delta token yet. + handler.QueueResponse(HttpStatusCode.OK, ChangesPage1); + Sync(deltaPath, checkpointPath, output); + var orphan = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + StripToPre21Shape(checkpointPath); + Assert.False(File.Exists(output)); + Assert.False(File.Exists(deltaPath)); + + // The same command line comes back to its own position. The changes it counts are + // in the temp beside the output it names, which is the evidence that shape is + // decided on - and the page that finishes it carries no token, so nothing but the + // checkpoint can have a say in the -Latest below. + handler.QueueResponse(HttpStatusCode.OK, ChangesPageNoToken); + var (warnings, _) = SyncStreams(deltaPath, checkpointPath, output); + + Assert.Contains(warnings, w => w.Contains( + "Recovered 2 items from an interrupted sync's temp file.", + StringComparison.Ordinal)); + Assert.DoesNotContain(warnings, w => w.Contains("no longer corroborate it", + StringComparison.Ordinal)); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b9\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(orphan), "the adopted temp outlived the recovery"); + Assert.False(File.Exists(checkpointPath), + "the completed sync left the position it recovered from behind"); + + // And -Latest is answerable again: the file that would have refused it for good + // went with the sync that finished. + handler.QueueResponse(HttpStatusCode.OK, LatestBaseline); + var (latest, _) = SyncStreams(deltaPath, checkpointPath, output, latest: true); + + Assert.DoesNotContain(latest, w => w.Contains("-Latest ignored", + StringComparison.Ordinal)); + Assert.Contains("$deltatoken=D9", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DeltaUndeletableCheckpointTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DeltaUndeletableCheckpointTests.cs new file mode 100644 index 0000000..7f5ef70 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/DeltaUndeletableCheckpointTests.cs @@ -0,0 +1,375 @@ +using System.Management.Automation; +using System.Net; +using System.Text; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// A recovery that gives up on the checkpoint has to give up on it whether or not the file goes +/// away. Every bail-out in the reconcile deletes the checkpoint and says what it does instead - +/// "Re-enumerating from the last saved delta token; no changes are lost" - and that delete can +/// fail: a checkpoint directory this account may read but not unlink from is the ordinary case, +/// and PaginationCheckpoint.Delete answers false for exactly it. The refusal used to be recorded +/// by nothing but the file's absence, so a failed delete left the run reloading the checkpoint it +/// had just given up on, resuming from its nextLink, putting the previous sync's rows in front of +/// this one's, and saving a delta token past items that are then in no file at all - under a +/// warning promising the opposite. +/// +/// Export-MgxCollection never had this: every bail-out in its reconcile returns false, and that +/// bool is what decides whether the run appends. These pin the same shape for the delta sync. +/// +[Collection("Pipeline")] +public class DeltaUndeletableCheckpointTests +{ + private const string Baseline = """ + {"value":[{"id":"a1"},{"id":"a2"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D1"} + """; + private const string ChangesPage1 = """ + {"value":[{"id":"b1"},{"id":"b2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2"} + """; + private const string ChangesPage2 = """ + {"value":[{"id":"b3"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D2"} + """; + private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + + private static readonly string[] ThisSyncsRows = + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3\"}"]; + + /// + /// Answers by URL rather than in order, so what the sync asks for first is what decides + /// which items it receives: a resume from the checkpoint's skiptoken gets what that + /// skiptoken addresses and nothing before it, which is the whole difference between + /// resuming and re-enumerating. Dies on that page while is set, + /// which is how the interrupted run that leaves the checkpoint dies. + /// + private sealed class ByUrlHandler : HttpMessageHandler + { + private readonly List _uris = []; + + public bool DieOnResume { get; set; } + + public List Uris { get { lock (_uris) { return [.. _uris]; } } } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var uri = request.RequestUri!.ToString(); + lock (_uris) { _uris.Add(uri); } + + var (status, body) = uri.Contains("skiptoken=B2", StringComparison.Ordinal) + ? DieOnResume + ? (HttpStatusCode.InternalServerError, ServerError) + : (HttpStatusCode.OK, ChangesPage2) + : uri.Contains("$deltatoken=D1", StringComparison.Ordinal) + ? (HttpStatusCode.OK, ChangesPage1) + : (HttpStatusCode.OK, Baseline); + + return Task.FromResult(new HttpResponseMessage(status) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } + + private static PowerShell Shell() + { + var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Delta.SyncMgxDelta).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + return ps; + } + + /// The warnings the run wrote. A run that dies is setup here, not a failure. + private static List Sync(string deltaPath, string checkpointPath, string outputPath) + { + using var ps = Shell(); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("OutputFile", outputPath); + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + return [.. ps.Streams.Warning.Select(w => w.Message)]; + } + + private static string NewDir() => Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-undeletable-{Guid.NewGuid():N}")).FullName; + + /// + /// Run 1 completes and records the D1 token; run 2 dies on the second page, leaving a + /// checkpoint that counts two items into the temp it names. That is the state every test + /// here starts from. + /// + private static void BaselineThenDeath( + ByUrlHandler handler, string deltaPath, string checkpointPath, string outputPath) + { + Sync(deltaPath, checkpointPath, outputPath); + handler.DieOnResume = true; + Sync(deltaPath, checkpointPath, outputPath); + handler.DieOnResume = false; + } + + /// + /// Make the checkpoint's directory one this account can read but not unlink from, which is + /// what makes PaginationCheckpoint.Delete answer false. Answers false itself where the host + /// does not enforce that - root ignores the mode bits, and there is no undeletable + /// checkpoint to be had there, so the caller returns rather than asserting something the + /// host cannot show. + /// + private static bool TrySeal(string dir) + { + if (OperatingSystem.IsWindows()) return false; + var canary = Path.Combine(dir, "deletable.probe"); + File.WriteAllText(canary, ""); + File.SetUnixFileMode(dir, UnixFileMode.UserRead | UnixFileMode.UserExecute); + try { File.Delete(canary); } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { return true; } + Unseal(dir); + return false; + } + + private static void Unseal(string dir) + { + if (OperatingSystem.IsWindows()) return; + File.SetUnixFileMode(dir, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + + /// + /// The counted items are in no file - the temp holding them is gone - and the checkpoint + /// naming it cannot be deleted. The bail-out's warning says this run re-enumerates from the + /// delta token, so that is what has to go on the wire: the two items the temp lost come + /// back, and the output holds this sync's rows rather than the previous sync's with the + /// remainder of this one behind them. + /// + [Fact] + public void A_missing_temp_bailout_reenumerates_when_the_checkpoint_cannot_be_deleted() + { + // The checkpoint sits in a directory the run cannot write, which is what makes its + // deletion fail. Windows expresses that through ACLs, not a mode. + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var vault = Directory.CreateDirectory(Path.Combine(dir, "vault")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(vault, "run.checkpoint"); + var outputPath = Path.Combine(dir, "out.jsonl"); + + var handler = new ByUrlHandler(); + using var transport = MgxTransportScope.Inject(handler); + try + { + BaselineThenDeath(handler, deltaPath, checkpointPath, outputPath); + + var checkpoint = PaginationCheckpoint.Load(checkpointPath)!; + Assert.Contains("skiptoken=B2", checkpoint.NextLink); + Assert.Equal(2, checkpoint.ItemsCollected); + Assert.NotNull(checkpoint.TempFile); + + // The temp holding b1 and b2 goes: a tmp sweeper, a container restart, a hand. The + // checkpoint still names it and still counts them. + foreach (var t in Directory.GetFiles(dir, "out.jsonl.*.tmp")) File.Delete(t); + if (!TrySeal(vault)) return; + + var before = handler.Uris.Count; + var warnings = Sync(deltaPath, checkpointPath, outputPath); + Unseal(vault); + + Assert.Contains(warnings, w => w.Contains("temp file is missing or incomplete")); + Assert.True(File.Exists(checkpointPath), + "the checkpoint was deletable after all, so no undeletable one was tested"); + + var wire = handler.Uris.Skip(before).ToList(); + Assert.True(wire[0].Contains("$deltatoken=D1", StringComparison.Ordinal), + $"the run resumed the checkpoint it gave up on; first request: {wire[0]}"); + Assert.DoesNotContain("skiptoken=B2", wire[0]); + + // "No changes are lost": b1 and b2 are back, and the previous sync's rows are not + // in front of them. + Assert.Equal(ThisSyncsRows, File.ReadAllLines(outputPath)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { Unseal(vault); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other bail-out a sync reaches with a checkpoint it wrote for itself: the checkpoint + /// counts its items into the output, and the output no longer holds them, so there is + /// nothing to trim back to. Same undeletable checkpoint, same promise, same properties - + /// and here a resume appends the remainder of the enumeration onto a file that never held + /// its first pages. + /// + [Fact] + public void A_trim_bailout_reenumerates_when_the_checkpoint_cannot_be_deleted() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var vault = Directory.CreateDirectory(Path.Combine(dir, "vault")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(vault, "run.checkpoint"); + var outputPath = Path.Combine(dir, "out.jsonl"); + + var handler = new ByUrlHandler(); + using var transport = MgxTransportScope.Inject(handler); + try + { + BaselineThenDeath(handler, deltaPath, checkpointPath, outputPath); + + // The shape a promoted cancellation leaves: the items are in the output itself, and + // the checkpoint records how many bytes of it they occupy. + File.WriteAllText(outputPath, "{\"id\":\"b1\"}\n{\"id\":\"b2\"}\n"); + foreach (var t in Directory.GetFiles(dir, "out.jsonl.*.tmp")) File.Delete(t); + var checkpoint = PaginationCheckpoint.Load(checkpointPath)!; + checkpoint.TempFile = null; + checkpoint.OutputFile = Path.GetFullPath(outputPath); + checkpoint.DataLength = new FileInfo(outputPath).Length; + checkpoint.Save(checkpointPath); + + // And the output is replaced by one that does not hold them. + File.WriteAllText(outputPath, ""); + if (!TrySeal(vault)) return; + + var before = handler.Uris.Count; + var warnings = Sync(deltaPath, checkpointPath, outputPath); + Unseal(vault); + + Assert.Contains(warnings, w => w.Contains("no longer holds the 2 items")); + Assert.True(File.Exists(checkpointPath), + "the checkpoint was deletable after all, so no undeletable one was tested"); + + var wire = handler.Uris.Skip(before).ToList(); + Assert.True(wire[0].Contains("$deltatoken=D1", StringComparison.Ordinal), + $"the run resumed the checkpoint it gave up on; first request: {wire[0]}"); + Assert.DoesNotContain("skiptoken=B2", wire[0]); + + Assert.Equal(ThisSyncsRows, File.ReadAllLines(outputPath)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { Unseal(vault); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The control, in the same undeletable directory: a checkpoint whose temp is where it says + /// it is is still resumed from, and the resume is the same one it has always been - one + /// request, for the page the checkpoint stopped at. What decides is the reconcile's answer + /// and not what is left on disk, so a run that can delete nothing at all resumes exactly as + /// it does when it can. + /// + [Fact] + public void A_healthy_checkpoint_still_resumes_when_it_cannot_be_deleted() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var vault = Directory.CreateDirectory(Path.Combine(dir, "vault")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(vault, "run.checkpoint"); + var outputPath = Path.Combine(dir, "out.jsonl"); + + var handler = new ByUrlHandler(); + using var transport = MgxTransportScope.Inject(handler); + try + { + BaselineThenDeath(handler, deltaPath, checkpointPath, outputPath); + + // Nothing is missing this time: the temp the checkpoint names holds the two items + // it counts. + Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + if (!TrySeal(vault)) return; + + var before = handler.Uris.Count; + var warnings = Sync(deltaPath, checkpointPath, outputPath); + Unseal(vault); + + Assert.Contains(warnings, w => w.Contains("Recovered 2 items")); + var wire = handler.Uris.Skip(before).ToList(); + Assert.Single(wire); + Assert.Contains("skiptoken=B2", wire[0]); + + Assert.Equal(ThisSyncsRows, File.ReadAllLines(outputPath)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { Unseal(vault); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The steady state a checkpoint nothing can remove leaves: it is still there next run, and + /// next run gives up on it again. Noisy - the warning repeats every time, which is what the + /// delete-failure warning already tells the caller to end by hand - and correct. What must + /// not happen is a later run reading the survivor as a position to resume from. + /// + [Fact] + public void A_second_run_against_the_surviving_checkpoint_gives_up_on_it_again() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var vault = Directory.CreateDirectory(Path.Combine(dir, "vault")).FullName; + var deltaPath = Path.Combine(dir, "state.json"); + var checkpointPath = Path.Combine(vault, "run.checkpoint"); + var outputPath = Path.Combine(dir, "out.jsonl"); + + var handler = new ByUrlHandler(); + using var transport = MgxTransportScope.Inject(handler); + try + { + BaselineThenDeath(handler, deltaPath, checkpointPath, outputPath); + foreach (var t in Directory.GetFiles(dir, "out.jsonl.*.tmp")) File.Delete(t); + if (!TrySeal(vault)) return; + + // Run 3 gives up on the checkpoint, cannot delete it, cannot save one of its own + // over it either, and dies where the run before it died - so run 4 meets the file + // run 3 met, unchanged, with the delta token still where it was. + handler.DieOnResume = true; + var beforeThird = handler.Uris.Count; + var third = Sync(deltaPath, checkpointPath, outputPath); + handler.DieOnResume = false; + + Assert.Contains(third, w => w.Contains("temp file is missing or incomplete")); + Assert.True(handler.Uris[beforeThird].Contains("$deltatoken=D1", StringComparison.Ordinal), + $"the run resumed the checkpoint it gave up on; first request: {handler.Uris[beforeThird]}"); + Assert.True(File.Exists(checkpointPath)); + Assert.Contains("skiptoken=B2", PaginationCheckpoint.Load(checkpointPath)!.NextLink); + + var beforeFourth = handler.Uris.Count; + var fourth = Sync(deltaPath, checkpointPath, outputPath); + Unseal(vault); + + Assert.Contains(fourth, w => w.Contains("temp file is missing or incomplete")); + var wire = handler.Uris.Skip(beforeFourth).ToList(); + Assert.True(wire[0].Contains("$deltatoken=D1", StringComparison.Ordinal), + $"the run resumed the checkpoint it gave up on; first request: {wire[0]}"); + Assert.DoesNotContain("skiptoken=B2", wire[0]); + + // Nothing doubled, and nothing from the runs before it in front of it. + Assert.Equal(ThisSyncsRows, File.ReadAllLines(outputPath)); + Assert.Contains("$deltatoken=D2", DeltaState.Load(deltaPath)!.DeltaLink); + } + finally + { + try { Unseal(vault); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/DiagnosticsDrainTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/DiagnosticsDrainTests.cs new file mode 100644 index 0000000..87dc342 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/DiagnosticsDrainTests.cs @@ -0,0 +1,44 @@ +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// When a request dies on an exception type no catch filter expects, the buffered verbose +/// messages - the retry history that explains what led up to the failure - still reach the +/// caller instead of being discarded with the client. +/// +[Collection("Pipeline")] +public class DiagnosticsDrainTests +{ + [Fact] + public void An_unexpected_exception_still_delivers_the_buffered_retry_history() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.ServiceUnavailable); + wire.QueueException(new InvalidOperationException("boom")); + + using (MgxTransportScope.Inject(wire, options: new ResilientGraphClientOptions + { + NoRateLimit = true, MaxRetryAttempts = 3, MaxRetryAfterSeconds = 1 + })) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users/u1") + .AddParameter("Verbose", true); + Assert.ThrowsAny(() => ps.Invoke()); + + Assert.Contains(ps.Streams.Verbose, + v => v.Message.Contains("Retry attempt", StringComparison.OrdinalIgnoreCase)); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ErrorActionMatrixTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ErrorActionMatrixTests.cs new file mode 100644 index 0000000..6824491 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/ErrorActionMatrixTests.cs @@ -0,0 +1,262 @@ +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// about_Mgx_Errors states the -ErrorAction and -ErrorVariable contract in prose and names only +/// Stop and SilentlyContinue. This drives all twenty-four preference-by-context cells and asserts, +/// per cell, whether the pipeline terminates, whether a record reaches $Error, whether +/// -ErrorVariable collects it, and whether anything is displayed. +/// +/// What was covered before this file, and by what: BatchErrorSurfacingTests pinned the batch +/// context under Stop twice (termination alone, with and without a dead-letter path) and under the +/// default preference once (one record on the error stream). Three cells of twenty-four, none of +/// them across all three properties; -ErrorVariable was asserted nowhere and Ignore appeared +/// nowhere. +/// +/// All four preferences are driven as common parameters. Ignore cannot be set as a preference +/// variable, so a preference-variable form could not cover the same grid. +/// +/// (Corpus: M365DSC-7273 sustained throttling - the throttling cells generalize it to what a +/// caller sees when a 429 outlives its retries; M365DSC-7198 batch semantics - the batch cells +/// extend it from Stop to all four preferences.) +/// +[Collection("Pipeline")] +public class ErrorActionMatrixTests +{ + private const string NotFoundBody = + """{"error":{"code":"Request_ResourceNotFound","message":"u1 does not exist"}}"""; + private const string ServerErrorBody = + """{"error":{"code":"InternalServerError","message":"boom"}}"""; + private const string ThrottledBody = + """{"error":{"code":"TooManyRequests","message":"slow down"}}"""; + private const string OneOkOneNotFound = """ + { "responses": [ + { "id": "1", "status": 200, "body": { "id": "u1" } }, + { "id": "2", "status": 404, "body": { "error": { "code": "Request_ResourceNotFound", "message": "u2 does not exist" } } } + ] } + """; + + private static readonly string[] Contexts = + [ + "DirectRequest", "ExhaustedRetry", "Batch", "Pagination", "CircuitBreaker", "Throttling" + ]; + + private static readonly string[] Preferences = + [ + "Stop", "Continue", "SilentlyContinue", "Ignore" + ]; + + public static TheoryData Cells + { + get + { + var cells = new TheoryData(); + foreach (var context in Contexts) + foreach (var preference in Preferences) + cells.Add(context, preference); + return cells; + } + } + + /// The wire, the options, an optional priming run, and the pipeline under test. + private sealed record Cell( + MockHttpHandler Wire, + ResilientGraphClientOptions Options, + string? Prime, + string Pipeline); + + private static ResilientGraphClientOptions Fast => new() + { + NoRateLimit = true, + MaxRetryAttempts = 1, + MaxRetryAfterSeconds = 1 + }; + + private static Cell BuildCell(string context) + { + var wire = new MockHttpHandler(); + switch (context) + { + // A 404 the retry policy does not retry: one request, one record. + case "DirectRequest": + wire.SetDefaultResponse(HttpStatusCode.NotFound, NotFoundBody); + return new Cell(wire, Fast, null, "Invoke-MgxRequest -Uri /users/u1"); + + // Every attempt fails, so the record is written after the last retry rather than + // on the first failure - the case about_Mgx_Errors calls out under RETRIES. + case "ExhaustedRetry": + wire.SetDefaultResponse(HttpStatusCode.ServiceUnavailable, ServerErrorBody); + return new Cell(wire, Fast, null, "Invoke-MgxRequest -Uri /users/u1"); + + // The chunk POST succeeds and one item inside it answers 404. + case "Batch": + wire.QueueResponse(HttpStatusCode.OK, OneOkOneNotFound); + return new Cell(wire, Fast, null, + "Invoke-MgxBatchRequest -Uri @('/users/u1','/users/u2') -Method GET"); + + // Page one is delivered, so output exists before the failure the record describes. + case "Pagination": + wire.QueueResponse(HttpStatusCode.OK, TestData.UsersPage1); + wire.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerErrorBody); + return new Cell(wire, Fast, null, "Invoke-MgxRequest -Uri /users -All"); + + // The priming run's two failed attempts meet the minimum throughput at a failure + // ratio of 1.0, so the breaker is open before the measured pipeline runs. + case "CircuitBreaker": + wire.SetDefaultResponse(HttpStatusCode.InternalServerError, ServerErrorBody); + return new Cell( + wire, + new ResilientGraphClientOptions + { + NoRateLimit = true, + MaxRetryAttempts = 1, + MaxRetryAfterSeconds = 1, + CircuitBreakerMinThroughput = 2, + CircuitBreakerFailureRatio = 0.5, + CircuitBreakerDurationSeconds = 30 + }, + "Invoke-MgxRequest -Uri /users/u1 -ErrorAction SilentlyContinue | Out-Null", + "Invoke-MgxRequest -Uri /users/u1"); + + case "Throttling": + wire.SetDefaultResponse(HttpStatusCode.TooManyRequests, ThrottledBody, + new Dictionary { ["Retry-After"] = "0" }); + return new Cell(wire, Fast, null, "Invoke-MgxRequest -Uri /users/u1"); + + default: + throw new ArgumentOutOfRangeException(nameof(context), context, "unknown failure context"); + } + } + + private static System.Management.Automation.PowerShell CreateShell() + { + var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + return ps; + } + + private sealed record Outcome( + bool Terminated, + int ErrorCount, + int VariableCount, + string FirstErrorId, + int ErrorStreamCount, + bool FailureServed); + + /// + /// Runs one cell inside PowerShell so the preference is interpreted where it is interpreted + /// in production - by the engine, not by a caller reading a C# property. + /// + private static Outcome Drive(string context, string preference) + { + var cell = BuildCell(context); + using var transport = MgxTransportScope.Inject(cell.Wire, options: cell.Options); + using var ps = CreateShell(); + + // The priming run goes through its own Invoke so the error stream can be emptied after + // it. That stream is the only thing telling Continue from SilentlyContinue - both leave + // a record in $Error and in -ErrorVariable, and only Continue displays one. + if (cell.Prime != null) + { + ps.AddScript(cell.Prime); + ps.Invoke(); + ps.Commands.Clear(); + } + + ps.Streams.Error.Clear(); + + // $Error is cleared after any priming run, so the count measures the pipeline under test. + var script = $$""" + $Error.Clear() + $ev = $null + $terminated = $false + try { {{cell.Pipeline}} -ErrorAction {{preference}} -ErrorVariable ev | Out-Null } + catch { $terminated = $true } + [pscustomobject]@{ + Terminated = $terminated + ErrorCount = $Error.Count + VariableCount = @($ev).Where({ $null -ne $_ }).Count + FirstErrorId = if ($Error.Count) { @($Error)[-1].FullyQualifiedErrorId } else { '' } + } + """; + + ps.AddScript(script); + var results = ps.Invoke(); + var outcome = Assert.Single(results); + return new Outcome( + (bool)outcome.Properties["Terminated"].Value, + (int)outcome.Properties["ErrorCount"].Value, + (int)outcome.Properties["VariableCount"].Value, + (string)outcome.Properties["FirstErrorId"].Value, + ps.Streams.Error.Count, + FailureServed(context, cell.Wire)); + } + + /// + /// That the wire produced the failure this context is about. Under Ignore every count the + /// grid reads is zero whether the request failed or succeeded, so the cell needs a signal + /// that does not come from the pipeline. A batch failure travels inside a 200, so there the + /// signal is the chunk going out and the queued response still carrying a failed item. + /// + private static bool FailureServed(string context, MockHttpHandler wire) => context == "Batch" + ? wire.RequestCount > 0 && OneOkOneNotFound.Contains("\"status\": 404", StringComparison.Ordinal) + : wire.ServedStatusCodes.Exists(status => (int)status >= 400); + + /// + /// The record each context is supposed to produce. Without this a context that quietly stopped + /// failing the way it claims - a 404 answered before pagination started, say - would still + /// satisfy every count in the grid. + /// + private static string ExpectedErrorId(string context) => context switch + { + "DirectRequest" => "Request_ResourceNotFound", + "ExhaustedRetry" => "InternalServerError", + "Batch" => "BatchItemError", + "Pagination" => "InternalServerError", + "CircuitBreaker" => "CircuitBroken", + "Throttling" => "TooManyRequests", + _ => throw new ArgumentOutOfRangeException(nameof(context), context, "unknown failure context") + }; + + [Theory] + [MemberData(nameof(Cells))] + public void A_preference_decides_termination_the_error_stack_and_the_variable( + string context, string preference) + { + var outcome = Drive(context, preference); + + Assert.Equal(preference == "Stop", outcome.Terminated); + + // Only Continue displays the record. SilentlyContinue keeps it, in $Error and in + // -ErrorVariable, and shows nothing - which is the whole difference between them, and + // the one about_Mgx_Errors describes. + Assert.Equal(preference == "Continue", outcome.ErrorStreamCount > 0); + + if (preference == "Ignore") + { + // Ignore is the one preference that leaves no trace: nothing to find afterwards, + // by either route. Every count below holds just as well for a request that + // succeeded, so the wire has to say it produced the failure this cell is about. + Assert.True(outcome.FailureServed, + $"{context}/{preference}: the wire never served a failure"); + Assert.Equal(0, outcome.ErrorCount); + Assert.Equal(0, outcome.VariableCount); + } + else + { + Assert.True(outcome.ErrorCount > 0, + $"{context}/{preference}: no record reached $Error"); + Assert.True(outcome.VariableCount > 0, + $"{context}/{preference}: -ErrorVariable collected nothing"); + Assert.StartsWith(ExpectedErrorId(context), outcome.FirstErrorId); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ExpandMgxRelationTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ExpandMgxRelationTests.cs index f2618ec..46831e5 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/ExpandMgxRelationTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/ExpandMgxRelationTests.cs @@ -1,6 +1,7 @@ -using System.Collections; +using System.Collections; using System.Management.Automation; using System.Net; +using Mgx.Cmdlets.Base; using System.Text.Json; using Mgx.Cmdlets.Cmdlets.Expand; using Mgx.IntegrationTests.Fakes; @@ -81,10 +82,9 @@ public void BuildUrl_AddsTopParameter() [Fact] public void GetStatusCodeFromException_ReturnsGraphServiceExceptionStatus() { - var cmdlet = new ExpandMgxRelation(); var ex = new Mgx.Engine.Models.GraphServiceException(HttpStatusCode.NotFound, "Not found"); - var status = InvokeMethod(cmdlet, "GetStatusCodeFromException", ex); + var status = MgxErrorPresentation.TryGetStatus(ex); Assert.Equal(HttpStatusCode.NotFound, status); } @@ -92,10 +92,9 @@ public void GetStatusCodeFromException_ReturnsGraphServiceExceptionStatus() [Fact] public void GetStatusCodeFromException_ReturnsHttpRequestExceptionStatus() { - var cmdlet = new ExpandMgxRelation(); var ex = new HttpRequestException("Error", null, HttpStatusCode.ServiceUnavailable); - var status = InvokeMethod(cmdlet, "GetStatusCodeFromException", ex); + var status = MgxErrorPresentation.TryGetStatus(ex); Assert.Equal(HttpStatusCode.ServiceUnavailable, status); } @@ -113,6 +112,6 @@ public void MapStatusToCategory_MapsCommonCodes() Assert.Equal(ErrorCategory.InvalidArgument, category); category = InvokeMethod(typeof(Mgx.Cmdlets.Base.MgxCmdletBase), "MapStatusToCategory", HttpStatusCode.ServiceUnavailable); - Assert.Equal(ErrorCategory.NotSpecified, category); + Assert.Equal(ErrorCategory.ResourceUnavailable, category); } } \ No newline at end of file diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ExpandRelationBufferTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ExpandRelationBufferTests.cs new file mode 100644 index 0000000..8336e8d --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/ExpandRelationBufferTests.cs @@ -0,0 +1,86 @@ +using System.Collections; +using System.Management.Automation; +using System.Net; + +namespace Mgx.IntegrationTests; + +/// +/// Expand-MgxRelation holds every piped object until the pipeline ends, because the fan-out +/// cannot start before it knows all the ids. That is a whole collection resident in memory, +/// and the 50,000th arrival is where the cmdlet says so - while the caller can still stop it +/// and put a -Filter or a -Top upstream. +/// +/// The count is the buffer's, not the fan-out's, so the objects below all carry one id: the +/// dedup leaves a single request to answer and what the test measures is the buffering. +/// +[Collection("Pipeline")] +public class ExpandRelationBufferTests +{ + private const int WarnAt = 50_000; + + private static PowerShell Shell() + { + var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + return ps; + } + + /// + /// objects down the pipe, and everything the run had to say. + /// Built here rather than generated in script: the shape under test is what the cmdlet + /// does with the objects, not what it costs PowerShell to make them. + /// + private static (int Output, List Warnings, List Errors, int Requests) + Expand(int count) + { + var input = new Hashtable[count]; + for (int i = 0; i < count; i++) + input[i] = new Hashtable(StringComparer.OrdinalIgnoreCase) { ["id"] = "u1" }; + + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); + using (MgxTransportScope.Inject(wire)) + { + using var ps = Shell(); + ps.Runspace.SessionStateProxy.SetVariable("objects", input); + ps.AddScript("$objects | Expand-MgxRelation -Uri '/users/{id}/manager' -As manager"); + var output = ps.Invoke(); + + return (output.Count, ps.Streams.Warning.Select(w => w.Message).ToList(), + ps.Streams.Error.ToList(), wire.RequestCount); + } + } + + [Fact] + public void The_fifty_thousandth_buffered_object_warns_what_is_being_held() + { + var (output, warnings, errors, requests) = Expand(WarnAt); + + Assert.Empty(errors); + + // One id, so one request: nothing here waited on a fan-out. + Assert.Equal(1, requests); + Assert.Equal(WarnAt, output); + + var warned = Assert.Single(warnings); + Assert.Contains("Buffered 50,000+ objects in memory", warned, StringComparison.Ordinal); + Assert.Contains("-Filter", warned, StringComparison.Ordinal); + Assert.Contains("-Top", warned, StringComparison.Ordinal); + } + + [Fact] + public void One_object_short_of_the_count_is_not_warned_about() + { + var (output, warnings, errors, _) = Expand(WarnAt - 1); + + Assert.Empty(errors); + Assert.Equal(WarnAt - 1, output); + Assert.Empty(warnings); + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ExportCheckpointIntegrityTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ExportCheckpointIntegrityTests.cs index c5e3f99..18e1b33 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/ExportCheckpointIntegrityTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/ExportCheckpointIntegrityTests.cs @@ -1,6 +1,5 @@ using System.Management.Automation; using System.Net; -using System.Reflection; using System.Text; using Mgx.Cmdlets.Base; using Mgx.Engine.Http; @@ -25,6 +24,13 @@ public class ExportCheckpointIntegrityTests """; private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + private const string EmptyPage = """ + {"value":[],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P3"} + """; + private const string LastPage = """ + {"value":[{"id":"u4"},{"id":"u5"}]} + """; + private sealed class ScriptedHandler : HttpMessageHandler { private readonly Queue<(HttpStatusCode Status, string Body)> _steps = new(); @@ -53,36 +59,38 @@ protected override Task SendAsync( } } - private static void InjectMock(HttpMessageHandler handler) + /// + /// Three pages with the middle one empty behind a nextLink, which is how a page boundary is + /// reached with no item written since the fetch. Every answer takes a real asynchronous hop: + /// a socket read always yields and Task.FromResult never does, and the thread the iterator + /// resumes on is what these pages are here to settle. + /// + private sealed class EmptyMiddlePageHandler : HttpMessageHandler { - ResiliencePipelineFactory.Reset(); - var t = typeof(MgxCmdletBase); - t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, new HttpClient(handler)); - t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - // Cleared with the fingerprint. A stale reference from an earlier test reads as a - // credential change and the injected client is rebuilt away - t.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_ownsHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, false); - t.GetField("s_graphEndpoint", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! - .SetValue(null, "https://graph.microsoft.com"); - t.GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! - .SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); - } + private readonly object _lock = new(); + public List Urls { get; } = []; - private static void CleanupMock() - { - var t = typeof(MgxCmdletBase); - t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - ResiliencePipelineFactory.Reset(); + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + lock (_lock) { Urls.Add(url); } + + await Task.Yield(); + + var body = url.Contains("skiptoken=P3", StringComparison.Ordinal) ? LastPage + : url.Contains("skiptoken=P2", StringComparison.Ordinal) ? EmptyPage + : Page1; + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }; + } } - private static long? Export(string outputPath, string? checkpointPath, bool all, int top = 0) + private static long? Export(string outputPath, string? checkpointPath, bool all, int top = 0, + string uri = "/users") { using var ps = PowerShell.Create(); ps.AddCommand("Import-Module") @@ -94,7 +102,7 @@ private static void CleanupMock() ps.Commands.Clear(); var cmd = ps.AddCommand("Export-MgxCollection") - .AddParameter("Uri", "/users") + .AddParameter("Uri", uri) .AddParameter("OutputFile", outputPath); if (checkpointPath != null) cmd.AddParameter("CheckpointPath", checkpointPath); if (all) cmd.AddParameter("All"); @@ -109,6 +117,32 @@ private static void CleanupMock() return null; } + /// + /// An export whose warnings and error records the caller reads, rather than only the item + /// count Export hands back. Nothing is swallowed here: a run that dies off the pipeline + /// thread has to fail the test that says it does not. + /// + private static (List Warnings, int Errors) ExportReporting( + string outputPath, string checkpointPath) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("All"); + ps.Invoke(); + return ([.. ps.Streams.Warning.Select(w => w.Message)], ps.Streams.Error.Count); + } + private static string[] Lines(string path) => File.Exists(path) ? File.ReadAllLines(path) : []; private static string NewDir() => @@ -130,7 +164,7 @@ public void A_resume_does_not_count_an_earlier_exports_output_as_this_ones_items try { var handler = new ScriptedHandler(); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); // Run 1: a one-item probe export. Completes, no checkpoint. handler.Queue(HttpStatusCode.OK, Page1); @@ -157,7 +191,6 @@ public void A_resume_does_not_count_an_earlier_exports_output_as_this_ones_items } finally { - CleanupMock(); try { Directory.Delete(dir, true); } catch { } } } @@ -178,7 +211,7 @@ public void A_transient_failure_keeps_the_temp_the_checkpoint_names() try { var handler = new ScriptedHandler(); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); handler.Queue(HttpStatusCode.OK, Page1); handler.Queue(HttpStatusCode.InternalServerError, ServerError); @@ -192,7 +225,6 @@ public void A_transient_failure_keeps_the_temp_the_checkpoint_names() } finally { - CleanupMock(); try { Directory.Delete(dir, true); } catch { } } } @@ -210,7 +242,7 @@ public void An_export_resumes_after_a_transient_failure_rather_than_starting_ove try { var handler = new ScriptedHandler(); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); handler.Queue(HttpStatusCode.OK, Page1); handler.Queue(HttpStatusCode.InternalServerError, ServerError); @@ -228,7 +260,65 @@ public void An_export_resumes_after_a_transient_failure_rather_than_starting_ove } finally { - CleanupMock(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The last step of a completed export is the move that puts its temp over the output, and + /// that move can fail with every page already fetched: a destination another process holds + /// open, a read-only file, a share that dropped. Deleting the checkpoint before it made + /// that failure read as "nothing was resumable", and the temp holding the whole export was + /// deleted with it - hours of enumeration, at the one moment all of it was on disk. The two + /// files have to be left exactly as an interruption leaves them, and the next run has to + /// resume from them. + /// + [Fact] + public void A_failed_promotion_keeps_the_checkpoint_and_the_temp_holding_the_export() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // Something at the output path the promotion cannot replace, so both pages are + // fetched and written and the move at the end of the run is what fails. + Directory.CreateDirectory(output); + + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + Assert.Null(Export(output, checkpoint, all: true)); + + Assert.True(File.Exists(checkpoint), + "the failed promotion deleted the position the next run resumes from"); + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.NotNull(cp!.TempFile); + var temp = Path.Combine(dir, cp.TempFile!); + Assert.True(File.Exists(temp), + "the failed promotion deleted the temp holding every item the export fetched"); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(temp)); + + // With the occupier gone: the temp is promoted, one page is fetched - the one the + // checkpoint recorded - and the export is not enumerated again from the start. + Directory.Delete(output); + handler.Queue(HttpStatusCode.OK, Page2); + var before = handler.Requests.Count; + var reported = Export(output, checkpoint, all: true); + + Assert.Equal(1, handler.Requests.Count - before); + Assert.Contains("skiptoken=P2", handler.Requests[before]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], Lines(output)); + Assert.Equal(3, reported); + Assert.False(File.Exists(checkpoint)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { try { Directory.Delete(dir, true); } catch { } } } @@ -246,7 +336,7 @@ public void A_resume_replaces_the_previous_export_rather_than_appending_to_it() try { var handler = new ScriptedHandler(); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); handler.Queue(HttpStatusCode.OK, Page1); handler.Queue(HttpStatusCode.OK, Page2); @@ -266,7 +356,6 @@ public void A_resume_replaces_the_previous_export_rather_than_appending_to_it() } finally { - CleanupMock(); try { Directory.Delete(dir, true); } catch { } } } @@ -285,7 +374,7 @@ public void A_surviving_temp_is_promoted_over_an_earlier_exports_output() try { var handler = new ScriptedHandler(); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); handler.Queue(HttpStatusCode.OK, Page1); Assert.Equal(1, Export(output, checkpointPath: null, all: false, top: 1)); @@ -301,6 +390,7 @@ public void A_surviving_temp_is_promoted_over_an_earlier_exports_output() ItemsCollected = 2, PageItemsAlreadyWritten = 0, TempFile = Path.GetFileName(temp), + OutputFile = output, DataLength = new FileInfo(temp).Length }.Save(checkpoint); @@ -313,7 +403,6 @@ public void A_surviving_temp_is_promoted_over_an_earlier_exports_output() } finally { - CleanupMock(); try { Directory.Delete(dir, true); } catch { } } } @@ -334,7 +423,7 @@ public void A_checkpoint_that_cannot_say_where_its_items_are_exports_again_rathe try { var handler = new ScriptedHandler(); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); // A previous export completed and left its output behind. File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n{\"id\":\"old3\"}\n"); @@ -361,7 +450,6 @@ public void A_checkpoint_that_cannot_say_where_its_items_are_exports_again_rathe } finally { - CleanupMock(); try { Directory.Delete(dir, true); } catch { } } } @@ -389,7 +477,7 @@ static string BigPage(string nextLink) => try { var handler = new ScriptedHandler(); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); // One real run, only so the checkpoint on disk carries the exact URL the cmdlet // builds for these parameters. @@ -431,7 +519,7 @@ static string BigPage(string nextLink) => Assert.Equal(lines.Length, lines.Distinct().Count()); Assert.Equal(1001, lines.Length); } - finally { CleanupMock(); try { Directory.Delete(dir, true); } catch { } } + finally { try { Directory.Delete(dir, true); } catch { } } } /// @@ -449,7 +537,7 @@ public void A_checkpoint_recording_a_negative_length_does_not_wedge_the_export() try { var handler = new ScriptedHandler(); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); File.WriteAllText(output, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); new PaginationCheckpoint @@ -473,7 +561,6 @@ public void A_checkpoint_recording_a_negative_length_does_not_wedge_the_export() } finally { - CleanupMock(); try { Directory.Delete(dir, true); } catch { } } } @@ -493,7 +580,7 @@ public void A_fresh_export_does_not_delete_a_temp_another_run_is_writing() try { var handler = new ScriptedHandler(); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); // Another export holds its temp open, exactly as StreamWriter does. using (var held = new FileStream(live, FileMode.Create, FileAccess.Write, FileShare.Read)) @@ -513,7 +600,265 @@ public void A_fresh_export_does_not_delete_a_temp_another_run_is_writing() } finally { - CleanupMock(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A checkpoint written by a release that recorded no output file, against the output it + /// was in fact collecting into. The files corroborate it - the output is there and holds + /// the bytes it counted - so an upgrade partway through an export still resumes. + /// + [Fact] + public void A_checkpoint_from_before_outputs_were_recorded_resumes_against_its_own_output() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + File.WriteAllText(output, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + DataLength = new FileInfo(output).Length, + }.Save(checkpoint); + + handler.Queue(HttpStatusCode.OK, Page2); + var before = handler.Requests.Count; + var reported = Export(output, checkpoint, all: true); + + Assert.Contains("skiptoken=P2", handler.Requests[before]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], Lines(output)); + Assert.Equal(3, reported); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The layout a pre-2.1.0 release left when a fresh export was killed: a checkpoint naming + /// no temp and recording no length, its items in a temp beside an output that was never + /// promoted. Deciding ownership from what such a checkpoint records refuses it for want of + /// anything recorded, and the export then swept the temp holding those items and enumerated + /// the collection again. What is on disk decides it instead - a temp carrying this output's + /// own name, and no output for its items to be confused with. + /// + [Fact] + public void A_checkpoint_from_before_temps_were_recorded_adopts_the_temp_holding_its_items() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // A fresh run's first page, in the temp it died holding. + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0 + }.Save(checkpoint); + + handler.Queue(HttpStatusCode.OK, Page2); + var before = handler.Requests.Count; + var reported = Export(output, checkpoint, all: true); + + // Resumed onto the recovered items rather than fetching them a second time. + Assert.Contains("skiptoken=P2", handler.Requests[before]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], Lines(output)); + Assert.Equal(3, reported); + Assert.False(File.Exists(temp), "the temp its items came from was left behind"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same shape of checkpoint, against an output it was never measured against. Nothing + /// in it names a file, and a null read as "mine" let a second export act on the first + /// one's position - here by declaring the temp it names unusable and deleting the + /// checkpoint, which is the one thing that made the first export's items unreachable. + /// + [Fact] + public void A_checkpoint_from_before_outputs_were_recorded_is_not_this_exports_by_default() + { + var dir = NewDir(); + var mine = Path.Combine(dir, "users.jsonl"); + var theirs = Path.Combine(dir, "other.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // An export to users.jsonl collects page 1 into its temp and dies on page 2. + handler.Queue(HttpStatusCode.OK, Page1); + Export(mine, checkpoint, all: true); + var temp = Assert.Single(Directory.GetFiles(dir, "users.jsonl.*.tmp")); + + // Its checkpoint, as the release before this one would have written it. + var upgraded = PaginationCheckpoint.Load(checkpoint)!; + upgraded.OutputFile = null; + upgraded.Save(checkpoint); + + // A second export shares the checkpoint path and fails on its first page. + Export(theirs, checkpoint, all: true); + + Assert.True(File.Exists(checkpoint), "the other export's position was deleted"); + Assert.True(File.Exists(temp), "the other export's temp was taken"); + + // The first export still resumes onto what it had already collected. + handler.Queue(HttpStatusCode.OK, Page2); + var reported = Export(mine, checkpoint, all: true); + + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], Lines(mine)); + Assert.Equal(3, reported); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same export, resumed with -Uri typed differently. Graph answers "/users" and + /// "/Users" from one collection, so both runs enumerate the same thing - but the recorded + /// resource was compared ordinally, which made the second run a different export: the + /// resume was refused, the collection was fetched again from its first page, and the + /// caller was told their checkpoint belonged to something else. + /// + [Fact] + public void A_checkpoint_written_under_another_spelling_of_the_resource_still_resumes() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + handler.Queue(HttpStatusCode.OK, Page1); + Export(output, checkpoint, all: true, uri: "/users"); + + handler.Queue(HttpStatusCode.OK, Page2); + var before = handler.Requests.Count; + var reported = Export(output, checkpoint, all: true, uri: "/Users"); + + Assert.Contains("skiptoken=P2", handler.Requests[before]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], Lines(output)); + Assert.Equal(3, reported); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The endpoint a session reports is not stable between runs: it comes back with a trailing + /// slash, through a gateway prefix, or from a sovereign cloud. None of that changes which + /// resource an export enumerates, but comparing whole URLs made it change whose checkpoint + /// the file on disk was - so the resume was refused and the collection re-enumerated from + /// the first page. + /// + [Fact] + public void A_checkpoint_written_under_another_spelling_of_the_endpoint_still_resumes() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + + using (MgxTransportScope.Inject(handler, endpoint: "https://graph.microsoft.com/")) + { + handler.Queue(HttpStatusCode.OK, Page1); + Export(output, checkpoint, all: true); + } + + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.Contains("//v1.0/users", cp!.Resource); + + using (MgxTransportScope.Inject(handler, endpoint: "https://graph.microsoft.com")) + { + handler.Queue(HttpStatusCode.OK, Page2); + var before = handler.Requests.Count; + var reported = Export(output, checkpoint, all: true); + + Assert.Contains("skiptoken=P2", handler.Requests[before]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], Lines(output)); + Assert.Equal(3, reported); + } + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A checkpoint the export cannot write, over pages whose middle one is empty behind a + /// nextLink. The page-boundary callback runs on whichever thread the iterator resumed on, + /// and a page that writes nothing resumes on the thread pool - so warning about the disk + /// from there ended the export at that page, taking the pages behind it with it. The + /// message is buffered on the client's warning channel and written by a drain on the + /// pipeline thread: one per failed save, and every page still reaches the output. + /// + [Fact] + public void A_failed_boundary_save_warns_and_the_export_goes_on() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + // Every save into this path fails: Save stages its bytes under this name first, and + // no account writes a file over a directory. The directory holding it is not sealed + // instead, the way a read-only disk would seal it, because the export writes its + // output there too and that write is not what is under test. + Directory.CreateDirectory(checkpoint + ".tmp"); + + var handler = new EmptyMiddlePageHandler(); + using var transport = MgxTransportScope.Inject(handler); + var (warnings, errors) = ExportReporting(output, checkpoint); + + Assert.Equal(0, errors); + + // Both boundaries carrying a nextLink try to save, and both fail. + Assert.Equal(2, warnings.Count( + w => w.StartsWith("Checkpoint save failed (page boundary)", StringComparison.Ordinal))); + + // The empty page sits behind the second request, so an export that died reporting + // its failed save never asks for the third - and the output ends two items short. + Assert.Equal(3, handler.Urls.Count); + Assert.Contains("skiptoken=P3", handler.Urls[2]); + Assert.Equal( + ["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u4\"}", "{\"id\":\"u5\"}"], + Lines(output)); + } + finally + { try { Directory.Delete(dir, true); } catch { } } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ExportCorruptCheckpointTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ExportCorruptCheckpointTests.cs new file mode 100644 index 0000000..6c3deac --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/ExportCorruptCheckpointTests.cs @@ -0,0 +1,216 @@ +using System.Management.Automation; +using System.Net; +using System.Text; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// A checkpoint file that cannot be read is not a resume. PaginationCheckpoint.Load answers +/// null for a truncated file, an empty one, a field of the wrong JSON type and a file the +/// account cannot open - and an export that decided to append from the checkpoint merely +/// EXISTING wrote a second complete copy of the collection onto the first, reporting the +/// item count of one pass while the file held two. +/// +[Collection("Pipeline")] +public class ExportCorruptCheckpointTests +{ + private const string Page1 = """ + {"value":[{"id":"u1"},{"id":"u2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2"} + """; + private const string Page2 = """ + {"value":[{"id":"u3"}]} + """; + private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + + private sealed class ScriptedHandler : HttpMessageHandler + { + private readonly Queue<(HttpStatusCode Status, string Body)> _steps = new(); + private readonly object _lock = new(); + + public void Queue(HttpStatusCode status, string body) + { + lock (_lock) _steps.Enqueue((status, body)); + } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + (HttpStatusCode Status, string Body) step; + lock (_lock) + step = _steps.Count > 0 ? _steps.Dequeue() : (HttpStatusCode.InternalServerError, ServerError); + return Task.FromResult(new HttpResponseMessage(step.Status) + { + RequestMessage = request, + Content = new StringContent(step.Body, Encoding.UTF8, "application/json") + }); + } + } + + private static long? Export(string outputPath, string? checkpointPath) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + var cmd = ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", outputPath) + .AddParameter("All"); + if (checkpointPath != null) cmd.AddParameter("CheckpointPath", checkpointPath); + try + { + foreach (var r in ps.Invoke()) + if (r?.BaseObject is Mgx.Cmdlets.Models.MgxExportResult summary) + return summary.ItemCount; + } + catch (CmdletInvocationException) { } + return null; + } + + private static string NewDir() => + Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-export-corrupt-{Guid.NewGuid():N}")).FullName; + + private static void ExportsOnceOver(string checkpointContent) + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // A completed export, with no checkpoint of its own. + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + Assert.Equal(3, Export(output, checkpointPath: null)); + + File.WriteAllText(checkpoint, checkpointContent); + + // Same export again, now pointed at a checkpoint nothing can read. + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + var reported = Export(output, checkpoint); + + var lines = File.ReadAllLines(output); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], lines); + Assert.Equal(3, reported); + Assert.Equal(lines.Length, reported); + Assert.False(File.Exists(checkpoint)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + [Fact] + public void A_truncated_checkpoint_does_not_append_a_whole_second_export() => + ExportsOnceOver("""{"resource":"https://graph.microsoft.com/v1.0/users?$top=999","nextLi"""); + + [Fact] + public void An_empty_checkpoint_does_not_append_a_whole_second_export() => + ExportsOnceOver(""); + + [Fact] + public void A_checkpoint_with_a_field_of_the_wrong_type_does_not_append_a_whole_second_export() => + ExportsOnceOver(""" + {"resource":"https://graph.microsoft.com/v1.0/users?$top=999", + "nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + "itemsCollected":"two","pageItemsAlreadyWritten":0,"tempFile":null,"dataLength":24} + """); + + /// + /// The same branch, reached by a file that is torn rather than closed: the crash that left + /// it got between the write and the rename. This is the one shape of "does not load" that + /// needs nothing of the filesystem to produce, so it is the shape that can be asserted + /// where a Unix file mode cannot go - Windows denies a read through an ACL, and the test + /// below returns before its first assertion there. + /// + [Fact] + public void A_checkpoint_this_run_cannot_parse_is_left_for_a_run_that_can() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + File.WriteAllText(checkpoint, + """{"resource":"https://graph.microsoft.com/v1.0/users?$top=999","nextLi"""); + var before = File.ReadAllBytes(checkpoint); + + // The export fails on its first page, so the success path - which deletes the + // checkpoint whatever it holds - is not what this measures. + Assert.Null(Export(output, checkpoint)); + + Assert.True(File.Exists(checkpoint), + "a position this run could not read was deleted anyway"); + Assert.Equal(before, File.ReadAllBytes(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// "Cannot be read" is a lock or a denying ACL as often as it is a torn file. Deleting the + /// checkpoint on that reading changed nothing about this run - a load that answers null + /// already forces a fresh export - while destroying a position the next run, or another + /// account, could still have resumed from. + /// + [Fact] + public void A_checkpoint_this_run_cannot_read_is_left_for_a_run_that_can() + { + // A file the account cannot open. Windows expresses that through ACLs, not a mode. + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = output, + DataLength = 24, + }.Save(checkpoint); + var before = File.ReadAllBytes(checkpoint); + File.SetUnixFileMode(checkpoint, UnixFileMode.None); + + // The export fails on its first page, so nothing that runs on success can be what + // removes the checkpoint. + Assert.Null(Export(output, checkpoint)); + + var survived = File.Exists(checkpoint); + if (survived) + { + File.SetUnixFileMode(checkpoint, UnixFileMode.UserRead | UnixFileMode.UserWrite); + Assert.Equal(before, File.ReadAllBytes(checkpoint)); + } + Assert.True(survived, "a position this run could not read was deleted anyway"); + } + finally + { + try { File.SetUnixFileMode(checkpoint, UnixFileMode.UserRead | UnixFileMode.UserWrite); } + catch { } + try { Directory.Delete(dir, true); } catch { } + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ExportDegradedQueryCheckpointTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ExportDegradedQueryCheckpointTests.cs new file mode 100644 index 0000000..2980d9f --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/ExportDegradedQueryCheckpointTests.cs @@ -0,0 +1,901 @@ +using System.Management.Automation; +using System.Net; +using System.Text; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// An endpoint that refuses $top, or the auto-added $count, sends the export round its attempt +/// loop again without that option - and the checkpoint it saves afterwards records the URL it +/// actually fetched. The recovery that runs before the loop compared only the URL the first +/// attempt builds, so an export's own checkpoint read as another export's: nothing was +/// recovered, the sweep deleted the temp holding its items, and the retry - which does build +/// that URL - appended the rest of the enumeration onto whatever file was at -OutputFile. +/// +[Collection("Pipeline")] +public class ExportDegradedQueryCheckpointTests +{ + private const string Page1 = """ + {"value":[{"id":"u1"},{"id":"u2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2"} + """; + private const string Page2 = """ + {"value":[{"id":"u3"}]} + """; + private const string TopUnsupported = """ + {"error":{"code":"Request_UnsupportedQuery","message":"$top is not supported on this endpoint"}} + """; + private const string CountUnsupported = """ + {"error":{"code":"BadRequest","message":"$count is not supported on this endpoint"}} + """; + private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + // A bare 400 off the resumed nextLink, with nothing to do with $count - which is all + // IsCountRejection needs to drop the auto-added one and send the loop round again. + private const string ExpiredSkipToken = """{"error":{"code":"BadRequest","message":"skip token expired"}}"""; + + private static readonly string[] OldExport = + ["{\"id\":\"old1\"}", "{\"id\":\"old2\"}", "{\"id\":\"old3\"}"]; + + /// + /// Answers by URL rather than in order: the option this endpoint refuses is refused every + /// time it appears, and page two fails until the second run asks for it. + /// + private sealed class RefusingHandler : HttpMessageHandler + { + private readonly object _lock = new(); + private readonly string _refused; + private readonly string _refusal; + + public RefusingHandler(string refused, string refusal) + { + _refused = refused; + _refusal = refusal; + } + + public List Requests { get; } = []; + public bool PageTwoFails { get; set; } = true; + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + HttpStatusCode status; + string body; + lock (_lock) + { + Requests.Add(url); + if (url.Contains(_refused, StringComparison.Ordinal)) + (status, body) = (HttpStatusCode.BadRequest, _refusal); + else if (!url.Contains("skiptoken=P2", StringComparison.Ordinal)) + (status, body) = (HttpStatusCode.OK, Page1); + else if (PageTwoFails) + (status, body) = (HttpStatusCode.InternalServerError, ServerError); + else + (status, body) = (HttpStatusCode.OK, Page2); + } + return Task.FromResult(new HttpResponseMessage(status) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } + + + /// + /// Refuses the option only once the first page has been collected: page one answers with + /// $top on it, and the nextLink that follows comes back Request_UnsupportedQuery. That is + /// what sends the attempt loop round again with a page-boundary checkpoint already on disk. + /// + private sealed class RefusingOnPageTwoHandler : HttpMessageHandler + { + private readonly object _lock = new(); + + public List Requests { get; } = []; + public bool PageTwoAnswers { get; set; } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + HttpStatusCode status; + string body; + lock (_lock) + { + Requests.Add(url); + if (url.Contains("skiptoken=P2", StringComparison.Ordinal)) + (status, body) = PageTwoAnswers + ? (HttpStatusCode.OK, Page2) + : (HttpStatusCode.BadRequest, TopUnsupported); + else if (url.Contains("$top=", StringComparison.Ordinal)) + (status, body) = (HttpStatusCode.OK, Page1); + else + // The retry, without the option the endpoint refused. It dies before its + // own first page boundary, so the checkpoint on disk is still the one the + // attempt before it saved. + (status, body) = (HttpStatusCode.InternalServerError, ServerError); + } + return Task.FromResult(new HttpResponseMessage(status) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } + + private static long? Export(string outputPath, string checkpointPath, string? filter = null) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + var cmd = ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("All"); + if (filter != null) cmd.AddParameter("Filter", filter); + try + { + foreach (var r in ps.Invoke()) + if (r?.BaseObject is Mgx.Cmdlets.Models.MgxExportResult summary) + return summary.ItemCount; + } + catch (CmdletInvocationException) { } + return null; + } + + /// + /// Answers the resumed nextLink with one bare 400 before it will serve page two. Page one + /// is served whatever form the URL is in, so the run that follows the refusal enumerates + /// the collection from the beginning. + /// + private sealed class RefusingTheResumedLinkHandler : HttpMessageHandler + { + private readonly object _lock = new(); + + public List Requests { get; } = []; + public bool PageTwoFails { get; set; } = true; + public bool RefuseTheResumedLinkOnce { get; set; } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + HttpStatusCode status; + string body; + lock (_lock) + { + Requests.Add(url); + if (!url.Contains("skiptoken=P2", StringComparison.Ordinal)) + (status, body) = (HttpStatusCode.OK, Page1); + else if (PageTwoFails) + (status, body) = (HttpStatusCode.InternalServerError, ServerError); + else if (RefuseTheResumedLinkOnce) + { + RefuseTheResumedLinkOnce = false; + (status, body) = (HttpStatusCode.BadRequest, ExpiredSkipToken); + } + else + (status, body) = (HttpStatusCode.OK, Page2); + } + return Task.FromResult(new HttpResponseMessage(status) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } + + private static (long? Count, List Warnings) ExportReporting( + string outputPath, string checkpointPath, string? filter) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + var cmd = ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("All"); + if (filter != null) cmd.AddParameter("Filter", filter); + long? count = null; + try + { + foreach (var r in ps.Invoke()) + if (r?.BaseObject is Mgx.Cmdlets.Models.MgxExportResult summary) + count = summary.ItemCount; + } + catch (CmdletInvocationException) { } + return (count, [.. ps.Streams.Warning.Select(w => w.Message)]); + } + + private static string NewDir() => + Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-export-degraded-{Guid.NewGuid():N}")).FullName; + + // Page one with no nextLink at all: an endpoint that pages the request carrying the option + // and answers the one without it in a single page. + private const string WholeCollection = """ + {"value":[{"id":"u1"},{"id":"u2"}]} + """; + + /// + /// Serves page one whatever form the URL is in, and hands back a nextLink carrying the same + /// options - so the refusal can land on the CONTINUATION, which is the shape that leaves a + /// page-boundary checkpoint and the temp it names on disk before the attempt loop goes + /// round again. The refusal on $count says nothing about $count: a bare 400 is all a run + /// that auto-added it asks for, and an expired skiptoken is the same 400. + /// + private sealed class RefusingTheContinuationHandler : HttpMessageHandler + { + private readonly object _lock = new(); + + public List Requests { get; } = []; + public bool RefuseCount { get; set; } = true; + public bool RefuseTop { get; set; } + /// A continuation with nothing left to refuse dies instead of answering. + public bool ContinuationFails { get; set; } + /// The retry's page one is the whole collection, so it saves no checkpoint. + public bool RetryTakesOnePage { get; set; } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + var hasCount = url.Contains("$count=true", StringComparison.Ordinal); + var hasTop = url.Contains("$top=", StringComparison.Ordinal); + HttpStatusCode status; + string body; + lock (_lock) + { + Requests.Add(url); + if (url.Contains("skiptoken=P2", StringComparison.Ordinal)) + { + if (RefuseCount && hasCount) + (status, body) = (HttpStatusCode.BadRequest, CountUnsupported); + else if (RefuseTop && hasTop) + (status, body) = (HttpStatusCode.BadRequest, TopUnsupported); + else if (ContinuationFails) + (status, body) = (HttpStatusCode.InternalServerError, ServerError); + else + (status, body) = (HttpStatusCode.OK, Page2); + } + else if (RetryTakesOnePage && !hasCount) + (status, body) = (HttpStatusCode.OK, WholeCollection); + else + { + var link = "https://graph.microsoft.com/v1.0/users?" + + (hasCount ? "$count=true&" : "") + + (hasTop ? "$top=100&" : "") + + "$skiptoken=P2"; + body = "{\"value\":[{\"id\":\"u1\"},{\"id\":\"u2\"}],\"@odata.nextLink\":\"" + link + "\"}"; + status = HttpStatusCode.OK; + } + } + return Task.FromResult(new HttpResponseMessage(status) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } + + /// + /// The endpoint refuses $top, and the -OutputFile already holds a completed export. The + /// interrupted run's items are in a temp; the run that follows has to recover them and + /// resume, and what it must never do is treat the earlier export's rows as its own first + /// pages. + /// + [Fact] + public void An_export_resumes_its_own_checkpoint_after_the_endpoint_refused_top() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new RefusingHandler("$top=", TopUnsupported); + using var transport = MgxTransportScope.Inject(handler); + + // A completed export is already sitting at -OutputFile. + File.WriteAllLines(output, OldExport); + + // Run one: refused $top, retried without it, collected page one into its temp and + // died on page two. + Export(output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.DoesNotContain("$top=", cp!.Resource); + Assert.NotNull(cp.TempFile); + Assert.Equal(OldExport, File.ReadAllLines(output)); + + // Run two: the same export again, with page two answering this time. + handler.PageTwoFails = false; + var reported = Export(output, checkpoint); + + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(3, reported); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same defect on the other option, in the shape where it costs the resume rather than + /// the file: the endpoint refuses the auto-added $count, and the run that died left no + /// output - which is what the accommodation this replaces needed to see before it looked at + /// the checkpoint's URL at all. + /// + [Fact] + public void An_export_resumes_its_own_checkpoint_after_the_endpoint_refused_count() + { + const string filter = "startsWith(displayName,'a')"; + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new RefusingHandler("$count=true", CountUnsupported); + using var transport = MgxTransportScope.Inject(handler); + + Export(output, checkpoint, filter); + Assert.False(File.Exists(output), "a fresh run promotes nothing when it dies"); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.DoesNotContain("$count=true", cp!.Resource); + + handler.PageTwoFails = false; + var first = handler.Requests.Count; + var reported = Export(output, checkpoint, filter); + + // Resumed: the first thing the second run asks for is the page the first never got. + Assert.Contains("skiptoken=P2", handler.Requests[first]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(3, reported); + Assert.False(File.Exists(temp), "the temp its items came from was left behind"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The retry's own sweep, against a checkpoint that is this run's. Dropping $count or $top + /// sends the attempt loop round again, and the attempt that died kept its temp precisely + /// because the checkpoint counting those items survived it. The sweep at the top of the + /// next attempt then deleted the file that checkpoint names, so the next invocation found a + /// position pointing at nothing and enumerated from the first page - unbounded work thrown + /// away, since a checkpoint is saved at every page boundary. + /// + [Fact] + public void A_retry_does_not_sweep_the_temp_this_runs_own_checkpoint_names() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new RefusingOnPageTwoHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // Page one lands in a temp and is checkpointed; the nextLink is refused, the loop + // retries without $top, and that attempt dies before a boundary of its own. + Assert.Null(Export(output, checkpoint)); + + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.NotNull(cp!.TempFile); + Assert.True(File.Exists(Path.Combine(dir, cp.TempFile!)), + "the retry's sweep deleted the temp this run's own checkpoint names"); + + // The next invocation resumes where the first left off rather than starting over. + handler.PageTwoAnswers = true; + var first = handler.Requests.Count; + var reported = Export(output, checkpoint); + + Assert.Contains("skiptoken=P2", handler.Requests[first]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(3, reported); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other end of the same rebuild. The run recovers the interrupted export's temp, + /// reports that it is resuming, and then the resumed nextLink comes back 400 - any 400 on a + /// run that auto-added $count is read as the endpoint refusing it, so the loop goes round + /// with the URL rebuilt without it. The checkpoint was written under the previous form and + /// is no longer recognized, and the resume just announced is withdrawn. Every sibling + /// refusal on this path says so; this one restarted the enumeration with nothing on any + /// stream but the recovery message, so the run looked from the outside like one that had + /// resumed. + /// + [Fact] + public void An_export_says_so_when_a_rebuilt_request_costs_it_the_resume_it_announced() + { + const string filter = "id ne null"; + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new RefusingTheResumedLinkHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // Run one: page one collected under the auto-added $count, page two dies. + Export(output, checkpoint, filter); + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.Contains("$count=true", cp!.Resource); + Assert.False(File.Exists(output), "a fresh run promotes nothing when it dies"); + + // Run two: the temp is recovered and the resume announced, then the nextLink is + // refused and the URL is rebuilt into a form the checkpoint was not written under. + handler.PageTwoFails = false; + handler.RefuseTheResumedLinkOnce = true; + var (reported, warnings) = ExportReporting(output, checkpoint, filter); + + Assert.Contains(warnings, w => w.Contains("Resuming from checkpoint")); + Assert.Contains(warnings, + w => w.Contains("under a form of the request the endpoint has since refused")); + + // And the run really did start over, which is what the warning is about. + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(3, reported); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// What a retry leaves beside the output once it succeeds. The attempt that died kept its + /// temp because a checkpoint counting those items survived it; the attempt that followed + /// saved a checkpoint of its own over that one, naming its own temp, and from that moment + /// nothing on disk referred to the first. It was spared anyway - the sweep is held off for + /// as long as this run remembers keeping it - so a reported-successful export left a + /// partial copy of the caller's data sitting beside the finished file, for some later + /// export's sweep to reclaim. + /// + [Fact] + public void A_successful_retry_leaves_no_temp_from_the_attempt_it_abandoned() + { + const string filter = "startsWith(displayName,'a')"; + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new RefusingTheContinuationHandler(); + using var transport = MgxTransportScope.Inject(handler); + + var reported = Export(output, checkpoint, filter); + + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(3, reported); + Assert.False(File.Exists(checkpoint)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same, chained: the endpoint refuses the auto-added $count on the continuation and + /// then $top on the next one, so two attempts are abandoned with a temp each before the + /// third finishes. + /// + [Fact] + public void A_second_retry_leaves_no_temp_from_either_attempt_before_it() + { + const string filter = "startsWith(displayName,'a')"; + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new RefusingTheContinuationHandler { RefuseTop = true }; + using var transport = MgxTransportScope.Inject(handler); + + var reported = Export(output, checkpoint, filter); + + // Page one and a refused continuation, twice, and then the pair that answered. + Assert.Equal(6, handler.Requests.Count); + Assert.DoesNotContain("$count=true", handler.Requests[5]); + Assert.DoesNotContain("$top=", handler.Requests[5]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(3, reported); + Assert.False(File.Exists(checkpoint)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And it goes at the moment it stops being anything, not at the end of the run. The retry + /// saves its first page-boundary checkpoint - which names its own temp, so the one before + /// it is now counted by nothing - and then dies, promoting nothing. What is left is the one + /// temp the checkpoint on disk names, and the run after it resumes from exactly that, so + /// what went was never something a resume needed. + /// + [Fact] + public void A_retry_deletes_the_temp_its_own_checkpoint_has_stopped_naming() + { + const string filter = "startsWith(displayName,'a')"; + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new RefusingTheContinuationHandler { ContinuationFails = true }; + using var transport = MgxTransportScope.Inject(handler); + + Assert.Null(Export(output, checkpoint, filter)); + + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.NotNull(cp!.TempFile); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(cp.TempFile, Path.GetFileName(temp)); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"], File.ReadAllLines(temp)); + + handler.ContinuationFails = false; + var first = handler.Requests.Count; + var reported = Export(output, checkpoint, filter); + + Assert.Contains("skiptoken=P2", handler.Requests[first]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(3, reported); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The retry that never saves a checkpoint at all: the endpoint pages the request carrying + /// the option it refuses and answers the one without it in a single page, so the attempt + /// that finishes reaches no page boundary and writes over nothing. The temp the attempt + /// before it left is released by the run finishing instead. + /// + [Fact] + public void A_retry_that_saves_no_checkpoint_of_its_own_leaves_no_temp_behind() + { + const string filter = "startsWith(displayName,'a')"; + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new RefusingTheContinuationHandler { RetryTakesOnePage = true }; + using var transport = MgxTransportScope.Inject(handler); + + var reported = Export(output, checkpoint, filter); + + // Page one, the continuation it refused, and the retry's single page: nothing the + // attempt that finished saved can have been what tidied up after the one before it. + Assert.Equal(3, handler.Requests.Count); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"], File.ReadAllLines(output)); + Assert.Equal(2, reported); + Assert.False(File.Exists(checkpoint)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other end of it: a retry that fetched every page and then could not move its temp + /// over the output. Nothing is promoted and nothing is finished, so the checkpoint and the + /// temp it names have to be left exactly as an interruption leaves them - the abandoned + /// attempt's temp is the only thing that goes - and the next run resumes from them. + /// + [Fact] + public void A_failed_promotion_after_a_retry_keeps_what_the_next_run_resumes_from() + { + const string filter = "startsWith(displayName,'a')"; + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new RefusingTheContinuationHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // Something at the output path the promotion cannot replace, so every page is + // fetched and written and the move at the end of the retry is what fails. + Directory.CreateDirectory(output); + + Assert.Null(Export(output, checkpoint, filter)); + + Assert.True(File.Exists(checkpoint), + "the failed promotion deleted the position the next run resumes from"); + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.NotNull(cp!.TempFile); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(cp.TempFile, Path.GetFileName(temp)); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(temp)); + + // With the occupier gone: the temp is promoted, the page the checkpoint recorded is + // the one page fetched, and the collection is not enumerated again from the start. + Directory.Delete(output); + var first = handler.Requests.Count; + var reported = Export(output, checkpoint, filter); + + Assert.Equal(1, handler.Requests.Count - first); + Assert.Contains("skiptoken=P2", handler.Requests[first]); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(3, reported); + Assert.False(File.Exists(checkpoint)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The scenario above, carried through to the invocation that finishes. The retry that dies + /// before a boundary of its own has a temp too, and the checkpoint on disk - the attempt + /// before it saved that - counts nothing in it: keeping it left a file no recovery can + /// reach beside the one that is resumed from, and the invocation that did resume appended + /// to the output, so no sweep ever came for it. A finished export leaves its output. + /// + [Fact] + public void A_retry_that_dies_before_a_boundary_leaves_only_the_temp_its_checkpoint_names() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new RefusingOnPageTwoHandler(); + using var transport = MgxTransportScope.Inject(handler); + + Assert.Null(Export(output, checkpoint)); + + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.NotNull(cp!.TempFile); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(cp.TempFile, Path.GetFileName(temp)); + + handler.PageTwoAnswers = true; + var reported = Export(output, checkpoint); + + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(3, reported); + Assert.False(File.Exists(checkpoint)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + // Ids of unequal width, so a byte count taken from one enumeration and applied to the file + // another left lands inside a line rather than on a boundary. + // + // The collection as the first run below reads it, paged two items and then one. An export + // is a snapshot of a collection that goes on changing, and the runs here are minutes apart: + // what the skip token names is a position in THIS enumeration, and the byte count beside it + // describes the bytes THIS enumeration wrote. + private const string GrowingPage1 = """ + {"value":[{"id":"u1"},{"id":"uu22"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2"} + """; + private const string GrowingPage2 = """ + {"value":[{"id":"uuu333"}]} + """; + + // The same collection with one item created at the front, which is what the runs after the + // first enumerate - paged two and two, under a skip token of their own. + private const string GrownPage1 = """ + {"value":[{"id":"a0"},{"id":"u1"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=Q2"} + """; + private const string GrownPage2 = """ + {"value":[{"id":"uu22"},{"id":"uuu333"}]} + """; + + // And the whole of it in a single page: how the endpoint answers the form of the request it + // does not refuse, so the attempt that finishes reaches no page boundary and saves nothing. + private const string GrownWholeCollection = """ + {"value":[{"id":"a0"},{"id":"u1"},{"id":"uu22"},{"id":"uuu333"}]} + """; + + private static readonly string[] GrownCollection = + ["{\"id\":\"a0\"}", "{\"id\":\"u1\"}", "{\"id\":\"uu22\"}", "{\"id\":\"uuu333\"}"]; + + private static HttpResponseMessage Json(HttpStatusCode status, string body) => + new(status) { Content = new StringContent(body, Encoding.UTF8, "application/json") }; + + /// What the resumed-link entry arms is called, + /// so a run can assert by name that does not + /// carry it - it is the one entry that ever answers the refusal these tests inject. + private const string FaultName = "fault-resumed-link"; + + /// + /// An endpoint that pages the form of the request carrying + /// and answers the form without it whole, over a collection that grows once the first run + /// is done with it. The refusal lands on the continuation, which is the shape that leaves a + /// page-boundary checkpoint on disk before the attempt loop goes round again; + /// says which of the three invocations is talking to it. + /// + private static void ProgramGrowingEndpoint( + MockHttpHandler handler, string refusedOption, string refusal, Func run) + { + handler + // The position the first run's enumeration handed out. It dies on it; the run that + // resumes is refused the option its URL carries; the run after that is answered + // from the snapshot the first run was reading. + .When(r => r.Uri.Contains("$skiptoken=P2", StringComparison.Ordinal), FaultName) + .Respond(_ => run() switch + { + 1 => Json(HttpStatusCode.InternalServerError, ServerError), + 2 => Json(HttpStatusCode.BadRequest, refusal), + _ => Json(HttpStatusCode.OK, GrowingPage2) + }) + // The position the enumeration over the grown collection hands out. + .When(r => r.Uri.Contains("$skiptoken=Q2", StringComparison.Ordinal)) + .Respond(_ => Json(HttpStatusCode.OK, GrownPage2)) + // Page one of the form the endpoint pages, against the collection as it stands. + .When(r => r.Uri.Contains(refusedOption, StringComparison.Ordinal)) + .Respond(_ => Json(HttpStatusCode.OK, run() == 1 ? GrowingPage1 : GrownPage1)) + // And the form it does not refuse. + .When(_ => true) + .Respond(_ => Json(HttpStatusCode.OK, GrownWholeCollection)); + } + + /// + /// What a rebuilt request leaves at -CheckpointPath, and what the run after it does with + /// that. The endpoint refuses $top on the continuation, so the attempt loop goes round with + /// a position of this run's already on disk, and the attempt that follows compares it + /// against a URL the run has stopped building and refuses it. That refusal says nothing + /// about whose the file is: it is this export's own position, into an enumeration this run + /// goes on to make again and replace the output with. Kept beside the finished output, it + /// is a position nothing comes back to, and the next run over the same command line reads + /// it as its own - the completed file cut back to the byte count of an enumeration two runs + /// old, and the rest appended from a skip token that counts items it no longer holds. + /// + [Fact] + public void A_checkpoint_refused_for_a_rebuilt_request_goes_with_the_completed_export() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var run = 1; + var handler = new MockHttpHandler(); + ProgramGrowingEndpoint(handler, "$top=", TopUnsupported, () => run); + using var transport = MgxTransportScope.Inject(handler); + + // Run one: page one collected into a temp under the automatic $top, and the + // continuation dies. The checkpoint records that form of the URL and names the temp. + var firstResult = Export(output, checkpoint); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + Assert.Null(firstResult); + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.Contains("$top=", cp!.Resource); + Assert.NotNull(cp.TempFile); + + // Run two: the temp is promoted and the resume announced, then the continuation + // comes back Request_UnsupportedQuery, the URL is rebuilt without $top, and the + // form the endpoint answers hands over the whole collection in one page - so the + // attempt that finishes never reaches a boundary and saves no checkpoint of its own. + run = 2; + var (secondCount, secondWarnings) = ExportReporting(output, checkpoint, null); + + Assert.Contains(secondWarnings, w => w.Contains("Resuming from checkpoint")); + Assert.Equal(GrownCollection, File.ReadAllLines(output)); + Assert.Equal(4, secondCount); + + // Taken here and weighed below, because what the file costs is what run three does. + var checkpointSurvivedRunTwo = File.Exists(checkpoint); + + // Run three, the same command line, with $top answered again. + run = 3; + var (thirdCount, thirdWarnings) = ExportReporting(output, checkpoint, null); + + Assert.Equal(GrownCollection, File.ReadAllLines(output)); + Assert.Equal(4, thirdCount); + Assert.DoesNotContain(thirdWarnings, + w => w.Contains("resum", StringComparison.OrdinalIgnoreCase)); + Assert.False(checkpointSurvivedRunTwo, + "a completed export left a position into the enumeration it had just replaced"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same on the other option: a -Filter run auto-adds $count, the continuation comes + /// back a bare 400, and the URL is rebuilt without it. The checkpoint the attempt before + /// saved is this export's own under the form that carried $count, and the run that refuses + /// it goes on to enumerate the collection again and replace the output. + /// + [Fact] + public void A_checkpoint_refused_for_a_dropped_count_goes_with_the_completed_export() + { + const string filter = "startsWith(displayName,'a')"; + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var run = 1; + var handler = new MockHttpHandler(); + ProgramGrowingEndpoint(handler, "$count=true", CountUnsupported, () => run); + using var transport = MgxTransportScope.Inject(handler); + + var firstResult = Export(output, checkpoint, filter); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + Assert.Null(firstResult); + var cp = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(cp); + Assert.Contains("$count=true", cp!.Resource); + Assert.NotNull(cp.TempFile); + + run = 2; + var (secondCount, secondWarnings) = ExportReporting(output, checkpoint, filter); + + Assert.Contains(secondWarnings, w => w.Contains("Resuming from checkpoint")); + Assert.Equal(GrownCollection, File.ReadAllLines(output)); + Assert.Equal(4, secondCount); + + var checkpointSurvivedRunTwo = File.Exists(checkpoint); + + run = 3; + var (thirdCount, thirdWarnings) = ExportReporting(output, checkpoint, filter); + + Assert.Equal(GrownCollection, File.ReadAllLines(output)); + Assert.Equal(4, thirdCount); + Assert.DoesNotContain(thirdWarnings, + w => w.Contains("resum", StringComparison.OrdinalIgnoreCase)); + Assert.False(checkpointSurvivedRunTwo, + "a completed export left a position into the enumeration it had just replaced"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ExportMgxCollectionRecoveryTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ExportMgxCollectionRecoveryTests.cs index 7c6c3ba..74f1a9d 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/ExportMgxCollectionRecoveryTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/ExportMgxCollectionRecoveryTests.cs @@ -1,4 +1,4 @@ -using System.Net; +using System.Net; using System.Text; using System.Text.Json; using Mgx.Engine.Pagination; @@ -229,31 +229,6 @@ public void A_checkpoint_whose_output_never_arrived_is_discarded() Assert.False(File.Exists(cpPath)); } - [Fact] - public void A_checkpoint_naming_a_temp_that_is_gone_starts_over() - { - var path = InWorkDir("orphan.jsonl"); - var cpPath = InWorkDir("orphan.checkpoint"); - new PaginationCheckpoint - { - Resource = "https://graph.microsoft.com/v1.0/users", - NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=p2", - ItemsCollected = 5, - TempFile = InWorkDir("orphan.jsonl.deadbeef.tmp"), - DataLength = 40 - }.Save(cpPath); - using var host = new MgxTestHost(Json(Page("u1"))); - - var result = host.Run(ps => ps.AddCommand("Export-MgxCollection") - .AddParameter("Uri", "/users") - .AddParameter("OutputFile", path) - .AddParameter("CheckpointPath", cpPath) - .AddParameter("All")); - - Assert.Contains(result.Warnings, w => w.Contains("temp file is missing or incomplete")); - Assert.Equal(1, ItemCount(result)); - } - [Fact] public void A_checkpoint_naming_a_temp_that_survived_promotes_it_and_resumes() { diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ExportSharedCheckpointTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ExportSharedCheckpointTests.cs new file mode 100644 index 0000000..76df0b2 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/ExportSharedCheckpointTests.cs @@ -0,0 +1,3030 @@ +using System.Management.Automation; +using System.Net; +using System.Text; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// A checkpoint's recorded length is a byte offset into "the output", and the recovery path +/// applies it to whatever -OutputFile the run names. Nothing in a checkpoint used to say which +/// file those bytes were counted in, so one -CheckpointPath reused by two exports cut a file +/// the checkpoint knew nothing about - mid-line, with the resumed pages appended onto the torn +/// byte, and no warning on any stream. +/// +[Collection("Pipeline")] +public class ExportSharedCheckpointTests +{ + private const string UsersPage1 = """ + {"value":[{"id":"u1"},{"id":"u2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2"} + """; + private const string UsersPage2 = """ + {"value":[{"id":"u3"}]} + """; + // The same page with a continuation behind it, so a resumed run writes and flushes a page + // before the request after it can be held open. + private const string UsersPage2Continued = """ + {"value":[{"id":"u3"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P3"} + """; + private const string UsersPage3 = """ + {"value":[{"id":"u4"}]} + """; + private const string GroupsPage1 = """ + {"value":[{"id":"g1"},{"id":"g2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/groups?$skiptoken=P2"} + """; + private const string GroupsPage2 = """ + {"value":[{"id":"g3"}]} + """; + private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + private const string TopUnsupported = """ + {"error":{"code":"Request_UnsupportedQuery","message":"$top is not supported on this endpoint"}} + """; + + private static readonly string[] OldExport = + ["{\"id\":\"old-0000001\"}", "{\"id\":\"old-0000002\"}", "{\"id\":\"old-0000003\"}"]; + + private sealed class ScriptedHandler : HttpMessageHandler + { + private readonly Queue<(HttpStatusCode Status, string Body)> _steps = new(); + private readonly object _lock = new(); + private int _served; + + /// How many requests this handler has answered. A run that stops adds none. + public int Served { get { lock (_lock) return _served; } } + + public void Queue(HttpStatusCode status, string body) + { + lock (_lock) _steps.Enqueue((status, body)); + } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + (HttpStatusCode Status, string Body) step; + lock (_lock) + { + _served++; + step = _steps.Count > 0 ? _steps.Dequeue() : (HttpStatusCode.InternalServerError, ServerError); + } + return Task.FromResult(new HttpResponseMessage(step.Status) + { + RequestMessage = request, + Content = new StringContent(step.Body, Encoding.UTF8, "application/json") + }); + } + } + + /// + /// Refuses the automatic $top on the page after the first: page one answers with the option + /// on it, the nextLink that follows comes back Request_UnsupportedQuery, and the retry + /// without it finds the whole collection in a single page. An attempt loop that goes round + /// with a page-boundary checkpoint already on disk, and finishes without saving one. + /// + private sealed class TopRefusedAfterPageOneHandler : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + var (status, body) = url.Contains("skiptoken=P2", StringComparison.Ordinal) + ? (HttpStatusCode.BadRequest, TopUnsupported) + : url.Contains("$top=", StringComparison.Ordinal) + ? (HttpStatusCode.OK, UsersPage1) + : (HttpStatusCode.OK, UsersPage2); + return Task.FromResult(new HttpResponseMessage(status) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } + + /// + /// A scripted handler that holds one request open until it is let go: the run parked there + /// has written and flushed everything the pages before it carried, and is appending still. + /// + private sealed class GatedHandler : HttpMessageHandler + { + private readonly Queue<(HttpStatusCode Status, string Body)> _steps = new(); + private readonly int _gateOn; + private readonly object _lock = new(); + private int _served; + + /// Which request, counting from one, is held at the gate. + internal GatedHandler(int gateOn) => _gateOn = gateOn; + + internal void Queue(HttpStatusCode status, string body) + { + lock (_lock) _steps.Enqueue((status, body)); + } + + /// Set the moment the request being gated arrives. + internal ManualResetEventSlim Arrived { get; } = new(false); + + /// Set by the test to let the parked request answer. + internal ManualResetEventSlim Released { get; } = new(false); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + int n; + (HttpStatusCode Status, string Body) step; + lock (_lock) + { + n = ++_served; + step = _steps.Count > 0 ? _steps.Dequeue() : (HttpStatusCode.OK, UsersPage2); + } + if (n == _gateOn) + { + Arrived.Set(); + Released.Wait(cancellationToken); + } + return Task.FromResult(new HttpResponseMessage(step.Status) + { + RequestMessage = request, + Content = new StringContent(step.Body, Encoding.UTF8, "application/json") + }); + } + } + + /// + /// Runs an export on this thread and parks it inside the window the hold exists to close: + /// past the reconcile, which has claimed the output and cut it back, and short of the writer + /// that used to open a handle of its own a page fetch later. The verbose the resume announces + /// itself with is written from exactly there, and a subscriber to that stream blocks the + /// pipeline where it is raised. + /// + private static (string[] Warnings, ErrorRecord[] Errors) ExportParkedAtResume( + string uri, string outputPath, string checkpointPath, + ManualResetEventSlim arrived, ManualResetEventSlim released) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + ps.Streams.Verbose.DataAdding += (_, e) => + { + if (e.ItemAdded is VerboseRecord v + && v.Message.Contains("Resuming from checkpoint", StringComparison.Ordinal) + && !arrived.IsSet) + { + arrived.Set(); + released.Wait(); + } + }; + + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", uri) + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("All") + .AddParameter("Verbose"); + List errors = []; + try { ps.Invoke(); } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + return ([.. ps.Streams.Warning.Select(w => w.Message)], [.. errors]); + } + + /// + /// An export built and waiting to be let go: the runspace is up and the module imported + /// before is set, so what releases is two runs + /// arriving at the reconcile together rather than two runspaces starting up. + /// + private static (string[] Warnings, ErrorRecord[] Errors) ExportReleasedWith( + string uri, string outputPath, string checkpointPath, + ManualResetEventSlim ready, ManualResetEventSlim go) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", uri) + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("All"); + + ready.Set(); + go.Wait(); + List errors = []; + try { ps.Invoke(); } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + return ([.. ps.Streams.Warning.Select(w => w.Message)], [.. errors]); + } + + /// + /// Answers by URL rather than from a queue, and holds the first request that arrives until it + /// is let go. Two runs racing over one command line each get the page they ask for however + /// they interleave, and the one that reaches the wire - which is the one that got past the + /// reconcile - keeps the output for as long as the test needs it to. + /// + private sealed class GatedUsersByUrlHandler : HttpMessageHandler + { + private int _served; + + /// How many requests this handler has answered or is holding. + internal int Served => Volatile.Read(ref _served); + + /// Set the moment the first request arrives. + internal ManualResetEventSlim Arrived { get; } = new(false); + + /// Set by the test to let the parked request answer. + internal ManualResetEventSlim Released { get; } = new(false); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + if (Interlocked.Increment(ref _served) == 1) + { + Arrived.Set(); + Released.Wait(cancellationToken); + } + var body = request.RequestUri!.ToString() + .Contains("skiptoken=P2", StringComparison.Ordinal) ? UsersPage2 : UsersPage1; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }); + } + } + + private static long? Export(string uri, string outputPath, string checkpointPath) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", uri) + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("All"); + try + { + foreach (var r in ps.Invoke()) + if (r?.BaseObject is Mgx.Cmdlets.Models.MgxExportResult summary) + return summary.ItemCount; + } + catch (CmdletInvocationException) { } + return null; + } + + private static void ExportNoCheckpoint(string uri, string outputPath) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", uri) + .AddParameter("OutputFile", outputPath) + .AddParameter("All"); + ps.Invoke(); + } + + private static string[] ExportWarnings(string uri, string outputPath, string checkpointPath) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", uri) + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("All"); + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + return [.. ps.Streams.Warning.Select(w => w.Message)]; + } + + /// + /// The same export with the error stream captured too. A run that will not touch the two + /// files the checkpoint stands for ends on a terminating error, and the id and the sentence + /// are the whole of what the caller has to act on. Both places PowerShell can put that + /// record are read, since which one it uses depends on how the command was added. + /// + private static (string[] Warnings, ErrorRecord[] Errors) ExportRun(string uri, + string outputPath, string checkpointPath) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", uri) + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("All"); + List errors = []; + try { ps.Invoke(); } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + return ([.. ps.Streams.Warning.Select(w => w.Message)], [.. errors]); + } + + /// + /// Everything a run that stops may not have moved: the checkpoint, the temp it names, the + /// output if there is one, and the directory's own entry list - which is what says no temp + /// of the stopped run's outlived it. + /// + private sealed class DiskState + { + public byte[] Checkpoint = []; + public byte[] Temp = []; + public byte[]? Output; + public string[] Entries = []; + } + + private static DiskState ReadDisk(string dir, string checkpointPath, string temp, + string outputPath) => new() + { + Checkpoint = File.ReadAllBytes(checkpointPath), + Temp = ReadShared(temp), + Output = File.Exists(outputPath) ? ReadShared(outputPath) : null, + Entries = [.. Directory.GetFiles(dir).Select(f => Path.GetFileName(f)) + .OrderBy(n => n, StringComparer.Ordinal)], + }; + + /// + /// The same, for the shape a resumed export leaves: its items are in the output itself and + /// the checkpoint names no temp, so there is no third file to read. + /// + private static DiskState ReadDisk(string dir, string checkpointPath, string outputPath) => new() + { + Checkpoint = File.ReadAllBytes(checkpointPath), + Output = ReadShared(outputPath), + Entries = [.. Directory.GetFiles(dir).Select(f => Path.GetFileName(f)) + .OrderBy(n => n, StringComparer.Ordinal)], + }; + + private static void AssertUnmoved(DiskState before, DiskState after) + { + Assert.Equal(before.Entries, after.Entries); + Assert.Equal(before.Checkpoint, after.Checkpoint); + Assert.Equal(before.Temp, after.Temp); + Assert.Equal(before.Output, after.Output); + } + + /// + /// Reads a file a live run still holds open, offering the sharing that holder asked for. + /// Windows checks the sharing both ways round - what the reader asks of the file, and what + /// it will let a handle already on it go on doing - so File.ReadAllBytes, which asks for + /// FileShare.Read and nothing more, is refused by a run holding the file to write it. + /// + private static byte[] ReadShared(string path) + { + using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + using var ms = new MemoryStream(); + fs.CopyTo(ms); + return ms.ToArray(); + } + + private static string NewDir() => + Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-export-shared-{Guid.NewGuid():N}")).FullName; + + /// + /// A FIFO at , which is one of the things a caller can leave standing + /// where -OutputFile points. .NET has no call for it, so the platform's own is used. + /// + private static void Mkfifo(string path) + { + using var mkfifo = System.Diagnostics.Process.Start("/usr/bin/mkfifo", [path]); + Assert.NotNull(mkfifo); + mkfifo.WaitForExit(); + Assert.Equal(0, mkfifo.ExitCode); + } + + /// + /// What each refusal says. A checkpoint recording another export's output is a second + /// export over one -CheckpointPath, and naming that is what the caller acts on. One + /// recording no output at all is not: it was believed while the files beside it + /// corroborated it, and a single export whose temp has since gone, or whose output has been + /// replaced, reaches the same refusal. Reporting a different export there names a cause the + /// caller can go and check and find nothing behind. + /// + [Fact] + public void A_refusal_reports_the_cause_it_can_show() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + File.WriteAllLines(output, OldExport); + + // Written before outputs were recorded, counting more bytes than this output holds. + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = null, + DataLength = new FileInfo(output).Length + 4096, + }.Save(checkpoint); + + var uncorroborated = ExportWarnings("/users", output, checkpoint); + Assert.Contains(uncorroborated, w => w.Contains("no longer corroborate")); + Assert.DoesNotContain(uncorroborated, w => w.Contains("belongs to a different export")); + + // The same position, recording an output this run is not writing to. + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = Path.Combine(dir, "elsewhere.jsonl"), + DataLength = 24, + }.Save(checkpoint); + + Assert.Contains(ExportWarnings("/users", output, checkpoint), + w => w.Contains("belongs to a different export")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// Two exports of the same resource to different output files, sharing one checkpoint path. + /// The second run's output must not be cut back to the first run's byte count. + /// + [Fact] + public void A_checkpoint_from_another_export_does_not_cut_this_ones_output() + { + var dir = NewDir(); + var outA = Path.Combine(dir, "outA.jsonl"); + var outB = Path.Combine(dir, "outB.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // Export A dies on page 2, twice: the second run promotes the temp, so the + // checkpoint it leaves records a length with no temp to explain it. + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", outA, checkpoint); + Export("/users", outA, checkpoint); + + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + Assert.Equal(new FileInfo(outA).Length, cp.DataLength); + + // Export B: same resource, its own output, which already holds an earlier export. + File.WriteAllLines(outB, OldExport); + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + var reported = Export("/users", outB, checkpoint); + + var lines = File.ReadAllLines(outB); + Assert.All(lines, l => Assert.StartsWith("{\"id\":\"", l)); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], lines); + Assert.Equal(3, reported); + + // A's own output is untouched by B. + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"], File.ReadAllLines(outA)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A checkpoint left by an export of a DIFFERENT resource, against the same output name. + /// The resource is compared before anything is cut, so a run that then fails leaves the + /// output it never replaced exactly as it found it. + /// + [Fact] + public void A_checkpoint_for_another_resource_does_not_destroy_an_output_the_run_never_replaces() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // An interrupted export of /groups, resumed once so its checkpoint names no temp. + handler.Queue(HttpStatusCode.OK, GroupsPage1); + Export("/groups", output, checkpoint); + Export("/groups", output, checkpoint); + Assert.Equal(2, File.ReadAllLines(output).Length); + + // A later export replaces the output without touching that checkpoint. + File.WriteAllLines(output, OldExport); + + // An export of /users to the same file, which fails on its first page. + var reported = Export("/users", output, checkpoint); + + Assert.Null(reported); + Assert.Equal(OldExport, File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A checkpoint that is not this export's is refused, and the refusal used to depend on + /// deleting it - an unchecked delete, so a checkpoint in a directory this account cannot + /// write survived, the loop below decided to append from it merely existing, and the other + /// export's remaining pages were appended onto a file that had never held its first ones. + /// + [Fact] + public void A_refused_checkpoint_that_cannot_be_deleted_appends_nothing_foreign() + { + // The checkpoint sits in a directory the run cannot write, which is what makes its + // deletion fail. Windows expresses that through ACLs, not a mode. + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var vault = Directory.CreateDirectory(Path.Combine(dir, "vault")).FullName; + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(vault, "run.checkpoint"); + try + { + File.WriteAllLines(output, OldExport); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = Path.Combine(dir, "elsewhere.jsonl"), + DataLength = new FileInfo(output).Length + }.Save(checkpoint); + File.SetUnixFileMode(vault, + UnixFileMode.UserRead | UnixFileMode.UserExecute); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + var reported = Export("/users", output, checkpoint); + + var lines = File.ReadAllLines(output); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], lines); + Assert.Equal(3, reported); + Assert.DoesNotContain(lines, l => l.Contains("old-")); + } + finally + { + try + { + File.SetUnixFileMode(vault, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// What the refusal leaves behind. The checkpoint and the staging file beside it are the + /// other export's resume position, and deleting them cost that export its progress while + /// this run went on to recreate the same collision at its first page boundary. + /// + [Fact] + public void A_refused_checkpoint_and_its_staging_file_are_left_where_they_are() + { + var dir = NewDir(); + var outA = Path.Combine(dir, "outA.jsonl"); + var outB = Path.Combine(dir, "outB.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // Export A dies on page 2 and leaves its position behind. + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", outA, checkpoint); + var before = File.ReadAllBytes(checkpoint); + // A save that died between writing the staging file and renaming it. + var staged = checkpoint + ".tmp"; + File.WriteAllBytes(staged, before); + + // Export B, its own output, sharing A's checkpoint path, failing on its first page. + Export("/users", outB, checkpoint); + + Assert.True(File.Exists(checkpoint), "the other export's resume position was deleted"); + Assert.Equal(before, File.ReadAllBytes(checkpoint)); + Assert.True(File.Exists(staged), "the other export's staging file was deleted"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// Two exports of the same resource to files that share a name in different directories, + /// over one checkpoint path. The name is all a leaf comparison sees, so the second export + /// adopted the first one's position and cut its file to the first one's byte count. + /// + [Fact] + public void A_checkpoint_for_the_same_name_in_another_directory_does_not_cut_this_ones_output() + { + var dir = NewDir(); + var outA = Path.Combine(Directory.CreateDirectory(Path.Combine(dir, "a")).FullName, "users.jsonl"); + var outB = Path.Combine(Directory.CreateDirectory(Path.Combine(dir, "b")).FullName, "users.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // Export A dies on page 2, twice: the second run promotes the temp, so the + // checkpoint it leaves records a length with no temp to explain it. + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", outA, checkpoint); + Export("/users", outA, checkpoint); + + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + Assert.Equal(new FileInfo(outA).Length, cp.DataLength); + + // Export B: same resource, same file name, its own directory, and its own earlier + // export already sitting there. + File.WriteAllLines(outB, OldExport); + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + var reported = Export("/users", outB, checkpoint); + + var lines = File.ReadAllLines(outB); + Assert.All(lines, l => Assert.StartsWith("{\"id\":\"", l)); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], lines); + Assert.Equal(3, reported); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"], File.ReadAllLines(outA)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A checkpoint for the same path under different query options - another -Top, another + /// -Filter - counts a different enumeration. Comparing paths and not queries let it through + /// the guard, so the output was cut to its byte count before the exact comparison that + /// refuses ever ran, and a run that then failed left the cut behind. + /// + [Fact] + public void A_checkpoint_whose_query_differs_does_not_cut_the_output_before_it_is_refused() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + File.WriteAllLines(output, OldExport); + var before = File.ReadAllBytes(output); + new PaginationCheckpoint + { + // Same path, half the page: an export of /users this one is not making. + Resource = "https://graph.microsoft.com/v1.0/users?$top=500", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = output, + DataLength = 24, + }.Save(checkpoint); + var checkpointBefore = File.ReadAllBytes(checkpoint); + + // This export fails on its first page, so anything the output loses, it loses here. + var reported = Export("/users", output, checkpoint); + + Assert.Null(reported); + Assert.Equal(before, File.ReadAllBytes(output)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// How far "it is left as it is" has to reach. The refused checkpoint's items are in the + /// temp it names, and the stale-temp sweep a few lines later deleted every temp beside the + /// output - including that one. The export it belongs to then came back to a position + /// pointing at a file that is gone and enumerated the collection again from the first page, + /// which for a long export is the whole cost the refusal was written to avoid. + /// + [Fact] + public void A_refused_checkpoints_temp_survives_the_run_that_refused_it() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // Export A dies on page 2: its two items are in its temp, the output was never + // promoted, and the checkpoint names both. + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", output, checkpoint); + + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.NotNull(cp.TempFile); + var temp = Path.Combine(dir, cp.TempFile); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"], File.ReadAllLines(temp)); + var before = File.ReadAllBytes(checkpoint); + + // Export B: another collection, the same -OutputFile and -CheckpointPath, failing + // before its first page boundary - so nothing it does on a boundary or at + // completion is what answers here. + Assert.Null(Export("/groups", output, checkpoint)); + + Assert.Equal(before, File.ReadAllBytes(checkpoint)); + Assert.True(File.Exists(temp), + "the refused checkpoint's items were swept away by the run that refused it"); + + // A, back where it left off, rather than at page 1. + handler.Queue(HttpStatusCode.OK, UsersPage2); + Assert.Equal(3, Export("/users", output, checkpoint)); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// How far "spare the temps a refused checkpoint names" reaches. The sweep only ever + /// deletes names a run gives its own temp - the output's name, a dot, 32 hex digits, + /// ".tmp" - so a refused checkpoint naming "users.jsonl.{guid}.tmp" beside an output called + /// "users" names a file this sweep could not touch either way. Skipping on it bought that + /// file nothing and cost this output its own orphans, which the pre-length adoption path + /// then picks up on a line count alone. + /// + [Fact] + public void A_refused_temp_this_sweep_could_never_reach_does_not_spare_this_outputs_orphans() + { + var dir = NewDir(); + var mine = Path.Combine(dir, "users"); + var theirs = Path.Combine(dir, "users.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // The export to "users" collects page one into its temp and dies on page two. + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", mine, checkpoint); + var orphan = Assert.Single(Directory.GetFiles(dir, "users.*.tmp")); + + // The export next door takes the shared checkpoint over: it refuses what it finds, + // saves its own position at its first page boundary, and dies on page two. The + // first export's temp is now an orphan - nothing on disk describes it. + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", theirs, checkpoint); + var theirTemp = Assert.Single(Directory.GetFiles(dir, "users.jsonl.*.tmp")); + + // "users" again. It refuses a checkpoint naming a temp beside a different output, + // and that name must not stand in the way of its own sweep. + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + Assert.Equal(3, Export("/users", mine, checkpoint)); + + Assert.False(File.Exists(orphan), + "a temp name the sweep could never reach suppressed this output's sweep"); + Assert.True(File.Exists(theirTemp), + "the refused checkpoint's own staging file was deleted"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + /// + /// What a refusing run leaves behind when it succeeds. The completion path deletes the + /// checkpoint on the way out - the position it describes is spent - and after a refusal the + /// file on disk is not this run's to spend: an export whose whole collection fits in one + /// page never writes one of its own, so the file deleted there was the other export's, the + /// temp the sweep had just been told to spare belonged to nothing, and that export came + /// back to page one. + /// + [Fact] + public void A_refused_checkpoint_survives_a_refusing_export_that_completes() + { + var dir = NewDir(); + var outA = Path.Combine(dir, "outA.jsonl"); + var outB = Path.Combine(dir, "outB.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // Export A dies on page 2: its two items are in its temp, the output was never + // promoted, and the checkpoint names both. + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", outA, checkpoint); + + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.NotNull(cp.TempFile); + var temp = Path.Combine(dir, cp.TempFile); + var before = File.ReadAllBytes(checkpoint); + + // Export B: its own output over A's checkpoint path, and one page of its own + // collection is all there is - so it refuses A's position as another export's, + // saves none of its own, and reaches the completion path. + handler.Queue(HttpStatusCode.OK, GroupsPage2); + var refusal = ExportWarnings("/groups", outB, checkpoint); + Assert.Contains(refusal, w => w.Contains("belongs to a different export")); + Assert.Equal(["{\"id\":\"g3\"}"], File.ReadAllLines(outB)); + + Assert.True(File.Exists(checkpoint), "the other export's resume position was deleted"); + Assert.Equal(before, File.ReadAllBytes(checkpoint)); + Assert.True(File.Exists(temp), + "the refused checkpoint's items were left with nothing describing them"); + + // A, back where it left off, rather than at page 1. + handler.Queue(HttpStatusCode.OK, UsersPage2); + Assert.Equal(3, Export("/users", outA, checkpoint)); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(outA)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other half: a refusal is not a standing exemption from the delete. A run that + /// refuses what it finds and then saves its own position at its first page boundary has + /// taken the path over, and what the completion path deletes is that. Left standing, it is + /// a position into an enumeration that has finished, naming a temp released a line later - + /// which every later export over that path refuses in turn, warning about a second export + /// that is not there. + /// + [Fact] + public void A_refusing_export_that_saved_its_own_checkpoint_deletes_it_on_completion() + { + var dir = NewDir(); + var outA = Path.Combine(dir, "outA.jsonl"); + var outB = Path.Combine(dir, "outB.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", outA, checkpoint); + Assert.True(File.Exists(checkpoint)); + + // Export B: two pages, so it refuses A's position and then writes its own over the + // same path at the boundary between them. + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + Assert.Equal(3, Export("/users", outB, checkpoint)); + + Assert.False(File.Exists(checkpoint), + "a position into a finished enumeration was left on disk"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And over a retry, where the attempt that saved the checkpoint is not the attempt that + /// finishes. An endpoint refusing the automatic $top sends the loop round again with a + /// boundary checkpoint of this run's already on disk, and the retry - everything it asks + /// for arriving in one page - saves none. Read per attempt, "this run never wrote over the + /// file it refused" is true of the retry and false of the run, and the position left behind + /// is the run's own. + /// + [Fact] + public void A_checkpoint_this_run_saved_before_a_retry_is_deleted_when_the_retry_completes() + { + var dir = NewDir(); + var outA = Path.Combine(dir, "outA.jsonl"); + var outB = Path.Combine(dir, "outB.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + // Export A dies on page 2 and leaves its position behind. + var handler = new ScriptedHandler(); + using (MgxTransportScope.Inject(handler)) + { + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", outA, checkpoint); + Assert.True(File.Exists(checkpoint)); + } + + // Export B refuses it, saves its own at its first page boundary, and is sent round + // the attempt loop by the refusal of $top on the page after. + using (MgxTransportScope.Inject(new TopRefusedAfterPageOneHandler())) + { + Assert.Equal(1, Export("/users", outB, checkpoint)); + } + + Assert.False(File.Exists(checkpoint), + "the position this run saved before its retry was left on disk"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And where a refusal is not about ownership at all, and where refusing is not enough. A + /// checkpoint whose named temp another run still holds counts rows that are in that temp and + /// in no other file, and it is the only thing on disk that counts them - so the holder + /// deletes it itself if it finishes, and keeps its temp for exactly as long as this file is + /// there if it does not. Exporting anyway is what takes it: the first page boundary saves + /// this run's own position over the same path, the completion door deletes that as its own, + /// and the holder's temp is left an orphan for the next sweep. There is no pass this run can + /// make that leaves the pair standing, so it makes none. + /// + [Fact] + public void A_held_temp_stops_the_export_before_anything_is_written() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // A run of this export dies on page 2: the two items it collected are in its temp, + // and the checkpoint beside it records this output and this enumeration. + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", output, checkpoint); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + + // It is not dead. It still holds that temp, the way a run between pages does, with + // the counted rows in it - and the rest of the collection is on the wire, one page + // of it, which is all the run below would have needed. + var counted = File.ReadAllText(temp); + using var live = new StreamWriter(temp, append: false); + live.Write(counted); + live.Flush(); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var before = ReadDisk(dir, checkpoint, temp, output); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + var after = ReadDisk(dir, checkpoint, temp, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointTempHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Contains("Another export is still writing the temp file", + stop.Exception.Message); + Assert.Contains(checkpoint, stop.Exception.Message); + Assert.Contains("the 2 items it records are that run's", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Wait for that export to finish, or give this one its own -OutputFile and " + + "-CheckpointPath.", + stop.Exception.Message); + + // Nothing that promises to leave both files and then replaces them. + Assert.DoesNotContain(warnings, w => w.Contains("exported from the beginning")); + Assert.DoesNotContain(warnings, w => w.Contains("Both files are left as they are")); + + // And nothing happened: no request, no output, and both files exactly as the holder + // left them - down to the directory's entry list, so no temp of this run's outlived + // it either. + Assert.Equal(servedBefore, handler.Served); + AssertUnmoved(before, after); + Assert.False(File.Exists(output), "the stopped run published an output"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same disk with two pages on the wire instead of one, which is the shape that made + /// leaving the file alone worth nothing: a run that reaches one page boundary saves its own + /// position over the very path it has just promised to leave, and walks out of a door that + /// deletes that as this run's own. A run that stops has no page boundary to reach. + /// + [Fact] + public void A_held_temp_stops_the_export_that_would_have_paged_twice() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", output, checkpoint); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"], File.ReadAllLines(temp)); + + var counted = File.ReadAllText(temp); + using var live = new StreamWriter(temp, append: false); + live.Write(counted); + live.Flush(); + + // A first page with a continuation, so the run would save a boundary checkpoint of + // its own, and a second that finishes the collection. + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + var before = ReadDisk(dir, checkpoint, temp, output); + var servedBefore = handler.Served; + var (_, errors) = ExportRun("/users", output, checkpoint); + var after = ReadDisk(dir, checkpoint, temp, output); + + Assert.StartsWith("CheckpointTempHeld", Assert.Single(errors).FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(servedBefore, handler.Served); + AssertUnmoved(before, after); + Assert.Equal(2, PaginationCheckpoint.Load(checkpoint)!.ItemsCollected); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The whole cost of getting that wrong, on one disk. Export A holds its temp with two rows + /// in it; export B over the same command line stops; A then dies, leaving the two files B + /// left alone. A's re-run is what recovers those rows - it promotes the temp over the output + /// and appends the rest of the enumeration - and it can only do that while the checkpoint + /// counting them is there. + /// + [Fact] + public void The_rows_a_held_temp_holds_come_back_after_the_export_that_stopped() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // A collects page one into its temp and is still going. + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", output, checkpoint); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"], File.ReadAllLines(temp)); + + var counted = File.ReadAllText(temp); + using (var live = new StreamWriter(temp, append: false)) + { + live.Write(counted); + live.Flush(); + + // B stops. Nothing is queued for it, and it asks for nothing. + var (_, errors) = ExportRun("/users", output, checkpoint); + Assert.StartsWith("CheckpointTempHeld", + Assert.Single(errors).FullyQualifiedErrorId, StringComparison.Ordinal); + } + + // A dies, leaving exactly the two files B left alone. + Assert.True(File.Exists(checkpoint), "B took A's position with it"); + Assert.True(File.Exists(temp), "B took A's temp with it"); + + // A's re-run. It promotes the two rows and resumes from the page its own checkpoint + // recorded, so the request below is the nextLink and not page one - which is what + // tells a recovery from a fresh export here. + handler.Queue(HttpStatusCode.OK, UsersPage2); + var warnings = ExportWarnings("/users", output, checkpoint); + + Assert.Contains(warnings, w => w.Contains("Recovered 2 items")); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other half of the same refusal, where there is no second export at all: a temp this + /// account cannot open read-write. The advice - grant write access, run again - is the only + /// thing that recovers those rows, so the run giving it must still leave the position they + /// are counted by. It stops rather than export over it. This follows the advice and measures + /// what comes back. + /// + [Fact] + public void An_unopenable_temp_stops_the_export_and_says_how_to_recover_it() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", output, checkpoint); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"], File.ReadAllLines(temp)); + + // 0444: the owner cannot open it read-write either, which is the claim failing on + // permissions rather than on another handle. + File.SetUnixFileMode(temp, UnixFileMode.UserRead); + + var before = ReadDisk(dir, checkpoint, temp, output); + var servedBefore = handler.Served; + var (_, errors) = ExportRun("/users", output, checkpoint); + var after = ReadDisk(dir, checkpoint, temp, output); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointTempUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Contains(Path.GetFileName(temp), stop.Exception.Message); + Assert.Contains("cannot be opened for writing by this account", stop.Exception.Message); + Assert.Contains("not on sharing", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Grant write access to that file and run again to recover them, or remove it " + + "and the checkpoint to export afresh.", + stop.Exception.Message); + Assert.Equal(servedBefore, handler.Served); + AssertUnmoved(before, after); + Assert.False(File.Exists(output), "the stopped run published an output"); + + // Follow it. + File.SetUnixFileMode(temp, UnixFileMode.UserRead | UnixFileMode.UserWrite); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var warnings = ExportWarnings("/users", output, checkpoint); + + Assert.Contains(warnings, w => w.Contains("Recovered 2 items")); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + try + { + foreach (var f in Directory.EnumerateFiles(dir)) + File.SetUnixFileMode(f, UnixFileMode.UserRead | UnixFileMode.UserWrite); + Directory.Delete(dir, true); + } + catch { } + } + } + /// + /// The disk a resumed export leaves at a page boundary. One run dies with its items in a + /// temp; the next promotes that temp over the output, appends the page it resumed onto, + /// saves its position against the output itself - there is no temp left to name - and dies + /// too. That last position is the one shape a shared -CheckpointPath offers no temp to + /// refuse a second run by. + /// + private static void ResumedAndAppending(ScriptedHandler handler, string output, + string checkpoint) + { + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", output, checkpoint); + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", output, checkpoint); + } + + /// + /// The refusal one file over from the temp claim, and the one it could not reach. A run that + /// has resumed once writes straight into -OutputFile and saves a position naming no temp at + /// all, so a second run over the same command line found nothing in the checkpoint to refuse + /// it by: it judged the file its own, cut it back to the recorded offset under a live writer + /// and appended, and the holder's next write landed past the hole that left. There is no + /// pass this run can make that leaves the pair standing either - the rows the checkpoint + /// counts are in the output now - so it makes none. + /// + [Fact] + public void A_held_output_stops_the_export_before_anything_is_written() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + ResumedAndAppending(handler, output, checkpoint); + + // The premise, off the file: this position counts its items into the output and + // names no temp, which is the shape the claim on the temp has nothing to say about. + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + Assert.Equal(Path.GetFullPath(output), cp.OutputFile); + Assert.Equal(new FileInfo(output).Length, cp.DataLength); + + // Export A is not dead. It holds the output the way a resumed run does - appending, + // sharing reads - and has written one more row since its last save, so the file is + // longer than the checkpoint records and cutting it back leaves a hole under it. + handler.Queue(HttpStatusCode.OK, UsersPage2); + DiskState before, after; + int servedBefore; + string[] warnings; + ErrorRecord[] errors; + using (var live = new StreamWriter( + new FileStream(output, FileMode.Append, FileAccess.Write, FileShare.Read))) + { + live.WriteLine("{\"id\":\"u9-live\"}"); + live.Flush(); + Assert.True(new FileInfo(output).Length > cp.DataLength, + "the holder wrote nothing past the recorded length, so no cut would show"); + + before = ReadDisk(dir, checkpoint, output); + servedBefore = handler.Served; + (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.ResourceBusy, stop.CategoryInfo.Category); + Assert.Contains($"Another export is still writing '{output}'", + stop.Exception.Message); + Assert.Contains(checkpoint, stop.Exception.Message); + Assert.Contains($"records {cp.ItemsCollected} items into", stop.Exception.Message); + Assert.Contains("would cut that file back under it", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Wait for that export to finish, or give this one its own -OutputFile and " + + "-CheckpointPath.", + stop.Exception.Message); + + // And not the sentence for a file whose contents this run never looked at. + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + Assert.DoesNotContain(warnings, w => w.Contains("exported from the beginning")); + + // Nothing ran: no request, and the two files and the directory's entry list + // exactly as the holder left them. Read after the record and not before it, so + // a run that took the checkpoint with it fails on the answer it did not give. + after = ReadDisk(dir, checkpoint, output); + Assert.Equal(servedBefore, handler.Served); + AssertUnmoved(before, after); + + // A goes on, and the row it writes next lands where its last one ended. + live.WriteLine("{\"id\":\"u10-live\"}"); + live.Flush(); + } + + // Ordinal, both of them: the culture-aware comparisons treat a NUL as ignorable, + // which is the one byte this is looking for. + var published = File.ReadAllText(output); + Assert.False(published.Contains('\0'), + "the cut left a hole for the holder's next write to land past"); + Assert.EndsWith("{\"id\":\"u9-live\"}" + Environment.NewLine + + "{\"id\":\"u10-live\"}" + Environment.NewLine, published, + StringComparison.Ordinal); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other half of the same claim, where there is no second export at all: the output the + /// checkpoint counts its items into cannot be opened for writing by this account. It was + /// reported as a file that no longer holds those items - a reading of the file's contents, + /// given about a file this run had not been able to open - and the position counting them + /// was deleted on the strength of it. The advice is the only thing that recovers them, so + /// the run giving it leaves both files standing. This follows the advice and measures what + /// comes back. + /// + [Fact] + public void An_unopenable_output_stops_the_export_and_says_how_to_recover_it() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + ResumedAndAppending(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + var recorded = cp.DataLength!.Value; + + // The rows the checkpoint counts are there, and a row written after its last save + // with them - so a run that can open the file cuts back to the recorded length. + File.AppendAllText(output, "{\"id\":\"u9-past-the-save\"}" + Environment.NewLine); + Assert.True(new FileInfo(output).Length > recorded); + + // 0444: the owner cannot open it read-write either, which is the claim failing on + // permissions rather than on another handle. + File.SetUnixFileMode(output, UnixFileMode.UserRead); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var before = ReadDisk(dir, checkpoint, output); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.PermissionDenied, stop.CategoryInfo.Category); + Assert.Contains($"'{output}' could not be opened for writing:", stop.Exception.Message); + Assert.Contains("is denied", stop.Exception.Message); + // The reason's own trailing period is trimmed before it is quoted, so the sentence + // this joins into is punctuated once, not twice. + Assert.DoesNotContain(".. ", stop.Exception.Message); + Assert.Contains("is denied. The resume checkpoint", stop.Exception.Message); + Assert.Contains($"The resume checkpoint at '{checkpoint}' records", + stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Grant write access to that file and run again to resume from it, or remove it " + + "and the checkpoint to export afresh.", + stop.Exception.Message); + + // Not said about a file whose contents this run could not read, and not acted on. + // The disk is read after the record and not before it, so a run that deleted the + // position fails on the answer it did not give. + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + Assert.Equal(servedBefore, handler.Served); + AssertUnmoved(before, ReadDisk(dir, checkpoint, output)); + Assert.True(File.Exists(checkpoint), + "the position counting the items the advice recovers was deleted"); + + // Follow it. The run that comes back cuts the output to the recorded length and + // appends the page it resumes onto, so the row written past the last save goes and + // nothing is written twice. + File.SetUnixFileMode(output, UnixFileMode.UserRead | UnixFileMode.UserWrite); + handler.Queue(HttpStatusCode.OK, UsersPage2); + var resumed = ExportWarnings("/users", output, checkpoint); + + Assert.DoesNotContain(resumed, w => w.Contains("no longer holds")); + Assert.Equal( + ["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u1\"}", "{\"id\":\"u2\"}", + "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + try + { + foreach (var f in Directory.EnumerateFiles(dir)) + File.SetUnixFileMode(f, UnixFileMode.UserRead | UnixFileMode.UserWrite); + Directory.Delete(dir, true); + } + catch { } + } + } + + /// + /// The answer the claim must not swallow. An output shorter than the length the checkpoint + /// records is not the file that checkpoint describes - the items it counts are in no file at + /// all - and nothing holds it and nothing refuses to open it. That still takes today's + /// route: say so, delete the position, and export from the beginning. + /// + [Fact] + public void A_stale_length_output_still_exports_from_the_beginning() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + ResumedAndAppending(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + + // The output is replaced by one that does not hold those items. + File.WriteAllText(output, "{\"id\":\"short\"}" + Environment.NewLine); + Assert.True(new FileInfo(output).Length < cp.DataLength); + + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + Assert.Empty(errors); + Assert.Contains(warnings, + w => w.Contains($"'{output}' no longer holds the {cp.ItemsCollected} items")); + Assert.False(File.Exists(checkpoint), "the stale position was left on disk"); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The disk a resumed export leaves at a page boundary, with every item in the output + /// exactly once. One run dies with its items in a temp; the next promotes that temp over the + /// output, saves its position against the output itself - there is no temp left to name - + /// and dies at its first request. That last position is the one shape a shared + /// -CheckpointPath offers no temp to refuse a second run by. + /// + private static void PromotedAndAppending(ScriptedHandler handler, string output, + string checkpoint) + { + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", output, checkpoint); + Export("/users", output, checkpoint); + } + + /// + /// Two exports over one command line, both past the stat the reconcile makes. The claim on + /// the output was asked for and let go inside the reconcile and the writer opened its own + /// handle a page fetch later, so for that window neither run held the file: both passed the + /// claim, both cut it back to the recorded offset, both appended, and the file came back + /// holding more lines than belonged in it with the checkpoint deleted over the lot. The + /// claim is the handle now, so the run that takes it keeps it across that window, and the + /// second run meets a file the first one has. + /// + /// A is parked in the window itself. The verbose announcing the resume is written after the + /// reconcile - the claim taken, the row past the last save cut away - and before the writer + /// is opened, which is exactly where the two runs used to overlap. + /// + [Fact] + public void Two_exports_released_together_leave_one_with_the_output() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + // A real second thread, not a scheduled task: the two runs have to be able to be inside + // the cmdlet at the same time, and joining one is not a blocking wait on work the test + // framework might have scheduled behind it. + (string[] Warnings, ErrorRecord[] Errors) aResult = default; + Exception? aFailed = null; + Thread? a = null; + using var arrived = new ManualResetEventSlim(false); + using var released = new ManualResetEventSlim(false); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + PromotedAndAppending(handler, output, checkpoint); + + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + Assert.Equal(Path.GetFullPath(output), cp.OutputFile); + var recorded = cp.DataLength!.Value; + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"], File.ReadAllLines(output)); + Assert.Equal(new FileInfo(output).Length, recorded); + + // A row written after the last save, so the cut A makes is visible from outside. + File.AppendAllText(output, "{\"id\":\"u9-past-the-save\"}" + Environment.NewLine); + Assert.True(new FileInfo(output).Length > recorded); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var servedBefore = handler.Served; + + a = new Thread(() => + { + try + { + aResult = ExportParkedAtResume("/users", output, checkpoint, + arrived, released); + } + catch (Exception ex) { aFailed = ex; } + }); + a.Start(); + Assert.True(arrived.Wait(TimeSpan.FromSeconds(60)), + "export A never reached the window between its reconcile and its writer"); + + // Past the reconcile: the row written after the last save has been cut away. And + // short of the writer: no page has been asked for, so nothing has been appended. + Assert.Equal(recorded, new FileInfo(output).Length); + Assert.Equal(servedBefore, handler.Served); + + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.ResourceBusy, stop.CategoryInfo.Category); + Assert.Contains($"Another export is still writing '{output}'", stop.Exception.Message); + Assert.Contains($"records {cp.ItemsCollected} items into", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + + // B sent no request and took nothing with it: the position A resumes from is still + // on disk, and the file is still exactly what A cut it back to. + Assert.Equal(servedBefore, handler.Served); + Assert.True(File.Exists(checkpoint), "B deleted the position A is resuming from"); + Assert.Equal(recorded, new FileInfo(output).Length); + + released.Set(); + Assert.True(a.Join(TimeSpan.FromSeconds(60)), "export A never finished"); + Assert.Null(aFailed); + Assert.Empty(aResult.Errors); + + // One run had the file. Every item once, in order, and the position spent. + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpoint), "the completed export left its position behind"); + } + finally + { + released.Set(); + a?.Join(TimeSpan.FromSeconds(30)); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// What a caller reading the output while a resumed export appends to it gets. The hold is + /// FileShare.None on Unix because that is the only mode .NET backs with LOCK_EX, and LOCK_EX + /// is the only lock that refuses a second writer there - LOCK_SH, which every other mode + /// takes and which the writer's own open used to be, does not. The cost is that it refuses + /// every .NET opener including a reader, and the help says so; tail and cat take no lock at + /// all and read the file as they always did. Windows needs no such trade: the handle's own + /// write access refuses a second writer, so FileShare.Read can let a reader keep looking. + /// + [Fact] + public void A_resumed_export_holds_the_output_against_a_writer_and_tail_still_reads_it() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + (string[] Warnings, ErrorRecord[] Errors) aResult = default; + Exception? aFailed = null; + Thread? a = null; + var gate = new GatedHandler(gateOn: 2); + try + { + { + // The setup runs on its own wire, so the gate below counts only A's requests. + var handler = new ScriptedHandler(); + using var scoped = MgxTransportScope.Inject(handler); + PromotedAndAppending(handler, output, checkpoint); + Assert.Null(PaginationCheckpoint.Load(checkpoint)!.TempFile); + } + + // A page with a continuation, then the page held at the gate. So by the time A is + // parked it has written and flushed a row of its own and is appending still. + gate.Queue(HttpStatusCode.OK, UsersPage2Continued); + gate.Queue(HttpStatusCode.OK, UsersPage3); + using var transport = MgxTransportScope.Inject(gate); + + a = new Thread(() => + { + try { aResult = ExportRun("/users", output, checkpoint); } + catch (Exception ex) { aFailed = ex; } + }); + a.Start(); + Assert.True(gate.Arrived.Wait(TimeSpan.FromSeconds(60)), + "export A never reached its second request"); + + // The property the hold exists for, on both platforms: no second writer. + var refusedWriter = Assert.ThrowsAny(() => + new FileStream(output, FileMode.Append, FileAccess.Write, FileShare.Read)); + Assert.Contains("being used by another process", refusedWriter.Message); + + if (OperatingSystem.IsWindows()) + { + using var reader = new FileStream(output, FileMode.Open, FileAccess.Read, + FileShare.ReadWrite); + Assert.True(reader.Length > 0); + } + else + { + var refusedReader = Assert.ThrowsAny(() => + new FileStream(output, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)); + Assert.Contains("being used by another process", refusedReader.Message); + + // And what the help promises instead. tail takes no lock, so the rows the run + // has written so far come back exactly as they are on disk - the two it resumed + // over and the one it has appended since. + using var tail = System.Diagnostics.Process.Start( + new System.Diagnostics.ProcessStartInfo + { + FileName = "tail", + ArgumentList = { "-c", "4096", output }, + RedirectStandardOutput = true, + RedirectStandardError = true, + }); + Assert.NotNull(tail); + var read = tail!.StandardOutput.ReadToEnd(); + Assert.True(tail.WaitForExit(30_000), "tail did not finish"); + Assert.Equal(0, tail.ExitCode); + Assert.Equal("{\"id\":\"u1\"}" + Environment.NewLine + + "{\"id\":\"u2\"}" + Environment.NewLine + + "{\"id\":\"u3\"}" + Environment.NewLine, read); + } + + gate.Released.Set(); + Assert.True(a.Join(TimeSpan.FromSeconds(60)), "export A never finished"); + Assert.Null(aFailed); + Assert.Empty(aResult.Errors); + + // And the file is everyone's again the moment the writing ends. + using (var after = new FileStream(output, FileMode.Open, FileAccess.ReadWrite, + FileShare.None)) + { + Assert.True(after.Length > 0); + } + Assert.Equal( + ["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}", "{\"id\":\"u4\"}"], + File.ReadAllLines(output)); + } + finally + { + gate.Released.Set(); + a?.Join(TimeSpan.FromSeconds(30)); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// An output in a directory this account cannot search. File.Exists answers false for it - + /// the same answer it gives for a file that is not there - so the run reported that the + /// output no longer held the items the checkpoint counts, and deleted the position counting + /// them, over a whole file it had never opened. The open is what decides now, and being + /// refused on permissions is not the same answer as finding nothing. + /// + /// And which permission. The refusal is an access failure naming the file, so the stop + /// quoted the runtime's words for it and told the caller to grant write access to that file + /// - a file they cannot stat, let alone chmod, through a parent that refuses to be looked + /// into. The grant that opens it is on the directory, and the run says which directory. + /// Told from a mode on the file, which reaches the same catch, by whether the entry can be + /// asked about at all: a directory missing its search bit refuses every name under it, and + /// one that has the bit answers an absent name with ENOENT. Not by whether the directory + /// itself is there - a stat of it needs the search bit on ITS parent, so a mode-000 + /// directory answers Directory.Exists true. This follows the advice and measures what comes + /// back. + /// + [Fact] + public void An_output_whose_directory_cannot_be_searched_stops_the_export() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var outDir = Directory.CreateDirectory(Path.Combine(dir, "out")).FullName; + var cpDir = Directory.CreateDirectory(Path.Combine(dir, "cp")).FullName; + var output = Path.Combine(outDir, "out.jsonl"); + var checkpoint = Path.Combine(cpDir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + PromotedAndAppending(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + var contents = File.ReadAllBytes(output); + var position = File.ReadAllBytes(checkpoint); + + // 000: the directory cannot be searched, so nothing in it can be reached by name. + // That the stat now says false is the premise - it is the answer this used to be + // decided on. Where it still says true the account is root, a mode says nothing, and + // there is nothing here to measure. + File.SetUnixFileMode(outDir, 0); + if (File.Exists(output)) return; + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.PermissionDenied, stop.CategoryInfo.Category); + Assert.Contains($"'{output}' could not be opened for writing: the directory " + + $"'{outDir}' cannot be searched.", stop.Exception.Message); + Assert.Contains($"The resume checkpoint at '{checkpoint}' records", + stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Grant access to that directory and run again to resume from it, or remove the " + + "checkpoint to export afresh.", + stop.Exception.Message); + + // Not the runtime's words for a mode on the file, and not the advice about it: that + // file is the one thing here the caller cannot reach to change or remove. + Assert.DoesNotContain("is denied", stop.Exception.Message); + Assert.DoesNotContain("Grant write access to that file", stop.Exception.Message); + // The reason carries no period of its own, so the sentence it joins is punctuated + // once. + Assert.DoesNotContain(".. ", stop.Exception.Message); + + // Not a reading of a file it never opened, and nothing acted on it. + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + Assert.Equal(servedBefore, handler.Served); + Assert.True(File.Exists(checkpoint), + "the position counting the items the advice recovers was deleted"); + Assert.Equal(position, File.ReadAllBytes(checkpoint)); + + // Follow it. The grant is on the directory, and the run that comes back resumes into + // the file the checkpoint records. + File.SetUnixFileMode(outDir, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + Assert.Equal(contents, File.ReadAllBytes(output)); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var resumed = ExportWarnings("/users", output, checkpoint); + Assert.DoesNotContain(resumed, w => w.Contains("from the beginning")); + Assert.False(File.Exists(checkpoint)); + } + finally + { + try + { + File.SetUnixFileMode(outDir, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + Directory.Delete(dir, true); + } + catch { } + } + } + + /// + /// A directory standing where the output should be. It is not a file, so File.Exists said + /// no and the run took the same route it takes for an output that is not there at all - + /// warning that the file no longer held the items, and deleting the position. Opening a + /// directory read-write is refused on both platforms, and that is an answer. + /// + [Fact] + public void A_directory_standing_at_the_output_path_stops_the_export() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + PromotedAndAppending(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + var position = File.ReadAllBytes(checkpoint); + + File.Delete(output); + Directory.CreateDirectory(output); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + // A directory is not a permission, whatever errno the open came back with: no grant + // makes one writable at an offset, and the way out is to take it off the path. + Assert.Equal(ErrorCategory.InvalidOperation, stop.CategoryInfo.Category); + Assert.Contains($"'{output}' could not be opened for writing: a directory stands at " + + "the path.", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Remove it (or point -OutputFile elsewhere) and run again to resume from it, or " + + "remove the checkpoint to export afresh.", + stop.Exception.Message); + Assert.DoesNotContain("Grant write access", stop.Exception.Message); + + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + Assert.Equal(servedBefore, handler.Served); + Assert.True(File.Exists(checkpoint), "the position was deleted over a directory"); + Assert.Equal(position, File.ReadAllBytes(checkpoint)); + Assert.True(Directory.Exists(output)); + Assert.Empty(Directory.GetFileSystemEntries(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The disk an interrupted fresh export leaves: its items are in a temp beside the output, + /// the checkpoint names that temp, and there is no output at all yet. The run after it + /// promotes the temp over the output and appends onto what it promoted. + /// + private static void InterruptedIntoATemp(ScriptedHandler handler, string output, + string checkpoint) + { + handler.Queue(HttpStatusCode.OK, UsersPage1); + Export("/users", output, checkpoint); + } + + /// + /// The other route that ends in an append, one file over. A run whose checkpoint names a + /// temp promotes it over the output and then appends onto it - and used to hold nothing + /// between the move that put the rows there and the writer's own open a page fetch later. + /// The checkpoint it repoints in that gap names no temp, which is the shape a second run + /// over the same command line reads as an output it may cut back and append to. So the + /// promotion takes the file it promoted into, and the second run meets a file the first + /// one has. + /// + [Fact] + public void A_promoting_export_holds_the_output_it_promoted_into() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + (string[] Warnings, ErrorRecord[] Errors) aResult = default; + Exception? aFailed = null; + Thread? a = null; + using var arrived = new ManualResetEventSlim(false); + using var released = new ManualResetEventSlim(false); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + InterruptedIntoATemp(handler, output, checkpoint); + + // The premise: the items are in a temp, the checkpoint names it, and nothing is at + // -OutputFile for anyone to be holding. + var before = PaginationCheckpoint.Load(checkpoint)!; + Assert.NotNull(before.TempFile); + Assert.False(File.Exists(output)); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var servedBefore = handler.Served; + + a = new Thread(() => + { + try + { + aResult = ExportParkedAtResume("/users", output, checkpoint, + arrived, released); + } + catch (Exception ex) { aFailed = ex; } + }); + a.Start(); + Assert.True(arrived.Wait(TimeSpan.FromSeconds(60)), + "export A never reached the window between its promotion and its writer"); + + // Past the promotion: the rows are in the output, the temp is gone, and the position + // on disk names no temp - which is exactly what a second run reads as an output it + // may take. And short of the writer: no page has been asked for. + // + // Measured off the directory entry, since the run holds the file and a reader of it + // is refused until the run ends. What it holds is read at the end, when it does not. + var promotedBytes = ("{\"id\":\"u1\"}" + Environment.NewLine + + "{\"id\":\"u2\"}" + Environment.NewLine).Length; + Assert.Equal(promotedBytes, new FileInfo(output).Length); + var repointed = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(repointed.TempFile); + Assert.Equal(promotedBytes, repointed.DataLength); + Assert.Equal(["out.jsonl", "run.checkpoint"], + Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal)); + Assert.Equal(servedBefore, handler.Served); + + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.ResourceBusy, stop.CategoryInfo.Category); + Assert.Contains($"Another export is still writing '{output}'", stop.Exception.Message); + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + + // B sent no request and took nothing with it. + Assert.Equal(servedBefore, handler.Served); + Assert.True(File.Exists(checkpoint), "B deleted the position A is resuming from"); + + released.Set(); + Assert.True(a.Join(TimeSpan.FromSeconds(60)), "export A never finished"); + Assert.Null(aFailed); + Assert.Empty(aResult.Errors); + + // One run had the file. Every item once, in order, and the position spent. + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpoint), "the completed export left its position behind"); + } + finally + { + released.Set(); + a?.Join(TimeSpan.FromSeconds(30)); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A -OutputFile the open fails on for a reason no list of access errnos carries. The + /// failures were told apart the other way round - a short list meant "cannot be opened" and + /// everything else meant a second run - so a symlink loop at the output path stopped the + /// export with "Another export is still writing", and the caller was told to wait for a run + /// that could not have existed: nothing can open that path, this export included. + /// + [Fact] + public void A_symlink_loop_at_the_output_path_stops_the_export_as_unopenable() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + ResumedAndAppending(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + + // The output's name now points at itself, one link along. ELOOP, which is neither a + // sharing violation nor a permission the caller could grant. + File.Delete(output); + var other = Path.Combine(dir, "other.jsonl"); + File.CreateSymbolicLink(output, other); + File.CreateSymbolicLink(other, output); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var checkpointBefore = File.ReadAllBytes(checkpoint); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + // Not a permission, and not the category a caller reads as one: nothing about this + // path is granted, and InvalidOperation is what says so. + Assert.Equal(ErrorCategory.InvalidOperation, stop.CategoryInfo.Category); + Assert.Contains($"'{output}' could not be opened for writing:", stop.Exception.Message); + // The runtime's own ELOOP wording, not a guess at "on permissions" - and nothing in + // it a caller could grant, so the advice that follows does not offer to. + Assert.Contains("Too many levels of symbolic links", stop.Exception.Message); + // No trailing period on this reason, so trimming one off it is a no-op: the single + // period the sentence joins on is the one it always had. + Assert.Contains($"Too many levels of symbolic links : '{output}'. The resume " + + "checkpoint", stop.Exception.Message); + Assert.Contains($"The resume checkpoint at '{checkpoint}' records", + stop.Exception.Message); + Assert.DoesNotContain("Another export is still writing", stop.Exception.Message); + Assert.DoesNotContain("Grant write access", stop.Exception.Message); + Assert.Contains( + "Fix that and run again to resume from it, or remove it and the checkpoint to " + + "export afresh.", + stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + + // Nothing ran, and the position counting the items is still there. + Assert.Equal(servedBefore, handler.Served); + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other shape that got past both guards: a FIFO at the output path. It opens read-write + /// without blocking, so the claim answered Taken, and the first question asked of the handle + /// afterwards - how long is the file - threw a NotSupportedException out of the run with the + /// FIFO still open. A stream with no length and no offset to write at is a file this run + /// cannot resume into, decided at the open and let go there. + /// + [Fact] + public void A_fifo_at_the_output_path_stops_the_export_as_unopenable() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + ResumedAndAppending(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + + File.Delete(output); + Mkfifo(output); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var checkpointBefore = File.ReadAllBytes(checkpoint); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.InvalidOperation, stop.CategoryInfo.Category); + Assert.Contains($"'{output}' could not be opened for writing: the file has no " + + "offset to write at.", stop.Exception.Message); + Assert.Contains($"The resume checkpoint at '{checkpoint}' records", + stop.Exception.Message); + Assert.DoesNotContain("Grant write access", stop.Exception.Message); + Assert.Contains( + "Fix that and run again to resume from it, or remove it and the checkpoint to " + + "export afresh.", + stop.Exception.Message); + Assert.IsNotType(stop.Exception); + Assert.Equal(servedBefore, handler.Served); + Assert.DoesNotContain(warnings, w => w.Contains("no longer holds")); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + + // And the handle it opened to find that out is closed: a claim of this test's own + // on the same FIFO succeeds, which it would not while the run still held one. + using var claimed = new FileStream(output, FileMode.Open, FileAccess.ReadWrite, + FileShare.None); + } + finally + { + try { File.Delete(output); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The promotion route's own claim on the file it replaces. A run whose checkpoint names a + /// temp puts those rows over the output and appends onto what it put there, and it asked for + /// the file only after the move - which a rename asks nothing about, on either side of it. So + /// the move went through over an output another run was holding and appending to: that run's + /// next write landed in an inode with no name left on it, and the claim taken a statement + /// later answered Taken about a file one statement old. The output is asked for first now, + /// and a held one is the whole answer. + /// + [Fact] + public void A_promotion_over_a_held_output_stops_and_replaces_nothing() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + InterruptedIntoATemp(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + var temp = Path.Combine(dir, cp.TempFile!); + Assert.True(File.Exists(temp), "the interrupted run left no temp to promote"); + Assert.False(File.Exists(output)); + + // A previous export's rows at -OutputFile, and the run that has them: appending, + // sharing reads, the way a resumed run holds its own output. + File.WriteAllLines(output, OldExport); + handler.Queue(HttpStatusCode.OK, UsersPage2); + using (var live = new StreamWriter( + new FileStream(output, FileMode.Append, FileAccess.Write, FileShare.Read)) + { AutoFlush = true }) + { + var before = ReadDisk(dir, checkpoint, temp, output); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.ResourceBusy, stop.CategoryInfo.Category); + Assert.Contains($"Another export is still writing '{output}'", + stop.Exception.Message); + // The temp, not the output, is where these items are counted on the promotion + // route, and going on replaces the output rather than cutting it back - the + // held sentence for a run that had already resumed says neither. + Assert.Contains($"The resume checkpoint at '{checkpoint}' records " + + $"{cp.ItemsCollected} items into the temp '{cp.TempFile}'", + stop.Exception.Message); + Assert.Contains($"going on would replace '{output}' with them under the run " + + "that holds it", stop.Exception.Message); + Assert.DoesNotContain("cut that file back", stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", + stop.Exception.Message); + + // No request, no recovery, and nothing moved: the temp still holds the rows the + // checkpoint counts, the checkpoint still names it, and the staged file the move + // would have come from was taken back. + Assert.Equal(servedBefore, handler.Served); + Assert.DoesNotContain(warnings, w => w.Contains("Recovered")); + AssertUnmoved(before, ReadDisk(dir, checkpoint, temp, output)); + Assert.False(File.Exists($"{output}.adopt")); + + // And the same inode, not a new file wearing the name: the holder's next write + // is read back through the path. + live.WriteLine("{\"id\":\"old-0000004\"}"); + } + + Assert.Equal([.. OldExport, "{\"id\":\"old-0000004\"}"], File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same claim, refused the other way, on the route where the move would have done the + /// damage itself: a FIFO at -OutputFile is replaced by a rename, so the caller's pipe and + /// whatever reads it would simply be gone - and the run would then have appended to a regular + /// file nobody was reading. + /// + [Fact] + public void A_promotion_over_a_fifo_output_stops_and_leaves_the_pipe() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + InterruptedIntoATemp(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + var temp = Path.Combine(dir, cp.TempFile!); + Assert.False(File.Exists(output)); + Mkfifo(output); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var checkpointBefore = File.ReadAllBytes(checkpoint); + var tempBefore = File.ReadAllBytes(temp); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.InvalidOperation, stop.CategoryInfo.Category); + Assert.Contains($"'{output}' could not be opened for writing: the file has no " + + "offset to write at.", stop.Exception.Message); + // The temp, not the output, is where these items are counted on the promotion + // route - the fixed lead-in above no longer says so on its own. + Assert.Contains($"The resume checkpoint at '{checkpoint}' records " + + $"{cp.ItemsCollected} items into the temp '{cp.TempFile}'", stop.Exception.Message); + Assert.Contains($"cannot promote them into '{output}'", stop.Exception.Message); + Assert.DoesNotContain("Grant write access", stop.Exception.Message); + Assert.Contains( + "Fix that and run again to recover them, or remove it and the checkpoint to " + + "export afresh.", + stop.Exception.Message); + Assert.Equal(servedBefore, handler.Served); + Assert.DoesNotContain(warnings, w => w.Contains("Recovered")); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.False(File.Exists($"{output}.adopt")); + + // Still a pipe, and still nobody's: a stream with no offset in it is what says the + // first, and a claim of this test's own would be refused if the run had kept one. + using var stillAFifo = new FileStream(output, FileMode.Open, FileAccess.ReadWrite, + FileShare.None); + Assert.False(stillAFifo.CanSeek, "the promotion replaced the caller's FIFO"); + } + finally + { + try { File.Delete(output); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The disk a promotion leaves for the instant between the temp's unlink and the save that + /// repoints the position: the rows are in the output at the length recorded for them, the + /// temp is gone, and the checkpoint still names it. Written out here rather than raced for, + /// so what the route does with each shape can be asked one shape at a time. + /// + private static long PromotedButNotYetRepointed(string dir, string output, string checkpoint) + { + var cp = PaginationCheckpoint.Load(checkpoint)!; + var temp = Path.Combine(dir, cp.TempFile!); + var recorded = cp.DataLength!.Value; + File.WriteAllBytes(output, File.ReadAllBytes(temp)[..(int)recorded]); + File.Delete(temp); + return recorded; + } + + /// + /// The window the second run walked through. A checkpoint naming a temp that is not on disk + /// was read as items in no file at all: the run said so, deleted the position counting them, + /// exported from the beginning and moved its own temp over the output at the end - which was + /// the promoting run's output, held and being appended to. The output is asked about before + /// any of that now, and a held one is the same stop the other two routes reach. + /// + [Fact] + public void A_vanished_temp_over_a_held_output_stops_the_export() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + InterruptedIntoATemp(handler, output, checkpoint); + var recorded = PromotedButNotYetRepointed(dir, output, checkpoint); + Assert.Equal(recorded, new FileInfo(output).Length); + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.NotNull(cp.TempFile); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + using var live = new StreamWriter( + new FileStream(output, FileMode.Append, FileAccess.Write, FileShare.Read)); + + var before = ReadDisk(dir, checkpoint, output); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Contains($"Another export is still writing '{output}'", stop.Exception.Message); + // The temp this checkpoint named is gone, not held - so the items are recorded + // (past tense) into a temp the sentence does not go on to name, and going on means + // exporting fresh rather than cutting the output back. + Assert.Contains($"The resume checkpoint at '{checkpoint}' recorded " + + $"{cp.ItemsCollected} items into a temp that is gone", stop.Exception.Message); + Assert.Contains($"going on would export from the beginning and replace '{output}' " + + "under the run that holds it", stop.Exception.Message); + Assert.DoesNotContain("cut that file back", stop.Exception.Message); + Assert.DoesNotContain(warnings, w => w.Contains("missing or incomplete")); + Assert.Equal(servedBefore, handler.Served); + AssertUnmoved(before, ReadDisk(dir, checkpoint, output)); + Assert.True(File.Exists(checkpoint), + "the position the promoting run is resuming from was deleted"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other half of the vanished-temp claim, where there is no second export at all: the + /// output the promoting run left behind cannot be opened for writing by this account. It used + /// to be reported as a file that never held the recorded items - the same misreading an + /// unopenable output got on every route - and the position counting them deleted on the + /// strength of it. The advice is the only thing that recovers here, so the run giving it + /// leaves the checkpoint standing over a temp that is, in fact, already gone. + /// + [Fact] + public void A_vanished_temp_over_an_unopenable_output_stops_the_export() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + InterruptedIntoATemp(handler, output, checkpoint); + var recorded = PromotedButNotYetRepointed(dir, output, checkpoint); + Assert.Equal(recorded, new FileInfo(output).Length); + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.NotNull(cp.TempFile); + + // 0444: the owner cannot open it read-write either, which is the claim failing on + // permissions rather than on another handle. + File.SetUnixFileMode(output, UnixFileMode.UserRead); + + // Nothing is queued for this attempt: the stop is decided before any request, and a + // response queued here and never served would still be waiting for the fresh export + // below, ahead of the two pages that export actually asks for. + var before = ReadDisk(dir, checkpoint, output); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointOutputUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.PermissionDenied, stop.CategoryInfo.Category); + Assert.Contains($"'{output}' could not be opened for writing:", stop.Exception.Message); + Assert.Contains("is denied", stop.Exception.Message); + // The reason's own trailing period is trimmed before it is quoted, so the sentence + // this joins into is punctuated once, not twice. + Assert.DoesNotContain(".. ", stop.Exception.Message); + Assert.Contains("is denied. The resume checkpoint", stop.Exception.Message); + Assert.Contains($"The resume checkpoint at '{checkpoint}' recorded " + + $"{cp.ItemsCollected} items into a temp that is gone", stop.Exception.Message); + Assert.Contains($"cannot export a fresh copy into '{output}' either", + stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.Contains( + "Grant write access to that file and run again, or remove it and the checkpoint " + + "to export afresh.", + stop.Exception.Message); + + Assert.DoesNotContain(warnings, w => w.Contains("missing or incomplete")); + Assert.Equal(servedBefore, handler.Served); + AssertUnmoved(before, ReadDisk(dir, checkpoint, output)); + Assert.True(File.Exists(checkpoint), + "the position counting the vanished temp's items was deleted"); + + // Follow it. Nothing was recoverable from the temp regardless, so the run that comes + // back exports fresh once it can reach the output at all. + File.SetUnixFileMode(output, UnixFileMode.UserRead | UnixFileMode.UserWrite); + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + var resumed = ExportWarnings("/users", output, checkpoint); + + Assert.Contains(resumed, w => w.Contains("missing or incomplete")); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpoint)); + } + finally + { + try + { + File.SetUnixFileMode(output, UnixFileMode.UserRead | UnixFileMode.UserWrite); + Directory.Delete(dir, true); + } + catch { } + } + } + + /// + /// And the shape that is no promotion at all. The temp is gone and the file at -OutputFile is + /// a previous export's, which nothing holds - and its length says nothing either way: two + /// rows of one shape are two rows of another shape's length, so a previous run's output + /// measures exactly what a promotion would have left about as often as a promoted one does. + /// This checkpoint counts its items into the temp, so resuming into that file would put rows + /// the caller has already consumed in front of this run's under no warning at all. It takes + /// the route it always took, which replaces the whole file and cannot leave a wrong one + /// behind. + /// + [Fact] + public void A_vanished_temp_over_an_unheld_output_still_exports_from_the_beginning() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + InterruptedIntoATemp(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + File.Delete(Path.Combine(dir, cp.TempFile!)); + + // A previous export's two rows, of the same shape and so of the same length as the + // two this checkpoint counts into the temp that has gone. + File.WriteAllLines(output, ["{\"id\":\"x1\"}", "{\"id\":\"x2\"}"]); + Assert.Equal(cp.DataLength, new FileInfo(output).Length); + + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + Assert.Empty(errors); + Assert.Contains(warnings, w => w.Contains("missing or incomplete")); + + // The whole file, and not the previous export's first two rows with this + // enumeration's remainder spliced onto them. + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The two runs the shapes above were taken from, released together on the promotion route. + /// Both completed: whichever lost the promotion found the temp gone, warned that the items it + /// counted were not on disk, deleted the position counting them, exported from the beginning + /// and moved its own temp over the file the other one was holding and appending to. A run + /// that loses a file ends there instead now, and nothing it would have taken with it moves - + /// whichever of the two files it lost, and whether one of them lost one or both did. + /// + [Fact] + public void Two_exports_released_together_on_the_promotion_route_leave_one_with_the_file() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var gate = new GatedUsersByUrlHandler(); + var results = new (string[] Warnings, ErrorRecord[] Errors)[2]; + var failures = new Exception?[2]; + var threads = new Thread[2]; + var ready = new[] { new ManualResetEventSlim(false), new ManualResetEventSlim(false) }; + using var go = new ManualResetEventSlim(false); + try + { + var setup = new ScriptedHandler(); + using (MgxTransportScope.Inject(setup)) + { + InterruptedIntoATemp(setup, output, checkpoint); + } + Assert.NotNull(PaginationCheckpoint.Load(checkpoint)!.TempFile); + Assert.False(File.Exists(output)); + + using var transport = MgxTransportScope.Inject(gate); + for (var i = 0; i < threads.Length; i++) + { + var slot = i; + threads[slot] = new Thread(() => + { + try + { + results[slot] = ExportReleasedWith("/users", output, checkpoint, + ready[slot], go); + } + catch (Exception ex) { failures[slot] = ex; } + }); + threads[slot].Start(); + } + + foreach (var r in ready) + Assert.True(r.Wait(TimeSpan.FromSeconds(60)), "an export never came up"); + go.Set(); + + // A run that got past the reconcile holds the output and is parked on its first + // request. It stays there until the other has finished, which is what a run that + // stops does with nothing on the wire at all - and which file it loses, the output + // or the name the promotion stages into, is the race. Both of them losing one is an + // outcome of it: then no run has the output and none of them asks for a page. + Assert.True( + SpinWait.SpinUntil(() => gate.Arrived.IsSet || threads.All(t => !t.IsAlive), + TimeSpan.FromSeconds(60)), + "neither export reached the wire or finished"); + if (gate.Arrived.IsSet) + { + Assert.True( + SpinWait.SpinUntil(() => threads.Any(t => !t.IsAlive), + TimeSpan.FromSeconds(60)), + "both exports were still running with one of them holding the output"); + } + + gate.Released.Set(); + foreach (var t in threads) + Assert.True(t.Join(TimeSpan.FromSeconds(60)), "an export never finished"); + Assert.All(failures, Assert.Null); + + // Every run that stopped did so on one of the files this checkpoint stands for, + // saying it wrote nothing and changed nothing, and none of them said the items it + // was protecting were lost. + var stops = results.SelectMany(r => r.Errors).ToArray(); + Assert.All(stops, s => Assert.StartsWith("Checkpoint", s.FullyQualifiedErrorId, + StringComparison.Ordinal)); + Assert.All(stops, s => Assert.True( + s.Exception.Message.Contains("This run stops here; nothing was written.") + || s.Exception.Message.Contains("Nothing was changed;"), + s.Exception.Message)); + Assert.DoesNotContain(results.SelectMany(r => r.Warnings), + w => w.Contains("missing or incomplete")); + + if (gate.Arrived.IsSet) + { + // One run had the file: every item once, in order, and the position spent by + // the run that finished rather than by the run that gave up on it. + Assert.Single(stops); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpoint)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + else + { + // Neither had it: the two files they were released on are the two files they + // leave, with every item the checkpoint counts still in the temp it names. + Assert.Equal(2, stops.Length); + Assert.False(File.Exists(output)); + var left = PaginationCheckpoint.Load(checkpoint); + Assert.NotNull(left!.TempFile); + Assert.True(File.Exists(Path.Combine(dir, left.TempFile!)), + "a run that changed nothing took the temp with it"); + } + + Assert.Empty(Directory.GetFiles(dir, "*.adopt")); + } + finally + { + gate.Released.Set(); + foreach (var t in threads) t?.Join(TimeSpan.FromSeconds(30)); + foreach (var r in ready) r.Dispose(); + gate.Arrived.Dispose(); + gate.Released.Dispose(); + gate.Dispose(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The write a promotion makes before it replaces anything, refused. A directory standing + /// where the staged copy goes is one of the ways it fails, and the answer used to come out of + /// the catch around the whole promotion as "nothing was promoted" - which the caller reads as + /// the temp having gone: it said the items it counted were not on disk, deleted the position + /// counting them, exported from the beginning and swept the temp that was holding every one. + /// The staging says so for itself now, and both files are where they were found. + /// + [Fact] + public void A_directory_at_the_staging_name_stops_the_export_and_changes_nothing() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + InterruptedIntoATemp(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + var temp = Path.Combine(dir, cp.TempFile!); + Assert.True(File.Exists(temp), "the interrupted run left no temp to promote"); + + var adopt = $"{output}.adopt"; + Directory.CreateDirectory(adopt); + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var before = ReadDisk(dir, checkpoint, temp, output); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointStagingFailed", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.WriteError, stop.CategoryInfo.Category); + Assert.Contains($"The {cp.ItemsCollected} items the resume checkpoint at " + + $"'{checkpoint}' records are in '{cp.TempFile}', whole, but staging them into " + + $"'{output}' failed at '{adopt}': ", stop.Exception.Message); + Assert.Contains("Nothing was changed; fix that and run again to recover them, or " + + "remove the temp and the checkpoint to export afresh.", stop.Exception.Message); + + // Not the sentence for a temp that has gone, and not the delete that goes with it. + // The runtime's own words, punctuated once: the reason is quoted inside a sentence + // that ends it, and a message already ending in a period put two there. + Assert.DoesNotContain(".. Nothing was changed;", stop.Exception.Message); + + Assert.DoesNotContain(warnings, w => w.Contains("missing or incomplete")); + Assert.DoesNotContain(warnings, w => w.Contains("Recovered")); + Assert.Equal(servedBefore, handler.Served); + AssertUnmoved(before, ReadDisk(dir, checkpoint, temp, output)); + Assert.True(Directory.Exists(adopt), "the run removed the caller's directory"); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same write refused the other way: another run holding the staging name, which is what + /// two exports over one command line reach a page apart. Nothing this run may take, and + /// nothing it may report as a temp that has gone. + /// + [Fact] + public void A_held_staging_name_stops_the_export_and_leaves_the_holders_file() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + InterruptedIntoATemp(handler, output, checkpoint); + var cp = PaginationCheckpoint.Load(checkpoint)!; + var temp = Path.Combine(dir, cp.TempFile!); + var adopt = $"{output}.adopt"; + + // The run that has it, holding the staged copy the way a promotion holds its own. + using var holder = new FileStream(adopt, FileMode.Create, FileAccess.ReadWrite, + FileShare.None); + holder.Write("{\"id\":\"theirs\"}\n"u8); + holder.Flush(); + var holderBytes = holder.Length; + + handler.Queue(HttpStatusCode.OK, UsersPage2); + var before = ReadDisk(dir, checkpoint, temp, output); + var servedBefore = handler.Served; + var (warnings, errors) = ExportRun("/users", output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointStagingFailed", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.WriteError, stop.CategoryInfo.Category); + Assert.Contains($"staging them into '{output}' failed at '{adopt}': ", + stop.Exception.Message); + Assert.Contains("Nothing was changed;", stop.Exception.Message); + Assert.DoesNotContain(warnings, w => w.Contains("missing or incomplete")); + Assert.Equal(servedBefore, handler.Served); + + // The temp and the checkpoint as they were, and the holder's own bytes still theirs: + // a promotion that could not stage does not truncate what it could not open. + AssertUnmoved(before, ReadDisk(dir, checkpoint, temp, output)); + Assert.Equal(holderBytes, holder.Length); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The copy a promotion stages, left behind by a run killed between the write and the + /// rename. No glob of the sweep's shape reaches that name - it is the output's own name with + /// a suffix, not "{output}.{32-hex}.tmp" - so it stood beside the output until somebody + /// noticed it, holding a partial copy of an earlier run's rows. The sweep takes it by name + /// now, and asks for it first, the way it asks for a temp. + /// + [Fact] + public void A_staged_copy_an_interrupted_promotion_left_is_swept() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var adopt = $"{output}.adopt"; + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // What a killed promotion leaves: its staged copy, and no checkpoint counting it. + File.WriteAllLines(adopt, OldExport); + var orphanTemp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllLines(orphanTemp, OldExport); + + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + Export("/users", output, Path.Combine(dir, "run.checkpoint")); + + Assert.False(File.Exists(adopt), "the staged copy outlived the run that swept"); + Assert.False(File.Exists(orphanTemp)); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And the one the sweep may not take: a promotion running right now holds the name it has + /// just staged, and a file another run holds is not an orphan whichever name it wears. + /// + [Fact] + public void A_staged_copy_another_run_holds_is_left_alone() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var adopt = $"{output}.adopt"; + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + using var holder = new FileStream(adopt, FileMode.Create, FileAccess.ReadWrite, + FileShare.None); + holder.Write("{\"id\":\"theirs\"}\n"u8); + holder.Flush(); + var holderBytes = holder.Length; + + handler.Queue(HttpStatusCode.OK, UsersPage1); + handler.Queue(HttpStatusCode.OK, UsersPage2); + Export("/users", output, Path.Combine(dir, "run.checkpoint")); + + Assert.True(File.Exists(adopt), "the sweep took a staged copy another run was holding"); + Assert.Equal(holderBytes, holder.Length); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// Both streams of an export, for a scenario whose evidence is on the verbose one: whether + /// the run swept the temps beside its output is not something a warning says. + /// + private static (string[] Warnings, string[] Verbose) ExportStreams(string uri, + string outputPath, string checkpointPath) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", uri) + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("All") + .AddParameter("Verbose"); + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + return ([.. ps.Streams.Warning.Select(w => w.Message)], + [.. ps.Streams.Verbose.Select(v => v.Message)]); + } + + /// + /// A refusal leaves the checkpoint standing because the export that wrote it resumes from + /// exactly that, and the items it counted are in a temp beside this output. A checkpoint + /// written before the temp's name was recorded names none - the shape every release before + /// 2.1 wrote - so keyed on that name the sweep in the same run found nothing to spare and + /// deleted the file the spared position was pointing at, under a warning that had promised + /// to leave things alone. The refusal takes the whole sweep with it instead. + /// + [Fact] + public void A_refusal_of_a_checkpoint_naming_no_temp_holds_the_whole_sweep_off() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var orphan = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // A groups export killed partway: its two items in a temp beside this output, no + // output of its own yet, and a position counting them that names neither file. + var counted = "{\"id\":\"g1\"}\n{\"id\":\"g2\"}\n"; + File.WriteAllText(orphan, counted); + File.WriteAllText(checkpoint, """ + {"resource":"https://graph.microsoft.com/v1.0/groups?$top=999", + "nextLink":"https://graph.microsoft.com/v1.0/groups?$skiptoken=P2", + "itemsCollected":2,"pageItemsAlreadyWritten":0} + """); + + // A users export over the same -CheckpointPath. It refuses that position and dies + // on its first request, so it leaves no output and no temp of its own. + var (warnings, verbose) = ExportStreams("/users", output, checkpoint); + + Assert.Contains(warnings, w => w.Contains("belongs to a different export", + StringComparison.Ordinal)); + Assert.Contains(verbose, v => v.Contains( + $"Left the temp files beside '{output}' alone: one of them may hold the items " + + "of the checkpoint this run refused.", StringComparison.Ordinal)); + Assert.True(File.Exists(orphan), + "the sweep took the temp the refused checkpoint stands for"); + Assert.Equal(counted, File.ReadAllText(orphan)); + Assert.True(File.Exists(checkpoint), "the refused position was deleted"); + Assert.False(File.Exists(output), "the run that stopped left an output behind"); + + // And the export that owns both comes back to find them where it left them. + handler.Queue(HttpStatusCode.OK, GroupsPage2); + var (recovered, _) = ExportStreams("/groups", output, checkpoint); + + Assert.Contains(recovered, w => w.Contains( + "Recovered 2 items from an interrupted export's temp file.", + StringComparison.Ordinal)); + Assert.Equal(["{\"id\":\"g1\"}", "{\"id\":\"g2\"}", "{\"id\":\"g3\"}"], + File.ReadAllLines(output)); + Assert.Empty(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// An export run with a -WarningAction of the caller's choosing, both streams captured. A + /// warning is a terminating error under Stop and nothing at all under SilentlyContinue, so + /// where the run writes one decides what a caller is left holding. + /// + private static (string[] Warnings, string[] Verbose, ErrorRecord[] Errors) ExportWith( + string uri, string outputPath, string checkpointPath, string warningAction) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", uri) + .AddParameter("OutputFile", outputPath) + .AddParameter("CheckpointPath", checkpointPath) + .AddParameter("All") + .AddParameter("Verbose") + .AddParameter("WarningAction", warningAction); + List errors = []; + try { ps.Invoke(); } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + // What -WarningAction Stop raises, and it is not a CmdletInvocationException: left + // uncaught it comes off the thread this runs on and takes the test host with it. + catch (ActionPreferenceStopException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + return ([.. ps.Streams.Warning.Select(w => w.Message)], + [.. ps.Streams.Verbose.Select(v => v.Message)], + [.. errors]); + } + + /// + /// The state a crash leaves partway through a checkpointed export, with a pipe standing at + /// the staging name: the previous export's output, this run's items in the temp beside it, + /// a checkpoint naming both, and an entry the recovery has to take off the name first. + /// + private static (string Temp, string Adopt) InterruptedWithAPipeAtTheStagingName(string dir, + string output, string checkpoint) + { + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = Path.GetFileName(temp), + OutputFile = output, + DataLength = new FileInfo(temp).Length, + }.Save(checkpoint); + var adopt = $"{output}.adopt"; + Mkfifo(adopt); + return (temp, adopt); + } + + /// + /// The sweep's removal warning was written where the deletion happens, which is above the + /// recovery it clears the way for. A warning is a terminating error under -WarningAction + /// Stop, so the run ended there: the pipe deleted, the items still in the temp, the output + /// still the previous export's, and a caller told only that something had been removed. The + /// warning is held until this run's own outcome for the recovery is known and written after + /// it, so a run stopped by it stops with the promotion made and the position repointed at + /// the file it landed in - resumable. + /// + [Fact] + public void A_sweep_under_WarningAction_Stop_leaves_the_recovery_made_and_resumable() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + string? adopt = null; + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + var (temp, adoptPath) = InterruptedWithAPipeAtTheStagingName(dir, output, checkpoint); + adopt = adoptPath; + + // On its own thread with a watchdog: an open of a pipe with no reader never comes + // back, and a test that called the export inline would hang the suite. + (string[] Warnings, string[] Verbose, ErrorRecord[] Errors) result = ([], [], []); + var running = new Thread(() => + result = ExportWith("/users", output, checkpoint, "Stop")) { IsBackground = true }; + running.Start(); + Assert.True(running.Join(TimeSpan.FromSeconds(30)), + "the run never returned: the staging open is waiting for a reader on the pipe"); + + // The run ended on a warning, which is what Stop makes of one. + var stop = Assert.Single(result.Errors); + Assert.StartsWith("ActionPreferenceStop", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + + // And what it ended with: the items promoted into the output, the temp spent, the + // pipe off the name, and a position that records the file they are in. + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"], File.ReadAllLines(output)); + Assert.False(File.Exists(temp), "the promoted temp was left on disk"); + Assert.False(File.Exists(adoptPath), "the pipe outlived the run"); + var cp = PaginationCheckpoint.Load(checkpoint)!; + Assert.Null(cp.TempFile); + Assert.Equal(Path.GetFullPath(output), cp.OutputFile); + Assert.Equal(new FileInfo(output).Length, cp.DataLength); + Assert.Equal(0, handler.Served); + } + finally + { + if (adopt != null) { try { File.Delete(adopt); } catch { } } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The other end of the same preference. -WarningAction SilentlyContinue takes the removal + /// off every stream a caller sees, and an entry beside their output was deleted with nothing + /// anywhere to say so. The verbose line stays where the deletion happens, as the record of + /// when it happened, so a caller who asks for verbose is told either way. + /// + [Fact] + public void A_sweep_under_WarningAction_SilentlyContinue_still_removes_and_says_so_in_verbose() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + string? adopt = null; + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + var (temp, adoptPath) = InterruptedWithAPipeAtTheStagingName(dir, output, checkpoint); + adopt = adoptPath; + handler.Queue(HttpStatusCode.OK, UsersPage2); + + (string[] Warnings, string[] Verbose, ErrorRecord[] Errors) result = ([], [], []); + var running = new Thread(() => + result = ExportWith("/users", output, checkpoint, "SilentlyContinue")) + { IsBackground = true }; + running.Start(); + Assert.True(running.Join(TimeSpan.FromSeconds(30)), + "the run never returned: the staging open is waiting for a reader on the pipe"); + + Assert.Empty(result.Errors); + Assert.Empty(result.Warnings); + Assert.Contains(result.Verbose, v => v.Contains( + $"Removed what stood at the staging name '{Path.GetFileName(adoptPath)}': " + + "an entry no copy could be staged in, such as a pipe or a socket.", + StringComparison.Ordinal)); + Assert.False(File.Exists(adoptPath), "the pipe outlived the run"); + Assert.False(File.Exists(temp), "the promoted temp was left on disk"); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.False(File.Exists(checkpoint), "the completed export left its position behind"); + } + finally + { + if (adopt != null) { try { File.Delete(adopt); } catch { } } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A staging failure after the sweep took something off the name. The removal cannot be a + /// warning of its own there - a warning in front of a terminating error is what + /// -WarningAction Stop ends the run on instead of the stop - so the stop says it: a reader + /// told only that the staging failed goes looking for a file that is no longer there. The + /// two halves of one disk state, in one sentence, and the same phrase the warning uses. + /// + [Fact] + public void The_staging_stop_names_what_the_sweep_took_off_the_name() + { + var message = typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).GetMethod( + "CheckpointStagingStopMessage", + System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic)!; + var checkpoint = new PaginationCheckpoint { ItemsCollected = 7 }; + string Build(string? removedKind) => (string)message.Invoke(null, [ + Activator.CreateInstance( + typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetNestedType("StagingFailure", + System.Reflection.BindingFlags.NonPublic)!, + ["/d/out.jsonl.adopt", "the volume is full"])!, + "/d/run.checkpoint", "/d/out.jsonl", checkpoint, "out.jsonl.abc.tmp", removedKind, + "Nothing was changed;"])!; + + var swept = Build("a link, and not what it pointed at"); + Assert.Contains("failed at '/d/out.jsonl.adopt': the volume is full, after removing what " + + "stood at the staging name: a link, and not what it pointed at. Nothing was " + + "changed;", swept); + + // And where the name was clear, the sentence says nothing about a removal. + var clear = Build(null); + Assert.Contains("failed at '/d/out.jsonl.adopt': the volume is full. Nothing was changed;", + clear); + Assert.DoesNotContain("after removing", clear); + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ExportWhatIfCheckpointTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ExportWhatIfCheckpointTests.cs new file mode 100644 index 0000000..526a61d --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/ExportWhatIfCheckpointTests.cs @@ -0,0 +1,918 @@ +using System.Globalization; +using System.Management.Automation; +using System.Management.Automation.Host; +using System.Management.Automation.Runspaces; +using System.Net; +using System.Security; +using System.Text; +using Mgx.Cmdlets.Base; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// -WhatIf on an export with a resume checkpoint. Recovery promotes temp files over outputs, +/// cuts outputs back to a recorded length and deletes checkpoints, and it ran above the +/// ShouldProcess gate - so the run that reported it would change nothing had already rewritten +/// the output and removed the position it was resuming from. The gate also has to name the +/// action a real run would take, which it cannot do from "a checkpoint file exists". +/// +[Collection("Pipeline")] +public class ExportWhatIfCheckpointTests +{ + private const string Page2 = """ + {"value":[{"id":"u3"}]} + """; + + private sealed class CountingHandler : HttpMessageHandler + { + public int Requests; + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref Requests); + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(Page2, Encoding.UTF8, "application/json") + }); + } + } + + /// The lines -WhatIf writes go to the host, so a test that reads them needs one. + private sealed class RecordingHost : PSHost + { + private readonly Guid _id = Guid.NewGuid(); + public RecordingHostUI Recorder { get; } = new(); + public override string Name => "MgxTestHost"; + public override Version Version => new(1, 0); + public override Guid InstanceId => _id; + public override PSHostUserInterface UI => Recorder; + public override CultureInfo CurrentCulture => CultureInfo.InvariantCulture; + public override CultureInfo CurrentUICulture => CultureInfo.InvariantCulture; + public override void EnterNestedPrompt() { } + public override void ExitNestedPrompt() { } + public override void NotifyBeginApplication() { } + public override void NotifyEndApplication() { } + public override void SetShouldExit(int exitCode) { } + } + + private sealed class RecordingHostUI : PSHostUserInterface + { + public List Lines { get; } = []; + public override PSHostRawUserInterface? RawUI => null; + public override void Write(string value) => Lines.Add(value); + public override void Write(ConsoleColor f, ConsoleColor b, string value) => Lines.Add(value); + public override void WriteLine(string value) => Lines.Add(value); + public override void WriteErrorLine(string value) => Lines.Add(value); + public override void WriteDebugLine(string value) => Lines.Add(value); + public override void WriteVerboseLine(string value) => Lines.Add(value); + public override void WriteWarningLine(string value) => Lines.Add(value); + public override void WriteProgress(long sourceId, ProgressRecord record) { } + public override string ReadLine() => string.Empty; + public override SecureString ReadLineAsSecureString() => new(); + public override Dictionary Prompt( + string caption, string message, System.Collections.ObjectModel.Collection descriptions) => []; + public override int PromptForChoice( + string caption, string message, + System.Collections.ObjectModel.Collection choices, int defaultChoice) => defaultChoice; + public override PSCredential PromptForCredential( + string caption, string message, string userName, string targetName) => PSCredential.Empty; + public override PSCredential PromptForCredential( + string caption, string message, string userName, string targetName, + PSCredentialTypes allowedCredentialTypes, PSCredentialUIOptions options) => PSCredential.Empty; + } + + private static string NewDir() => + Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-whatif-{Guid.NewGuid():N}")).FullName; + + /// Runs the export under -WhatIf and returns every line the gate wrote. + private static (List Lines, bool HadErrors) WhatIf(string output, string checkpoint) + { + var host = new RecordingHost(); + using var runspace = RunspaceFactory.CreateRunspace(host); + runspace.Open(); + using var ps = PowerShell.Create(); + ps.Runspace = runspace; + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", output) + .AddParameter("CheckpointPath", checkpoint) + .AddParameter("All") + .AddParameter("WhatIf", true); + ps.Invoke(); + return (host.Recorder.Lines, ps.HadErrors); + } + + private static void Export(string output, string checkpoint) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", output) + .AddParameter("CheckpointPath", checkpoint) + .AddParameter("All"); + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + } + + /// + /// The applying run, with its warnings and its terminating record. A run that stops on the + /// staging name gets there before the client, so this needs no transport either - which is + /// what lets one test put the two passes over one disk state side by side. + /// + private static (string[] Warnings, ErrorRecord[] Errors) ExportRun(string output, + string checkpoint) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", output) + .AddParameter("CheckpointPath", checkpoint) + .AddParameter("All"); + List errors = []; + try { ps.Invoke(); } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + return ([.. ps.Streams.Warning.Select(w => w.Message)], [.. errors]); + } + + /// + /// A checkpoint from a release that recorded neither the temp's name nor its length, which + /// is the one shape adoption exists for: a line count, the newest temp beside the output, + /// and no output at all. + /// + private static void SaveOrphanCheckpoint(string checkpoint, long items) => + File.WriteAllText(checkpoint, $$""" + {"resource":"https://graph.microsoft.com/v1.0/users?$top=999", + "nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + "itemsCollected":{{items}},"pageItemsAlreadyWritten":0} + """); + + private static void Mkfifo(string path) + { + using var mkfifo = System.Diagnostics.Process.Start("/usr/bin/mkfifo", [path]); + Assert.NotNull(mkfifo); + mkfifo.WaitForExit(); + Assert.Equal(0, mkfifo.ExitCode); + } + + /// + /// What a hard kill leaves partway through a fresh checkpointed export: the previous + /// export's output, this run's items in a temp beside it, and a checkpoint naming both. + /// A real run promotes the temp over the output and appends the rest; -WhatIf has to say + /// so and touch none of the three. It also runs above the client, so it needs no session. + /// + [Fact] + public void WhatIf_promotes_nothing_and_names_the_append_a_real_run_would_do() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + try + { + File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n{\"id\":\"old3\"}\n"); + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = Path.GetFileName(temp), + OutputFile = output, + DataLength = new FileInfo(temp).Length, + }.Save(checkpoint); + + var outputBefore = File.ReadAllBytes(output); + var tempBefore = File.ReadAllBytes(temp); + var checkpointBefore = File.ReadAllBytes(checkpoint); + + // No transport and no Get-MgContext: reaching the client would end the run here. + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf asked for a Graph connection"); + Assert.Contains(lines, l => l.Contains("Append JSONL data") && l.Contains(output)); + Assert.Equal(outputBefore, File.ReadAllBytes(output)); + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.False(File.Exists(output + ".adopt")); + + // And the run it described does exactly that. + var handler = new CountingHandler(); + using var transport = MgxTransportScope.Inject(handler); + Export(output, checkpoint); + + Assert.Equal( + ["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same gate over a checkpoint that is another export's. A real run refuses it and + /// exports from the beginning, so that is what -WhatIf has to report - and the checkpoint, + /// which the refusal used to delete, is still there afterwards. + /// + [Fact] + public void WhatIf_over_another_exports_checkpoint_names_an_export_and_deletes_nothing() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = Path.Combine(dir, "elsewhere.jsonl"), + DataLength = 12, + }.Save(checkpoint); + + var outputBefore = File.ReadAllBytes(output); + var checkpointBefore = File.ReadAllBytes(checkpoint); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf asked for a Graph connection"); + Assert.Contains(lines, l => l.Contains("Export JSONL data") && l.Contains(output)); + Assert.DoesNotContain(lines, l => l.Contains("Append JSONL data")); + Assert.Equal(outputBefore, File.ReadAllBytes(output)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same gate over a checkpoint whose temp another export is still writing. A real run + /// takes nothing from that file and exports from the beginning, so reporting an append + /// described a recovery that would not have happened - and the report is the only thing a + /// caller running -WhatIf gets to act on. + /// + [Fact] + public void WhatIf_over_a_temp_a_running_export_holds_names_an_export() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + try + { + File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n"); + + // Opened the way an export opens its own temp, with two rows flushed and the run + // that wrote them still going. + using var live = new StreamWriter(temp, append: false); + live.WriteLine("{\"id\":\"u1\"}"); + live.WriteLine("{\"id\":\"u2\"}"); + live.Flush(); + + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = Path.GetFileName(temp), + OutputFile = output, + DataLength = new FileInfo(temp).Length, + }.Save(checkpoint); + + var outputBefore = File.ReadAllBytes(output); + var tempBefore = ReadShared(temp); + var checkpointBefore = File.ReadAllBytes(checkpoint); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf asked for a Graph connection"); + Assert.Contains(lines, l => l.Contains("Export JSONL data") && l.Contains(output)); + Assert.DoesNotContain(lines, l => l.Contains("Append JSONL data")); + Assert.Equal(outputBefore, File.ReadAllBytes(output)); + Assert.Equal(tempBefore, ReadShared(temp)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.False(File.Exists(output + ".adopt")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And what the preview says about it. A real run over these two files stops, which is not + /// one of the two actions the ShouldProcess gate names against -OutputFile - the gate says + /// which of append and export a run would do, and this run would do neither - so the + /// sentence goes out beside it, on the stream every other checkpoint refusal is written to. + /// In the future tense, and it does not throw: -WhatIf reports what a run would do without + /// ending the pipeline on it, and the caller may have piped a dozen more commands behind + /// this one. + /// + [Fact] + public void WhatIf_over_a_held_temp_says_the_run_would_stop() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + try + { + File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n"); + + using var live = new StreamWriter(temp, append: false); + live.WriteLine("{\"id\":\"u1\"}"); + live.WriteLine("{\"id\":\"u2\"}"); + live.Flush(); + + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = Path.GetFileName(temp), + OutputFile = output, + DataLength = new FileInfo(temp).Length, + }.Save(checkpoint); + + var outputBefore = File.ReadAllBytes(output); + var tempBefore = ReadShared(temp); + var checkpointBefore = File.ReadAllBytes(checkpoint); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf ended the pipeline on a run it only described"); + Assert.Contains(lines, l => + l.Contains("Another export is still writing the temp file") + && l.Contains("This run would stop here; nothing would be written.")); + Assert.DoesNotContain(lines, l => l.Contains("nothing was written")); + Assert.Equal(outputBefore, File.ReadAllBytes(output)); + Assert.Equal(tempBefore, ReadShared(temp)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.Equal([temp], Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same preview one file over. A checkpoint that names no temp counts its items into the + /// output itself, and a real run over one another export still holds stops there - so the + /// gate has to say so, in the tense of a pass that does none of it. The claim it needs is + /// taken and let go here like every other question above the gate: the output, the position + /// and the directory are exactly what they were. + /// + [Fact] + public void WhatIf_over_a_held_output_says_the_run_would_stop() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + File.WriteAllLines(output, ["{\"id\":\"u1\"}", "{\"id\":\"u2\"}"]); + var recorded = new FileInfo(output).Length; + + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = null, + OutputFile = output, + DataLength = recorded, + }.Save(checkpoint); + + // The run that owns it is still going: appending, sharing reads, one row past the + // length its last save recorded. + using var live = new StreamWriter( + new FileStream(output, FileMode.Append, FileAccess.Write, FileShare.Read)); + live.WriteLine("{\"id\":\"u3-live\"}"); + live.Flush(); + + var outputBefore = ReadShared(output); + var checkpointBefore = File.ReadAllBytes(checkpoint); + var entriesBefore = Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray(); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf ended the pipeline on a run it only described"); + Assert.Contains(lines, l => + l.Contains($"Another export is still writing '{output}'") + && l.Contains("This run would stop here; nothing would be written.")); + Assert.DoesNotContain(lines, l => l.Contains("nothing was written")); + Assert.Equal(outputBefore, ReadShared(output)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.Equal(entriesBefore, Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray()); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// What -WhatIf leaves of the directory itself, and not only of the files in it. The + /// recovery the gate reports on compares file names, and which spellings a directory keeps + /// apart is a question answered by creating a file in it and looking for it under the other + /// casing - so a run that reported it would change nothing was asking it by writing in the + /// caller's own output directory. The answer is read off the directory's entries instead, + /// which A_directory_holding_a_lettered_entry_decides_its_case_rule_without_writing pins at + /// the helper, and the entry list here is what says no probe was written on the way through. + /// + /// The entry list and not the write time: the preview makes the create the promotion makes + /// at its own staging name, since that create is what a directory this account cannot write + /// in refuses and asking only whether something already stood there previewed a recovery + /// the run would have stopped on. An entry made and taken back moves the directory's write + /// time, and what the preview owes the caller is that nothing is left of it. + /// + /// The answer this process has cached for the directory is forgotten first, so the question + /// is actually reached here rather than answered from the run that set the scene up. The + /// directory holds three lettered entries by then, which is what a run may read the answer + /// off instead - and reading takes nothing from either. + /// + [Fact] + public void WhatIf_leaves_the_output_directorys_entries_alone() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + try + { + File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n{\"id\":\"old3\"}\n"); + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = Path.GetFileName(temp), + OutputFile = output, + DataLength = new FileInfo(temp).Length, + }.Save(checkpoint); + + MgxCmdletBase.SetDirectoryCaseSensitivity(dir, null); + + var namesBefore = Directory.GetFileSystemEntries(dir).OrderBy(n => n).ToArray(); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf asked for a Graph connection"); + Assert.Contains(lines, l => l.Contains("Append JSONL data") && l.Contains(output)); + Assert.Equal(namesBefore, Directory.GetFileSystemEntries(dir).OrderBy(n => n).ToArray()); + } + finally + { + MgxCmdletBase.SetDirectoryCaseSensitivity(dir, null); + try { Directory.Delete(dir, true); } catch { } + } + } + + // Reads a file the test's own live writer still holds open: on Windows the reader + // must offer ReadWrite sharing or the holder's write access denies the open. + private static byte[] ReadShared(string path) + { + using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + using var ms = new MemoryStream(); + fs.CopyTo(ms); + return ms.ToArray(); + } + + /// + /// The preview on the promotion route. A checkpoint naming a temp is recovered by replacing + /// the output with that temp's rows and appending onto them, and a real run over an output + /// another export holds stops there instead - so the gate has to say so. It never asked about + /// the output at all on this route, and named the append the run would not have got to. + /// + [Fact] + public void WhatIf_over_a_held_output_on_the_promotion_route_says_the_run_would_stop() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + try + { + File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n"); + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = Path.GetFileName(temp), + OutputFile = output, + DataLength = new FileInfo(temp).Length, + }.Save(checkpoint); + + // The export that has the output: appending, sharing reads. + using var live = new StreamWriter( + new FileStream(output, FileMode.Append, FileAccess.Write, FileShare.Read)); + live.WriteLine("{\"id\":\"old3-live\"}"); + live.Flush(); + + var outputBefore = ReadShared(output); + var tempBefore = File.ReadAllBytes(temp); + var checkpointBefore = File.ReadAllBytes(checkpoint); + var entriesBefore = Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray(); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf ended the pipeline on a run it only described"); + Assert.Contains(lines, l => + l.Contains($"Another export is still writing '{output}'") + && l.Contains("This run would stop here; nothing would be written.")); + Assert.DoesNotContain(lines, l => l.Contains("nothing was written")); + Assert.Equal(outputBefore, ReadShared(output)); + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.Equal(entriesBefore, Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray()); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The name the promotion stages its copy under, asked about too. A real run whose staging + /// name is another run's - or a directory - stops there with both files left as found, and + /// the gate could not see that at all: it named the recovery a run would have made if it had + /// got past a write it never would have made. + /// + [Fact] + public void WhatIf_over_a_held_staging_name_says_the_run_would_stop() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + var adopt = $"{output}.adopt"; + try + { + File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n"); + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = Path.GetFileName(temp), + OutputFile = output, + DataLength = new FileInfo(temp).Length, + }.Save(checkpoint); + + // The run that has the staging name, holding it the way a promotion holds the copy + // it has just written. + using var holder = new FileStream(adopt, FileMode.Create, FileAccess.ReadWrite, + FileShare.None); + holder.Write("{\"id\":\"theirs\"}\n"u8); + holder.Flush(); + var holderBytes = holder.Length; + + var outputBefore = File.ReadAllBytes(output); + var tempBefore = File.ReadAllBytes(temp); + var checkpointBefore = File.ReadAllBytes(checkpoint); + var entriesBefore = Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray(); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf ended the pipeline on a run it only described"); + Assert.Contains(lines, l => + l.Contains($"staging them into '{output}' failed at '{adopt}'") + && l.Contains("Nothing would be changed;")); + Assert.DoesNotContain(lines, l => l.Contains("Nothing was changed;")); + Assert.DoesNotContain(lines, l => l.Contains("Recovering")); + + Assert.Equal(outputBefore, File.ReadAllBytes(output)); + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.Equal(entriesBefore, Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray()); + Assert.Equal(holderBytes, holder.Length); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + private static void Chmod(string mode, string path) + { + using var chmod = System.Diagnostics.Process.Start("/bin/chmod", [mode, path]); + Assert.NotNull(chmod); + chmod.WaitForExit(); + Assert.Equal(0, chmod.ExitCode); + } + + /// + /// -WhatIf over an output directory this account may read and not write. Recovery stages a + /// copy beside the output, so a real run stops on a create it cannot make - and the gate + /// asked only whether something already stood at the staging name, found nothing, and named + /// the recovery the run would then have failed to make. It makes the create too, at no + /// length, so the preview says what the run would say. + /// + [Fact] + public void WhatIf_over_a_read_only_output_directory_says_the_run_would_stop() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + var adopt = $"{output}.adopt"; + try + { + File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n"); + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = Path.GetFileName(temp), + OutputFile = output, + DataLength = new FileInfo(temp).Length, + }.Save(checkpoint); + + var outputBefore = File.ReadAllBytes(output); + var tempBefore = File.ReadAllBytes(temp); + var checkpointBefore = File.ReadAllBytes(checkpoint); + var entriesBefore = Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray(); + Chmod("555", dir); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + Chmod("755", dir); + + Assert.False(hadErrors, "-WhatIf ended the pipeline on a run it only described"); + var stop = Assert.Single(lines.Where(l => + l.Contains($"staging them into '{output}' failed at '{adopt}'"))); + Assert.Contains("Nothing would be changed;", stop); + Assert.DoesNotContain("Nothing was changed;", stop); + Assert.DoesNotContain(lines, l => l.Contains("Recovering")); + + // The reason is quoted inside a sentence the caller punctuates, and the runtime's + // words for a refused create end in a period of their own. + Assert.DoesNotContain("..", stop.Replace(dir, "")); + + Assert.Equal(outputBefore, File.ReadAllBytes(output)); + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.Equal(entriesBefore, Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray()); + } + finally + { + try { Chmod("755", dir); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A FIFO at the staging name, under -WhatIf. The preview swept it, which made a deletion as + /// real as a run's on the one pass that is supposed to touch nothing: a caller asking what + /// would be changed had an entry beside their output removed in order to be told that + /// nothing would be. It is left standing now and named instead, in a sentence saying what + /// the run would take off the name and what it would put there - and the create test that + /// follows a clear name is skipped, since nothing at the name can be tested without taking + /// it off. + /// + [Fact] + public void WhatIf_leaves_a_fifo_at_the_staging_name_and_names_what_a_run_would_remove() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + var adopt = $"{output}.adopt"; + try + { + File.WriteAllText(output, "{\"id\":\"old1\"}\n{\"id\":\"old2\"}\n"); + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users?$top=999", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + PageItemsAlreadyWritten = 0, + TempFile = Path.GetFileName(temp), + OutputFile = output, + DataLength = new FileInfo(temp).Length, + }.Save(checkpoint); + + Mkfifo(adopt); + + var outputBefore = File.ReadAllBytes(output); + var tempBefore = File.ReadAllBytes(temp); + var checkpointBefore = File.ReadAllBytes(checkpoint); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf ended the pipeline on a run it only described"); + Assert.Contains(lines, l => l.Contains( + $"The run would remove what stands at the staging name '{Path.GetFileName(adopt)}' " + + "(an entry no copy could be staged in, such as a pipe or a socket) and stage " + + "its copy there.")); + Assert.Contains(lines, l => l.Contains("Append JSONL data") && l.Contains(output)); + Assert.DoesNotContain(lines, l => l.Contains("failed at")); + Assert.DoesNotContain(lines, l => l.Contains("Removed what stood at")); + + // The pipe is where the preview found it, and so is everything else. + Assert.True(File.Exists(adopt), "the preview removed what it said a run would"); + Assert.Equal(outputBefore, File.ReadAllBytes(output)); + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + } + finally + { + try { File.Delete(adopt); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A checkpoint from before the temp's name and length were recorded, with a directory at + /// the staging name. Adoption stages its copy under that name like a promotion does, and + /// the preview asked neither question: it reported the append the recovery would have + /// made, and the run then found the name unusable, said the output file was missing, + /// deleted the position counting the temp's items and exported from the beginning over it. + /// Both passes sweep and create there now, so the preview stops where the run stops and + /// says so in the run's own words. + /// + [Fact] + public void WhatIf_over_an_orphan_checkpoint_stopped_at_the_staging_name_says_the_run_would_stop() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + var adopt = $"{output}.adopt"; + try + { + // No output at all, which is the only state adoption runs in. + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + SaveOrphanCheckpoint(checkpoint, 2); + Directory.CreateDirectory(adopt); + File.WriteAllText(Path.Combine(adopt, "somebody-elses"), "keep\n"); + + var tempBefore = File.ReadAllBytes(temp); + var checkpointBefore = File.ReadAllBytes(checkpoint); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf ended the pipeline on a run it only described"); + var previewed = Assert.Single(lines.Where(l => + l.Contains($"records are in '{Path.GetFileName(temp)}', whole, but staging them " + + $"into '{output}' failed at '{adopt}': a directory stands at the " + + "path."))); + Assert.Contains("Nothing would be changed;", previewed); + Assert.DoesNotContain(lines, l => l.Contains("output file is missing")); + Assert.DoesNotContain(lines, l => l.Contains("Recovered")); + + // Nothing of the three moved, and the preview did not delete the position counting + // the temp's items. + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.False(File.Exists(output)); + Assert.Equal("keep\n", File.ReadAllText(Path.Combine(adopt, "somebody-elses"))); + + // And the run the preview described stops the same way, on the same sentence in the + // applying tense, with the checkpoint still there to come back to. + var (warnings, errors) = ExportRun(output, checkpoint); + + var stop = Assert.Single(errors); + Assert.StartsWith("CheckpointStagingFailed", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(ErrorCategory.WriteError, stop.CategoryInfo.Category); + Assert.Contains($"records are in '{Path.GetFileName(temp)}', whole, but staging them " + + $"into '{output}' failed at '{adopt}': a directory stands at the path.", + stop.Exception.Message); + Assert.Contains("Nothing was changed;", stop.Exception.Message); + Assert.DoesNotContain(warnings, w => w.Contains("output file is missing")); + + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.False(File.Exists(output)); + Assert.Equal("keep\n", File.ReadAllText(Path.Combine(adopt, "somebody-elses"))); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same checkpoint with a FIFO at the staging name, on the adoption route. The preview + /// names what the run would take off the name and leaves the pipe where it is; the run that + /// follows takes it off and adopts. Leaving it is not what wedges a later run - the run's + /// own pass clears it before it opens anything - and removing it was a deletion made by a + /// pass whose whole promise is that it makes none. + /// + [Fact] + public void WhatIf_over_an_orphan_checkpoint_names_the_fifo_and_the_run_that_follows_adopts() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + var adopt = $"{output}.adopt"; + var kind = "an entry no copy could be staged in, such as a pipe or a socket"; + var removed = $"Removed what stood at the staging name '{Path.GetFileName(adopt)}': " + + kind + "."; + var wouldRemove = "The run would remove what stands at the staging name " + + $"'{Path.GetFileName(adopt)}' ({kind}) and stage its copy there."; + try + { + File.WriteAllText(temp, "{\"id\":\"u1\"}\n{\"id\":\"u2\"}\n"); + SaveOrphanCheckpoint(checkpoint, 2); + Mkfifo(adopt); + + var tempBefore = File.ReadAllBytes(temp); + var checkpointBefore = File.ReadAllBytes(checkpoint); + + var (lines, hadErrors) = WhatIf(output, checkpoint); + + Assert.False(hadErrors, "-WhatIf ended the pipeline on a run it only described"); + Assert.Contains(lines, l => l.Contains(wouldRemove)); + Assert.Contains(lines, l => l.Contains("Append JSONL data") && l.Contains(output)); + Assert.DoesNotContain(lines, l => l.Contains("failed at")); + Assert.DoesNotContain(lines, l => l.Contains("Removed what stood at")); + + // The pipe is where the preview found it, and so are the two files the checkpoint + // stands for: the temp, and the output that is still not there. + Assert.True(File.Exists(adopt), "the preview removed what it said a run would"); + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.False(File.Exists(output)); + + // And the applying run over the state the preview left - the pipe still at the name + // - takes it off, warns, and adopts, on a thread with a watchdog: the failure this + // pins is a run that does not come back at all. + var handler = new CountingHandler(); + using var transport = MgxTransportScope.Inject(handler); + string[] warnings = []; + var running = new Thread(() => warnings = ExportRun(output, checkpoint).Warnings) + { IsBackground = true }; + running.Start(); + Assert.True(running.Join(TimeSpan.FromSeconds(30)), + "the run never returned: the staging open is waiting for a reader on the FIFO"); + + Assert.Contains(warnings, w => w.Contains(removed)); + Assert.Contains(warnings, w => w.Contains("Recovered 2 items")); + Assert.False(File.Exists(adopt)); + Assert.Equal( + ["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { File.Delete(adopt); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/GraphSdkBridgeTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/GraphSdkBridgeTests.cs deleted file mode 100644 index f751a36..0000000 --- a/tests/Mgx.IntegrationTests/Cmdlets/GraphSdkBridgeTests.cs +++ /dev/null @@ -1,497 +0,0 @@ -using System.Net; -using System.Reflection; -using System.Text; -using Mgx.Cmdlets.Base; -using Mgx.Cmdlets.Cmdlets.Configuration; -using Mgx.Engine.Http; -using Mgx.IntegrationTests.Engine; -using Mgx.IntegrationTests.Fakes; -using Mgx.IntegrationTests.Infrastructure; - -namespace Mgx.IntegrationTests; - -/// -/// The reflection bridge into the Graph SDK, driven through a stubbed GraphSession. -/// -[Collection(ResilienceCollection.Name)] -public class GraphSdkBridgeTests : IDisposable -{ - private readonly string? _modulePath = Environment.GetEnvironmentVariable("PSModulePath"); - - public GraphSdkBridgeTests() - { - // Enable-MgxResilience probes with Invoke-MgGraphRequest, which would auto-load a real - // Microsoft.Graph.Authentication from the developer machine and leave its GraphSession - // resolvable for every later test in the process - Environment.SetEnvironmentVariable("PSModulePath", string.Empty); - ResetStatics(); - } - - public void Dispose() - { - Environment.SetEnvironmentVariable("PSModulePath", _modulePath); - ResetStatics(); - GC.SuppressFinalize(this); - } - - private static void ResetStatics() - { - MgxCmdletBase.s_typeResolverForTests = null; - EnableMgxResilience.IsEnabled = false; - EnableMgxResilience.OriginalSdkClient = null; - EnableMgxResilience.ResilientSdkClient = null; - EnableMgxResilience.ActiveHandler = null; - MgxCmdletBase.ResetHttpClient(); - } - - private static HttpClient? BuiltClient() => (HttpClient?)typeof(MgxCmdletBase) - .GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.GetValue(null); - - private static StubHttpMessageHandler Ok() => - new StubHttpMessageHandler().Enqueue(_ => new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent("""{ "id": "u1" }""", Encoding.UTF8, "application/json") - }); - - // --- identity --- - - [Fact] - public void A_connected_session_produces_a_stable_fingerprint() - { - using var stub = new GraphSdkStub(); - - var first = MgxCmdletBase.BuildAuthFingerprint(stub.Session.AuthContext, "https://graph.microsoft.com"); - var second = MgxCmdletBase.BuildAuthFingerprint(stub.Session.AuthContext, "https://graph.microsoft.com"); - - Assert.NotEmpty(first); - Assert.Equal(first, second); - } - - [Fact] - public void A_different_tenant_produces_a_different_fingerprint() - { - using var stub = new GraphSdkStub(); - var context = (FakeAuthContext)stub.Session.AuthContext!; - var before = MgxCmdletBase.BuildAuthFingerprint(context, null); - - context.TenantId = "33333333-3333-3333-3333-333333333333"; - - Assert.NotEqual(before, MgxCmdletBase.BuildAuthFingerprint(context, null)); - } - - [Fact] - public void Scope_order_does_not_change_the_fingerprint() - { - var a = new FakeAuthContext { Scopes = ["User.Read.All", "Group.Read.All"] }; - var b = new FakeAuthContext { Scopes = ["Group.Read.All", "User.Read.All"] }; - - Assert.Equal(MgxCmdletBase.BuildAuthFingerprint(a, null), MgxCmdletBase.BuildAuthFingerprint(b, null)); - } - - [Fact] - public void The_session_endpoint_is_what_the_module_targets() - { - using var stub = new GraphSdkStub(); - ((FakeGraphEnvironment)stub.Session.Environment!).GraphEndpoint = "https://graph.microsoft.us"; - - Assert.Equal("https://graph.microsoft.us", MgxCmdletBase.GetGraphEndpoint(null, null)); - } - - [Fact] - public void A_session_without_an_instance_reports_no_endpoint() - { - using var stub = new GraphSdkStub(); - stub.ClearInstance(); - - Assert.Null(MgxCmdletBase.GetGraphEndpoint(null, null)); - } - - // --- client build --- - - [Fact] - public void A_connected_session_gets_an_auth_only_client_of_its_own() - { - using var stub = new GraphSdkStub(); - var warnings = new List(); - - MgxCmdletBase.TryPreInitHttpClient(warnings.Add, _ => { }); - - Assert.NotNull(BuiltClient()); - Assert.Empty(warnings); - } - - [Fact] - public void Without_the_sdk_auth_helpers_no_client_is_built() - { - using var stub = new GraphSdkStub(withAuthHelpers: false); - - MgxCmdletBase.TryPreInitHttpClient(_ => { }, _ => { }); - - Assert.Null(BuiltClient()); - } - - [Fact] - public void A_disconnected_session_builds_nothing() - { - using var stub = new GraphSdkStub(connected: false); - - MgxCmdletBase.TryPreInitHttpClient(_ => { }, _ => { }); - - Assert.Null(BuiltClient()); - } - - // --- Enable / Disable round trip --- - - [Fact] - public void Enable_wraps_the_sdk_client_and_Disable_puts_the_original_back() - { - using var stub = new GraphSdkStub(); - var original = new HttpClient { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; - stub.Session.GraphHttpClient = original; - using var host = new MgxTestHost(Ok()); - - var enable = host.Run(ps => ps.AddCommand("Enable-MgxResilience")); - - Assert.Null(enable.Terminating); - Assert.True(EnableMgxResilience.IsEnabled); - Assert.NotSame(original, stub.Session.GraphHttpClient); - Assert.Same(stub.Session.GraphHttpClient, EnableMgxResilience.ResilientSdkClient); - Assert.Equal(original.BaseAddress, stub.Session.GraphHttpClient!.BaseAddress); - - var disable = host.Run(ps => ps.AddCommand("Disable-MgxResilience")); - - Assert.Null(disable.Terminating); - Assert.False(EnableMgxResilience.IsEnabled); - Assert.Same(original, stub.Session.GraphHttpClient); - } - - [Fact] - public void Enabling_twice_leaves_the_first_wrap_in_place() - { - using var stub = new GraphSdkStub(); - stub.Session.GraphHttpClient = new HttpClient(); - using var host = new MgxTestHost(Ok()); - - host.Run(ps => ps.AddCommand("Enable-MgxResilience")); - var wrapped = stub.Session.GraphHttpClient; - host.Run(ps => ps.AddCommand("Enable-MgxResilience")); - - Assert.Same(wrapped, stub.Session.GraphHttpClient); - } - - [Fact] - public void Enable_under_WhatIf_leaves_the_sdk_client_alone() - { - using var stub = new GraphSdkStub(); - var original = new HttpClient(); - stub.Session.GraphHttpClient = original; - using var host = new MgxTestHost(Ok()); - - host.Run(ps => ps.AddCommand("Enable-MgxResilience").AddParameter("WhatIf")); - - Assert.False(EnableMgxResilience.IsEnabled); - Assert.Same(original, stub.Session.GraphHttpClient); - } - - [Fact] - public void Enable_without_a_graph_session_says_so() - { - using var stub = GraphSdkStub.WithoutTheSdk(); - using var host = new MgxTestHost(Ok()); - - var result = host.Run(ps => ps.AddCommand("Enable-MgxResilience")); - - Assert.NotNull(result.Terminating); - Assert.Equal("GraphSessionNotFound", result.Terminating!.FullyQualifiedErrorId.Split(',')[0]); - } - - [Fact] - public void Enable_with_the_session_torn_down_says_so() - { - using var stub = new GraphSdkStub(); - stub.ClearInstance(); - using var host = new MgxTestHost(Ok()); - - var result = host.Run(ps => ps.AddCommand("Enable-MgxResilience")); - - Assert.NotNull(result.Terminating); - Assert.Equal("GraphSessionNull", result.Terminating!.FullyQualifiedErrorId.Split(',')[0]); - } - - [Fact] - public void Enable_with_no_sdk_client_to_wrap_says_so() - { - using var stub = new GraphSdkStub(); - stub.Session.GraphHttpClient = null; - using var host = new MgxTestHost(Ok()); - - var result = host.Run(ps => ps.AddCommand("Enable-MgxResilience")); - - Assert.NotNull(result.Terminating); - Assert.Equal("HttpClientNotFound", result.Terminating!.FullyQualifiedErrorId.Split(',')[0]); - } - - [Fact] - public void Disable_before_Enable_only_warns() - { - using var stub = new GraphSdkStub(); - stub.Session.GraphHttpClient = new HttpClient(); - using var host = new MgxTestHost(Ok()); - - var result = host.Run(ps => ps.AddCommand("Disable-MgxResilience")); - - Assert.Null(result.Terminating); - Assert.Contains(result.Warnings, w => w.Contains("not currently enabled")); - } - - [Fact] - public void Disable_under_WhatIf_leaves_the_wrap_installed() - { - using var stub = new GraphSdkStub(); - stub.Session.GraphHttpClient = new HttpClient(); - using var host = new MgxTestHost(Ok()); - - host.Run(ps => ps.AddCommand("Enable-MgxResilience")); - var wrapped = stub.Session.GraphHttpClient; - - host.Run(ps => ps.AddCommand("Disable-MgxResilience").AddParameter("WhatIf")); - - Assert.True(EnableMgxResilience.IsEnabled); - Assert.Same(wrapped, stub.Session.GraphHttpClient); - } - - [Fact] - public void Disable_restores_the_original_even_when_something_else_swapped_the_client() - { - using var stub = new GraphSdkStub(); - var original = new HttpClient(); - stub.Session.GraphHttpClient = original; - using var host = new MgxTestHost(Ok()); - - host.Run(ps => ps.AddCommand("Enable-MgxResilience")); - var stranger = new HttpClient(); - stub.Session.GraphHttpClient = stranger; - - var result = host.Run(ps => ps.AddCommand("Disable-MgxResilience")); - - Assert.Contains(result.Warnings, w => w.Contains("not the one injected")); - Assert.Same(original, stub.Session.GraphHttpClient); - } - - [Fact] - public void Get_MgxResilience_reports_the_wrap_once_it_is_installed() - { - using var stub = new GraphSdkStub(); - stub.Session.GraphHttpClient = new HttpClient(); - using var host = new MgxTestHost(Ok()); - - host.Run(ps => ps.AddCommand("Enable-MgxResilience")); - var result = host.Run(ps => ps.AddCommand("Get-MgxResilience")); - - Assert.Equal(true, Assert.Single(result.Output).Properties["IsEnabled"].Value); - } - - // --- identity change --- - - [Fact] - public void A_new_identity_re_injects_the_wrap_against_the_new_sdk_client() - { - using var stub = new GraphSdkStub(); - stub.Session.GraphHttpClient = new HttpClient(); - using var host = new MgxTestHost(Ok()); - host.Run(ps => ps.AddCommand("Enable-MgxResilience")); - var firstWrap = EnableMgxResilience.ResilientSdkClient; - - // Connect-MgGraph replaces the SDK client along with the credential - var reconnected = new HttpClient(); - stub.Session.GraphHttpClient = reconnected; - var warnings = new List(); - var verbose = new List(); - - EnableMgxResilience.RefreshInjectedClient(warnings.Add, verbose.Add); - - Assert.True(EnableMgxResilience.IsEnabled); - Assert.NotSame(firstWrap, EnableMgxResilience.ResilientSdkClient); - Assert.Same(reconnected, EnableMgxResilience.OriginalSdkClient); - Assert.Same(EnableMgxResilience.ResilientSdkClient, stub.Session.GraphHttpClient); - } - - [Fact] - public void A_new_identity_that_kept_our_wrap_drops_it_and_asks_for_a_re_enable() - { - using var stub = new GraphSdkStub(); - stub.Session.GraphHttpClient = new HttpClient(); - using var host = new MgxTestHost(Ok()); - host.Run(ps => ps.AddCommand("Enable-MgxResilience")); - - var warnings = new List(); - EnableMgxResilience.RefreshInjectedClient(warnings.Add, _ => { }); - - Assert.False(EnableMgxResilience.IsEnabled); - Assert.Null(stub.Session.GraphHttpClient); - Assert.Contains(warnings, w => w.Contains("run Enable-MgxResilience again")); - } - - // --- the client the cmdlets actually run on --- - - private static MgxTestHost SdkBackedHost(ResilientGraphClientOptions? options = null) => - new(Ok(), "https://graph.microsoft.com", options, useTestTransport: false); - - [Fact] - public void A_request_runs_on_the_client_built_from_the_session() - { - using var stub = new GraphSdkStub(); - using var host = SdkBackedHost(); - - var result = host.Run(ps => ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users")); - - Assert.Null(result.Terminating); - Assert.Equal("https://graph.microsoft.com/v1.0/users", - Assert.Single(FakeAuthenticationHandler.Sent)); - Assert.NotNull(BuiltClient()); - } - - [Fact] - public void A_sovereign_endpoint_on_the_session_is_what_the_request_targets() - { - using var stub = new GraphSdkStub(); - ((FakeGraphEnvironment)stub.Session.Environment!).GraphEndpoint = "https://graph.microsoft.us"; - using var host = SdkBackedHost(); - - host.Run(ps => ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users")); - - Assert.StartsWith("https://graph.microsoft.us/", Assert.Single(FakeAuthenticationHandler.Sent), - StringComparison.Ordinal); - } - - [Fact] - public void A_second_request_on_the_same_identity_reuses_the_client() - { - using var stub = new GraphSdkStub(); - using var host = SdkBackedHost(); - - host.Run(ps => ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users")); - var first = BuiltClient(); - host.Run(ps => ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/groups")); - - Assert.Same(first, BuiltClient()); - } - - [Fact] - public void A_reconnect_under_a_different_tenant_rebuilds_the_client() - { - using var stub = new GraphSdkStub(); - using var host = SdkBackedHost(); - - host.Run(ps => ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users")); - var first = BuiltClient(); - - stub.Session.AuthContext = new FakeAuthContext { TenantId = "44444444-4444-4444-4444-444444444444" }; - var result = host.Run(ps => ps.AddCommand("Invoke-MgxRequest") - .AddParameter("Uri", "/users") - .AddParameter("Verbose")); - - Assert.NotSame(first, BuiltClient()); - Assert.Contains(result.Verbose, v => v.Contains("Graph identity changed")); - } - - [Fact] - public void A_new_total_timeout_rebuilds_the_client_it_is_baked_into() - { - using var stub = new GraphSdkStub(); - using var host = SdkBackedHost(); - - host.Run(ps => ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users")); - var first = BuiltClient(); - - host.Run(ps => ps.AddCommand("Set-MgxOption").AddParameter("TotalTimeoutSeconds", 45)); - var result = host.Run(ps => ps.AddCommand("Invoke-MgxRequest") - .AddParameter("Uri", "/users") - .AddParameter("Verbose")); - - Assert.NotSame(first, BuiltClient()); - Assert.Equal(TimeSpan.FromSeconds(105), BuiltClient()!.Timeout); - Assert.Contains(result.Verbose, v => v.Contains("TotalTimeoutSeconds changed")); - } - - [Fact] - public void A_disconnected_session_stops_the_cmdlet_before_any_request() - { - using var stub = new GraphSdkStub(connected: false); - using var host = SdkBackedHost(); - - var result = host.Run(ps => ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users")); - - Assert.NotNull(result.Terminating); - Assert.Equal("NotConnected", result.Terminating!.FullyQualifiedErrorId.Split(',')[0]); - Assert.Empty(FakeAuthenticationHandler.Sent); - } - - [Fact] - public void No_graph_module_at_all_names_the_install_step_instead_of_the_connect_step() - { - using var stub = GraphSdkStub.WithoutTheSdk(); - using var host = SdkBackedHost(); - - var result = host.Run(ps => ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users")); - - Assert.Equal("GraphAuthModuleNotLoaded", result.Terminating!.FullyQualifiedErrorId.Split(',')[0]); - } - - [Fact] - public void A_session_the_module_cannot_build_a_client_from_falls_back_and_then_gives_up() - { - using var stub = new GraphSdkStub(withAuthHelpers: false); - using var host = SdkBackedHost(); - - var result = host.Run(ps => ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users")); - - Assert.Contains(result.Warnings, w => w.Contains("Falling back to SDK client")); - Assert.Equal("HttpClientInitFailed", result.Terminating!.FullyQualifiedErrorId.Split(',')[0]); - } - - [Fact] - public void A_session_the_module_cannot_build_from_borrows_the_sdk_client_when_there_is_one() - { - using var stub = new GraphSdkStub(withAuthHelpers: false); - var borrowed = new HttpClient(new FakeAuthenticationHandler(new object(), new HttpClientHandler())); - stub.Session.GraphHttpClient = borrowed; - using var host = SdkBackedHost(); - - var result = host.Run(ps => ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users")); - - Assert.Null(result.Terminating); - Assert.Same(borrowed, BuiltClient()); - } - - [Fact] - public void The_sdk_client_going_stale_under_a_borrowed_transport_rebuilds_it() - { - using var stub = new GraphSdkStub(withAuthHelpers: false); - stub.Session.GraphHttpClient = - new HttpClient(new FakeAuthenticationHandler(new object(), new HttpClientHandler())); - using var host = SdkBackedHost(); - - host.Run(ps => ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users")); - - var replaced = new HttpClient(new FakeAuthenticationHandler(new object(), new HttpClientHandler())); - stub.Session.GraphHttpClient = replaced; - host.Run(ps => ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users")); - - Assert.Same(replaced, BuiltClient()); - } - - [Fact] - public void The_sdk_retry_handler_is_disarmed_inside_the_wrap() - { - // The resolver only sees assemblies the process has already loaded - _ = new Microsoft.Kiota.Http.HttpClientLibrary.Middleware.Options.RetryHandlerOption(); - using var stub = new GraphSdkStub(); - - var options = EnableMgxResilience.BuildInnerRetryOverride(); - - var option = Assert.Single(options!).Value; - Assert.NotNull(option); - Assert.Equal(0, option!.GetType().GetProperty("MaxRetry")!.GetValue(option)); - } -} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/HeaderWireTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/HeaderWireTests.cs new file mode 100644 index 0000000..6147360 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/HeaderWireTests.cs @@ -0,0 +1,37 @@ +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// Header values from a -Headers hashtable reach the wire as HTTP header text. +/// +[Collection("Pipeline")] +public class HeaderWireTests +{ + [Fact] + public void An_array_header_value_joins_as_an_http_list_not_a_type_name() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + using (MgxTransportScope.Inject(wire)) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("Invoke-MgxRequest -Uri /users/u1 -Headers @{ Prefer = @('outlook.timezone=\"UTC\"', 'IdType=\"ImmutableId\"') }"); + ps.Invoke(); + + var captured = wire.CapturedRequests; + Assert.NotEmpty(captured); + var prefer = Assert.Contains("Prefer", captured[0].Headers); + Assert.Equal("outlook.timezone=\"UTC\", IdType=\"ImmutableId\"", string.Join(", ", prefer)); + Assert.DoesNotContain("System.", string.Join(",", prefer)); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/InvokeMgxRequestCollectionTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/InvokeMgxRequestCollectionTests.cs index 203a688..11c2ad5 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/InvokeMgxRequestCollectionTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/InvokeMgxRequestCollectionTests.cs @@ -1,4 +1,4 @@ -using System.Collections; +using System.Collections; using System.Management.Automation; using System.Net; using System.Text; @@ -51,6 +51,61 @@ public void Raw_emits_the_json_text_rather_than_a_hashtable() Assert.Contains("\"id\"", Assert.IsType(Assert.Single(result.Output).BaseObject)); } + [Fact] + public void Envelope_keeps_a_one_item_collection_a_collection() + { + using var host = new MgxTestHost(Json(Page("u1"))); + + var result = host.Run(ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users?$filter=id eq 'u1'") + .AddParameter("Envelope")); + + var payload = (Hashtable)Assert.Single(result.Output).BaseObject; + var items = Assert.IsAssignableFrom(payload["value"]).Cast().ToList(); + Assert.Equal("u1", (string)((Hashtable)Assert.Single(items))["id"]!); + } + + [Fact] + public void Envelope_leaves_an_entity_payload_as_it_is() + { + using var host = new MgxTestHost(Json("""{ "id": "u1" }""")); + + var result = host.Run(ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users/u1") + .AddParameter("Envelope")); + + Assert.Equal("u1", Id(Assert.Single(result.Output))); + } + + [Fact] + public void Envelope_holds_every_item_and_still_strips_the_transport_metadata() + { + using var host = new MgxTestHost(Json( + """{ "@odata.count": 7, "value": [ { "id": "u1" }, { "id": "u2" } ] }""")); + + var result = host.Run(ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("Envelope")); + + var payload = (Hashtable)Assert.Single(result.Output).BaseObject; + Assert.Equal(2, Assert.IsAssignableFrom(payload["value"]).Cast().Count()); + Assert.False(payload.ContainsKey("@odata.count")); + } + + [Fact] + public void Envelope_with_raw_emits_the_payload_whole() + { + using var host = new MgxTestHost(Json(Page("u1"))); + + var result = host.Run(ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("Envelope") + .AddParameter("Raw")); + + var text = Assert.IsType(Assert.Single(result.Output).BaseObject); + Assert.Contains("\"value\"", text); + Assert.Contains("\"u1\"", text); + } [Fact] public void A_collection_payload_from_an_entity_request_is_unwrapped_and_flagged_as_truncated() { diff --git a/tests/Mgx.IntegrationTests/Cmdlets/NamedTempPromotionOwnershipTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/NamedTempPromotionOwnershipTests.cs new file mode 100644 index 0000000..2dc2473 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/NamedTempPromotionOwnershipTests.cs @@ -0,0 +1,775 @@ +using System.Management.Automation; +using System.Net; +using System.Text; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// Recovery from a checkpoint that NAMES its temp - every checkpoint this release writes. +/// The temp is taken by name rather than by the glob adoption searches with, which says which +/// file holds the counted items but nothing at all about whether anyone is still writing it: +/// a run whose export dies partway leaves that state on disk, and so does one that is still +/// going. Promotion copied the rows into its own output and unlinked the file underneath the +/// run that was filling it, which then wrote into an unlinked inode and failed its own +/// promotion naming the OUTPUT. A scheduled export overlapping itself reaches this with +/// nothing misconfigured. +/// +[Collection("Pipeline")] +public class NamedTempPromotionOwnershipTests +{ + private const string Page1 = """ + {"value":[{"id":"u1"},{"id":"u2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2"} + """; + private const string Page2 = """ + {"value":[{"id":"u3"}]} + """; + private const string Baseline = """ + {"value":[{"id":"a1"},{"id":"a2"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D1"} + """; + private const string ChangesPage1 = """ + {"value":[{"id":"b1"},{"id":"b2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users/delta?$skiptoken=B2"} + """; + private const string ChangesPage2 = """ + {"value":[{"id":"b3"}],"@odata.deltaLink":"https://graph.microsoft.com/v1.0/users/delta?$deltatoken=D2"} + """; + private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + + /// + /// Answers what it was queued with, and 500s once the queue is empty - so a run dying on + /// the wire needs no assumption about how many times the client retries first. + /// + private sealed class ScriptedHandler : HttpMessageHandler + { + private readonly Queue<(HttpStatusCode Status, string Body)> _steps = new(); + private readonly object _lock = new(); + + public void Queue(HttpStatusCode status, string body) + { + lock (_lock) _steps.Enqueue((status, body)); + } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + (HttpStatusCode Status, string Body) step; + lock (_lock) + step = _steps.Count > 0 ? _steps.Dequeue() : (HttpStatusCode.InternalServerError, ServerError); + return Task.FromResult(new HttpResponseMessage(step.Status) + { + RequestMessage = request, + Content = new StringContent(step.Body, Encoding.UTF8, "application/json") + }); + } + } + + private static string NewDir() => + Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-promote-owner-{Guid.NewGuid():N}")).FullName; + + /// + /// What a run leaves on the streams. Errors carries the terminating record a run that will + /// not touch a checkpoint's held or unopenable temp ends on, read from both places + /// PowerShell can put it. + /// + private sealed record RunResult(List Warnings, List Output, + List Errors); + + private static PowerShell Shell(bool withContext = false) + { + var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + if (withContext) + { + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + } + return ps; + } + + private static RunResult Export(string output, string checkpoint) + { + using var ps = Shell(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", output) + .AddParameter("CheckpointPath", checkpoint) + .AddParameter("All"); + // A run that dies is the subject here, so a terminating error is an expected outcome. + List results = []; + List errors = []; + try { results = [.. ps.Invoke()]; } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + return new RunResult([.. ps.Streams.Warning.Select(w => w.Message)], results, errors); + } + + /// + /// The same export with the verbose stream captured. What the stale-temp sweep left alone, + /// and why, is said there and nowhere else. + /// + private static List ExportVerbose(string output, string checkpoint) + { + using var ps = Shell(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", output) + .AddParameter("CheckpointPath", checkpoint) + .AddParameter("All") + .AddParameter("Verbose", true); + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + return [.. ps.Streams.Verbose.Select(v => v.Message)]; + } + + private static RunResult Sync(string deltaPath, string checkpoint, string output) + { + using var ps = Shell(withContext: true); + ps.AddCommand("Sync-MgxDelta") + .AddParameter("Uri", "/users/delta") + .AddParameter("DeltaPath", deltaPath) + .AddParameter("CheckpointPath", checkpoint) + .AddParameter("OutputFile", output); + List results = []; + List errors = []; + try { results = [.. ps.Invoke()]; } + catch (CmdletInvocationException ex) { errors.Add(ex.ErrorRecord); } + errors.AddRange(ps.Streams.Error); + return new RunResult([.. ps.Streams.Warning.Select(w => w.Message)], results, errors); + } + + /// + /// Reopens a temp the way the cmdlet opens its own - new StreamWriter(path, append: false), + /// which asks for FileShare.Read - with the bytes the checkpoint counted already in it. + /// This is the run that has not died: it is between pages, and its next flush is still to + /// come. + /// + private static StreamWriter HoldLikeALiveRun(string tempPath) + { + var counted = File.ReadAllText(tempPath); + var writer = new StreamWriter(tempPath, append: false); + writer.Write(counted); + writer.Flush(); + return writer; + } + + /// + /// The reproduction. A run dies after its first page, leaving a temp and a checkpoint + /// naming it; the same export is running again against that -OutputFile before the first + /// one has finished. The second run must not take the file the first is writing: it copied + /// those rows into its own output, unlinked the temp, and the first run's closing move then + /// failed with a FileNotFoundException reported against the output it never got to write. + /// + /// Nor may it export around them, which is what taking them looks like a page later: the + /// output it would replace is the one the checkpoint records, and the position it would save + /// goes to the same path. It stops, and says which of the two files to deal with. + /// + [Fact] + public void An_export_does_not_promote_a_temp_a_running_export_still_holds() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // The first run: one page collected, then the wire dies. A real temp, and a real + // checkpoint naming it with the flushed length. + handler.Queue(HttpStatusCode.OK, Page1); + Export(output, checkpoint); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.False(File.Exists(output)); + Assert.Equal(2, PaginationCheckpoint.Load(checkpoint)!.ItemsCollected); + + // It is not dead. It still holds that temp, with its two rows in it. + using var live = HoldLikeALiveRun(temp); + + // The second run, same -OutputFile and same -CheckpointPath, with the whole + // collection waiting for it on the wire. + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + var second = Export(output, checkpoint); + + Assert.True(File.Exists(temp), "a running export's temp was unlinked under it"); + Assert.DoesNotContain(second.Warnings, w => w.Contains("Recovered")); + var stop = Assert.Single(second.Errors); + Assert.StartsWith("CheckpointTempHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Contains("Another export is still writing the temp file", stop.Exception.Message); + Assert.Contains(checkpoint, stop.Exception.Message); + Assert.Contains("This run stops here; nothing was written.", stop.Exception.Message); + Assert.DoesNotContain("cannot be opened for writing by this account", + stop.Exception.Message); + + // And it published no output of its own: the file at -OutputFile is still the one + // the checkpoint counts bytes of, which here is no file at all. + Assert.False(File.Exists(output), "the stopped run published an output"); + + // And the first run goes on, undisturbed: the page it fetched since its last save + // lands, and the move it ends with finds the file it has been writing all along. + live.WriteLine("{\"id\":\"u3-live\"}"); + live.Flush(); + Assert.True(File.Exists(temp)); + live.Dispose(); + File.Move(temp, output, overwrite: true); + Assert.Equal( + ["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3-live\"}"], + File.ReadAllLines(output)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// What the refusal leaves behind. A temp that is missing or short means the counted items + /// are in no file, and the checkpoint describing them is deleted on the way to exporting + /// again - but a temp another run is writing is the position that run comes back to, and + /// deleting it is the same mistake as unlinking the temp, one file over. Both files are + /// still there, and still say exactly what they said. + /// + [Fact] + public void A_refused_promotion_leaves_the_running_exports_checkpoint_and_temp_alone() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + handler.Queue(HttpStatusCode.OK, Page1); + Export(output, checkpoint); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + + var checkpointBefore = File.ReadAllBytes(checkpoint); + var tempBefore = File.ReadAllBytes(temp); + + using (HoldLikeALiveRun(temp)) + { + // The second run gets nothing off the wire, so it never saves a checkpoint of + // its own: whatever is on disk afterwards is what the refusal left. + Export(output, checkpoint); + + Assert.Equal(checkpointBefore, File.ReadAllBytes(checkpoint)); + Assert.Equal(tempBefore, ReadShared(temp)); + Assert.False(File.Exists(output), "the other run's rows were published as an output"); + Assert.False(File.Exists(output + ".adopt")); + Assert.Equal([temp], Directory.GetFiles(dir, "out.jsonl.*.tmp")); + } + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The ordinary resume, which is the same state on disk minus the holder: the run that + /// wrote the temp is gone, so its items are recovered into the output and the enumeration + /// carries on from the page the checkpoint records. A claim that cannot tell the two apart + /// would cost every resume this release writes. + /// + [Fact] + public void An_export_promotes_the_named_temp_of_a_run_that_is_gone() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + handler.Queue(HttpStatusCode.OK, Page1); + Export(output, checkpoint); + Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + + handler.Queue(HttpStatusCode.OK, Page2); + var resumed = Export(output, checkpoint); + + Assert.Contains(resumed.Warnings, w => + w.Contains("Recovered 2 items from an interrupted export's temp file")); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(3L, Assert.IsType( + Assert.Single(resumed.Output).BaseObject).ItemCount); + Assert.Empty(Directory.GetFiles(dir, "*.tmp")); + Assert.False(File.Exists(checkpoint), "a completed export deletes its checkpoint"); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// Where the claim must not go. A checkpoint written before the output file was recorded is + /// diagnosed as this export's by the temp it names corroborating it - the same resolution + /// the promotion below uses - so a claim taken during that reading would answer "this names + /// no file of mine" for a temp that is merely open at the moment, and a settled ownership + /// diagnosis would swing on which second the export started in. The checkpoint is still + /// this export's; it is the promotion that refuses. + /// + [Fact] + public void A_held_temp_does_not_change_who_a_checkpoint_without_an_output_belongs_to() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + try + { + using var live = new StreamWriter(temp, append: false); + live.WriteLine("{\"id\":\"u1\"}"); + live.WriteLine("{\"id\":\"u2\"}"); + live.Flush(); + + // No "outputFile" field: the shape a release that did not record one leaves. + File.WriteAllText(checkpoint, $$""" + {"resource":"https://graph.microsoft.com/v1.0/users?$top=999", + "nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + "itemsCollected":2,"pageItemsAlreadyWritten":0, + "tempFile":"{{Path.GetFileName(temp)}}", + "dataLength":{{new FileInfo(temp).Length}}} + """); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + var run = Export(output, checkpoint); + + Assert.StartsWith("CheckpointTempHeld", Assert.Single(run.Errors).FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.DoesNotContain(run.Warnings, w => w.Contains("no longer corroborate it")); + Assert.True(File.Exists(temp), "a running export's temp was unlinked under it"); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The same helper, reached the same way, from the other cmdlet. A sync has one thing more + /// to lose: taking the live run's temp and then resuming past its items advances the delta + /// token over changes that are in no file this run can produce, and a delta token that has + /// moved cannot be asked for them again. Re-enumerating instead moves it just the same, one + /// completed page later, so the run ends before either. + /// + [Fact] + public void A_sync_does_not_promote_a_temp_a_running_sync_still_holds() + { + var dir = NewDir(); + var delta = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // A baseline sync, then one that dies after its first page of changes. + handler.Queue(HttpStatusCode.OK, Baseline); + Sync(delta, checkpoint, output); + handler.Queue(HttpStatusCode.OK, ChangesPage1); + Sync(delta, checkpoint, output); + var temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + Assert.Equal(["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], File.ReadAllLines(temp)); + + using var live = HoldLikeALiveRun(temp); + + // The overlapping sync, which re-enumerates from the delta token instead. + handler.Queue(HttpStatusCode.OK, ChangesPage1); + handler.Queue(HttpStatusCode.OK, ChangesPage2); + var second = Sync(delta, checkpoint, output); + + Assert.True(File.Exists(temp), "a running sync's temp was unlinked under it"); + Assert.DoesNotContain(second.Warnings, w => w.Contains("Recovered")); + var stop = Assert.Single(second.Errors); + Assert.StartsWith("CheckpointTempHeld", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Contains("Another sync is still writing the temp file", stop.Exception.Message); + Assert.Contains("advance the delta token past them", stop.Exception.Message); + Assert.DoesNotContain("cannot be opened for writing by this account", + stop.Exception.Message); + + // Nothing of the second run's is on disk: the output is still the baseline's, and + // the token is still the one the interrupted run was working from. + Assert.Equal(["{\"id\":\"a1\"}", "{\"id\":\"a2\"}"], File.ReadAllLines(output)); + + live.WriteLine("{\"id\":\"b3-live\"}"); + live.Flush(); + live.Dispose(); + Assert.Equal( + ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}", "{\"id\":\"b3-live\"}"], + File.ReadAllLines(temp)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// Takes write access away from a file the way each platform expresses it. Unix mode bits + /// and a Windows deny entry are different mechanisms and neither exists on the other host, + /// so the choice is made per OS and each branch executes on the host it is for. + /// + /// The read-only attribute refuses the open the same way and is not the form used here, + /// because it refuses the unlink as well: one of these cases is a sweep that takes the name + /// back, and under the attribute the copy outlived it. A deny entry for the account the + /// tests run as refuses reading and writing the file and says nothing about deleting it, + /// which is the directory's to grant - so the entry the run meets is a file it cannot open + /// and can still take off the name, which is what mode 000 is on the other host. + /// + /// + private static void MakeUnwritable(string path) + { + if (OperatingSystem.IsWindows()) + { + // The premise itself, so a run that then opened the file would be measuring nothing. + Assert.Equal(0, Icacls(path, "/deny", $"{Environment.UserName}:(R,W)")); + } + else + { + File.SetUnixFileMode(path, + UnixFileMode.UserRead | UnixFileMode.GroupRead | UnixFileMode.OtherRead); + } + } + + /// Puts it back, so the directory can be removed on either platform. + private static void MakeWritable(string path) + { + try + { + if (!File.Exists(path)) return; + if (OperatingSystem.IsWindows()) + Icacls(path, "/remove:d", Environment.UserName); + else + File.SetUnixFileMode(path, UnixFileMode.UserRead | UnixFileMode.UserWrite); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException + or System.ComponentModel.Win32Exception) { } + } + + /// + /// One icacls call over the file, waited for, with its exit code for the caller to read. + /// .NET's own access-control types are Windows-only and this assembly is built for a target + /// that has none of them, so the platform's own command is what edits an entry here. + /// + private static int Icacls(string path, params string[] arguments) + { + var start = new System.Diagnostics.ProcessStartInfo("icacls") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + }; + start.ArgumentList.Add(path); + foreach (var argument in arguments) start.ArgumentList.Add(argument); + using var icacls = System.Diagnostics.Process.Start(start); + Assert.NotNull(icacls); + icacls.WaitForExit(); + return icacls.ExitCode; + } + + /// + /// Whether this account can still open the file read-write after that. CI runs as an + /// ordinary user and cannot, which is the case these tests are about; a root shell opens a + /// 0444 file and reaches the ordinary recovery instead. Both answers are asserted, so the + /// test executes and counts whichever host it lands on rather than being skipped on one. + /// + private static bool StillOpenableReadWrite(string path) + { + try + { + using var _ = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite); + return true; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { return false; } + } + + /// + /// The other way the exclusive claim fails, which the refusal used to report as the first. + /// A temp this account cannot open read-write - mode bits, an ACL - stops the run for the + /// same reason and leaves the same two files behind, but there is no second export anywhere: + /// the sentence saying another run was still writing it sent the caller looking for + /// something that does not exist instead of at the file's permissions. + /// + [Fact] + public void An_export_names_a_temp_it_cannot_open_as_a_permissions_problem() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + string? temp = null; + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + handler.Queue(HttpStatusCode.OK, Page1); + Export(output, checkpoint); + temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + + MakeUnwritable(temp); + var openable = StillOpenableReadWrite(temp); + + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + var second = Export(output, checkpoint); + + if (openable) + { + // Running as root: the claim succeeds and this is the ordinary resume. + Assert.Contains(second.Warnings, w => + w.Contains("Recovered 2 items from an interrupted export's temp file")); + } + else + { + var stop = Assert.Single(second.Errors); + Assert.StartsWith("CheckpointTempUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Contains("cannot be opened for writing by this account", + stop.Exception.Message); + Assert.Contains(Path.GetFileName(temp), stop.Exception.Message); + Assert.Contains("Grant write access to that file and run again", + stop.Exception.Message); + Assert.DoesNotContain("Another export is still writing the temp file", + stop.Exception.Message); + Assert.True(File.Exists(temp), "the refused temp was swept by the run that stopped"); + Assert.False(File.Exists(output), "the stopped run published an output"); + } + } + finally + { + if (temp != null) MakeWritable(temp); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same distinction from the other cmdlet, which reaches the same claim. A sync stops + /// either way; what it can honestly say about the cause is what differs. + /// + [Fact] + public void A_sync_names_a_temp_it_cannot_open_as_a_permissions_problem() + { + var dir = NewDir(); + var delta = Path.Combine(dir, "state.json"); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + string? temp = null; + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + handler.Queue(HttpStatusCode.OK, Baseline); + Sync(delta, checkpoint, output); + handler.Queue(HttpStatusCode.OK, ChangesPage1); + Sync(delta, checkpoint, output); + temp = Assert.Single(Directory.GetFiles(dir, "out.jsonl.*.tmp")); + + MakeUnwritable(temp); + var openable = StillOpenableReadWrite(temp); + + handler.Queue(HttpStatusCode.OK, ChangesPage1); + handler.Queue(HttpStatusCode.OK, ChangesPage2); + var second = Sync(delta, checkpoint, output); + + if (openable) + { + Assert.Contains(second.Warnings, w => + w.Contains("Recovered 2 items from an interrupted sync's temp file")); + } + else + { + var stop = Assert.Single(second.Errors); + Assert.StartsWith("CheckpointTempUnopenable", stop.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Contains("cannot be opened for writing by this account", + stop.Exception.Message); + Assert.Contains(Path.GetFileName(temp), stop.Exception.Message); + Assert.Contains("Grant write access to that file and run again", + stop.Exception.Message); + Assert.DoesNotContain("Another sync is still writing the temp file", + stop.Exception.Message); + Assert.True(File.Exists(temp), "the refused temp was swept by the run that stopped"); + } + } + finally + { + if (temp != null) MakeWritable(temp); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The sweep's own account of a temp it passed over. It asks for the file exclusively and + /// leaves what it cannot claim, which is right for both ways the claim fails and honest + /// about only one of them: an orphan this account cannot open read-write has no second run + /// behind it, and calling it one sent the caller looking for an export that is not there. + /// + /// Which of the second kind it is is the open's own answer and not the line's to guess. Said + /// as "the open was refused on permissions, not on sharing", the same sentence went out for + /// a directory, for a pipe with no length to write at and for a path too long for the + /// volume - and a caller who went and granted write access to the file found nothing had + /// changed. The reason the open gave is quoted instead. + /// + /// No checkpoint here, so no resume is pending and the sweep runs over every temp beside + /// the output. The file is left where it is either way; only the sentence differs. + /// + [Fact] + public void The_sweep_names_a_temp_it_cannot_open_by_the_reason_the_open_gave() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var orphan = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + try + { + File.WriteAllLines(orphan, ["{\"id\":\"ORPHAN-1\"}"]); + MakeUnwritable(orphan); + var openable = StillOpenableReadWrite(orphan); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + + var verbose = ExportVerbose(output, checkpoint); + + if (openable) + { + // Running as root: the claim succeeds and the file is an ordinary orphan. + Assert.Contains(verbose, v => + v.Contains("Deleted an orphaned temp file") + && v.Contains(Path.GetFileName(orphan))); + } + else + { + var line = Assert.Single(verbose, v => v.Contains(Path.GetFileName(orphan))); + Assert.Contains("this run cannot open it for writing - ", line); + Assert.DoesNotContain("not on sharing", line); + Assert.DoesNotContain("another run is writing to it", line); + // The reason the open gave, quoted, and it names the file it was refused on. + Assert.Contains(orphan, line[line.IndexOf(" - ", StringComparison.Ordinal)..]); + Assert.True(File.Exists(orphan), "the sweep deleted a temp it could not claim"); + } + + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + MakeWritable(orphan); + try { Directory.Delete(dir, true); } catch { } + } + } + + // Reads a file the test's own live writer still holds open: on Windows the reader + // must offer ReadWrite sharing or the holder's write access denies the open. + private static byte[] ReadShared(string path) + { + using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + using var ms = new MemoryStream(); + fs.CopyTo(ms); + return ms.ToArray(); + } + + /// + /// A FIFO at , which .NET has no call for. + /// + private static void Mkfifo(string path) + { + using var mkfifo = System.Diagnostics.Process.Start("/usr/bin/mkfifo", [path]); + Assert.NotNull(mkfifo); + mkfifo.WaitForExit(); + Assert.Equal(0, mkfifo.ExitCode); + } + + /// + /// The staging name in the stale-temp sweep, which reached it with the claim-then-delete a + /// temp takes. That claim answers for a regular file and for nothing else: a pipe and a + /// socket came back unopenable and were left standing - under a sentence calling them a file + /// this account could not open - and a dangling link was left as well, so the next + /// promotion met exactly the entry its own sweep exists to clear. Both go now, through that + /// sweep, and the run says which kind it took. + /// + [Theory] + [InlineData("pipe")] + [InlineData("dangling link")] + public void The_sweep_clears_what_stands_at_the_staging_name(string kind) + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var adopt = $"{output}.adopt"; + try + { + if (kind == "pipe") Mkfifo(adopt); + else File.CreateSymbolicLink(adopt, Path.Combine(dir, "nothing-is-here")); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + + var verbose = ExportVerbose(output, Path.Combine(dir, "run.checkpoint")); + + Assert.Contains(verbose, v => v.Contains( + $"Removed what stood at the staging name '{Path.GetFileName(adopt)}': " + + (kind == "pipe" + ? "an entry no copy could be staged in, such as a pipe or a socket." + : "a link, and not what it pointed at."), StringComparison.Ordinal)); + // File.Exists answers for a pipe and for a dangling link alike on this platform, so + // it is what says the entry itself is gone rather than what it pointed at. + Assert.False(File.Exists(adopt), "the entry outlived the sweep"); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + try { File.Delete(adopt); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// What the sweep of the staging name calls a regular file it cannot open. The only kinds it + /// told apart were a link and a copy nothing held, so a 0444 or a mode-000 file - the claim + /// refused, the unlink allowed by the directory it sits in - was reported as "an entry no + /// copy could be staged in, such as a pipe or a socket": a kind of entry it is not, and one + /// the caller cannot have made by the means they actually used. + /// + [Fact] + public void The_sweep_calls_a_staged_copy_it_cannot_open_what_it_is() + { + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + var adopt = $"{output}.adopt"; + try + { + File.WriteAllText(adopt, "{\"id\":\"half-staged\"}\n"); + MakeUnwritable(adopt); + var openable = StillOpenableReadWrite(adopt); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + + var verbose = ExportVerbose(output, Path.Combine(dir, "run.checkpoint")); + + // Running as root the claim succeeds and it is an ordinary leftover copy. + Assert.Contains(verbose, v => v.Contains( + $"Removed what stood at the staging name '{Path.GetFileName(adopt)}': " + + (openable + ? "a copy left by an interrupted promotion." + : "a copy this account cannot open."), StringComparison.Ordinal)); + Assert.DoesNotContain(verbose, v => v.Contains("such as a pipe or a socket")); + Assert.False(File.Exists(adopt), "the copy outlived the sweep"); + } + finally + { + MakeWritable(adopt); + try { Directory.Delete(dir, true); } catch { } + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/OrphanTempOwnershipTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/OrphanTempOwnershipTests.cs new file mode 100644 index 0000000..c6fa4ce --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/OrphanTempOwnershipTests.cs @@ -0,0 +1,509 @@ +using System.Management.Automation; +using System.Net; +using System.Reflection; +using System.Text; +using Mgx.Cmdlets.Base; + +namespace Mgx.IntegrationTests; + +/// +/// Which temp files belong to an output. The recovery paths search for them with +/// "{output}.*.tmp", which Directory.EnumerateFiles reads as a glob - and '*' spans dots, so +/// an export to "users" matched a file left by an export to "users.jsonl". Adopting one is how +/// another export's rows end up in this one's output. +/// +[Collection("Pipeline")] +public class OrphanTempOwnershipTests +{ + private const string Page1 = """ + {"value":[{"id":"u1"},{"id":"u2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2"} + """; + private const string Page2 = """ + {"value":[{"id":"u3"}]} + """; + private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + + private static readonly MethodInfo Adopt = + typeof(MgxCmdletBase).GetMethod( + "TryAdoptOrphanedTemp", BindingFlags.Static | BindingFlags.NonPublic)!; + + /// + /// The two arguments, followed by an empty slot for each out parameter the method carries. + /// Sized off the method, since reflection wants a slot for every one of them. + /// + private static bool TryAdopt(string outputPath, long itemCount) + { + var slots = new object?[Adopt.GetParameters().Length]; + slots[0] = outputPath; + slots[1] = itemCount; + return (bool)Adopt.Invoke(null, slots)!; + } + + private sealed class ScriptedHandler : HttpMessageHandler + { + private readonly Queue<(HttpStatusCode Status, string Body)> _steps = new(); + private readonly object _lock = new(); + + public void Queue(HttpStatusCode status, string body) + { + lock (_lock) _steps.Enqueue((status, body)); + } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + (HttpStatusCode Status, string Body) step; + lock (_lock) + step = _steps.Count > 0 ? _steps.Dequeue() : (HttpStatusCode.InternalServerError, ServerError); + return Task.FromResult(new HttpResponseMessage(step.Status) + { + RequestMessage = request, + Content = new StringContent(step.Body, Encoding.UTF8, "application/json") + }); + } + } + + private static string NewDir() => + Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-temp-owner-{Guid.NewGuid():N}")).FullName; + + /// + /// Whether this directory's filesystem answers to a name in any case. Where it does not, + /// two spellings are two files and refusing the other one's temp is the right answer, so + /// the case test below has nothing to assert. + /// + private static bool NamesAreCaseInsensitive(string dir) + { + var probe = Path.Combine(dir, $"Case-{Guid.NewGuid():N}"); + File.WriteAllText(probe, ""); + try { return File.Exists(Path.Combine(dir, Path.GetFileName(probe).ToLowerInvariant())); } + finally { try { File.Delete(probe); } catch { } } + } + + private static void Export(string output, string checkpoint) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", output) + .AddParameter("CheckpointPath", checkpoint) + .AddParameter("All"); + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + } + + /// + /// Plain names, no glob characters anywhere: "users" and "users.jsonl" are two outputs in + /// one directory, and only the second one's export wrote that temp. + /// + [Fact] + public void A_temp_left_by_a_longer_named_export_is_not_adopted() + { + var dir = NewDir(); + try + { + var mine = Path.Combine(dir, "users"); + var theirs = Path.Combine(dir, $"users.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllLines(theirs, ["{\"id\":\"THEIRS-1\"}", "{\"id\":\"THEIRS-2\"}"]); + + Assert.False(TryAdopt(mine, 2L)); + + Assert.False(File.Exists(mine)); + Assert.True(File.Exists(theirs)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The same thing through the cmdlet, from the state that reaches adoption: a checkpoint + /// written before the temp name and length were recorded, and no output file. + /// + [Fact] + public void An_export_does_not_adopt_a_longer_named_exports_temp() + { + var dir = NewDir(); + var output = Path.Combine(dir, "users"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var theirs = Path.Combine(dir, $"users.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllLines(theirs, ["{\"id\":\"THEIRS-1\"}", "{\"id\":\"THEIRS-2\"}"]); + File.WriteAllText(checkpoint, """ + {"resource":"https://graph.microsoft.com/v1.0/users?$top=999", + "nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + "itemsCollected":2,"pageItemsAlreadyWritten":0} + """); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", output) + .AddParameter("CheckpointPath", checkpoint) + .AddParameter("All"); + ps.Invoke(); + + var lines = File.ReadAllLines(output); + Assert.DoesNotContain(lines, l => l.Contains("THEIRS")); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], lines); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A file the caller keeps beside their output. The glob that finds temps reaches it, and + /// nothing about the name says mgx wrote it - a run's own temp is the output's name, a dot, + /// 32 hex digits and ".tmp". Adoption copies what it takes into the output and then deletes + /// it, so a backup taken by hand was consumed as an interrupted export's items. + /// + [Fact] + public void A_file_that_is_not_a_run_temp_is_not_adopted() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "users.jsonl"); + var backup = Path.Combine(dir, "users.jsonl.backup.tmp"); + File.WriteAllLines(backup, ["{\"id\":\"BACKUP-1\"}", "{\"id\":\"BACKUP-2\"}"]); + + Assert.False(TryAdopt(output, 2L)); + + Assert.False(File.Exists(output)); + Assert.True(File.Exists(backup)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The same thing through the cmdlet, from the state that reaches adoption: a checkpoint + /// written before the temp name and length were recorded, and no output file. + /// + [Fact] + public void An_export_does_not_adopt_a_file_it_never_wrote() + { + var dir = NewDir(); + var output = Path.Combine(dir, "users.jsonl"); + var backup = Path.Combine(dir, "users.jsonl.backup.tmp"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + File.WriteAllLines(backup, ["{\"id\":\"BACKUP-1\"}", "{\"id\":\"BACKUP-2\"}"]); + File.WriteAllText(checkpoint, """ + {"resource":"https://graph.microsoft.com/v1.0/users?$top=999", + "nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + "itemsCollected":2,"pageItemsAlreadyWritten":0} + """); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + + Export(output, checkpoint); + + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.Equal(["{\"id\":\"BACKUP-1\"}", "{\"id\":\"BACKUP-2\"}"], + File.ReadAllLines(backup)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The same export, resumed with -OutputFile spelled differently. On a filesystem that + /// answers to either spelling the temp is this export's own, and the glob that finds it + /// matches case-insensitively - so a predicate comparing ordinally threw away the run's + /// own work and re-enumerated the collection from the first page without saying so. + /// + [Fact] + public void An_export_adopts_its_own_temp_when_the_output_is_spelled_differently() + { + var dir = NewDir(); + try + { + if (!NamesAreCaseInsensitive(dir)) return; + + var upper = Path.Combine(dir, "Users.jsonl"); + var lower = Path.Combine(dir, "users.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // Page 1 reaches the temp; page 2 fails, so the temp and its checkpoint survive. + handler.Queue(HttpStatusCode.OK, Page1); + Export(upper, checkpoint); + Assert.Single(Directory.GetFiles(dir, "Users.jsonl.*.tmp")); + + // The resume, typed the other way. Only the page the dead run never reached. + handler.Queue(HttpStatusCode.OK, Page2); + Export(lower, checkpoint); + + Assert.Equal( + ["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(lower)); + Assert.Empty(Directory.GetFiles(dir, "*.tmp")); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// Whether the temps of two spellings cross is the filesystem's answer and not the + /// platform's. The comparison and the glob both took their rule from the operating system, + /// so a case-sensitive APFS volume - and a Windows directory flagged case-sensitive on its + /// own - was matched case-insensitively against a filesystem that keeps the two apart: + /// "Users.jsonl" and "users.jsonl" are two exports there, and one's temp was reachable by + /// the other's adoption and by its sweep. + /// + /// How this host gets there differs; what it asserts does not. Where the directory's own + /// probe answers case-sensitive the two files really are two, and where it does not the + /// answer is pinned so the same decision path still runs. Neither is skipped. + /// + [Fact] + public void An_export_does_not_adopt_the_other_spelling_of_its_output_where_case_is_kept() + { + var dir = NewDir(); + var pinned = !MgxCmdletBase.DirectoryNamesAreCaseSensitive(dir, mayWrite: true); + if (pinned) MgxCmdletBase.SetDirectoryCaseSensitivity(dir, true); + try + { + var mine = Path.Combine(dir, "users.jsonl"); + var theirs = Path.Combine(dir, $"Users.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllLines(theirs, ["{\"id\":\"THEIRS-1\"}", "{\"id\":\"THEIRS-2\"}"]); + + Assert.False(TryAdopt(mine, 2L)); + + Assert.False(File.Exists(mine)); + Assert.True(File.Exists(theirs)); + Assert.Equal(["{\"id\":\"THEIRS-1\"}", "{\"id\":\"THEIRS-2\"}"], + File.ReadAllLines(theirs)); + } + finally + { + if (pinned) MgxCmdletBase.SetDirectoryCaseSensitivity(dir, null); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same rule on the sweep, which is where it costs the other export its work rather + /// than this one its output: a fresh run deletes every "{output}.{guid}.tmp" beside it as + /// an orphan, and a temp whose name differs from this output's only in case is not beside + /// this output at all where the filesystem keeps the two apart. It was deleted from under + /// the run that was writing it. + /// + [Fact] + public void An_export_does_not_sweep_the_other_spelling_of_its_output_where_case_is_kept() + { + var dir = NewDir(); + var pinned = !MgxCmdletBase.DirectoryNamesAreCaseSensitive(dir, mayWrite: true); + if (pinned) MgxCmdletBase.SetDirectoryCaseSensitivity(dir, true); + try + { + var mine = Path.Combine(dir, "users.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var theirs = Path.Combine(dir, $"Users.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllLines(theirs, ["{\"id\":\"THEIRS-1\"}"]); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + + Export(mine, checkpoint); + + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(mine)); + Assert.True(File.Exists(theirs), + "the sweep took a temp that belongs to another output"); + Assert.Equal(["{\"id\":\"THEIRS-1\"}"], File.ReadAllLines(theirs)); + } + finally + { + if (pinned) MgxCmdletBase.SetDirectoryCaseSensitivity(dir, null); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A second export running against the same output right now. Adoption picks the newest + /// matching temp with only a line count to go on, and the file a live run is writing into + /// is always the newest one - so recovering a pre-2.1.0 checkpoint copied that run's rows + /// into this run's deliverable and unlinked the file underneath it, reported on the warning + /// stream as a successful recovery with nothing on the error stream. The sweep beside it + /// spares the same file; adoption has to. + /// + [Fact] + public void An_export_does_not_adopt_a_temp_a_running_export_still_holds() + { + var dir = NewDir(); + var output = Path.Combine(dir, "users.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var live = Path.Combine(dir, $"users.jsonl.{Guid.NewGuid():N}.tmp"); + try + { + File.WriteAllText(checkpoint, """ + {"resource":"https://graph.microsoft.com/v1.0/users?$top=999", + "nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + "itemsCollected":2,"pageItemsAlreadyWritten":0} + """); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + + // The other run, holding its temp open the way an export does, two rows flushed. + using (var writer = new StreamWriter(live, append: false)) + { + writer.WriteLine("{\"id\":\"OTHER-1\"}"); + writer.WriteLine("{\"id\":\"OTHER-2\"}"); + writer.Flush(); + + Export(output, checkpoint); + + Assert.True(File.Exists(live), "a running export's temp was unlinked under it"); + var lines = File.ReadAllLines(output); + Assert.DoesNotContain(lines, l => l.Contains("OTHER")); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], lines); + + writer.WriteLine("{\"id\":\"OTHER-3\"}"); + } + + Assert.Equal( + ["{\"id\":\"OTHER-1\"}", "{\"id\":\"OTHER-2\"}", "{\"id\":\"OTHER-3\"}"], + File.ReadAllLines(live)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The probe file the case question is answered with is deleted by hand and by the handle + /// it was opened on - and on Unix that handle unlinks where the stream is disposed, so a + /// process killed outright leaves one behind, named after nothing any run comes back to. + /// The next run that writes here takes it back before writing its own. + /// + /// A leftover is put there by hand, since the state is only reachable by killing a process + /// mid-probe. The cached answer is forgotten first so the export below actually asks. + /// + [Fact] + public void A_run_reclaims_the_case_probe_file_an_earlier_run_left() + { + var dir = NewDir(); + var output = Path.Combine(dir, "users.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + var litter = Path.Combine(dir, $".mgx-case-{Guid.NewGuid():N}"); + try + { + File.WriteAllText(litter, ""); + MgxCmdletBase.SetDirectoryCaseSensitivity(dir, null); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + + Export(output, checkpoint); + + Assert.False(File.Exists(litter), + "a leftover case probe outlived a run that wrote in the same directory"); + Assert.Empty(Directory.GetFiles(dir, ".mgx-case-*")); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + } + finally + { + MgxCmdletBase.SetDirectoryCaseSensitivity(dir, null); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The question answered from what is already on disk. An entry that is there under one + /// spelling is there under the inverted spelling too exactly when the volume folds case, so + /// a directory holding any lettered name settles it with nothing written - which is what + /// keeps the answer out of the caller's directory's write time on every path, gated or not. + /// + /// What the volume actually does is taken with the test's own file first, so the assertion + /// is against this host rather than against its platform's default: either answer executes + /// here and neither is skipped. + /// + [Fact] + public void A_directory_holding_a_lettered_entry_decides_its_case_rule_without_writing() + { + var dir = NewDir(); + try + { + var insensitive = NamesAreCaseInsensitive(dir); + File.WriteAllText(Path.Combine(dir, "Entry.txt"), ""); + MgxCmdletBase.SetDirectoryCaseSensitivity(dir, null); + + var namesBefore = Directory.GetFileSystemEntries(dir).OrderBy(n => n).ToArray(); + var writtenBefore = Directory.GetLastWriteTimeUtc(dir); + + Assert.Equal(!insensitive, + MgxCmdletBase.DirectoryNamesAreCaseSensitive(dir, mayWrite: true)); + + Assert.Equal(namesBefore, Directory.GetFileSystemEntries(dir).OrderBy(n => n).ToArray()); + Assert.Equal(writtenBefore, Directory.GetLastWriteTimeUtc(dir)); + } + finally + { + MgxCmdletBase.SetDirectoryCaseSensitivity(dir, null); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// What reaches the caller's file. The checkpoint predates the recorded length, so adoption + /// has only a line count to go on - and a temp cut inside the row that count reaches hands + /// the fragment back as a line like any other. It went into the JSONL output as an item, + /// was reported as a recovery, and the rest of the enumeration was appended behind it, + /// leaving a row nothing can parse in the middle of the deliverable. + /// + [Fact] + public void An_export_does_not_adopt_a_temp_cut_inside_the_row_the_checkpoint_counts() + { + var dir = NewDir(); + var output = Path.Combine(dir, "users.jsonl"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + File.WriteAllText(Path.Combine(dir, $"users.jsonl.{Guid.NewGuid():N}.tmp"), + "{\"id\":\"t1\"}\n{\"id\":\"t2\"}\n{\"id\":\"to"); + File.WriteAllText(checkpoint, """ + {"resource":"https://graph.microsoft.com/v1.0/users?$top=999", + "nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + "itemsCollected":3,"pageItemsAlreadyWritten":0} + """); + + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + + Export(output, checkpoint); + + var lines = File.ReadAllLines(output); + foreach (var line in lines) + System.Text.Json.JsonDocument.Parse(line).Dispose(); + Assert.Equal(["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], lines); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/OutFileGuardTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/OutFileGuardTests.cs new file mode 100644 index 0000000..316db94 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/OutFileGuardTests.cs @@ -0,0 +1,54 @@ +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// A second piped item against one -OutFile is refused BEFORE its content is fetched - +/// previously the whole body was downloaded and then thrown away with the refusal. +/// +[Collection("Pipeline")] +public class OutFileGuardTests +{ + [Fact] + public void A_second_piped_item_is_refused_without_fetching_it() + { + var wire = new MockHttpHandler(); + wire.QueueBytes(HttpStatusCode.OK, [1, 2, 3, 4], "application/octet-stream"); + wire.QueueBytes(HttpStatusCode.OK, [5, 6, 7, 8], "application/octet-stream"); + + // Get-MgxContent refuses a transport mgx does not own, so this one claims ownership. + using var transport = MgxTransportScope.Inject(wire, owned: true); + + var outFile = Path.Combine(Path.GetTempPath(), $"mgx-outfile-{Guid.NewGuid():N}.bin"); + try + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript($$""" + @( + [Hashtable]@{ id = 'item1'; parentReference = @{ driveId = 'd1' } } + [Hashtable]@{ id = 'item2'; parentReference = @{ driveId = 'd1' } } + ) | Get-MgxContent -OutFile '{{outFile}}' + """); + ps.Invoke(); + + var error = Assert.Single(ps.Streams.Error); + Assert.StartsWith("OutFileWithMultipleInputs", error.FullyQualifiedErrorId); + + // The first item was fetched and written; the second was refused pre-fetch. + Assert.Single(wire.CapturedRequests); + Assert.Equal(new byte[] { 1, 2, 3, 4 }, File.ReadAllBytes(outFile)); + } + finally + { + File.Delete(outFile); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/RequestWhatIfCheckpointTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/RequestWhatIfCheckpointTests.cs new file mode 100644 index 0000000..e73f203 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/RequestWhatIfCheckpointTests.cs @@ -0,0 +1,574 @@ +using System.Globalization; +using System.Management.Automation; +using System.Management.Automation.Host; +using System.Management.Automation.Runspaces; +using System.Net; +using System.Security; +using System.Text; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// -WhatIf on a list run carrying -CheckpointPath. The read path saves the next position over +/// that file at every page boundary, deletes it on completion, and deletes it where it turns +/// out to describe another enumeration - so previewing a run destroyed the position the caller +/// resumes from. The reads are not what is gated: a GET under -WhatIf is sent, pages and emits +/// its objects, which is the convention Invoke-MgxRequest documents. +/// +[Collection("Pipeline")] +public class RequestWhatIfCheckpointTests +{ + private const string Page1 = """ + {"value":[{"id":"u1"},{"id":"u2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2"} + """; + private const string Page2 = """ + {"value":[{"id":"u3"}]} + """; + private const string NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2"; + + private const string EmptyPage = """ + {"value":[],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P3"} + """; + private const string LastPage = """ + {"value":[{"id":"u4"},{"id":"u5"}]} + """; + + /// + /// Two pages, plus a snapshot of the checkpoint file as each request arrives. The + /// page-boundary save lands between the first response and the second request and the + /// completion delete takes it away again, so mid-run is the only point from which a test + /// can see whether a run wrote a checkpoint at all. + /// + private sealed class PagingHandler(string? checkpointPath) : HttpMessageHandler + { + private readonly object _lock = new(); + public List Urls { get; } = []; + public List CheckpointExisted { get; } = []; + public List CheckpointContent { get; } = []; + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + lock (_lock) + { + Urls.Add(url); + var existed = checkpointPath != null && File.Exists(checkpointPath); + CheckpointExisted.Add(existed); + CheckpointContent.Add(existed ? File.ReadAllText(checkpointPath!) : null); + } + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent( + url.Contains("skiptoken", StringComparison.Ordinal) ? Page2 : Page1, + Encoding.UTF8, "application/json") + }); + } + } + + /// + /// Three pages with the middle one empty behind a nextLink, which is how a page boundary is + /// reached with no item yielded since the fetch. Every answer takes a real asynchronous hop: + /// a socket read always yields and Task.FromResult never does, and the thread the iterator + /// resumes on is what these pages are here to settle. + /// + private sealed class EmptyMiddlePageHandler : HttpMessageHandler + { + private readonly object _lock = new(); + public List Urls { get; } = []; + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + lock (_lock) { Urls.Add(url); } + + await Task.Yield(); + + var body = url.Contains("skiptoken=P3", StringComparison.Ordinal) ? LastPage + : url.Contains("skiptoken=P2", StringComparison.Ordinal) ? EmptyPage + : Page1; + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }; + } + } + + /// + /// Page 1 has items and a nextLink; whatever answers the request that follows is not JSON at + /// all. Awaits Task.Yield() for the same reason the empty-page handler does: every answer + /// takes a real asynchronous hop, so the page-boundary callback - which fires once page 1's + /// two items are both yielded, before the second request goes out - runs on whichever thread + /// that hop left it on rather than synchronously on the caller's. + /// + private sealed class MalformedSecondPageHandler : HttpMessageHandler + { + private readonly object _lock = new(); + public List Urls { get; } = []; + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var url = request.RequestUri!.ToString(); + lock (_lock) { Urls.Add(url); } + + await Task.Yield(); + + var body = url.Contains("skiptoken", StringComparison.Ordinal) + ? "not a collection envelope at all" + : Page1; + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, Encoding.UTF8, "application/json") + }; + } + } + + /// The lines -WhatIf writes go to the host, so a test that reads them needs one. + private sealed class RecordingHost : PSHost + { + private readonly Guid _id = Guid.NewGuid(); + public RecordingHostUI Recorder { get; } = new(); + public override string Name => "MgxTestHost"; + public override Version Version => new(1, 0); + public override Guid InstanceId => _id; + public override PSHostUserInterface UI => Recorder; + public override CultureInfo CurrentCulture => CultureInfo.InvariantCulture; + public override CultureInfo CurrentUICulture => CultureInfo.InvariantCulture; + public override void EnterNestedPrompt() { } + public override void ExitNestedPrompt() { } + public override void NotifyBeginApplication() { } + public override void NotifyEndApplication() { } + public override void SetShouldExit(int exitCode) { } + } + + private sealed class RecordingHostUI : PSHostUserInterface + { + public List Lines { get; } = []; + public override PSHostRawUserInterface? RawUI => null; + public override void Write(string value) => Lines.Add(value); + public override void Write(ConsoleColor f, ConsoleColor b, string value) => Lines.Add(value); + public override void WriteLine(string value) => Lines.Add(value); + public override void WriteErrorLine(string value) => Lines.Add(value); + public override void WriteDebugLine(string value) => Lines.Add(value); + public override void WriteVerboseLine(string value) => Lines.Add(value); + public override void WriteWarningLine(string value) => Lines.Add(value); + public override void WriteProgress(long sourceId, ProgressRecord record) { } + public override string ReadLine() => string.Empty; + public override SecureString ReadLineAsSecureString() => new(); + public override Dictionary Prompt( + string caption, string message, System.Collections.ObjectModel.Collection descriptions) => []; + public override int PromptForChoice( + string caption, string message, + System.Collections.ObjectModel.Collection choices, int defaultChoice) => defaultChoice; + public override PSCredential PromptForCredential( + string caption, string message, string userName, string targetName) => PSCredential.Empty; + public override PSCredential PromptForCredential( + string caption, string message, string userName, string targetName, + PSCredentialTypes allowedCredentialTypes, PSCredentialUIOptions options) => PSCredential.Empty; + } + + private static string NewDir() => + Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-req-whatif-{Guid.NewGuid():N}")).FullName; + + /// + /// Walks the collection and hands back the lines the host was given, how many objects the + /// run emitted, whether it errored, the warnings it wrote, and the ids of the errors it + /// wrote. + /// + private static (List Lines, int Items, bool HadErrors, List Warnings, List Errors) Enumerate( + string? checkpoint, bool whatIf) + { + var host = new RecordingHost(); + using var runspace = RunspaceFactory.CreateRunspace(host); + runspace.Open(); + using var ps = PowerShell.Create(); + ps.Runspace = runspace; + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + var cmd = ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("All") + .AddParameter("PageSize", 2); + if (checkpoint != null) cmd.AddParameter("CheckpointPath", checkpoint); + if (whatIf) cmd.AddParameter("WhatIf", true); + var output = ps.Invoke(); + return (host.Recorder.Lines, output.Count, ps.HadErrors, + [.. ps.Streams.Warning.Select(w => w.Message)], + [.. ps.Streams.Error.Select(e => e.FullyQualifiedErrorId)]); + } + + /// + /// The URL this enumeration asks for. A checkpoint is the run's own only when the resource + /// it records matches that string exactly, and writing the query shape into the test would + /// pin the URL builder instead of the gate. + /// + private static string EnumerationUrl() + { + var handler = new PagingHandler(checkpointPath: null); + using var transport = MgxTransportScope.Inject(handler); + Enumerate(checkpoint: null, whatIf: false); + return handler.Urls[0]; + } + + /// + /// A checkpoint another enumeration left over the same -CheckpointPath. A real run refuses + /// it, deletes it and pages from the start; the preview pages from the start too - same + /// requests, same objects - and leaves the caller's file exactly as it found it. + /// + [Fact] + public void WhatIf_leaves_another_enumerations_checkpoint_byte_identical() + { + var dir = NewDir(); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/groups?$top=2", + NextLink = "https://graph.microsoft.com/v1.0/groups?$skiptoken=P2", + ItemsCollected = 2, + }.Save(checkpoint); + var before = File.ReadAllBytes(checkpoint); + + var handler = new PagingHandler(checkpoint); + using var transport = MgxTransportScope.Inject(handler); + var (lines, items, hadErrors, _, _) = Enumerate(checkpoint, whatIf: true); + + Assert.False(hadErrors); + + // The reads are not what the gate withholds: both pages were fetched and every + // object went down the pipeline. + Assert.Equal(2, handler.Urls.Count); + Assert.Equal(3, items); + + // The file is what it withholds, and the caller's is where it was. + Assert.True(File.Exists(checkpoint), "-WhatIf deleted the caller's checkpoint"); + Assert.Equal(before, File.ReadAllBytes(checkpoint)); + + // Named on the way past, so the caller running the preview is told which file a + // real run would have written. + Assert.Contains(lines, + l => l.Contains("Save resume checkpoint") && l.Contains(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A completion marker (NextLink null) left by this enumeration's own previous run. A real + /// run takes that as "already finished" and deletes it before paging from the start again; + /// the preview takes the same route through the same Delete call, gated the same way as the + /// page-boundary save and the mismatched-resource branch above - and mayWriteCheckpoint is + /// the only thing standing between it and deleting a marker -WhatIf is not supposed to touch. + /// + [Fact] + public void WhatIf_leaves_a_completion_marker_checkpoint_byte_identical() + { + var dir = NewDir(); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + new PaginationCheckpoint + { + Resource = EnumerationUrl(), + NextLink = null, + ItemsCollected = 7, + }.Save(checkpoint); + var before = File.ReadAllBytes(checkpoint); + + var handler = new PagingHandler(checkpoint); + using var transport = MgxTransportScope.Inject(handler); + var (_, items, hadErrors, _, _) = Enumerate(checkpoint, whatIf: true); + + Assert.False(hadErrors); + + // A completion marker carries no position to resume from, so both pages are asked + // for and all three objects go down the pipeline either way. + Assert.Equal(2, handler.Urls.Count); + Assert.Equal(3, items); + + Assert.True(File.Exists(checkpoint), "-WhatIf deleted the caller's completion-marker checkpoint"); + Assert.Equal(before, File.ReadAllBytes(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// This enumeration's own checkpoint, but its nextLink is one NextLinkValidator refuses (here, + /// relative rather than https) - the SSRF gate, not a stale or foreign resource. A real run + /// cannot resume from a link it will not follow, so it deletes the checkpoint and pages from + /// the start; the preview takes the same branch through the same Delete call, and withholds it + /// the same way. + /// + [Fact] + public void WhatIf_leaves_a_refused_nextlink_checkpoint_byte_identical() + { + var dir = NewDir(); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + new PaginationCheckpoint + { + Resource = EnumerationUrl(), + NextLink = "pages/4", + ItemsCollected = 7, + }.Save(checkpoint); + var before = File.ReadAllBytes(checkpoint); + + var handler = new PagingHandler(checkpoint); + using var transport = MgxTransportScope.Inject(handler); + var (_, items, hadErrors, _, _) = Enumerate(checkpoint, whatIf: true); + + Assert.False(hadErrors); + + // The refused link carries no usable position either, so this run pages from the + // start the same as the completion-marker case above. + Assert.Equal(2, handler.Urls.Count); + Assert.Equal(3, items); + + Assert.True(File.Exists(checkpoint), + "-WhatIf deleted the caller's checkpoint with a nextLink the validator refuses"); + Assert.Equal(before, File.ReadAllBytes(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same run with nothing at -CheckpointPath to begin with. A preview that leaves a + /// checkpoint behind has written a resume position for an enumeration that never happened, + /// and one that writes and then deletes has still had the caller's file for the length of + /// the run - so the page boundary, not the end, is where this is settled. + /// + [Fact] + public void WhatIf_writes_no_checkpoint_of_its_own() + { + var dir = NewDir(); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new PagingHandler(checkpoint); + using var transport = MgxTransportScope.Inject(handler); + var (_, items, hadErrors, _, _) = Enumerate(checkpoint, whatIf: true); + + Assert.False(hadErrors); + Assert.Equal(3, items); + Assert.Equal(2, handler.CheckpointExisted.Count); + Assert.All(handler.CheckpointExisted, + existed => Assert.False(existed, "-WhatIf saved a checkpoint of its own")); + Assert.False(File.Exists(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A checkpoint this enumeration wrote. The preview is of the run the caller would make + /// next, and that run resumes - so the preview resumes too, and asks only for the page the + /// checkpoint stopped at. What it cannot do is finish by deleting the position it read. + /// + [Fact] + public void WhatIf_resumes_from_a_matching_checkpoint_without_consuming_it() + { + var dir = NewDir(); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + new PaginationCheckpoint + { + Resource = EnumerationUrl(), + NextLink = NextLink, + ItemsCollected = 2, + }.Save(checkpoint); + var before = File.ReadAllBytes(checkpoint); + + var handler = new PagingHandler(checkpoint); + using var transport = MgxTransportScope.Inject(handler); + var (_, items, hadErrors, _, _) = Enumerate(checkpoint, whatIf: true); + + Assert.False(hadErrors); + Assert.Single(handler.Urls); + Assert.Contains("skiptoken=P2", handler.Urls[0]); + Assert.Equal(1, items); + Assert.True(File.Exists(checkpoint), "-WhatIf deleted the checkpoint it resumed from"); + Assert.Equal(before, File.ReadAllBytes(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The run the previews above describe. Nothing about the checkpoint changes for a real + /// run: the next position is saved at the page boundary, and the file is gone once the + /// enumeration completes. + /// + [Fact] + public void A_real_run_saves_at_the_page_boundary_and_deletes_on_completion() + { + var dir = NewDir(); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new PagingHandler(checkpoint); + using var transport = MgxTransportScope.Inject(handler); + var (_, items, hadErrors, _, _) = Enumerate(checkpoint, whatIf: false); + + Assert.False(hadErrors); + Assert.Equal(3, items); + Assert.False(handler.CheckpointExisted[0]); + Assert.True(handler.CheckpointExisted[1], "no checkpoint was saved at the page boundary"); + Assert.Contains("skiptoken=P2", handler.CheckpointContent[1]!); + Assert.False(File.Exists(checkpoint)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same real run, against a checkpoint it cannot write. The page-boundary callback runs + /// on whichever thread the iterator resumed on, and a page whose value is empty resumes on + /// the thread pool with no yield in between - so a cmdlet API call from there throws + /// PSInvalidOperationException, and warning the caller about the disk ended the enumeration + /// at the empty page instead. The message goes onto the client's buffered warning channel + /// and the pipeline thread's drain writes it: one per failed save, and the pages behind the + /// empty one are still asked for and still emitted. + /// + [Fact] + public void A_failed_boundary_save_warns_and_the_enumeration_goes_on() + { + var dir = NewDir(); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + BlockSaves(dir, checkpoint); + + var handler = new EmptyMiddlePageHandler(); + using var transport = MgxTransportScope.Inject(handler); + var (_, items, hadErrors, warnings, _) = Enumerate(checkpoint, whatIf: false); + + Assert.False(hadErrors); + + // Both boundaries that carry a nextLink try to save, and both fail against a + // checkpoint that cannot be written. + var failures = warnings + .Where(w => w.StartsWith("Checkpoint save failed", StringComparison.Ordinal)) + .ToList(); + Assert.Equal(2, failures.Count); + + // The empty page sits behind the second request, so a run that died reporting its + // failed save never asks for the third - and the two items on it never arrive. + Assert.Equal(3, handler.Urls.Count); + Assert.Contains("skiptoken=P3", handler.Urls[2]); + Assert.Equal(4, items); + } + finally + { + RestoreMode(dir); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same blocked checkpoint, but the run this time does not reach the successful exit + /// the previous test measures: page 1's boundary save fails same as before, and what + /// answers the next request is not JSON at all. That exit is the JsonException catch, which + /// returns straight after writing its error record - there is no next item left to carry the + /// per-item drain, and until the enumeration loop's exits all shared a single finally, that + /// catch was one of three that returned without draining, so the buffered "Checkpoint save + /// failed" warning stayed on the queue and was never written. + /// + [Fact] + public void A_failed_boundary_save_warns_even_when_the_next_page_does_not_parse() + { + var dir = NewDir(); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + BlockSaves(dir, checkpoint); + + var handler = new MalformedSecondPageHandler(); + using var transport = MgxTransportScope.Inject(handler); + var (_, items, hadErrors, warnings, errors) = Enumerate(checkpoint, whatIf: false); + + Assert.True(hadErrors); + var errorId = Assert.Single(errors); + Assert.StartsWith("MalformedJsonResponse", errorId); + + var failures = warnings + .Where(w => w.StartsWith("Checkpoint save failed", StringComparison.Ordinal)) + .ToList(); + Assert.Single(failures); + + // Page 1's two items were already on the pipeline before the second request's body + // failed to parse. + Assert.Equal(2, items); + } + finally + { + RestoreMode(dir); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// Makes every checkpoint save into this directory fail, whatever the host allows. A mode + /// the account cannot write is the honest version of it, but root ignores mode bits and + /// Windows expresses this through ACLs; where a canary shows the mode does not bite, the + /// name Save stages its bytes under is taken by a directory instead, which no account can + /// write a file over. + /// + private static void BlockSaves(string dir, string checkpoint) + { + if (!OperatingSystem.IsWindows()) + { + File.SetUnixFileMode(dir, UnixFileMode.UserRead | UnixFileMode.UserExecute); + try + { + var canary = Path.Combine(dir, "writable.probe"); + File.WriteAllText(canary, ""); + File.Delete(canary); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return; + } + RestoreMode(dir); + } + + Directory.CreateDirectory(checkpoint + ".tmp"); + } + + private static void RestoreMode(string dir) + { + if (OperatingSystem.IsWindows()) return; + try + { + File.SetUnixFileMode(dir, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ResilienceInjectionScenarioTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ResilienceInjectionScenarioTests.cs new file mode 100644 index 0000000..ede3fed --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/ResilienceInjectionScenarioTests.cs @@ -0,0 +1,560 @@ +using System.Net; +using System.Reflection; +using Mgx.Cmdlets; +using Mgx.Cmdlets.Base; +using Mgx.Cmdlets.Cmdlets.Configuration; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// The existing suites cover request-level behavior through the wrap - disarming Kiota's inner +/// retry handler across 429 and 503, correlation-id stamping, oversized bodies, lazy one-shot +/// option resolution - and two properties of the wrapper client. None of the injection +/// scenarios is covered as a scenario, because each needs a GraphSession and this process has +/// no Graph SDK. GraphSessionScope supplies one. +/// +/// Coexistence with Az/PnP/Exchange is not here: module load and ALC resolution are +/// irreversible within a process, so that belongs to the out-of-process ecosystem matrix. +/// +[Collection("Pipeline")] +public class ResilienceInjectionScenarioTests +{ + private static HttpClient? BuildWrapper(HttpClient sdkClient, List warnings) => + EnableMgxResilience.BuildResilientSdkClient(sdkClient, warnings.Add); + + /// Puts the process in the state Enable-MgxResilience leaves behind on success. + private static HttpClient Enable(GraphSessionScope scope, HttpClient sdkClient) + { + var wrapper = BuildWrapper(sdkClient, [])!; + EnableMgxResilience.OriginalSdkClient = sdkClient; + EnableMgxResilience.ResilientSdkClient = wrapper; + EnableMgxResilience.IsEnabled = true; + scope.Session.GraphHttpClient = wrapper; + return wrapper; + } + + /// + /// Asserts that is exactly one resilience layer over + /// . A second layer answers requests as happily as one, so nothing + /// above the wire shows it: each layer retries the one beneath it, so a throttled request + /// multiplies its attempts, telemetry counts each attempt once per layer, and the pacer + /// halves once per layer. + /// + private static void AssertOneLayerOver(HttpClient wrapper, HttpClient genuine, string context = "") + { + var (layers, innermost) = MeasureWrap(wrapper); + Assert.True(layers == 1, $"{context}the wrap is {layers} layers deep"); + Assert.True(ReferenceEquals(genuine, innermost), + $"{context}the wrap bridges to something other than the genuine SDK client"); + } + + /// + /// Walks the handler chain to the client at the bottom of it, counting resilience layers on + /// the way. The object graph, not mgx's bookkeeping about it: what the wire sees is the chain. + /// + private static (int Layers, HttpClient Innermost) MeasureWrap(HttpClient client) + { + var layers = 0; + while (HandlerOf(client) is ResilientDelegatingHandler resilient) + { + layers++; + var bridge = resilient.InnerHandler; + Assert.NotNull(bridge); + var target = bridge!.GetType().GetField("_sdkClient", BindingFlags.Instance | BindingFlags.NonPublic); + Assert.True(target != null, $"{bridge.GetType().Name} is not the SDK bridge handler"); + client = (HttpClient)target!.GetValue(bridge)!; + } + return (layers, client); + } + + /// The handler an HttpClient sends through; it keeps it in a private field. + private static HttpMessageHandler? HandlerOf(HttpClient client) + { + var field = typeof(HttpMessageInvoker).GetField("_handler", BindingFlags.Instance | BindingFlags.NonPublic); + Assert.True(field != null, "HttpMessageInvoker no longer keeps its handler in _handler"); + return (HttpMessageHandler?)field!.GetValue(client); + } + + private sealed class OkHandler : HttpMessageHandler + { + public int Requests { get; private set; } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Requests++; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent("""{"value":[]}""", + System.Text.Encoding.UTF8, "application/json") + }); + } + } + + // --- 1. the SDK replaced our client --- + + [Fact] + public void A_replaced_graph_http_client_is_wrapped_again_rather_than_left_raw() + { + using var scope = GraphSessionScope.Arm(); + using var first = new HttpClient(new OkHandler()) { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + var wrapper = Enable(scope, first); + + // Connect-MgGraph builds a fresh SDK client and hands it to the session, discarding ours. + using var replacement = new HttpClient(new OkHandler()) { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + scope.Session.GraphHttpClient = replacement; + + EnableMgxResilience.RefreshInjectedClient(_ => { }, _ => { }); + + var active = scope.Session.GraphHttpClient; + Assert.True(EnableMgxResilience.IsEnabled, "the wrap was dropped instead of re-injected"); + Assert.NotSame(replacement, active); // the caller is not left on the raw SDK client + Assert.NotSame(wrapper, active); // nor on the wrapper that bridged to the old one + Assert.Same(active, EnableMgxResilience.ResilientSdkClient); + Assert.Same(replacement, EnableMgxResilience.OriginalSdkClient); + } + + // --- 2. reconnect versus credential change --- + + [Fact] + public void A_same_identity_reconnect_keeps_the_limiter_a_credential_change_resets_it() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + using var sdkClient = new HttpClient(wire); + + const string tenant = "11111111-1111-1111-1111-111111111111"; + const string appA = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"; + const string appB = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"; + + using var scope = GraphSessionScope.Arm(sdkClient, GraphSessionScope.AuthContextFor(tenant, appA)); + + // No transport seam here: this is about the identity check inside GetClient, which the + // seam sits above. BuildCleanHttpClient cannot run without the SDK's auth helpers, so + // GetClient falls back to the session's own client - the borrowed-transport path. + // A real limiter, not NoRateLimit: the property under test is that the warm limiter + // instance survives, and NoRateLimit leaves the factory with none to keep. The burst + // covers every request this test sends, so nothing waits on it. + var options = new ResilientGraphClientOptions + { + MaxRetryAttempts = 1, + RateLimitBurst = 50, + RateLimitPerSecond = 50, + NoAdaptivePacing = true + }; + MgxCmdletBase.SetClientOptions(options); + ResiliencePipelineFactory.Reset(); + + Run(); + var afterFirst = ResiliencePipelineFactory.GetOrCreate(options).RateLimiter; + + // Reconnect: a new AuthContext object carrying the same credentials. The instance + // changed, so the client is rebuilt - but the identity did not, so the limiter that has + // already learned this tenant's ceiling must survive rather than earn a fresh burst. + scope.Session.AuthContext = GraphSessionScope.AuthContextFor(tenant, appA); + Run(); + Assert.Same(afterFirst, ResiliencePipelineFactory.GetOrCreate(options).RateLimiter); + + // A different application on the same tenant is a different credential. + scope.Session.AuthContext = GraphSessionScope.AuthContextFor(tenant, appB); + Run(); + Assert.NotSame(afterFirst, ResiliencePipelineFactory.GetOrCreate(options).RateLimiter); + + void Run() + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users/u1") + .AddParameter("WarningAction", System.Management.Automation.ActionPreference.SilentlyContinue); + var output = ps.Invoke(); + Assert.True(output.Count > 0, + "the request did not reach the session's client: " + + string.Join(" | ", ps.Streams.Error.Select(e => e.FullyQualifiedErrorId))); + } + } + + // --- 3. one wrapper at a time --- + + [Fact] + public void A_second_sdk_client_leaves_exactly_one_wrapper_armed() + { + using var scope = GraphSessionScope.Arm(); + using var first = new HttpClient(new OkHandler()); + Enable(scope, first); + var firstHandler = EnableMgxResilience.ActiveHandler; + Assert.NotNull(firstHandler); + + using var second = new HttpClient(new OkHandler()); + scope.Session.GraphHttpClient = second; + EnableMgxResilience.RefreshInjectedClient(_ => { }, _ => { }); + + // ActiveHandler is what Disable-MgxResilience and module teardown reach for. Two armed + // handlers would mean one of them keeps a pipeline and a bridge alive unreachably. + Assert.NotNull(EnableMgxResilience.ActiveHandler); + Assert.NotSame(firstHandler, EnableMgxResilience.ActiveHandler); + Assert.Same(scope.Session.GraphHttpClient, EnableMgxResilience.ResilientSdkClient); + Assert.Same(second, EnableMgxResilience.OriginalSdkClient); + } + + // --- 4. module removal --- + + [Fact] + public void Module_removal_puts_the_session_back_on_the_genuine_client() + { + using var scope = GraphSessionScope.Arm(); + using var sdkClient = new HttpClient(new OkHandler()) + { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + var wrapper = Enable(scope, sdkClient); + var handlerBeforeRemoval = EnableMgxResilience.ActiveHandler; + Assert.NotNull(handlerBeforeRemoval); + + // The teardown itself, not a copy of it. AlcInitializer.OnRemove is this call plus + // DetachAssemblyLoadHandler. + AlcInitializer.ReleaseStaticState(); + + Assert.Null(EnableMgxResilience.ActiveHandler); + Assert.Null(EnableMgxResilience.ResilientSdkClient); + Assert.Null(EnableMgxResilience.OriginalSdkClient); + Assert.False(EnableMgxResilience.IsEnabled); + + // What the teardown leaves installed is what the SDK goes on sending through, so the + // session has to be back on the genuine client before the references go. A wrapper left + // there routes every SDK request through a handler belonging to a module that is no + // longer loaded, for as long as nobody imports mgx again. + Assert.Same(sdkClient, scope.Session.GraphHttpClient); + + var warnings = EnableThroughTheCmdlet(); + + Assert.Empty(warnings); + Assert.True(EnableMgxResilience.IsEnabled); + Assert.NotNull(EnableMgxResilience.ActiveHandler); + Assert.NotSame(handlerBeforeRemoval, EnableMgxResilience.ActiveHandler); + Assert.NotSame(wrapper, EnableMgxResilience.ResilientSdkClient); + Assert.Same(scope.Session.GraphHttpClient, EnableMgxResilience.ResilientSdkClient); + Assert.Same(sdkClient, EnableMgxResilience.OriginalSdkClient); + AssertOneLayerOver(EnableMgxResilience.ResilientSdkClient!, sdkClient); + } + + [Fact] + public void Three_remove_and_re_enable_cycles_leave_the_wrap_one_layer_deep() + { + using var scope = GraphSessionScope.Arm(); + using var sdkClient = new HttpClient(new OkHandler()) + { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + scope.Session.GraphHttpClient = sdkClient; + + for (var cycle = 1; cycle <= 3; cycle++) + { + var where = $"cycle {cycle}: "; + Assert.Empty(EnableThroughTheCmdlet()); + + var wrapper = EnableMgxResilience.ResilientSdkClient; + Assert.True(wrapper != null, $"{where}resilience was not injected"); + Assert.True(ReferenceEquals(wrapper, scope.Session.GraphHttpClient), + $"{where}the session is not on the wrapper mgx built"); + Assert.True(ReferenceEquals(sdkClient, EnableMgxResilience.OriginalSdkClient), + $"{where}the client Disable-MgxResilience would restore is not the genuine one"); + AssertOneLayerOver(wrapper!, sdkClient, where); + + // Remove-Module. The next iteration's Enable-MgxResilience is the re-import: it runs + // the cmdlet against whatever this teardown left on the session. + AlcInitializer.ReleaseStaticState(); + Assert.True(ReferenceEquals(sdkClient, scope.Session.GraphHttpClient), + $"{where}removal left something other than the genuine client on the session"); + } + } + + /// + /// Runs Enable-MgxResilience where PowerShell runs it. The cmdlet reads the session through + /// the same reflection a re-imported module would, so what the previous import left behind is + /// what it decides against. + /// + private static IReadOnlyList EnableThroughTheCmdlet() => RunCmdlet("Enable-MgxResilience"); + + private static IReadOnlyList DisableThroughTheCmdlet() => RunCmdlet("Disable-MgxResilience"); + + private static IReadOnlyList RunCmdlet(string name) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(EnableMgxResilience).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand(name); + ps.Invoke(); + + Assert.Empty(ps.Streams.Error.Select(e => e.FullyQualifiedErrorId)); + return [.. ps.Streams.Warning.Select(w => w.Message)]; + } + + // --- 5. taking the injection off again --- + + [Fact] + public void Disable_after_a_module_removal_says_so_and_leaves_the_genuine_client() + { + using var scope = GraphSessionScope.Arm(); + using var sdkClient = new HttpClient(new OkHandler()) + { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + Enable(scope, sdkClient); + + AlcInitializer.ReleaseStaticState(); + + var warnings = DisableThroughTheCmdlet(); + + // Removal already unwound the injection, so there is nothing left to disable and saying + // so is the honest answer - but only because the session is on the genuine client. + Assert.Contains(warnings, w => w.Contains("not currently enabled", StringComparison.Ordinal)); + Assert.Same(sdkClient, scope.Session.GraphHttpClient); + } + + [Fact] + public void Disable_takes_off_a_wrapper_that_no_state_points_at_any_more() + { + using var scope = GraphSessionScope.Arm(); + using var sdkClient = new HttpClient(new OkHandler()) + { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + var wrapper = Enable(scope, sdkClient); + + // What a removal that could not reach GraphSession leaves: the wrapper is installed and + // every reference mgx held to it, and to the client under it, is gone. + EnableMgxResilience.IsEnabled = false; + EnableMgxResilience.ActiveHandler = null; + EnableMgxResilience.ResilientSdkClient = null; + EnableMgxResilience.OriginalSdkClient = null; + Assert.Same(wrapper, scope.Session.GraphHttpClient); + + var warnings = DisableThroughTheCmdlet(); + + Assert.Empty(warnings); + Assert.Same(sdkClient, scope.Session.GraphHttpClient); + } + + [Fact] + public void Disable_after_a_re_import_restores_the_genuine_client_not_the_previous_wrapper() + { + using var scope = GraphSessionScope.Arm(); + using var sdkClient = new HttpClient(new OkHandler()) + { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + var firstWrapper = Enable(scope, sdkClient); + + AlcInitializer.ReleaseStaticState(); + Assert.Empty(EnableThroughTheCmdlet()); + Assert.NotSame(firstWrapper, scope.Session.GraphHttpClient); + + var warnings = DisableThroughTheCmdlet(); + + Assert.Empty(warnings); + Assert.Same(sdkClient, scope.Session.GraphHttpClient); + Assert.False(EnableMgxResilience.IsEnabled); + Assert.Null(EnableMgxResilience.ResilientSdkClient); + Assert.Null(EnableMgxResilience.OriginalSdkClient); + Assert.Null(EnableMgxResilience.ActiveHandler); + } + + // --- 6. the option factory the cmdlet installs --- + + [Fact] + public async Task The_wrap_the_cmdlet_builds_sends_the_request_its_option_factory_annotates() + { + // The chain the cmdlet builds is ResilientDelegatingHandler -> bridge -> SDK client, and + // nothing in it consumes the retry option when the SDK's handlers are not there. The + // factory is the cmdlet's own, so whatever it does on this path - answer, or throw over a + // type the process has not loaded - is what the handler has to absorb, and the request + // has to go out either way. + var wire = new OkHandler(); + using var sdkClient = new HttpClient(wire) { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + var warnings = new List(); + + var wrapper = BuildWrapper(sdkClient, warnings); + try + { + Assert.NotNull(wrapper); + Assert.Empty(warnings); + + // The factory the cmdlet installs is BuildInnerRetryOverride itself, so the guard + // that already covers a throwing factory covers the production chain too. + var handler = EnableMgxResilience.ActiveHandler; + Assert.NotNull(handler); + var factory = handler.AdditionalRequestOptionsFactory; + Assert.NotNull(factory); + Assert.Equal(nameof(EnableMgxResilience.BuildInnerRetryOverride), factory.Method.Name); + + // Which outcome this run gets is not this file's to choose: BuildInnerRetryOverride + // resolves the SDK's option type across the loaded assemblies, so it depends on + // whether anything has pulled Kiota in yet. Asserting only the request would pass + // under either without saying which happened, so the handler's own note decides. + var notes = new List(); + handler.VerboseWriter = notes.Add; + var resolved = Record.Exception(() => factory()); + + using var response = await wrapper!.GetAsync("users"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(1, wire.Requests); + + var absorbed = notes.Where(note => + note.StartsWith("Could not configure the SDK's retry option", StringComparison.Ordinal)).ToList(); + if (resolved == null) + { + Assert.Empty(absorbed); + Assert.NotEmpty(factory()!); + } + else + { + Assert.Single(absorbed); + Assert.Contains(resolved.Message, absorbed[0], StringComparison.Ordinal); + } + } + finally + { + wrapper?.Dispose(); + EnableMgxResilience.ActiveHandler = null; + ResiliencePipelineFactory.Reset(); + } + } + + // --- 7. mgx's own requests while the injection is armed --- + + /// + /// Enable-MgxResilience leaves a wrapper of ours on GraphSession.GraphHttpClient. When + /// BuildCleanHttpClient cannot run - the Graph SDK's auth helpers are not in this process, and + /// in production any SDK build that moved them, or any throw out of provider construction - + /// GetClient borrows the session's client instead, and borrowing that wrapper puts mgx's own + /// pipeline on top of the one already inside it. + /// + /// Two pipelines answer a request as happily as one, so nothing above the wire says which is + /// running. What multiplies is the retry budget the caller configured: each layer retries the + /// one beneath it, so a throttled request sends the square of its attempts at a service that + /// is already throttling, and telemetry books each attempt once per layer. + /// + /// + [Fact] + public void The_borrowed_session_client_is_the_genuine_one_not_the_wrapper_over_it() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.TooManyRequests, + """{"error":{"code":"TooManyRequests","message":"slow down"}}""", + new Dictionary { ["Retry-After"] = "0" }); + using var sdkClient = new HttpClient(wire) + { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + + using var scope = GraphSessionScope.Arm( + sdkClient, + GraphSessionScope.AuthContextFor( + "11111111-1111-1111-1111-111111111111", "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")); + + // Two retries, so one pipeline serves three attempts to a request that keeps being + // refused and two serve nine - far enough apart that no timing or ordering detail reads + // as the other. The breaker never evaluates at either count, so the second invocation + // reaches the wire on the same terms as the first. Options before Enable: the wrapper + // captures a pipeline when it is built, and it has to be the one under test. + MgxCmdletBase.SetClientOptions(new ResilientGraphClientOptions + { + MaxRetryAttempts = 2, + MaxRetryAfterSeconds = 1, + NoRateLimit = true, + NoAdaptivePacing = true, + CircuitBreakerMinThroughput = 1000 + }); + ResiliencePipelineFactory.Reset(); + + var wrapper = Enable(scope, sdkClient); + + // No transport seam armed: the borrow is inside GetClient, which the seam sits above. + var warnings = new List(); + Invoke(warnings); + var firstRun = wire.RequestCount; + Invoke(warnings); + var secondRun = wire.RequestCount - firstRun; + + Assert.Same(wrapper, scope.Session.GraphHttpClient); // the injection is still armed + Assert.Equal(3, firstRun); + Assert.Equal(3, secondRun); + + // The transport GetClient built its ResilientGraphClient on, measured on the object graph + // rather than on mgx's bookkeeping about it. That client is the one resilience layer this + // request path is meant to have, so anything under the transport is a second one. + var (layers, innermost) = MeasureWrap(BorrowedTransport()); + Assert.True(layers == 0, $"mgx borrowed a transport {layers} resilience layers deep"); + Assert.Same(sdkClient, innermost); + + // The borrowed client is compared against the session's on every invocation, to catch a + // Connect-MgGraph that swapped it. Comparing the wrapper the session holds against the + // genuine client mgx borrowed reads as a swap every time: it rebuilds, and warns again, + // for the rest of the session. + Assert.Equal(1, warnings.Count(w => + w.Contains("Falling back to SDK client", StringComparison.Ordinal))); + + void Invoke(List collected) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("Invoke-MgxRequest -Uri /users/u1 -ErrorAction SilentlyContinue | Out-Null"); + ps.Invoke(); + collected.AddRange(ps.Streams.Warning.Select(w => w.Message)); + } + } + + /// + /// What the teardown releases, and what it leaves standing. The handler and the client + /// wrapped around it go with the statics; the Polly pipeline does not - the factory keeps + /// one instance and hands the same one to every client it builds, so it survives an SDK + /// injection being taken off. Releasing it would mean Reset(), and that would take the + /// circuit-breaker history, the rate limiter and the learned pacing that mgx's own cmdlets + /// go on using. + /// + [Fact] + public void Disable_leaves_the_pipeline_the_factory_shares_with_mgxs_own_cmdlets() + { + using var scope = GraphSessionScope.Arm(); + using var sdkClient = new HttpClient(new OkHandler()) + { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + Enable(scope, sdkClient); + + var options = new ResilientGraphClientOptions + { + MaxRetryAttempts = 1, + NoRateLimit = true, + NoAdaptivePacing = true + }; + var before = ResiliencePipelineFactory.GetOrCreate(options).Pipeline; + + try + { + DisableThroughTheCmdlet(); + + Assert.Null(EnableMgxResilience.ResilientSdkClient); + Assert.Null(EnableMgxResilience.ActiveHandler); + Assert.Same(sdkClient, scope.Session.GraphHttpClient); + Assert.Same(before, ResiliencePipelineFactory.GetOrCreate(options).Pipeline); + } + finally + { + ResiliencePipelineFactory.Reset(); + } + } + + /// + /// The transport GetClient last built a client on. MgxCmdletBase keeps it private, and it is + /// read the way MeasureWrap reads a handler chain: what mgx sends through is the property + /// under test, and no output above it tells one transport from another. + /// + private static HttpClient BorrowedTransport() + { + var field = typeof(MgxCmdletBase).GetField( + "s_graphHttpClient", BindingFlags.Static | BindingFlags.NonPublic); + Assert.True(field != null, "MgxCmdletBase no longer keeps its transport in s_graphHttpClient"); + var transport = field!.GetValue(null) as HttpClient; + Assert.True(transport != null, "GetClient did not build a transport"); + return transport!; + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ResilienceWrapTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ResilienceWrapTests.cs index c9ff656..44575ea 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/ResilienceWrapTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/ResilienceWrapTests.cs @@ -1,5 +1,7 @@ using System.Reflection; +using Mgx.Cmdlets.Base; using Mgx.Cmdlets.Cmdlets.Configuration; +using Mgx.Cmdlets.Models; using Mgx.Engine.Http; namespace Mgx.IntegrationTests; @@ -15,11 +17,9 @@ public class ResilienceWrapTests { private static HttpClient? BuildWrapper(HttpClient sdkClient, List warnings) { - var build = typeof(EnableMgxResilience).GetMethod( - "BuildResilientSdkClient", BindingFlags.NonPublic | BindingFlags.Static)!; try { - return (HttpClient?)build.Invoke(null, [sdkClient, (Action)warnings.Add]); + return EnableMgxResilience.BuildResilientSdkClient(sdkClient, warnings.Add); } finally { @@ -110,4 +110,307 @@ public async Task A_throttle_through_the_wrapper_is_visible_to_mgx() Assert.True(snapshot.ThrottleRetries > 0, $"mgx recorded {snapshot.ThrottleRetries} throttle retries for a 429 the wire served"); } + + // --- the preview --- + + /// + /// Puts the process where a successful Enable-MgxResilience leaves it, and then has the SDK + /// replace the wrapper - a Connect-MgGraph or a Set-MgRequestContext does that. It is the one + /// arrangement whose Enable does more than report itself already active, so it is the one that + /// says what a preview costs. + /// + private static HttpClient InjectThenLetTheSdkReplaceIt( + GraphSessionScope scope, HttpClient genuine, HttpClient replacement) + { + var wrapper = InjectAndLeaveItInstalled(scope, genuine); + scope.Session.GraphHttpClient = replacement; + return wrapper; + } + + /// + /// Puts the process where a successful Enable-MgxResilience leaves it and stops there: the + /// wrapper is still the client the SDK is sending through, so it is this module's to take off. + /// + private static HttpClient InjectAndLeaveItInstalled(GraphSessionScope scope, HttpClient genuine) + { + var wrapper = EnableMgxResilience.BuildResilientSdkClient(genuine, _ => { })!; + EnableMgxResilience.OriginalSdkClient = genuine; + EnableMgxResilience.ResilientSdkClient = wrapper; + EnableMgxResilience.IsEnabled = true; + scope.Session.GraphHttpClient = wrapper; + return wrapper; + } + + /// Runs a cmdlet where PowerShell runs it, and returns the warnings it wrote. + private static IReadOnlyList RunCmdlet(string name, bool whatIf = false) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(EnableMgxResilience).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + var command = ps.AddCommand(name); + if (whatIf) command.AddParameter("WhatIf", true); + ps.Invoke(); + + Assert.Empty(ps.Streams.Error.Select(e => e.FullyQualifiedErrorId)); + return [.. ps.Streams.Warning.Select(w => w.Message)]; + } + + private static HttpClient NewSdkClient() => + new() { BaseAddress = new Uri("https://graph.microsoft.com/v1.0/") }; + + /// What Get-MgxResilience reports right now. + private static MgxResilienceOutput Reported() + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(EnableMgxResilience).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Get-MgxResilience"); + var results = ps.Invoke(); + return Assert.IsType(Assert.Single(results).BaseObject); + } + + /// + /// -WhatIf reports what a run would do; it does not do any of it. The injection is shared + /// state - the wrapper the SDK is sending through, the handler holding it, and the pipeline + /// factory's circuit-breaker history, rate limiter and learned pacing - and a session that + /// asks what would happen keeps all of it. + /// + [Fact] + public void A_preview_leaves_the_injection_and_the_pipeline_where_it_found_them() + { + using var scope = GraphSessionScope.Arm(); + using var genuine = NewSdkClient(); + using var replacement = NewSdkClient(); + var wrapper = InjectThenLetTheSdkReplaceIt(scope, genuine, replacement); + + var handler = EnableMgxResilience.ActiveHandler; + Assert.NotNull(handler); + var (pipeline, rateLimiter) = ResiliencePipelineFactory.GetOrCreate(MgxCmdletBase.s_clientOptions); + + Assert.Empty(RunCmdlet("Enable-MgxResilience", whatIf: true)); + + Assert.Same(wrapper, EnableMgxResilience.ResilientSdkClient); + Assert.Same(handler, EnableMgxResilience.ActiveHandler); + Assert.Same(genuine, EnableMgxResilience.OriginalSdkClient); + Assert.True(EnableMgxResilience.IsEnabled); + Assert.Same(replacement, scope.Session.GraphHttpClient); + + // The factory hands back what it is holding, so the same pipeline and the same limiter + // mean nothing rebuilt them - and rebuilding is how the history in them is lost. + var (pipelineAfter, rateLimiterAfter) = + ResiliencePipelineFactory.GetOrCreate(MgxCmdletBase.s_clientOptions); + Assert.Same(pipeline, pipelineAfter); + Assert.Same(rateLimiter, rateLimiterAfter); + } + + /// + /// The acceptance for the ShouldProcess gate says a preview performs none of the run's state + /// changes. Two of those changes are the pre-initialization of mgx's own client and the SDK + /// probe - and the probe is the one that sends a real Graph request and makes the SDK build a + /// client on the session. This drives the one arrangement where the probe would run: a + /// connected session whose GraphHttpClient is not built yet. + /// + [Fact] + public void A_preview_runs_neither_the_pre_init_nor_the_sdk_probe() + { + var marker = Path.Combine(Path.GetTempPath(), $"mgx-preview-probe-{Guid.NewGuid():N}.txt"); + + using var scope = GraphSessionScope.Arm( + graphHttpClient: null, + authContext: GraphSessionScope.AuthContextFor("tenant-a", "app-a")); + + var (pipeline, limiter) = ResiliencePipelineFactory.GetOrCreate(MgxCmdletBase.s_clientOptions); + + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(EnableMgxResilience).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + // Stands in for the SDK cmdlet the init probe shells out to. Its only job is to record + // that the probe reached the wire at all. + ps.AddScript( + "function global:Invoke-MgGraphRequest { param($Method,$Uri,$ErrorAction,$WarningAction) " + + $"Set-Content -LiteralPath '{marker}' -Value 'probe ran' }}"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Enable-MgxResilience").AddParameter("WhatIf", true); + var thrown = Record.Exception(() => ps.Invoke()); + + try + { + Assert.False(File.Exists(marker), + "-WhatIf ran the SDK init probe: it sent a Graph request and made the SDK build a client"); + Assert.Null(thrown); + Assert.Empty(ps.Streams.Error.Select(e => e.FullyQualifiedErrorId)); + Assert.Null(scope.Session.GraphHttpClient); + Assert.False(EnableMgxResilience.IsEnabled); + Assert.Null(EnableMgxResilience.ResilientSdkClient); + Assert.Null(EnableMgxResilience.ActiveHandler); + + var (pipelineAfter, limiterAfter) = + ResiliencePipelineFactory.GetOrCreate(MgxCmdletBase.s_clientOptions); + Assert.Same(pipeline, pipelineAfter); + Assert.Same(limiter, limiterAfter); + } + finally + { + if (File.Exists(marker)) File.Delete(marker); + } + } + + /// + /// Disable-MgxResilience deliberately leaves the pipeline factory alone: its circuit-breaker + /// history, rate limiter and learned pacing are shared with mgx's own cmdlets, which go on + /// running after the SDK injection comes off. A preview of the Enable must not do to that + /// state what the Disable itself refuses to do, and must leave the Disable the wrapper it is + /// there to release. + /// + [Fact] + public void A_disable_after_a_preview_releases_the_wrapper_the_preview_left() + { + using var scope = GraphSessionScope.Arm(); + using var genuine = NewSdkClient(); + using var replacement = NewSdkClient(); + var wrapper = InjectThenLetTheSdkReplaceIt(scope, genuine, replacement); + var (pipeline, rateLimiter) = ResiliencePipelineFactory.GetOrCreate(MgxCmdletBase.s_clientOptions); + + RunCmdlet("Enable-MgxResilience", whatIf: true); + + // What the Disable is about to release, still there for it to release. + Assert.Same(wrapper, EnableMgxResilience.ResilientSdkClient); + + RunCmdlet("Disable-MgxResilience"); + + Assert.Same(replacement, scope.Session.GraphHttpClient); + Assert.False(EnableMgxResilience.IsEnabled); + Assert.Null(EnableMgxResilience.ResilientSdkClient); + Assert.Null(EnableMgxResilience.ActiveHandler); + + var (pipelineAfter, rateLimiterAfter) = + ResiliencePipelineFactory.GetOrCreate(MgxCmdletBase.s_clientOptions); + Assert.Same(pipeline, pipelineAfter); + Assert.Same(rateLimiter, rateLimiterAfter); + } + + // --- whose client is on the session when the injection comes off --- + + /// + /// A Connect-MgGraph to another tenant builds a fresh SDK client and hands it to the session, + /// so the client Enable-MgxResilience saved is the previous tenant's. Installing that on the + /// way out puts those credentials back under every SDK cmdlet - and says the session was + /// restored to original behavior while it does it. The injection's own state still goes: what + /// the session is holding is not this module's, but the handler, the wrapper and the flag are. + /// + [Fact] + public void A_disable_leaves_a_client_the_sdk_replaced_after_the_injection() + { + using var scope = GraphSessionScope.Arm(); + using var genuine = NewSdkClient(); + using var replacement = NewSdkClient(); + InjectThenLetTheSdkReplaceIt(scope, genuine, replacement); + + var warnings = RunCmdlet("Disable-MgxResilience"); + + Assert.Same(replacement, scope.Session.GraphHttpClient); + Assert.Contains(warnings, w => w.Contains("left as it was found", StringComparison.Ordinal)); + Assert.False(EnableMgxResilience.IsEnabled); + Assert.Null(EnableMgxResilience.ResilientSdkClient); + Assert.Null(EnableMgxResilience.OriginalSdkClient); + Assert.Null(EnableMgxResilience.ActiveHandler); + + // The state Get-MgxResilience reports is the state the release left, not a session that + // still has something to re-inject. + var reported = Reported(); + Assert.False(reported.IsEnabled); + Assert.False(reported.IsActive); + Assert.Null(reported.Warning); + } + + /// + /// The other branch, unchanged: the wrapper is still the client the SDK is sending through, + /// so it is this module's to take off and the client it bridges to is the one to put back. + /// + [Fact] + public void A_disable_restores_the_original_while_the_wrapper_is_still_installed() + { + using var scope = GraphSessionScope.Arm(); + using var genuine = NewSdkClient(); + var wrapper = InjectAndLeaveItInstalled(scope, genuine); + + var warnings = RunCmdlet("Disable-MgxResilience"); + + Assert.Empty(warnings); + Assert.Same(genuine, scope.Session.GraphHttpClient); + Assert.False(EnableMgxResilience.IsEnabled); + Assert.Null(EnableMgxResilience.ResilientSdkClient); + Assert.Null(EnableMgxResilience.OriginalSdkClient); + Assert.Null(EnableMgxResilience.ActiveHandler); + + var reported = Reported(); + Assert.False(reported.IsEnabled); + Assert.False(reported.IsActive); + Assert.Null(reported.Warning); + } + + /// What the gate wrote when Disable-MgxResilience was previewed. + private static List PreviewTheDisable() => + RecordingHost.Preview( + typeof(EnableMgxResilience).Assembly, + ps => ps.AddCommand("Disable-MgxResilience").AddParameter("WhatIf", true)); + + private const string RestoreAction = "Restore original SDK HttpClient"; + private const string LeaveAloneAction = "leave the SDK HttpClient as found"; + + /// + /// A preview of the branch above. It restores nothing - the client the session is holding is + /// not this module's to take off - so a caller told a restore would happen was told the one + /// thing this run will not do, and the report is all -WhatIf gives them to act on. + /// + [Fact] + public void A_preview_over_a_client_the_sdk_replaced_names_the_leave_alone() + { + using var scope = GraphSessionScope.Arm(); + using var genuine = NewSdkClient(); + using var replacement = NewSdkClient(); + var wrapper = InjectThenLetTheSdkReplaceIt(scope, genuine, replacement); + + var lines = PreviewTheDisable(); + + Assert.Contains(lines, l => l.Contains(LeaveAloneAction, StringComparison.Ordinal)); + Assert.DoesNotContain(lines, l => l.Contains(RestoreAction, StringComparison.Ordinal)); + + // And it did none of it: the release the action names is still the Disable's to do. + Assert.Same(replacement, scope.Session.GraphHttpClient); + Assert.Same(wrapper, EnableMgxResilience.ResilientSdkClient); + Assert.True(EnableMgxResilience.IsEnabled); + } + + /// + /// The other branch previewed. Here the wrapper is still installed, so a restore is exactly + /// what a real run would do and exactly what the gate has to say. + /// + [Fact] + public void A_preview_over_the_wrapper_still_installed_names_the_restore() + { + using var scope = GraphSessionScope.Arm(); + using var genuine = NewSdkClient(); + var wrapper = InjectAndLeaveItInstalled(scope, genuine); + + var lines = PreviewTheDisable(); + + Assert.Contains(lines, l => l.Contains(RestoreAction, StringComparison.Ordinal)); + Assert.DoesNotContain(lines, l => l.Contains(LeaveAloneAction, StringComparison.Ordinal)); + + Assert.Same(wrapper, scope.Session.GraphHttpClient); + Assert.Same(genuine, EnableMgxResilience.OriginalSdkClient); + Assert.True(EnableMgxResilience.IsEnabled); + } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/ResponseFidelityTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/ResponseFidelityTests.cs new file mode 100644 index 0000000..60ead76 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/ResponseFidelityTests.cs @@ -0,0 +1,447 @@ +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// Responses that are not the JSON entity the caller expected - empty bodies, HTML error +/// pages from a proxy, truncated JSON - surface as verbose output or error records, never +/// as an unhandled exception. (Corpus: GraphSDK-1425/2088, non-JSON responses.) +/// +[Collection("Pipeline")] +public class ResponseFidelityTests +{ + private static System.Management.Automation.PowerShell CreateShell() + { + var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + return ps; + } + + [Fact] + public void A_204_on_a_get_emits_nothing_and_no_error() + { + var wire = new MockHttpHandler(); + wire.QueueEmpty(HttpStatusCode.NoContent); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1"); + var output = ps.Invoke(); + + Assert.Empty(output); + Assert.Empty(ps.Streams.Error); + } + } + + [Fact] + public void A_200_with_a_zero_length_body_emits_nothing_and_no_error() + { + var wire = new MockHttpHandler(); + wire.QueueEmpty(HttpStatusCode.OK); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1"); + var output = ps.Invoke(); + + Assert.Empty(output); + Assert.Empty(ps.Streams.Error); + } + } + + [Fact] + public void An_html_body_is_an_error_record_naming_the_content_type() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "proxy says no", contentType: "text/html"); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1"); + var output = ps.Invoke(); + + Assert.Empty(output); + var error = Assert.Single(ps.Streams.Error); + Assert.StartsWith("NonJsonResponse", error.FullyQualifiedErrorId); + Assert.Contains("text/html", error.Exception.Message); + } + } + + [Fact] + public void Raw_passes_a_non_json_body_through_as_text() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "plain text payload", contentType: "text/plain"); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1").AddParameter("Raw", true); + var output = ps.Invoke(); + + var item = Assert.Single(output); + Assert.Equal("plain text payload", item.BaseObject); + Assert.Empty(ps.Streams.Error); + } + } + + [Fact] + public void Malformed_json_is_an_error_record_with_a_body_snippet() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "{\"id\": \"u1\", \"displayNa"); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1"); + var output = ps.Invoke(); + + Assert.Empty(output); + var error = Assert.Single(ps.Streams.Error); + Assert.StartsWith("MalformedJsonResponse", error.FullyQualifiedErrorId); + Assert.Contains("displayNa", error.Exception.Message); + } + } + + [Fact] + public void A_malformed_page_during_enumeration_is_an_error_record_not_a_crash() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "not a collection envelope at all"); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users").AddParameter("All", true); + ps.Invoke(); + + var error = Assert.Single(ps.Streams.Error); + Assert.StartsWith("MalformedJsonResponse", error.FullyQualifiedErrorId); + } + } + + [Fact] + public void A_bom_prefixed_json_body_parses() + { + var wire = new MockHttpHandler(); + var bom = new byte[] { 0xEF, 0xBB, 0xBF }; + var body = System.Text.Encoding.UTF8.GetBytes("""{"id":"u1","displayName":"BOM"}"""); + wire.QueueBytes(HttpStatusCode.OK, [.. bom, .. body], "application/json"); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1"); + var output = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + var item = Assert.Single(output); + Assert.Equal("BOM", ((System.Collections.Hashtable)item.BaseObject)["displayName"]); + } + } + + /// + /// The fan-out read is the same read. It deserialized the raw bytes, so the BOM that the + /// single-entity path accepts lost every entity to a per-ID FanOutError, and the declared + /// charset, the empty body and -Raw were decided differently there than everywhere else. + /// + [Fact] + public void A_bom_prefixed_body_parses_on_the_entity_fan_out_too() + { + var wire = new MockHttpHandler(); + var bom = new byte[] { 0xEF, 0xBB, 0xBF }; + var body = System.Text.Encoding.UTF8.GetBytes("""{"id":"u1","displayName":"BOM"}"""); + // Both IDs are answered the same way: the two requests race, so the queue order + // cannot say which entity gets which response. + wire.QueueBytes(HttpStatusCode.OK, [.. bom, .. body], "application/json"); + wire.QueueBytes(HttpStatusCode.OK, [.. bom, .. body], "application/json"); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddScript("'u1','u2' | Invoke-MgxRequest -Uri '/users/{id}'"); + var output = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + Assert.Equal(2, output.Count); + Assert.All(output, item => + Assert.Equal("BOM", ((System.Collections.Hashtable)item.BaseObject)["displayName"])); + } + } + + [Fact] + public void An_entity_fan_out_body_is_decoded_by_its_declared_charset() + { + var wire = new MockHttpHandler(); + var body = System.Text.Encoding.Unicode.GetBytes("""{"id":"u1","displayName":"charset"}"""); + wire.QueueBytes(HttpStatusCode.OK, body, "application/json; charset=utf-16"); + wire.QueueBytes(HttpStatusCode.OK, body, "application/json; charset=utf-16"); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddScript("'u1','u2' | Invoke-MgxRequest -Uri '/users/{id}'"); + var output = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + Assert.Equal(2, output.Count); + Assert.All(output, item => + Assert.Equal("charset", ((System.Collections.Hashtable)item.BaseObject)["displayName"])); + } + } + + [Fact] + public void Raw_passes_a_non_json_fan_out_body_through_as_text() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "plain text payload", contentType: "text/plain"); + wire.QueueResponse(HttpStatusCode.OK, "plain text payload", contentType: "text/plain"); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddScript("'u1','u2' | Invoke-MgxRequest -Uri '/users/{id}' -Raw"); + var output = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + Assert.Equal(2, output.Count); + Assert.All(output, item => Assert.Equal("plain text payload", item.BaseObject)); + } + } + + /// + /// The relation fan-out reads a body the same way, and deserialized the raw bytes: a + /// byte-order mark lost the relation to a per-URL error and left the object carrying null. + /// + [Fact] + public void A_bom_prefixed_relation_body_is_attached() + { + var wire = new MockHttpHandler(); + var bom = new byte[] { 0xEF, 0xBB, 0xBF }; + var body = System.Text.Encoding.UTF8.GetBytes("""{"id":"m1","displayName":"BOM"}"""); + wire.QueueBytes(HttpStatusCode.OK, [.. bom, .. body], "application/json"); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddScript( + "[pscustomobject]@{ id = 'u1' } | Expand-MgxRelation -Uri '/users/{id}/manager' -As manager -Flatten"); + var output = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + var item = Assert.Single(output); + var manager = Assert.IsType( + item.Properties["manager"].Value); + Assert.Equal("BOM", manager["displayName"]); + } + } + + /// + /// A relation is not always an entity. Graph answers /$count with the number in the body + /// and text/plain on it, in both the bare and the charset-carrying form, and the count is + /// what the caller asked for - Expand-MgxRelation has no -Raw to receive it any other way. + /// + [Theory] + [InlineData("text/plain")] + [InlineData("text/plain; charset=utf-8")] + [InlineData("application/json")] + public void A_count_relation_attaches_the_count_whatever_type_it_declares(string contentType) + { + var wire = new MockHttpHandler(); + wire.QueueBytes(HttpStatusCode.OK, System.Text.Encoding.UTF8.GetBytes("5"), contentType); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddScript( + "[pscustomobject]@{ id = 'g1' } | Expand-MgxRelation -Uri '/groups/{id}/members/$count'" + + " -As memberCount -Flatten -ConsistencyLevel eventual"); + var output = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + var item = Assert.Single(output); + var count = Assert.IsType( + item.Properties["memberCount"].Value); + Assert.Equal(5L, count["Value"]); + } + } + + /// The ordinary relation, read as it always was. + [Fact] + public void A_json_relation_body_is_attached_as_the_entity() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, """{"id":"m1","displayName":"Ann"}"""); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddScript( + "[pscustomobject]@{ id = 'u1' } | Expand-MgxRelation -Uri '/users/{id}/manager' -As manager -Flatten"); + var output = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + var item = Assert.Single(output); + var manager = Assert.IsType( + item.Properties["manager"].Value); + Assert.Equal("m1", manager["id"]); + Assert.Equal("Ann", manager["displayName"]); + } + } + + /// + /// The other half of reading a body by what it carries: a proxy's HTML page is no relation + /// in any encoding, and stays a per-URL error naming what came back instead. + /// + [Fact] + public void An_html_relation_body_is_an_error_naming_the_content_type() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "proxy says no", contentType: "text/html"); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddScript( + "[pscustomobject]@{ id = 'u1' } | Expand-MgxRelation -Uri '/users/{id}/manager' -As manager -Flatten"); + var output = ps.Invoke(); + + var error = Assert.Single(ps.Streams.Error); + Assert.StartsWith("ExpandRelationError", error.FullyQualifiedErrorId, StringComparison.Ordinal); + Assert.Contains("text/html", error.Exception.Message, StringComparison.Ordinal); + var item = Assert.Single(output); + Assert.Null(item.Properties["manager"].Value); + } + } + + [Fact] + public void A_utf16_body_is_decoded_by_its_declared_charset() + { + var wire = new MockHttpHandler(); + wire.QueueBytes(HttpStatusCode.OK, + System.Text.Encoding.Unicode.GetBytes("plain utf-16 text"), + "text/plain; charset=utf-16"); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1").AddParameter("Raw", true); + var output = ps.Invoke(); + + var item = Assert.Single(output); + Assert.Equal("plain utf-16 text", item.BaseObject); + } + } + + /// + /// A charset .NET knows and refuses to construct. Encoding.GetEncoding answers + /// NotSupportedException for utf-7 rather than the ArgumentException every unknown name + /// raises, so the fallback to UTF-8 was one exception type short and a response header + /// ended the pipeline with a .NET deprecation message. + /// + [Theory] + [InlineData("application/json; charset=utf-7")] + [InlineData("application/json; charset=\"UTF-7\"")] + [InlineData("application/json; charset=windows-1252")] + public void A_charset_dotnet_will_not_construct_falls_back_to_utf8(string contentType) + { + var wire = new MockHttpHandler(); + wire.QueueBytes(HttpStatusCode.OK, + System.Text.Encoding.UTF8.GetBytes("{\"displayName\":\"charset\"}"), + contentType); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users/u1"); + var output = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + var item = Assert.Single(output); + Assert.Equal("charset", ((System.Collections.Hashtable)item.BaseObject)["displayName"]); + } + } + + /// Same header on the write path, which reads the body the same way. + [Fact] + public void A_charset_dotnet_will_not_construct_does_not_end_a_write() + { + var wire = new MockHttpHandler(); + wire.QueueBytes(HttpStatusCode.OK, + System.Text.Encoding.UTF8.GetBytes("{\"displayName\":\"charset\"}"), + "application/json; charset=utf-7"); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users/u1") + .AddParameter("Method", "PATCH") + .AddParameter("Body", "{}") + .AddParameter("Confirm", false); + var output = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + var item = Assert.Single(output); + Assert.Equal("charset", ((System.Collections.Hashtable)item.BaseObject)["displayName"]); + } + } + + [Fact] + public void A_malformed_export_page_is_an_error_record_not_a_crash() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "not a collection envelope"); + using var transport = MgxTransportScope.Inject(wire); + var outFile = Path.Combine(Path.GetTempPath(), $"mgx-export-{Guid.NewGuid():N}.jsonl"); + try + { + using var ps = CreateShell(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", outFile); + ps.Invoke(); + + var error = Assert.Single(ps.Streams.Error); + Assert.StartsWith("MalformedJsonResponse", error.FullyQualifiedErrorId); + } + finally + { + File.Delete(outFile); + } + } + + [Fact] + public void A_304_from_a_conditional_download_is_not_an_error() + { + var wire = new MockHttpHandler(); + wire.QueueEmpty((HttpStatusCode)304); + // Get-MgxContent refuses a transport mgx does not own, so this one claims ownership. + using (MgxTransportScope.Inject(wire, owned: true)) + { + using var ps = CreateShell(); + ps.AddScript(""" + Get-MgxContent '/me/drive/items/x/content' -Headers @{ 'If-None-Match' = '"etag"' } + """); + var output = ps.Invoke(); + + Assert.Empty(output); + Assert.Empty(ps.Streams.Error); + } + } + + [Fact] + public void A_write_answered_with_html_is_an_error_record() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "gateway", contentType: "text/html"); + using (MgxTransportScope.Inject(wire)) + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("Method", "POST") + .AddParameter("Body", new System.Collections.Hashtable { ["displayName"] = "x" }) + .AddParameter("Confirm", false); + var output = ps.Invoke(); + + Assert.Empty(output); + var error = Assert.Single(ps.Streams.Error); + Assert.StartsWith("NonJsonResponse", error.FullyQualifiedErrorId); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/SetOptionGuardTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/SetOptionGuardTests.cs index 0569e27..f8bad89 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/SetOptionGuardTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/SetOptionGuardTests.cs @@ -1,6 +1,11 @@ using System.Management.Automation; +using System.Net; +using System.Reflection; +using Mgx.Cmdlets; using Mgx.Cmdlets.Base; +using Mgx.Cmdlets.Models; using Mgx.Engine.Http; +using Microsoft.Graph.PowerShell.Authentication; namespace Mgx.IntegrationTests; @@ -82,4 +87,338 @@ public void Setting_NoRateLimit_beside_BatchItemsPerSecond_says_which_one_wins() reset.Invoke(); } } + + /// What Get-MgxOption reports right now. + private static MgxOptionOutput Reported() + { + using var ps = Shell(); + ps.AddCommand("Get-MgxOption"); + var results = ps.Invoke(); + return Assert.IsType(Assert.Single(results).BaseObject); + } + + /// What Get-MgxTelemetry reports right now. + private static MgxTelemetryOutput Telemetry() + { + using var ps = Shell(); + ps.AddCommand("Get-MgxTelemetry"); + var results = ps.Invoke(); + return Assert.IsType(Assert.Single(results).BaseObject); + } + + /// + /// No parameters bound (beyond the common ones PowerShell always adds) is the guard's + /// no-op case, and the gate now sits after that guard: a bare `Set-MgxOption` never reaches + /// ShouldProcess, so it previews nothing and would prompt for nothing under -Confirm either. + /// + [Fact] + public void No_parameters_previews_nothing() + { + var lines = RecordingHost.Preview( + typeof(Mgx.Cmdlets.Cmdlets.Configuration.SetMgxOption).Assembly, + ps => ps.AddCommand("Set-MgxOption") + .AddParameter("Verbose", true).AddParameter("WhatIf", true)); + + Assert.DoesNotContain(lines, l => l.Contains("What if:", StringComparison.Ordinal)); + } + + /// + /// The target a preview names is what the run would set. It was built from every bound + /// parameter, and PowerShell binds the common ones, so `Set-MgxOption -Verbose -WhatIf` + /// used to preview a change to "MgxOptions (Verbose, WhatIf)" - naming two parameters that + /// set no option at all, for a run the guard reports as unchanged. -Reset names what it + /// drops rather than what it sets, so a bound option shows up there as "ignoring", not as + /// part of the reset itself. + /// + [Theory] + [InlineData(false, "MaxRetryAttempts", 3, "MgxOptions (MaxRetryAttempts)")] + [InlineData(true, null, 0, "MgxOptions (reset to defaults)")] + [InlineData(true, "RateLimitBurst", 500, "MgxOptions (reset to defaults; ignoring RateLimitBurst)")] + public void A_preview_target_names_only_the_options_a_run_would_set( + bool reset, string? option, int value, string expected) + { + var lines = RecordingHost.Preview( + typeof(Mgx.Cmdlets.Cmdlets.Configuration.SetMgxOption).Assembly, + ps => + { + var command = ps.AddCommand("Set-MgxOption"); + if (reset) command.AddParameter("Reset", true); + if (option != null) command.AddParameter(option, value); + command.AddParameter("Verbose", true).AddParameter("WhatIf", true); + }); + + // The gate quotes its target, so the closing quote is what says the name ends there. + Assert.Contains(lines, l => l.Contains($"\"{expected}\"", StringComparison.Ordinal)); + } + + /// + /// -Reset beside a bound option drops the option - it always has - but used to do so without + /// a word about it, which reads as the option having taken effect over the defaults it did + /// not survive. The warning names what it ignored, and the value confirms it really was. + /// + [Fact] + public void Reset_beside_an_option_warns_and_names_it() + { + try + { + using var ps = Shell(); + ps.AddCommand("Set-MgxOption") + .AddParameter("Reset", true) + .AddParameter("RateLimitBurst", 500); + ps.Invoke(); + + var warned = string.Join(" ", ps.Streams.Warning.Select(w => w.Message)); + Assert.Contains("-Reset resets every option", warned); + Assert.Contains("-RateLimitBurst", warned); + Assert.Contains("ignored", warned); + + Assert.Equal(ResilientGraphClientOptions.Default.RateLimitBurst, Reported().RateLimitBurst); + } + finally + { + using var reset = Shell(); + reset.AddCommand("Set-MgxOption").AddParameter("Reset", true); + reset.Invoke(); + } + } + + /// + /// -NoRateLimit is sticky: it stays off until something turns it back on, and every later + /// call starts from the options it left behind. A caller who then tunes the limiter is + /// asking for a limiter, so a bound -RateLimitBurst or -RateLimitPerSecond clears it - + /// otherwise the burst lands on an option nothing reads and the session keeps running + /// unlimited with a rate on the books. + /// + [Theory] + [InlineData("RateLimitBurst", 777)] + [InlineData("RateLimitPerSecond", 77)] + public void A_rate_parameter_turns_rate_limiting_back_on(string parameter, int value) + { + try + { + using (var off = Shell()) + { + off.AddCommand("Set-MgxOption").AddParameter("NoRateLimit", true); + off.Invoke(); + } + + // What the second call has to clear. Without this the assertions below hold on a + // session that never turned the limiter off in the first place. + Assert.True(Reported().NoRateLimit, + "-NoRateLimit did not take, so nothing below is being cleared"); + + using (var tune = Shell()) + { + tune.AddCommand("Set-MgxOption").AddParameter(parameter, value); + tune.Invoke(); + } + + var reported = Reported(); + Assert.False(reported.NoRateLimit); + Assert.Equal(value, parameter == "RateLimitBurst" + ? reported.RateLimitBurst + : reported.RateLimitPerSecond); + } + finally + { + using var reset = Shell(); + reset.AddCommand("Set-MgxOption").AddParameter("Reset", true); + reset.Invoke(); + } + } + + /// + /// The counters are per-process too, and they are one session's record of what it sent: a + /// removal that leaves them set has Get-MgxTelemetry crediting a fresh import with the + /// requests of the import before it - the same shape as the options and the endpoint a + /// removal already puts back. + /// + [Fact] + public void A_module_removal_releases_the_telemetry_counters() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, "{\"value\":[]}"); + + MgxTelemetryCollector.Current.Reset(); + using (MgxTransportScope.Inject(wire)) + using (var request = Shell()) + { + request.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/users"); + request.Invoke(); + Assert.Empty(request.Streams.Error.Select(e => e.FullyQualifiedErrorId)); + } + + // What the removal has to release: this import's own traffic, reported as current. + Assert.True(Telemetry().Requests > 0, + "the request the seam served was never counted, so the release below proves nothing"); + + // A removal is ReleaseStaticState and then the detach; an import is OnImport. + AlcInitializer.ReleaseStaticState(); + new AlcInitializer().OnImport(); + + Assert.Equal(0L, Telemetry().Requests); + } + + /// + /// The options a session sets are per-process state, and a removal that leaves them set gives + /// the next import somebody else's tuning under a fresh module - with Get-MgxOption reporting + /// it as current, so nothing says the reset the re-import looks like never happened. + /// + [Fact] + public void A_module_removal_puts_the_options_back_to_their_defaults() + { + try + { + using (var set = Shell()) + { + set.AddCommand("Set-MgxOption") + .AddParameter("RateLimitBurst", 999) + .AddParameter("RateLimitPerSecond", 42) + .AddParameter("NoRateLimit", true) + .AddParameter("RateLimitQueueLimit", 12345) + .AddParameter("NoAdaptivePacing", true) + .AddParameter("MaxRetryAfterSeconds", 7) + .AddParameter("MaxRetryAttempts", 3) + .AddParameter("TotalTimeoutSeconds", 111) + .AddParameter("AttemptTimeoutSeconds", 9) + .AddParameter("CircuitBreakerDurationSeconds", 8) + .AddParameter("CircuitBreakerFailureRatio", 0.55) + .AddParameter("CircuitBreakerMinThroughput", 6) + .AddParameter("CircuitBreakerSamplingDurationSeconds", 11) + .AddParameter("BatchChunkConcurrency", 4) + .AddParameter("BatchItemsPerSecond", 0); + set.Invoke(); + } + + // Every settable option away from its default, so the removal below has all of the + // option surface to put back - not just the one property this test used to read. + var changed = Reported(); + Assert.Equal(999, changed.RateLimitBurst); + Assert.Equal(42, changed.RateLimitPerSecond); + Assert.True(changed.NoRateLimit); + Assert.Equal(12345, changed.RateLimitQueueLimit); + Assert.True(changed.NoAdaptivePacing); + Assert.Equal(7, changed.MaxRetryAfterSeconds); + Assert.Equal(3, changed.MaxRetryAttempts); + Assert.Equal(111, changed.TotalTimeoutSeconds); + Assert.Equal(9, changed.AttemptTimeoutSeconds); + Assert.Equal(8, changed.CircuitBreakerDurationSeconds); + Assert.Equal(0.55, changed.CircuitBreakerFailureRatio); + Assert.Equal(6, changed.CircuitBreakerMinThroughput); + Assert.Equal(11, changed.CircuitBreakerSamplingDurationSeconds); + Assert.Equal(4, changed.BatchChunkConcurrency); + Assert.Equal(0, changed.BatchItemsPerSecond); + + // A removal is ReleaseStaticState and then the detach; an import is OnImport. + AlcInitializer.ReleaseStaticState(); + new AlcInitializer().OnImport(); + + // The whole surface back to Default, not only the property this test used to check. + var restored = Reported(); + var defaults = ResilientGraphClientOptions.Default; + Assert.Equal(defaults.RateLimitBurst, restored.RateLimitBurst); + Assert.Equal(defaults.RateLimitPerSecond, restored.RateLimitPerSecond); + Assert.Equal(defaults.NoRateLimit, restored.NoRateLimit); + Assert.Equal(defaults.RateLimitQueueLimit, restored.RateLimitQueueLimit); + Assert.Equal(defaults.NoAdaptivePacing, restored.NoAdaptivePacing); + Assert.Equal(defaults.MaxRetryAfterSeconds, restored.MaxRetryAfterSeconds); + Assert.Equal(defaults.MaxRetryAttempts, restored.MaxRetryAttempts); + Assert.Equal(defaults.TotalTimeoutSeconds, restored.TotalTimeoutSeconds); + Assert.Equal(defaults.AttemptTimeoutSeconds, restored.AttemptTimeoutSeconds); + Assert.Equal(defaults.CircuitBreakerDurationSeconds, restored.CircuitBreakerDurationSeconds); + Assert.Equal(defaults.CircuitBreakerFailureRatio, restored.CircuitBreakerFailureRatio); + Assert.Equal(defaults.CircuitBreakerMinThroughput, restored.CircuitBreakerMinThroughput); + Assert.Equal(defaults.CircuitBreakerSamplingDurationSeconds, restored.CircuitBreakerSamplingDurationSeconds); + Assert.Equal(defaults.BatchChunkConcurrency, restored.BatchChunkConcurrency); + Assert.Equal(defaults.BatchItemsPerSecond, restored.BatchItemsPerSecond); + } + finally + { + using var reset = Shell(); + reset.AddCommand("Set-MgxOption").AddParameter("Reset", true); + reset.Invoke(); + } + } + + /// + /// The timeout the last client build sized its transport to. MgxCmdletBase keeps it private, + /// and nothing above it reports what a build is comparing against, so the field is the state + /// under test. + /// + private static FieldInfo CachedTimeoutField() + { + var field = typeof(MgxCmdletBase).GetField( + "s_cachedTotalTimeoutSeconds", BindingFlags.Static | BindingFlags.NonPublic); + Assert.True(field != null, + "MgxCmdletBase no longer keeps the cached timeout in s_cachedTotalTimeoutSeconds"); + return field!; + } + + /// + /// The endpoint and that timeout are per-process too, and derived rather than configured: a + /// removal that leaves them set has the next import composing URLs against the cloud the + /// import before it connected to, and its first build comparing against a + /// TotalTimeoutSeconds nobody in the new session chose, until that build derives both again. + /// + [Fact] + public void A_module_removal_releases_the_endpoint_and_timeout_a_build_derived() + { + const string sovereign = "https://microsoftgraph.chinacloudapi.cn"; + var timeout = CachedTimeoutField(); + var previousEndpoint = MgxCmdletBase.s_graphEndpoint; + var previousTimeout = timeout.GetValue(null); + try + { + // Where a build against a sovereign cloud leaves the process. + MgxCmdletBase.s_graphEndpoint = sovereign; + timeout.SetValue(null, 900); + + // A removal is ReleaseStaticState and then the detach; an import is OnImport. + AlcInitializer.ReleaseStaticState(); + new AlcInitializer().OnImport(); + + Assert.Equal(MgxCmdletBase.DefaultGraphEndpoint, MgxCmdletBase.s_graphEndpoint); + Assert.Equal(0, timeout.GetValue(null)); + + // And the endpoint the next build derives comes off the session it finds, not off + // anything the removal left behind. Asserted on the static a build writes, through a + // build: GetGraphEndpoint on its own reads the session and nothing else, so it answers + // the same whether the release ran or not, and whether anything drives the + // re-derivation or not. + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, "{\"value\":[]}"); + using var sessionClient = new HttpClient(wire); + + // No transport seam: it sits above the auth probe in GetClient, and the endpoint is + // derived on the build below it. GetClient cannot build a client of its own against a + // session this fake, so it borrows the one on the session - which is the mock. + using var scope = GraphSessionScope.Arm( + sessionClient, GraphSessionScope.AuthContextFor( + "11111111-1111-1111-1111-111111111111", + "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")); + scope.Session.Environment = new FakeGraphEnvironment { GraphEndpoint = sovereign }; + + using (var request = Shell()) + { + request.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("WarningAction", ActionPreference.SilentlyContinue); + request.Invoke(); + Assert.Empty(request.Streams.Error.Select(e => e.FullyQualifiedErrorId)); + } + + Assert.Equal(sovereign, MgxCmdletBase.s_graphEndpoint); + + // The static is what the cmdlets compose from, so the request that went out is the + // reading of it that matters. + Assert.StartsWith( + $"{sovereign}/v1.0/users", + Assert.Single(wire.CapturedRequests).Uri, StringComparison.Ordinal); + } + finally + { + MgxCmdletBase.s_graphEndpoint = previousEndpoint; + timeout.SetValue(null, previousTimeout); + } + } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/SkipLogicTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/SkipLogicTests.cs new file mode 100644 index 0000000..460e6b3 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/SkipLogicTests.cs @@ -0,0 +1,80 @@ +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// -SkipNotFound / -SkipForbidden behave identically on all four paths that implement +/// them: single request, GET fan-out, write fan-out, and relation expansion. The decision +/// is shared code now; these hold each path to it. +/// +[Collection("Pipeline")] +public class SkipLogicTests +{ + private const string NotFoundBody = + """{ "error": { "code": "Request_ResourceNotFound", "message": "missing" } }"""; + + private static (int Output, List Errors, List Warnings) + Run(string script) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript(script); + var output = ps.Invoke(); + return (output.Count, ps.Streams.Error.ToList(), + ps.Streams.Warning.Select(w => w.Message).ToList()); + } + + [Theory] + [InlineData("Invoke-MgxRequest -Uri /users/u1 -SkipNotFound")] + [InlineData("'u1','u2' | Invoke-MgxRequest -Uri '/users/{id}' -SkipNotFound")] + [InlineData("'u1','u2' | Invoke-MgxRequest -Uri '/users/{id}' -Method PATCH -Body @{ x = 1 } -Confirm:$false -SkipNotFound")] + [InlineData("[pscustomobject]@{ id = 'u1' } | Expand-MgxRelation -Uri '/users/{id}/manager' -As manager -SkipNotFound")] + public void A_404_is_skipped_without_an_error_on_every_path(string script) + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.NotFound, NotFoundBody); + using (MgxTransportScope.Inject(wire)) + { + var (_, errors, _) = Run(script); + Assert.Empty(errors); + } + } + + [Theory] + [InlineData("Invoke-MgxRequest -Uri /users/u1")] + [InlineData("'u1','u2' | Invoke-MgxRequest -Uri '/users/{id}'")] + [InlineData("'u1','u2' | Invoke-MgxRequest -Uri '/users/{id}' -Method PATCH -Body @{ x = 1 } -Confirm:$false")] + [InlineData("[pscustomobject]@{ id = 'u1' } | Expand-MgxRelation -Uri '/users/{id}/manager' -As manager")] + public void The_same_404_is_an_error_without_the_switch(string script) + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.NotFound, NotFoundBody); + using (MgxTransportScope.Inject(wire)) + { + var (_, errors, _) = Run(script); + Assert.NotEmpty(errors); + } + } + + [Fact] + public void Skipped_items_are_counted_in_one_warning_summary() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.NotFound, NotFoundBody); + using (MgxTransportScope.Inject(wire)) + { + var (output, errors, warnings) = Run( + "'u1','u2','u3' | Invoke-MgxRequest -Uri '/users/{id}' -SkipNotFound"); + Assert.Equal(0, output); + Assert.Empty(errors); + Assert.Contains(warnings, w => w.Contains("Skipped 3")); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/StaleTempSweepOwnershipTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/StaleTempSweepOwnershipTests.cs new file mode 100644 index 0000000..998e448 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/StaleTempSweepOwnershipTests.cs @@ -0,0 +1,145 @@ +using System.Management.Automation; +using System.Net; +using System.Text; + +namespace Mgx.IntegrationTests; + +/// +/// The stale-temp sweep runs on every fresh export, and it searched with "{output}.*.tmp" - +/// a glob whose '*' spans dots. An export to "users" therefore swept the temp of an +/// interrupted export to "users.jsonl", whose checkpoint still named it. That checkpoint then +/// resolves to a missing file and the interrupted run re-enumerates from page one, with +/// nothing to say that another mgx run deleted what it had collected. +/// +[Collection("Pipeline")] +public class StaleTempSweepOwnershipTests +{ + private const string Page1 = """ + {"value":[{"id":"u1"},{"id":"u2"}],"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=P2"} + """; + private const string Page2 = """ + {"value":[{"id":"u3"}]} + """; + private const string ServerError = """{"error":{"code":"InternalServerError","message":"boom"}}"""; + + private sealed class ScriptedHandler : HttpMessageHandler + { + private readonly Queue<(HttpStatusCode Status, string Body)> _steps = new(); + private readonly object _lock = new(); + + public void Queue(HttpStatusCode status, string body) + { + lock (_lock) _steps.Enqueue((status, body)); + } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + (HttpStatusCode Status, string Body) step; + lock (_lock) + step = _steps.Count > 0 ? _steps.Dequeue() : (HttpStatusCode.InternalServerError, ServerError); + return Task.FromResult(new HttpResponseMessage(step.Status) + { + RequestMessage = request, + Content = new StringContent(step.Body, Encoding.UTF8, "application/json") + }); + } + } + + private static PowerShell Shell() + { + var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + return ps; + } + + private static void Export(string output, string? checkpoint, bool all, int top = 0) + { + using var ps = Shell(); + var cmd = ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", output); + if (checkpoint != null) cmd.AddParameter("CheckpointPath", checkpoint); + if (all) cmd.AddParameter("All"); + if (top > 0) cmd.AddParameter("Top", top); + try { ps.Invoke(); } + catch (CmdletInvocationException) { } + } + + [Fact] + public void A_fresh_export_does_not_sweep_a_longer_named_exports_temp() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-sweep-owner-{Guid.NewGuid():N}")).FullName; + var theirs = Path.Combine(dir, "users.jsonl"); + var mine = Path.Combine(dir, "users"); + var checkpoint = Path.Combine(dir, "run.checkpoint"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + // An export to "users.jsonl" collects page 1 into its temp and dies on page 2. + handler.Queue(HttpStatusCode.OK, Page1); + Export(theirs, checkpoint, all: true); + var temp = Assert.Single(Directory.GetFiles(dir, "users.jsonl.*.tmp")); + Assert.False(File.Exists(theirs)); + + // An unrelated export to "users" in the same directory. + handler.Queue(HttpStatusCode.OK, Page1); + Export(mine, checkpoint: null, all: false, top: 1); + Assert.True(File.Exists(temp), "another export's temp file was deleted by this export's sweep"); + + // The interrupted export resumes onto what it had already collected. + handler.Queue(HttpStatusCode.OK, Page2); + Export(theirs, checkpoint, all: true); + Assert.Equal( + ["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(theirs)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The sweep deletes files it did not create, on the strength of their names, so the name + /// has to be one a run could have given its own temp: the output's, a dot, 32 hex digits, + /// ".tmp". Reaching wider took "users.jsonl.backup.tmp" - somebody else's file, in a + /// directory mgx was only ever asked to write one output into. + /// + [Fact] + public void A_fresh_export_does_not_sweep_a_tmp_no_run_could_have_written() + { + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-sweep-shape-{Guid.NewGuid():N}")).FullName; + var output = Path.Combine(dir, "users.jsonl"); + var backup = Path.Combine(dir, "users.jsonl.backup.tmp"); + try + { + var handler = new ScriptedHandler(); + using var transport = MgxTransportScope.Inject(handler); + + var kept = new[] { "{\"id\":\"kept-1\"}", "{\"id\":\"kept-2\"}" }; + File.WriteAllLines(backup, kept); + + handler.Queue(HttpStatusCode.OK, Page1); + handler.Queue(HttpStatusCode.OK, Page2); + Export(output, checkpoint: null, all: true); + + Assert.Equal( + ["{\"id\":\"u1\"}", "{\"id\":\"u2\"}", "{\"id\":\"u3\"}"], + File.ReadAllLines(output)); + Assert.True(File.Exists(backup), "a file mgx never wrote was deleted by the sweep"); + Assert.Equal(kept, File.ReadAllLines(backup)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/StalledBodyTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/StalledBodyTests.cs new file mode 100644 index 0000000..3c52669 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/StalledBodyTests.cs @@ -0,0 +1,116 @@ +using System.Management.Automation; +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// A server that sends response headers and then stops sending the body, without closing the +/// connection. Requests go out with HttpCompletionOption.ResponseHeadersRead, so neither +/// HttpClient.Timeout nor the pipeline's attempt timeout bounds the content read - the engine's +/// own BodyReadTimeout is the only thing that does. The collection path applied it; the single +/// request, the write and the entity fan-out read the body on the bare cmdlet token and waited +/// for a byte that never came, until Ctrl-C. +/// +[Collection("Pipeline")] +public class StalledBodyTests +{ + private sealed class StallingHandler(HttpStatusCode status) : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var content = new StallingContent(); + content.Headers.TryAddWithoutValidation("Content-Type", "application/json"); + return Task.FromResult(new HttpResponseMessage(status) + { + RequestMessage = request, + Content = content + }); + } + } + + private static MgxTransportScope InjectStallingTransport( + HttpStatusCode status = HttpStatusCode.OK) => + MgxTransportScope.Inject(new StallingHandler(status), options: new ResilientGraphClientOptions + { + NoRateLimit = true, + MaxRetryAttempts = 1, + AttemptTimeoutSeconds = 2 + }); + + /// + /// Runs the cmdlet on a worker so a hang fails the test instead of wedging the suite. + /// + private static IReadOnlyList RunAndCollectErrors(Action build) + { + var work = Task.Run(() => + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + + build(ps); + // A terminating error carries the record too, and a stall that ends the pipeline + // rather than reporting an error record is exactly what these tests are about. + try { ps.Invoke(); } + catch (RuntimeException ex) { return (IReadOnlyList)[ex.ErrorRecord]; } + return (IReadOnlyList)ps.Streams.Error.ToList(); + }); + + Assert.True(work.Wait(TimeSpan.FromSeconds(30)), + "the cmdlet was still running 30s after the server stalled the response body"); + return work.Result; + } + + [Theory] + [InlineData("GET")] + [InlineData("PATCH")] + public void A_stalled_body_ends_the_request_instead_of_hanging(string method) + { + using var transport = InjectStallingTransport(); + var errors = RunAndCollectErrors(ps => + { + var cmd = ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users/u1") + .AddParameter("Method", method); + if (method != "GET") cmd.AddParameter("Body", "{}").AddParameter("Confirm", false); + }); + + Assert.Contains(errors, e => + e.ToString().Contains(ResilientGraphClient.BodyReadTimedOutMessage, StringComparison.Ordinal)); + } + + [Fact] + public void A_stalled_body_ends_an_entity_fan_out_instead_of_hanging() + { + using var transport = InjectStallingTransport(); + var errors = RunAndCollectErrors(ps => + ps.AddScript("'u1','u2' | Invoke-MgxRequest -Uri '/users/{id}'")); + + Assert.Contains(errors, e => + e.ToString().Contains(ResilientGraphClient.BodyReadTimedOutMessage, StringComparison.Ordinal)); + } + + /// + /// The error body on the collection path. It was the one body read in the client without + /// the timeout wrapper, so a server that stalled while sending a 500 ended the pipeline + /// with a raw TaskCanceledException: outside the error contract, and past the filter the + /// cmdlet catches its transport failures with. + /// + [Fact] + public void A_stalled_error_body_on_the_all_path_is_reported_as_a_transport_failure() + { + using var transport = InjectStallingTransport(HttpStatusCode.InternalServerError); + var errors = RunAndCollectErrors(ps => + ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("All", true)); + + var error = Assert.Single(errors); + Assert.Equal(ResilientGraphClient.BodyReadTimedOutMessage, error.Exception.Message); + Assert.StartsWith("HttpError", error.FullyQualifiedErrorId, StringComparison.Ordinal); + } +} diff --git a/tests/Mgx.IntegrationTests/Cmdlets/TopWithAllTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/TopWithAllTests.cs index a06ed49..56603c9 100644 --- a/tests/Mgx.IntegrationTests/Cmdlets/TopWithAllTests.cs +++ b/tests/Mgx.IntegrationTests/Cmdlets/TopWithAllTests.cs @@ -1,6 +1,5 @@ using System.Management.Automation; using System.Net; -using System.Reflection; using System.Text; using Mgx.Cmdlets.Base; using Mgx.Engine.Http; @@ -45,40 +44,10 @@ protected override Task SendAsync( } } - private static void InjectMock(HttpMessageHandler handler) - { - ResiliencePipelineFactory.Reset(); - var t = typeof(MgxCmdletBase); - t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, new HttpClient(handler)); - t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)! - .SetValue(null, MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - // Cleared with the fingerprint. A stale reference from an earlier test reads as a - // credential change and the injected client is rebuilt away - t.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_ownsHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, false); - t.GetField("s_graphEndpoint", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! - .SetValue(null, "https://graph.microsoft.com"); - t.GetField("s_clientOptions", BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public)! - .SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); - } - - private static void CleanupMock() - { - var t = typeof(MgxCmdletBase); - t.GetField("s_graphHttpClient", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthFingerprint", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthContextRef", BindingFlags.NonPublic | BindingFlags.Static)!.SetValue(null, null); - ResiliencePipelineFactory.Reset(); - } - private static (int Count, int Requests) Enumerate(int top, bool all, int pageSize) { var handler = new PagedHandler(pageSize, maxPages: 40); - InjectMock(handler); - try + using (MgxTransportScope.Inject(handler)) { using var ps = PowerShell.Create(); ps.AddCommand("Import-Module") @@ -98,7 +67,6 @@ private static (int Count, int Requests) Enumerate(int top, bool all, int pageSi var output = ps.Invoke(); return (output.Count, handler.RequestCount); } - finally { CleanupMock(); } } [Fact] @@ -135,7 +103,7 @@ public void Export_stops_at_Top_rather_than_walking_the_whole_collection() // The sibling of the Invoke-MgxRequest case. Both cmdlets document -Top as a total // maximum, so they must not disagree about what -All -Top means. var handler = new PagedHandler(pageSize: 50, maxPages: 40); - InjectMock(handler); + using var transport = MgxTransportScope.Inject(handler); var dir = Directory.CreateDirectory( Path.Combine(Path.GetTempPath(), $"mgx-topall-{Guid.NewGuid():N}")).FullName; var output = Path.Combine(dir, "out.jsonl"); @@ -162,7 +130,6 @@ public void Export_stops_at_Top_rather_than_walking_the_whole_collection() } finally { - CleanupMock(); try { Directory.Delete(dir, true); } catch { } } } diff --git a/tests/Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs b/tests/Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs new file mode 100644 index 0000000..7fe1c21 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Cmdlets/UriEncodingTests.cs @@ -0,0 +1,296 @@ +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// What the caller wrote is what goes on the wire: a '#' in a path stays in the path +/// instead of vanishing as a URI fragment, hostile filter values are escaped once, and +/// pre-encoded input is not encoded twice. (Corpus: GraphSDK-1947, GraphSDK-2709/2942, +/// M365DSC-5354.) +/// +[Collection("Pipeline")] +public class UriEncodingTests +{ + private static string WireUriFor(MockHttpHandler wire, Action configure) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + configure(ps); + ps.Invoke(); + var captured = wire.CapturedRequests; + Assert.NotEmpty(captured); + return captured[0].Uri; + } + + [Fact] + public void A_hash_in_a_path_reaches_the_wire_instead_of_becoming_a_fragment() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + using (MgxTransportScope.Inject(wire)) + { + var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/me/drive/root:/reports/a#b.txt")); + + Assert.Contains("/reports/a%23b.txt", uri); + Assert.DoesNotContain("#", uri); + } + } + + [Fact] + public void A_raw_filter_is_escaped_once() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); + using (MgxTransportScope.Inject(wire)) + { + var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("Filter", "displayName eq 'O''Brien & Söhne'")); + + Assert.Contains("$filter=displayName%20eq%20%27O%27%27Brien%20%26%20S%C3%B6hne%27", uri); + } + } + + [Fact] + public void A_pre_encoded_filter_is_not_encoded_twice() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); + using (MgxTransportScope.Inject(wire)) + { + var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("Filter", "displayName%20eq%20%27Bob%27")); + + Assert.Contains("$filter=displayName%20eq%20%27Bob%27", uri); + Assert.DoesNotContain("%25", uri); + } + } + + [Fact] + public void A_literal_percent_that_is_not_a_triplet_is_escaped() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); + using (MgxTransportScope.Inject(wire)) + { + var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("Filter", "displayName eq '50% off'")); + + Assert.Contains("50%25%20off", uri); + } + } + + [Fact] + public void A_piped_drive_item_id_is_escaped_into_the_content_path() + { + // Drive item ids can carry '!' and ':'; SharePoint site ids carry ','. The path is + // built from pipeline data, so the injection point escapes rather than trusts. + var wire = new MockHttpHandler(); + wire.QueueBytes(HttpStatusCode.OK, [1, 2, 3], "application/octet-stream"); + // Get-MgxContent refuses borrowed transports (the SDK's RedirectHandler would + // auto-follow the content 302); this injected client is mgx-owned. + using (MgxTransportScope.Inject(wire, owned: true)) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript(""" + [Hashtable]@{ + id = '01ABC!DEF:XYZ' + parentReference = @{ driveId = 'b!x_y' } + } | Get-MgxContent + """); + ps.Invoke(); + + var errText = string.Join(" | ", ps.Streams.Error.Select(e => e.FullyQualifiedErrorId + ": " + e.Exception.Message)); + var captured = wire.CapturedRequests; + Assert.True(captured.Count > 0, $"no request sent; errors: {errText}"); + Assert.Contains("/drives/b%21x_y/items/01ABC%21DEF%3AXYZ/content", captured[0].Uri); + } + } + + [Fact] + public void Every_hostile_filter_round_trips_through_encoding() + { + // The corpus generalizes GraphSDK-2709/2942: the wire value must decode back to + // exactly what the caller wrote, whatever it contains. + foreach (var filter in HostileInputs.FilterValues) + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); + using (MgxTransportScope.Inject(wire)) + { + var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("Filter", filter)); + // OriginalString keeps a raw '#' that the transport would then drop as a + // fragment - so round-tripping alone is not proof. No raw '#', '&' or '+' + // may survive in the encoded value; then the decode must give back the input. + var query = uri[(uri.IndexOf("$filter=", StringComparison.Ordinal) + "$filter=".Length)..]; + query = query.Split('&')[0]; + Assert.DoesNotContain('#', query); + Assert.DoesNotContain('+', query); + Assert.Equal(filter, System.Uri.UnescapeDataString(query)); + } + } + } + + [Fact] + public void A_top_already_in_the_uri_is_not_sent_twice() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); + using (MgxTransportScope.Inject(wire)) + { + var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users?$top=5") + .AddParameter("All", true)); + + Assert.Single(System.Text.RegularExpressions.Regex.Matches(uri, "top=")); + Assert.Contains("$top=5", uri); + } + } + + [Fact] + public void A_filter_already_in_the_uri_wins_over_the_parameter_with_a_warning() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); + using (MgxTransportScope.Inject(wire)) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users?$filter=accountEnabled eq true") + .AddParameter("Filter", "displayName eq 'x'"); + ps.Invoke(); + + var uri = wire.CapturedRequests[0].Uri; + Assert.Single(System.Text.RegularExpressions.Regex.Matches(uri, "filter=")); + Assert.Contains("accountEnabled", uri); + Assert.DoesNotContain("displayName", uri); + // Deferring must be loud: the caller's -Filter was not sent. + Assert.Contains(ps.Streams.Warning, w => w.Message.Contains("-Filter")); + } + } + + [Fact] + public void An_encoded_option_name_in_the_uri_still_defers_the_parameter() + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.EmptyCollection); + using (MgxTransportScope.Inject(wire)) + { + var uri = WireUriFor(wire, ps => ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users?%24top=10") + .AddParameter("All", true)); + + // %24top IS $top to the server; a second $top would draw a 400. + Assert.Single(System.Text.RegularExpressions.Regex.Matches(uri, "top=")); + } + } + + [Fact] + public void A_caller_count_in_the_uri_does_not_arm_the_count_degradation_retry() + { + // Endpoint 400s; if mgx thinks its auto-$count caused it, the "retry without + // count" rebuild is byte-identical and burns a duplicate request. + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.BadRequest, + """{ "error": { "code": "BadRequest", "message": "no" } }"""); + using (MgxTransportScope.Inject(wire)) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("Invoke-MgxRequest '/users?$count=true' -Filter \"displayName eq 'x'\" -ErrorAction SilentlyContinue"); + ps.Invoke(); + + // One request (plus zero byte-identical degradation retries): the URI's own + // $count was never mgx's to drop. + Assert.Single(wire.CapturedRequests); + } + } + + [Fact] + public void Every_hostile_path_segment_survives_id_templating() + { + // The PathSegments corpus, driven through the one path-injection mechanism a + // caller has: {id} templating. The captured request path must carry the fully + // escaped segment - no raw '#', '%', space, or comma damage. + foreach (var segment in HostileInputs.PathSegments) + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + using (MgxTransportScope.Inject(wire)) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Invoke-MgxRequest").AddParameter("Uri", "/items/{id}"); + ps.Invoke(new[] { segment }); + + var sent = Assert.Single(wire.CapturedRequests); + Assert.Contains($"/items/{System.Uri.EscapeDataString(segment)}", sent.Uri); + Assert.DoesNotContain('#', sent.Uri); + } + } + } + + /// + /// An absolute -Uri concatenated onto the versioned base URL would silently produce + /// /v1.0/https:/... on the wire. Get-MgxContent and Sync-MgxDelta already refuse it; + /// the other three URI-taking cmdlets refuse it the same way. + /// + [Theory] + [InlineData("Invoke-MgxRequest -Uri ' https://graph.microsoft.com/v1.0/users'")] + [InlineData("Invoke-MgxRequest -Uri 'https://graph.microsoft.com/v1.0/users'")] + [InlineData("[PSCustomObject]@{ id = 'g1' } | Expand-MgxRelation -Uri 'https://graph.microsoft.com/v1.0/groups/{id}/members' -As members")] + [InlineData("Export-MgxCollection -Uri 'https://graph.microsoft.com/v1.0/users' -OutputFile ([IO.Path]::GetTempFileName())")] + public void An_absolute_uri_is_refused_not_mangled(string command) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript(command); + var output = ps.Invoke(); + + Assert.Empty(output); + var error = Assert.Single(ps.Streams.Error); + Assert.StartsWith("AbsoluteUriNotAllowed", error.FullyQualifiedErrorId); + Assert.Contains("must be a relative path", error.Exception.Message); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Batch/BatchEdgeCaseTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/BatchEdgeCaseTests.cs index 1ee49c5..293b5b8 100644 --- a/tests/Mgx.IntegrationTests/Engine/Batch/BatchEdgeCaseTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Batch/BatchEdgeCaseTests.cs @@ -67,7 +67,7 @@ public async Task Batch_All20Items_429_RetriedAndSucceed() using var httpClient = new HttpClient(handler); using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + var batchClient = new GraphBatchClient(client); var operations = Enumerable.Range(1, 20) .Select(i => new BatchOperation($"/users/user{i}", "GET")) @@ -103,7 +103,7 @@ public async Task Batch_PartialRetry_10Succeed_10Throttled_ThenAllSucceed() using var httpClient = new HttpClient(handler); using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + var batchClient = new GraphBatchClient(client); var operations = Enumerable.Range(1, 20) .Select(i => new BatchOperation($"/users/user{i}", "GET")) @@ -123,8 +123,64 @@ public async Task Batch_PartialRetry_10Succeed_10Throttled_ThenAllSucceed() // ── R2-3: Mismatched response count ─────────────────────────────────────── + [Fact] + public async Task Batch_MismatchedResponseCount_IsReportedAsAChunkFailure() + { + // Send 5 items but server returns only 3 responses + var truncatedResponse = """ + { + "responses": [ + { "id": "1", "status": 200, "body": { "id": "user1" } }, + { "id": "2", "status": 200, "body": { "id": "user2" } }, + { "id": "3", "status": 200, "body": { "id": "user3" } } + ] + } + """; + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, truncatedResponse); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); + + var operations = Enumerable.Range(1, 5) + .Select(i => new BatchOperation($"/users/user{i}", "GET")) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(operations); + + var ex = Assert.IsType(result.ChunkFailure); + Assert.Contains("count mismatch", ex.Message); + Assert.Contains("sent 5", ex.Message); + Assert.Contains("received 3", ex.Message); + // The envelope cannot be matched to what was sent, so no item has an answer - but the + // requests went out, and every one of them is accounted for by position. + Assert.Equal(5, result.Results.Count); + Assert.DoesNotContain(result.Results, r => r.Response.Status == GraphBatchClient.NotSentStatus); + } + // ── R2-8: 0 items — empty batch ────────────────────────────────────────── + [Fact] + public async Task Batch_ZeroItems_ReturnsEmptyResult_NoHttpCalls() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, """{ "responses": [] }"""); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); + + var operations = new List(); + + var result = await batchClient.ExecuteBatchIndexedAsync(operations); + + Assert.Empty(result.Results); + Assert.Equal(0, handler.RequestCount); + Assert.Equal(0, result.Telemetry.TotalRequests); + } + // ── R2-8: 1 item — single item batch ───────────────────────────────────── [Fact] @@ -143,7 +199,7 @@ public async Task Batch_SingleItem_OneHttpCall_OneResult() using var httpClient = new HttpClient(handler); using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + var batchClient = new GraphBatchClient(client); var operations = new List { new("/users/user1", "GET") }; @@ -168,7 +224,7 @@ public async Task Batch_Exactly20Items_SingleHttpCall() using var httpClient = new HttpClient(handler); using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + var batchClient = new GraphBatchClient(client); var operations = Enumerable.Range(1, 20) .Select(i => new BatchOperation($"/users/user{i}", "GET")) @@ -198,7 +254,7 @@ public async Task Batch_21Items_TwoHttpCalls() using var httpClient = new HttpClient(handler); using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + var batchClient = new GraphBatchClient(client); var operations = Enumerable.Range(1, 21) .Select(i => new BatchOperation($"/users/user{i}", "GET")) diff --git a/tests/Mgx.IntegrationTests/Engine/Batch/BatchWriteTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/BatchWriteTests.cs index 333f045..99455e4 100644 --- a/tests/Mgx.IntegrationTests/Engine/Batch/BatchWriteTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Batch/BatchWriteTests.cs @@ -1,4 +1,4 @@ -using System.Net; +using System.Net; using System.Text.Json; using System.Text.Json.Nodes; using Mgx.Cmdlets.Cmdlets.Batch; @@ -10,13 +10,6 @@ namespace Mgx.IntegrationTests; [Collection("Pipeline")] public class BatchWriteTests { - /// Caps every batch retry backoff at one second, the option minimum. - private static ResilientGraphClientOptions FastRetryOptions => new() - { - NoRateLimit = true, - MaxRetryAfterSeconds = 1 - }; - private static readonly string BatchSuccessResponse = """ { "responses": [ @@ -42,8 +35,8 @@ public async Task BatchPost_SetsMethodAndBody() handler.SetDefaultResponse(HttpStatusCode.OK, BatchSuccessResponse); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); var body = JsonSerializer.Deserialize("""{"displayName":"Test"}"""); var operations = new List @@ -76,8 +69,8 @@ public async Task BatchGet_ItemHeaders_AppliedToEachItem() handler.SetDefaultResponse(HttpStatusCode.OK, BatchGetResponse); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1) + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client) { ItemHeaders = new Dictionary { @@ -123,8 +116,8 @@ public async Task BatchPost_ItemHeaders_MergedWithContentType() handler.SetDefaultResponse(HttpStatusCode.OK, singlePostResponse); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1) + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client) { ItemHeaders = new Dictionary { @@ -166,9 +159,9 @@ public async Task BatchGet_NoItemHeaders_NoHeadersInJson() handler.SetDefaultResponse(HttpStatusCode.OK, singleGetResponse); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); // No ItemHeaders set (default null) - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + var batchClient = new GraphBatchClient(client); var operations = new List { new("/users") }; @@ -189,8 +182,8 @@ public async Task BatchGet_BackwardCompatible() handler.SetDefaultResponse(HttpStatusCode.OK, BatchGetResponse); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); // Old-style call with string URLs var urls = new List { "/users/user1", "/users/user2" }; @@ -226,8 +219,8 @@ public async Task BatchPost_NonIdempotentRetryGuard() handler.QueueResponse(HttpStatusCode.OK, post503Response); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); var body = JsonSerializer.Deserialize("""{"displayName":"Test"}"""); var operations = new List { new("/users", "POST", body) }; @@ -265,8 +258,8 @@ public async Task BatchGet_503Retries() handler.QueueResponse(HttpStatusCode.OK, getSuccessResponse); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); var operations = new List { new("/users/user1", "GET") }; @@ -303,8 +296,8 @@ public async Task BatchGet_500Retries() handler.QueueResponse(HttpStatusCode.OK, getSuccessResponse); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); var operations = new List { new("/users/user1", "GET") }; var result = await batchClient.ExecuteBatchIndexedAsync(operations); @@ -338,8 +331,8 @@ public async Task BatchGet_502Retries() handler.QueueResponse(HttpStatusCode.OK, getSuccessResponse); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); var operations = new List { new("/users/user1", "GET") }; var result = await batchClient.ExecuteBatchIndexedAsync(operations); @@ -365,8 +358,8 @@ public async Task BatchPost_DoesNotRetryOn500() handler.QueueResponse(HttpStatusCode.OK, post500Response); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); var body = JsonSerializer.Deserialize("""{"displayName":"Test"}"""); var operations = new List { new("/users", "POST", body) }; @@ -393,8 +386,8 @@ public async Task BatchPost_DoesNotRetryOn502() handler.QueueResponse(HttpStatusCode.OK, post502Response); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); var body = JsonSerializer.Deserialize("""{"displayName":"Test"}"""); var operations = new List { new("/users", "POST", body) }; @@ -420,8 +413,8 @@ public async Task BatchPatch_SetsMethodAndBody() handler.SetDefaultResponse(HttpStatusCode.OK, patchResponse); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); var body = JsonSerializer.Deserialize("""{"department":"Engineering"}"""); var operations = new List { new("/users/user1", "PATCH", body) }; @@ -452,8 +445,8 @@ public async Task BatchDelete_NoBody() handler.SetDefaultResponse(HttpStatusCode.OK, deleteResponse); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); var operations = new List { @@ -501,8 +494,8 @@ public async Task BatchPost_429RetryAfter_RetriesAfterDelay() handler.QueueResponse(HttpStatusCode.OK, successResponse); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); var body = JsonSerializer.Deserialize("""{"displayName":"Test"}"""); var operations = new List { new("/users", "POST", body) }; @@ -536,8 +529,8 @@ public async Task BatchGet_ChunkingOver20_SendsMultipleBatches() handler.QueueResponse(HttpStatusCode.OK, chunk2Response); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); // 25 operations: should produce 2 batch calls (20 + 5) var operations = Enumerable.Range(1, 25) @@ -572,8 +565,8 @@ public async Task BatchPost_MixedResults_PartialSuccess() handler.QueueResponse(HttpStatusCode.OK, mixedResponse); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); var body = JsonSerializer.Deserialize("""{"displayName":"Test Group"}"""); var operations = new List @@ -606,9 +599,9 @@ public async Task BatchGet_BetaApiVersion_SendsToBetaEndpoint() handler.SetDefaultResponse(HttpStatusCode.OK, BatchGetResponse); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); // Construct with beta base URL (mirrors what InvokeMgxBatchRequest.VersionedBaseUrl produces) - var batchClient = new GraphBatchClient(client, "https://graph.microsoft.com/beta", maxRetryAfterSeconds: 1); + var batchClient = new GraphBatchClient(client, "https://graph.microsoft.com/beta"); var urls = new List { "/users/user1", "/users/user2" }; await batchClient.ExecuteBatchAsync(urls); @@ -636,7 +629,7 @@ public async Task BatchGet_RetryExhausted_ReturnsLastError() handler.QueueResponse(HttpStatusCode.OK, get503Response); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); var operations = new List { new("/users/user1", "GET") }; @@ -652,7 +645,7 @@ public async Task BatchGet_RetryExhausted_ReturnsLastError() } [Fact] - public async Task BatchGet_ResponseCountMismatch_Throws() + public async Task BatchGet_ResponseCountMismatch_IsReportedAsAChunkFailure() { // Send 3 requests but mock returns only 2 response items var truncatedResponse = """ @@ -668,8 +661,8 @@ public async Task BatchGet_ResponseCountMismatch_Throws() handler.SetDefaultResponse(HttpStatusCode.OK, truncatedResponse); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); var ops = new List { @@ -678,8 +671,9 @@ public async Task BatchGet_ResponseCountMismatch_Throws() new("/users/c"), }; - var ex = await Assert.ThrowsAsync( - () => batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None)); + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + var ex = Assert.IsType(result.ChunkFailure); Assert.Contains("count mismatch", ex.Message); Assert.Contains("sent 3", ex.Message); Assert.Contains("received 2", ex.Message); @@ -704,8 +698,8 @@ public async Task BatchGet_CrossChunkBackpressure_DelaysNextChunk() handler.QueueResponse(HttpStatusCode.OK, chunk2Success); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); var operations = Enumerable.Range(1, 25) .Select(i => new BatchOperation($"/users/user{i}", "GET")) @@ -738,8 +732,8 @@ public async Task BatchGet_NoThrottling_NoCrossChunkDelay() handler.QueueResponse(HttpStatusCode.OK, chunk2Success); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); var operations = Enumerable.Range(1, 25) .Select(i => new BatchOperation($"/users/user{i}", "GET")) @@ -788,7 +782,7 @@ public async Task BatchGet_BatchLevelRetry_RecoverAfterChunkExhaustion() handler.QueueResponse(HttpStatusCode.OK, getSuccessResponse); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); var operations = new List { new("/users/user1", "GET") }; @@ -831,7 +825,7 @@ public async Task BatchPost_BatchLevelRetry_429Recovery() handler.QueueResponse(HttpStatusCode.OK, successResponse); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); var body = JsonSerializer.Deserialize("""{"displayName":"Test"}"""); @@ -852,7 +846,10 @@ public async Task BatchPost_BatchLevelRetry_429Recovery() [Fact] public async Task BatchGet_VerboseWriter_LogsClampEvent() { - // The clamp is what the test pays in wall clock and sits at the option minimum + // Server requests 300s Retry-After, client clamps it to the cap it was built with. + // The cap is 1s rather than the 120s default so the clamp is observable without the + // test sleeping out the default. + // VerboseWriter should receive a clamping message. var throttledResponse = """ { "responses": [ @@ -873,7 +870,7 @@ public async Task BatchGet_VerboseWriter_LogsClampEvent() handler.QueueResponse(HttpStatusCode.OK, successResponse); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); var verboseMessages = new List(); @@ -883,7 +880,8 @@ public async Task BatchGet_VerboseWriter_LogsClampEvent() await batchClient.ExecuteBatchIndexedAsync(operations); batchClient.DrainVerboseMessages(); - Assert.Contains(verboseMessages, m => m.Contains("300s") && m.Contains("clamped to 1s")); + // Verify a clamping message was logged + Assert.Contains(verboseMessages, m => m.Contains("300s") && m.Contains("clamped")); } private static string BuildBatchResponse(int count, int status, int? retryAfterSeconds = null) @@ -910,10 +908,10 @@ public async Task BatchWrite_Pacing_AppliedBetweenChunks() handler.SetDefaultResponse(HttpStatusCode.OK, BuildBatchResponse(20, 201)); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); var body = JsonSerializer.Deserialize("""{"displayName":"Test"}"""); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1, batchItemsPerSecond: 20); + var batchClient = new GraphBatchClient(client, batchItemsPerSecond: 20); var ops = Enumerable.Range(1, 40) .Select(i => new BatchOperation($"/users", "POST", body)) .ToArray(); @@ -938,9 +936,9 @@ public async Task BatchGet_Pacing_DisabledWhenZero() handler.SetDefaultResponse(HttpStatusCode.OK, BuildBatchResponse(20, 200)); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1, batchItemsPerSecond: 0); + var batchClient = new GraphBatchClient(client, batchItemsPerSecond: 0); var ops = Enumerable.Range(1, 40) .Select(i => new BatchOperation($"/users/{i}")) .ToArray(); @@ -971,9 +969,9 @@ public async Task BatchGet_Pacing_SkippedWhenBackpressureActive() handler.SetDefaultResponse(HttpStatusCode.OK, BuildBatchResponse(20, 200)); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1, batchItemsPerSecond: 20); + var batchClient = new GraphBatchClient(client, batchItemsPerSecond: 20); var ops = Enumerable.Range(1, 40) .Select(i => new BatchOperation($"/users/{i}")) .ToArray(); @@ -996,7 +994,7 @@ public async Task BatchWrite_Pacing_CrossCallDelayApplied() handler.SetDefaultResponse(HttpStatusCode.OK, BuildBatchResponse(20, 201)); using var httpClient = new HttpClient(handler); - using var rgcClient = new ResilientGraphClient(httpClient, FastRetryOptions); + using var rgcClient = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); var body = JsonSerializer.Deserialize("""{"displayName":"Test"}"""); var ops = Enumerable.Range(1, 20) @@ -1004,12 +1002,12 @@ public async Task BatchWrite_Pacing_CrossCallDelayApplied() .ToArray(); // Call 1: establishes pacing baseline - var client1 = new GraphBatchClient(rgcClient, maxRetryAfterSeconds: 1, batchItemsPerSecond: 20); + var client1 = new GraphBatchClient(rgcClient, batchItemsPerSecond: 20); await client1.ExecuteBatchIndexedAsync(ops); // Call 2: should be delayed by cross-call pacing var sw = System.Diagnostics.Stopwatch.StartNew(); - var client2 = new GraphBatchClient(rgcClient, maxRetryAfterSeconds: 1, batchItemsPerSecond: 20); + var client2 = new GraphBatchClient(rgcClient, batchItemsPerSecond: 20); var result = await client2.ExecuteBatchIndexedAsync(ops); sw.Stop(); @@ -1029,18 +1027,18 @@ public async Task BatchWrite_Pacing_CrossCallDisabledWhenZero() handler.SetDefaultResponse(HttpStatusCode.OK, BuildBatchResponse(20, 201)); using var httpClient = new HttpClient(handler); - using var rgcClient = new ResilientGraphClient(httpClient, FastRetryOptions); + using var rgcClient = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); var body = JsonSerializer.Deserialize("""{"displayName":"Test"}"""); var ops = Enumerable.Range(1, 20) .Select(i => new BatchOperation($"/users", "POST", body)) .ToArray(); - var client1 = new GraphBatchClient(rgcClient, maxRetryAfterSeconds: 1, batchItemsPerSecond: 0); + var client1 = new GraphBatchClient(rgcClient, batchItemsPerSecond: 0); await client1.ExecuteBatchIndexedAsync(ops); var sw = System.Diagnostics.Stopwatch.StartNew(); - var client2 = new GraphBatchClient(rgcClient, maxRetryAfterSeconds: 1, batchItemsPerSecond: 0); + var client2 = new GraphBatchClient(rgcClient, batchItemsPerSecond: 0); var result = await client2.ExecuteBatchIndexedAsync(ops); sw.Stop(); @@ -1059,7 +1057,7 @@ public async Task BatchGet_Pacing_SkippedForGetOnlyBatch() handler.SetDefaultResponse(HttpStatusCode.OK, BuildBatchResponse(20, 200)); using var httpClient = new HttpClient(handler); - using var rgcClient = new ResilientGraphClient(httpClient, FastRetryOptions); + using var rgcClient = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); var body = JsonSerializer.Deserialize("""{"displayName":"Test"}"""); @@ -1067,7 +1065,7 @@ public async Task BatchGet_Pacing_SkippedForGetOnlyBatch() var writeOps = Enumerable.Range(1, 20) .Select(i => new BatchOperation($"/users", "POST", body)) .ToArray(); - var client1 = new GraphBatchClient(rgcClient, maxRetryAfterSeconds: 1, batchItemsPerSecond: 20); + var client1 = new GraphBatchClient(rgcClient, batchItemsPerSecond: 20); await client1.ExecuteBatchIndexedAsync(writeOps); // Call 2: GET-only batch — should NOT be paced @@ -1075,7 +1073,7 @@ public async Task BatchGet_Pacing_SkippedForGetOnlyBatch() .Select(i => new BatchOperation($"/users/{i}")) .ToArray(); var sw = System.Diagnostics.Stopwatch.StartNew(); - var client2 = new GraphBatchClient(rgcClient, maxRetryAfterSeconds: 1, batchItemsPerSecond: 20); + var client2 = new GraphBatchClient(rgcClient, batchItemsPerSecond: 20); var result = await client2.ExecuteBatchIndexedAsync(getOps); sw.Stop(); @@ -1095,7 +1093,7 @@ public async Task BatchGet_Pacing_CrossCallCappedToOneChunk() handler.SetDefaultResponse(HttpStatusCode.OK, BuildBatchResponse(20, 200)); using var httpClient = new HttpClient(handler); - using var rgcClient = new ResilientGraphClient(httpClient, FastRetryOptions); + using var rgcClient = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); var body = JsonSerializer.Deserialize("""{"displayName":"Test"}"""); @@ -1103,7 +1101,7 @@ public async Task BatchGet_Pacing_CrossCallCappedToOneChunk() var largeOps = Enumerable.Range(1, 40) .Select(i => new BatchOperation($"/users", "POST", body)) .ToArray(); - var client1 = new GraphBatchClient(rgcClient, maxRetryAfterSeconds: 1, batchItemsPerSecond: 20); + var client1 = new GraphBatchClient(rgcClient, batchItemsPerSecond: 20); await client1.ExecuteBatchIndexedAsync(largeOps); // Call 2: 20-item write batch — cross-call delay should be ~1000ms (capped), not ~2000ms @@ -1111,7 +1109,7 @@ public async Task BatchGet_Pacing_CrossCallCappedToOneChunk() .Select(i => new BatchOperation($"/users", "POST", body)) .ToArray(); var sw = System.Diagnostics.Stopwatch.StartNew(); - var client2 = new GraphBatchClient(rgcClient, maxRetryAfterSeconds: 1, batchItemsPerSecond: 20); + var client2 = new GraphBatchClient(rgcClient, batchItemsPerSecond: 20); await client2.ExecuteBatchIndexedAsync(smallOps); sw.Stop(); @@ -1147,7 +1145,7 @@ public async Task BatchWrite_Pacing_ConcurrentCallsAlwaysPace() handler.SetDefaultResponse(HttpStatusCode.OK, BuildBatchResponse(20, 201)); using var httpClient = new HttpClient(handler); - using var rgcClient = new ResilientGraphClient(httpClient, FastRetryOptions); + using var rgcClient = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); var body = JsonSerializer.Deserialize("""{"displayName":"Test"}"""); var ops = Enumerable.Range(1, 20) @@ -1155,7 +1153,7 @@ public async Task BatchWrite_Pacing_ConcurrentCallsAlwaysPace() .ToArray(); // Seed the pacing state so all concurrent readers have something to read - var seed = new GraphBatchClient(rgcClient, maxRetryAfterSeconds: 1, batchItemsPerSecond: 20); + var seed = new GraphBatchClient(rgcClient, batchItemsPerSecond: 20); await seed.ExecuteBatchIndexedAsync(ops); // 20 concurrent calls — each should be paced because the static state has @@ -1164,7 +1162,7 @@ public async Task BatchWrite_Pacing_ConcurrentCallsAlwaysPace() var tasks = Enumerable.Range(0, 20).Select(async _ => { var sw = System.Diagnostics.Stopwatch.StartNew(); - var client = new GraphBatchClient(rgcClient, maxRetryAfterSeconds: 1, batchItemsPerSecond: 20); + var client = new GraphBatchClient(rgcClient, batchItemsPerSecond: 20); await client.ExecuteBatchIndexedAsync(ops); sw.Stop(); @@ -1200,7 +1198,7 @@ public async Task R2_3a_AllItemsFail_20Items_ExhaustsRetries() handler.SetDefaultResponse(HttpStatusCode.OK, BuildBatchResponse(20, 429, retryAfterSeconds: 0)); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); var ops = Enumerable.Range(1, 20) @@ -1227,8 +1225,8 @@ public async Task R2_3a_AllItemsFail_500_NonRetryableForPost() handler.SetDefaultResponse(HttpStatusCode.OK, BuildBatchResponse(20, 500)); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); var body = JsonSerializer.Deserialize("""{"displayName":"Test"}"""); var ops = Enumerable.Range(1, 20) @@ -1257,8 +1255,8 @@ public async Task R2_3b_PartialRetry_SomeFailSomeSucceed() handler.SetDefaultResponse(HttpStatusCode.OK, allSuccessResponse); // Retries: all succeed using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); var ops = Enumerable.Range(1, 20) .Select(i => new BatchOperation($"/users/{i}")) @@ -1275,7 +1273,7 @@ public async Task R2_3b_PartialRetry_SomeFailSomeSucceed() } [Fact] - public async Task R2_3c_MismatchedResponseIds_Throws() + public async Task R2_3c_MismatchedResponseIds_AreReportedAsAChunkFailure() { // Response IDs don't match request IDs — should throw InvalidOperationException var mismatchedResponse = """ @@ -1294,8 +1292,8 @@ public async Task R2_3c_MismatchedResponseIds_Throws() handler.SetDefaultResponse(HttpStatusCode.OK, mismatchedResponse); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); var ops = new[] { @@ -1303,16 +1301,18 @@ public async Task R2_3c_MismatchedResponseIds_Throws() new BatchOperation("/users/2") }; - var ex = await Assert.ThrowsAsync( - () => batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None)); + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + var ex = Assert.IsType(result.ChunkFailure); Assert.Contains("missing result", ex.Message); } [Fact] - public async Task R2_3d_Phase2_TransportError_PropagatesException() + public async Task R2_3d_Phase2_TransportError_IsReportedWithTheResults() { - // Per-chunk retries exhaust on 503. Phase 2 batch-level retry hits - // a transport error (HttpRequestException). Should propagate. + // Per-chunk retries exhaust on 503. Phase 2 batch-level retry hits a transport error + // (HttpRequestException). The pass runs once every chunk has been sent, so the run + // already holds the outcome of the whole batch: the failure is reported alongside it. var handler = new MockHttpHandler(); // Per-chunk: 4 attempts all 503 for (int i = 0; i < 4; i++) @@ -1326,8 +1326,7 @@ public async Task R2_3d_Phase2_TransportError_PropagatesException() NoRateLimit = true, MaxRetryAttempts = 1, AttemptTimeoutSeconds = 5, - TotalTimeoutSeconds = 30, - MaxRetryAfterSeconds = 1 + TotalTimeoutSeconds = 30 }); var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); @@ -1337,9 +1336,14 @@ public async Task R2_3d_Phase2_TransportError_PropagatesException() new BatchOperation("/users/2") }; - // The transport error during Phase 2 should propagate - await Assert.ThrowsAnyAsync( - () => batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None)); + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.NotNull(result.ChunkFailure); + // The 503s are what the server said about these two operations, and the retry pass + // failing did not change that. + Assert.Equal(2, result.Results.Count); + Assert.All(result.Results, r => Assert.Equal(503, r.Response.Status)); + Assert.Equal(2, result.Telemetry.Failed); } // ═══════════════════════════════════════════════════════════════ @@ -1358,7 +1362,7 @@ public async Task R2_5_BatchWithRateLimiting_Succeeds() using var httpClient = new HttpClient(handler); using var client = new ResilientGraphClient(httpClient); // Default options — rate limiter ON - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + var batchClient = new GraphBatchClient(client); var ops = Enumerable.Range(1, 40) .Select(i => new BatchOperation($"/users/{i}")) .ToArray(); @@ -1383,8 +1387,8 @@ public async Task R2_6_CancellationDuringRetryDelay_CancelsPromptly() handler.SetDefaultResponse(HttpStatusCode.OK, BuildBatchResponse(2, 429, retryAfterSeconds: 30)); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); var ops = new[] { @@ -1408,6 +1412,81 @@ await Assert.ThrowsAnyAsync( // R2-8: Chunk boundary tests (0, 1, 20, 21 items) // ═══════════════════════════════════════════════════════════════ + [Fact] + public async Task R2_8_ZeroItems_ReturnsEmpty() + { + var handler = new MockHttpHandler(); + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); + + var result = await batchClient.ExecuteBatchIndexedAsync( + Array.Empty(), CancellationToken.None); + + Assert.Empty(result.Results); + Assert.Equal(0, handler.RequestCount); // No HTTP calls for empty input + } + + [Fact] + public async Task R2_8_OneItem_SingleChunk() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, BuildBatchResponse(1, 200)); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); + + var ops = new[] { new BatchOperation("/users/1") }; + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.Single(result.Results); + Assert.Equal(200, result.Results[0].Response.Status); + Assert.Equal(1, handler.RequestCount); // Exactly 1 batch POST + } + + [Fact] + public async Task R2_8_TwentyItems_ExactlyOneChunk() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, BuildBatchResponse(20, 200)); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); + + var ops = Enumerable.Range(1, 20) + .Select(i => new BatchOperation($"/users/{i}")) + .ToArray(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.Equal(20, result.Results.Count); + Assert.Equal(1, handler.RequestCount); // Exactly 1 chunk + } + + [Fact] + public async Task R2_8_TwentyOneItems_TwoChunks() + { + var handler = new MockHttpHandler(); + // First chunk: 20 items, second chunk: 1 item + handler.QueueResponse(HttpStatusCode.OK, BuildBatchResponse(20, 200)); + handler.SetDefaultResponse(HttpStatusCode.OK, BuildBatchResponse(1, 200)); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client, batchItemsPerSecond: 0); // No pacing delay for speed + + var ops = Enumerable.Range(1, 21) + .Select(i => new BatchOperation($"/users/{i}")) + .ToArray(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.Equal(21, result.Results.Count); + Assert.Equal(2, handler.RequestCount); // 2 chunks: 20 + 1 + } + // ═══════════════════════════════════════════════════════════════ // Helper: Build mixed batch response (some succeed, some fail) // ═══════════════════════════════════════════════════════════════ @@ -1441,7 +1520,7 @@ public void RedactSensitiveFields_RedactsPasswordProfile() } """); - InvokeMgxBatchRequest.RedactSensitiveFields(node); + node = InvokeMgxBatchRequest.RedactSensitiveFields(node); var obj = node!.AsObject(); Assert.Equal("Test User", obj["displayName"]!.GetValue()); @@ -1460,7 +1539,7 @@ public void RedactSensitiveFields_RedactsNestedPassword() } """); - InvokeMgxBatchRequest.RedactSensitiveFields(node); + node = InvokeMgxBatchRequest.RedactSensitiveFields(node); var obj = node!.AsObject(); Assert.True(obj["accountEnabled"]!.GetValue()); @@ -1479,7 +1558,7 @@ public void RedactSensitiveFields_PreservesNonSensitiveFields() } """); - InvokeMgxBatchRequest.RedactSensitiveFields(node); + node = InvokeMgxBatchRequest.RedactSensitiveFields(node); var obj = node!.AsObject(); Assert.Equal("Test", obj["displayName"]!.GetValue()); @@ -1490,8 +1569,8 @@ public void RedactSensitiveFields_PreservesNonSensitiveFields() [Fact] public void RedactSensitiveFields_HandlesNullNode() { - // Should not throw - InvokeMgxBatchRequest.RedactSensitiveFields(null); + // Should not throw, and a body that was nothing is still nothing. + Assert.Null(InvokeMgxBatchRequest.RedactSensitiveFields(null)); } [Fact] @@ -1505,7 +1584,7 @@ public void RedactSensitiveFields_RedactsKeyAndPasswordCredentials() } """); - InvokeMgxBatchRequest.RedactSensitiveFields(node); + node = InvokeMgxBatchRequest.RedactSensitiveFields(node); var obj = node!.AsObject(); Assert.Equal("App", obj["displayName"]!.GetValue()); @@ -1522,7 +1601,7 @@ public void RedactSensitiveFields_DeepNesting_FiveLevels() } """); - InvokeMgxBatchRequest.RedactSensitiveFields(node); + node = InvokeMgxBatchRequest.RedactSensitiveFields(node); var d = node!["a"]!["b"]!["c"]!["d"]!.AsObject(); Assert.Equal("***REDACTED***", d["password"]!.GetValue()); @@ -1540,7 +1619,7 @@ public void RedactSensitiveFields_RedactsClientSecretAndAppPassword() } """); - InvokeMgxBatchRequest.RedactSensitiveFields(node); + node = InvokeMgxBatchRequest.RedactSensitiveFields(node); var obj = node!.AsObject(); Assert.Equal("App", obj["displayName"]!.GetValue()); @@ -1562,7 +1641,7 @@ public void RedactSensitiveFields_RedactsInsideArrays() } """); - InvokeMgxBatchRequest.RedactSensitiveFields(node); + node = InvokeMgxBatchRequest.RedactSensitiveFields(node); var users = node!["users"]!.AsArray(); Assert.Equal("Alice", users[0]!["displayName"]!.GetValue()); @@ -1571,6 +1650,40 @@ public void RedactSensitiveFields_RedactsInsideArrays() Assert.Equal("***REDACTED***", users[1]!["clientSecret"]!.GetValue()); } + /// + /// The URL rule reads path segments. Split naively, an absolute URL hands its scheme and + /// host to the same test as its path, and a tenant whose Graph host is called + /// keyvault.example.com loses every body it ever dead-letters - the file keeps its + /// Url/Method/Status and nothing to read. + /// + [Theory] + [InlineData("https://keyvault.example.com/v1.0/groups/g1", false)] + [InlineData("https://tokens.contoso.com/v1.0/users/u1", false)] + [InlineData("https://graph.microsoft.com/v1.0/users/u1/resetPassword", true)] + [InlineData("https://graph.microsoft.com/v1.0/applications/a1/addKey", true)] + [InlineData("/trustFramework/keySets/ks1/uploadSecret", true)] + [InlineData("/servicePrincipals/sp1/synchronization/secrets", true)] + [InlineData("/users/u1/resetPassword", true)] + [InlineData("/groups/g1?$select=displayName", false)] + // A query string, not a path. On Unix a leading slash parses as a file URI and the parser + // folds the query into the path, so these read as one more segment unless the scheme is + // checked - and they would then answer differently here and on Windows. + [InlineData("/users/u1?password=1", false)] + [InlineData("/servicePrincipals/x?$select=keyCredentials", false)] + [InlineData("users/u1?password=1", false)] + // %50 is 'P'. Graph reads the path decoded, so this is the same reset the two rows above + // are; the parser unescapes it in the absolute form and the batch envelope carries the + // relative one, which is where a name could hide behind an escape. + [InlineData("/users/u1/microsoft.graph.reset%50assword", true)] + [InlineData("https://graph.microsoft.com/v1.0/users/u1/microsoft.graph.reset%50assword", true)] + // An escape that decodes to nothing anyone named, and one that does not decode at all. + [InlineData("/groups/g1/%6Dembers", false)] + [InlineData("/groups/100%25", false)] + public void UrlNamesSecret_ReadsPathSegmentsOnly(string url, bool expected) + { + Assert.Equal(expected, InvokeMgxBatchRequest.UrlNamesSecret(url)); + } + [Fact] public void RedactSensitiveFields_RedactsRootLevelArray() { @@ -1583,7 +1696,7 @@ public void RedactSensitiveFields_RedactsRootLevelArray() ] """); - InvokeMgxBatchRequest.RedactSensitiveFields(node); + node = InvokeMgxBatchRequest.RedactSensitiveFields(node); var arr = node!.AsArray(); Assert.Equal("Alice", arr[0]!["displayName"]!.GetValue()); @@ -1592,6 +1705,1384 @@ public void RedactSensitiveFields_RedactsRootLevelArray() Assert.Equal("***REDACTED***", arr[1]!["passwordProfile"]!.GetValue()); } + // ═══════════════════════════════════════════════════════════════ + // Dead-letter: what a written line is allowed to carry + // ═══════════════════════════════════════════════════════════════ + + private const string OneItemRejected = """ + { "responses": [ + { "id": "1", "status": 400, "body": { "error": { "code": "BadRequest", "message": "rejected" } } } + ] } + """; + + private const string OneItemAccepted = """ + { "responses": [ + { "id": "1", "status": 200, "body": { "id": "u1" } } + ] } + """; + + private static System.Management.Automation.PowerShell CreateShell() + { + var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + return ps; + } + + /// + /// Reads the dead-letter file while a sibling run still holds it open to append to. Windows + /// checks the sharing both ways round - what the reader asks of the file, and what it will + /// let a handle already on it go on doing - so File.ReadAllLines, which asks for + /// FileShare.Read and nothing more, is refused by the run this test left holding it. The + /// sharing that holder asked for is offered back, which is what a reader of a file a live + /// run is appending to has to do on either platform. + /// + private static string[] ReadSharedLines(string path) + { + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, + FileShare.ReadWrite); + using var reader = new StreamReader(stream); + var lines = new List(); + while (reader.ReadLine() is { } line) lines.Add(line); + return [.. lines]; + } + + /// + /// Sends one item the server refuses and returns the line the dead-letter file got for it. + /// The assertions are about the file, not about the redactor: what a caller can read off + /// disk is the whole question, and the URL rule only exists in the writer. + /// + private static string DeadLetterLineFor(string url, string method, System.Collections.Hashtable body) + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, OneItemRejected); + using var transport = MgxTransportScope.Inject(handler); + + var path = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable + { + ["Url"] = url, + ["Method"] = method, + ["Body"] = body + } + }) + .AddParameter("DeadLetterPath", path); + ps.Invoke(); + + Assert.True(File.Exists(path), "dead-letter file was not written"); + return Assert.Single(File.ReadAllLines(path)); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void RedactedDeadLetterLine_OmitsANewPasswordAtTheTopOfTheBody() + { + var line = DeadLetterLineFor("/users/u1", "PATCH", new System.Collections.Hashtable + { + ["displayName"] = "Keep This", + ["newPassword"] = "sentinel-new-password" + }); + + Assert.DoesNotContain("sentinel-new-password", line); + Assert.Contains("***REDACTED***", line); + // The diagnostic half of the line is what the file is for; only the credential goes. + Assert.Contains("Keep This", line); + } + + [Fact] + public void RedactedDeadLetterLine_OmitsAPreSharedKeyAndASharedSecretMidBody() + { + var line = DeadLetterLineFor("/deviceManagement/deviceConfigurations", "POST", + new System.Collections.Hashtable + { + ["displayName"] = "Corp Wi-Fi", + ["wifiSettings"] = new System.Collections.Hashtable + { + ["preSharedKey"] = "sentinel-wifi-psk" + }, + ["vpnSettings"] = new System.Collections.Hashtable + { + ["sharedSecret"] = "sentinel-vpn-secret" + } + }); + + Assert.DoesNotContain("sentinel-wifi-psk", line); + Assert.DoesNotContain("sentinel-vpn-secret", line); + Assert.Contains("Corp Wi-Fi", line); + } + + [Fact] + public void RedactedDeadLetterLine_OmitsASecretTextNestedInAPasswordCredential() + { + var line = DeadLetterLineFor("/applications/app1", "PATCH", new System.Collections.Hashtable + { + ["displayName"] = "Corp App", + ["passwordCredential"] = new System.Collections.Hashtable + { + ["displayName"] = "rotation", + ["secretText"] = "sentinel-secret-text" + } + }); + + Assert.DoesNotContain("sentinel-secret-text", line); + Assert.Contains("Corp App", line); + } + + [Fact] + public void RedactedDeadLetterLine_OmitsAPasswordInsideAnArrayOfArrays() + { + // An array directly inside an array: no object between them, so a redactor that only + // steps from an array into an object never reaches the field. A -Body is whatever the + // caller built, so the writer cannot assume a shape. + var line = DeadLetterLineFor("/deviceManagement/configurationPolicies", "POST", + new System.Collections.Hashtable + { + ["name"] = "Corp Baseline", + ["settings"] = new object[] + { + new object[] + { + new System.Collections.Hashtable { ["password"] = "sentinel-nested-array" } + } + } + }); + + Assert.DoesNotContain("sentinel-nested-array", line); + Assert.Contains("Corp Baseline", line); + } + + [Fact] + public void RedactedDeadLetterLine_OmitsEveryNameTheExactListCovered() + { + var line = DeadLetterLineFor("/users/u1", "PATCH", new System.Collections.Hashtable + { + ["displayName"] = "Corp User", + ["passwordProfile"] = new System.Collections.Hashtable + { + ["password"] = "sentinel-passwordprofile" + }, + ["password"] = "sentinel-password", + ["secretText"] = "sentinel-secrettext", + ["keyCredentials"] = new object[] + { + new System.Collections.Hashtable { ["key"] = "sentinel-keycredentials" } + }, + ["passwordCredentials"] = new object[] + { + new System.Collections.Hashtable { ["secretText"] = "sentinel-passwordcredentials" } + }, + ["clientSecret"] = "sentinel-clientsecret", + ["appPassword"] = "sentinel-apppassword", + ["clientAssertion"] = "sentinel-clientassertion" + }); + + foreach (var sentinel in new[] + { + "sentinel-passwordprofile", "sentinel-password", "sentinel-secrettext", + "sentinel-keycredentials", "sentinel-passwordcredentials", "sentinel-clientsecret", + "sentinel-apppassword", "sentinel-clientassertion" + }) + { + Assert.DoesNotContain(sentinel, line); + } + + Assert.Contains("Corp User", line); + } + + [Fact] + public void RedactedDeadLetterLine_ReplacesTheWholeBodyWhenTheUrlNamesAnUploadSecret() + { + // The credential is the field called `k`; nothing about that name says so. + var line = DeadLetterLineFor("/trustFramework/keySets/ks1/uploadSecret", "POST", + new System.Collections.Hashtable + { + ["use"] = "sig", + ["k"] = "sentinel-upload-secret" + }); + + Assert.DoesNotContain("sentinel-upload-secret", line); + + var written = JsonNode.Parse(line)!.AsObject(); + Assert.Equal("***REDACTED***", written["Body"]!.GetValue()); + // What the line is still worth: everything a caller needs to decide on the item. + Assert.Equal("/trustFramework/keySets/ks1/uploadSecret", written["Url"]!.GetValue()); + Assert.Equal("POST", written["Method"]!.GetValue()); + Assert.Equal(400, written["Status"]!.GetValue()); + Assert.Contains("BadRequest", written["Error"]!.GetValue()); + } + + [Fact] + public void RedactedDeadLetterLine_ReplacesTheWholeBodyForSynchronizationSecrets() + { + // Here the credential is one `value` among a list of them. + var line = DeadLetterLineFor("/servicePrincipals/sp1/synchronization/secrets", "PUT", + new System.Collections.Hashtable + { + ["value"] = new object[] + { + new System.Collections.Hashtable + { + ["key"] = "ClientSecret", + ["value"] = "sentinel-sync-secret" + } + } + }); + + Assert.DoesNotContain("sentinel-sync-secret", line); + + var written = JsonNode.Parse(line)!.AsObject(); + Assert.Equal("***REDACTED***", written["Body"]!.GetValue()); + } + + /// + /// A dead-letter line re-piped the way -DeadLetterPath's recipe says to. A Body that is the + /// marker whole is refused already - the marker is not JSON - but a Body carrying it in one + /// field parses, and went to the tenant with the literal marker where the credential was. + /// + [Fact] + public void RePipedBody_CarryingTheRedactionMarkerInAField_IsRefusedAndNotSent() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, OneItemRejected); + using var transport = MgxTransportScope.Inject(handler); + + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable + { + ["Url"] = "/users/u1", + ["Method"] = "PATCH", + ["Body"] = new System.Collections.Hashtable + { + ["displayName"] = "Keep", + ["passwordProfile"] = "***REDACTED***" + } + }, + new System.Collections.Hashtable + { + ["Url"] = "/groups/g1", + ["Method"] = "PATCH", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "the-other-item" } + } + }); + ps.Invoke(); + + var error = Assert.Single(ps.Streams.Error, + e => e.FullyQualifiedErrorId.StartsWith("InvalidBatchItemBody", StringComparison.Ordinal)); + Assert.Contains("PATCH /users/u1", error.Exception.Message); + // The field is named: it says which value the file took, and so which one to supply. + Assert.Contains("passwordProfile", error.Exception.Message); + + // The rest of the file still goes out, and the marker never reaches the wire. + var envelope = Assert.Single(handler.Requests).Content! + .ReadAsStringAsync().GetAwaiter().GetResult(); + Assert.DoesNotContain("REDACTED", envelope); + Assert.Contains("the-other-item", envelope); + } + + /// + /// An ordinary call, never a dead-letter line: a hashtable body a caller wrote by hand, whose + /// own text happens to carry the marker. The refusal is the same one a re-piped line gets, + /// because nothing in the text tells the two apart, so the message has to hold for both - + /// naming the field, and both remedies: a re-piped line is rebuilt from its Url and Method, + /// and a body of the caller's own must not carry that text at all. + /// + [Fact] + public void OrdinaryBody_CarryingTheMarkerText_IsRefusedNamingTheFieldAndBothRemedies() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, """{"responses":[{"id":"1","status":200,"body":{}}]}"""); + using var transport = MgxTransportScope.Inject(handler); + + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable + { + ["Url"] = "/users/u1", + ["Method"] = "PATCH", + ["Body"] = new System.Collections.Hashtable + { + ["displayName"] = "Redaction demo ***REDACTED*** room" + } + } + }); + ps.Invoke(); + + var error = Assert.Single(ps.Streams.Error, + e => e.FullyQualifiedErrorId.StartsWith("InvalidBatchItemBody", StringComparison.Ordinal)); + Assert.Equal( + "Body for PATCH /users/u1 carries the redaction marker '***REDACTED***' at " + + "'displayName'. A dead-letter line is not the body that was sent: rebuild the " + + "request from its Url and Method, supplying the credential again. A body of your " + + "own must not carry that text.", + error.Exception.Message); + + // A body that never came from the file is refused the same way, before anything is sent. + Assert.Empty(handler.Requests); + } + + /// + /// The same recipe on a line whose marker is in a KEY, not a value. The writer cuts a + /// pre-authenticated URL used as a field name at its capability parameter and re-adds the + /// property under the cut name, suffixing #2 when two keys cut to the same string - so the + /// marker can be in a line with no string value carrying it. A guard reading values only + /// passed such a line through, and the tenant got the literal marker as a field name. + /// + [Fact] + public void RePipedBody_CarryingTheRedactionMarkerInAKey_IsRefusedAndNothingIsSent() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, OneItemRejected); + using var transport = MgxTransportScope.Inject(handler); + + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable + { + ["Url"] = "/users/u1", + ["Method"] = "PATCH", + ["Body"] = """{"https://h/f?sig=***REDACTED***#2":"1"}""" + } + }); + ps.Invoke(); + + var error = Assert.Single(ps.Streams.Error, + e => e.FullyQualifiedErrorId.StartsWith("InvalidBatchItemBody", StringComparison.Ordinal)); + Assert.Contains("PATCH /users/u1", error.Exception.Message); + // The key as written, collision suffix and all: it is what has to be supplied again. + Assert.Contains("https://h/f?sig=***REDACTED***#2", error.Exception.Message); + + // The only item was refused, so no chunk was built and nothing reached the wire. + Assert.Empty(handler.Requests); + } + + /// + /// The same recipe on the shape the URL rule leaves, where the marker sits INSIDE a value: + /// a pre-authenticated URL is cut after its capability parameter, so what stands in the + /// file is the URL prefix with the marker after it and never the marker alone. Comparing a + /// value to the marker whole let those lines through, and the tenant got a + /// "?sig=***REDACTED***" URL no host will honor in place of the one the caller believes it + /// supplied. Three shapes, because the writer cuts a URL in all three: a property's value, + /// one nested in an array of objects, and a bare array element with no property name at all. + /// + [Fact] + public void RePipedBody_CarryingACutCapabilityUrlInAValue_IsRefusedAndNothingIsSent() + { + (System.Collections.Hashtable Body, string Field)[] shapes = + [ + (new System.Collections.Hashtable + { + ["uploadUri"] = "https://s.blob.core.windows.net/c/b?sv=2021&sig=sentinel-sas" + }, "uploadUri"), + (new System.Collections.Hashtable + { + ["files"] = new object[] + { + new System.Collections.Hashtable + { + ["uploadUri"] = "https://h/f?tempauth=sentinel-jwt" + } + } + }, "files[0].uploadUri"), + (new System.Collections.Hashtable + { + ["a"] = new object[] { "https://h/f?sig=sentinel-bare" } + }, "a[0]") + ]; + + foreach (var (body, field) in shapes) + { + // A real failed item's line, read back the way -DeadLetterPath's recipe reads it. + var written = JsonNode.Parse(DeadLetterLineFor("/users/u1", "PATCH", body))!; + var rePiped = written["Body"]!.ToJsonString(); + Assert.Contains(InvokeMgxBatchRequest.RedactionMarker, rePiped, StringComparison.Ordinal); + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, OneItemRejected); + using var transport = MgxTransportScope.Inject(handler); + + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable + { + ["Url"] = "/users/u1", + ["Method"] = "PATCH", + ["Body"] = rePiped + } + }); + ps.Invoke(); + + var error = Assert.Single(ps.Streams.Error, + e => e.FullyQualifiedErrorId.StartsWith("InvalidBatchItemBody", StringComparison.Ordinal)); + Assert.Contains("PATCH /users/u1", error.Exception.Message); + // The field is named at its own depth, which is what says what to supply again. + Assert.Contains(field, error.Exception.Message); + + // The only item was refused, so no chunk was built and nothing reached the wire. + Assert.Empty(handler.Requests); + } + } + + /// + /// A raw-string body the JSON parser accepts and the marker guard cannot read. An unpaired + /// surrogate escape is a legal string token whose value has no UTF-16 form, so the parse + /// succeeds and the read of that one string throws - and the guard reads every string of + /// every body. It is one item's body, so it is refused as one, like a body that is not JSON + /// at all: the guard is the only thing that decides whether a body may be sent, so a body + /// it cannot read is a body it cannot clear. + /// + [Fact] + public void Body_TheMarkerGuardCannotRead_IsRefusedAndTheOtherItemIsSent() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, + """{"responses":[{"id":"1","status":201,"body":{"id":"g1"}}]}"""); + using var transport = MgxTransportScope.Inject(handler); + + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable + { + ["Url"] = "/users", + ["Method"] = "POST", + ["Body"] = """{"displayName":"Jos\ud83d"}""" + }, + new System.Collections.Hashtable + { + ["Url"] = "/groups", + ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "the-other-item" } + } + }); + var output = ps.Invoke(); + + var error = Assert.Single(ps.Streams.Error, + e => e.FullyQualifiedErrorId.StartsWith("InvalidBatchItemBody", StringComparison.Ordinal)); + Assert.StartsWith( + "Body for POST /users could not be read for the redaction check: ", + error.Exception.Message, StringComparison.Ordinal); + // The runtime's own words for what it could not read, and the remedy. The reason's + // trailing period is trimmed rather than doubled onto the sentence quoting it. + Assert.Contains("surrogate", error.Exception.Message, StringComparison.Ordinal); + Assert.EndsWith(". Send it as a hashtable or a string this run can read.", + error.Exception.Message, StringComparison.Ordinal); + Assert.DoesNotContain("..", error.Exception.Message, StringComparison.Ordinal); + Assert.Equal("/users", error.TargetObject); + + // One item was refused; the other one went, was answered, and is in the output. The run + // used to end on the unreadable body with nothing sent and no record of anything. + var envelope = Assert.Single(handler.Requests).Content! + .ReadAsStringAsync().GetAwaiter().GetResult(); + Assert.Contains("the-other-item", envelope, StringComparison.Ordinal); + Assert.DoesNotContain("Jos", envelope, StringComparison.Ordinal); + + var result = Assert.Single(output).BaseObject as System.Collections.Hashtable; + Assert.NotNull(result); + Assert.Equal("/groups", result["Url"]); + Assert.Equal(201, result["Status"]); + } + + [Fact] + public void FindRedactionMarker_NamesTheFieldAtAnyDepth() + { + var nested = JsonSerializer.Deserialize( + """{"a":{"b":[{"c":"x"},{"keyCredentials":"***REDACTED***"}]}}"""); + Assert.Equal("a.b[1].keyCredentials", InvokeMgxBatchRequest.FindRedactionMarker(nested)); + + // A value is matched as a substring, not compared whole. The writer's cut leaves the + // URL up to the capability parameter in front of the marker, so a comparison saw only + // the cuts that took a whole value; and nothing in the text tells the file's marker + // from a caller's own, since the file's is a prefix plus the marker too. A caller who + // writes "***REDACTED***x" himself is refused with the rest - renaming a field costs + // less than a request the tenant cannot honor. + var ownText = JsonSerializer.Deserialize("""{"a":["***REDACTED***x"],"b":1}"""); + Assert.Equal("a[0]", InvokeMgxBatchRequest.FindRedactionMarker(ownText)); + + // The cut form the URL rule actually leaves in a value, at the depth the writer reaches. + var cutUrl = JsonSerializer.Deserialize( + """{"files":[{"uploadUri":"https://h/f?tempauth=***REDACTED***"}]}"""); + Assert.Equal("files[0].uploadUri", InvokeMgxBatchRequest.FindRedactionMarker(cutUrl)); + + // A name is matched the same way: a key carrying the marker is never the marker alone, + // and the path returned is the name as written. + var inAKey = JsonSerializer.Deserialize( + """{"keep":"v","https://h/f?sig=***REDACTED***#2":"1"}"""); + Assert.Equal("https://h/f?sig=***REDACTED***#2", + InvokeMgxBatchRequest.FindRedactionMarker(inAKey)); + + // Nested, so the path is built the same way the value case builds it. + var nestedKey = JsonSerializer.Deserialize( + """{"a":[{"https://h/f?tempauth=***REDACTED***":"1"}]}"""); + Assert.Equal("a[0].https://h/f?tempauth=***REDACTED***", + InvokeMgxBatchRequest.FindRedactionMarker(nestedKey)); + } + + /// + /// Intune's content-file body, on the line a caller reads off disk. The credential is the + /// URL: azureStorageUri and uploadUrl fetch and fill the blob on the strength of the SAS + /// alone, and no name in the body says so. A -Debug trace of the same body has hidden them + /// all along; BodyRedactionTests holds the two side by side. + /// + [Fact] + public void RedactedDeadLetterLine_CutsAPreAuthenticatedUrlAtItsCapabilityParameter() + { + var line = DeadLetterLineFor( + "/deviceAppManagement/mobileApps/a1/contentVersions/1/files/f1", "PATCH", + new System.Collections.Hashtable + { + ["azureStorageUri"] = "https://s.blob.core.windows.net/c/b?sv=2021-08-06&sig=sentinel-sas&se=2026", + ["@microsoft.graph.downloadUrl"] = "https://public.bl.files.1drv.com/y4msentinel-path/f.bin" + }); + + Assert.DoesNotContain("sentinel-sas", line); + Assert.DoesNotContain("sentinel-path", line); + + var body = JsonNode.Parse(line)!["Body"]!.AsObject(); + // The host and the parameter name stay. Which blob and which kind of URL is the half + // the file is read for, and losing it would leave the line saying only that one failed. + Assert.Equal("https://s.blob.core.windows.net/c/b?sv=2021-08-06&sig=***REDACTED***", + body["azureStorageUri"]!.GetValue()); + // The capability is in the path here, so the property name is what reaches it and the + // whole value goes. + Assert.Equal("***REDACTED***", body["@microsoft.graph.downloadUrl"]!.GetValue()); + } + + /// + /// A pre-authenticated URL used as the item URL rather than as a value inside a body. An + /// upload session's uploadUrl and a @microsoft.graph.downloadUrl are requests a caller + /// makes, so the URL a batch item names is as much a credential as one a body carries - and + /// the line wrote it verbatim while cutting the identical URL beside it in the Body. + /// + /// Both halves, from the run's own artifacts: the line on disk, and the envelope as the + /// trace formats it. NormalizeToRelativeUrl takes the scheme and host off before the POST, + /// so the two forms are not the same text and each needed its own rule. + /// + [Fact] + public void ACapabilityUrlUsedAsTheItemUrl_IsCutInTheFileAndInTheTrace() + { + const string itemUrl = "https://s.blob.core.windows.net/c/b?sv=2021&sig=sentinel-item-sas"; + + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, OneItemRejected); + using var transport = MgxTransportScope.Inject(handler); + + var path = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + string line; + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable + { + ["Url"] = itemUrl, + ["Method"] = "PUT", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "Keep This" } + } + }) + .AddParameter("DeadLetterPath", path); + ps.Invoke(); + + Assert.True(File.Exists(path), "dead-letter file was not written"); + line = Assert.Single(File.ReadAllLines(path)); + } + finally + { + File.Delete(path); + } + + Assert.DoesNotContain("sentinel-item-sas", line); + var written = JsonNode.Parse(line)!.AsObject(); + // Cut where a body's value would be cut: the host and the parameter name stay, so the + // line still says which blob and which kind of URL failed. + Assert.Equal("https://s.blob.core.windows.net/c/b?sv=2021&sig=***REDACTED***", + written["Url"]!.GetValue()); + // The rest of the line is what the file is read for. + Assert.Equal("PUT", written["Method"]!.GetValue()); + Assert.Contains("Keep This", line); + + // The same URL on the wire, relative by then, in the trace of the envelope that carried + // it. A trace is what users paste into issue reports. + var captured = Assert.Single(handler.CapturedRequests); + var trace = GraphRequestTracer.FormatRequest( + new HttpRequestMessage(HttpMethod.Post, "https://graph.microsoft.com/v1.0/$batch"), + captured.Body, 1); + Assert.DoesNotContain("sentinel-item-sas", trace, StringComparison.Ordinal); + // The envelope's serializer escapes an ampersand, so the query in the trace is not the + // query in the file - "?sv=2021\u0026sig=". What both keep is the parameter name, which + // is what says which kind of URL went. + Assert.Contains("/c/b?sv=2021", trace, StringComparison.Ordinal); + Assert.Contains("sig=", trace, StringComparison.Ordinal); + } + + /// + /// The 429s a batch met and the time it spent in Retry-After sleeps, in the session view + /// after a run that -ErrorAction Stop ended on an item error. Those counters describe what + /// the wire did, which is settled by the time the batch returns; they used to be recorded at + /// the very end of the reporting, past the dead-letter write, the chunk failure and one + /// error record per failed item, so a run that stopped on any of those told Get-MgxTelemetry + /// nothing - and a throttled, retried batch is exactly the run whose numbers a caller opens + /// that view to find. + /// + /// Both preferences, because the point is that they agree: the stop must cost the caller + /// nothing the run already measured. + /// + [Theory] + [InlineData(System.Management.Automation.ActionPreference.Continue)] + [InlineData(System.Management.Automation.ActionPreference.Stop)] + public void SessionTelemetry_SurvivesAStopOnTheItemErrors( + System.Management.Automation.ActionPreference errorAction) + { + const string throttledAndRejected = """ + { "responses": [ + { "id": "1", "status": 429, "headers": { "Retry-After": "0" } }, + { "id": "2", "status": 400, "body": { "error": { "code": "BadRequest", "message": "rejected" } } } + ] } + """; + const string retrySucceeded = """ + { "responses": [ { "id": "1", "status": 201, "body": { "id": "u1" } } ] } + """; + + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, throttledAndRejected); + handler.QueueResponse(HttpStatusCode.OK, retrySucceeded); + using var transport = MgxTransportScope.Inject(handler); + + MgxTelemetryCollector.Current.Reset(); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable { ["Url"] = "/users", ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "a" } }, + new System.Collections.Hashtable { ["Url"] = "/groups", ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "b" } } + }) + .AddParameter("ErrorAction", errorAction); + + if (errorAction == System.Management.Automation.ActionPreference.Stop) + Assert.ThrowsAny(() => ps.Invoke()); + else + ps.Invoke(); + + var session = MgxTelemetryCollector.Current.GetSummary(); + // One item was answered 429 and retried, and the retry waited. + Assert.Equal(1, session.BatchItemThrottles); + Assert.True(session.RetryDelayMs > 0, $"RetryDelayMs was {session.RetryDelayMs}"); + } + finally + { + MgxTelemetryCollector.Current.Reset(); + } + } + + private const string TwoItemsRejected = """ + { "responses": [ + { "id": "1", "status": 400, "body": { "error": { "code": "BadRequest", "message": "rejected" } } }, + { "id": "2", "status": 400, "body": { "error": { "code": "BadRequest", "message": "rejected" } } } + ] } + """; + + /// + /// Sends items the server refuses and returns every line the dead-letter file got, with the + /// warnings the run wrote. Nothing here asserts a line count: what a body the writer cannot + /// read costs the item and the items after it is the question. A run that ends on such a + /// body throws out of EndProcessing, so ps.Invoke() throwing is itself the failure - and it + /// names the exception, which is what the assertions below could not. + /// + private static (string[] Lines, string[] Warnings) DeadLetterRun(string response, params object[] items) + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, response); + using var transport = MgxTransportScope.Inject(handler); + + var path = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", items) + .AddParameter("DeadLetterPath", path); + ps.Invoke(); + + return (File.Exists(path) ? File.ReadAllLines(path) : [], + ps.Streams.Warning.Select(w => w.Message).ToArray()); + } + finally + { + File.Delete(path); + } + } + + /// + /// A raw-string body with the same property name twice. Graph takes it, and JsonNode.Parse + /// takes it; enumerating the JsonObject it built is what throws, so the failure lands in + /// the writer with the request already sent and answered. The line is owed either way. + /// + [Fact] + public void DeadLetterLine_ForABodyTheRedactorCannotWalk_CarriesTheMarkerAndWarns() + { + var (lines, warnings) = DeadLetterRun(TwoItemsRejected, + new System.Collections.Hashtable + { + ["Url"] = "/users", + ["Method"] = "POST", + ["Body"] = """{"displayName":"first","displayName":"second"}""" + }, + new System.Collections.Hashtable + { + ["Url"] = "/groups", + ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "the-second-item" } + }); + + Assert.Equal(2, lines.Length); + var first = JsonNode.Parse(lines[0])!.AsObject(); + Assert.Equal("/users", first["Url"]!.GetValue()); + Assert.Equal("***REDACTED***", first["Body"]!.GetValue()); + // The body was never read, so nothing in it may be written - not even the half a + // duplicate key left readable. + Assert.DoesNotContain("first", lines[0]); + + // The item after it still gets its line: one unreadable body is not the run. + Assert.Contains("the-second-item", lines[1]); + + // Not "Contains(withheld)" alone: the outcome warning now folds the withheld count + // into its own text too, so that predicate would match both. "POST /users" is what + // is unique to the per-item warning under test here. + var warning = Assert.Single(warnings, w => w.Contains("POST /users") && w.Contains("withheld")); + Assert.Contains("could not be read for redaction", warning); + } + + /// + /// The same body on the second of two items. The writer holds one StreamWriter open across + /// the loop, so an exception in the middle of it is what decides whether the lines already + /// written reach disk at all. + /// + [Fact] + public void DeadLetterFile_IsNotCutShortByABodyTheRedactorCannotWalk() + { + var (lines, warnings) = DeadLetterRun(TwoItemsRejected, + new System.Collections.Hashtable + { + ["Url"] = "/groups", + ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "the-first-item" } + }, + new System.Collections.Hashtable + { + ["Url"] = "/users", + ["Method"] = "POST", + ["Body"] = """{"displayName":"first","displayName":"second"}""" + }); + + Assert.Equal(2, lines.Length); + Assert.Contains("the-first-item", lines[0]); + Assert.Equal("***REDACTED***", JsonNode.Parse(lines[1])!["Body"]!.GetValue()); + Assert.Single(warnings, w => w.Contains("POST /users") && w.Contains("withheld")); + } + + /// + /// WriteWarning throws under -WarningAction Stop. The withheld-body warning used to be + /// written right after the dead-letter file closed - before the item errors and before + /// WriteBatchTelemetry - so that throw cut the run off before either ran: two refused items + /// produced two dead-letter lines and zero error records. The warning now runs last, so the + /// warning that actually stops this run is WriteBatchTelemetry's own outcome warning (it + /// still has one to give, since both items failed), and reaching it proves the item errors + /// and the telemetry it summarizes already ran. That outcome warning now folds the withheld + /// count in too, so it is what proves this run was stopped there rather than by the + /// dedicated per-item warning after it - the fold names the count without naming the item, + /// and the dedicated warning (never reached here) is the other way around. + /// + [Fact] + public void WarningActionStop_WithAnUnreadableBody_StillRecordsTheItemErrorsAndTheTelemetry() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TwoItemsRejected); + using var transport = MgxTransportScope.Inject(handler); + + var path = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable + { + ["Url"] = "/users", + ["Method"] = "POST", + ["Body"] = """{"displayName":"first","displayName":"second"}""" + }, + new System.Collections.Hashtable + { + ["Url"] = "/groups", + ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "the-second-item" } + } + }) + .AddParameter("DeadLetterPath", path) + .AddParameter("WarningAction", System.Management.Automation.ActionPreference.Stop); + + var ex = Assert.ThrowsAny(() => ps.Invoke()); + + // Reaching WriteBatchTelemetry's own outcome warning means the per-item errors and + // the telemetry summary it closes already happened. It is the outcome warning, not + // the dedicated per-item one, because it names the count rather than the item. + Assert.Contains("2 of 2 batch items failed", ex.Message); + Assert.Contains("1 body was withheld from the dead-letter file", ex.Message); + Assert.DoesNotContain("POST /users", ex.Message); + + var itemErrors = ps.Streams.Error + .Where(e => e.FullyQualifiedErrorId.StartsWith("BatchItemError", StringComparison.Ordinal)) + .ToList(); + Assert.Equal(2, itemErrors.Count); + + Assert.True(File.Exists(path), "dead-letter file was not written"); + Assert.Equal(2, File.ReadAllLines(path).Length); + } + finally + { + File.Delete(path); + } + } + + /// + /// A -DeadLetterPath the run cannot open, answered before the first POST. The open used to + /// happen where the lines are written, at the end, so a path like this was discovered with + /// the batch already applied: the caller had made every one of those changes and had no + /// record of which to retry. Nothing is sent now, and the message says so - which is what + /// makes the failure recoverable by fixing the path and running the same command again. + /// + /// An error record rather than a warning, at the default preference: a caller running + /// -WarningAction SilentlyContinue would otherwise lose the only line saying the file the + /// batch was to be recorded in is not there. + /// + [Fact] + public void DeadLetterPathThatCannotBeOpened_IsRefusedBeforeAnythingIsSent() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TwoItemsRejected); + using var transport = MgxTransportScope.Inject(handler); + + var dir = Path.Combine(Path.GetTempPath(), $"mgx-dl-dir-{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable + { + ["Url"] = "/users", + ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "a" } + }, + new System.Collections.Hashtable + { + ["Url"] = "/groups", + ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "b" } + } + }) + .AddParameter("DeadLetterPath", dir); + var output = ps.Invoke(); + + // One record, and no item errors behind it: there are no items to have errors. + var record = Assert.Single(ps.Streams.Error); + Assert.StartsWith("DeadLetterWriteFailed", record.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.Equal(System.Management.Automation.ErrorCategory.WriteError, + record.CategoryInfo.Category); + Assert.Equal(dir, record.TargetObject); + Assert.StartsWith("Failed to write dead-letter file '", record.Exception.Message, + StringComparison.Ordinal); + Assert.Contains(dir, record.Exception.Message); + Assert.EndsWith(" before the batch was sent; nothing was sent.", + record.Exception.Message, StringComparison.Ordinal); + // The reason the open refused is kept, not flattened into the text alone. + Assert.NotNull(record.Exception.InnerException); + + // Nothing went out, so there is no output and nothing to warn about. + Assert.Empty(handler.Requests); + Assert.Empty(output); + Assert.Empty(ps.Streams.Warning); + Assert.Empty(Directory.GetFiles(dir)); + } + finally + { + Directory.Delete(dir); + } + } + + /// + /// The same unusable path on a batch every item of which would have been applied. It is + /// refused just the same, and refused before the batch: the file is part of what the caller + /// asked for, and the run that could not have it is the run that has not yet spent anything. + /// The old order had no way to be this: the path was found out at the end, with the batch + /// applied, so refusing there would have told a scripted caller their batch failed when the + /// whole of it had succeeded, and the failure was softened to a warning to avoid saying so. + /// + [Fact] + public void DeadLetterPathThatCannotBeOpened_IsRefusedEvenWhenTheBatchWouldHaveSucceeded() + { + var dir = Path.Combine(Path.GetTempPath(), $"mgx-dl-dir-{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + try + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, BatchSuccessResponse); + using var transport = MgxTransportScope.Inject(handler); + + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] { "/users/u1", "/users/u2" }) + .AddParameter("DeadLetterPath", dir); + var output = ps.Invoke(); + + var record = Assert.Single(ps.Streams.Error); + Assert.StartsWith("DeadLetterWriteFailed", record.FullyQualifiedErrorId, + StringComparison.Ordinal); + Assert.EndsWith(" before the batch was sent; nothing was sent.", + record.Exception.Message, StringComparison.Ordinal); + + // And it is true of the run: the two GETs never went. + Assert.Empty(handler.Requests); + Assert.Empty(output); + Assert.Empty(ps.Streams.Warning); + } + finally + { + Directory.Delete(dir); + } + } + + /// + /// The point of making it an error: under -ErrorAction Stop the run ends on it, and it ends + /// before the batch, so the caller is told the file is missing rather than told which items + /// to look for in it - and nothing has been applied to go looking for. The message names + /// the resolved path, which is the only place the path appears. + /// + [Fact] + public void ErrorActionStop_WithADeadLetterPathThatIsADirectory_TerminatesOnTheWriteFailure() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TwoItemsRejected); + using var transport = MgxTransportScope.Inject(handler); + + var dir = Path.Combine(Path.GetTempPath(), $"mgx-dl-dir-{Guid.NewGuid():N}"); + Directory.CreateDirectory(dir); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] + { + new System.Collections.Hashtable + { + ["Url"] = "/users", + ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "a" } + }, + new System.Collections.Hashtable + { + ["Url"] = "/groups", + ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "b" } + } + }) + .AddParameter("DeadLetterPath", dir) + .AddParameter("ErrorAction", System.Management.Automation.ActionPreference.Stop); + + var ex = Assert.ThrowsAny( + () => ps.Invoke()); + + Assert.StartsWith("DeadLetterWriteFailed", + ex.ErrorRecord.FullyQualifiedErrorId, StringComparison.Ordinal); + // ActionPreferenceStopException wraps the record it stopped on, and its own message + // prefixes the preference text - the record's is the message the cmdlet wrote. + Assert.StartsWith("Failed to write dead-letter file '", + ex.ErrorRecord.Exception.Message, StringComparison.Ordinal); + Assert.Contains(dir, ex.ErrorRecord.Exception.Message); + + // Stopped on the open, so nothing was POSTed and there are no item errors to have + // been cut off - which is the whole difference: the caller can fix the path and run + // the same command again without deciding anything about duplicates. + Assert.EndsWith(" before the batch was sent; nothing was sent.", + ex.ErrorRecord.Exception.Message, StringComparison.Ordinal); + Assert.Empty(handler.Requests); + Assert.DoesNotContain(ps.Streams.Error, + e => e.FullyQualifiedErrorId.StartsWith("BatchItemError", StringComparison.Ordinal)); + } + finally + { + Directory.Delete(dir); + } + } + + /// + /// -DeadLetterPath on a provider that is not the file system. The path was resolved without + /// asking which provider answered, and every other provider resolves to something this run + /// cannot append to - "Env:\MGX_DEAD_LETTER" comes back as "MGX_DEAD_LETTER", a name + /// relative to nothing - so the run made a file of that name in whatever directory it was + /// standing in and reported the dead letters written to it. + /// + [Fact] + public void DeadLetterPathOnAnotherProvider_IsRefusedAndNothingIsSent() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TwoItemsRejected); + using var transport = MgxTransportScope.Inject(handler); + + // Where the resolved name lands when nothing refuses it: a bare name is relative to + // the current directory. Cleared first, because the file it names is exactly what a run + // without this check leaves behind. + var strayed = Path.Combine(Directory.GetCurrentDirectory(), "MGX_DEAD_LETTER"); + File.Delete(strayed); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] { "/users/u1", "/users/u2" }) + .AddParameter("DeadLetterPath", @"Env:\MGX_DEAD_LETTER"); + + var ex = Assert.ThrowsAny(() => ps.Invoke()); + + Assert.StartsWith("DeadLetterPathNotFileSystem", + ex.ErrorRecord.FullyQualifiedErrorId, StringComparison.Ordinal); + Assert.Equal(@"-DeadLetterPath 'Env:\MGX_DEAD_LETTER' is not a file system path.", + ex.ErrorRecord.Exception.Message); + Assert.Empty(handler.Requests); + Assert.False(File.Exists(strayed), $"a file was written to {strayed}"); + } + finally + { + File.Delete(strayed); + } + } + + /// + /// The same resolution on an empty path, which answers with the working directory itself - + /// so the run opened a directory to append to, or, where a provider gave it a name, made a + /// file nobody asked for. Refused at binding, which is where a parameter with nothing in it + /// belongs. + /// + [Fact] + public void DeadLetterPathThatIsEmpty_IsRefusedAndNothingIsSent() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TwoItemsRejected); + using var transport = MgxTransportScope.Inject(handler); + + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] { "/users/u1", "/users/u2" }) + .AddParameter("DeadLetterPath", string.Empty); + + var ex = Assert.ThrowsAny(() => ps.Invoke()); + + Assert.Contains("DeadLetterPath", ex.Message, StringComparison.Ordinal); + Assert.Empty(handler.Requests); + } + + /// + /// A batch in which nothing failed. The file is opened before the first POST, so it exists + /// by the time the run knows whether it has a line for it - and a run that has none used to + /// leave an empty file standing where a caller watching that path for failures would find + /// one after every clean run. It is removed, and nothing is said about it: there was nothing + /// in it to report. + /// + [Fact] + public void ACleanBatch_LeavesNoDeadLetterFileBehind() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, BatchSuccessResponse); + using var transport = MgxTransportScope.Inject(handler); + + var path = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] { "/users/u1", "/users/u2" }) + .AddParameter("DeadLetterPath", path); + var output = ps.Invoke(); + + Assert.Equal(2, output.Count); + Assert.False(File.Exists(path), "an empty dead-letter file was left behind"); + Assert.Empty(ps.Streams.Error); + Assert.Empty(ps.Streams.Warning); + } + finally + { + File.Delete(path); + } + } + + /// + /// The same clean run over a file that was already there. Those lines are an earlier run's + /// record of what is still outstanding, and this run has no business removing them because + /// its own items all succeeded. + /// + [Fact] + public void ACleanBatch_LeavesAnExistingDeadLetterFileAlone() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, BatchSuccessResponse); + using var transport = MgxTransportScope.Inject(handler); + + var path = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + File.WriteAllText(path, "{\"Url\":\"/users/earlier\"}" + Environment.NewLine); + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] { "/users/u1", "/users/u2" }) + .AddParameter("DeadLetterPath", path); + ps.Invoke(); + + Assert.True(File.Exists(path), "an earlier run's dead-letter file was removed"); + Assert.Equal("{\"Url\":\"/users/earlier\"}", Assert.Single(File.ReadAllLines(path))); + } + finally + { + File.Delete(path); + } + } + + /// + /// Two batches over one -DeadLetterPath, which is what running them in parallel gives. A + /// opens the path first - creating it - and is held on the wire; B runs to completion with a + /// refused item and appends its line beside A's open handle, which the share mode admits; + /// then A is released with nothing of its own to write. + /// + /// A created the file and wrote no lines, so the removal is A's to consider - and B's dead + /// letter is in it. The removal asks the file rather than the run: the length A's own handle + /// reports is B's line, and the exclusive claim behind it is what a sibling still appending + /// refuses. The file stays, and nothing is said about it: A has nothing to report about a + /// record that is not its own. + /// + [Fact] + public void ABatchThatCreatedTheFile_LeavesASiblingsDeadLetterAlone() + { + var release = new ManualResetEventSlim(false); + var handler = new MockHttpHandler(); + handler.When(r => r.BodyText != null && r.BodyText.Contains("b-item"), "b") + .Respond(HttpStatusCode.OK, OneItemRejected); + handler.When(r => r.BodyText != null && r.BodyText.Contains("a-item"), "a") + .Respond(_ => + { + release.Wait(TimeSpan.FromSeconds(30)); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + OneItemAccepted, System.Text.Encoding.UTF8, "application/json") + }; + }); + using var transport = MgxTransportScope.Inject(handler); + + var path = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + var aWarnings = new List(); + var aErrors = new List(); + Exception? aFailure = null; + var a = new Thread(() => + { + try + { + using var ps = CreateShell(); + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] { "/users/a-item" }) + .AddParameter("DeadLetterPath", path); + ps.Invoke(); + aWarnings.AddRange(ps.Streams.Warning.Select(w => w.Message)); + aErrors.AddRange(ps.Streams.Error.Select(e => e.FullyQualifiedErrorId)); + } + catch (Exception ex) { aFailure = ex; } + }); + + try + { + a.Start(); + + // A opens the file before its first POST, so its creation precedes the hold. + var deadline = DateTime.UtcNow.AddSeconds(20); + while (!File.Exists(path) && DateTime.UtcNow < deadline) Thread.Sleep(20); + Assert.True(File.Exists(path), "the held run never created the dead-letter file"); + + using (var ps = CreateShell()) + { + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", new object[] { "/users/b-item" }) + .AddParameter("DeadLetterPath", path); + ps.Invoke(); + } + Assert.Contains("b-item", Assert.Single(ReadSharedLines(path)), StringComparison.Ordinal); + + release.Set(); + Assert.True(a.Join(TimeSpan.FromSeconds(60)), "the held run did not finish"); + Assert.Null(aFailure); + + Assert.True(File.Exists(path), "a sibling's dead-letter file was removed"); + Assert.Contains("b-item", Assert.Single(ReadSharedLines(path)), StringComparison.Ordinal); + Assert.Empty(aErrors); + Assert.Empty(aWarnings); + } + finally + { + release.Set(); + a.Join(TimeSpan.FromSeconds(60)); + release.Dispose(); + if (File.Exists(path)) File.Delete(path); + } + } + + /// + /// The other half of the fold, at the default preference: an unreadable body, not a write + /// failure. Both the outcome warning's count and the per-item warning naming the item reach + /// the stream, since nothing here stops the run early. + /// + /// The text is asserted whole, because the fold is punctuation and a Contains on either + /// half of it passes either way. The base sentence used to close with its own period before + /// the clause was appended, and nothing closed the sentence after it: "...batch items + /// failed.; 1 body was withheld from the dead-letter file They failed after all retry + /// attempts." + /// + [Fact] + public void OutcomeWarning_CountsTheBodyItWithheld() + { + var (_, warnings) = DeadLetterRun(TwoItemsRejected, + new System.Collections.Hashtable + { + ["Url"] = "/users", + ["Method"] = "POST", + ["Body"] = """{"displayName":"first","displayName":"second"}""" + }, + new System.Collections.Hashtable + { + ["Url"] = "/groups", + ["Method"] = "POST", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "the-second-item" } + }); + + var outcome = Assert.Single(warnings, w => w.Contains("2 of 2 batch items failed")); + Assert.Equal( + "2 of 2 batch items failed; 1 body was withheld from the dead-letter file. " + + "They failed after all retry attempts. Check $Error for details on each item.", + outcome); + } + + /// + /// The duplicate key one level below where the old walk stopped. The array-of-array descent + /// reaches it, which is what turns a body that used to be written unredacted into one that + /// throws, so the two cases have to be pinned together. + /// + [Fact] + public void DeadLetterLine_ForADuplicateKeyInsideAnArrayOfArrays_CarriesTheMarkerAndWarns() + { + var (lines, warnings) = DeadLetterRun(OneItemRejected, + new System.Collections.Hashtable + { + ["Url"] = "/deviceManagement/configurationPolicies", + ["Method"] = "POST", + ["Body"] = """{"settings":[[{"displayName":"first","displayName":"second"}]]}""" + }); + + var line = Assert.Single(lines); + Assert.Equal("***REDACTED***", JsonNode.Parse(line)!["Body"]!.GetValue()); + Assert.DoesNotContain("settings", line); + Assert.Single(warnings, w => w.Contains("POST /deviceManagement/configurationPolicies") + && w.Contains("withheld")); + } + + /// + /// A body that is one bare string: a pre-authenticated URL and nothing else, which is a + /// legal JSON document and what a PUT to an upload session can carry. The walk steps into an + /// object's properties and an array's elements, and this body has neither, so it returned + /// having read nothing and the live signature was written to disk - while the identical URL + /// under any property name was cut, and a -Debug trace of the same body cut it too. + /// + [Fact] + public void DeadLetterLine_CutsABodyThatIsOneBareCapabilityUrl() + { + var (lines, warnings) = DeadLetterRun(OneItemRejected, + new System.Collections.Hashtable + { + ["Url"] = "/users/u1", + ["Method"] = "PATCH", + ["Body"] = "\"https://c-my.sharepoint.com/x?tempauth=sentinel-root-jwt\"" + }); + + var line = Assert.Single(lines); + Assert.DoesNotContain("sentinel-root-jwt", line); + // Cut, not withheld whole: the host and the parameter name are the half the file is read + // for, and nothing about this body was unreadable. + Assert.Equal("https://c-my.sharepoint.com/x?tempauth=***REDACTED***", + JsonNode.Parse(line)!["Body"]!.GetValue()); + Assert.DoesNotContain(warnings, w => w.Contains("withheld")); + } + + /// + /// The order the -DeadLetterPath doc states, on the line a caller parses field by field. + /// JsonObject writes properties in insertion order, so the order is the writer's to keep. + /// + [Fact] + public void DeadLetterLine_HoldsItsFieldsInTheDocumentedOrder() + { + var line = DeadLetterLineFor("/groups/g1", "PATCH", new System.Collections.Hashtable + { + ["displayName"] = "Corp Group" + }); + + Assert.Equal( + new[] { "Timestamp", "Url", "Method", "Status", "Body", "Error" }, + JsonNode.Parse(line)!.AsObject().Select(p => p.Key).ToArray()); + } + + [Fact] + public void RedactedDeadLetterLine_KeepsTheBodyWhenNeitherTheUrlNorTheNamesSayCredential() + { + // Over-redaction is the accepted cost, not the default: a body with nothing to hide + // reaches the file whole, which is what makes the file worth reading. + var line = DeadLetterLineFor("/groups/g1", "PATCH", new System.Collections.Hashtable + { + ["displayName"] = "Corp Group", + ["mailNickname"] = "corp-group" + }); + + Assert.DoesNotContain("***REDACTED***", line); + Assert.Contains("Corp Group", line); + Assert.Contains("corp-group", line); + } + // ═══════════════════════════════════════════════════════════════ // Additional batch edge case tests // ═══════════════════════════════════════════════════════════════ @@ -1611,8 +3102,8 @@ public async Task BatchMixed_GetAndPost_BothProcessed() """); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); var body = JsonSerializer.Deserialize("""{"displayName":"Test"}"""); var operations = new List @@ -1630,13 +3121,42 @@ public async Task BatchMixed_GetAndPost_BothProcessed() Assert.Equal(200, result.Results[2].Response.Status); } + [Fact] + public async Task BatchResponseCountMismatch_IsReportedAsAChunkFailure() + { + var handler = new MockHttpHandler(); + handler.QueueResponse(HttpStatusCode.OK, """ + { + "responses": [ + { "id": "1", "status": 200, "body": { "id": "user1" } } + ] + } + """); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); + + var operations = new List + { + new("/users/1"), + new("/users/2"), + new("/users/3") + }; + + var result = await batchClient.ExecuteBatchIndexedAsync(operations); + + var ex = Assert.IsType(result.ChunkFailure); + Assert.Contains("response count mismatch", ex.Message, StringComparison.OrdinalIgnoreCase); + } + [Fact] public async Task BatchEmptyOperations_ReturnsEmptyResult() { var handler = new MockHttpHandler(); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); var result = await batchClient.ExecuteBatchIndexedAsync(Array.Empty()); @@ -1646,18 +3166,18 @@ public async Task BatchEmptyOperations_ReturnsEmptyResult() } [Fact] - public async Task BatchEmptyResponsesFromGraph_ThrowsInvalidOperationException() + public async Task BatchEmptyResponsesFromGraph_IsReportedAsAChunkFailure() { var handler = new MockHttpHandler(); handler.QueueResponse(HttpStatusCode.OK, """{ "responses": [] }"""); using var httpClient = new HttpClient(handler); - using var client = new ResilientGraphClient(httpClient, FastRetryOptions); - var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var batchClient = new GraphBatchClient(client); - var ex = await Assert.ThrowsAsync( - () => batchClient.ExecuteBatchIndexedAsync(new[] { new BatchOperation("/users/1") })); + var result = await batchClient.ExecuteBatchIndexedAsync(new[] { new BatchOperation("/users/1") }); + var ex = Assert.IsType(result.ChunkFailure); Assert.Contains("empty or malformed", ex.Message, StringComparison.OrdinalIgnoreCase); } } diff --git a/tests/Mgx.IntegrationTests/Engine/Batch/BulkWriteTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/BulkWriteTests.cs index 3dec881..e970998 100644 --- a/tests/Mgx.IntegrationTests/Engine/Batch/BulkWriteTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Batch/BulkWriteTests.cs @@ -52,6 +52,39 @@ public async Task BulkWriteAsync_AllSucceed_ReturnsCorrectCounts() ResiliencePipelineFactory.Reset(); } + /// + /// A body that opens with a UTF-8 byte-order mark is a body. Deserialized from the bytes it + /// was refused for an invalid start character, and the write - a 201, applied on the server - + /// was reported as a status-0 failure of the request that carried it. + /// + [Fact] + public async Task BulkWriteAsync_BomPrefixedBody_CountsAsSucceededAndParses() + { + ResiliencePipelineFactory.Reset(); + var handler = new MockHttpHandler(); + byte[] bom = [0xEF, 0xBB, 0xBF]; + var body = System.Text.Encoding.UTF8.GetBytes(TestData.SingleUser); + handler.QueueBytes(HttpStatusCode.Created, [.. bom, .. body], "application/json"); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, NoRateLimitOptions); + var fanOut = new ConcurrentFanOut(client, maxConcurrency: 2); + + var ops = new List<(string id, string url)> + { + ("op1", "https://graph.microsoft.com/v1.0/users") + }; + + var result = await fanOut.BulkWriteAsync(HttpMethod.Post, ops, """{"displayName":"Test"}"""); + + Assert.Equal(1, result.Succeeded); + Assert.Empty(result.Errors); + var response = Assert.Single(result.Responses); + Assert.Equal("user1", response.Response.GetProperty("id").GetString()); + + ResiliencePipelineFactory.Reset(); + } + [Fact] public async Task BulkWriteAsync_EmptyOperations_ReturnsEmptyResult() { @@ -248,9 +281,11 @@ public async Task BulkWriteAsync_InvalidJsonBody_NoDoubleCount() Assert.Equal(1, result.Failed); Assert.Single(result.Errors); Assert.Equal("bad-json", result.Errors[0].Id); - // StatusCode 0 proves error came from deserialization-failure catch path, - // not from the HTTP-error path (which would carry the real status code) - Assert.Equal(0, result.Errors[0].StatusCode); + // What the body did is what tells the deserialization failure from the HTTP-error + // path. The status is the one the server answered with: the write reached it, and a + // 0 there says it never did. + Assert.Equal(201, result.Errors[0].StatusCode); + Assert.Contains("invalid start of a value", result.Errors[0].Message, StringComparison.Ordinal); ResiliencePipelineFactory.Reset(); } @@ -299,6 +334,72 @@ public async Task BulkWriteAsync_MixedErrors_PreservesStatusCodes() ResiliencePipelineFactory.Reset(); } + /// + /// A server that sends the headers of an error and then stops sending the body. The stall + /// was recorded as if it were the body, run through the Graph-error parser that does not + /// know it, and reached the caller as "HTTP 500" - which is the one thing it already knew. + /// + [Fact] + public async Task BulkWriteAsync_StalledErrorBody_KeepsTheStallInTheRecord() + { + ResiliencePipelineFactory.Reset(); + + using var httpClient = new HttpClient(new StallingErrorHandler()); + using var client = new ResilientGraphClient(httpClient, NoRateLimitOptions) + { + BodyReadTimeout = TimeSpan.FromSeconds(2) + }; + var fanOut = new ConcurrentFanOut(client, maxConcurrency: 1); + + var ops = new List<(string id, string url)> + { + ("op1", "https://graph.microsoft.com/v1.0/users/u1") + }; + + var result = await fanOut.BulkWriteAsync(HttpMethod.Patch, ops, """{"displayName":"Test"}"""); + + Assert.Equal(1, result.Failed); + var error = Assert.Single(result.Errors); + Assert.Equal(500, error.StatusCode); + Assert.Equal(ResilientGraphClient.BodyReadTimedOutMessage, error.Message); + + ResiliencePipelineFactory.Reset(); + } + + /// + /// The same stall after the server answered 201. The write is on the server; only the read + /// of what it created failed. Recorded as status 0 - the code for a write that never + /// reached an HTTP status - an applied POST read as a connection failure, which is the one + /// kind a caller may safely repeat, and repeating it creates the entity twice. + /// + [Fact] + public async Task BulkWriteAsync_StalledBodyAfterCreated_KeepsTheStatusTheServerAnswered() + { + ResiliencePipelineFactory.Reset(); + + using var httpClient = new HttpClient(new StallingErrorHandler(HttpStatusCode.Created)); + using var client = new ResilientGraphClient(httpClient, NoRateLimitOptions) + { + BodyReadTimeout = TimeSpan.FromSeconds(2) + }; + var fanOut = new ConcurrentFanOut(client, maxConcurrency: 1); + + var ops = new List<(string id, string url)> + { + ("a1", "https://graph.microsoft.com/v1.0/groups/g1/members/$ref") + }; + + var result = await fanOut.BulkWriteAsync(HttpMethod.Post, ops, + """{"@odata.id":"https://graph.microsoft.com/v1.0/users/a1"}"""); + + Assert.Equal(1, result.Failed); + var error = Assert.Single(result.Errors); + Assert.Equal(201, error.StatusCode); + Assert.Equal(ResilientGraphClient.BodyReadTimedOutMessage, error.Message); + + ResiliencePipelineFactory.Reset(); + } + // ── Per-item exception swallowing verification ───────────────────── /// @@ -417,3 +518,26 @@ protected override Task SendAsync( throw _exceptionFactory(); } } + +/// +/// Sends the headers of the given status and one byte of body, then nothing, honoring only +/// the read's own token - the shape of a response whose body stalls. The status defaults to +/// a 500; a write is answered 201 and stalls exactly the same way. The body is the shared +/// , whose content read stream stalls too, so a reader that takes +/// the body as a stream reaches its read timeout rather than waiting on a buffer forever. +/// +public class StallingErrorHandler(HttpStatusCode status = HttpStatusCode.InternalServerError) + : HttpMessageHandler +{ + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var content = new StallingContent(); + content.Headers.TryAddWithoutValidation("Content-Type", "application/json"); + return Task.FromResult(new HttpResponseMessage(status) + { + RequestMessage = request, + Content = content + }); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Batch/FaultInjectionBatchTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/FaultInjectionBatchTests.cs new file mode 100644 index 0000000..998ec58 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Batch/FaultInjectionBatchTests.cs @@ -0,0 +1,306 @@ +using System.Net; +using System.Net.Sockets; +using System.Text; +using System.Text.Json; +using Mgx.Engine.Errors; +using Mgx.Engine.Http; +using Mgx.Engine.Models; +using Polly.Timeout; + +namespace Mgx.IntegrationTests; + +/// +/// Every fault in the catalog, injected at subrequest 23 of a 25-operation batch - the third +/// subrequest of the second chunk, after the first chunk's twenty were answered. A status lands +/// inside the 200 envelope, which is the only place Graph can put one; a transport failure lands +/// on that chunk's own POST, because a sub-response has no way to carry one. What each case +/// expects comes from : the item loop asks it about the sub-response's +/// status with the item's own method, and the pipeline asks it about the POST, which is a POST. +/// +[Collection("Pipeline")] +public class FaultInjectionBatchTests +{ + private const int Operations = 25; + private const int ChunkSize = 20; + private const string TargetUrl = "/users/u23"; + + /// What every armed fault in this file is called, so a theory can assert by name + /// that does not carry it. + private const string FaultName = "fault-chunk-2"; + + private static readonly ResilientGraphClientOptions Options = new() + { + NoRateLimit = true, + NoAdaptivePacing = true, + MaxRetryAttempts = 1, + AttemptTimeoutSeconds = 1, + TotalTimeoutSeconds = 30, + }; + + private static readonly TimeSpan ShortBodyRead = TimeSpan.FromMilliseconds(250); + + /// The envelope stops in the middle of a sub-response. + private const string TruncatedEnvelope = """{"responses":[{"id":"1","status":200"""; + + private static List AllOperations() => + [.. Enumerable.Range(1, Operations).Select(i => new BatchOperation($"/users/u{i}"))]; + + /// + /// Answers a $batch envelope from the envelope that was sent: 200 for every subrequest, + /// except the one whose url is . Keyed on the url rather than the + /// sub-response id because a retry re-numbers the ids - the item that failed comes back as + /// id 1 - and the url is what identifies it across attempts. + /// + private static HttpResponseMessage Envelope(MockRequest request, string? faultUrl, int faultStatus) + { + using var sent = JsonDocument.Parse(request.BodyText!); + var items = new List(); + foreach (var subrequest in sent.RootElement.GetProperty("requests").EnumerateArray()) + { + var id = subrequest.GetProperty("id").GetString(); + var url = subrequest.GetProperty("url").GetString(); + var faulted = faultUrl != null && string.Equals(url, faultUrl, StringComparison.Ordinal); + var status = faulted ? faultStatus : 200; + // Retry-After on the throttled sub-response, so the item loop's backoff is the + // server's and not an exponential one this suite would have to wait out. + var headers = faulted && status == 429 ? ",\"headers\":{\"Retry-After\":\"0\"}" : ""; + var body = faulted + ? ",\"body\":{\"error\":{\"code\":\"Injected_" + status + "\"}}" + : ",\"body\":{\"id\":\"" + url!.Split('/')[^1] + "\"}"; + items.Add("{\"id\":\"" + id + "\",\"status\":" + status + headers + body + "}"); + } + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + "{\"responses\":[" + string.Join(",", items) + "]}", Encoding.UTF8, "application/json") + }; + } + + /// The chunk carrying the target subrequest, on any attempt - a retry re-sends it alone. + private static bool CarriesTheTarget(MockRequest request) => + request.BodyText?.Contains("\"" + TargetUrl + "\"", StringComparison.Ordinal) == true; + + private static MockHttpHandler TwoChunks(Action> armSecondChunk) + { + var handler = new MockHttpHandler(); + armSecondChunk(handler.When(CarriesTheTarget, FaultName).OnAttempt(1)); + handler.When(CarriesTheTarget).Respond(r => Envelope(r, faultUrl: null, faultStatus: 0)); + // Everything else: the first chunk, answered in full. + handler.When(_ => true).Respond(r => Envelope(r, faultUrl: null, faultStatus: 0)); + return handler; + } + + private static void ArmAtTheTargetSubrequest(MockResponseRule rule, FaultEntry entry) + { + switch (entry.Shape) + { + case FaultShape.Status: + // Graph answers $batch with 200 and puts the subrequest's failure in its own + // sub-response, so this is the only shape a status can take here. + rule.Respond(r => Envelope(r, TargetUrl, entry.Status)); + break; + case FaultShape.Transport: + rule.Throw(entry.Transport!()); + break; + case FaultShape.HeldPastAttemptTimeout: + rule.AfterDelay(TimeSpan.FromSeconds(5)) + .Respond(r => Envelope(r, faultUrl: null, faultStatus: 0)); + break; + case FaultShape.StalledBody: + rule.RespondStalled(HttpStatusCode.OK, prefix: """{"responses":["""); + break; + case FaultShape.TruncatedJson: + rule.Respond(HttpStatusCode.OK, TruncatedEnvelope); + break; + default: + throw new InvalidOperationException( + $"{entry.Kind} does not take a wire shape in a batch; it needs its own case."); + } + } + + private static async Task RunBatchAsync(MockHttpHandler handler) + { + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, Options) + { + BodyReadTimeout = ShortBodyRead + }; + var batchClient = new GraphBatchClient(client); + return await batchClient.ExecuteBatchIndexedAsync(AllOperations()); + } + + private static int StatusOf(BatchExecutionResult result, string url) => + result.Results.Single(r => r.Operation.Url == url).Response.Status; + + [Theory] + [MemberData(nameof(FaultCatalog.AllKinds), MemberType = typeof(FaultCatalog))] + public async Task Each_fault_at_the_target_subrequest(FaultKind kind) + { + var entry = FaultCatalog.Entry(kind); + var cell = entry.At(InjectionPoint.BatchSubrequest); + if (cell.Coverage == FaultCoverage.NotApplicable) + { + FaultInjectionRows.AssertNotApplicable(entry, InjectionPoint.BatchSubrequest); + return; + } + + ResiliencePipelineFactory.Reset(); + try + { + var handler = TwoChunks(rule => ArmAtTheTargetSubrequest(rule, entry)); + var result = await RunBatchAsync(handler); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + if (entry.Shape == FaultShape.Status) + AssertAFaultedSubResponse(entry, handler, result, cell.Coverage); + else + AssertARefusedChunk(entry, handler, result); + } + finally + { + ResiliencePipelineFactory.Reset(); + } + } + + /// + /// A status inside the envelope. The item loop asks the same policy the request pipeline + /// does, with the item's own method - GET here, so idempotent. + /// + private static void AssertAFaultedSubResponse( + FaultEntry entry, MockHttpHandler handler, BatchExecutionResult result, FaultCoverage coverage) + { + Assert.Null(result.ChunkFailure); + Assert.Equal(Operations, result.Results.Count); + Assert.Empty(result.NotSent); + + // Every subrequest but the target was answered 200, in both chunks. + foreach (var (operation, response) in result.Results.Where(r => r.Operation.Url != TargetUrl)) + Assert.InRange(response.Status, 200, 299); + + if (FaultCatalog.RetriedAsABatchItem(entry, "GET")) + { + // The retry is on the wire: chunk 1, chunk 2, and chunk 2 again carrying the one item. + Assert.Equal(3, handler.RequestCount); + Assert.Equal(1, result.Telemetry.ItemRetries); + Assert.InRange(StatusOf(result, TargetUrl), 200, 299); + Assert.Equal(Operations, result.Telemetry.Succeeded); + Assert.Equal(0, result.Telemetry.Failed); + Assert.Contains(TargetUrl, handler.CapturedRequests[2].BodyText); + Assert.DoesNotContain("/users/u21", handler.CapturedRequests[2].BodyText); + } + else + { + // One POST per chunk and no more: the item keeps the status the server gave it, and + // the batch-level retry pass does not pick up a status the policy will not retry. + Assert.Equal(2, handler.RequestCount); + Assert.Equal(0, result.Telemetry.ItemRetries); + Assert.Equal(0, result.Telemetry.BatchLevelRetries); + Assert.Equal(entry.Status, StatusOf(result, TargetUrl)); + Assert.Equal(Operations - 1, result.Telemetry.Succeeded); + Assert.Equal(1, result.Telemetry.Failed); + + if (coverage == FaultCoverage.Pinned) + AssertThePin(entry, handler); + } + } + + /// + /// What the two pinned status rows add over the plain non-retryable case: an expired token is + /// not refreshed and re-sent, and a read-after-write 404 is not waited out. + /// + private static void AssertThePin(FaultEntry entry, MockHttpHandler handler) + { + switch (entry.Kind) + { + case FaultKind.ExpiredToken: + Assert.Equal(MgxErrorClass.Authentication, entry.Class); + // Two POSTs, both answered 200 at the envelope level: nothing re-authenticated + // and sent the chunk again. + Assert.Equal([HttpStatusCode.OK, HttpStatusCode.OK], handler.ServedStatusCodes); + break; + case FaultKind.ConsistencyDelay: + // MgxErrorClass.Consistency is declared for the read-after-write window and has + // no producer; a 404 is a 404, and the item loop does not retry one. + Assert.Equal(MgxErrorClass.NotFound, entry.Class); + Assert.False(FaultCatalog.RetriedAsABatchItem(entry, "GET")); + break; + default: + Assert.Fail($"{entry.Kind} is pinned inside a batch envelope with nothing to record."); + break; + } + } + + /// + /// A transport failure on the chunk's own POST. The POST is not idempotent, so the policy + /// does not retry it whatever the class - and the chunk before it keeps everything the server + /// answered, which is the record of what it applied. + /// + private static void AssertARefusedChunk( + FaultEntry entry, MockHttpHandler handler, BatchExecutionResult result) + { + Assert.False(FaultCatalog.RetriedOnANonIdempotentRequest(entry)); + Assert.Equal(2, handler.RequestCount); + Assert.NotNull(result.ChunkFailure); + AssertTheFaultSurfaced(entry, result.ChunkFailure!); + Assert.Equal(Operations, result.Results.Count); + + var answered = result.Results.Take(ChunkSize).ToList(); + var refused = result.Results.Skip(ChunkSize).ToList(); + + // The first chunk's twenty are not unmade by the second chunk's failure. + Assert.All(answered, r => Assert.InRange(r.Response.Status, 200, 299)); + Assert.Equal(ChunkSize, result.Telemetry.Succeeded); + + // The refused chunk's POST went out, so none of its items may read as never sent: the + // difference between a request that may have reached the service and one that did not. + Assert.Equal(Operations - ChunkSize, refused.Count); + Assert.All(refused, r => + { + Assert.NotEqual(GraphBatchClient.NotSentStatus, r.Response.Status); + Assert.True(r.Response.Status >= 400, + $"{r.Operation.Url} came back {r.Response.Status}, which reads as a success"); + }); + Assert.Empty(result.NotSent); + } + + /// + /// The armed fault itself, as the caller gets it. A chunk failure is handed back on the + /// result rather than thrown, so this asks of + /// what the pagination theory asks of the exception the iterator raises: the kind that was + /// armed has to be the kind that surfaced, or the row proves only that something went wrong. + /// + private static void AssertTheFaultSurfaced(FaultEntry entry, Exception failure) + { + switch (entry.Shape) + { + case FaultShape.Transport: + // Straight through: the send throws, the POST is not retried, and the exception + // the mock raised is the one the result carries, reset SocketException and all. + var reset = Assert.IsType(failure); + Assert.Equal(entry.Transport!().Message, reset.Message); + Assert.IsType(reset.InnerException); + break; + case FaultShape.HeldPastAttemptTimeout: + // The pipeline's own per-attempt timeout, not the server's answer: the held + // response never arrives, so what surfaces names the timeout that gave up on it. + var timedOut = Assert.IsType(failure); + Assert.Equal(TimeSpan.FromSeconds(Options.AttemptTimeoutSeconds), timedOut.Timeout); + break; + case FaultShape.StalledBody: + // Headers arrived, so the timeout is the body read's and says so. + var stalled = Assert.IsType(failure); + Assert.Equal(ResilientGraphClient.BodyReadTimedOutMessage, stalled.Message); + break; + case FaultShape.TruncatedJson: + // The envelope is a 200 that stops mid-document; the parse is what fails, and + // it fails inside the responses array rather than at the top of the envelope. + var parse = Assert.IsAssignableFrom(failure); + Assert.Contains("responses", parse.Message, StringComparison.Ordinal); + break; + default: + Assert.Fail($"{entry.Kind} surfaced as {failure.GetType().Name} with no case to check it."); + break; + } + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Batch/ParallelChunkPartialResultTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/ParallelChunkPartialResultTests.cs new file mode 100644 index 0000000..57c920c --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Batch/ParallelChunkPartialResultTests.cs @@ -0,0 +1,854 @@ +using System.Net; +using Mgx.Engine.Http; +using Mgx.Engine.Models; + +namespace Mgx.IntegrationTests; + +/// +/// The partial-result contract of a chunked batch does not depend on how many chunks are in +/// flight at once. With -BatchChunkConcurrency above 1 the chunks run in parallel, and a chunk +/// whose own POST fails used to take the whole run down with it: Task.WhenAll rethrew and the +/// results of the chunks that had already been applied on the server were discarded with the +/// array holding them. +/// +[Collection("Pipeline")] +public class ParallelChunkPartialResultTests +{ + private const string Rejection = + "{\"error\":{\"code\":\"BadRequest\",\"message\":\"batch rejected\"}}"; + + private static string BatchResponse(int count, int status) => + "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, count).Select(i => + $"{{ \"id\": \"{i}\", \"status\": {status} }}")) + "] }"; + + /// + /// Rejects one chunk by its CONTENT, not by arrival order, so sequential and parallel runs + /// see the same wire behavior. + /// + /// Nothing is answered until the whole opening wave is on the wire, so no chunk of it is + /// stopped before it has sent: left to the scheduler the refusal can be recorded while a + /// chunk that holds a permit has not reached the handler yet, and that chunk then turns + /// back correctly at a wire count the test reads as the thing it exists to catch. + /// + /// + /// The rejected chunk is then answered first, so the permit it holds is released only after + /// the run has recorded the refusal. The rest of the wave is held until the run says it has + /// stopped sending, so that permit is the only one a queued chunk can be admitted on and + /// the refusal is recorded before the last chunk asks whether to send. A POST arriving from + /// outside the wave releases them as well: that is the thing the guard exists to prevent + /// and the thing the counts have to be able to see, so a run that sends one is counted + /// rather than held here behind it. + /// + /// + private sealed class RejectOneChunkHandler : HttpMessageHandler + { + private readonly string _marker; + private readonly int _wave; + private readonly TaskCompletionSource _inFlight = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _settled = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _posts; + private int _arrived; + private int _landing; + public int Posts => Volatile.Read(ref _posts); + + /// Body text carried by the one chunk to reject. + /// Chunks other than that one which hold a permit from the start. + /// Zero for a sequential run, which holds one chunk at a time and needs no gate. + public RejectOneChunkHandler(string marker, int wave) + { + _marker = marker; + _wave = wave; + if (wave == 0) _settled.TrySetResult(); + } + + /// Armed on the run: the refusal is recorded and nothing further may send. + public void RunStopped() => _settled.TrySetResult(); + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _posts); + if (Interlocked.Increment(ref _arrived) >= _wave + 1) _inFlight.TrySetResult(); + await Task.Yield(); + var body = await request.Content!.ReadAsStringAsync(cancellationToken); + await _inFlight.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + if (body.Contains(_marker, StringComparison.Ordinal)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent(Rejection, System.Text.Encoding.UTF8, "application/json") + }; + } + + if (Interlocked.Increment(ref _landing) > _wave) _settled.TrySetResult(); + await _settled.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(BatchResponse(20, 204), + System.Text.Encoding.UTF8, "application/json") + }; + } + } + + /// + /// Four chunks, so at concurrency 3 one of them has to wait for a permit and the guard that + /// decides whether a waiting chunk still goes out is actually consulted. At 60 operations + /// every chunk held a permit from the start and the guard was never reached. + /// + private static async Task<(BatchExecutionResult Result, int Posts)> Run(int concurrency, int wave) + { + ResiliencePipelineFactory.Reset(); + // "u21" is the first operation of the second chunk and appears in no other chunk. + var wire = new RejectOneChunkHandler("/users/u21\"", wave); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client, batchChunkConcurrency: concurrency) + { + SendingStopped = wire.RunStopped + }; + + var ops = Enumerable.Range(1, 80) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + return (result, wire.Posts); + } + + // Sequentially the run stops at the refusal: two POSTs, two chunks never sent. In parallel + // three chunks are on the wire and the fourth is still queued when the refusal is recorded, + // so it never goes out either - one chunk refused, one stopped, two applied. + [Theory] + [InlineData(1, 0, 2, 20, 40)] + [InlineData(3, 2, 3, 40, 20)] + public async Task A_failed_chunk_keeps_the_results_of_the_chunks_that_landed( + int concurrency, int wave, int expectedPosts, int expectedLanded, int expectedNotSent) + { + var (result, posts) = await Run(concurrency, wave); + + Assert.NotNull(result.ChunkFailure); + Assert.IsType(result.ChunkFailure); + + // Every operation is accounted for by position, so a caller can line the results up + // against its own input list. + Assert.Equal(80, result.Results.Count); + Assert.Equal(expectedPosts, posts); + + var landed = result.Results.Count(r => r.Response.Status == 204); + var refused = result.Results.Count(r => r.Response.Status == 400); + var notSent = result.Results.Count(r => r.Response.Status == GraphBatchClient.NotSentStatus); + + // Exact counts: an inequality here passes just as well when a chunk that should have + // been stopped went out anyway, which is the thing worth knowing. + Assert.Equal(expectedLanded, landed); + Assert.Equal(20, refused); + Assert.Equal(expectedNotSent, notSent); + Assert.Equal(notSent, result.NotSent.Count); + } + + /// + /// Answers nothing until as many chunks are on the wire as the run can hold at once. The + /// two failures then reach the run at the same time rather than one after the other, and a + /// chunk that was going to send is not stopped before it has - left to the scheduler a + /// chunk can still be waiting to start when the refusals land, and it is then correctly + /// reported as never sent, which is a different outcome from the one asked about here. + /// + /// Chunks the run holds at once, so all of them and no more. + private sealed class TwoChunksRefusedTogetherHandler(int inFlight) : HttpMessageHandler + { + private int _arrived; + private readonly TaskCompletionSource _all = new(TaskCreationOptions.RunContinuationsAsynchronously); + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + await Task.Yield(); + var body = await request.Content!.ReadAsStringAsync(cancellationToken); + var refusal = body.Contains("/users/u1\"", StringComparison.Ordinal) ? "chunk A" + : body.Contains("/users/u21\"", StringComparison.Ordinal) ? "chunk B" + : null; + if (Interlocked.Increment(ref _arrived) == inFlight) _all.TrySetResult(); + await _all.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + if (refusal == null) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(BatchResponse(20, 204), + System.Text.Encoding.UTF8, "application/json") + }; + } + + return new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent( + $"{{\"error\":{{\"code\":\"BadRequest\",\"message\":\"{refusal} rejected\"}}}}", + System.Text.Encoding.UTF8, "application/json") + }; + } + } + + [Fact] + public async Task A_second_chunk_failing_at_the_same_time_is_still_reported() + { + ResiliencePipelineFactory.Reset(); + using var httpClient = new HttpClient(new TwoChunksRefusedTogetherHandler(inFlight: 3)); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var messages = new List(); + var batchClient = new GraphBatchClient(client, batchChunkConcurrency: 3) + { + VerboseWriter = messages.Add + }; + + var ops = Enumerable.Range(1, 60) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + batchClient.DrainVerboseMessages(); + + Assert.NotNull(result.ChunkFailure); + // One of the two is the failure the caller is handed. The other lost a race it had no + // stake in, and used to disappear with no record that it had happened at all. + var reported = result.ChunkFailure!.Message.Contains("chunk A", StringComparison.Ordinal) + ? "chunk A" : "chunk B"; + var lost = reported == "chunk A" ? "chunk B" : "chunk A"; + Assert.Contains(messages, m => m.Contains(lost, StringComparison.Ordinal)); + + // Both refused chunks report their own items as attempted, not as never sent. + for (int i = 0; i < 40; i++) + Assert.Equal(400, result.Results[i].Response.Status); + for (int i = 40; i < 60; i++) + Assert.Equal(204, result.Results[i].Response.Status); + } + + /// + /// With chunks in flight together there is no "earlier" chunk to blame: the two that were + /// refused ran alongside each other, and the one that never went out was behind both. + /// + [Fact] + public void An_operation_that_was_never_sent_is_not_blamed_on_an_earlier_chunk() + { + using (MgxTransportScope.Inject(new TwoChunksRefusedTogetherHandler(inFlight: 2), + options: new ResilientGraphClientOptions + { + NoRateLimit = true, + MaxRetryAttempts = 1, + BatchChunkConcurrency = 2 + })) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", Enumerable.Range(1, 60).Select(i => $"/users/u{i}").ToArray()) + .AddParameter("Method", "PATCH"); + ps.Invoke(); + + // Two chunks hold both permits until both are refused, so the third can only reach + // the semaphore after a refusal has been recorded: it never goes out. + var notSent = ps.Streams.Error + .Where(e => e.FullyQualifiedErrorId.StartsWith("BatchItemNotSent", StringComparison.Ordinal)) + .ToList(); + Assert.Equal(20, notSent.Count); + Assert.All(notSent, e => Assert.DoesNotContain("earlier", e.Exception.Message, + StringComparison.OrdinalIgnoreCase)); + Assert.All(notSent, e => Assert.Contains("was not sent", e.Exception.Message, + StringComparison.Ordinal)); + } + } + + /// + /// One chunk's call fails with no HTTP response at all - the shape a stalled $batch body, + /// an open circuit or a dead connection arrives in. + /// + /// Nothing is answered until as many chunks are on the wire as the run can carry at once, so + /// the one that fails does so with the others already sent. Left to the scheduler a chunk can + /// still be waiting to start when the failure lands - it is then correctly stopped, which is + /// a different outcome, asked about by a different test. + /// + /// + private sealed class OneChunkThrowsHandler(int inFlight) : HttpMessageHandler + { + private readonly TaskCompletionSource _all = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _arrived; + private int _posts; + public int Posts => Volatile.Read(ref _posts); + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _posts); + if (Interlocked.Increment(ref _arrived) == inFlight) _all.TrySetResult(); + await Task.Yield(); + await _all.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + var body = await request.Content!.ReadAsStringAsync(cancellationToken); + if (body.Contains("/users/u21\"", StringComparison.Ordinal)) + throw new HttpRequestException("connection reset"); + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(BatchResponse(20, 204), + System.Text.Encoding.UTF8, "application/json") + }; + } + } + + /// + /// A chunk can fail without a status to fail with. Every one of those shapes used to travel + /// out of the task and take the results array with it, so the chunks the server had already + /// applied came back as nothing. + /// + [Theory] + [InlineData(1)] + [InlineData(3)] + public async Task A_chunk_that_fails_without_a_response_keeps_the_other_chunks(int concurrency) + { + ResiliencePipelineFactory.Reset(); + // Sixty operations are three chunks, so at concurrency 3 the run can hold all three at + // once and the handler answers none of them until it has all three. The failure is then + // reached with nothing left to stop, which is the case this asserts on. + var wire = new OneChunkThrowsHandler(Math.Min(concurrency, 3)); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client, batchChunkConcurrency: concurrency); + + var ops = Enumerable.Range(1, 60) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.IsType(result.ChunkFailure); + Assert.Equal(60, result.Results.Count); + + // The first chunk was applied on the server whichever way the second one died. + for (int i = 0; i < 20; i++) + Assert.Equal(204, result.Results[i].Response.Status); + + // The failed chunk's own items went out and may have been applied. Without a status of + // their own they read as attempted-and-unknown, never as unattempted. + for (int i = 20; i < 40; i++) + Assert.Equal(503, result.Results[i].Response.Status); + + // Sequentially the third chunk never goes out; in parallel it was already in flight. + Assert.Equal(concurrency == 1 ? 20 : 0, result.NotSent.Count); + } + + /// + /// Three chunks, all in flight at once. One is refused; one is throttled and would retry a + /// second later; one is applied. The throttled chunk's retry is the only POST that could + /// still go out after the refusal is recorded, so the wire count says whether anything did. + /// + /// The order the question needs is gated, not raced. Nothing is answered until all three + /// chunks are on the wire, so none of them is stopped before it has sent and the count of + /// three means three chunks that really went out. The refusal then goes back first, and the + /// throttle only once the run has recorded it - so the throttled chunk meets the stop at + /// its next attempt, instead of a one-second Retry-After having to outlast the refusal on a + /// loaded machine to reach the same place. + /// + /// + private sealed class RefuseOneThrottleOneHandler : HttpMessageHandler + { + private readonly TaskCompletionSource _allInFlight = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _stopped = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _arrived; + private int _posts; + public int Posts => Volatile.Read(ref _posts); + + /// Armed on the run: the refusal is recorded and nothing further may send. + public void RunStopped() => _stopped.TrySetResult(); + + private static string Throttled(int count) => + "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, count).Select(i => + $"{{ \"id\": \"{i}\", \"status\": 429, \"headers\": {{ \"Retry-After\": \"1\" }} }}")) + "] }"; + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _posts); + if (Interlocked.Increment(ref _arrived) == 3) _allInFlight.TrySetResult(); + await Task.Yield(); + var body = await request.Content!.ReadAsStringAsync(cancellationToken); + // Answered only once the caller has started the others, so all three are genuinely + // in flight - a handler that answers inline serializes them and the guard never has + // two chunks to arbitrate. + await _allInFlight.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + + if (body.Contains("/users/u21\"", StringComparison.Ordinal)) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent(Rejection, System.Text.Encoding.UTF8, "application/json") + }; + } + if (body.Contains("/users/u1\"", StringComparison.Ordinal)) + { + await _stopped.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(Throttled(20), System.Text.Encoding.UTF8, "application/json") + }; + } + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(BatchResponse(20, 204), + System.Text.Encoding.UTF8, "application/json") + }; + } + } + + [Fact] + public async Task Nothing_further_is_posted_once_a_chunk_has_been_refused() + { + ResiliencePipelineFactory.Reset(); + var wire = new RefuseOneThrottleOneHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client, batchChunkConcurrency: 3) + { + SendingStopped = wire.RunStopped + }; + + var ops = Enumerable.Range(1, 60) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.NotNull(result.ChunkFailure); + // One POST per chunk and not one more: no per-chunk retry after the refusal, and no + // follow-up batch either. Each of those would be a write the caller is told nothing about. + Assert.Equal(3, wire.Posts); + Assert.Equal(0, result.Telemetry.BatchLevelRetries); + + // The throttled chunk stops where it is, holding what the server last said about it. + for (int i = 0; i < 20; i++) + Assert.Equal(429, result.Results[i].Response.Status); + for (int i = 20; i < 40; i++) + Assert.Equal(400, result.Results[i].Response.Status); + for (int i = 40; i < 60; i++) + Assert.Equal(204, result.Results[i].Response.Status); + Assert.Empty(result.NotSent); + } + + /// + /// Throttles one chunk for a minute, and refuses another only once the throttled chunk is + /// into the wait between its attempts - the run says when, rather than the refusal being + /// held back long enough that it usually is. The refusal then arrives with the backoff + /// running, which is the only state the question of whether the wait ends with it can be + /// asked from. Nothing is answered until all three chunks are on the wire, so none of them + /// is stopped before it has sent. + /// + private sealed class RefusesDuringARetryWaitHandler : HttpMessageHandler + { + private readonly TaskCompletionSource _inFlight = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _waiting = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _arrived; + private int _posts; + public int Posts => Volatile.Read(ref _posts); + + /// Armed on the run: the throttled chunk is in the wait between two attempts. + public void RetryWaitStarted() => _waiting.TrySetResult(); + + private static string Throttled(int count) => + "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, count).Select(i => + $"{{ \"id\": \"{i}\", \"status\": 429, \"headers\": {{ \"Retry-After\": \"60\" }} }}")) + "] }"; + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _posts); + if (Interlocked.Increment(ref _arrived) == 3) _inFlight.TrySetResult(); + await Task.Yield(); + var body = await request.Content!.ReadAsStringAsync(cancellationToken); + await _inFlight.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + if (body.Contains("/users/u1\"", StringComparison.Ordinal)) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(Throttled(20), + System.Text.Encoding.UTF8, "application/json") + }; + } + if (body.Contains("/users/u21\"", StringComparison.Ordinal)) + { + await _waiting.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + return new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent(Rejection, System.Text.Encoding.UTF8, "application/json") + }; + } + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(BatchResponse(20, 204), + System.Text.Encoding.UTF8, "application/json") + }; + } + } + + /// + /// The run stops sending the moment a chunk is refused, and a chunk already waiting out a + /// Retry-After has nothing left to wait for. Consulted only before the backoff started, the + /// stop left the call blocked for the rest of the delay - here a minute, and up to the + /// two-minute cap plus jitter against a server that asks for longer. + /// + [Fact] + public async Task A_refusal_ends_a_retry_wait_that_is_already_running() + { + ResiliencePipelineFactory.Reset(); + var wire = new RefusesDuringARetryWaitHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client, batchChunkConcurrency: 3) + { + RetryWaitStarted = wire.RetryWaitStarted + }; + + var ops = Enumerable.Range(1, 60) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + sw.Stop(); + + Assert.NotNull(result.ChunkFailure); + Assert.True(sw.Elapsed < TimeSpan.FromSeconds(15), + $"the call waited out {sw.Elapsed.TotalSeconds:F1}s of a Retry-After it had stopped honoring"); + // Nothing more went out: the wait ended because the run stopped, not because it expired. + Assert.Equal(3, wire.Posts); + for (int i = 0; i < 20; i++) + Assert.Equal(429, result.Results[i].Response.Status); + for (int i = 20; i < 40; i++) + Assert.Equal(400, result.Results[i].Response.Status); + for (int i = 40; i < 60; i++) + Assert.Equal(204, result.Results[i].Response.Status); + } + + /// + /// The chunk that was rejected went out on the wire, so its operations carry the refusal. + /// Reporting them as never sent would send a caller back to re-submit writes that may + /// already have been applied. + /// + [Fact] + public async Task The_rejected_chunks_operations_carry_the_refusal_not_a_not_sent_status() + { + var (result, _) = await Run(3, wave: 2); + + Assert.DoesNotContain(result.NotSent, op => op.Url == "/users/u21"); + for (int i = 20; i < 40; i++) + Assert.Equal(400, result.Results[i].Response.Status); + for (int i = 0; i < 20; i++) + Assert.Equal(204, result.Results[i].Response.Status); + } + + /// + /// The verbose summary is the run's own account of what it did. A retry counted when it was + /// decided on rather than when it went out is a request that never left: the refusal ends + /// the backoff, the chunk breaks at its next attempt, and the run reports twenty item + /// retries over a wire that carried one POST per chunk. + /// + [Fact] + public async Task Retries_a_refusal_stopped_are_not_counted_as_retries_that_went_out() + { + ResiliencePipelineFactory.Reset(); + var wire = new RefusesDuringARetryWaitHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client, batchChunkConcurrency: 3) + { + RetryWaitStarted = wire.RetryWaitStarted + }; + + var ops = Enumerable.Range(1, 60) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.NotNull(result.ChunkFailure); + // One POST per chunk and no fourth: the twenty the throttled chunk was going to resend + // never went out. + Assert.Equal(3, wire.Posts); + Assert.Equal(0, result.Telemetry.ItemRetries); + // The throttles and the wait are not in question - those twenty 429s arrived, and the + // chunk did spend time on the backoff before the refusal ended it. + Assert.Equal(20, result.Telemetry.ThrottleEncounters); + } + + /// + /// Throttles one chunk's ENVELOPE - a 429 on the $batch POST itself, with a Retry-After the + /// transport honors - and refuses a second chunk once that throttle has landed, while a + /// third is still on the wire. The throttled chunk is inside the transport's own retry, a + /// layer below the loop that reads the stop, and the wire count says whether its writes + /// went out again after the run had stopped sending. + /// + /// The third chunk is answered only once the run has recorded the refusal, so it is on the + /// wire at that moment by the run's own account of when the moment was. The caller holds + /// the throttled chunk's second attempt at the head of the attempt until the same point, + /// so that attempt meets the stop rather than racing a five-second Retry-After against the + /// round trip that carries the refusal. Nothing at all is answered until all three chunks + /// are on the wire, so none of them is stopped before it has sent. + /// + /// + private sealed class ThrottlesTheEnvelopeThenRefusesHandler : HttpMessageHandler + { + private readonly TaskCompletionSource _inFlight = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _throttled = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _stopped = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _arrived; + private int _posts; + private int _throttledChunkPosts; + public int Posts => Volatile.Read(ref _posts); + public int ThrottledChunkPosts => Volatile.Read(ref _throttledChunkPosts); + + /// Armed on the run: the refusal is recorded and nothing further may send. + public void RunStopped() => _stopped.TrySetResult(); + + /// Completes with that same point, for a caller ordering an attempt against it. + public Task Stopped => _stopped.Task; + + private static HttpResponseMessage Answered(HttpRequestMessage request) => + new(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(BatchResponse(20, 204), + System.Text.Encoding.UTF8, "application/json") + }; + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _posts); + if (Interlocked.Increment(ref _arrived) == 3) _inFlight.TrySetResult(); + await Task.Yield(); + var body = await request.Content!.ReadAsStringAsync(cancellationToken); + await _inFlight.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + + if (body.Contains("/users/u1\"", StringComparison.Ordinal)) + { + if (Interlocked.Increment(ref _throttledChunkPosts) == 1) + { + var throttle = new HttpResponseMessage(HttpStatusCode.TooManyRequests) + { + RequestMessage = request, + Content = new StringContent(Rejection, System.Text.Encoding.UTF8, "application/json") + }; + throttle.Headers.RetryAfter = + new System.Net.Http.Headers.RetryConditionHeaderValue(TimeSpan.FromSeconds(5)); + _throttled.TrySetResult(); + return throttle; + } + // A second POST of this chunk is the run sending after it stopped. Answered + // rather than refused, so the writes it applied show up in the result and the + // test fails on what happened rather than on how it was reported. + return Answered(request); + } + + if (body.Contains("/users/u21\"", StringComparison.Ordinal)) + { + await _throttled.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + return new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent(Rejection, System.Text.Encoding.UTF8, "application/json") + }; + } + + // Still on the wire when the refusal above is recorded: the server is about to + // answer all twenty of its writes, and that answer is the record of what it applied. + await _stopped.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + return Answered(request); + } + } + + /// + /// A chunk inside the transport's own throttle retry when a sibling is refused. The stop + /// reaches the decision to send again, and not the attempt on the wire: nothing further + /// goes out, a chunk that was mid-request keeps the answers the server gave it, and the + /// call comes back rather than waiting out a Retry-After the run has stopped honoring. + /// + [Fact] + public async Task A_chunk_inside_the_transports_retry_stops_with_the_run() + { + ResiliencePipelineFactory.Reset(); + var wire = new ThrottlesTheEnvelopeThenRefusesHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + // The second attempt of the throttled chunk is the POST in question. Held at the head of + // the attempt until the refusal has been recorded, so whether it goes out is decided by + // the stop and not by which of the two finished first. + client.AttemptEntryGate = async attempt => + { + if (attempt < 2) return; + await wire.Stopped.WaitAsync(TimeSpan.FromSeconds(30)); + }; + var batchClient = new GraphBatchClient(client, batchChunkConcurrency: 3) + { + SendingStopped = wire.RunStopped + }; + + var ops = Enumerable.Range(1, 60) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + sw.Stop(); + + Assert.NotNull(result.ChunkFailure); + + // Nothing further went out. The throttled chunk's second POST is the only one that + // could, and it carries twenty writes the caller would be told nothing about. + Assert.Equal(1, wire.ThrottledChunkPosts); + Assert.Equal(3, wire.Posts); + + // The chunk that was on the wire keeps what the server said about each of its items. + // Ending its request to stop the run would trade twenty known outcomes for twenty + // writes that may or may not have been applied. + for (int i = 40; i < 60; i++) + Assert.Equal(204, result.Results[i].Response.Status); + + // The throttled chunk carries the status its envelope was refused with, the same one it + // would carry had the transport run out of retries instead of being stopped. + for (int i = 0; i < 20; i++) + Assert.Equal(429, result.Results[i].Response.Status); + for (int i = 20; i < 40; i++) + Assert.Equal(400, result.Results[i].Response.Status); + Assert.Empty(result.NotSent); + + // The Retry-After was five seconds. Waiting it out holds the call open for a delay the + // run has already decided not to act on. + Assert.True(sw.Elapsed < TimeSpan.FromSeconds(3), + $"the call waited out {sw.Elapsed.TotalSeconds:F1}s of a Retry-After it had stopped honoring"); + } + + /// + /// Refuses whichever POST holds the one permit that was free, and answers every later one. + /// The refusal waits for the other chunks to be queued at the limiter - counted there, not + /// allowed time to get there - so a later POST is a chunk that waited out its permit and + /// sent anyway, not one that reached the queue after the refusal had already been recorded. + /// + private sealed class RefusesTheFirstPostHandler( + System.Threading.RateLimiting.TokenBucketRateLimiter limiter, int queued) : HttpMessageHandler + { + private int _posts; + public int Posts => Volatile.Read(ref _posts); + + private int Queued => (int)limiter.GetStatistics()!.CurrentQueuedCount; + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + if (Interlocked.Increment(ref _posts) == 1) + { + var waited = System.Diagnostics.Stopwatch.StartNew(); + while (Queued < queued) + { + if (waited.Elapsed > TimeSpan.FromSeconds(30)) + throw new TimeoutException( + $"only {Queued} of {queued} chunks reached the permit queue"); + await Task.Delay(5, cancellationToken); + } + return new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent(Rejection, System.Text.Encoding.UTF8, "application/json") + }; + } + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(BatchResponse(20, 204), + System.Text.Encoding.UTF8, "application/json") + }; + } + } + + /// + /// A rate limiter of one permit a second, which the tuning help recommends verbatim for + /// OneNote and Planner, and chunk parallelism above one. Two chunks are parked waiting for a + /// permit when the third is refused; the stop they were given has to reach the wait they are + /// in, or each POSTs its twenty writes as its permit comes up - seconds after the run + /// recorded that it had stopped sending, and with nothing that reports on them. + /// + [Fact] + public async Task Chunks_waiting_for_a_rate_limit_permit_do_not_post_after_a_refusal() + { + ResiliencePipelineFactory.Reset(); + // The limiter the run's requests queue at, held so the handler can count who is in that + // queue rather than guess how long they take to reach it. + var (pipeline, limiter) = ResiliencePipelineFactory.GetOrCreate(new ResilientGraphClientOptions + { + RateLimitBurst = 1, + RateLimitPerSecond = 1, + RateLimitQueueLimit = 100, + MaxRetryAttempts = 1 + }); + var wire = new RefusesTheFirstPostHandler(limiter!, queued: 2); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, pipeline, limiter); + var batchClient = new GraphBatchClient(client, batchChunkConcurrency: 3); + + var ops = Enumerable.Range(1, 60) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + sw.Stop(); + + Assert.NotNull(result.ChunkFailure); + // One permit was free at the start and the chunk that took it was refused. The next two + // permits fall due a second and two seconds later; neither may carry a POST. + Assert.Equal(1, wire.Posts); + Assert.True(sw.Elapsed < TimeSpan.FromSeconds(3), + $"the run held on for {sw.Elapsed.TotalSeconds:F1}s of permits it had stopped needing"); + + // Forty operations never reached the server, and that is what they are reported as. A + // status here instead would say the writes were attempted and may have been applied, + // which is the one thing that would stop a caller resubmitting them. + Assert.Equal(40, result.NotSent.Count); + var refused = result.Results.Count(r => r.Response.Status == 400); + var neverSent = result.Results.Count(r => r.Response.Status == GraphBatchClient.NotSentStatus); + Assert.Equal(20, refused); + Assert.Equal(40, neverSent); + Assert.DoesNotContain(result.Results, r => r.Response.Status == 503); + + // Retries counted in front of a POST that never went out are requests the summary + // claims left when nothing did. + Assert.Equal(0, result.Telemetry.ItemRetries); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs b/tests/Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs index 70aed50..eee555d 100644 --- a/tests/Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Batch/PartialBatchResultTests.cs @@ -1,3 +1,5 @@ +using System.Management.Automation; +using System.Management.Automation.Host; using System.Net; using Mgx.Engine.Http; using Mgx.Engine.Models; @@ -70,6 +72,567 @@ public async Task A_failed_chunk_does_not_discard_the_chunks_that_already_succee $"only {known} of 30 operations came back; the first chunk's 20 results were discarded"); } + /// + /// Answers the chunk in part - some items applied, the rest throttled - and then refuses the + /// retry that carries the throttled ones. The items the first answer covered were applied on + /// the server whatever happens to the retry. + /// + private sealed class HalfLandsThenRefusesHandler : HttpMessageHandler + { + private int _posts; + public int Posts => Volatile.Read(ref _posts); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + if (Interlocked.Increment(ref _posts) == 1) + { + var body = "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, 20).Select(i => + $"{{ \"id\": \"{i}\", \"status\": {(i <= 15 ? 204 : 429)} }}")) + "] }"; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") + }); + } + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent( + "{\"error\":{\"code\":\"BadRequest\",\"message\":\"batch rejected\"}}", + System.Text.Encoding.UTF8, "application/json") + }); + } + } + + [Fact] + public async Task A_chunk_whose_retry_is_refused_keeps_the_items_the_server_answered() + { + ResiliencePipelineFactory.Reset(); + var wire = new HalfLandsThenRefusesHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + + var ops = Enumerable.Range(1, 20) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.NotNull(result.ChunkFailure); + // The server answered these 15 with 204: it applied them. A later attempt of the same + // chunk being refused does not unmake that. + for (int i = 0; i < 15; i++) + Assert.Equal(204, result.Results[i].Response.Status); + // The five the refusal carried were sent and may have been applied, so they read as a + // failure of the request - never as an operation that was not attempted. + for (int i = 15; i < 20; i++) + Assert.Equal(400, result.Results[i].Response.Status); + Assert.Empty(result.NotSent); + + // The chunk that failed is the chunk that met the throttling. Counted only off the + // chunks that returned normally, a run throttled into a refusal reported no throttling + // at all - and no retries, and no time spent waiting on either. + Assert.Equal(5, result.Telemetry.ThrottleEncounters); + Assert.Equal(5, result.Telemetry.ItemRetries); + Assert.True(result.Telemetry.TotalRetryDelayMs > 0, + "the retry delay this chunk waited out was not reported"); + } + + [Fact] + public void Items_a_refused_chunk_already_had_answers_for_stay_out_of_the_dead_letter_file() + { + var deadLetter = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + try + { + using (MgxTransportScope.Inject(new HalfLandsThenRefusesHandler())) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", Enumerable.Range(1, 20).Select(i => $"/users/u{i}").ToArray()) + .AddParameter("Method", "PATCH") + .AddParameter("DeadLetterPath", deadLetter); + ps.Invoke(); + } + + // The dead-letter file is documented as re-pipeable. A line for a write the server + // confirmed is an instruction to send it a second time. + var lines = File.Exists(deadLetter) ? File.ReadAllLines(deadLetter) : []; + Assert.Equal(5, lines.Length); + for (int i = 1; i <= 15; i++) + Assert.DoesNotContain(lines, l => l.Contains($"\"/users/u{i}\"", StringComparison.Ordinal)); + for (int i = 16; i <= 20; i++) + Assert.Contains(lines, l => l.Contains($"\"/users/u{i}\"", StringComparison.Ordinal)); + } + finally + { + if (File.Exists(deadLetter)) File.Delete(deadLetter); + } + } + + /// + /// Three chunks: one applied, one refused, one never sent. The counts have to add up to + /// something a reader can act on - "60 succeeded, 0 failed" over a run that sent one chunk + /// is worse than no summary at all. + /// + [Fact] + public async Task Operations_that_were_never_sent_count_as_neither_succeeded_nor_failed() + { + ResiliencePipelineFactory.Reset(); + using var httpClient = new HttpClient(new FirstChunkOnlyHandler()); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + + var ops = Enumerable.Range(1, 60) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.Equal(60, result.Telemetry.TotalRequests); + Assert.Equal(20, result.Telemetry.Succeeded); + Assert.Equal(20, result.Telemetry.Failed); + Assert.Equal(20, result.NotSent.Count); + } + + /// + /// Both runs stop, and stopping is not what tells them apart. In the first the refusal came + /// before the batch-level retry pass, which was then skipped for every candidate. In the + /// second the pass had already POSTed twenty items and applied fifteen of them when the + /// follow-up carrying the other five was refused. The line the caller sees turns on which + /// of those happened, not on the stop they share: told the pass never went out, nobody has + /// reason to check the tenant before creating fifteen users a second time. + /// + [Fact] + public void A_retry_pass_that_went_out_is_not_reported_as_one_that_was_withheld() + { + var withheld = Say(new FirstChunkOnlyHandler(), 60, "PATCH"); + var sent = Say(new RetryPassLandsThenIsRefusedHandler(), 20, "POST", + new System.Collections.Hashtable { ["displayName"] = "u" }); + + Assert.Contains("the retry pass was withheld", withheld.Warning, StringComparison.Ordinal); + // Nothing of the pass reached the wire, so the run credits itself with none of it. + Assert.DoesNotContain("Batch-level retries", withheld.Summary, StringComparison.Ordinal); + + Assert.DoesNotContain("withheld", sent.Warning, StringComparison.Ordinal); + Assert.Contains("The retry pass was sent and then refused", sent.Warning, StringComparison.Ordinal); + Assert.Contains(" Batch-level retries: 20.", sent.Summary, StringComparison.Ordinal); + + // Neither run took its items through every attempt they had, so neither may say it did. + Assert.DoesNotContain("after all retry attempts", withheld.Warning, StringComparison.Ordinal); + Assert.DoesNotContain("after all retry attempts", sent.Warning, StringComparison.Ordinal); + } + + /// Answers every chunk in full; five items of the first come back 404. + private sealed class AnswersEveryChunkHandler : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var body = "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, 20).Select(i => + $"{{ \"id\": \"{i}\", \"status\": {(i <= 15 ? 200 : 404)} }}")) + "] }"; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") + }); + } + } + + /// + /// Nothing stopped this run: every chunk was POSTed and answered, and the items that failed + /// had whatever attempts they qualified for. The warning says so, and says nothing about + /// operations left unsent, because there were none. + /// + [Fact] + public void A_run_that_was_not_stopped_still_reports_its_failures_as_retried() + { + using (MgxTransportScope.Inject(new AnswersEveryChunkHandler())) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", Enumerable.Range(1, 20).Select(i => $"/users/u{i}").ToArray()) + .AddParameter("Method", "GET"); + ps.Invoke(); + + var warning = Assert.Single(ps.Streams.Warning, + w => w.Message.Contains("batch items failed", StringComparison.Ordinal)).Message; + Assert.Contains("5 of 20 batch items failed.", warning, StringComparison.Ordinal); + Assert.Contains("after all retry attempts", warning, StringComparison.Ordinal); + Assert.DoesNotContain("were not sent", warning, StringComparison.Ordinal); + } + } + + /// + /// The dead-letter file takes the refused and the never-sent alike, and the line that says + /// what was written has to keep them apart: one may already have been applied on the server + /// and the other certainly was not, which is the whole of what a caller re-piping the file + /// is deciding about. + /// + [Fact] + public void The_dead_letter_line_does_not_call_an_unsent_operation_a_failed_one() + { + var deadLetter = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + try + { + using (MgxTransportScope.Inject(new FirstChunkOnlyHandler())) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", Enumerable.Range(1, 60).Select(i => $"/users/u{i}").ToArray()) + .AddParameter("Method", "PATCH") + .AddParameter("DeadLetterPath", deadLetter) + .AddParameter("Verbose", true); + ps.Invoke(); + + Assert.Equal(40, File.ReadAllLines(deadLetter).Length); + Assert.Contains(ps.Streams.Verbose, v => + v.Message.Contains("Wrote 20 failed and 20 not-sent items to dead-letter file", + StringComparison.Ordinal)); + } + } + finally + { + if (File.Exists(deadLetter)) File.Delete(deadLetter); + } + } + + /// + /// Answers both items of the only chunk and leaves the "status" member off the second - the + /// truncated-or-rewritten envelope the count check names, one member short instead of one + /// sub-response short. + /// + private sealed class OmitsAStatusHandler : HttpMessageHandler + { + private int _posts; + public int Posts => Volatile.Read(ref _posts); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _posts); + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent( + "{ \"responses\": [ { \"id\": \"1\", \"status\": 204, \"body\": null }, " + + "{ \"id\": \"2\", \"body\": null } ] }", + System.Text.Encoding.UTF8, "application/json") + }); + } + } + + /// + /// The POST went out and the server answered it. Whatever the envelope failed to say about + /// the second write, the one thing it cannot be is a write that never left - which is what a + /// status deserialized to 0 says, that being NotSentStatus itself. + /// + [Fact] + public async Task An_item_answered_without_a_status_is_not_handed_back_as_never_sent() + { + ResiliencePipelineFactory.Reset(); + var wire = new OmitsAStatusHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + + var ops = Enumerable.Range(1, 2) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.Equal(1, wire.Posts); + Assert.NotEqual(GraphBatchClient.NotSentStatus, result.Results[1].Response.Status); + Assert.True(result.Results[1].Response.Status >= 400, + $"a write the server answered came back as {result.Results[1].Response.Status}"); + Assert.Empty(result.NotSent); + + // The item the envelope did answer keeps that answer, as it does for every other shape + // of chunk failure. + Assert.Equal(204, result.Results[0].Response.Status); + Assert.Equal(1, result.Telemetry.Succeeded); + Assert.Equal(1, result.Telemetry.Failed); + + var failure = Assert.IsType(result.ChunkFailure); + Assert.Contains("without a status", failure.Message); + } + + /// Answers the first chunk in full, then an envelope that answers nobody. + private sealed class SecondChunkGarbledHandler : HttpMessageHandler + { + private int _posts; + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var body = Interlocked.Increment(ref _posts) == 1 + ? BatchResponse(1, 20, 200) + : "{ \"responses\": [ { \"id\": \"1\", \"status\": 200 } ] }"; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") + }); + } + } + + /// + /// An envelope that does not answer what was sent is a protocol violation, and it is + /// reported as one. It is not a reason to discard the chunk the server did answer. + /// + [Fact] + public async Task A_chunk_answered_with_a_garbled_envelope_keeps_the_chunk_before_it() + { + ResiliencePipelineFactory.Reset(); + using var httpClient = new HttpClient(new SecondChunkGarbledHandler()); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + + var ops = Enumerable.Range(1, 40) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + var failure = Assert.IsType(result.ChunkFailure); + Assert.Contains("count mismatch", failure.Message); + for (int i = 0; i < 20; i++) + Assert.Equal(200, result.Results[i].Response.Status); + for (int i = 20; i < 40; i++) + Assert.Equal(503, result.Results[i].Response.Status); + } + + /// + /// Answers the first chunk, throttles the second through every attempt it has, and then + /// fails the follow-up batch that carries the throttled items - the retry pass, which runs + /// once every chunk has been sent. + /// + private sealed class RefusesTheRetryPassHandler(bool garbled) : HttpMessageHandler + { + private int _posts; + + private static string Throttled(int count) => + "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, count).Select(i => + $"{{ \"id\": \"{i}\", \"status\": 429, \"headers\": {{ \"Retry-After\": \"1\" }} }}")) + "] }"; + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var post = Interlocked.Increment(ref _posts); + var body = await request.Content!.ReadAsStringAsync(cancellationToken); + if (body.Contains("/users/u1\"", StringComparison.Ordinal)) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(BatchResponse(1, 20, 204), + System.Text.Encoding.UTF8, "application/json") + }; + } + // The second chunk's own four attempts; the fifth POST of its items is the + // follow-up batch. + if (post <= 5) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(Throttled(20), + System.Text.Encoding.UTF8, "application/json") + }; + } + // Two shapes of failure: a refusal with a status, and an envelope that answers + // one of the twenty requests it was sent. + return garbled + ? new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent("{ \"responses\": [ { \"id\": \"1\", \"status\": 204 } ] }", + System.Text.Encoding.UTF8, "application/json") + } + : new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent( + "{\"error\":{\"code\":\"BadRequest\",\"message\":\"batch rejected\"}}", + System.Text.Encoding.UTF8, "application/json") + }; + } + } + + /// + /// The retry pass runs after every chunk has been sent, so by the time it fails the run + /// holds the outcome of the whole batch. Throwing hands back the failure and nothing else: + /// twenty writes the server confirmed, and twenty it throttled, are lost with the array. + /// + [Fact] + public async Task A_refused_retry_pass_keeps_what_the_run_already_knew() + { + ResiliencePipelineFactory.Reset(); + using var httpClient = new HttpClient(new RefusesTheRetryPassHandler(garbled: false)); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + + var ops = Enumerable.Range(1, 40) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.IsType(result.ChunkFailure); + Assert.Equal(40, result.Results.Count); + // The first chunk was applied. The retried items keep the last thing the server said + // about them, which is what they are: throttled, not retried, still to be re-submitted. + for (int i = 0; i < 20; i++) + Assert.Equal(204, result.Results[i].Response.Status); + for (int i = 20; i < 40; i++) + Assert.Equal(429, result.Results[i].Response.Status); + Assert.Equal(20, result.Telemetry.Succeeded); + Assert.Equal(20, result.Telemetry.Failed); + Assert.Empty(result.NotSent); + } + + /// + /// The other shape the retry pass fails in. A truncated envelope is an InvalidOperationException, + /// which the cmdlet does not catch, so the run's results left EndProcessing unreported: no + /// output objects, no error records, nothing on the pipeline but the exception. + /// + [Fact] + public void A_garbled_retry_pass_still_reports_the_run() + { + using (MgxTransportScope.Inject(new RefusesTheRetryPassHandler(garbled: true))) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", Enumerable.Range(1, 40).Select(i => $"/users/u{i}").ToArray()) + .AddParameter("Method", "PATCH"); + var output = ps.Invoke(); + + Assert.Equal(40, output.Count); + Assert.Contains(ps.Streams.Error, + e => e.FullyQualifiedErrorId.StartsWith("BatchChunkFailed", StringComparison.Ordinal)); + } + } + + /// + /// Answers the $batch POST with a status Graph never sends. A proxy or a captive portal + /// between the client and Graph does, and the chunk is refused either way. + /// + private sealed class NotModifiedHandler : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + => Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotModified) + { + RequestMessage = request, + Content = new StringContent("not modified", + System.Text.Encoding.UTF8, "text/html") + }); + } + + /// + /// A refusal below 400 is still a refusal. Recorded verbatim it put every item of the + /// chunk in the range a success is counted in, so twenty writes that never reached Graph + /// were reported as twenty that had. + /// + [Fact] + public async Task A_chunk_refused_with_a_3xx_gives_its_items_a_failing_status() + { + ResiliencePipelineFactory.Reset(); + using var httpClient = new HttpClient(new NotModifiedHandler()); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + + var ops = Enumerable.Range(1, 20) + .Select(i => new BatchOperation($"/users/u{i}", "PATCH", null)) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.NotNull(result.ChunkFailure); + Assert.All(result.Results, r => Assert.True(r.Response.Status >= 400, + $"a refused write was recorded as status {r.Response.Status}")); + Assert.Equal(0, result.Telemetry.Succeeded); + Assert.Equal(20, result.Telemetry.Failed); + Assert.Empty(result.NotSent); + } + + /// + /// The dead-letter file keeps nothing under 400, so the same 3xx left the run with no + /// record of the twenty writes to re-submit. + /// + [Fact] + public void Items_refused_with_a_3xx_are_written_to_the_dead_letter_file() + { + var deadLetter = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + try + { + using (MgxTransportScope.Inject(new NotModifiedHandler())) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", Enumerable.Range(1, 20).Select(i => $"/users/u{i}").ToArray()) + .AddParameter("Method", "PATCH") + .AddParameter("DeadLetterPath", deadLetter); + ps.Invoke(); + } + + var lines = File.Exists(deadLetter) ? File.ReadAllLines(deadLetter) : []; + Assert.Equal(20, lines.Length); + } + finally + { + if (File.Exists(deadLetter)) File.Delete(deadLetter); + } + } + /// Counts requests; answers an empty batch envelope. private sealed class CountingHandler : HttpMessageHandler { @@ -97,23 +660,8 @@ protected override Task SendAsync( [Fact] public void WhatIf_does_not_send_a_read_only_batch() { - ResiliencePipelineFactory.Reset(); var wire = new CountingHandler(); - var t = typeof(Mgx.Cmdlets.Base.MgxCmdletBase); - t.GetField("s_graphHttpClient", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)! - .SetValue(null, new HttpClient(wire)); - t.GetField("s_cachedAuthFingerprint", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)! - .SetValue(null, Mgx.Cmdlets.Base.MgxCmdletBase.BuildAuthFingerprint( - new { TenantId = "test-tenant-00000000-0000-0000-0000-000000000000" }, null)); - t.GetField("s_ownsHttpClient", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)! - .SetValue(null, false); - t.GetField("s_graphEndpoint", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.Public)! - .SetValue(null, "https://graph.microsoft.com"); - t.GetField("s_clientOptions", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.Public)! - .SetValue(null, new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); - ResiliencePipelineFactory.Reset(); - - try + using (MgxTransportScope.Inject(wire)) { using var ps = System.Management.Automation.PowerShell.Create(); ps.AddCommand("Import-Module") @@ -133,11 +681,608 @@ public void WhatIf_does_not_send_a_read_only_batch() Assert.Equal(0, wire.Requests); Assert.Empty(output); } + } + + /// + /// Throttles a whole chunk until its per-item retries are gone, so every item goes into the + /// batch-level retry pass. That pass's first attempt applies most of them and throttles the + /// rest; the follow-up carrying the rest is refused. The items the pass applied are writes + /// the server confirmed, and the only place they are recorded is the answer it gave. + /// + private sealed class RetryPassLandsThenIsRefusedHandler : HttpMessageHandler + { + private int _posts; + public int Posts => Volatile.Read(ref _posts); + + private static string Throttled(int count) => + "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, count).Select(i => + $"{{ \"id\": \"{i}\", \"status\": 429, \"headers\": {{ \"Retry-After\": \"1\" }} }}")) + "] }"; + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var post = Interlocked.Increment(ref _posts); + // Four posts exhaust the chunk's own retries, leaving all 20 items throttled and + // eligible for the batch-level pass. + if (post <= 4) + return Task.FromResult(Ok(Throttled(20))); + // The pass's own first attempt: fifteen creates applied, five still throttled. + if (post == 5) + { + var body = "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, 20).Select(i => + i <= 15 + ? $"{{ \"id\": \"{i}\", \"status\": 201, \"body\": {{ \"id\": \"u{i}\" }} }}" + : $"{{ \"id\": \"{i}\", \"status\": 429, \"headers\": {{ \"Retry-After\": \"1\" }} }}")) + + "] }"; + return Task.FromResult(Ok(body)); + } + // The follow-up carrying the five is refused outright. + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent( + "{\"error\":{\"code\":\"BadRequest\",\"message\":\"batch rejected\"}}", + System.Text.Encoding.UTF8, "application/json") + }); + + HttpResponseMessage Ok(string body) => new(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") + }; + } + } + + /// + /// The batch-level retry pass is a send like any other, and the server answers it. When a + /// later attempt of that pass is refused, the items the earlier attempt applied still have + /// to come back with the status the server gave them: reported with the throttle that sent + /// them into the pass, fifteen confirmed creates read as work still outstanding, and the + /// dead-letter file the caller replays creates them a second time. + /// + [Fact] + public async Task Items_the_retry_pass_applied_keep_that_answer_when_its_follow_up_is_refused() + { + ResiliencePipelineFactory.Reset(); + var wire = new RetryPassLandsThenIsRefusedHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var batchClient = new GraphBatchClient(client, maxRetryAfterSeconds: 1); + + var ops = Enumerable.Range(1, 20) + .Select(i => new BatchOperation("/users", "POST", + System.Text.Json.JsonSerializer.Deserialize( + $"{{\"displayName\":\"u{i}\"}}"))) + .ToList(); + + var result = await batchClient.ExecuteBatchIndexedAsync(ops, CancellationToken.None); + + Assert.Equal(6, wire.Posts); + Assert.NotNull(result.ChunkFailure); + // The fifteen the pass created. The server said 201; nothing after that unsays it. + for (int i = 0; i < 15; i++) + Assert.Equal(201, result.Results[i].Response.Status); + // The five the refused follow-up carried keep the last thing the server said about + // them - a throttle, which is a failure, and never a status below 400. + for (int i = 15; i < 20; i++) + Assert.Equal(429, result.Results[i].Response.Status); + Assert.Empty(result.NotSent); + Assert.Equal(15, result.Telemetry.Succeeded); + Assert.Equal(5, result.Telemetry.Failed); + } + + // ------------------------------------------------------------------------------------ + // What a run says about itself, transcribed per scenario. + // + // Four lines carry it: the dead-letter line, the chunk-failure target, the verbose summary + // and the warning - the last being all a caller running without -Verbose sees. They are + // read together by whoever is deciding what to re-submit, so each one being right on its + // own is not enough; they have to agree as a set. + // ------------------------------------------------------------------------------------ + + /// The four lines of one run, as the streams carried them. + private sealed record Prose( + string Summary, + string? Warning, + string? ChunkFailureTarget, + string? DeadLetterLine, + int DeadLetterLines, + IReadOnlyList Verbose); + + /// + /// Drives the cmdlet over and collects the four lines. Always with a + /// dead-letter path and -Verbose, so every scenario is asked the same question. + /// + private static Prose Say(HttpMessageHandler wire, int count, string method, object? body = null) + { + var deadLetter = Path.Combine(Path.GetTempPath(), $"mgx-dl-{Guid.NewGuid():N}.jsonl"); + try + { + using (MgxTransportScope.Inject(wire)) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + var cmd = ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", Enumerable.Range(1, count).Select(i => $"/users/u{i}").ToArray()) + .AddParameter("Method", method) + .AddParameter("DeadLetterPath", deadLetter) + .AddParameter("Verbose", true); + if (body != null) cmd.AddParameter("Body", body); + ps.Invoke(); + + var verbose = ps.Streams.Verbose.Select(v => v.Message).ToList(); + return new Prose( + Summary: Assert.Single(verbose, v => v.StartsWith("Batch: ", StringComparison.Ordinal)), + Warning: ps.Streams.Warning.Select(w => w.Message) + .FirstOrDefault(w => w.Contains("batch items failed", StringComparison.Ordinal)), + ChunkFailureTarget: ps.Streams.Error + .Where(e => e.FullyQualifiedErrorId.StartsWith("BatchChunkFailed", StringComparison.Ordinal)) + .Select(e => e.TargetObject as string) + .FirstOrDefault(), + DeadLetterLine: verbose.FirstOrDefault(v => v.StartsWith("Wrote ", StringComparison.Ordinal)), + DeadLetterLines: File.Exists(deadLetter) ? File.ReadAllLines(deadLetter).Length : 0, + Verbose: verbose); + } + } finally { - t.GetField("s_graphHttpClient", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!.SetValue(null, null); - t.GetField("s_cachedAuthFingerprint", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!.SetValue(null, null); - ResiliencePipelineFactory.Reset(); + if (File.Exists(deadLetter)) File.Delete(deadLetter); + } + } + + /// Answers every chunk in full, every item applied. + private sealed class AppliesEveryItemHandler : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(BatchResponse(1, 20, 204), + System.Text.Encoding.UTF8, "application/json") + }); + } + + /// + /// Nothing failed, so three of the four lines have nothing to say and do not say it. The + /// warning in particular: a run that worked must not put one on the stream at all. + /// + [Fact] + public void A_clean_run_says_so_and_says_nothing_else() + { + var prose = Say(new AppliesEveryItemHandler(), 20, "PATCH"); + + Assert.Contains("Batch: 20 succeeded, 0 failed out of 20 requests", prose.Summary, StringComparison.Ordinal); + Assert.DoesNotContain("not sent", prose.Summary, StringComparison.Ordinal); + Assert.DoesNotContain("Batch-level retries", prose.Summary, StringComparison.Ordinal); + Assert.Null(prose.Warning); + Assert.Null(prose.ChunkFailureTarget); + Assert.Null(prose.DeadLetterLine); + Assert.Equal(0, prose.DeadLetterLines); + } + + /// + /// The refused chunk is the last one, so nothing was left unsent and no line mentions + /// anything that was. The retry pass genuinely never went out: the run stopped before it. + /// + [Fact] + public void A_chunk_refused_with_nothing_behind_it_reports_no_unsent_work() + { + var prose = Say(new FirstChunkOnlyHandler(), 25, "PATCH"); + + Assert.Contains("Batch: 20 succeeded, 5 failed out of 25 requests", prose.Summary, StringComparison.Ordinal); + Assert.DoesNotContain("not sent", prose.Summary, StringComparison.Ordinal); + Assert.DoesNotContain("Batch-level retries", prose.Summary, StringComparison.Ordinal); + Assert.Equal( + "5 of 25 batch items failed." + + " A chunk failed, so the run stopped sending and the retry pass was withheld." + + " Check $Error for details on each item.", + prose.Warning); + Assert.Equal("5 of 25 operations failed, 0 were not sent", prose.ChunkFailureTarget); + Assert.StartsWith("Wrote 5 failed items to dead-letter file: ", prose.DeadLetterLine, StringComparison.Ordinal); + Assert.Equal(5, prose.DeadLetterLines); + } + + /// + /// A chunk behind the refused one never went out, and all four lines have to keep it apart + /// from the writes that were refused: one of those may have been applied on the server and + /// the other certainly was not. + /// + [Fact] + public void Operations_never_sent_are_named_by_every_line_that_counts_them() + { + var prose = Say(new FirstChunkOnlyHandler(), 60, "PATCH"); + + Assert.Contains("Batch: 20 succeeded, 20 failed, 20 not sent out of 60 requests", prose.Summary, StringComparison.Ordinal); + Assert.Equal( + "20 of 60 batch items failed and 20 were not sent." + + " A chunk failed, so the run stopped sending and the retry pass was withheld." + + " Check $Error for details on each item.", + prose.Warning); + Assert.Equal("20 of 60 operations failed, 20 were not sent", prose.ChunkFailureTarget); + Assert.StartsWith("Wrote 20 failed and 20 not-sent items to dead-letter file: ", + prose.DeadLetterLine, StringComparison.Ordinal); + Assert.Equal(40, prose.DeadLetterLines); + } + + /// + /// Throttles the chunk through every attempt it has, then answers the batch-level pass in + /// full: fifteen applied, five not found. Nothing stopped the run. + /// + private sealed class ThrottlesThenAnswersThePassHandler : HttpMessageHandler + { + private int _posts; + public int Posts => Volatile.Read(ref _posts); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var post = Interlocked.Increment(ref _posts); + var body = post <= 4 + ? "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, 20).Select(i => + $"{{ \"id\": \"{i}\", \"status\": 429, \"headers\": {{ \"Retry-After\": \"1\" }} }}")) + "] }" + : "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, 20).Select(i => + $"{{ \"id\": \"{i}\", \"status\": {(i <= 15 ? 204 : 404)} }}")) + "] }"; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") + }); + } + } + + /// + /// The pass went out and the server answered it. Nothing stopped the run, so the five that + /// failed had every attempt they qualified for and the warning says so. + /// + [Fact] + public void A_retry_pass_that_ran_to_the_end_leaves_the_run_unstopped() + { + var wire = new ThrottlesThenAnswersThePassHandler(); + var prose = Say(wire, 20, "PATCH"); + + Assert.Equal(5, wire.Posts); + Assert.Contains("Batch: 15 succeeded, 5 failed out of 20 requests", prose.Summary, StringComparison.Ordinal); + Assert.Contains(" Batch-level retries: 20.", prose.Summary, StringComparison.Ordinal); + Assert.Equal( + "5 of 20 batch items failed." + + " They failed after all retry attempts." + + " Check $Error for details on each item.", + prose.Warning); + Assert.Null(prose.ChunkFailureTarget); + Assert.StartsWith("Wrote 5 failed items to dead-letter file: ", prose.DeadLetterLine, StringComparison.Ordinal); + Assert.Equal(5, prose.DeadLetterLines); + } + + /// + /// The pass carried twenty items, applied fifteen of them, and was refused on the follow-up + /// carrying the other five. Twenty went out on the wire, which is what the count reports. + /// + [Fact] + public void A_retry_pass_refused_after_it_applied_work_still_counts_what_it_sent() + { + var wire = new RetryPassLandsThenIsRefusedHandler(); + var prose = Say(wire, 20, "POST", new System.Collections.Hashtable { ["displayName"] = "u" }); + + Assert.Equal(6, wire.Posts); + Assert.Contains("Batch: 15 succeeded, 5 failed out of 20 requests", prose.Summary, StringComparison.Ordinal); + Assert.Contains(" Batch-level retries: 20.", prose.Summary, StringComparison.Ordinal); + Assert.Equal( + "5 of 20 batch items failed." + + " The retry pass was sent and then refused, so the run stopped sending." + + " Check $Error for details on each item.", + prose.Warning); + Assert.Equal("5 of 20 operations failed, 0 were not sent", prose.ChunkFailureTarget); + Assert.StartsWith("Wrote 5 failed items to dead-letter file: ", prose.DeadLetterLine, StringComparison.Ordinal); + Assert.Equal(5, prose.DeadLetterLines); + } + + /// + /// Throttles both chunks through every attempt they have, so all forty items become + /// candidates for the batch-level pass - two chunks of it - and refuses the first chunk + /// that pass sends. The loop breaks there, so the second chunk of the pass never goes out. + /// + private sealed class RefusesTheFirstChunkOfThePassHandler : HttpMessageHandler + { + private int _posts; + public int Posts => Volatile.Read(ref _posts); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + // Four attempts each for two chunks exhausts the per-chunk retries of both. + if (Interlocked.Increment(ref _posts) <= 8) + { + var body = "{ \"responses\": [" + string.Join(",", Enumerable.Range(1, 20).Select(i => + $"{{ \"id\": \"{i}\", \"status\": 429, \"headers\": {{ \"Retry-After\": \"1\" }} }}")) + "] }"; + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json") + }); + } + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.BadRequest) + { + RequestMessage = request, + Content = new StringContent( + "{\"error\":{\"code\":\"BadRequest\",\"message\":\"batch rejected\"}}", + System.Text.Encoding.UTF8, "application/json") + }); + } + } + + /// + /// Forty items were candidates for the pass and twenty of them were POSTed: the refusal of + /// the first chunk ends the pass, and the twenty behind it are never re-sent. Candidacy is + /// its own line and says forty; what went out is the count, and it is not the same number. + /// + [Fact] + public void A_retry_pass_refused_on_its_first_chunk_never_sends_the_second() + { + var wire = new RefusesTheFirstChunkOfThePassHandler(); + var prose = Say(wire, 40, "PATCH"); + + // Eight per-chunk attempts and one POST of the pass. The pass's second chunk is the + // POST that would make this nine, and it never happened. + Assert.Equal(9, wire.Posts); + Assert.Contains("Batch: 0 succeeded, 40 failed out of 40 requests", prose.Summary, StringComparison.Ordinal); + // Twenty, not forty: the second chunk of the pass is the POST that never happened. + Assert.Contains(" Batch-level retries: 20.", prose.Summary, StringComparison.Ordinal); + Assert.Equal( + "40 of 40 batch items failed." + + " The retry pass was sent and then refused, so the run stopped sending." + + " Check $Error for details on each item.", + prose.Warning); + Assert.Equal("40 of 40 operations failed, 0 were not sent", prose.ChunkFailureTarget); + Assert.StartsWith("Wrote 40 failed items to dead-letter file: ", prose.DeadLetterLine, StringComparison.Ordinal); + Assert.Equal(40, prose.DeadLetterLines); + // Candidacy is stated separately and is not in question: forty items did exhaust their + // per-chunk retries, whatever the pass then managed to send. + Assert.Contains(prose.Verbose, v => + v.Contains("Batch-level retry: 40 items exhausted per-chunk retries", StringComparison.Ordinal)); + } + + /// The lines -WhatIf writes go to the host, so a test that reads them needs one. + private sealed class GateHost : PSHost + { + private readonly Guid _id = Guid.NewGuid(); + public GateHostUI Recorder { get; } = new(); + public override string Name => "MgxBatchGateHost"; + public override Version Version => new(1, 0); + public override Guid InstanceId => _id; + public override PSHostUserInterface UI => Recorder; + public override System.Globalization.CultureInfo CurrentCulture => System.Globalization.CultureInfo.InvariantCulture; + public override System.Globalization.CultureInfo CurrentUICulture => System.Globalization.CultureInfo.InvariantCulture; + public override void EnterNestedPrompt() { } + public override void ExitNestedPrompt() { } + public override void NotifyBeginApplication() { } + public override void NotifyEndApplication() { } + public override void SetShouldExit(int exitCode) { } + } + + private sealed class GateHostUI : PSHostUserInterface + { + public List Lines { get; } = []; + public override PSHostRawUserInterface? RawUI => null; + public override void Write(string value) => Lines.Add(value); + public override void Write(ConsoleColor f, ConsoleColor b, string value) => Lines.Add(value); + public override void WriteLine(string value) => Lines.Add(value); + public override void WriteErrorLine(string value) => Lines.Add(value); + public override void WriteDebugLine(string value) => Lines.Add(value); + public override void WriteVerboseLine(string value) => Lines.Add(value); + public override void WriteWarningLine(string value) => Lines.Add(value); + public override void WriteProgress(long sourceId, ProgressRecord record) { } + public override string ReadLine() => string.Empty; + public override System.Security.SecureString ReadLineAsSecureString() => new(); + public override Dictionary Prompt( + string caption, string message, System.Collections.ObjectModel.Collection descriptions) => []; + public override int PromptForChoice( + string caption, string message, + System.Collections.ObjectModel.Collection choices, int defaultChoice) => defaultChoice; + public override PSCredential PromptForCredential( + string caption, string message, string userName, string targetName) => PSCredential.Empty; + public override PSCredential PromptForCredential( + string caption, string message, string userName, string targetName, + PSCredentialTypes allowedCredentialTypes, PSCredentialUIOptions options) => PSCredential.Empty; + } + + /// + /// Runs the batch under -WhatIf over per-item methods and returns the line the gate wrote. + /// The wire is armed and counted: a gate that lets the batch through is a different failure + /// from one that describes it wrongly, and the two have to be told apart. + /// + private static (string Line, int Requests) GateSays(params string[] methods) + { + var wire = new CountingHandler(); + using (MgxTransportScope.Inject(wire)) + { + var host = new GateHost(); + using var runspace = System.Management.Automation.Runspaces.RunspaceFactory.CreateRunspace(host); + runspace.Open(); + using var ps = System.Management.Automation.PowerShell.Create(); + ps.Runspace = runspace; + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", methods + .Select((m, i) => (object)new System.Collections.Hashtable + { + ["Url"] = $"/users/u{i + 1}", + ["Method"] = m + }) + .ToArray()) + .AddParameter("WhatIf", true); + ps.Invoke(); + + return (Assert.Single(host.Recorder.Lines, l => l.Contains("What if:", StringComparison.Ordinal)), + wire.Requests); + } + } + + /// + /// -WhatIf is the one surface whose whole purpose is stating what is about to happen, and + /// the target it states is the batch's own account of itself. The total is deliberately the + /// whole batch, reads included - they spend resource units and emit output - but a write + /// verb across that total names requests the batch does not hold: one DELETE among nineteen + /// reads announced twenty deletions, and five creates among fifteen reads announced twenty + /// creates, byte-identical to a batch of twenty that really was all creates. + /// + [Theory] + // A single write among reads: the verb is the alarming part, and it belonged to one item. + [InlineData(1, "DELETE", 19, "Send batch\" on target \"20 requests (1 DELETE) via $batch\"")] + [InlineData(5, "POST", 15, "Send batch\" on target \"20 requests (5 POST) via $batch\"")] + // No reads to distinguish: the whole batch is that one method, and the verb is the total. + [InlineData(20, "POST", 0, "Send batch\" on target \"POST 20 requests via $batch\"")] + [InlineData(0, "POST", 20, "Send batch\" on target \"GET 20 requests via $batch\"")] + public void The_gate_names_the_writes_a_batch_holds_not_the_batch_as_writes( + int writes, string writeMethod, int reads, string expected) + { + var methods = Enumerable.Repeat(writeMethod, writes) + .Concat(Enumerable.Repeat("GET", reads)) + .ToArray(); + + var (line, requests) = GateSays(methods); + + Assert.Contains(expected, line, StringComparison.Ordinal); + Assert.Equal(0, requests); + } + + /// + /// Runs the batch under -WhatIf over per-item bodies and returns the gate's line, the error + /// records it wrote and what reached the wire. -WhatIf's whole purpose is saying what would + /// happen, so a gate that describes requests the run would refuse is the defect, and the + /// records naming them are half of the answer. + /// + private static (string Line, string[] Errors, int Requests) GateSaysOfBodies( + params object[] items) + { + var wire = new CountingHandler(); + using (MgxTransportScope.Inject(wire)) + { + var host = new GateHost(); + using var runspace = System.Management.Automation.Runspaces.RunspaceFactory.CreateRunspace(host); + runspace.Open(); + using var ps = System.Management.Automation.PowerShell.Create(); + ps.Runspace = runspace; + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Batch.InvokeMgxBatchRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + + ps.AddCommand("Invoke-MgxBatchRequest") + .AddParameter("Uri", items) + .AddParameter("WhatIf", true); + ps.Invoke(); + + return ( + Assert.Single(host.Recorder.Lines, l => l.Contains("What if:", StringComparison.Ordinal)), + ps.Streams.Error.Select(e => e.Exception.Message).ToArray(), + wire.Requests); } } + + /// + /// The gate over a batch two of whose items will not be sent: one body the marker guard + /// refuses, one the JSON parser refuses. The count used to be everything that was piped + /// in, so the line announced three requests where the run would make one - at the one + /// surface whose purpose is saying what is about to happen - and said nothing at all about + /// the two it would refuse. The validation and the guard run before the gate now, so the + /// count is what would be sent and the refusals are named beside it. + /// + [Fact] + public void The_gate_names_the_count_that_will_be_sent_and_the_refusals() + { + var (line, errors, requests) = GateSaysOfBodies( + new System.Collections.Hashtable + { + ["Url"] = "/users/u1", ["Method"] = "PATCH", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "sent" } + }, + new System.Collections.Hashtable + { + ["Url"] = "/users/u2", ["Method"] = "PATCH", + ["Body"] = new System.Collections.Hashtable { ["passwordProfile"] = "***REDACTED***" } + }, + new System.Collections.Hashtable + { + ["Url"] = "/users/u3", ["Method"] = "PATCH", ["Body"] = "not json at all" + }); + + // One request, singular, and the two refusals as their own count. + Assert.Contains("Send batch\" on target \"PATCH 1 request via $batch; 2 refused\"", + line, StringComparison.Ordinal); + + // The refusals are error records, written under -WhatIf as without it: a body this run + // will not send is not an action the gate governs. + Assert.Equal(2, errors.Length); + Assert.Contains(errors, e => e.Contains("PATCH /users/u2", StringComparison.Ordinal) + && e.Contains("redaction marker", StringComparison.Ordinal)); + Assert.Contains(errors, e => e.Contains("PATCH /users/u3", StringComparison.Ordinal) + && e.Contains("not valid JSON", StringComparison.Ordinal)); + + Assert.Equal(0, requests); + } + + /// + /// Nothing refused, so nothing is named: the line a batch with no refusals gets is the one + /// it always got, and the clause is not appended empty. + /// + [Fact] + public void The_gate_names_no_refusals_when_there_are_none() + { + var (line, errors, requests) = GateSaysOfBodies( + new System.Collections.Hashtable + { + ["Url"] = "/users/u1", ["Method"] = "PATCH", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "a" } + }, + new System.Collections.Hashtable + { + ["Url"] = "/users/u2", ["Method"] = "PATCH", + ["Body"] = new System.Collections.Hashtable { ["displayName"] = "b" } + }); + + Assert.Contains("Send batch\" on target \"PATCH 2 requests via $batch\"", + line, StringComparison.Ordinal); + Assert.DoesNotContain("refused", line, StringComparison.Ordinal); + Assert.Empty(errors); + Assert.Equal(0, requests); + } + + /// + /// Several write methods alongside reads keep the shape they already had: the total, then + /// the writes by method. It states a total and a subset of it, and nothing in it is a + /// count of requests that are not there. + /// + [Fact] + public void The_gate_keeps_naming_each_write_method_of_a_mixed_batch() + { + var methods = Enumerable.Repeat("POST", 5) + .Concat(Enumerable.Repeat("PATCH", 3)) + .Concat(Enumerable.Repeat("GET", 12)) + .ToArray(); + + var (line, requests) = GateSays(methods); + + Assert.Contains("Send batch\" on target \"20 requests (5 POST, 3 PATCH) via $batch\"", + line, StringComparison.Ordinal); + Assert.Equal(0, requests); + } } diff --git a/tests/Mgx.IntegrationTests/Engine/ErrorBodyShapeTests.cs b/tests/Mgx.IntegrationTests/Engine/ErrorBodyShapeTests.cs index e9214a1..0b9c6bc 100644 --- a/tests/Mgx.IntegrationTests/Engine/ErrorBodyShapeTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/ErrorBodyShapeTests.cs @@ -70,4 +70,59 @@ public void Falls_back_to_the_status_line_when_the_shape_is_wrong() var ex = new GraphServiceException(HttpStatusCode.ServiceUnavailable, """{"error":404}"""); Assert.Contains("503", ex.Message); } + + [Fact] + public void Carries_the_inner_error_message() + { + var ex = new GraphServiceException( + HttpStatusCode.BadRequest, + """{"error":{"code":"","message":"The request is invalid.","innerError":{"message":"configuration.DisplayName : The DisplayName field is required.","date":"2026-09-21T19:08:59"}}}"""); + + Assert.Equal("configuration.DisplayName : The DisplayName field is required.", ex.InnerErrorMessage); + Assert.Contains("The request is invalid.", ex.Message); + Assert.Contains("configuration.DisplayName", ex.Message); + } + + [Fact] + public void Takes_the_deepest_message_of_a_nested_inner_error() + { + var ex = new GraphServiceException( + HttpStatusCode.BadRequest, + """{"error":{"code":"BadRequest","message":"top","innerError":{"message":"middle","innerError":{"message":"deepest"}}}}"""); + + Assert.Equal("deepest", ex.InnerErrorMessage); + Assert.Contains("deepest", ex.Message); + } + + [Fact] + public void Leaves_the_message_alone_when_the_inner_error_carries_no_message() + { + var ex = new GraphServiceException( + HttpStatusCode.BadRequest, + """{"error":{"code":"Request_BadRequest","message":"Invalid request","innerError":{"date":"2024-01-01T00:00:00","request-id":"12345"}}}"""); + + Assert.Null(ex.InnerErrorMessage); + Assert.StartsWith("Request_BadRequest: Invalid request", ex.Message); + } + + [Fact] + public void Does_not_repeat_an_inner_error_that_matches_the_message() + { + var ex = new GraphServiceException( + HttpStatusCode.BadRequest, + """{"error":{"code":"BadRequest","message":"same text","innerError":{"message":"same text"}}}"""); + + Assert.Equal("same text", ex.InnerErrorMessage); + Assert.Equal(1, ex.Message.Split("same text").Length - 1); + } + + [Fact] + public void Stops_walking_a_chain_nested_past_any_real_depth() + { + var ex = new GraphServiceException( + HttpStatusCode.BadRequest, + """{"error":{"code":"BadRequest","message":"top","innerError":{"message":"a","innerError":{"message":"b","innerError":{"message":"c","innerError":{"message":"d","innerError":{"message":"e","innerError":{"message":"f","innerError":{"message":"g","innerError":{"message":"h","innerError":{"message":"i"}}}}}}}}}}}"""); + + Assert.False(string.IsNullOrWhiteSpace(ex.Message)); + } } diff --git a/tests/Mgx.IntegrationTests/Engine/Errors/ErrorClassifierTests.cs b/tests/Mgx.IntegrationTests/Engine/Errors/ErrorClassifierTests.cs new file mode 100644 index 0000000..3206294 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Errors/ErrorClassifierTests.cs @@ -0,0 +1,52 @@ +using Mgx.Engine.Errors; + +namespace Mgx.IntegrationTests; + +public class ErrorClassifierTests +{ + [Theory] + [InlineData(429, MgxErrorClass.Throttle)] + [InlineData(500, MgxErrorClass.TransientServer)] + [InlineData(502, MgxErrorClass.TransientServer)] + [InlineData(503, MgxErrorClass.TransientServer)] + [InlineData(504, MgxErrorClass.TransientServer)] + [InlineData(408, MgxErrorClass.TransientTransport)] + [InlineData(401, MgxErrorClass.Authentication)] + [InlineData(403, MgxErrorClass.Authorization)] + [InlineData(404, MgxErrorClass.NotFound)] + [InlineData(409, MgxErrorClass.Conflict)] + [InlineData(412, MgxErrorClass.Conflict)] + [InlineData(400, MgxErrorClass.InvalidRequest)] + [InlineData(405, MgxErrorClass.InvalidRequest)] + [InlineData(413, MgxErrorClass.InvalidRequest)] + [InlineData(415, MgxErrorClass.InvalidRequest)] + [InlineData(422, MgxErrorClass.InvalidRequest)] + [InlineData(501, MgxErrorClass.InvalidRequest)] + [InlineData(410, MgxErrorClass.Permanent)] // delta resync belongs to a later phase + [InlineData(505, MgxErrorClass.Permanent)] // unknown 5xx: never retried, stays that way + [InlineData(507, MgxErrorClass.Permanent)] + [InlineData(418, MgxErrorClass.Permanent)] + public void Status_maps_to_its_class(int status, MgxErrorClass expected) + => Assert.Equal(expected, MgxErrorClassifier.Classify(status).Class); + + [Fact] + public void A_response_carries_status_and_unclamped_retry_after() + { + using var response = new HttpResponseMessage((System.Net.HttpStatusCode)429); + response.Headers.RetryAfter = new System.Net.Http.Headers.RetryConditionHeaderValue( + TimeSpan.FromSeconds(999)); + var info = MgxErrorClassifier.Classify(response); + Assert.Equal(MgxErrorClass.Throttle, info.Class); + Assert.Equal(429, info.StatusCode); + Assert.Equal(TimeSpan.FromSeconds(999), info.ServerRetryAfter); // facts unclamped; policy clamps + } + + [Fact] + public void Cancellation_is_permanent_regardless_of_exception_shape() + { + Assert.Equal(MgxErrorClass.Permanent, + MgxErrorClassifier.Classify(new TaskCanceledException(), cancellationRequested: true).Class); + Assert.Equal(MgxErrorClass.Permanent, + MgxErrorClassifier.Classify(new Polly.Timeout.TimeoutRejectedException(), cancellationRequested: true).Class); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Errors/ErrorPolicyParityTests.cs b/tests/Mgx.IntegrationTests/Engine/Errors/ErrorPolicyParityTests.cs new file mode 100644 index 0000000..34d83bf --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Errors/ErrorPolicyParityTests.cs @@ -0,0 +1,174 @@ +using System.Net.Sockets; +using Mgx.Engine.Errors; +using Polly.CircuitBreaker; +using Polly.Timeout; + +namespace Mgx.IntegrationTests; + +/// +/// Pins classifier + policy to the decisions the retry predicate, the circuit-breaker +/// predicate, the batch item loop, and the download pipeline each made on their own +/// before consolidation, across every status. The oracles below are transcriptions of +/// v2.1.2's inline code. Scope, stated honestly: the status tables are held exactly; +/// the batch theory drives the real IsRetryable; the pipeline predicates themselves are +/// lambdas exercised by the behavior suites (RetryTests, CircuitBreakerTests). Three +/// deliberate widenings in the exception dimension are pinned as INTENDED below: +/// SocketException, IOException (reachable as HttpIOException), and a bare +/// OperationCanceledException now classify as transport failures and retry/count where +/// v2.1.2 fell through to no-retry. +/// +public class ErrorPolicyParityTests +{ + // ResiliencePipelineFactory.cs retry predicate, as it stood. + private static bool OracleRetry(int status, bool isIdempotent) + { + if (status == 429) return true; + if (!isIdempotent) return false; + return status is 500 or 502 or 503 or 504 or 408; + } + + // ResiliencePipelineFactory.cs circuit-breaker predicate, as it stood. + private static bool OracleCircuit(int status) => status is 500 or 502 or 503 or 504; + + // GraphBatchClient.IsRetryable, as it stood. + private static bool OracleBatch(int status, string method) + { + if (status == 429) return true; + if (string.Equals(method, "POST", StringComparison.OrdinalIgnoreCase)) return false; + return status is 408 or 500 or 502 or 503 or 504; + } + + // GraphContentClient download pipeline ShouldHandle, as it stood. + private static bool OracleDownload(int status) => status is 429 or 500 or 502 or 503 or 504; + + public static TheoryData AllStatuses() + { + var d = new TheoryData(); + for (var s = 100; s < 600; s++) d.Add(s); + return d; + } + + [Theory] + [MemberData(nameof(AllStatuses))] + public void Retry_decision_matches_the_old_predicate_for_every_status(int status) + { + var cls = MgxErrorClassifier.Classify(status).Class; + Assert.Equal(OracleRetry(status, isIdempotent: true), MgxErrorPolicy.ShouldRetry(cls, isIdempotent: true)); + Assert.Equal(OracleRetry(status, isIdempotent: false), MgxErrorPolicy.ShouldRetry(cls, isIdempotent: false)); + } + + [Theory] + [MemberData(nameof(AllStatuses))] + public void Circuit_decision_matches_the_old_predicate_for_every_status(int status) + { + var info = MgxErrorClassifier.Classify(status); + Assert.Equal(OracleCircuit(status), MgxErrorPolicy.CountsAsCircuitFailure(info)); + } + + private static readonly System.Reflection.MethodInfo s_realBatchIsRetryable = + typeof(Mgx.Engine.Http.GraphBatchClient).GetMethod("IsRetryable", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static)!; + + [Theory] + [MemberData(nameof(AllStatuses))] + public void Batch_decision_matches_the_old_predicate_for_every_status(int status) + { + // The REAL private predicate, not a re-implementation of its glue - so a drift in + // GraphBatchClient.IsRetryable itself (say, its case-insensitive POST check) fails + // here rather than staying green behind a faithful copy. + foreach (var method in new[] { "GET", "POST", "PATCH", "PUT", "DELETE", "post" }) + { + var actual = (bool)s_realBatchIsRetryable.Invoke(null, [status, method])!; + Assert.Equal(OracleBatch(status, method), actual); + } + } + + [Theory] + [MemberData(nameof(AllStatuses))] + public void Download_decision_matches_the_old_predicate_for_every_status(int status) + { + var info = MgxErrorClassifier.Classify(status); + Assert.Equal(OracleDownload(status), MgxErrorPolicy.ShouldRetryDownload(info, exception: null)); + } + + [Fact] + public void Exception_decisions_match_the_old_predicates() + { + // Retry predicate: HttpRequestException retried (idempotent only, via the gate); + // TaskCanceled/TimeoutRejected retried unless the caller cancelled; everything else not. + static bool Retry(Exception ex, bool cancelled, bool idempotent) + => MgxErrorPolicy.ShouldRetry(MgxErrorClassifier.Classify(ex, cancelled).Class, idempotent); + + Assert.True(Retry(new HttpRequestException(), cancelled: false, idempotent: true)); + Assert.False(Retry(new HttpRequestException(), cancelled: false, idempotent: false)); + Assert.True(Retry(new TaskCanceledException(), cancelled: false, idempotent: true)); + Assert.False(Retry(new TaskCanceledException(), cancelled: true, idempotent: true)); + Assert.True(Retry(new TimeoutRejectedException(), cancelled: false, idempotent: true)); + Assert.False(Retry(new TimeoutRejectedException(), cancelled: true, idempotent: true)); + Assert.False(Retry(new BrokenCircuitException(), cancelled: false, idempotent: true)); + Assert.False(Retry(new InvalidOperationException(), cancelled: false, idempotent: true)); + + // Circuit predicate: transport failures count, except after cancellation. + static bool Circuit(Exception ex, bool cancelled) + => MgxErrorPolicy.CountsAsCircuitFailure(MgxErrorClassifier.Classify(ex, cancelled)); + + Assert.True(Circuit(new HttpRequestException(), cancelled: false)); + Assert.True(Circuit(new TaskCanceledException(), cancelled: false)); + Assert.False(Circuit(new TaskCanceledException(), cancelled: true)); + Assert.True(Circuit(new TimeoutRejectedException(), cancelled: false)); + Assert.False(Circuit(new BrokenCircuitException(), cancelled: false)); + + // Download pipeline: HttpRequestException only. + static bool Download(Exception ex) + => MgxErrorPolicy.ShouldRetryDownload(MgxErrorClassifier.Classify(ex, false), ex); + + Assert.True(Download(new HttpRequestException())); + Assert.False(Download(new TaskCanceledException())); + Assert.False(Download(new TimeoutRejectedException())); + Assert.False(Download(new SocketException())); + } + + [Fact] + public void The_intended_widenings_are_pinned_as_intended() + { + // v2.1.2's inline predicates fell through to no-retry for these; classification + // makes them transport failures, which retries (idempotent) and counts toward the + // breaker. Deliberate - a "response ended prematurely" HttpIOException is exactly + // as transient as the HttpRequestException that usually wraps it. If this test + // surprises you, the change log entry to read is 2.1.3's classifier note. + static (bool Retry, bool Circuit) Decide(Exception ex) + { + var info = MgxErrorClassifier.Classify(ex, cancellationRequested: false); + return (MgxErrorPolicy.ShouldRetry(info.Class, isIdempotent: true), + MgxErrorPolicy.CountsAsCircuitFailure(info)); + } + + Assert.Equal((true, true), Decide(new SocketException())); + Assert.Equal((true, true), Decide(new IOException())); + Assert.Equal((true, true), Decide(new OperationCanceledException())); + + // And the narrowing: a per-attempt timeout after the caller cancelled no longer + // retries (2.1.2's fix, preserved through classification). + var cancelled = MgxErrorClassifier.Classify(new TimeoutRejectedException(), cancellationRequested: true); + Assert.False(MgxErrorPolicy.ShouldRetry(cancelled.Class, isIdempotent: true)); + Assert.False(MgxErrorPolicy.CountsAsCircuitFailure(cancelled)); + } + + [Fact] + public void The_breaker_reads_cancellation_from_the_exceptions_own_token() + { + // The CB call site derives cancellationRequested from the exception's token - + // matching v2.1.2's Handle(e => !e.CancellationToken...) - + // while the retry predicate reads the context token. Pin the distinction: a TCE + // carrying a cancelled token must not count, one with a default token must. + using var cts = new CancellationTokenSource(); + cts.Cancel(); + var carried = new TaskCanceledException(null, null, cts.Token); + Assert.False(MgxErrorPolicy.CountsAsCircuitFailure( + MgxErrorClassifier.Classify(carried, carried.CancellationToken.IsCancellationRequested))); + + var bare = new TaskCanceledException(); + Assert.True(MgxErrorPolicy.CountsAsCircuitFailure( + MgxErrorClassifier.Classify(bare, bare.CancellationToken.IsCancellationRequested))); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Errors/FaultInjectionCatalog.cs b/tests/Mgx.IntegrationTests/Engine/Errors/FaultInjectionCatalog.cs new file mode 100644 index 0000000..5a3eb03 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Errors/FaultInjectionCatalog.cs @@ -0,0 +1,482 @@ +using System.Net.Sockets; +using System.Text.Json; +using Mgx.Engine.Errors; +using Mgx.Engine.Http; +using Polly.Timeout; + +namespace Mgx.IntegrationTests; + +/// +/// The fault classes an operation has to survive, named once. Each one says how the mock +/// produces it and what makes of it; what the operation is +/// then expected to do comes from at the point of injection, +/// never from a status list restated in a test. +/// +public enum FaultKind +{ + Unauthorized, + Forbidden, + NotFound, + Conflict, + Throttled, + ServerError, + ConnectionReset, + AttemptTimeout, + BodyTimeout, + MalformedJson, + RepeatedNextLink, + UnusableNextLink, + ExpiredToken, + ChangedAuthContext, + ConsistencyDelay, +} + +/// How the mock produces a fault, which is also what decides what the classifier sees. +public enum FaultShape +{ + /// An HTTP status, on the request or inside a $batch envelope. + Status, + + /// An exception thrown instead of an answer. + Transport, + + /// An answer held past the per-attempt timeout. + HeldPastAttemptTimeout, + + /// Headers, then a body that delivers a prefix and stops. + StalledBody, + + /// A 200 whose body stops mid-document. + TruncatedJson, + + /// A page whose @odata.nextLink is its own URL. + SelfNextLink, + + /// A page whose @odata.nextLink is not a link mgx will follow. + UnusableNextLink, + + /// The session's AuthContext object replaced while the operation runs. + SwappedAuthContext, +} + +/// The four points a fault is injected at. +public enum InjectionPoint +{ + /// Page N of a pagination, after earlier pages were delivered. + Page, + + /// Item N of a fan-out, after earlier items succeeded. + FanOutItem, + + /// Subrequest N of a $batch, after earlier subrequests were answered. + BatchSubrequest, + + /// Mid-body of a content download, after earlier bytes arrived. + DownloadBody, +} + +public enum FaultCoverage +{ + /// The case runs and asserts the outcome the policy's decision implies. + Covered, + + /// The case runs and records today's behavior, which the policy does not decide. + Pinned, + + /// The fault cannot land at this point. says why. + NotApplicable, +} + +/// What one fault does at one injection point, and the shape it takes there. +public sealed record FaultCell(FaultCoverage Coverage, string Note); + +/// One row of the catalog: a fault, how it is produced, and where it can land. +public sealed record FaultEntry +{ + public required FaultKind Kind { get; init; } + + public required FaultShape Shape { get; init; } + + /// The status the fault carries, or 0 when it carries none. + public int Status { get; init; } + + /// The exception the fault is thrown as, for . + public Func? Transport { get; init; } + + /// + /// The class assigns, or null when the classifier never + /// sees this fault at all - a self-referencing nextLink and a swapped auth context are not + /// failures of a request. holds each non-null value + /// to what the classifier actually answers, so this column cannot drift away from it. + /// + public MgxErrorClass? Class { get; init; } + + /// How the mock produces it. + public required string Production { get; init; } + + public required IReadOnlyDictionary Cells { get; init; } + + public FaultCell At(InjectionPoint point) => Cells[point]; +} + +/// +/// The one fault catalog. The four injection-point theories read it; each of them iterates +/// every kind, so a cell that cannot be reached shows up as an explicit "not applicable" row +/// with its reason rather than as a case nobody wrote. +/// +public static class FaultCatalog +{ + private static FaultCell Covered(string note) => new(FaultCoverage.Covered, note); + + private static FaultCell Pinned(string note) => new(FaultCoverage.Pinned, note); + + private static FaultCell NotApplicable(string why) => new(FaultCoverage.NotApplicable, why); + + private const string StatusNotInABody = + "a status arrives on the response line, before the first byte of the body; once bytes " + + "are flowing there is no status left to change. Hop 2's own 401/403 - the expired " + + "pre-authenticated URL - is a request-start fault, held in GraphContentClientTests."; + + private const string NoLinksInABatch = + "the batch client never reads @odata.nextLink out of a sub-response; nothing in a $batch " + + "follows a link, so there is no link to poison."; + + private const string NoLinksInADownload = "a content download has no pages and follows no link."; + + private static IReadOnlyDictionary Cells( + FaultCell page, FaultCell fanOut, FaultCell batch, FaultCell download) => + new Dictionary + { + [InjectionPoint.Page] = page, + [InjectionPoint.FanOutItem] = fanOut, + [InjectionPoint.BatchSubrequest] = batch, + [InjectionPoint.DownloadBody] = download, + }; + + /// A plain status fault: the same shape at the first three points, impossible mid-body. + private static FaultEntry StatusEntry(FaultKind kind, int status, MgxErrorClass cls) => new() + { + Kind = kind, + Shape = FaultShape.Status, + Status = status, + Class = cls, + Production = $"the request is answered {status}", + Cells = Cells( + Covered($"page 3 is answered {status} after pages 1 and 2 were delivered"), + Covered($"the third url is answered {status} after the first two succeeded"), + Covered($"sub-response 23 carries status {status} inside the chunk's 200 envelope"), + NotApplicable(StatusNotInABody)), + }; + + public static IReadOnlyList Entries { get; } = + [ + StatusEntry(FaultKind.Unauthorized, 401, MgxErrorClass.Authentication), + StatusEntry(FaultKind.Forbidden, 403, MgxErrorClass.Authorization), + StatusEntry(FaultKind.NotFound, 404, MgxErrorClass.NotFound), + StatusEntry(FaultKind.Conflict, 409, MgxErrorClass.Conflict), + StatusEntry(FaultKind.Throttled, 429, MgxErrorClass.Throttle), + StatusEntry(FaultKind.ServerError, 503, MgxErrorClass.TransientServer), + + new FaultEntry + { + Kind = FaultKind.ConnectionReset, + Shape = FaultShape.Transport, + Transport = () => new HttpRequestException( + "The connection was reset.", new SocketException((int)SocketError.ConnectionReset)), + Class = MgxErrorClass.TransientTransport, + Production = "the send throws HttpRequestException over a reset SocketException", + Cells = Cells( + Covered("the page-3 GET is thrown on instead of answered"), + Covered("the third url's GET is thrown on instead of answered"), + Covered("the chunk carrying subrequest 23 has its own POST thrown on - a " + + "sub-response cannot carry a transport failure, only a status"), + Covered("the body throws IOException after delivering its first bytes; a reset " + + "mid-stream reaches the reader as an IOException, not as the " + + "HttpRequestException a failed send raises, and both classify the same")), + }, + + new FaultEntry + { + Kind = FaultKind.AttemptTimeout, + Shape = FaultShape.HeldPastAttemptTimeout, + Class = MgxErrorClass.TransientTransport, + Production = "the answer is held past AttemptTimeoutSeconds, so the pipeline's " + + "per-attempt timeout raises TimeoutRejectedException", + Cells = Cells( + Covered("page 3's answer is held past the attempt timeout"), + Covered("the third url's answer is held past the attempt timeout"), + Covered("the chunk carrying subrequest 23 has its POST held past the attempt timeout"), + NotApplicable( + "the per-attempt timeout ends when the response headers arrive; past that " + + "only the per-read idle timeout bounds the body, which is the body-timeout row")), + }, + + new FaultEntry + { + Kind = FaultKind.BodyTimeout, + Shape = FaultShape.StalledBody, + Class = MgxErrorClass.TransientTransport, + Production = "headers arrive, then the body delivers a prefix and stops without closing", + Cells = Cells( + Pinned("page 3's body stalls. The class is retryable for a GET, but the body is " + + "read after the resilience pipeline has already returned the response, so " + + "nothing retries it and the pagination ends at the stall"), + Pinned("the third url's body stalls, and for the same reason nothing retries it"), + Covered("the chunk's envelope stalls; the chunk POST is not idempotent, so the " + + "policy does not retry it either way"), + Pinned("the download body stalls mid-stream. ShouldRetryDownload answers true for " + + "the HttpRequestException the idle timeout raises, but the copy runs " + + "outside the download pipeline, so nothing retries it")), + }, + + new FaultEntry + { + Kind = FaultKind.MalformedJson, + Shape = FaultShape.TruncatedJson, + Class = MgxErrorClass.Permanent, + Production = "a 200 whose body stops in the middle of the value array", + Cells = Cells( + Covered("page 3 is answered 200 with a body that stops mid-array"), + Covered("the third url is answered 200 with a body that stops mid-array"), + Covered("the chunk's envelope stops mid-array"), + NotApplicable("download bytes are opaque; nothing on the content path parses them")), + }, + + new FaultEntry + { + Kind = FaultKind.RepeatedNextLink, + Shape = FaultShape.SelfNextLink, + Class = null, + Production = "a page whose @odata.nextLink is its own URL", + Cells = Cells( + Pinned("page 3 points back at page 3. Nothing stops a self-loop of non-empty " + + "pages today - only -Top / maxItems ends it; an empty self-loop ends at " + + "the consecutive-empty-page limit"), + Pinned("the third url's page points back at itself, ended the same two ways"), + NotApplicable(NoLinksInABatch), + NotApplicable(NoLinksInADownload)), + }, + + new FaultEntry + { + Kind = FaultKind.UnusableNextLink, + Shape = FaultShape.UnusableNextLink, + Class = MgxErrorClass.Permanent, + Production = "a page whose @odata.nextLink is not an absolute https URL, which " + + "NextLinkValidator refuses with an InvalidOperationException", + Cells = Cells( + Covered("page 3 carries a nextLink that is not a URL"), + Covered("the third url's page carries a nextLink that is not a URL"), + NotApplicable(NoLinksInABatch), + NotApplicable(NoLinksInADownload)), + }, + + new FaultEntry + { + Kind = FaultKind.ExpiredToken, + Shape = FaultShape.Status, + Status = 401, + Class = MgxErrorClass.Authentication, + Production = "a 401 that arrives only after earlier requests of the same operation " + + "were answered - the token expired while the operation was running", + Cells = Cells( + Pinned("page 3 is answered 401 after two pages were delivered: no retry, no " + + "re-authentication, and the pages already delivered stand"), + Pinned("the third url is answered 401 after the first two succeeded"), + Pinned("sub-response 23 carries 401 after the earlier subrequests were answered"), + NotApplicable(StatusNotInABody)), + }, + + new FaultEntry + { + Kind = FaultKind.ChangedAuthContext, + Shape = FaultShape.SwappedAuthContext, + Class = null, + Production = "the Graph session's AuthContext object is replaced with one carrying a " + + "different application while the operation is running", + Cells = Cells( + Pinned("the session's AuthContext is swapped while page 2 is being answered; the " + + "running invocation finishes on the client it resolved, and the next " + + "invocation is the one that notices"), + Pinned("the session's AuthContext is swapped while the second id is being " + + "answered, with the same outcome"), + NotApplicable( + "the auth context is read once per invocation, in GetClient, before the " + + "first request goes out; no chunk POST re-reads it, so a swap has no " + + "subrequest-level point to land at. What it does to a running invocation " + + "is pinned at the page and fan-out points."), + NotApplicable( + "hop 2 is token-free by construction - no Authorization header, no auth " + + "context - so nothing about the download body depends on one")), + }, + + new FaultEntry + { + Kind = FaultKind.ConsistencyDelay, + Shape = FaultShape.Status, + Status = 404, + Class = MgxErrorClass.NotFound, + Production = "a 404 on the first attempt that would be a 200 on any later one - the " + + "read-after-write window", + Cells = Cells( + Pinned("page 3 answers 404 first and 200 afterwards; the classifier assigns " + + "NotFound and the policy does not retry, so the 200 is never asked for"), + Pinned("the third url answers 404 first and 200 afterwards, with the same outcome"), + Pinned("sub-response 23 carries 404, which the batch item loop does not retry"), + NotApplicable(StatusNotInABody)), + }, + ]; + + public static FaultEntry Entry(FaultKind kind) => + Entries.SingleOrDefault(e => e.Kind == kind) + ?? throw new InvalidOperationException($"No catalog entry for {kind}."); + + /// Every kind, for a theory that has to show a row per kind at its own point. + public static TheoryData AllKinds + { + get + { + var data = new TheoryData(); + foreach (var kind in Enum.GetValues()) + data.Add(kind); + return data; + } + } + + // ── The policy's decisions, asked rather than restated ────────────────── + + /// What the retry policy says about this fault on an idempotent request (a GET page, + /// a fan-out read). + public static bool RetriedOnAnIdempotentRequest(FaultEntry entry) => + entry.Class is { } cls && MgxErrorPolicy.ShouldRetry(cls, isIdempotent: true); + + /// What it says on a non-idempotent request - the $batch POST itself. + public static bool RetriedOnANonIdempotentRequest(FaultEntry entry) => + entry.Class is { } cls && MgxErrorPolicy.ShouldRetry(cls, isIdempotent: false); + + /// What it says about a $batch sub-response carrying this fault's status. + public static bool RetriedAsABatchItem(FaultEntry entry, string method) => + entry.Class is { } cls && MgxErrorPolicy.ShouldRetry( + cls, isIdempotent: !string.Equals(method, "POST", StringComparison.OrdinalIgnoreCase)); + + /// What the download-host retry filter says about it. + public static bool RetriedByTheDownloadPipeline(FaultEntry entry, Exception? exception) => + entry.Class is { } cls + && MgxErrorPolicy.ShouldRetryDownload(new MgxErrorInfo(cls, entry.Status), exception); + + /// Whether this fault counts toward opening the circuit. + public static bool CountsAgainstTheBreaker(FaultEntry entry) => + entry.Class is { } cls && MgxErrorPolicy.CountsAsCircuitFailure(new MgxErrorInfo(cls, entry.Status)); + + /// + /// What the classifier answers for the fault this row produces. The + /// column is held to this, so the catalog cannot claim a class the classifier does not assign. + /// + public static MgxErrorClass? ClassifierAnswer(FaultEntry entry) => entry.Shape switch + { + FaultShape.Status => MgxErrorClassifier.Classify(entry.Status).Class, + FaultShape.Transport => MgxErrorClassifier.Classify(entry.Transport!(), cancellationRequested: false).Class, + // The pipeline's per-attempt timeout, as Polly raises it. + FaultShape.HeldPastAttemptTimeout => + MgxErrorClassifier.Classify(new TimeoutRejectedException(), cancellationRequested: false).Class, + // What ResilientGraphClient turns a timed-out body read into. + FaultShape.StalledBody => MgxErrorClassifier.Classify( + new HttpRequestException(ResilientGraphClient.BodyReadTimedOutMessage), + cancellationRequested: false).Class, + FaultShape.TruncatedJson => MgxErrorClassifier.Classify( + MalformedJsonException(), cancellationRequested: false).Class, + // What NextLinkValidator.ValidateOrThrow raises. + FaultShape.UnusableNextLink => MgxErrorClassifier.Classify( + new InvalidOperationException("Pagination stopped"), cancellationRequested: false).Class, + _ => null, + }; + + /// A real JsonException, from a real truncated document rather than a constructed one. + public static JsonException MalformedJsonException() + { + try + { + JsonDocument.Parse(TruncatedPageBody).Dispose(); + } + catch (JsonException ex) + { + return ex; + } + throw new InvalidOperationException("The truncated body in the catalog now parses."); + } + + /// A 200 body that stops in the middle of the value array. + public const string TruncatedPageBody = """{"value":[{"id":"u5"""; +} + +/// +/// The catalog's own guards: the table has a row per kind, every row's class is the class the +/// classifier assigns, and nothing is marked impossible everywhere. +/// +public class FaultInjectionCatalogTests +{ + [Theory] + [MemberData(nameof(FaultCatalog.AllKinds), MemberType = typeof(FaultCatalog))] + public void Every_kind_has_exactly_one_row(FaultKind kind) + { + var entry = FaultCatalog.Entry(kind); + Assert.Equal(kind, entry.Kind); + Assert.False(string.IsNullOrWhiteSpace(entry.Production)); + foreach (var point in Enum.GetValues()) + { + var cell = entry.At(point); + Assert.False(string.IsNullOrWhiteSpace(cell.Note), + $"{kind} at {point} has no note saying what shape it takes or why it cannot land there"); + } + } + + [Theory] + [MemberData(nameof(FaultCatalog.AllKinds), MemberType = typeof(FaultCatalog))] + public void The_class_a_row_claims_is_the_one_the_classifier_assigns(FaultKind kind) + { + // The catalog names the class so the table can be read on its own; this holds that + // column to MgxErrorClassifier rather than letting it become a second opinion. + // ErrorPolicyParityTests owns status-to-decision for every status; nothing here + // restates those rows. + var entry = FaultCatalog.Entry(kind); + Assert.Equal(entry.Class, FaultCatalog.ClassifierAnswer(entry)); + } + + [Theory] + [MemberData(nameof(FaultCatalog.AllKinds), MemberType = typeof(FaultCatalog))] + public void No_kind_is_impossible_at_every_point(FaultKind kind) + { + // "Where physically possible" has to be auditable from the table, which means a kind + // that is not applicable anywhere would be a kind nobody covers at all. + var entry = FaultCatalog.Entry(kind); + Assert.Contains(Enum.GetValues(), + point => entry.At(point).Coverage != FaultCoverage.NotApplicable); + } + + [Fact] + public void The_consistency_class_still_has_no_producer() + { + // MgxErrorClass.Consistency is declared for the read-after-write window and nothing + // assigns it yet, which is why the eventual-consistency rows below record a NotFound + // that is not retried instead of asserting a retry. The producer is 2.3.0 work. + for (var status = 100; status < 600; status++) + Assert.NotEqual(MgxErrorClass.Consistency, MgxErrorClassifier.Classify(status).Class); + + Exception[] shapes = + [ + new HttpRequestException(), + new SocketException((int)SocketError.ConnectionReset), + new IOException(), + new TaskCanceledException(), + new TimeoutRejectedException(), + new InvalidOperationException(), + ]; + foreach (var exception in shapes) + { + Assert.NotEqual(MgxErrorClass.Consistency, + MgxErrorClassifier.Classify(exception, cancellationRequested: false).Class); + Assert.NotEqual(MgxErrorClass.Consistency, + MgxErrorClassifier.Classify(exception, cancellationRequested: true).Class); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Errors/RetryAfterPolicyTests.cs b/tests/Mgx.IntegrationTests/Engine/Errors/RetryAfterPolicyTests.cs new file mode 100644 index 0000000..7b4ac37 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Errors/RetryAfterPolicyTests.cs @@ -0,0 +1,61 @@ +using System.Net.Http.Headers; +using Mgx.Engine.Errors; + +namespace Mgx.IntegrationTests; + +public class RetryAfterPolicyTests +{ + private static readonly TimeSpan Cap = TimeSpan.FromSeconds(120); + + [Fact] + public void Delta_form_is_honored_and_clamped() + { + Assert.Equal(TimeSpan.FromSeconds(30), + RetryAfterPolicy.Resolve(new RetryConditionHeaderValue(TimeSpan.FromSeconds(30)), Cap)); + Assert.Equal(Cap, + RetryAfterPolicy.Resolve(new RetryConditionHeaderValue(TimeSpan.FromSeconds(600)), Cap)); + Assert.Equal(TimeSpan.Zero, + RetryAfterPolicy.Resolve(new RetryConditionHeaderValue(TimeSpan.Zero), Cap)); + } + + [Fact] + public void Date_form_is_honored_in_the_future_and_ignored_in_the_past() + { + var future = RetryAfterPolicy.Resolve( + new RetryConditionHeaderValue(DateTimeOffset.UtcNow.AddSeconds(40)), Cap); + Assert.NotNull(future); + Assert.InRange(future!.Value.TotalSeconds, 35, 41); + + Assert.Null(RetryAfterPolicy.Resolve( + new RetryConditionHeaderValue(DateTimeOffset.UtcNow.AddSeconds(-40)), Cap)); + + var farFuture = RetryAfterPolicy.Resolve( + new RetryConditionHeaderValue(DateTimeOffset.UtcNow.AddHours(2)), Cap); + Assert.Equal(Cap, farFuture); + } + + [Fact] + public void Absent_header_defers_to_backoff() + => Assert.Null(RetryAfterPolicy.Resolve(null, Cap)); + + [Fact] + public void Batch_seconds_form_rounds_up_floors_at_zero_and_clamps() + { + Assert.True(RetryAfterPolicy.TryResolveSeconds("30", 120, out var req, out var clamped)); + Assert.Equal(30, req); + Assert.Equal(30, clamped); + + Assert.True(RetryAfterPolicy.TryResolveSeconds("600", 120, out req, out clamped)); + Assert.Equal(600, req); + Assert.Equal(120, clamped); + + // A past HTTP date computes negative; the sleep floors at zero. + var past = DateTimeOffset.UtcNow.AddSeconds(-30).ToString("R"); + Assert.True(RetryAfterPolicy.TryResolveSeconds(past, 120, out req, out clamped)); + Assert.True(req <= 0); + Assert.Equal(0, clamped); + + Assert.False(RetryAfterPolicy.TryResolveSeconds("not-a-value", 120, out _, out _)); + Assert.False(RetryAfterPolicy.TryResolveSeconds(null, 120, out _, out _)); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/GraphBatchClientTests.cs b/tests/Mgx.IntegrationTests/Engine/GraphBatchClientTests.cs index b68e9a0..5631809 100644 --- a/tests/Mgx.IntegrationTests/Engine/GraphBatchClientTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/GraphBatchClientTests.cs @@ -1,4 +1,4 @@ -using System.Net; +using System.Net; using System.Text; using System.Text.Json; using Mgx.Engine.Http; @@ -214,7 +214,11 @@ public async Task A_failing_batch_POST_is_returned_as_a_chunk_failure() var ex = Assert.IsType(result.ChunkFailure); Assert.Equal(HttpStatusCode.Forbidden, ex.StatusCode); - Assert.Single(result.NotSent); + + // NotSent holds the operations after the failed chunk. This chunk's own operation went + // out and carries the failure's own status, so it is not one of them. + Assert.Empty(result.NotSent); + Assert.Equal((int)HttpStatusCode.Forbidden, result.Results[0].Response.Status); } [Fact] diff --git a/tests/Mgx.IntegrationTests/Engine/Http/AdaptiveRequestPacerTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/AdaptiveRequestPacerTests.cs index 6598570..3147473 100644 --- a/tests/Mgx.IntegrationTests/Engine/Http/AdaptiveRequestPacerTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Http/AdaptiveRequestPacerTests.cs @@ -1,4 +1,5 @@ using System.Diagnostics; +using System.Globalization; using System.Net; using Mgx.Engine.Http; @@ -13,6 +14,7 @@ namespace Mgx.IntegrationTests.Engine; /// Stateful tests run inside PacerScope (which re-enables the gate the suite-wide /// ModuleInitializer turned off) and in the Pipeline collection so nothing runs concurrently /// with them against the process-static state. +/// (Corpus: M365DSC-7273, sustained throttling.) /// [Collection("Pipeline")] public class AdaptiveRequestPacerTests @@ -185,6 +187,34 @@ public void RecordResponse_captures_the_throttle_percentage_gauge() Assert.Contains("proximity 95%", state); } + [Fact] + public void DescribeState_renders_the_latency_ratio_with_an_invariant_separator() + { + // Get-MgxTelemetry's PacingState is parsed by the benchmark harness, so a comma-decimal + // locale must not turn "1.8x" into "1,8x". + using var scope = new PacerScope(); + + var original = CultureInfo.CurrentCulture; + try + { + CultureInfo.CurrentCulture = new CultureInfo("de-DE"); + + // The baseline is a slow EMA, so a second, slower sample leaves a fractional ratio: + // baseline 100ms -> 112ms against a last latency of 200ms. + AdaptiveRequestPacer.RecordLatency(WorkloadBucket.Directory, 100); + AdaptiveRequestPacer.RecordLatency(WorkloadBucket.Directory, 200); + + var state = AdaptiveRequestPacer.DescribeState(); + Assert.NotNull(state); + Assert.Contains("1.8x of 112ms baseline", state); + Assert.DoesNotContain(",8x", state); + } + finally + { + CultureInfo.CurrentCulture = original; + } + } + [Fact] public async Task Proximity_gauge_paces_requests_beyond_what_the_rate_cap_alone_would() { diff --git a/tests/Mgx.IntegrationTests/Engine/Http/BodyRedactionTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/BodyRedactionTests.cs index 9dcea33..e3d26a8 100644 --- a/tests/Mgx.IntegrationTests/Engine/Http/BodyRedactionTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Http/BodyRedactionTests.cs @@ -1,4 +1,8 @@ using System.Net; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Nodes; +using Mgx.Cmdlets.Cmdlets.Batch; using Mgx.Engine.Http; namespace Mgx.IntegrationTests.Engine.Http; @@ -51,6 +55,11 @@ public void Redacts_pre_authenticated_urls(string body, string secret) [InlineData("""{"u":"https://example.com/x?design=KEEPME"}""", "KEEPME")] [InlineData("""{"u":"https://example.com/x?config=KEEPME"}""", "KEEPME")] [InlineData("""{"u":"https://example.com/x?assign=KEEPME"}""", "KEEPME")] + // Relative paths, which the item-url pattern reads: a paging link and a $select naming a + // property that starts with "sig" are both the thing a trace is read for. + [InlineData("""{"url":"/users?$skiptoken=KEEPME"}""", "KEEPME")] + [InlineData("""{"url":"/users?$select=signInActivity,KEEPME"}""", "KEEPME")] + [InlineData("""{"url":"/me/drive/root:/KEEPME.xlsx:/content"}""", "KEEPME")] public void Does_not_redact_diagnostics_or_ordinary_urls(string body, string keep) { Assert.Contains(keep, Trace(body), StringComparison.Ordinal); @@ -63,12 +72,175 @@ public void Still_redacts_credential_named_properties() Assert.DoesNotContain("hunter2", trace, StringComparison.Ordinal); } + /// + /// The trace and the dead-letter file read one list of credential name fragments + /// (SensitiveNames), so a field the file hides is missing from -Debug output too. These + /// three are the fragments the file has that a name-by-name tracer would not: a federated + /// credential's assertion, a profile passphrase, a connection string. + /// + [Theory] + // A workload-identity JWT: the value IS the credential. + [InlineData("""{"clientAssertion":"eyJhbGci.LEAKED.c2ln"}""", "LEAKED")] + // An imported PFX and a Wi-Fi profile both spell it this way. + [InlineData("""{"passphrase":"LEAKED"}""", "LEAKED")] + // A connection string carries its credential inline, so the whole value goes. + [InlineData("""{"connectionString":"Server=db;User=svc;Password=LEAKED"}""", "LEAKED")] + public void Redacts_the_names_the_dead_letter_file_redacts(string body, string secret) + { + var trace = Trace(body); + Assert.DoesNotContain(secret, trace, StringComparison.Ordinal); + Assert.Contains("", trace); + } + /// /// Three items, not two. Sanitize makes two passes that can each match a downloadUrl - one /// by property name, one by value - so with only two items a first-match-only regression /// was masked: pass one redacted FIRST, pass two redacted SECOND, and the test stayed green /// while every third and subsequent capability URL leaked. /// + /// + /// The dead-letter writer's walk over the same body the tracer is given. It reads parsed + /// JSON rather than text, so the two cannot be one function, but the fragment, the + /// parameters and the cut are one definition in SensitiveNames - and a URL the trace hides + /// and the file writes is a live credential on disk, which is the worse half of the pair. + /// + /// Serialized the way the writer serializes it: the default encoder escapes an ampersand, + /// and the query string a caller reads off the line has to be the one that was sent. + /// + private static string FileLine(string body) + { + var node = JsonNode.Parse(body); + node = InvokeMgxBatchRequest.RedactSensitiveFields(node); + return node!.ToJsonString(new JsonSerializerOptions + { + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }); + } + + [Theory] + // Intune's content-file body. The SAS is the credential and neither name says so, so only + // the value rule reaches it; what survives the cut says which blob and which parameter. + [InlineData("""{"azureStorageUri":"https://s.blob.core.windows.net/c/b?sv=2021-08-06&sig=LEAKEDSAS&se=2026-12-31"}""", + "LEAKEDSAS", "https://s.blob.core.windows.net/c/b?sv=2021-08-06&sig=")] + [InlineData("""{"uploadUrl":"https://s.blob.core.windows.net/c/b?sv=2021&sig=LEAKEDSAS"}""", + "LEAKEDSAS", "https://s.blob.core.windows.net/c/b?sv=2021&sig=")] + [InlineData("""{"u":"https://c-my.sharepoint.com/x?tempauth=LEAKED&ApiVersion=2.0"}""", + "LEAKED", "https://c-my.sharepoint.com/x?tempauth=")] + // The capability sits in the path, so the property name is the only thing that reaches it. + [InlineData("""{"@microsoft.graph.downloadUrl":"https://public.bl.files.1drv.com/y4mLEAKED/file.bin"}""", + "LEAKED", null)] + [InlineData("""{"value":[{"@content.downloadUrl":"https://c-my.sharepoint.com/x?tempauth=LEAKED"}]}""", + "LEAKED", null)] + // An array element has no property name at all: the value rule is what is left. + [InlineData("""{"links":["https://c.sharepoint.com/d?guestaccesstoken=LEAKED"]}""", + "LEAKED", "https://c.sharepoint.com/d?guestaccesstoken=")] + // The capability sits in the property NAME rather than in any value: the tracer matches + // any quoted string in the text alike, key or value, but the file's walk used to read + // only values, so this key reached the file whole. + [InlineData("""{"https://h/f?sig=LEAKED":"v"}""", + "LEAKED", "https://h/f?sig=")] + // A capability URL used as a key whose text ALSO carries a SensitiveNames fragment - + // "guestaccesstoken" contains "token", "authkey" contains "key", "passwordreset" contains + // "password". The fragment check used to run first, take the branch that only clears the + // value beside the key, and leave the key - secret included - verbatim; the trace's own + // first pass has this same fragment-matches-the-key case, but its later, URL-only pass + // still finds and cuts the key as text, which is why the trace never leaked here even + // though the file did. + [InlineData("""{"https://h/f?guestaccesstoken=LEAKED":"v"}""", + "LEAKED", "https://h/f?guestaccesstoken=")] + [InlineData("""{"https://h/f?authkey=LEAKED":"v"}""", + "LEAKED", "https://h/f?authkey=")] + [InlineData("""{"https://h/passwordreset?sig=LEAKED":"v"}""", + "LEAKED", "https://h/passwordreset?sig=")] + // A capability URL as a key takes the value beside it, in the trace as in the file: the key + // IS the credential, so what a body files under one is what that URL was minted for. The + // trace's URL pass cuts such a key as text but stops at the key's closing quote, so a value + // the file cleared stood beside a key both of them had cut - and only where the key's own + // text carried no fragment, which is the signature deciding. + [InlineData("""{"https://h/f?sig=abc":"KEEPME"}""", + "KEEPME", "https://h/f?sig=")] + // The whole body is one string. There is no property and no element for the walk to step + // into, so it returned having read nothing and the URL reached the file entire; the tracer + // matches a quoted string wherever it stands and had cut this one all along. + [InlineData("\"https://s.blob.core.windows.net/c/b?sv=2021&sig=LEAKEDSAS\"", + "LEAKEDSAS", "https://s.blob.core.windows.net/c/b?sv=2021&sig=")] + [InlineData("\"https://c-my.sharepoint.com/x?tempauth=LEAKED\"", + "LEAKED", "https://c-my.sharepoint.com/x?tempauth=")] + // A capability URL with its scheme and host gone, which is the form a caller reaches for + // when they hand a batch item a URL Graph gave them relative - the item's own url, and any + // body value beside it. The tracer reads that form; the file's rule was anchored on the + // scheme, so the same value was cut in -Debug output and written to disk entire. + [InlineData("""{"u":"/drives/d/items/i/content?tempauth=LEAKEDJWT"}""", + "LEAKEDJWT", "/drives/d/items/i/content?tempauth=")] + [InlineData("""{"uploadUrl":"/c/b?sv=2021&sig=LEAKEDSAS&se=2026"}""", + "LEAKEDSAS", "/c/b?sv=2021&sig=")] + [InlineData("""{"links":["/d?guestaccesstoken=LEAKED"]}""", + "LEAKED", "/d?guestaccesstoken=")] + // The same form used as a property name, which the key rule reads through the same cut. + [InlineData("""{"/f?authkey=LEAKED":"v"}""", + "LEAKED", "/f?authkey=")] + public void The_dead_letter_file_hides_what_the_trace_hides(string body, string secret, string? kept) + { + var trace = Trace(body); + var line = FileLine(body); + + Assert.DoesNotContain(secret, trace, StringComparison.Ordinal); + Assert.DoesNotContain(secret, line, StringComparison.Ordinal); + + if (kept != null) + { + Assert.Contains(kept, trace, StringComparison.Ordinal); + Assert.Contains(kept, line, StringComparison.Ordinal); + } + } + + /// + /// The other direction, on the side that had no value rule at all. A paging link is a URL + /// the file is read for, and over-redacting it costs the same thing over-redacting a trace + /// costs. + /// + [Theory] + [InlineData("""{"@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=RFNwdAoAAQAAA"}""", "RFNwdAoAAQAAA")] + [InlineData("""{"webUrl":"https://c.sharepoint.com/Shared%20Documents/KEEPME.xlsx"}""", "KEEPME")] + [InlineData("""{"u":"https://example.com/x?design=KEEPME"}""", "KEEPME")] + // The relative form the rule above now reads, on the other side of it: a paging link, a + // $select naming a property that starts with "sig", and a path with neither. + [InlineData("""{"url":"/users?$skiptoken=KEEPME"}""", "KEEPME")] + [InlineData("""{"url":"/users?$select=signInActivity,KEEPME"}""", "KEEPME")] + [InlineData("""{"url":"/me/drive/root:/KEEPME.xlsx:/content"}""", "KEEPME")] + public void The_dead_letter_file_keeps_the_urls_the_trace_keeps(string body, string keep) + { + Assert.Contains(keep, Trace(body), StringComparison.Ordinal); + Assert.Contains(keep, FileLine(body), StringComparison.Ordinal); + } + + /// + /// A $batch item's own url, as the envelope carries it. Invoke-MgxBatchRequest normalizes + /// each url to a relative path before the POST - that is the form Graph's batch format takes + /// - so a pre-authenticated URL a caller passed as an item URL has lost the scheme both URL + /// patterns anchor on by the time the request is traced, and the tempauth JWT stood in + /// -Debug output beside a dead-letter line that cut the same URL. + /// + /// Traced as a request, which is what a batch envelope is. + /// + [Theory] + [InlineData("""{"requests":[{"id":"1","method":"PUT","url":"/x?tempauth=LEAKEDJWT"}]}""", + "LEAKEDJWT", "/x?tempauth=")] + [InlineData("""{"requests":[{"id":"1","method":"PUT","url":"/c/b?sv=2021&sig=LEAKEDSAS&se=2026"}]}""", + "LEAKEDSAS", "/c/b?sv=2021&sig=")] + [InlineData("""{"requests":[{"id":"1","method":"GET","url":"/d?guestaccesstoken=LEAKED"}]}""", + "LEAKED", "/d?guestaccesstoken=")] + public void Redacts_a_capability_url_that_has_lost_its_scheme(string envelope, string secret, string kept) + { + var trace = GraphRequestTracer.FormatRequest( + new HttpRequestMessage(HttpMethod.Post, "https://graph.microsoft.com/v1.0/$batch"), + System.Text.Encoding.UTF8.GetBytes(envelope), 1); + + Assert.DoesNotContain(secret, trace, StringComparison.Ordinal); + // Which host is gone with the scheme, but which kind of URL still reads off the line. + Assert.Contains(kept, trace, StringComparison.Ordinal); + } + [Fact] public void Redacts_every_url_in_a_body_not_just_the_first() { @@ -77,4 +249,81 @@ public void Redacts_every_url_in_a_body_not_just_the_first() Assert.DoesNotContain("SECOND", trace, StringComparison.Ordinal); Assert.DoesNotContain("THIRD", trace, StringComparison.Ordinal); } + + /// + /// A capability-URL key whose only SensitiveNames fragment lies in the secret after the + /// capability parameter: "abc.token.def" carries "token", and "https://h/f?tempauth=" on + /// its own carries none. The cut takes the fragment away with the secret, so a fragment + /// check reading only the cut key found nothing and left the value beside the key whole - + /// a value under a pre-authenticated URL is as much a credential as one under + /// "guestaccesstoken", whose fragment happens to survive its own cut. The trace never had + /// the gap: its name pass reads the key as written, before its URL pass cuts anything. + /// + [Fact] + public void A_key_whose_fragment_is_only_in_the_secret_still_clears_the_value_beside_it() + { + const string body = """{"https://h/f?tempauth=abc.token.def":"SECRETVALUE"}"""; + + var trace = Trace(body); + Assert.DoesNotContain("abc.token.def", trace, StringComparison.Ordinal); + Assert.DoesNotContain("SECRETVALUE", trace, StringComparison.Ordinal); + + var line = FileLine(body); + Assert.DoesNotContain("abc.token.def", line, StringComparison.Ordinal); + Assert.DoesNotContain("SECRETVALUE", line, StringComparison.Ordinal); + + var cutKey = $"https://h/f?tempauth={InvokeMgxBatchRequest.RedactionMarker}"; + var obj = JsonNode.Parse(line)!.AsObject(); + Assert.True(obj.ContainsKey(cutKey), line); + Assert.Equal(InvokeMgxBatchRequest.RedactionMarker, obj[cutKey]!.GetValue()); + // Which host and which kind of URL still reads off the line, as with any other cut. + Assert.Contains("https://h/f?tempauth=", line, StringComparison.Ordinal); + } + + /// + /// A capability-URL key whose text carries no SensitiveNames fragment at all. The cut on the + /// key is what says the property is a credential - a pre-authenticated URL as a field name + /// makes whatever sits beside it the thing that URL was minted for - but the value beside it + /// was decided separately, on whether the key including its secret happened to read as a + /// credential name. So "?sig=abc" kept its value and "?sig=abcsecretdef" cleared it, and + /// which of the two a caller gets is the signature's spelling rather than the property's + /// meaning. + /// + [Fact] + public void A_capability_url_key_clears_the_value_beside_it_whatever_its_secret_spells() + { + var cutKey = $"https://h/f?sig={InvokeMgxBatchRequest.RedactionMarker}"; + + var line = FileLine("""{"https://h/f?sig=abc":"KEEPME"}"""); + var obj = JsonNode.Parse(line)!.AsObject(); + Assert.True(obj.ContainsKey(cutKey), line); + Assert.Equal(InvokeMgxBatchRequest.RedactionMarker, obj[cutKey]!.GetValue()); + Assert.DoesNotContain("KEEPME", line, StringComparison.Ordinal); + + // The same key with a fragment inside the signature, which was the only spelling that + // used to take the value with it. + var withFragment = JsonNode.Parse( + FileLine("""{"https://h/f?sig=abcsecretdef":"KEEPME"}"""))!.AsObject(); + Assert.Equal(InvokeMgxBatchRequest.RedactionMarker, withFragment[cutKey]!.GetValue()); + } + + /// + /// Two keys that cut to the identical redacted string - both are "?sig=", the same + /// capability parameter, differing only in the secret after it. JsonObject's indexer + /// replaces an existing key rather than refusing it, so re-adding the second under the + /// same cut name would silently drop the first property from the file. Both have to + /// survive, which means they cannot end up with the same name. + /// + [Fact] + public void Two_keys_that_redact_to_the_same_string_both_survive() + { + var line = FileLine("""{"https://h/f?sig=AAA":"1","https://h/f?sig=BBB":"2"}"""); + var obj = JsonNode.Parse(line)!.AsObject(); + + Assert.DoesNotContain("AAA", line, StringComparison.Ordinal); + Assert.DoesNotContain("BBB", line, StringComparison.Ordinal); + Assert.Equal(2, obj.Count); + Assert.True(obj.ContainsKey($"https://h/f?sig={InvokeMgxBatchRequest.RedactionMarker}")); + Assert.True(obj.ContainsKey($"https://h/f?sig={InvokeMgxBatchRequest.RedactionMarker}#2")); + } } diff --git a/tests/Mgx.IntegrationTests/Engine/Http/ConfigurableOptionsTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/ConfigurableOptionsTests.cs index 146edbe..e7aa1ae 100644 --- a/tests/Mgx.IntegrationTests/Engine/Http/ConfigurableOptionsTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Http/ConfigurableOptionsTests.cs @@ -171,11 +171,43 @@ public void CircuitBreakerFailureRatio_AcceptsValidValues(double value) [Theory] [InlineData(0)] [InlineData(-1)] + [InlineData(1)] [InlineData(1001)] public void CircuitBreakerMinThroughput_RejectsInvalidValues(int value) { - Assert.Throws( + var ex = Assert.Throws( () => new ResilientGraphClientOptions { CircuitBreakerMinThroughput = value }); + // Refused in the option's own terms, with the range it will take. Polly refuses 1 as + // well, but it does so while a later call is building the pipeline. + Assert.Contains("between 2 and 1,000", ex.Message); + } + + [Theory] + [InlineData(2)] + [InlineData(40)] + [InlineData(1000)] + public void CircuitBreakerMinThroughput_EveryAcceptedValueBuildsAPipeline(int value) + { + // The floor, the default and the ceiling. A value the option takes but the pipeline + // cannot hold would fail on the next request instead of at the set site, which is the + // whole point of the option validating at all. + ResiliencePipelineFactory.Reset(); + try + { + var options = new ResilientGraphClientOptions + { + NoRateLimit = true, + CircuitBreakerMinThroughput = value + }; + + var (pipeline, _) = ResiliencePipelineFactory.GetOrCreate(options); + + Assert.NotNull(pipeline); + } + finally + { + ResiliencePipelineFactory.Reset(); + } } [Theory] diff --git a/tests/Mgx.IntegrationTests/Engine/Http/DownloadHeaderTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/DownloadHeaderTests.cs new file mode 100644 index 0000000..293f9f5 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/DownloadHeaderTests.cs @@ -0,0 +1,110 @@ +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// Conditional headers cross to the download host; auth and correlation headers never do. +/// Before this, hop 2 carried only Range, so a caller's If-None-Match silently vanished +/// and a conditional download re-fetched the full content every time. +/// +[Collection("Pipeline")] +public class DownloadHeaderTests : IDisposable +{ + private const string CdnUrl = "https://contoso-my.sharepoint.com/_layouts/15/download.aspx?UniqueId=abc&tempauth=TOKEN"; + + private readonly MockHttpHandler _cdnHandler = new(); + + public DownloadHeaderTests() + { + GraphContentClient.DownloadClientForTests = new HttpClient(_cdnHandler); + } + + public void Dispose() + { + GraphContentClient.DownloadClientForTests?.Dispose(); + GraphContentClient.DownloadClientForTests = null; + } + + [Fact] + public async Task Conditional_headers_reach_the_download_host() + { + _cdnHandler.QueueBytes(HttpStatusCode.NotModified, [], "application/octet-stream"); + + var headers = new Dictionary + { + ["If-None-Match"] = "\"etag-1\"", + ["If-Modified-Since"] = "Mon, 24 Aug 2026 10:00:00 GMT", + ["If-Match"] = "\"graph-etag\"", // write validator: stays on hop 1 + ["Authorization"] = "Bearer should-never-cross", + ["client-request-id"] = "should-never-cross", + ["ConsistencyLevel"] = "eventual", + }; + + // 304 is a non-success status; the client surfaces it as an error after reading the + // body - what matters here is what went on the wire. + try + { + using var result = await GraphContentClient.GetFromDownloadUrlAsync( + CdnUrl, range: null, TimeSpan.FromSeconds(5), CancellationToken.None, headers); + } + catch (Mgx.Engine.Models.GraphServiceException) { /* 304 surfaces as an error; fine */ } + + var sent = Assert.Single(_cdnHandler.CapturedRequests); + Assert.Equal("\"etag-1\"", Assert.Single(Assert.Contains("If-None-Match", sent.Headers))); + Assert.Contains("If-Modified-Since", sent.Headers); + Assert.DoesNotContain("Authorization", sent.Headers); + Assert.DoesNotContain("client-request-id", sent.Headers); + Assert.DoesNotContain("ConsistencyLevel", sent.Headers); + Assert.DoesNotContain("If-Match", sent.Headers); + } + + [Fact] + public async Task Lowercase_conditional_headers_forward_on_the_two_hop_path() + { + // The two-hop path copies the caller dictionary; the copy must keep case + // insensitivity or hop 2 silently drops non-canonical casings. + ResiliencePipelineFactory.Reset(); + var graphWire = new MockHttpHandler(); + graphWire.QueueResponse(HttpStatusCode.Found, headers: new Dictionary + { + ["Location"] = CdnUrl, + }); + _cdnHandler.QueueBytes(HttpStatusCode.OK, [1, 2, 3], "application/octet-stream"); + + using var client = new ResilientGraphClient(new HttpClient(graphWire), + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }); + var headers = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["if-none-match"] = "\"lower-etag\"", + ["client-request-id"] = "caller-id", + }; + using var result = await client.GetContentAsync( + "https://graph.microsoft.com/v1.0/me/drive/items/x/content", + range: null, headers, CancellationToken.None); + + var hop2 = Assert.Single(_cdnHandler.CapturedRequests); + Assert.Equal("\"lower-etag\"", Assert.Single(Assert.Contains("If-None-Match", + new Dictionary(hop2.Headers, StringComparer.OrdinalIgnoreCase)))); + + // And hop 1 must carry the caller's correlation id exactly once. + var hop1 = Assert.Single(graphWire.CapturedRequests); + var ids = Assert.Contains("client-request-id", + new Dictionary(hop1.Headers, StringComparer.OrdinalIgnoreCase)); + Assert.Equal("caller-id", Assert.Single(ids)); + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task A_download_without_caller_headers_sends_none() + { + _cdnHandler.QueueBytes(HttpStatusCode.OK, [1, 2, 3], "application/octet-stream"); + + using var result = await GraphContentClient.GetFromDownloadUrlAsync( + CdnUrl, range: null, TimeSpan.FromSeconds(5), CancellationToken.None); + + var sent = Assert.Single(_cdnHandler.CapturedRequests); + Assert.DoesNotContain("If-None-Match", sent.Headers); + Assert.Equal(HttpStatusCode.OK, result.StatusCode); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Http/FaultInjectionDownloadTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/FaultInjectionDownloadTests.cs new file mode 100644 index 0000000..53179ef --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/FaultInjectionDownloadTests.cs @@ -0,0 +1,189 @@ +using System.Net; +using System.Text; +using Mgx.Engine.Errors; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// Every fault in the catalog, held against the middle of a content download: hop 1 redirects to +/// the download host, hop 2 answers 200, and the body faults after its first bytes have already +/// reached the caller. Two kinds can land there at all - a stall and a reset - and the rest are +/// explicit "not applicable" rows rather than tests nobody wrote: a status arrives on the +/// response line, a nextLink has no meaning here, and hop 2 is token-free by construction. +/// +[Collection("Pipeline")] +public class FaultInjectionDownloadTests : IDisposable +{ + private const string GraphContentUrl = + "https://graph.microsoft.com/v1.0/me/drive/items/01ABC/content"; + private const string CdnUrl = + "https://contoso-my.sharepoint.com/_layouts/15/download.aspx?UniqueId=abc&tempauth=SECRET"; + + /// The bytes the caller has already been handed when the fault lands. + private const string DeliveredPrefix = "the-first-bytes"; + + /// What the armed fault in this file is called, so the theory can assert by name + /// that does not carry it. + private const string FaultName = "fault-mid-body"; + + /// Short enough that a stalled body ends the case in a fraction of a second. + private static readonly TimeSpan IdleTimeout = TimeSpan.FromMilliseconds(200); + + private readonly MockHttpHandler _graphHandler = new(); + private readonly MockHttpHandler _cdnHandler = new(); + private readonly HttpClient _graphHttpClient; + private readonly ResilientGraphClient _client; + + public FaultInjectionDownloadTests() + { + ResiliencePipelineFactory.Reset(); + _graphHttpClient = new HttpClient(_graphHandler); + _client = new ResilientGraphClient(_graphHttpClient, + new ResilientGraphClientOptions { NoRateLimit = true, NoAdaptivePacing = true, MaxRetryAttempts = 1 }); + GraphContentClient.DownloadClientForTests = new HttpClient(_cdnHandler); + } + + public void Dispose() + { + GraphContentClient.DownloadClientForTests?.Dispose(); + GraphContentClient.DownloadClientForTests = null; + _client.Dispose(); + _graphHttpClient.Dispose(); + ResiliencePipelineFactory.Reset(); + GC.SuppressFinalize(this); + } + + /// + /// A body that delivers its prefix and then fails the read, the way a connection dropped + /// mid-transfer reaches a reader: an IOException out of the stream, not the + /// HttpRequestException a failed send raises. Overriding the content read stream is what + /// makes it a mid-body failure rather than a failure to produce a body at all. + /// + private sealed class ResetMidBodyContent : HttpContent + { + private readonly byte[] _prefix = Encoding.UTF8.GetBytes(DeliveredPrefix); + + protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context) + => throw new IOException("The response ended prematurely."); + + protected override Task CreateContentReadStreamAsync() + => Task.FromResult(new ResettingStream(_prefix)); + + protected override bool TryComputeLength(out long length) + { + length = 0; + return false; + } + + private sealed class ResettingStream(byte[] prefix) : Stream + { + private int _delivered; + + public override ValueTask ReadAsync( + Memory buffer, CancellationToken cancellationToken = default) + { + if (_delivered < prefix.Length) + { + var count = Math.Min(buffer.Length, prefix.Length - _delivered); + prefix.AsSpan(_delivered, count).CopyTo(buffer.Span); + _delivered += count; + return ValueTask.FromResult(count); + } + + throw new IOException("The response ended prematurely."); + } + + public override Task ReadAsync( + byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => ReadAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => _delivered; set => throw new NotSupportedException(); } + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + } + } + + private void ArmMidBody(FaultEntry entry) + { + _graphHandler.QueueResponse(HttpStatusCode.Found, null, new() { ["Location"] = CdnUrl }); + + switch (entry.Shape) + { + case FaultShape.StalledBody: + _cdnHandler.When(_ => true, FaultName).RespondStalled( + HttpStatusCode.OK, prefix: DeliveredPrefix, contentType: "application/octet-stream"); + break; + case FaultShape.Transport: + _cdnHandler.When(_ => true, FaultName).Respond(_ => new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ResetMidBodyContent() + }); + break; + default: + throw new InvalidOperationException( + $"{entry.Kind} does not take a wire shape mid-body; it needs its own case."); + } + } + + [Theory] + [MemberData(nameof(FaultCatalog.AllKinds), MemberType = typeof(FaultCatalog))] + public async Task Each_fault_mid_download_body(FaultKind kind) + { + var entry = FaultCatalog.Entry(kind); + var cell = entry.At(InjectionPoint.DownloadBody); + if (cell.Coverage == FaultCoverage.NotApplicable) + { + FaultInjectionRows.AssertNotApplicable(entry, InjectionPoint.DownloadBody); + return; + } + + ArmMidBody(entry); + + using var result = await _client.GetContentAsync(GraphContentUrl); + Assert.True(result.FromDownloadHost); + Assert.DoesNotContain(FaultName, _cdnHandler.UnansweredRules); + + using var destination = new MemoryStream(); + var thrown = await Record.ExceptionAsync(() => GraphContentClient.CopyWithIdleTimeoutAsync( + result.Content, destination, maxBytes: null, IdleTimeout, CancellationToken.None)); + + Assert.NotNull(thrown); + + // The bytes that arrived before the fault are the caller's; the copy does not unwrite them. + Assert.Equal(DeliveredPrefix, Encoding.UTF8.GetString(destination.ToArray())); + + // The classifier sees the failure the read raised, and it is the class the catalog claims. + Assert.Equal(entry.Class, + MgxErrorClassifier.Classify(thrown!, cancellationRequested: false).Class); + + // Whatever the download-host retry filter would say about it, nothing acts on it: the + // body copy runs after GetContentAsync returned, outside the pipeline that filter belongs + // to. A kind the filter would retry is therefore a pinned row, not a covered one. + var wouldRetry = FaultCatalog.RetriedByTheDownloadPipeline(entry, thrown); + Assert.Equal(wouldRetry ? FaultCoverage.Pinned : FaultCoverage.Covered, cell.Coverage); + Assert.Single(_cdnHandler.Requests); + Assert.Single(_graphHandler.Requests); + + switch (entry.Shape) + { + case FaultShape.StalledBody: + var stalled = Assert.IsType(thrown); + Assert.Contains("stalled", stalled.Message, StringComparison.OrdinalIgnoreCase); + break; + case FaultShape.Transport: + Assert.IsAssignableFrom(thrown); + break; + default: + Assert.Fail($"{entry.Kind} surfaced as {thrown!.GetType().Name} with no case to check it."); + break; + } + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Http/GraphContentClientTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/GraphContentClientTests.cs index 55b1c4e..01ec286 100644 --- a/tests/Mgx.IntegrationTests/Engine/Http/GraphContentClientTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Http/GraphContentClientTests.cs @@ -2,7 +2,6 @@ using System.Management.Automation; using System.Net; using System.Net.Http.Headers; -using System.Reflection; using System.Text; using Mgx.Cmdlets.Base; using Mgx.Engine.Http; @@ -175,40 +174,12 @@ public async Task A_transport_that_auto_followed_the_redirect_is_refused() private const string TestTenantId = "test-tenant-00000000-0000-0000-0000-000000000000"; /// - /// Points MgxCmdletBase's cached transport at this class's Graph mock and declares whether - /// mgx owns it (false = the Graph SDK's client, which ships a RedirectHandler). Reflection - /// because the statics are private; the fingerprint has to match what the test shell's - /// stubbed Get-MgContext produces, or GetClient() sees a credential change and rebuilds. + /// Points MgxCmdletBase's transport at this class's Graph mock and declares whether mgx owns + /// it (false = the Graph SDK's client, which ships a RedirectHandler). The mock client is + /// this class's own, so the scope borrows it rather than disposing it at the end of a test. /// - private void InjectCmdletTransport(bool owned, HttpClient? client = null) - { - ResiliencePipelineFactory.Reset(); - var options = new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }; - SetCmdletStatic("s_graphHttpClient", client ?? _graphHttpClient); - SetCmdletStatic("s_cachedAuthFingerprint", - MgxCmdletBase.BuildAuthFingerprint(new { TenantId = TestTenantId }, null)); - SetCmdletStatic("s_cachedAuthContextRef", null); - SetCmdletStatic("s_ownsHttpClient", owned); - SetCmdletStatic("s_graphEndpoint", "https://graph.microsoft.com"); - SetCmdletStatic("s_clientOptions", options); - SetCmdletStatic("s_cachedTotalTimeoutSeconds", options.TotalTimeoutSeconds); - } - - private static void ResetCmdletTransport() - { - SetCmdletStatic("s_graphHttpClient", null); - SetCmdletStatic("s_cachedAuthFingerprint", null); - SetCmdletStatic("s_cachedAuthContextRef", null); - SetCmdletStatic("s_ownsHttpClient", false); - SetCmdletStatic("s_cachedTotalTimeoutSeconds", 0); - SetCmdletStatic("s_clientOptions", ResilientGraphClientOptions.Default); - ResiliencePipelineFactory.Reset(); - } - - private static void SetCmdletStatic(string field, object? value) => - typeof(MgxCmdletBase) - .GetField(field, BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Static)! - .SetValue(null, value); + private MgxTransportScope InjectCmdletTransport(bool owned, HttpClient? client = null) => + MgxTransportScope.Inject(client ?? _graphHttpClient, owned); private static PowerShell CreateContentShell() { @@ -241,37 +212,30 @@ public void GetMgxContent_refuses_to_fetch_over_a_transport_mgx_does_not_own() // validated - the exact leak the two-hop design exists to prevent. _graphHandler.SetDefaultResponse(HttpStatusCode.OK, "bytes-that-must-not-be-fetched"); - try + // 1. Borrowed SDK transport: refuse, and refuse before the request is sent. + using (InjectCmdletTransport(owned: false)) + using (var ps = CreateContentShell()) { - // 1. Borrowed SDK transport: refuse, and refuse before the request is sent. - InjectCmdletTransport(owned: false); - using (var ps = CreateContentShell()) - { - ps.AddCommand("Get-MgxContent").AddParameter("Uri", contentUri); - var thrown = Record.Exception(() => ps.Invoke()); + ps.AddCommand("Get-MgxContent").AddParameter("Uri", contentUri); + var thrown = Record.Exception(() => ps.Invoke()); - Assert.Contains(ErrorIds(ps, thrown), - id => id.Contains("ContentRequiresOwnedTransport", StringComparison.Ordinal)); - Assert.Equal(0, _graphHandler.RequestCount); - } - - // 2. The identical call on the mgx-owned transport has to go through, or step 1 - // proves nothing: a cmdlet that always refuses would pass it too. - InjectCmdletTransport(owned: true); - using (var ps = CreateContentShell()) - { - ps.AddCommand("Get-MgxContent").AddParameter("Uri", contentUri); - var results = ps.Invoke(); - - Assert.Empty(ps.Streams.Error); - var bytes = Assert.IsType(Assert.Single(results).BaseObject); - Assert.Equal("bytes-that-must-not-be-fetched", Encoding.UTF8.GetString(bytes)); - Assert.Equal(1, _graphHandler.RequestCount); - } + Assert.Contains(ErrorIds(ps, thrown), + id => id.Contains("ContentRequiresOwnedTransport", StringComparison.Ordinal)); + Assert.Equal(0, _graphHandler.RequestCount); } - finally + + // 2. The identical call on the mgx-owned transport has to go through, or step 1 + // proves nothing: a cmdlet that always refuses would pass it too. + using (InjectCmdletTransport(owned: true)) + using (var ps = CreateContentShell()) { - ResetCmdletTransport(); + ps.AddCommand("Get-MgxContent").AddParameter("Uri", contentUri); + var results = ps.Invoke(); + + Assert.Empty(ps.Streams.Error); + var bytes = Assert.IsType(Assert.Single(results).BaseObject); + Assert.Equal("bytes-that-must-not-be-fetched", Encoding.UTF8.GetString(bytes)); + Assert.Equal(1, _graphHandler.RequestCount); } } @@ -348,19 +312,14 @@ public override void Flush() { } { var handler = new SizedBodyHandler(declaredLength, actualBytes); using var http = new HttpClient(handler); - try + using (InjectCmdletTransport(owned: true, client: http)) { - InjectCmdletTransport(owned: true, client: http); using var ps = CreateContentShell(); ps.AddCommand("Get-MgxContent").AddParameter("Uri", "/me/drive/items/01ABC/content"); Collection output = []; var thrown = Record.Exception(() => output = ps.Invoke()); return (output, [.. ErrorIds(ps, thrown)], handler.BytesServed); } - finally - { - ResetCmdletTransport(); - } } [Fact] @@ -444,9 +403,8 @@ protected override Task SendAsync( { var handler = new RangeHonouringHandler(body); using var http = new HttpClient(handler); - try + using (InjectCmdletTransport(owned: true, client: http)) { - InjectCmdletTransport(owned: true, client: http); using var ps = CreateContentShell(); ps.AddCommand("Get-MgxContent").AddParameter("Uri", "/me/drive/items/01ABC/content"); foreach (var (name, value) in rangeParameters) @@ -457,10 +415,6 @@ protected override Task SendAsync( Assert.Empty(ps.Streams.Error); return (Assert.IsType(Assert.Single(output).BaseObject), handler.RangeHeader); } - finally - { - ResetCmdletTransport(); - } } [Fact] @@ -518,9 +472,8 @@ protected override Task SendAsync( { var handler = new RangeIgnoringHandler(body); using var http = new HttpClient(handler); - try + using (InjectCmdletTransport(owned: true, client: http)) { - InjectCmdletTransport(owned: true, client: http); using var ps = CreateContentShell(); ps.AddCommand("Get-MgxContent").AddParameter("Uri", "/me/drive/items/01ABC/content"); foreach (var (name, value) in rangeParameters) @@ -531,10 +484,6 @@ protected override Task SendAsync( Assert.Empty(ps.Streams.Error); return (Assert.IsType(Assert.Single(output).BaseObject), handler.RangeHeader); } - finally - { - ResetCmdletTransport(); - } } [Fact] @@ -562,9 +511,9 @@ public void GetMgxContent_honours_Offset_locally_when_the_server_ignores_the_ran var outFile = Path.Combine(Path.GetTempPath(), $"mgx-offset-{Guid.NewGuid():N}.bin"); var handler = new RangeIgnoringHandler(body); using var http = new HttpClient(handler); + using var transport = InjectCmdletTransport(owned: true, client: http); try { - InjectCmdletTransport(owned: true, client: http); using var ps = CreateContentShell(); ps.AddCommand("Get-MgxContent") .AddParameter("Uri", "/me/drive/items/01ABC/content") @@ -578,7 +527,6 @@ public void GetMgxContent_honours_Offset_locally_when_the_server_ignores_the_ran } finally { - ResetCmdletTransport(); if (File.Exists(outFile)) File.Delete(outFile); } } diff --git a/tests/Mgx.IntegrationTests/Engine/Http/HeaderCultureTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/HeaderCultureTests.cs new file mode 100644 index 0000000..bb2aa25 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/HeaderCultureTests.cs @@ -0,0 +1,92 @@ +using System.Globalization; +using System.Management.Automation; +using System.Net; +using System.Reflection; +using Mgx.Cmdlets.Base; + +namespace Mgx.IntegrationTests; + +/// +/// A -Headers value that is not a string is rendered by mgx before it goes on the wire, and +/// object.ToString() renders under the thread's culture: the same script sent "X-Ratio: 0,5" +/// on a de-DE machine and "X-Ratio: 0.5" on an en-US one. +/// +[Collection("Pipeline")] +public class HeaderCultureTests +{ + private static readonly MethodInfo Build = + typeof(MgxCmdletBase).GetMethod("BuildRequestHeaders", + BindingFlags.Static | BindingFlags.NonPublic)!; + + private static Dictionary Headers(System.Collections.Hashtable values) => + (Dictionary)Build.Invoke(null, [null, values])!; + + private static void InCulture(string name, Action body) + { + var previous = CultureInfo.CurrentCulture; + CultureInfo.CurrentCulture = CultureInfo.GetCultureInfo(name); + try { body(); } + finally { CultureInfo.CurrentCulture = previous; } + } + + [Theory] + [InlineData("de-DE")] + [InlineData("en-US")] + [InlineData("fr-FR")] + public void A_non_string_header_value_renders_the_same_in_every_culture(string culture) + { + InCulture(culture, () => + { + var headers = Headers(new System.Collections.Hashtable + { + ["X-Ratio"] = 0.5d, + ["X-Decimal"] = 1.7m, + ["X-Count"] = 100, + ["X-List"] = new object[] { 0.5d, 100 } + }); + + Assert.Equal("0.5", headers["X-Ratio"]); + Assert.Equal("1.7", headers["X-Decimal"]); + Assert.Equal("100", headers["X-Count"]); + Assert.Equal("0.5, 100", headers["X-List"]); + }); + } + + /// A -Headers key is typed object as well, and was rendered under the culture. + [Theory] + [InlineData("de-DE")] + [InlineData("en-US")] + [InlineData("fr-FR")] + public void A_non_string_header_name_renders_the_same_in_every_culture(string culture) + { + InCulture(culture, () => + { + var headers = Headers(new System.Collections.Hashtable { [1.5d] = "x" }); + + Assert.Equal("1.5", Assert.Single(headers.Keys)); + Assert.Equal("x", headers["1.5"]); + }); + } + + /// The value as sent, read back off the captured request. + [Fact] + public void The_value_on_the_wire_does_not_depend_on_the_operators_locale() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "{\"id\":\"u1\"}"); + using var transport = MgxTransportScope.Inject(wire); + + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript( + "[System.Threading.Thread]::CurrentThread.CurrentCulture = 'de-DE'; " + + "Invoke-MgxRequest -Uri '/users/u1' -Headers @{ 'X-Ratio' = 0.5 }"); + ps.Invoke(); + + var sent = Assert.Single(wire.CapturedRequests); + Assert.Equal("0.5", Assert.Single(sent.Headers["X-Ratio"])); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Http/HeaderFidelityTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/HeaderFidelityTests.cs new file mode 100644 index 0000000..3285387 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/HeaderFidelityTests.cs @@ -0,0 +1,194 @@ +using System.Net; +using System.Text; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// Caller headers reach the wire as given: content headers land on the content collection +/// instead of being silently dropped, a caller's correlation id is not doubled by mgx's +/// own, and the merge is case-insensitive like HTTP header names are. +/// (Corpus: GraphSDK-2328, missing required headers.) +/// +[Collection("Pipeline")] +public class HeaderFidelityTests +{ + private static readonly ResilientGraphClientOptions Options = new() + { + NoRateLimit = true, + MaxRetryAttempts = 3, + TotalTimeoutSeconds = 30, + AttemptTimeoutSeconds = 10 + }; + + private static (MockHttpHandler wire, ResilientGraphClient client) NewClient() + { + ResiliencePipelineFactory.Reset(); + var wire = new MockHttpHandler(); + var client = new ResilientGraphClient(new HttpClient(wire), Options); + return (wire, client); + } + + [Fact] + public async Task A_caller_content_type_replaces_the_default_instead_of_vanishing() + { + var (wire, client) = NewClient(); + using (client) + { + wire.QueueResponse(HttpStatusCode.OK, "{}"); + using var response = await client.SendAsync(HttpMethod.Put, + "https://graph.microsoft.com/v1.0/drives/d/items/i/content", + new StringContent("hello", Encoding.UTF8, "application/json"), + new Dictionary { ["Content-Type"] = "text/plain" }, + CancellationToken.None); + + var sent = Assert.Single(wire.CapturedRequests); + var contentType = Assert.Contains("Content-Type", sent.ContentHeaders); + Assert.Equal("text/plain", Assert.Single(contentType)); + } + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task A_content_header_survives_every_retry_attempt() + { + var (wire, client) = NewClient(); + using (client) + { + wire.QueueFailuresThenSuccess(1, HttpStatusCode.ServiceUnavailable, "{}"); + using var response = await client.SendAsync(HttpMethod.Put, + "https://graph.microsoft.com/v1.0/drives/d/items/i/content", + new StringContent("hello", Encoding.UTF8, "application/json"), + new Dictionary { ["Content-Type"] = "text/plain" }, + CancellationToken.None); + + Assert.Equal(2, wire.CapturedRequests.Count); + foreach (var sent in wire.CapturedRequests) + { + var contentType = Assert.Contains("Content-Type", sent.ContentHeaders); + Assert.Equal("text/plain", Assert.Single(contentType)); + } + } + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task A_header_no_collection_accepts_warns_instead_of_vanishing_silently() + { + var (wire, client) = NewClient(); + using (client) + { + wire.QueueResponse(HttpStatusCode.OK, TestData.SingleUser); + // A GET has no content, so a content header has nowhere to go. + using var response = await client.GetAsync( + "https://graph.microsoft.com/v1.0/users/u1", + CancellationToken.None, + new Dictionary { ["Content-Type"] = "text/plain" }); + + var warnings = new List(); + client.WarningWriter = warnings.Add; + client.DrainWarningMessages(); + Assert.Contains(warnings, w => w.Contains("Content-Type")); + } + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task A_caller_correlation_id_is_not_doubled() + { + var (wire, client) = NewClient(); + using (client) + { + wire.QueueResponse(HttpStatusCode.OK, TestData.SingleUser); + using var response = await client.GetAsync( + "https://graph.microsoft.com/v1.0/users/u1", + CancellationToken.None, + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["client-request-id"] = "caller-supplied-id" + }); + + var sent = Assert.Single(wire.CapturedRequests); + var ids = Assert.Contains("client-request-id", sent.Headers); + Assert.Equal("caller-supplied-id", Assert.Single(ids)); + } + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public void Case_variant_duplicate_keys_merge_to_one_header() + { + // A genuinely case-variant duplicate: an ordinal Hashtable holds both spellings; + // the merge must produce ONE entry (whichever value enumeration hands it last). + var duplicates = new System.Collections.Hashtable(); + duplicates["If-Match"] = "\"etag-a\""; + duplicates["if-match"] = "\"etag-b\""; + Assert.Equal(2, duplicates.Count); + + var headers = TestCmdlet.BuildHeaders(null, duplicates); + Assert.NotNull(headers); + Assert.Single(headers!.Keys, k => string.Equals(k, "If-Match", StringComparison.OrdinalIgnoreCase)); + Assert.True(headers.ContainsKey("if-match")); + Assert.True(headers.ContainsKey("IF-MATCH")); + } + + [Fact] + public async Task A_malformed_header_name_warns_instead_of_crashing_a_write() + { + var (wire, client) = NewClient(); + using (client) + { + wire.QueueResponse(HttpStatusCode.OK, "{}"); + using var response = await client.SendAsync(HttpMethod.Post, + "https://graph.microsoft.com/v1.0/users", + new StringContent("{}", Encoding.UTF8, "application/json"), + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["Bad Header"] = "v", + }, + CancellationToken.None); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + var warnings = new List(); + client.WarningWriter = warnings.Add; + client.DrainWarningMessages(); + Assert.Contains(warnings, w => w.Contains("Bad Header")); + } + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task A_caller_content_length_is_ignored_with_a_warning() + { + var (wire, client) = NewClient(); + using (client) + { + wire.QueueResponse(HttpStatusCode.OK, "{}"); + using var response = await client.SendAsync(HttpMethod.Post, + "https://graph.microsoft.com/v1.0/users", + new StringContent("0123456789", Encoding.UTF8, "application/json"), + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["Content-Length"] = "3", + }, + CancellationToken.None); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + var sent = Assert.Single(wire.CapturedRequests); + Assert.Equal(10, sent.Body!.Length); // the real body, not a truncation + var warnings = new List(); + client.WarningWriter = warnings.Add; + client.DrainWarningMessages(); + Assert.Contains(warnings, w => w.Contains("Content-Length")); + } + ResiliencePipelineFactory.Reset(); + } + + /// BuildRequestHeaders is protected static; surfaced for assertion. + private sealed class TestCmdlet : Mgx.Cmdlets.Base.MgxCmdletBase + { + public static Dictionary? BuildHeaders( + string? consistencyLevel, System.Collections.Hashtable? extra) + => BuildRequestHeaders(consistencyLevel, extra); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Http/RateLimiterLifetimeTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/RateLimiterLifetimeTests.cs new file mode 100644 index 0000000..c1dd757 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/RateLimiterLifetimeTests.cs @@ -0,0 +1,87 @@ +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// The rate limiter's lifetime against option changes. Every ResilientGraphClient and every +/// ResilientDelegatingHandler captures the limiter it was built with as a readonly field, and +/// the Enable-MgxResilience handler is not rebuilt when options change - so a limiter released +/// by GetOrCreate or Reset is still the one live traffic goes through. Disposing it on a timer +/// left those sessions throwing ObjectDisposedException minutes after a Set-MgxOption call. +/// +[Collection("Pipeline")] +public class RateLimiterLifetimeTests +{ + /// + /// Longer than the window the old timer waited, so a scheduled disposal has fired by the + /// time the held handler sends. + /// + private static readonly TimeSpan PastTheOldDisposalWindow = TimeSpan.FromSeconds(1.5); + + private static (ResilientDelegatingHandler Handler, System.Threading.RateLimiting.TokenBucketRateLimiter Limiter) + BuildHeldHandler(HttpMessageHandler inner) + { + // TotalTimeoutSeconds was what the old code used as the disposal delay; 1s keeps the + // wait short without changing what is being proven. + var options = new ResilientGraphClientOptions { TotalTimeoutSeconds = 1 }; + var (pipeline, rateLimiter) = ResiliencePipelineFactory.GetOrCreate(options); + Assert.NotNull(rateLimiter); + return (new ResilientDelegatingHandler(pipeline, rateLimiter) { InnerHandler = inner }, rateLimiter); + } + + [Fact] + public async Task A_held_handler_keeps_its_limiter_when_options_change() + { + ResiliencePipelineFactory.Reset(); + try + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + var (handler, limiter) = BuildHeldHandler(wire); + using var held = handler; + using var httpClient = new HttpClient(held); + + // Set-MgxOption builds a fresh options instance, so GetOrCreate rebuilds. The + // old code read the disposal delay off the INCOMING options, so the replacement + // has to carry the same short timeout or the wait below outlasts nothing. + ResiliencePipelineFactory.GetOrCreate( + new ResilientGraphClientOptions { RateLimitPerSecond = 10, TotalTimeoutSeconds = 1 }); + await Task.Delay(PastTheOldDisposalWindow); + + var response = await httpClient.GetAsync("https://graph.microsoft.com/v1.0/users/user1"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(1, wire.RequestCount); + // The lease path above already went through the limiter; reading its statistics + // fails on a disposed instance and says so more precisely than the send does. + Assert.NotNull(limiter.GetStatistics()); + } + finally { ResiliencePipelineFactory.Reset(); } + } + + [Fact] + public async Task A_held_handler_keeps_its_limiter_across_a_reset() + { + ResiliencePipelineFactory.Reset(); + try + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + var (handler, limiter) = BuildHeldHandler(wire); + using var held = handler; + using var httpClient = new HttpClient(held); + + // A tenant change drops the cached pipeline and limiter outright. + ResiliencePipelineFactory.Reset(); + await Task.Delay(PastTheOldDisposalWindow); + + var response = await httpClient.GetAsync("https://graph.microsoft.com/v1.0/users/user1"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(1, wire.RequestCount); + Assert.NotNull(limiter.GetStatistics()); + } + finally { ResiliencePipelineFactory.Reset(); } + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Http/ResilienceBridgeTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/ResilienceBridgeTests.cs index 2eb898c..e1dd027 100644 --- a/tests/Mgx.IntegrationTests/Engine/Http/ResilienceBridgeTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Http/ResilienceBridgeTests.cs @@ -246,4 +246,140 @@ protected override Task SendAsync( }); } } + + [Fact] + public async Task Bridge_stamps_one_correlation_id_per_logical_request() + { + ResiliencePipelineFactory.Reset(); + var wire = new MockHttpHandler(); + wire.QueueFailuresThenSuccess(1, HttpStatusCode.ServiceUnavailable, "{}"); + + var (pipeline, _) = ResiliencePipelineFactory.GetOrCreate( + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 3 }); + using var handler = new ResilientDelegatingHandler(pipeline, null) { InnerHandler = wire }; + using var client = new HttpClient(handler); + + var response = await client.SendAsync( + new HttpRequestMessage(HttpMethod.Get, "https://graph.microsoft.com/v1.0/users")); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + var captured = wire.CapturedRequests; + Assert.Equal(2, captured.Count); + var first = Assert.Single(Assert.Contains("client-request-id", captured[0].Headers)); + var second = Assert.Single(Assert.Contains("client-request-id", captured[1].Headers)); + Assert.Equal(first, second); + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task A_known_oversized_body_passes_to_the_inner_chain_untouched() + { + // The pipeline cannot replay what it will not buffer, so it does not manage it: + // no clone, no retry stamp, no per-attempt timeout - the inner chain sees the + // ORIGINAL request exactly as the SDK built it. + ResiliencePipelineFactory.Reset(); + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "{}"); + + var (pipeline, _) = ResiliencePipelineFactory.GetOrCreate( + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 3 }); + using var handler = new ResilientDelegatingHandler(pipeline, null) { InnerHandler = wire }; + using var client = new HttpClient(handler); + + var big = new byte[5 * 1024 * 1024]; + var request = new HttpRequestMessage(HttpMethod.Put, "https://graph.microsoft.com/v1.0/drives/d/items/i/content") + { + Content = new ByteArrayContent(big), + }; + var response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + var sent = Assert.Single(wire.CapturedRequests); + Assert.Equal(big.Length, sent.Body!.Length); + // Untouched: none of the wrap's stamps. + Assert.DoesNotContain("SdkVersion", sent.Headers); + Assert.DoesNotContain("client-request-id", sent.Headers); + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task The_wrap_adds_no_retries_to_a_known_oversized_body() + { + // A 429 comes back as-is from the wrap's side; whether it retries is the inner + // SDK chain's decision, exactly as without Enable-MgxResilience. (The mock IS + // the inner chain here, so one request means the wrap added none.) + ResiliencePipelineFactory.Reset(); + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.TooManyRequests); + wire.QueueResponse(HttpStatusCode.OK, "{}"); + + var (pipeline, _) = ResiliencePipelineFactory.GetOrCreate( + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 3 }); + using var handler = new ResilientDelegatingHandler(pipeline, null) { InnerHandler = wire }; + using var client = new HttpClient(handler); + + var request = new HttpRequestMessage(HttpMethod.Put, "https://graph.microsoft.com/v1.0/drives/d/items/i/content") + { + Content = new ByteArrayContent(new byte[5 * 1024 * 1024]), + }; + var response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + Assert.Equal(1, wire.RequestCount); + ResiliencePipelineFactory.Reset(); + } + + [Fact] + public async Task An_unknown_length_body_keeps_full_retry_protection_whatever_its_size() + { + // No declared length means the read is unavoidable; once buffered, it replays. + ResiliencePipelineFactory.Reset(); + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.ServiceUnavailable); + wire.QueueResponse(HttpStatusCode.OK, "{}"); + + var (pipeline, _) = ResiliencePipelineFactory.GetOrCreate( + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 3 }); + using var handler = new ResilientDelegatingHandler(pipeline, null) { InnerHandler = wire }; + using var client = new HttpClient(handler); + + var big = new byte[5 * 1024 * 1024]; + var chunked = new StreamContent(new NonSeekableStream(big)); + var request = new HttpRequestMessage(HttpMethod.Put, "https://graph.microsoft.com/v1.0/drives/d/items/i/content") + { + Content = chunked, + }; + var response = await client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(2, wire.RequestCount); + Assert.All(wire.CapturedRequests, r => Assert.Equal(big.Length, r.Body!.Length)); + ResiliencePipelineFactory.Reset(); + } + + private sealed class NonSeekableStream(byte[] data) : MemoryStream(data) + { + public override bool CanSeek => false; + } + + [Fact] + public async Task Bridge_keeps_the_sdk_correlation_id_when_one_is_already_set() + { + ResiliencePipelineFactory.Reset(); + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, "{}"); + + var (pipeline, _) = ResiliencePipelineFactory.GetOrCreate( + new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 3 }); + using var handler = new ResilientDelegatingHandler(pipeline, null) { InnerHandler = wire }; + using var client = new HttpClient(handler); + + var request = new HttpRequestMessage(HttpMethod.Get, "https://graph.microsoft.com/v1.0/users"); + request.Headers.TryAddWithoutValidation("client-request-id", "sdk-set-id"); + await client.SendAsync(request); + + var sent = Assert.Single(wire.CapturedRequests); + Assert.Equal("sdk-set-id", Assert.Single(Assert.Contains("client-request-id", sent.Headers))); + ResiliencePipelineFactory.Reset(); + } } diff --git a/tests/Mgx.IntegrationTests/Engine/Http/RetryTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/RetryTests.cs index 6eabc13..a5b29c9 100644 --- a/tests/Mgx.IntegrationTests/Engine/Http/RetryTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Http/RetryTests.cs @@ -1,8 +1,11 @@ using System.Diagnostics; using System.Net; using System.Text; +using Mgx.Engine.Errors; using Mgx.Engine.Http; +using Mgx.Engine.Pagination; using Mgx.IntegrationTests.Fakes; +using Polly.Timeout; namespace Mgx.IntegrationTests; @@ -721,6 +724,396 @@ public async Task ResilientDelegatingHandler_AlwaysClonesRequest() ResiliencePipelineFactory.Reset(); } + /// + /// A keyed entry answers the request its predicate picks out and no other, so a fault sits + /// at a point in the operation instead of at a position in the send order. Everything the + /// predicate misses falls through to the default. + /// + [Fact] + public async Task Mock_KeyedResponse_AnswersOnlyTheMatchingRequest() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + handler.When(r => r.Uri.EndsWith("/users/user2", StringComparison.Ordinal)) + .Respond(HttpStatusCode.NotFound); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + + foreach (var id in new[] { "user1", "user2", "user3" }) + (await client.GetAsync($"https://graph.microsoft.com/v1.0/users/{id}")).Dispose(); + + Assert.Equal( + new[] { HttpStatusCode.OK, HttpStatusCode.NotFound, HttpStatusCode.OK }, + handler.ServedStatusCodes); + Assert.Equal(3, handler.RequestCount); + } + + /// + /// The attempt index counts the requests matching one entry's predicate, so a fault scoped + /// to the second leaves the first alone, and the retry the fault provokes falls through to + /// whatever answers next. + /// + [Fact] + public async Task Mock_KeyedResponse_ScopedToAttempt2_LeavesAttempt1Alone() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + handler.When(r => r.Uri.EndsWith("/users/user1", StringComparison.Ordinal)) + .OnAttempt(2) + .Respond((HttpStatusCode)429, null, new() { ["Retry-After"] = "0" }); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + + using var first = await client.GetAsync("https://graph.microsoft.com/v1.0/users/user1"); + Assert.Equal(HttpStatusCode.OK, first.StatusCode); + Assert.Equal(1, handler.RequestCount); + + using var second = await client.GetAsync("https://graph.microsoft.com/v1.0/users/user1"); + Assert.Equal(HttpStatusCode.OK, second.StatusCode); + + Assert.Equal( + new[] { HttpStatusCode.OK, (HttpStatusCode)429, HttpStatusCode.OK }, + handler.ServedStatusCodes); + } + + /// + /// A body that delivers a few bytes and then stops. The class is named rather than compared + /// against another exception of the same type, which no classification of a stalled body + /// could differ from: the contract is TransientTransport, the class a connection failure + /// carries, so the pipeline may retry an idempotent request and nothing reads the stall as + /// the server's final answer. + /// + [Fact] + public async Task Mock_StalledBody_SurfacesAsTheTransportClassTheClassifierAssigns() + { + var handler = new MockHttpHandler(); + handler.When(r => r.Uri.EndsWith("/users", StringComparison.Ordinal)) + .RespondStalled(HttpStatusCode.OK, prefix: """{"value":["""); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }) + { + BodyReadTimeout = TimeSpan.FromSeconds(2) + }; + + using var response = await client.GetAsync("https://graph.microsoft.com/v1.0/users"); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + var thrown = await Assert.ThrowsAsync( + () => client.ReadBodyAsStringAsync(response)); + Assert.Equal(ResilientGraphClient.BodyReadTimedOutMessage, thrown.Message); + + var stalled = MgxErrorClassifier.Classify(thrown, cancellationRequested: false); + Assert.Equal(MgxErrorClass.TransientTransport, stalled.Class); + } + + /// + /// An answer held past the attempt timeout never reaches the caller, so nothing is recorded + /// as served for it. Both attempts time out and the served list stays empty, which is what + /// makes it usable as evidence that a fault landed. + /// + [Fact] + public async Task Mock_HeldAnswer_TimedOutByTheAttempt_RecordsNothingAsServed() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + handler.When(r => r.Uri.EndsWith("/users/slow", StringComparison.Ordinal)) + .AfterDelay(TimeSpan.FromSeconds(5)) + .Respond(HttpStatusCode.OK, TestData.SingleUser); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions + { + NoRateLimit = true, + AttemptTimeoutSeconds = 1, + TotalTimeoutSeconds = 30, + MaxRetryAttempts = 1 + }); + + var thrown = await Record.ExceptionAsync( + () => client.GetAsync("https://graph.microsoft.com/v1.0/users/slow")); + + Assert.IsType(thrown); + Assert.Equal(2, handler.RequestCount); + Assert.Empty(handler.ServedStatusCodes); + Assert.Empty(handler.UnansweredRules); + } + + /// + /// Two answers, one held and one immediate, are recorded in the order the handler handed + /// them back rather than the order it chose them: the held one resolved first and arrives + /// last. + /// + [Fact] + public async Task Mock_HeldAnswer_IsRecordedWhenItIsHandedBack() + { + var handler = new MockHttpHandler(); + handler.When(r => r.Uri.EndsWith("/a", StringComparison.Ordinal)) + .AfterDelay(TimeSpan.FromSeconds(2)) + .Respond(HttpStatusCode.OK, """{"which":"a"}"""); + handler.When(r => r.Uri.EndsWith("/b", StringComparison.Ordinal)) + .Respond(HttpStatusCode.Accepted, """{"which":"b"}"""); + + using var httpClient = new HttpClient(handler); + var held = httpClient.GetAsync("https://graph.microsoft.com/v1.0/a"); + await Task.Delay(TimeSpan.FromMilliseconds(300)); + using var immediate = await httpClient.GetAsync("https://graph.microsoft.com/v1.0/b"); + using var released = await held; + + Assert.Equal(HttpStatusCode.OK, released.StatusCode); + Assert.Equal(HttpStatusCode.Accepted, immediate.StatusCode); + Assert.Equal( + new[] { HttpStatusCode.Accepted, HttpStatusCode.OK }, + handler.ServedStatusCodes); + } + + /// + /// A fault programmed after a generator that also matches its request never answers: the + /// first matching entry wins, the enumeration completes, every item arrives, and the run + /// looks exactly like one with no fault in it. The unanswered view is the only thing that + /// says so - the shadowed entry's own match counter still advanced. Programmed the other + /// way round the same fault lands. + /// + [Fact] + public async Task Mock_ShadowedFault_IsNamedInUnansweredRules() + { + const string FaultName = "page 3 fails once"; + var collection = new PagedCollection(total: 6, pageSize: 2); + + var shadowed = new MockHttpHandler(); + collection.ServeOn(shadowed); + ArmPageThreeFault(shadowed, FaultName); + + var injected = new MockHttpHandler(); + ArmPageThreeFault(injected, FaultName); + collection.ServeOn(injected); + + var expected = new[] { "user1", "user2", "user3", "user4", "user5", "user6" }; + Assert.Equal(expected, await EnumerateAsync(shadowed, collection)); + Assert.Equal(expected, await EnumerateAsync(injected, collection)); + + Assert.DoesNotContain(HttpStatusCode.ServiceUnavailable, shadowed.ServedStatusCodes); + Assert.Contains(HttpStatusCode.ServiceUnavailable, injected.ServedStatusCodes); + + Assert.Equal(new[] { FaultName }, shadowed.UnansweredRules); + Assert.Empty(injected.UnansweredRules); + + var shadowedFault = shadowed.ProgrammedRules.Single(rule => rule.Name == FaultName); + Assert.Equal(1, shadowedFault.Matches); + Assert.Equal(0, shadowedFault.Answers); + } + + private static void ArmPageThreeFault(MockHttpHandler handler, string name) => + handler.When(r => r.Uri.Contains("$skiptoken=page3", StringComparison.Ordinal), name) + .OnAttempt(1) + .Respond(HttpStatusCode.ServiceUnavailable, null, new() { ["Retry-After"] = "0" }); + + private static async Task> EnumerateAsync( + MockHttpHandler handler, PagedCollection collection) + { + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions + { + NoRateLimit = true, + AttemptTimeoutSeconds = 5, + TotalTimeoutSeconds = 60, + MaxRetryAttempts = 2 + }); + + var ids = new List(); + await foreach (var item in new PageIterator(client) + .StreamAllWithCountAsync(collection.InitialUrl, 0, null)) + { + ids.Add(item.GetProperty("id").GetString()!); + } + return ids; + } + + /// + /// The attempt index counts the requests that matched one entry, not the ones it answered. + /// The first request is answered by the first entry; the second entry counted that request + /// too, which is what puts its own fault on the second. + /// + [Fact] + public async Task Mock_AttemptIndex_CountsMatchesPerEntry_NotAnswers() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + handler.When(r => r.Uri.EndsWith("/users/user1", StringComparison.Ordinal), "first request") + .OnAttempt(1) + .Respond(HttpStatusCode.NotFound); + handler.When(r => r.Uri.EndsWith("/users/user1", StringComparison.Ordinal), "second request") + .OnAttempt(2) + .Respond(HttpStatusCode.Conflict); + + // The bare client: a 404 and a 409 are answers, and nothing here should retry them. + using var httpClient = new HttpClient(handler); + for (var i = 0; i < 2; i++) + (await httpClient.GetAsync("https://graph.microsoft.com/v1.0/users/user1")).Dispose(); + + Assert.Equal( + new[] { HttpStatusCode.NotFound, HttpStatusCode.Conflict }, + handler.ServedStatusCodes); + Assert.Empty(handler.UnansweredRules); + Assert.Equal(new[] { 2, 2 }, handler.ProgrammedRules.Select(rule => rule.Matches)); + Assert.Equal(new[] { 1, 1 }, handler.ProgrammedRules.Select(rule => rule.Answers)); + } + + /// + /// The attempt index over the retries of one request: the queue answers the first attempt, + /// the keyed entry the retry it provoked, and the retry after that falls through to the + /// default. One request, three attempts, three different answers. + /// + [Fact] + public async Task Mock_KeyedResponse_ScopedToAttempt2_AnswersTheRetryOfOneRequest() + { + var handler = new MockHttpHandler(); + handler.SetDefaultResponse(HttpStatusCode.OK, TestData.SingleUser); + handler.QueueResponse(HttpStatusCode.ServiceUnavailable); + handler.When(r => r.Uri.EndsWith("/users/user1", StringComparison.Ordinal)) + .OnAttempt(2) + .Respond((HttpStatusCode)429, null, new() { ["Retry-After"] = "0" }); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions + { + NoRateLimit = true, + AttemptTimeoutSeconds = 5, + TotalTimeoutSeconds = 60, + MaxRetryAttempts = 4 + }); + + using var response = await client.GetAsync("https://graph.microsoft.com/v1.0/users/user1"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(3, handler.RequestCount); + Assert.All(handler.CapturedRequests, r => Assert.EndsWith("/users/user1", r.Uri, StringComparison.Ordinal)); + Assert.Equal( + new[] { HttpStatusCode.ServiceUnavailable, (HttpStatusCode)429, HttpStatusCode.OK }, + handler.ServedStatusCodes); + Assert.Empty(handler.UnansweredRules); + } + + /// A byte body comes back as programmed, content type and all. + [Fact] + public async Task Mock_RespondBytes_HandsBackTheBytesAndTheContentType() + { + var payload = new byte[] { 0x00, 0x01, 0xFE, 0xFF }; + var handler = new MockHttpHandler(); + handler.When(r => r.Uri.EndsWith("/content", StringComparison.Ordinal)) + .RespondBytes(HttpStatusCode.OK, payload, "application/octet-stream"); + + using var httpClient = new HttpClient(handler); + using var response = await httpClient.GetAsync("https://graph.microsoft.com/v1.0/content"); + + Assert.Equal(payload, await response.Content.ReadAsByteArrayAsync()); + Assert.Equal("application/octet-stream", response.Content.Headers.ContentType?.ToString()); + Assert.Equal(new[] { HttpStatusCode.OK }, handler.ServedStatusCodes); + } + + /// A status with no body at all, and the headers still arrive. + [Fact] + public async Task Mock_RespondEmpty_HandsBackAStatusAndNoBody() + { + var handler = new MockHttpHandler(); + handler.When(r => r.Method == HttpMethod.Delete) + .RespondEmpty(HttpStatusCode.NoContent, new() { ["request-id"] = "empty-1" }); + + using var httpClient = new HttpClient(handler); + using var response = await httpClient.DeleteAsync("https://graph.microsoft.com/v1.0/users/user1"); + + Assert.Equal(HttpStatusCode.NoContent, response.StatusCode); + Assert.Empty(await response.Content.ReadAsByteArrayAsync()); + Assert.Equal("empty-1", Assert.Single(response.Headers.GetValues("request-id"))); + Assert.Equal(new[] { HttpStatusCode.NoContent }, handler.ServedStatusCodes); + } + + /// + /// A thrown answer reaches the caller as itself and records no status: there was none. + /// + [Fact] + public async Task Mock_Throw_SurfacesTheExceptionAndRecordsNothingAsServed() + { + var handler = new MockHttpHandler(); + handler.When(r => r.Uri.EndsWith("/users", StringComparison.Ordinal)) + .Throw(new HttpRequestException("connection reset")); + + using var httpClient = new HttpClient(handler); + var thrown = await Assert.ThrowsAsync( + () => httpClient.GetAsync("https://graph.microsoft.com/v1.0/users")); + + Assert.Equal("connection reset", thrown.Message); + Assert.Equal(1, handler.RequestCount); + Assert.Empty(handler.ServedStatusCodes); + Assert.Empty(handler.UnansweredRules); + } + + /// + /// A computed answer: one entry answers every request in the collection, each from the + /// request itself, and the status it computed is what gets recorded. + /// + [Fact] + public async Task Mock_FactoryAnswer_ComputesTheResponseAndRecordsItsStatus() + { + var handler = new MockHttpHandler(); + handler.When(r => r.Uri.Contains("/users/", StringComparison.Ordinal)) + .Respond(r => + { + var id = r.Uri[(r.Uri.LastIndexOf('/') + 1)..]; + return new HttpResponseMessage( + id == "user2" ? HttpStatusCode.NotFound : HttpStatusCode.OK) + { + Content = new StringContent( + $$"""{"id":"{{id}}"}""", Encoding.UTF8, "application/json") + }; + }); + + using var httpClient = new HttpClient(handler); + var bodies = new List(); + foreach (var id in new[] { "user1", "user2" }) + { + using var response = await httpClient.GetAsync($"https://graph.microsoft.com/v1.0/users/{id}"); + bodies.Add(await response.Content.ReadAsStringAsync()); + } + + Assert.Equal(new[] { """{"id":"user1"}""", """{"id":"user2"}""" }, bodies); + Assert.Equal( + new[] { HttpStatusCode.OK, HttpStatusCode.NotFound }, + handler.ServedStatusCodes); + } + + /// + /// The scripted handler's keyed path, keyed on the $batch envelope: the predicate reads the + /// request body, so a fault can name the subrequest it is aimed at. The entry answers that + /// envelope and consumes nothing from the script. + /// + [Fact] + public async Task Stub_KeyedResponse_MatchesOnTheBatchBody() + { + var handler = new StubHttpMessageHandler(); + handler.When(r => r.BodyText?.Contains("""{"id":"7"}""", StringComparison.Ordinal) == true, + "subrequest 7 fails") + .Respond(HttpStatusCode.InternalServerError, """{"error":{"code":"aimed"}}"""); + handler.EnqueueJson(HttpStatusCode.OK, """{"responses":[]}"""); + + using var httpClient = new HttpClient(handler); + using var first = await PostBatchAsync(httpClient, """{"requests":[{"id":"1"}]}"""); + using var aimed = await PostBatchAsync(httpClient, """{"requests":[{"id":"7"}]}"""); + + Assert.Equal(HttpStatusCode.OK, first.StatusCode); + Assert.Equal(HttpStatusCode.InternalServerError, aimed.StatusCode); + Assert.Equal("""{"error":{"code":"aimed"}}""", await aimed.Content.ReadAsStringAsync()); + Assert.Equal(2, handler.RequestCount); + Assert.Empty(handler.UnansweredRules); + } + + private static Task PostBatchAsync(HttpClient client, string body) => + client.PostAsync( + "https://graph.microsoft.com/v1.0/$batch", + new StringContent(body, Encoding.UTF8, "application/json")); + private sealed class CapturingMockHandler(List captured) : HttpMessageHandler { protected override Task SendAsync( diff --git a/tests/Mgx.IntegrationTests/Engine/Http/SendStopTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/SendStopTests.cs new file mode 100644 index 0000000..0477799 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/SendStopTests.cs @@ -0,0 +1,233 @@ +using System.Net; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// A caller that has stopped sending - a batch chunk refused while its siblings are still going - +/// hands the transport a stop token. What that stop is allowed to reach is the whole contract: +/// the decision to send again, never the attempt already on the wire, whose answer is the only +/// record of what the server applied. Driven here against the transport directly, so the states +/// the guard arbitrates are named rather than inferred from a chunked batch. +/// +[Collection("Pipeline")] +public class SendStopTests +{ + private static readonly ResilientGraphClientOptions Options = new() + { + NoRateLimit = true, + MaxRetryAttempts = 2, + TotalTimeoutSeconds = 120, + AttemptTimeoutSeconds = 60 + }; + + /// + /// Holds the first POST open until the test says so, and answers it with a status the + /// pipeline would ordinarily retry. The stop is set while that POST is on the wire. + /// + private sealed class HoldsThePostOpenHandler : HttpMessageHandler + { + private readonly TaskCompletionSource _entered = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _release = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _posts; + + public Task Entered => _entered.Task; + public void Release() => _release.TrySetResult(); + public int Posts => Volatile.Read(ref _posts); + + protected override async Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _posts); + _entered.TrySetResult(); + await _release.Task.WaitAsync(TimeSpan.FromSeconds(30), cancellationToken); + return new HttpResponseMessage(HttpStatusCode.TooManyRequests) + { + RequestMessage = request, + Content = new StringContent("{}", System.Text.Encoding.UTF8, "application/json") + }; + } + } + + /// + /// The attempt is on the wire when the stop arrives. Ending it there would trade the answer + /// the server is about to give for a write that may or may not have been applied - the very + /// thing the caller stopped in order to keep. The stop reaches the next attempt instead, + /// which never goes out. + /// + [Fact] + public async Task A_stop_that_lands_on_an_attempt_in_flight_leaves_it_alone() + { + ResiliencePipelineFactory.Reset(); + var wire = new HoldsThePostOpenHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, Options); + using var stop = new CancellationTokenSource(); + + var call = client.PostAsync("https://graph.microsoft.com/v1.0/$batch", + new StringContent("{}"), CancellationToken.None, + paceGate: false, stopRetries: stop.Token); + + await wire.Entered.WaitAsync(TimeSpan.FromSeconds(30)); + stop.Cancel(); + wire.Release(); + + using var response = await call.WaitAsync(TimeSpan.FromSeconds(30)); + + // The answer to the attempt that was flying survives the stop, in full. + Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode); + // A 429 on a POST is the one thing this pipeline retries, and it did not: the stop + // reached the decision to send again. + Assert.Equal(1, wire.Posts); + } + + /// + /// Answers the first POST with a 429 asking for a minute, so the request is unambiguously + /// inside its backoff when the stop arrives. + /// + private sealed class ThrottlesForAMinuteHandler : HttpMessageHandler + { + private readonly TaskCompletionSource _answered = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _posts; + + public Task Answered => _answered.Task; + public int Posts => Volatile.Read(ref _posts); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Interlocked.Increment(ref _posts); + var response = new HttpResponseMessage(HttpStatusCode.TooManyRequests) + { + RequestMessage = request, + Content = new StringContent("{}", System.Text.Encoding.UTF8, "application/json") + }; + response.Headers.RetryAfter = + new System.Net.Http.Headers.RetryConditionHeaderValue(TimeSpan.FromSeconds(60)); + _answered.TrySetResult(); + return Task.FromResult(response); + } + } + + /// + /// Between attempts there is nothing on the wire to protect, and a backoff the caller has + /// stopped honoring is time spent holding the run open for a request it will not send. The + /// wait ends with the stop, and what the server last said is what the request reports - a + /// stopped retry is not a request that got no answer. + /// + [Fact] + public async Task A_stop_between_attempts_ends_the_wait_and_keeps_the_last_status() + { + ResiliencePipelineFactory.Reset(); + var wire = new ThrottlesForAMinuteHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, Options); + using var stop = new CancellationTokenSource(); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + var call = client.PostAsync("https://graph.microsoft.com/v1.0/$batch", + new StringContent("{}"), CancellationToken.None, + paceGate: false, stopRetries: stop.Token); + + await wire.Answered.WaitAsync(TimeSpan.FromSeconds(30)); + // Long enough that the retry decision is behind us and the backoff is running: the + // question is whether a wait already under way ends, not whether one is refused. + await Task.Delay(500); + stop.Cancel(); + + var failure = await Assert.ThrowsAsync( + () => call.WaitAsync(TimeSpan.FromSeconds(30))); + sw.Stop(); + + Assert.Equal(ResilientGraphClient.RetriesStoppedMessage, failure.Message); + // The 429 is what the server last said. A stopped retry that reported no status at all + // would have the caller's items clamped to a generic failure instead. + Assert.Equal(HttpStatusCode.TooManyRequests, failure.StatusCode); + Assert.Equal(1, wire.Posts); + Assert.True(sw.Elapsed < TimeSpan.FromSeconds(30), + $"the call waited out {sw.Elapsed.TotalSeconds:F1}s of a Retry-After it had stopped honoring"); + } + + /// + /// Throttles the first POST with a Retry-After of zero so the retry follows at once, and + /// answers anything after it. Counts every request it is handed without consulting the + /// token: a transport has the request before a cancel reaches it, so a send counted here + /// went out whatever the token said a moment later. + /// + private sealed class CountsEverySendHandler : HttpMessageHandler + { + private int _sends; + + public int Sends => Volatile.Read(ref _sends); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + var n = Interlocked.Increment(ref _sends); + var response = new HttpResponseMessage( + n == 1 ? HttpStatusCode.TooManyRequests : HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent("{}", System.Text.Encoding.UTF8, "application/json") + }; + if (n == 1) + response.Headers.RetryAfter = + new System.Net.Http.Headers.RetryConditionHeaderValue(TimeSpan.Zero); + return Task.FromResult(response); + } + } + + /// + /// The stop arrives after the next attempt has been decided on but before it claims the + /// request - the stretch in which the attempt is being built. Nothing is on the wire to + /// protect there, so the attempt ends where it stands. Reading the stop and sending anyway + /// would leave the caller told that no further attempt was sent over one the server has: + /// the stop sets the state and cancels the link as two steps, and a request handed to the + /// transport in between is already gone when the token lands. + /// + [Fact] + public async Task A_stop_that_lands_before_the_next_attempt_claims_it_keeps_it_off_the_wire() + { + ResiliencePipelineFactory.Reset(); + var wire = new CountsEverySendHandler(); + using var httpClient = new HttpClient(wire); + using var client = new ResilientGraphClient(httpClient, Options); + using var stop = new CancellationTokenSource(); + + var reachedSecondAttempt = + new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + client.AttemptEntryGate = async attempt => + { + if (attempt < 2) return; + reachedSecondAttempt.TrySetResult(); + await release.Task; + }; + + var call = client.PostAsync("https://graph.microsoft.com/v1.0/$batch", + new StringContent("{}"), CancellationToken.None, + paceGate: false, stopRetries: stop.Token); + + await reachedSecondAttempt.Task.WaitAsync(TimeSpan.FromSeconds(30)); + // Cancel runs the registration on this thread, so both halves of the stop - the state + // it takes and the link it cancels - are behind us before the attempt is let go. The + // window is entered on purpose rather than waited for. + stop.Cancel(); + release.SetResult(); + + var outcome = await Record.ExceptionAsync(async () => + { + using var response = await call.WaitAsync(TimeSpan.FromSeconds(30)); + }); + + // The attempt the stop overtook never reached the transport. + Assert.Equal(1, wire.Sends); + var failure = Assert.IsType(outcome); + // ...so the sentence the caller is given about it is true. + Assert.Equal(ResilientGraphClient.RetriesStoppedMessage, failure.Message); + Assert.Equal(HttpStatusCode.TooManyRequests, failure.StatusCode); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Http/TransportSwapTests.cs b/tests/Mgx.IntegrationTests/Engine/Http/TransportSwapTests.cs new file mode 100644 index 0000000..f60f0ef --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Http/TransportSwapTests.cs @@ -0,0 +1,82 @@ +using System.Net; +using System.Reflection; +using System.Text; +using Mgx.Cmdlets.Base; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// A cmdlet builds one ResilientGraphClient over the shared transport and pages through it for +/// the life of its enumeration. Anything that swaps the transport - Connect-MgGraph to another +/// identity, Set-MgxOption, Enable-MgxResilience, Remove-Module - must leave that client able +/// to fetch its next page. The replaced HttpClient used to be disposed TotalTimeoutSeconds +/// later, a per-request timeout used as a resource lifetime, so a long-running operation died +/// mid-stream with ObjectDisposedException. +/// +[Collection("Pipeline")] +public class TransportSwapTests +{ + private sealed class PageHandler : HttpMessageHandler + { + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) => + Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request, + Content = new StringContent("""{"value":[{"id":"u1"}]}""", + Encoding.UTF8, "application/json") + }); + } + + private static readonly BindingFlags Statics = BindingFlags.NonPublic | BindingFlags.Static; + + private static ResilientGraphClientOptions ShortTimeoutOptions() + { + var options = new ResilientGraphClientOptions { NoRateLimit = true, MaxRetryAttempts = 1 }; + // A legal value (the range is 1..3600), set so the old disposal timer would fire + // inside the test rather than five minutes after it. + typeof(ResilientGraphClientOptions) + .GetField("_totalTimeoutSeconds", BindingFlags.Instance | BindingFlags.NonPublic)! + .SetValue(options, 1); + return options; + } + + [Fact] + public async Task A_client_a_running_cmdlet_holds_survives_a_transport_reset() + { + ResiliencePipelineFactory.Reset(); + var t = typeof(MgxCmdletBase); + var options = ShortTimeoutOptions(); + var transport = new HttpClient(new PageHandler()); + t.GetField("s_graphHttpClient", Statics)!.SetValue(null, transport); + t.GetField("s_ownsHttpClient", Statics)!.SetValue(null, true); + t.GetField("s_clientOptions", Statics | BindingFlags.Public)!.SetValue(null, options); + + try + { + // What a cmdlet holds for the whole of its enumeration. + using var held = new ResilientGraphClient(transport, options); + + var page1 = await held.SendAsync(HttpMethod.Get, "https://graph.microsoft.com/v1.0/users"); + Assert.Equal(HttpStatusCode.OK, page1.StatusCode); + + // Another thread swaps the transport out from under it. + MgxCmdletBase.ResetHttpClient(); + await Task.Delay(TimeSpan.FromSeconds(2.5)); + + var page2 = await held.SendAsync(HttpMethod.Get, "https://graph.microsoft.com/v1.0/users?$skiptoken=P2"); + Assert.Equal(HttpStatusCode.OK, page2.StatusCode); + } + finally + { + t.GetField("s_graphHttpClient", Statics)!.SetValue(null, null); + t.GetField("s_ownsHttpClient", Statics)!.SetValue(null, false); + t.GetField("s_clientOptions", Statics | BindingFlags.Public)! + .SetValue(null, ResilientGraphClientOptions.Default); + t.GetField("s_cachedAuthFingerprint", Statics)!.SetValue(null, null); + t.GetField("s_cachedAuthContextRef", Statics)!.SetValue(null, null); + ResiliencePipelineFactory.Reset(); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/AtomicSaveHygieneTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/AtomicSaveHygieneTests.cs index 146f0a0..7088a0b 100644 --- a/tests/Mgx.IntegrationTests/Engine/Pagination/AtomicSaveHygieneTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/AtomicSaveHygieneTests.cs @@ -40,6 +40,280 @@ public void A_failed_checkpoint_save_leaves_no_staging_file() } } +/// +/// The three scratch names the engine writes beside a file it is keeping: the write probe +/// ValidateWriteAccess makes, and the "<path>.tmp" each atomic save stages through. All +/// three were written with an open that creates or truncates, so whatever stood at the name was +/// what the run wrote into: a symlink there took the write into the link's target - a file +/// outside the run's business, truncated or filled with state - and a FIFO with no reader +/// blocked the open for as long as the process lived, in one case inside BeginProcessing before +/// the run had anything a cancellation could unwind. The name is cleared first now, by the same +/// primitive the promotion's staging name goes through, and created by an open that refuses to +/// use anything already there. +/// +[Collection("Pipeline")] +public class ScratchNameHygieneTests +{ + /// Which of the three names a case is about. + public enum Site + { + /// DeltaState.ValidateWriteAccess's "<path>.probe". + WriteProbe, + + /// DeltaState.Save's "<path>.tmp". + DeltaSave, + + /// PaginationCheckpoint.Save's "<path>.tmp". + CheckpointSave, + } + + private static string NewDir() => + Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-scratch-{Guid.NewGuid():N}")).FullName; + + /// + /// The scratch name a site writes through, and the write itself. The paths are the ones the + /// engine derives - Save normalizes before it appends the suffix - so a test cannot name a + /// scratch file the run does not use. + /// + private static (string Scratch, string Target, Action Write) SiteUnderTest(Site site, + string dir) + { + if (site == Site.CheckpointSave) + { + var target = Path.Combine(dir, "run.checkpoint"); + var checkpoint = new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users", + NextLink = "https://graph.microsoft.com/v1.0/users?$skiptoken=P2", + ItemsCollected = 2, + }; + return (Path.GetFullPath(target) + ".tmp", target, () => checkpoint.Save(target)); + } + + var statePath = Path.Combine(dir, "state.json"); + if (site == Site.DeltaSave) + { + var state = new DeltaState + { + DeltaLink = "https://graph.microsoft.com/v1.0/users/delta?$deltatoken=T", + Resource = "/users/delta", + }; + return (Path.GetFullPath(statePath) + ".tmp", statePath, () => state.Save(statePath)); + } + + return (statePath + ".probe", statePath, () => DeltaState.ValidateWriteAccess(statePath)); + } + + private static void Mkfifo(string path) + { + using var mkfifo = System.Diagnostics.Process.Start("/usr/bin/mkfifo", [path]); + Assert.NotNull(mkfifo); + mkfifo.WaitForExit(); + Assert.Equal(0, mkfifo.ExitCode); + } + + /// + /// A symlink at the scratch name. The link comes off the name and the file it pointed at is + /// not touched - which is the whole of what a run can honestly do about a link it did not + /// put there. + /// + [Theory] + [InlineData(Site.WriteProbe)] + [InlineData(Site.DeltaSave)] + [InlineData(Site.CheckpointSave)] + public void A_symlink_at_a_scratch_name_comes_off_it_and_its_target_is_untouched(Site site) + { + var dir = NewDir(); + try + { + var (scratch, target, write) = SiteUnderTest(site, dir); + var victim = Path.Combine(dir, "SOMEBODY-ELSES.txt"); + var kept = "line one\nline two\nline three\n"; + File.WriteAllText(victim, kept); + File.CreateSymbolicLink(scratch, victim); + + write(); + + Assert.Equal(kept, File.ReadAllText(victim)); + Assert.False(File.Exists(scratch), "the scratch name outlived the write"); + if (site != Site.WriteProbe) + Assert.True(File.Exists(target), "the file the save was keeping is not there"); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// A FIFO at the scratch name. Nothing is on the other end of it, so an open that waits for + /// a peer waits for as long as the process lives - past SIGINT, measured - and the write + /// runs on a background thread here so a regression cannot take the suite with it. + /// + [Theory] + [InlineData(Site.WriteProbe)] + [InlineData(Site.DeltaSave)] + [InlineData(Site.CheckpointSave)] + public void A_fifo_at_a_scratch_name_comes_off_it_without_wedging_the_write(Site site) + { + if (OperatingSystem.IsWindows()) return; + var dir = NewDir(); + try + { + var (scratch, target, write) = SiteUnderTest(site, dir); + Mkfifo(scratch); + + Exception? failed = null; + var writer = new Thread(() => + { + try { write(); } + catch (Exception ex) { failed = ex; } + }) + { + // A thread parked in an open cannot be interrupted, so it must not be one the + // process waits for on the way out. + IsBackground = true, + }; + writer.Start(); + + Assert.True(writer.Join(TimeSpan.FromSeconds(20)), + "the write is still waiting for a reader on the pipe at its scratch name"); + Assert.Null(failed); + Assert.False(File.Exists(scratch), "the pipe outlived the write"); + if (site != Site.WriteProbe) + Assert.True(File.Exists(target), "the file the save was keeping is not there"); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// Taking a scratch name back, under the handle that holds it. Released in front of the + /// delete, the name is decided and unheld for as long as the discard takes, and a second run + /// that sweeps it in that interval creates its own file at the name and has it unlinked from + /// under the handle it was holding it by. The entry goes and the handle goes with it. + /// + [Fact] + public void Discarding_a_scratch_name_takes_the_entry_and_the_handle_with_it() + { + var dir = NewDir(); + try + { + var path = Path.Combine(dir, "run.checkpoint.tmp"); + var held = new FileStream(path, FileMode.CreateNew, FileAccess.ReadWrite, + FileShare.None); + + ScratchName.Discard(path, held); + + Assert.False(File.Exists(path), "the entry outlived the discard"); + Assert.Throws(() => held.Length); + + // And a name nothing stands at is one a discard has nothing to say about. + ScratchName.Discard(path, null); + Assert.False(File.Exists(path)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// Two runs over one -DeltaPath, or one -OutputFile, reach the probe together as a matter of + /// course, and the name they both use is the same one. A file the other run holds there is + /// its own probe - taken out a moment ago, in this very directory, which is the question + /// this probe is asking - so it answers yes and leaves the file alone. Refused for the + /// other run's sake, the probe ended a run in BeginProcessing over nothing that was wrong + /// with the path. + /// + [Fact] + public void A_probe_another_run_is_holding_answers_the_question_it_asks() + { + var dir = NewDir(); + try + { + var (scratch, _, write) = SiteUnderTest(Site.WriteProbe, dir); + using var theirs = new FileStream(scratch, FileMode.CreateNew, FileAccess.ReadWrite, + FileShare.None); + + write(); + + Assert.True(File.Exists(scratch), "a live run's own probe was taken off the name"); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// Runs probing one path together. The probe name is derived from the path, so two runs over + /// one -DeltaPath - or two over one -OutputFile - meet at it as a matter of course, and the + /// probe is written to be passed by both: a name another run is holding answers the question + /// this one came to ask. The refusal a create came back with was read off a second look at + /// the name instead, which arrives after the run that made the entry has taken it away - so + /// the second run saw a clear name, called the create's refusal the path's own, and ended in + /// BeginProcessing over a path nothing was wrong with. + /// + /// Windows has a second way to refuse them, and the run is not allowed to stop on that one + /// either: a delete of the name that has been started and has not finished leaves the entry + /// there with every open refused, so the next create - and the next sweep's unlink - meets + /// ERROR_ACCESS_DENIED, which is also what a directory this account cannot write answers. + /// Both are waited out and only a refusal that outlasts the wait is the path's own. At this + /// density the state is reached constantly: over 40 runs of this shape it refused 154 creates + /// and 64 unlinks, and every run of the 40 ended in BeginProcessing over a path nothing was + /// wrong with. + /// + /// + [Fact] + public void Runs_probing_one_path_together_do_not_refuse_each_other() + { + var dir = NewDir(); + try + { + var (_, _, write) = SiteUnderTest(Site.WriteProbe, dir); + var failures = new Exception?[4]; + var threads = new Thread[failures.Length]; + using var go = new ManualResetEventSlim(false); + for (var i = 0; i < threads.Length; i++) + { + var slot = i; + threads[slot] = new Thread(() => + { + go.Wait(TimeSpan.FromSeconds(30)); + for (var n = 0; n < 200 && failures[slot] == null; n++) + { + try { write(); } + catch (Exception ex) { failures[slot] = ex; } + } + }); + threads[slot].Start(); + } + go.Set(); + foreach (var t in threads) + Assert.True(t.Join(TimeSpan.FromSeconds(60)), "a probing run never finished"); + + Assert.All(failures, f => Assert.Null(f)); + Assert.Empty(Directory.GetFiles(dir, "*.probe")); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// A directory at the scratch name is not the run's to delete and could hold anything, so + /// the write says so instead of failing on an open that cannot say what refused it. The + /// probe's refusal is the write-access error it has always been, naming the scratch file and + /// what stood there. + /// + [Fact] + public void A_directory_at_the_probe_name_is_the_write_access_refusal() + { + var dir = NewDir(); + try + { + var (scratch, target, write) = SiteUnderTest(Site.WriteProbe, dir); + Directory.CreateDirectory(scratch); + + var stop = Assert.Throws(write); + Assert.Contains($"Cannot write to delta state path '{target}'", stop.Message); + Assert.Contains($"a directory stands at '{Path.GetFileName(scratch)}'", stop.Message); + Assert.True(Directory.Exists(scratch), "the refusal deleted the directory"); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } +} + /// /// A delta state file from before this field existed carries no apiVersion, so the mismatch /// check is skipped and the run proceeds against whatever version the stored deltaLink names. diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/BoundarySizeTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/BoundarySizeTests.cs new file mode 100644 index 0000000..4ae05f3 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/BoundarySizeTests.cs @@ -0,0 +1,259 @@ +using System.Management.Automation; +using System.Net; +using System.Text; +using System.Text.Json; +using Mgx.Engine.Http; +using Mgx.Engine.Pagination; +using Mgx.IntegrationTests.Fakes; + +namespace Mgx.IntegrationTests; + +/// +/// Collection sizes that sit on a page seam: exactly full, one over, and the 999/1000/1001 +/// band around Graph's largest page. An off-by-one in the follow-the-nextLink loop drops the +/// last page or the last item, and only these sizes expose it. +/// (Corpus: M365DSC-7274, silent truncation.) +/// +[Collection("Pipeline")] +public class BoundarySizeTests +{ + /// One page of items, with a nextLink unless it is last. + private static string Page(int firstIndex, int count, int pageNumber, bool isLast) + { + var json = new StringBuilder("""{"value":["""); + for (var i = 0; i < count; i++) + { + if (i > 0) json.Append(','); + json.Append($$"""{"id":"user{{firstIndex + i}}"}"""); + } + json.Append(']'); + if (!isLast) + { + json.Append(""","@odata.nextLink":"https://graph.microsoft.com/v1.0/users?$skiptoken=page"""); + json.Append(pageNumber + 1).Append('"'); + } + json.Append('}'); + return json.ToString(); + } + + /// + /// What a complete enumeration means, whatever armed the wire: every item, each exactly + /// once, from the first id to the last, over exactly the pages the nextLink chain describes. + /// Count alone does not say it - a loop that re-fetches a page it has already read arrives + /// at the right total whenever the last page is short by as much as it repeated. + /// + private static async Task AssertWholeCollection( + MockHttpHandler handler, string initialUrl, int total, int expectedPages) + { + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var iterator = new PageIterator(client); + + var ids = new List(total); + await foreach (var item in iterator.StreamAllWithCountAsync(initialUrl, maxItems: 0, onCount: null)) + { + ids.Add(item.GetProperty("id").GetString()!); + } + + Assert.Equal(total, ids.Count); + Assert.Distinct(ids); + Assert.Equal("user1", ids[0]); + Assert.Equal($"user{total}", ids[^1]); + Assert.Equal(expectedPages, handler.RequestCount); + Assert.All(handler.Requests.Skip(1).Select((r, i) => (r, i)), + x => Assert.Equal($"https://graph.microsoft.com/v1.0/users?$skiptoken=page{x.i + 2}", + x.r.RequestUri!.ToString())); + } + + /// + /// is the shape Graph actually returns when the + /// collection divides evenly: the last full page still carries a nextLink, and following + /// it yields an empty page. Without it a full page is simply the end, which is a + /// single-request enumeration that crosses no seam at all. + /// + [Theory] + [InlineData(100, 100, false)] // exactly one full page, ended by its own absent nextLink + [InlineData(100, 100, true)] // the same size, ended by an empty page behind a nextLink + [InlineData(101, 100, false)] // one item spills into a second page + [InlineData(999, 999, false)] // exactly Graph's largest page + [InlineData(999, 999, true)] // and the same, with the empty page Graph appends + [InlineData(1000, 999, false)] // one item past it + [InlineData(1001, 999, false)] // two items past it + public async Task Every_item_comes_back_at_a_page_boundary_size(int total, int pageSize, bool trailingEmptyPage) + { + var handler = new MockHttpHandler(); + var expectedPages = 0; + for (var sent = 0; sent < total; sent += pageSize) + { + var count = Math.Min(pageSize, total - sent); + handler.QueueResponse(HttpStatusCode.OK, + Page(sent + 1, count, expectedPages + 1, isLast: sent + count == total && !trailingEmptyPage)); + expectedPages++; + } + if (trailingEmptyPage) + { + handler.QueueResponse(HttpStatusCode.OK, Page(total + 1, 0, expectedPages + 1, isLast: true)); + expectedPages++; + } + + // MockHttpHandler answers in queue order regardless of URL, so the counts hold even for + // an iterator that re-sent the initial URL. The URL assertion is what pins the chain + // here; the generated cases below key the answer on the URL instead, so there a repeat + // hands back a page already seen and the distinctness assertion is what catches it. + await AssertWholeCollection(handler, + $"https://graph.microsoft.com/v1.0/users?$top={pageSize}", total, expectedPages); + } + + /// + /// The same seams over pages computed from the request rather than queued in advance, which + /// is what lets 100,000 items be a size like any other: the queue would need 101 responses + /// built before the first one is sent. + /// + [Theory] + [InlineData(100, 100, false)] + [InlineData(100, 100, true)] + [InlineData(101, 100, false)] + [InlineData(999, 999, false)] + [InlineData(999, 999, true)] + [InlineData(1000, 999, false)] + [InlineData(1001, 999, false)] + [InlineData(100_000, 999, false)] // 101 pages, the size no queue can hold + public async Task Every_item_comes_back_from_a_generated_collection(int total, int pageSize, bool trailingEmptyPage) + { + var pages = new PagedCollection(total, pageSize, trailingEmptyPage: trailingEmptyPage); + var handler = new MockHttpHandler(); + pages.ServeOn(handler); + + await AssertWholeCollection(handler, pages.InitialUrl, total, pages.PageCount); + } + + /// + /// One seam over the scripted handler as well. The generator is programmed on both of the + /// suite's handlers, and the scripted one resolves a keyed entry ahead of its own queue - + /// which it has none of here, so a page it did not key would be a request with nothing to + /// answer it rather than a page quietly repeated. + /// + [Fact] + public async Task Every_item_comes_back_from_a_generated_collection_over_the_scripted_handler() + { + const int total = 1001; + var pages = new PagedCollection(total, pageSize: 999); + var handler = new StubHttpMessageHandler(); + pages.ServeOn(handler); + + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, new ResilientGraphClientOptions { NoRateLimit = true }); + var iterator = new PageIterator(client); + + var ids = new List(total); + await foreach (var item in iterator.StreamAllWithCountAsync(pages.InitialUrl, maxItems: 0, onCount: null)) + { + ids.Add(item.GetProperty("id").GetString()!); + } + + Assert.Equal(total, ids.Count); + Assert.Distinct(ids); + Assert.Equal("user1", ids[0]); + Assert.Equal($"user{total}", ids[^1]); + Assert.Equal(pages.PageCount, handler.RequestCount); + Assert.All(handler.Requests.Skip(1).Select((r, i) => (r, i)), + x => Assert.Equal(pages.NextLinkTo(x.i + 2), x.r.Uri)); + } + + /// + /// One pass of the largest size through the cmdlet, because the iterator's guarantee is not + /// the caller's: Export-MgxCollection writes each item to a temp as it arrives and moves + /// that over the output at the end, and a line written twice or a page lost between the + /// iterator and the file is invisible to every assertion above. -PageSize is left at its + /// default, so the first request also states what that default is worth. + /// + [Fact] + public void An_export_of_a_hundred_thousand_items_writes_each_one_once() + { + const int total = 100_000; + var pages = new PagedCollection(total, pageSize: 999); + var handler = new MockHttpHandler(); + pages.ServeOn(handler); + + using var transport = MgxTransportScope.Inject(handler); + var dir = Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-boundary-{Guid.NewGuid():N}")).FullName; + var output = Path.Combine(dir, "out.jsonl"); + try + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Export-MgxCollection") + .AddParameter("Uri", "/users") + .AddParameter("OutputFile", output) + .AddParameter("All") + .AddParameter("WarningAction", ActionPreference.SilentlyContinue); + ps.Invoke(); + + // Before the file is read at all: an export that wrote an error record produced a + // file that says nothing about what the run was asked to do, and counting its lines + // would report on the wrong thing. + Assert.Empty(ps.Streams.Error.Select(e => e.FullyQualifiedErrorId)); + + var seen = new HashSet(total); + var lines = 0; + foreach (var line in File.ReadLines(output)) + { + lines++; + using var item = JsonDocument.Parse(line); + var id = item.RootElement.GetProperty("id").GetString(); + Assert.True(seen.Add(id!), $"line {lines} repeats id {id}"); + } + + Assert.Equal(total, lines); + Assert.Equal(pages.PageCount, handler.RequestCount); + Assert.Equal("https://graph.microsoft.com/v1.0/users?$top=999", + handler.CapturedRequests[0].Uri); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + } + } +} + +/// +/// The generator's own guards. It stands in for Graph at a page seam, so a shape the service +/// never returns has to be refused rather than served, and a page past the end of the collection +/// is the defect the boundary tests exist to catch rather than something to answer. +/// +public class PagedCollectionTests +{ + [Fact] + public void A_trailing_empty_page_is_refused_on_a_total_that_does_not_divide_evenly() + { + // 1001 items at 999 a page end on a short page of two. Graph's empty page comes behind + // a full one; behind a short one it is a nextLink no collection ever carries. + var refused = Assert.Throws( + () => new PagedCollection(1001, pageSize: 999, trailingEmptyPage: true)); + Assert.Equal("trailingEmptyPage", refused.ParamName); + + // The evenly dividing sizes the boundary theory passes keep their empty page. + Assert.Equal(2, new PagedCollection(999, pageSize: 999, trailingEmptyPage: true).PageCount); + Assert.Equal(3, new PagedCollection(1998, pageSize: 999, trailingEmptyPage: true).PageCount); + } + + [Fact] + public void A_page_past_the_end_is_refused_rather_than_answered() + { + var pages = new PagedCollection(1001, pageSize: 999); + Assert.Equal(2, pages.PageCount); + + // The last page is short and carries no nextLink, so nothing should ask for a third. + // An empty body here would let an iterator that asked anyway look correct. + var past = Assert.Throws(() => pages.BodyFor(pages.PageCount + 1)); + Assert.Contains("Page 3", past.Message, StringComparison.Ordinal); + Assert.Throws(() => pages.BodyFor(0)); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/CheckpointRetentionTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/CheckpointRetentionTests.cs index 999d963..d2bf915 100644 --- a/tests/Mgx.IntegrationTests/Engine/Pagination/CheckpointRetentionTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/CheckpointRetentionTests.cs @@ -19,8 +19,17 @@ public class CheckpointRetentionTests typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( "TryAdoptOrphanedTemp", BindingFlags.Static | BindingFlags.NonPublic)!; - private static bool TryAdopt(string outputPath, long itemCount) => - (bool)Adopt.Invoke(null, [outputPath, itemCount])!; + /// + /// The two arguments, followed by an empty slot for each out parameter the method carries. + /// Sized off the method, since reflection wants a slot for every one of them. + /// + private static bool TryAdopt(string outputPath, long itemCount) + { + var slots = new object?[Adopt.GetParameters().Length]; + slots[0] = outputPath; + slots[1] = itemCount; + return (bool)Adopt.Invoke(null, slots)!; + } private static string NewDir() { diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/FaultInjectionFanOutTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/FaultInjectionFanOutTests.cs new file mode 100644 index 0000000..8ec1002 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/FaultInjectionFanOutTests.cs @@ -0,0 +1,400 @@ +using System.Management.Automation; +using System.Net; +using System.Text; +using System.Text.Json; +using Mgx.Cmdlets.Base; +using Mgx.Engine.Errors; +using Mgx.Engine.Http; +using Mgx.Engine.Models; +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// Every fault in the catalog, injected at the third item of a fan-out whose first two items +/// have already succeeded. The rule the fan-out exists to keep is that one item failing does not +/// discard the others' data, so each case reads both: what happened to the third item, measured +/// against 's decision for its class, and that the first two are +/// still there. +/// +[Collection("Pipeline")] +public class FaultInjectionFanOutTests +{ + private const string Url1 = "https://graph.microsoft.com/v1.0/groups/g1/members"; + private const string Url2 = "https://graph.microsoft.com/v1.0/groups/g2/members"; + private const string Url3 = "https://graph.microsoft.com/v1.0/groups/g3/members"; + + /// What every armed fault in this file is called, so a theory can assert by name + /// that does not carry it. + private const string FaultName = "fault-item-3"; + + private static readonly ResilientGraphClientOptions Options = new() + { + NoRateLimit = true, + NoAdaptivePacing = true, + MaxRetryAttempts = 1, + AttemptTimeoutSeconds = 1, + TotalTimeoutSeconds = 30, + }; + + private static readonly TimeSpan ShortBodyRead = TimeSpan.FromMilliseconds(250); + + private static string PageBody(string[] ids, string? nextLink) + { + var value = string.Join(",", ids.Select(id => "{\"id\":\"" + id + "\"}")); + var link = nextLink == null ? "" : ",\"@odata.nextLink\":\"" + nextLink + "\""; + return "{\"value\":[" + value + "]" + link + "}"; + } + + private static string EntityBody(string id) => "{\"id\":\"" + id + "\"}"; + + private static Dictionary NoBackoff() => new() { ["Retry-After"] = "0" }; + + private static string ErrorBody(int status) => + "{\"error\":{\"code\":\"Injected_" + status + "\",\"message\":\"injected at item 3\"}}"; + + /// + /// Three urls of two items each, with the fault armed on the third url's first attempt only. + /// A later attempt at it is answered normally, so whether a retry happens is the policy's + /// decision rather than the mock's. + /// + private static MockHttpHandler ThreeUrls(Action> armThird) + { + var handler = new MockHttpHandler(); + handler.When(r => r.Uri.Contains("/g1/")).Respond(HttpStatusCode.OK, PageBody(["m1", "m2"], null)); + handler.When(r => r.Uri.Contains("/g2/")).Respond(HttpStatusCode.OK, PageBody(["m3", "m4"], null)); + armThird(handler.When(r => r.Uri.Contains("/g3/"), FaultName).OnAttempt(1)); + handler.When(r => r.Uri.Contains("/g3/")).Respond(HttpStatusCode.OK, PageBody(["m5", "m6"], null)); + return handler; + } + + private static void ArmAtTheThirdItem(MockResponseRule rule, FaultEntry entry) + { + switch (entry.Shape) + { + case FaultShape.Status: + rule.Respond((HttpStatusCode)entry.Status, ErrorBody(entry.Status), NoBackoff()); + break; + case FaultShape.Transport: + rule.Throw(entry.Transport!()); + break; + case FaultShape.HeldPastAttemptTimeout: + rule.AfterDelay(TimeSpan.FromSeconds(5)) + .Respond(HttpStatusCode.OK, PageBody(["m5", "m6"], null)); + break; + case FaultShape.StalledBody: + rule.RespondStalled(HttpStatusCode.OK, prefix: """{"value":["""); + break; + case FaultShape.TruncatedJson: + rule.Respond(HttpStatusCode.OK, FaultCatalog.TruncatedPageBody); + break; + case FaultShape.UnusableNextLink: + rule.Respond(HttpStatusCode.OK, PageBody(["m5", "m6"], "members?page=2")); + break; + default: + throw new InvalidOperationException( + $"{entry.Kind} does not take a wire shape at a fan-out item; it needs its own case."); + } + } + + private static async Task FanOutAsync(MockHttpHandler handler, int maxItemsPerUrl = 0) + { + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, Options) + { + BodyReadTimeout = ShortBodyRead + }; + // One at a time, so "the third item" is the third request on the wire and not a race. + var fanOut = new ConcurrentFanOut(client, maxConcurrency: 1); + return await fanOut.FetchAllAsync([Url1, Url2, Url3], maxItemsPerUrl); + } + + [Theory] + [MemberData(nameof(FaultCatalog.AllKinds), MemberType = typeof(FaultCatalog))] + public async Task Each_fault_at_the_third_fan_out_item(FaultKind kind) + { + var entry = FaultCatalog.Entry(kind); + var cell = entry.At(InjectionPoint.FanOutItem); + if (cell.Coverage == FaultCoverage.NotApplicable) + { + FaultInjectionRows.AssertNotApplicable(entry, InjectionPoint.FanOutItem); + return; + } + + ResiliencePipelineFactory.Reset(); + try + { + if (cell.Coverage == FaultCoverage.Pinned) + { + await PinAtTheThirdItem(entry); + return; + } + + var handler = ThreeUrls(rule => ArmAtTheThirdItem(rule, entry)); + var result = await FanOutAsync(handler); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + if (FaultCatalog.RetriedOnAnIdempotentRequest(entry)) + { + Assert.False(result.HasErrors); + Assert.Equal(3, result.Results.Count); + Assert.Equal(2, result.Results[Url3].Length); + Assert.Equal(4, handler.RequestCount); + } + else + { + AssertTheFirstTwoSurvived(handler, result); + Assert.Equal(3, handler.RequestCount); + var failed = Assert.Single(result.Errors); + Assert.Equal(Url3, failed.Key); + AssertTheFaultSurfaced(entry, failed.Value); + } + } + finally + { + ResiliencePipelineFactory.Reset(); + } + } + + /// + /// The whole point of the fan-out's per-url error bag: the two urls that worked keep their + /// data whatever happened to the third, and the third was the third request on the wire. + /// + private static void AssertTheFirstTwoSurvived(MockHttpHandler handler, FanOutResult result) + { + Assert.True(result.HasErrors); + Assert.Equal(2, result.Results.Count); + Assert.Equal(2, result.Results[Url1].Length); + Assert.Equal(2, result.Results[Url2].Length); + Assert.DoesNotContain(Url3, result.Results.Keys); + Assert.Contains("/g3/", handler.CapturedRequests[2].Uri); + } + + private static void AssertTheFaultSurfaced(FaultEntry entry, Exception thrown) + { + switch (entry.Shape) + { + case FaultShape.Status: + var graph = Assert.IsType(thrown); + Assert.Equal(entry.Status, (int)graph.StatusCode); + Assert.Equal(entry.Class, MgxErrorClassifier.Classify((int)graph.StatusCode).Class); + break; + case FaultShape.TruncatedJson: + Assert.IsAssignableFrom(thrown); + break; + case FaultShape.UnusableNextLink: + var refused = Assert.IsType(thrown); + Assert.Contains("nextLink", refused.Message, StringComparison.OrdinalIgnoreCase); + break; + case FaultShape.StalledBody: + var stalled = Assert.IsType(thrown); + Assert.Equal(ResilientGraphClient.BodyReadTimedOutMessage, stalled.Message); + break; + default: + Assert.Fail($"{entry.Kind} surfaced as {thrown.GetType().Name} with no case to check it."); + break; + } + } + + // ── The rows that record today's behavior rather than a policy decision ── + + private static async Task PinAtTheThirdItem(FaultEntry entry) + { + switch (entry.Kind) + { + case FaultKind.BodyTimeout: await PinAStalledItemBody(entry); return; + case FaultKind.RepeatedNextLink: await PinASelfReferencingNextLink(); return; + case FaultKind.ExpiredToken: await PinAnExpiredTokenAtTheThirdItem(entry); return; + case FaultKind.ConsistencyDelay: await PinAConsistencyDelay(entry); return; + case FaultKind.ChangedAuthContext: PinAnAuthContextSwapBetweenItems(); return; + default: + Assert.Fail($"{entry.Kind} is pinned at a fan-out item with nothing to run."); + return; + } + } + + private static async Task PinAStalledItemBody(FaultEntry entry) + { + var handler = ThreeUrls(rule => ArmAtTheThirdItem(rule, entry)); + var result = await FanOutAsync(handler); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + // The policy would retry this class on a GET. + Assert.True(FaultCatalog.RetriedOnAnIdempotentRequest(entry)); + + // Nothing does: the body is read after the pipeline handed the response back, so the + // third item gets one attempt and its stall lands in the error bag. + AssertTheFirstTwoSurvived(handler, result); + Assert.Equal(3, handler.RequestCount); + var stalled = Assert.IsType(Assert.Single(result.Errors).Value); + Assert.Equal(ResilientGraphClient.BodyReadTimedOutMessage, stalled.Message); + } + + private static async Task PinASelfReferencingNextLink() + { + // Non-empty pages: the third url's page points back at itself, and only the per-url item + // cap ends the walk. + var looping = new MockHttpHandler(); + looping.When(r => r.Uri.Contains("/g1/")).Respond(HttpStatusCode.OK, PageBody(["m1", "m2"], null)); + looping.When(r => r.Uri.Contains("/g2/")).Respond(HttpStatusCode.OK, PageBody(["m3", "m4"], null)); + looping.When(r => r.Uri.Contains("/g3/")).Respond(HttpStatusCode.OK, PageBody(["m5", "m6"], Url3)); + + var looped = await FanOutAsync(looping, maxItemsPerUrl: 6); + + Assert.False(looped.HasErrors); + Assert.Equal(6, looped.Results[Url3].Length); + Assert.Equal(5, looping.RequestCount); + Assert.Equal(3, looping.CapturedRequests.Count(r => r.Uri.Contains("/g3/"))); + + // Empty pages: the consecutive-empty-page limit is what does stop a self-loop today. + var empty = new MockHttpHandler(); + empty.When(r => r.Uri.Contains("/g1/")).Respond(HttpStatusCode.OK, PageBody(["m1", "m2"], null)); + empty.When(r => r.Uri.Contains("/g2/")).Respond(HttpStatusCode.OK, PageBody(["m3", "m4"], null)); + empty.When(r => r.Uri.Contains("/g3/")).Respond(HttpStatusCode.OK, PageBody([], Url3)); + + var stopped = await FanOutAsync(empty); + + Assert.False(stopped.HasErrors); + Assert.Empty(stopped.Results[Url3]); + Assert.Equal(5, empty.RequestCount); + } + + private static async Task PinAnExpiredTokenAtTheThirdItem(FaultEntry entry) + { + var handler = ThreeUrls(rule => ArmAtTheThirdItem(rule, entry)); + var result = await FanOutAsync(handler); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + Assert.Equal(MgxErrorClass.Authentication, entry.Class); + Assert.False(FaultCatalog.RetriedOnAnIdempotentRequest(entry)); + + AssertTheFirstTwoSurvived(handler, result); + var graph = Assert.IsType(Assert.Single(result.Errors).Value); + Assert.Equal(HttpStatusCode.Unauthorized, graph.StatusCode); + + // One attempt at the third item, and no fresh token asked for: the good page the mock is + // holding for a second attempt is never fetched. + Assert.Equal( + [HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.Unauthorized], + handler.ServedStatusCodes); + } + + private static async Task PinAConsistencyDelay(FaultEntry entry) + { + var handler = ThreeUrls(rule => ArmAtTheThirdItem(rule, entry)); + var result = await FanOutAsync(handler); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + // A read-after-write 404 classifies as NotFound, which the policy does not retry, so the + // 200 waiting behind it is never asked for. MgxErrorClass.Consistency, declared for this + // window, still has no producer. + Assert.Equal(MgxErrorClass.NotFound, entry.Class); + Assert.False(FaultCatalog.RetriedOnAnIdempotentRequest(entry)); + + AssertTheFirstTwoSurvived(handler, result); + var graph = Assert.IsType(Assert.Single(result.Errors).Value); + Assert.Equal(HttpStatusCode.NotFound, graph.StatusCode); + Assert.Equal( + [HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.NotFound], + handler.ServedStatusCodes); + } + + private static void PinAnAuthContextSwapBetweenItems() + { + const string Tenant = "11111111-1111-1111-1111-111111111111"; + const string AppA = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"; + const string AppB = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"; + + var handler = new MockHttpHandler(); + handler.When(r => r.Uri.Contains("/users/u1")).Respond(HttpStatusCode.OK, EntityBody("u1")); + handler.When(r => r.Uri.Contains("/users/u3")).Respond(HttpStatusCode.OK, EntityBody("u3")); + + using var wire = new HttpClient(handler); + // The transport seam sits above the auth probe, and the probe is what is under test, so + // this drives the real GetClient path against the session's own client. + using var scope = GraphSessionScope.Arm(wire, GraphSessionScope.AuthContextFor(Tenant, AppA)); + + // Swapped while the second id is being answered - one item into the fan-out. + handler.When(r => r.Uri.Contains("/users/u2")).OnAttempt(1).Respond(_ => + { + scope.Session.AuthContext = GraphSessionScope.AuthContextFor(Tenant, AppB); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(EntityBody("u2"), Encoding.UTF8, "application/json") + }; + }); + handler.When(r => r.Uri.Contains("/users/u2")).Respond(HttpStatusCode.OK, EntityBody("u2")); + + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + + var (firstCount, firstVerbose) = RunFanOut(ps); + Assert.Equal(3, firstCount); + Assert.DoesNotContain(firstVerbose, v => v.Contains("Graph identity changed")); + + var (secondCount, secondVerbose) = RunFanOut(ps); + Assert.Equal(3, secondCount); + Assert.Contains(secondVerbose, v => v.Contains("Graph identity changed")); + + static (int Output, List Verbose) RunFanOut(PowerShell ps) + { + ps.Commands.Clear(); + ps.Streams.ClearStreams(); + ps.AddScript( + "'u1','u2','u3' | Invoke-MgxRequest -Uri '/users/{id}' -Concurrency 1 " + + "-Verbose -WarningAction SilentlyContinue"); + var output = ps.Invoke(); + Assert.Empty(ps.Streams.Error.Select(e => e.FullyQualifiedErrorId)); + return (output.Count, [.. ps.Streams.Verbose.Select(v => v.Message)]); + } + } + + // ── The same faults through the {id} fan-out, over the transport seam ──── + + [Theory] + [InlineData(FaultKind.Throttled)] + [InlineData(FaultKind.Forbidden)] + public void A_fault_at_the_third_id_of_Invoke_MgxRequest(FaultKind kind) + { + var entry = FaultCatalog.Entry(kind); + var handler = new MockHttpHandler(); + handler.When(r => r.Uri.Contains("/users/u1")).Respond(HttpStatusCode.OK, EntityBody("u1")); + handler.When(r => r.Uri.Contains("/users/u2")).Respond(HttpStatusCode.OK, EntityBody("u2")); + handler.When(r => r.Uri.Contains("/users/u3"), FaultName).OnAttempt(1) + .Respond((HttpStatusCode)entry.Status, ErrorBody(entry.Status), NoBackoff()); + handler.When(r => r.Uri.Contains("/users/u3")).Respond(HttpStatusCode.OK, EntityBody("u3")); + + using (MgxTransportScope.Inject(handler, options: Options)) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript( + "'u1','u2','u3' | Invoke-MgxRequest -Uri '/users/{id}' -Concurrency 1 " + + "-WarningAction SilentlyContinue"); + var output = ps.Invoke(); + var errors = ps.Streams.Error.ToList(); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + if (FaultCatalog.RetriedOnAnIdempotentRequest(entry)) + { + Assert.Empty(errors); + Assert.Equal(3, output.Count); + } + else + { + // One failed id is one non-terminating record; the other two are still emitted. + Assert.Equal(2, output.Count); + var record = Assert.Single(errors); + Assert.Equal("u3", record.TargetObject); + Assert.Equal( + MgxErrorPresentation.Category(entry.Class!.Value), + record.CategoryInfo.Category); + } + } + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/FaultInjectionPaginationTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/FaultInjectionPaginationTests.cs new file mode 100644 index 0000000..f48f9bf --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/FaultInjectionPaginationTests.cs @@ -0,0 +1,534 @@ +using System.Management.Automation; +using System.Net; +using System.Text; +using System.Text.Json; +using Mgx.Cmdlets.Base; +using Mgx.Engine.Errors; +using Mgx.Engine.Http; +using Mgx.Engine.Models; +using Mgx.Engine.Pagination; +using Polly.CircuitBreaker; + +namespace Mgx.IntegrationTests; + +/// +/// Every fault in the catalog, injected at page 3 of a pagination that has already delivered +/// pages 1 and 2. What each case asserts comes from 's decision for +/// the class the classifier assigns: a retryable class has to show the mock serving the retry +/// and the pagination finishing, a non-retryable one exactly one attempt and the pages before +/// the fault still delivered. Nothing here restates a status table - the parity suite owns that. +/// +[Collection("Pipeline")] +public class FaultInjectionPaginationTests +{ + private const string CollectionUrl = "https://graph.microsoft.com/v1.0/users"; + private const string Page2Url = CollectionUrl + "?$skiptoken=page2"; + private const string Page3Url = CollectionUrl + "?$skiptoken=page3"; + + /// What every armed fault in this file is called, so a theory can assert by name + /// that does not carry it. + private const string FaultName = "fault-page-3"; + + /// + /// One retry is all any of these needs, and a one-second attempt timeout is what makes the + /// held-answer case fire without costing the suite anything. + /// + private static readonly ResilientGraphClientOptions Options = new() + { + NoRateLimit = true, + NoAdaptivePacing = true, + MaxRetryAttempts = 1, + AttemptTimeoutSeconds = 1, + TotalTimeoutSeconds = 30, + }; + + private static readonly TimeSpan ShortBodyRead = TimeSpan.FromMilliseconds(250); + + private static string PageBody(string[] ids, string? nextLink) + { + var value = string.Join(",", ids.Select(id => "{\"id\":\"" + id + "\"}")); + var link = nextLink == null ? "" : ",\"@odata.nextLink\":\"" + nextLink + "\""; + return "{\"value\":[" + value + "]" + link + "}"; + } + + private static Dictionary NoBackoff() => new() { ["Retry-After"] = "0" }; + + private static string ErrorBody(int status) => + "{\"error\":{\"code\":\"Injected_" + status + "\",\"message\":\"injected at page 3\"}}"; + + /// + /// Three pages of two items each, with the fault armed on page 3's first attempt only. The + /// same programming serves every kind: a later attempt at page 3 is answered normally, so + /// whether the retry happens at all is the policy's decision and not the mock's. + /// + private static MockHttpHandler ThreePages(Action> armPage3) + { + var handler = new MockHttpHandler(); + handler.When(r => r.Uri.Contains("skiptoken=page2")) + .Respond(HttpStatusCode.OK, PageBody(["u3", "u4"], Page3Url)); + armPage3(handler.When(r => r.Uri.Contains("skiptoken=page3"), FaultName).OnAttempt(1)); + handler.When(r => r.Uri.Contains("skiptoken=page3")) + .Respond(HttpStatusCode.OK, PageBody(["u5", "u6"], null)); + handler.SetDefaultResponse(HttpStatusCode.OK, PageBody(["u1", "u2"], Page2Url)); + return handler; + } + + private static void ArmAtPageThree(MockResponseRule rule, FaultEntry entry) + { + switch (entry.Shape) + { + case FaultShape.Status: + rule.Respond((HttpStatusCode)entry.Status, ErrorBody(entry.Status), NoBackoff()); + break; + case FaultShape.Transport: + rule.Throw(entry.Transport!()); + break; + case FaultShape.HeldPastAttemptTimeout: + rule.AfterDelay(TimeSpan.FromSeconds(5)) + .Respond(HttpStatusCode.OK, PageBody(["u5", "u6"], null)); + break; + case FaultShape.StalledBody: + rule.RespondStalled(HttpStatusCode.OK, prefix: """{"value":["""); + break; + case FaultShape.TruncatedJson: + rule.Respond(HttpStatusCode.OK, FaultCatalog.TruncatedPageBody); + break; + case FaultShape.UnusableNextLink: + // Not a URL at all, so the validator refuses it for a reason that has nothing to + // do with the host - SsrfValidationTests owns the cross-host refusal. + rule.Respond(HttpStatusCode.OK, PageBody(["u5", "u6"], "pages/4")); + break; + default: + throw new InvalidOperationException( + $"{entry.Kind} does not take a wire shape at a page; it needs its own case."); + } + } + + private static async Task<(List Items, Exception? Thrown)> PaginateAsync( + MockHttpHandler handler, int maxItems = 0, ResilientGraphClientOptions? options = null) + { + using var httpClient = new HttpClient(handler); + using var client = new ResilientGraphClient(httpClient, options ?? Options) + { + BodyReadTimeout = ShortBodyRead + }; + var iterator = new PageIterator(client); + var items = new List(); + var thrown = await Record.ExceptionAsync(async () => + { + await foreach (var item in iterator.StreamAllWithCountAsync(CollectionUrl, maxItems, null)) + items.Add(item); + }); + return (items, thrown); + } + + [Theory] + [MemberData(nameof(FaultCatalog.AllKinds), MemberType = typeof(FaultCatalog))] + public async Task Each_fault_at_page_three(FaultKind kind) + { + var entry = FaultCatalog.Entry(kind); + var cell = entry.At(InjectionPoint.Page); + if (cell.Coverage == FaultCoverage.NotApplicable) + { + FaultInjectionRows.AssertNotApplicable(entry, InjectionPoint.Page); + return; + } + + ResiliencePipelineFactory.Reset(); + try + { + if (cell.Coverage == FaultCoverage.Pinned) + { + await PinAtPageThree(entry); + return; + } + + var handler = ThreePages(rule => ArmAtPageThree(rule, entry)); + var (items, thrown) = await PaginateAsync(handler); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + if (FaultCatalog.RetriedOnAnIdempotentRequest(entry)) + { + Assert.Null(thrown); + Assert.Equal(6, items.Count); + // page 1, page 2, page 3 refused, page 3 served: the retry is on the wire. + Assert.Equal(4, handler.RequestCount); + } + else + { + Assert.NotNull(thrown); + // Everything delivered before the fault is still delivered - including page 3's + // own items when the fault is in the link that page carried rather than in the + // page: the items were handed over before the link was read. + Assert.Equal(entry.Shape == FaultShape.UnusableNextLink ? 6 : 4, items.Count); + Assert.Equal(3, handler.RequestCount); + AssertTheFaultSurfaced(entry, thrown!); + } + } + finally + { + ResiliencePipelineFactory.Reset(); + } + } + + private static void AssertTheFaultSurfaced(FaultEntry entry, Exception thrown) + { + switch (entry.Shape) + { + case FaultShape.Status: + var graph = Assert.IsType(thrown); + Assert.Equal(entry.Status, (int)graph.StatusCode); + Assert.Equal(entry.Class, MgxErrorClassifier.Classify((int)graph.StatusCode).Class); + break; + case FaultShape.TruncatedJson: + Assert.IsAssignableFrom(thrown); + break; + case FaultShape.UnusableNextLink: + var refused = Assert.IsType(thrown); + Assert.Contains("nextLink", refused.Message, StringComparison.OrdinalIgnoreCase); + break; + case FaultShape.StalledBody: + var stalled = Assert.IsType(thrown); + Assert.Equal(ResilientGraphClient.BodyReadTimedOutMessage, stalled.Message); + break; + default: + Assert.Fail($"{entry.Kind} surfaced as {thrown.GetType().Name} with no case to check it."); + break; + } + } + + // ── The rows that record today's behavior rather than a policy decision ── + + private static async Task PinAtPageThree(FaultEntry entry) + { + switch (entry.Kind) + { + case FaultKind.BodyTimeout: await PinAStalledPageBody(entry); return; + case FaultKind.RepeatedNextLink: await PinASelfReferencingNextLink(); return; + case FaultKind.ExpiredToken: await PinAnExpiredTokenAtPageThree(entry); return; + case FaultKind.ConsistencyDelay: await PinAConsistencyDelay(entry); return; + case FaultKind.ChangedAuthContext: PinAnAuthContextSwapBetweenPages(); return; + default: + Assert.Fail($"{entry.Kind} is pinned at a page with nothing to run."); + return; + } + } + + private static async Task PinAStalledPageBody(FaultEntry entry) + { + var handler = ThreePages(rule => ArmAtPageThree(rule, entry)); + var (items, thrown) = await PaginateAsync(handler); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + // The policy would retry this class on a GET. + Assert.True(FaultCatalog.RetriedOnAnIdempotentRequest(entry)); + + // Nothing does. The page body is read after the resilience pipeline has already handed + // the response back, so the retry predicate never sees the failure: one attempt at page + // 3, and the pagination ends there with the pages before it delivered. + var stalled = Assert.IsType(thrown); + Assert.Equal(ResilientGraphClient.BodyReadTimedOutMessage, stalled.Message); + Assert.Equal(4, items.Count); + Assert.Equal(3, handler.RequestCount); + } + + private static async Task PinASelfReferencingNextLink() + { + // Non-empty pages: the link points back at the page that produced it, every time, and + // nothing in the iterator notices. Only the item cap ends the walk - which is why this + // case has to pass one. + var looping = new MockHttpHandler(); + looping.When(r => r.Uri.Contains("skiptoken=page2")) + .Respond(HttpStatusCode.OK, PageBody(["u3", "u4"], Page3Url)); + looping.When(r => r.Uri.Contains("skiptoken=page3")) + .Respond(HttpStatusCode.OK, PageBody(["u5", "u6"], Page3Url)); + looping.SetDefaultResponse(HttpStatusCode.OK, PageBody(["u1", "u2"], Page2Url)); + + var (items, thrown) = await PaginateAsync(looping, maxItems: 10); + + Assert.Null(thrown); + Assert.Equal(10, items.Count); + Assert.Equal(5, looping.RequestCount); + Assert.Equal(3, looping.CapturedRequests.Count(r => r.Uri.Contains("skiptoken=page3"))); + + // Empty pages: the consecutive-empty-page limit is what does stop a self-loop today. + var empty = new MockHttpHandler(); + empty.When(r => r.Uri.Contains("skiptoken=page2")) + .Respond(HttpStatusCode.OK, PageBody(["u3", "u4"], Page3Url)); + empty.When(r => r.Uri.Contains("skiptoken=page3")) + .Respond(HttpStatusCode.OK, PageBody([], Page3Url)); + empty.SetDefaultResponse(HttpStatusCode.OK, PageBody(["u1", "u2"], Page2Url)); + + var (emptyItems, emptyThrown) = await PaginateAsync(empty); + + Assert.Null(emptyThrown); + Assert.Equal(4, emptyItems.Count); + Assert.Equal(5, empty.RequestCount); + } + + private static async Task PinAnExpiredTokenAtPageThree(FaultEntry entry) + { + var handler = ThreePages(rule => ArmAtPageThree(rule, entry)); + var (items, thrown) = await PaginateAsync(handler); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + Assert.Equal(MgxErrorClass.Authentication, entry.Class); + Assert.False(FaultCatalog.RetriedOnAnIdempotentRequest(entry)); + + var graph = Assert.IsType(thrown); + Assert.Equal(HttpStatusCode.Unauthorized, graph.StatusCode); + Assert.Equal(4, items.Count); + + // One attempt, and no fresh token asked for: the mock is holding a good page 3 that the + // run never comes back for. Mid-operation re-authentication is not something mgx does. + Assert.Equal( + [HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.Unauthorized], + handler.ServedStatusCodes); + } + + private static async Task PinAConsistencyDelay(FaultEntry entry) + { + var handler = ThreePages(rule => ArmAtPageThree(rule, entry)); + var (items, thrown) = await PaginateAsync(handler); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + // A read-after-write 404 classifies as NotFound, which the policy does not retry, so the + // 200 the mock would serve on a second attempt is never asked for. MgxErrorClass + // .Consistency is declared for this window and has no producer yet. + Assert.Equal(MgxErrorClass.NotFound, entry.Class); + Assert.False(FaultCatalog.RetriedOnAnIdempotentRequest(entry)); + + var graph = Assert.IsType(thrown); + Assert.Equal(HttpStatusCode.NotFound, graph.StatusCode); + Assert.Equal(4, items.Count); + Assert.Equal( + [HttpStatusCode.OK, HttpStatusCode.OK, HttpStatusCode.NotFound], + handler.ServedStatusCodes); + } + + private static void PinAnAuthContextSwapBetweenPages() + { + const string Tenant = "11111111-1111-1111-1111-111111111111"; + const string AppA = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"; + const string AppB = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"; + + var handler = new MockHttpHandler(); + handler.When(r => r.Uri.Contains("skiptoken=page3")) + .Respond(HttpStatusCode.OK, PageBody(["u5", "u6"], null)); + handler.SetDefaultResponse(HttpStatusCode.OK, PageBody(["u1", "u2"], Page2Url)); + + using var wire = new HttpClient(handler); + // No transport seam here: the seam sits above the auth probe, and the probe is the thing + // under test. GetClient falls back to the session's own client, so the mock is reached + // through the identity check rather than around it. + using var scope = GraphSessionScope.Arm(wire, GraphSessionScope.AuthContextFor(Tenant, AppA)); + + // The swap lands while page 2 is being answered - after page 2's request went out on the + // client the invocation resolved, and before page 3's does. + handler.When(r => r.Uri.Contains("skiptoken=page2")).OnAttempt(1).Respond(_ => + { + scope.Session.AuthContext = GraphSessionScope.AuthContextFor(Tenant, AppB); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + PageBody(["u3", "u4"], Page3Url), Encoding.UTF8, "application/json") + }; + }); + handler.When(r => r.Uri.Contains("skiptoken=page2")) + .Respond(HttpStatusCode.OK, PageBody(["u3", "u4"], Page3Url)); + + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + + var (firstCount, firstVerbose) = RunAll(ps); + Assert.Equal(6, firstCount); + Assert.DoesNotContain(firstVerbose, v => v.Contains("Graph identity changed")); + + var (secondCount, secondVerbose) = RunAll(ps); + Assert.Equal(6, secondCount); + Assert.Contains(secondVerbose, v => v.Contains("Graph identity changed")); + + static (int Output, List Verbose) RunAll(PowerShell ps) + { + ps.Commands.Clear(); + ps.Streams.ClearStreams(); + ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("All", true) + .AddParameter("Verbose", true) + .AddParameter("WarningAction", ActionPreference.SilentlyContinue); + var output = ps.Invoke(); + Assert.Empty(ps.Streams.Error.Select(e => e.FullyQualifiedErrorId)); + return (output.Count, [.. ps.Streams.Verbose.Select(v => v.Message)]); + } + } + + // ── The breaker dimension of the same faults ───────────────────────────── + + /// + /// The breaker's minimum throughput and sampling window at the floors the options allow, so + /// four attempts are enough to judge, and a long open window so nothing half-opens while the + /// runs behind the first one are being made. + /// + private static readonly ResilientGraphClientOptions BreakerOptions = new() + { + NoRateLimit = true, + NoAdaptivePacing = true, + MaxRetryAttempts = 1, + AttemptTimeoutSeconds = 1, + TotalTimeoutSeconds = 30, + CircuitBreakerMinThroughput = 2, + CircuitBreakerFailureRatio = 0.4, + CircuitBreakerSamplingDurationSeconds = 5, + CircuitBreakerDurationSeconds = 60, + }; + + /// How many runs the drive below makes before it gives up on the circuit opening. + private const int BreakerRuns = 3; + + /// Pages 1 and 2, then page 3 twice - its own attempt and the one retry. + private const int RequestsPerRun = 4; + + /// + /// The dimension the per-fault theory above does not ask about: whether repeating a fault + /// opens the circuit. answers it per class, + /// and the two classes that answer differently are what makes the answer worth anything - a + /// 503 is the service failing and a 429 is the service pacing us, and a circuit that opened + /// on the second would stop a run exactly when the right response is to slow down and keep + /// going. + /// + [Fact] + public async Task Repeating_a_page_three_fault_opens_the_circuit_only_for_a_counted_class() + { + var counted = FaultCatalog.Entry(FaultKind.ServerError); + var excluded = FaultCatalog.Entry(FaultKind.Throttled); + + var server = await DriveAgainstTheBreakerAsync(counted); + var throttle = await DriveAgainstTheBreakerAsync(excluded); + + // Each side is what the policy says of its own class. + Assert.Equal(FaultCatalog.CountsAgainstTheBreaker(counted), server.Opened); + Assert.Equal(FaultCatalog.CountsAgainstTheBreaker(excluded), throttle.Opened); + + // And the two answers differ, which neither assertion above can say on its own: a policy + // that counted throttles as well would agree with both of them and open the circuit on + // the service's own pacing. + Assert.NotEqual(server.Opened, throttle.Opened); + + // What the difference costs on the wire. Once the circuit is open the run behind it sends + // nothing at all - the refusal is the client's, before the request. The class the breaker + // ignores keeps sending every attempt, run after run, and what surfaces is still the + // status the server gave. + Assert.Equal(0, server.RequestsOnTheLastRun); + Assert.IsType(server.LastFailure); + Assert.Equal(RequestsPerRun, throttle.RequestsOnTheLastRun); + Assert.Equal(HttpStatusCode.TooManyRequests, + Assert.IsType(throttle.LastFailure).StatusCode); + } + + /// + /// Page 3 answered with this fault's status on every attempt, run and re-run against one + /// pipeline so the outcomes accumulate in a single sampling window. Returns whether any run + /// was refused by the open circuit, what the last run cost on the wire, and what it surfaced. + /// + private static async Task<(bool Opened, int RequestsOnTheLastRun, Exception LastFailure)> + DriveAgainstTheBreakerAsync(FaultEntry entry) + { + ResiliencePipelineFactory.Reset(); + try + { + var opened = false; + var requests = 0; + Exception? last = null; + for (var run = 0; run < BreakerRuns; run++) + { + var handler = new MockHttpHandler(); + // Not named and not asserted: once the circuit opens, a later run's client-side + // refusal never sends a request at all, so this entry is legitimately unanswered + // on that run - a blanket assertion here would fail the very outcome the test + // means to observe. + handler.When(r => r.Uri.Contains("skiptoken=page3")) + .Respond((HttpStatusCode)entry.Status, ErrorBody(entry.Status), NoBackoff()); + handler.When(r => r.Uri.Contains("skiptoken=page2")) + .Respond(HttpStatusCode.OK, PageBody(["u3", "u4"], Page3Url)); + handler.SetDefaultResponse(HttpStatusCode.OK, PageBody(["u1", "u2"], Page2Url)); + + var (_, thrown) = await PaginateAsync(handler, options: BreakerOptions); + + Assert.NotNull(thrown); + last = thrown; + opened |= thrown is BrokenCircuitException; + requests = handler.RequestCount; + } + return (opened, requests, last!); + } + finally + { + ResiliencePipelineFactory.Reset(); + } + } + + // ── The same faults through the cmdlet, over the transport seam ────────── + + [Theory] + [InlineData(FaultKind.Throttled)] + [InlineData(FaultKind.Forbidden)] + public void A_fault_at_page_three_of_Invoke_MgxRequest_All(FaultKind kind) + { + var entry = FaultCatalog.Entry(kind); + var handler = ThreePages(rule => ArmAtPageThree(rule, entry)); + + using (MgxTransportScope.Inject(handler, options: Options)) + { + using var ps = PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddCommand("Invoke-MgxRequest") + .AddParameter("Uri", "/users") + .AddParameter("All", true) + .AddParameter("WarningAction", ActionPreference.SilentlyContinue); + var output = ps.Invoke(); + var errors = ps.Streams.Error.ToList(); + Assert.DoesNotContain(FaultName, handler.UnansweredRules); + + if (FaultCatalog.RetriedOnAnIdempotentRequest(entry)) + { + Assert.Empty(errors); + Assert.Equal(6, output.Count); + } + else + { + // The contract: only a failure that makes the whole invocation meaningless + // terminates, so the pages already delivered are delivered and the page that + // failed is one non-terminating record. + Assert.Equal(4, output.Count); + var record = Assert.Single(errors); + Assert.Equal( + MgxErrorPresentation.Category(entry.Class!.Value), + record.CategoryInfo.Category); + } + } + } +} + +/// +/// What a theory says about a cell it cannot run. The acceptance is "every fault at each point +/// where it is physically possible", so a point that cannot host a fault has to say which fault +/// and why, in the same run as the ones it does host. +/// +internal static class FaultInjectionRows +{ + internal static void AssertNotApplicable(FaultEntry entry, InjectionPoint point) + { + var cell = entry.At(point); + Assert.Equal(FaultCoverage.NotApplicable, cell.Coverage); + Assert.False(string.IsNullOrWhiteSpace(cell.Note), + $"{entry.Kind} is marked impossible at {point} without saying why"); + Assert.Contains(Enum.GetValues(), + other => entry.At(other).Coverage != FaultCoverage.NotApplicable); + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/HeldEntryTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/HeldEntryTests.cs new file mode 100644 index 0000000..4b2d3f1 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/HeldEntryTests.cs @@ -0,0 +1,149 @@ +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// Serializes the test classes that set HeldEntry's claim seam, which is one static for the +/// whole process: run in parallel, one class's assignment is the other's, and tests written +/// against the instant between a claim and its check would fire each other's. +/// +[CollectionDefinition("ClaimSeam")] +public class ClaimSeamCollection; + +/// +/// The question every claim on Unix is finished with: is the file this run holds still the file +/// standing at the name it asked for? A handle says nothing about that on its own - the lock +/// .NET takes for FileShare.None is granted several syscalls after the open that resolved the +/// name, and rename and unlink honor no lock - so the three ways a name moves under a handle are +/// asked about here, one test each. +/// +/// Those three are Unix's alone, and not because the answer would differ: Windows refuses to +/// unlink or rename a file another handle has open at all, so a name cannot move under a claim +/// there and the three states cannot be reached to be asked about. That refusal is the whole +/// reason the primitive answers Windows at the open and asks the platform nothing. +/// +/// +[Collection("ClaimSeam")] +public class HeldEntryTests +{ + private static string NewDir() => + Directory.CreateDirectory( + Path.Combine(Path.GetTempPath(), $"mgx-held-{Guid.NewGuid():N}")).FullName; + + private static FileStream Claim(string path) => + new(path, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None); + + /// A file nothing has touched since the claim: the name is still this handle's. + [Fact] + public void A_handle_on_the_file_at_the_name_still_stands_there() + { + var dir = NewDir(); + try + { + var path = Path.Combine(dir, "out.jsonl"); + using var held = Claim(path); + held.Write("{\"id\":\"a\"}\n"u8); + held.Flush(); + + Assert.True(HeldEntry.StillStandsAt(held, path)); + } + finally { Directory.Delete(dir, true); } + } + + /// + /// Unlinked under the handle, which is what a second run's sweep of the name does: the inode + /// is this run's for as long as it holds it and has no name left on it at all. The file still + /// reads and still takes writes, and every one of them is into a file no name reaches. + /// + [Fact] + public void A_file_unlinked_under_the_handle_no_longer_stands_at_the_name() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var path = Path.Combine(dir, "out.jsonl"); + using var held = Claim(path); + File.Delete(path); + + Assert.False(HeldEntry.StillStandsAt(held, path)); + } + finally { Directory.Delete(dir, true); } + } + + /// + /// Renamed away, which leaves the handle's own inode with a name - just not this one. Nothing + /// about the file has changed and the name reaches nothing. + /// + [Fact] + public void A_file_renamed_away_no_longer_stands_at_the_name() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var path = Path.Combine(dir, "out.jsonl"); + using var held = Claim(path); + File.Move(path, Path.Combine(dir, "elsewhere.jsonl")); + + Assert.False(HeldEntry.StillStandsAt(held, path)); + } + finally { Directory.Delete(dir, true); } + } + + /// + /// A second run's file standing at the name instead, with this run's own inode still named + /// somewhere else: the one case a link count cannot answer. The name exists, the handle is + /// open over a file with a name on it, and they are two different files - which is exactly + /// the state a promotion whose claim was granted late came away in. + /// + [Fact] + public void Another_file_at_the_name_is_not_the_one_the_handle_holds() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var path = Path.Combine(dir, "out.jsonl"); + using var held = Claim(path); + File.Move(path, Path.Combine(dir, "elsewhere.jsonl")); + File.WriteAllText(path, "{\"id\":\"theirs\"}\n"); + + Assert.False(HeldEntry.StillStandsAt(held, path)); + } + finally { Directory.Delete(dir, true); } + } + + /// + /// And the seam the deterministic tests of the rule are written against fires with the path + /// that was claimed, in the instant before the question is asked. One static for the whole + /// process, so it hears every claim any run in it takes: what a test written against it may + /// say anything about is its own name, which is what this counts. + /// + [Fact] + public void The_seam_fires_with_the_path_the_claim_was_taken_on() + { + var dir = NewDir(); + var seen = new List(); + try + { + var path = Path.Combine(dir, "out.jsonl"); + using var held = Claim(path); + HeldEntry.BeforeTheClaimIsVerified = claimed => + { + if (string.Equals(claimed, path, StringComparison.Ordinal)) seen.Add(claimed); + }; + + Assert.True(HeldEntry.StillStandsAt(held, path)); + Assert.Equal([path], seen); + } + finally + { + HeldEntry.BeforeTheClaimIsVerified = null; + Directory.Delete(dir, true); + } + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/NamedTempPromotionTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/NamedTempPromotionTests.cs new file mode 100644 index 0000000..5180acc --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/NamedTempPromotionTests.cs @@ -0,0 +1,1781 @@ +using System.Reflection; + +namespace Mgx.IntegrationTests.Engine.Pagination; + +/// +/// Promotion of the temp a checkpoint names, which is how every checkpoint this release writes +/// recovers: the file is taken by name and by the length recorded for it, so nothing has to be +/// guessed about which run's items are in it. What the name does not say is whether that run is +/// finished. Promotion replaces the output with those bytes and then unlinks the temp, so the +/// same claim adoption takes in OrphanAdoptionTests has to be taken here - a file nobody can be +/// found holding is an interrupted run's leftovers, and one that cannot be claimed belongs to a +/// run that is still writing it. +/// +[Collection("ClaimSeam")] +public class NamedTempPromotionTests +{ + // An instance method, because what follows a promotion is an append onto the file it landed + // in: the run keeps that file from the move that puts the rows there until its writing ends, + // and the handle lives on the cmdlet. + private static readonly MethodInfo Promote = + typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( + "TryPromoteNamedTemp", BindingFlags.Instance | BindingFlags.NonPublic)!; + + private static readonly MethodInfo CanPromote = + typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( + "CanPromoteNamedTemp", BindingFlags.Static | BindingFlags.NonPublic)!; + + /// + /// The route a resumed run takes the output by, which is the shortest way to a claim whose + /// caller lets the handle go on its own path rather than in a finally: it asks for the file, + /// reads the claim's answer, and keeps nothing where the answer is not Taken. + /// + private static readonly MethodInfo Resume = + typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( + "TakeOutputForCheckpoint", BindingFlags.Instance | BindingFlags.NonPublic)!; + + /// + /// A directory a run may write in, which is how the tests below find every directory and how + /// they leave it. + /// + private const UnixFileMode Searchable = + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute; + + /// + /// And one that answers for every name in it and takes no new file: the mode a create is + /// refused by, with the stat of the name it was refused at still answering. + /// + private const UnixFileMode ReadAndSearch = UnixFileMode.UserRead | UnixFileMode.UserExecute; + + /// + /// The mode on a directory. Set through here rather than at each site, because the claim + /// seam below is a lambda and a lambda is its own body to the platform analyzer: it cannot + /// see the Windows guard the test opened with, and the guard has to be in front of the call. + /// + private static void SetMode(string dir, UnixFileMode mode) + { + if (OperatingSystem.IsWindows()) return; + File.SetUnixFileMode(dir, mode); + } + + /// + /// The promotion's answer, by name: the enum it returns is protected on the cmdlet base, so + /// a test outside that hierarchy cannot name the type. "Taken" is a promotion that landed + /// and kept the output it landed in; every other answer left both files where they were. + /// + private static string Invoke(Mgx.Cmdlets.Base.MgxCmdletBase run, + string outputPath, string tempFileName, long dataLength) => + Promote.Invoke(run, Slots(Promote, outputPath, tempFileName, dataLength))!.ToString()!; + + /// + /// The three arguments, followed by an empty slot for each out parameter the method carries. + /// Reflection wants a slot for every one of them and this suite asks about them by position + /// rather than by count - the reason a refusal carries at 3, the staging failure at 4 - + /// so the array is sized off the method itself. + /// + private static object?[] Slots(MethodInfo method, string outputPath, string tempFileName, + long dataLength) + { + var slots = new object?[method.GetParameters().Length]; + slots[0] = outputPath; + slots[1] = tempFileName; + slots[2] = dataLength; + return slots; + } + + /// + /// The preview's answer, by the same name. It reports what the applying pass would answer - + /// which of "Taken" and "NotPromoted" a temp claim gives it, and the two refusals the output + /// standing at the path gives it - and promotes nothing to find out. + /// + private static string InvokeCan(string outputPath, string tempFileName, long dataLength) => + CanPromote.Invoke(null, Slots(CanPromote, outputPath, tempFileName, dataLength))! + .ToString()!; + + private static string NewDir() + { + var d = Path.Combine(Path.GetTempPath(), $"mgx-promote-{Guid.NewGuid():N}"); + Directory.CreateDirectory(d); + return d; + } + + /// + /// Reads back a file the run standing in for a second export still holds open to write it. + /// Windows checks the sharing both ways round - what the reader asks of the file, and what + /// the reader will let a handle already on it go on doing - so File.ReadAllLines, which asks + /// for FileShare.Read and nothing more, is refused by the writer this test put there itself. + /// The sharing the holder asked for is offered back, which is what a reader of a file a live + /// run is appending to has to do on either platform. + /// + private static string[] ReadSharedLines(string path) + { + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, + FileShare.ReadWrite); + using var reader = new StreamReader(stream); + var lines = new List(); + while (reader.ReadLine() is { } line) lines.Add(line); + return [.. lines]; + } + + /// + /// The state a crash leaves: the previous run's output, this run's items in the temp beside + /// it, and a checkpoint naming both. The temp replaces the output, trimmed to the length + /// recorded, and goes. + /// + [Fact] + public void Promotes_a_named_temp_nothing_holds() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + // The writer's position at the last flush, which is what a checkpoint records. + var counted = new FileInfo(temp).Length; + File.AppendAllText(temp, "{\"id\":\"after-the-flush\"}\n"); + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + Assert.Equal("Taken", InvokeCan(output, Path.GetFileName(temp), counted)); + Assert.Equal("Taken", Invoke(run, output, Path.GetFileName(temp), counted)); + + // The rows are read once the run that took the file lets it go, which is what + // disposing it does: while the hold stands, a reader of that file is refused. + Assert.ThrowsAny(() => File.ReadAllLines(output)); + run.Dispose(); + + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + Assert.Empty(Directory.GetFiles(dir, "*.tmp")); + } + finally { Directory.Delete(dir, true); } + } + + /// + /// The same three files, with the run that wrote the temp still writing it. Promotion + /// copied its rows into this run's output and unlinked the file underneath it, so that run + /// wrote into an unlinked inode and its own closing move failed against a name that was no + /// longer there. Being named by a checkpoint says which file holds the items; it says + /// nothing about whether anyone still has it open. + /// + [Fact] + public void Refuses_a_named_temp_a_live_run_still_holds() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var live = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + + // Opened the way an export opens its own temp: new StreamWriter(path, append). + using (var writer = new StreamWriter(live, append: false)) + { + writer.WriteLine("{\"id\":\"a\"}"); + writer.WriteLine("{\"id\":\"b\"}"); + writer.Flush(); + var counted = new FileInfo(live).Length; + Assert.True(counted > 0); + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + Assert.Equal("NotPromoted", InvokeCan(output, Path.GetFileName(live), counted)); + Assert.Equal("NotPromoted", + Invoke(run, output, Path.GetFileName(live), counted)); + + // Nothing was promoted, so nothing was taken either: the output is the previous + // run's, unheld, and readable. + Assert.Equal(["{\"id\":\"old\"}"], File.ReadAllLines(output)); + Assert.False(File.Exists($"{output}.adopt")); + Assert.True(File.Exists(live), "a running export's temp was promoted out from under it"); + + // The run that owns it goes on writing into the file it still has. + writer.WriteLine("{\"id\":\"c\"}"); + } + + Assert.Equal( + ["{\"id\":\"a\"}", "{\"id\":\"b\"}", "{\"id\":\"c\"}"], + File.ReadAllLines(live)); + } + finally { Directory.Delete(dir, true); } + } + + /// + /// The third file. A promotion replaces the output and then appends onto what it replaced, + /// so the output another run is holding is one it must not replace at all - and a rename + /// asks nothing of the lock: the move went through, the holder's next write landed in an + /// inode with no name on it, and the claim taken afterwards answered Taken about a file one + /// statement old. The output is asked for before the move now, and a held one is the whole + /// answer. + /// + [Fact] + public void Refuses_an_output_another_run_holds_and_replaces_nothing() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"theirs\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBefore = File.ReadAllBytes(temp); + + // The run that has it: appending, sharing reads, the way a resumed run holds it. + using var live = new StreamWriter( + new FileStream(output, FileMode.Open, FileAccess.ReadWrite, FileShare.Read)) + { AutoFlush = true }; + live.BaseStream.Seek(0, SeekOrigin.End); + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + Assert.Equal("OutputHeld", InvokeCan(output, Path.GetFileName(temp), counted)); + Assert.Equal("OutputHeld", Invoke(run, output, Path.GetFileName(temp), counted)); + + // Nothing was replaced, nothing was staged, and the temp the checkpoint names is + // still there with the checkpoint's own bytes in it. + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.False(File.Exists($"{output}.adopt")); + + // The same inode, not a new file of the same name: the holder's next write is read + // back through the path. + live.WriteLine("{\"id\":\"theirs-2\"}"); + Assert.Equal( + ["{\"id\":\"theirs\"}", "{\"id\":\"theirs-2\"}"], + ReadSharedLines(output)); + } + finally { Directory.Delete(dir, true); } + } + + /// + /// The other refusal, which nothing could reach before: an output path that is not a file + /// this run can write at an offset. A FIFO opens read-write and answers no length, and the + /// move would have replaced it with a regular file - so the caller's pipe, and whatever is + /// reading it, would be gone. + /// + [Fact] + public void Refuses_an_output_it_cannot_open_and_replaces_nothing() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var output = Path.Combine(dir, "out.jsonl"); + try + { + using (var mkfifo = System.Diagnostics.Process.Start("/usr/bin/mkfifo", [output])) + { + Assert.NotNull(mkfifo); + mkfifo.WaitForExit(); + Assert.Equal(0, mkfifo.ExitCode); + } + + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBefore = File.ReadAllBytes(temp); + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + Assert.Equal("OutputUnopenable", InvokeCan(output, Path.GetFileName(temp), counted)); + Assert.Equal("OutputUnopenable", Invoke(run, output, Path.GetFileName(temp), counted)); + + Assert.Equal(tempBefore, File.ReadAllBytes(temp)); + Assert.False(File.Exists($"{output}.adopt")); + + // Still a pipe, and still nobody's: a stream with no offset in it is what says so, + // and a claim of this test's own would be refused if the run had kept one. + using var stillAFifo = new FileStream(output, FileMode.Open, FileAccess.ReadWrite, + FileShare.None); + Assert.False(stillAFifo.CanSeek, "the move replaced the caller's FIFO with a file"); + } + finally + { + try { File.Delete(output); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A temp big enough that the copy beside the output takes long enough to be watched, so a + /// question can be asked of the output path while the promotion is in the middle of staging. + /// + private static long WriteTempOfAtLeast(string temp, long bytes) + { + const string row = "{\"id\":\"0000000000000000000000000000000000000000000000000000\"}\n"; + var block = string.Concat(Enumerable.Repeat(row, 1024)); + using (var writer = new StreamWriter(temp, append: false)) + while (writer.BaseStream.Length < bytes) + writer.Write(block); + return new FileInfo(temp).Length; + } + + /// + /// The interval a claim released in front of the rename left open. The promotion asked for + /// the standing output and let it go, and only then renamed over it - and rename(2) honors no + /// lock on either side of it, so a run that took the output in between had it replaced under + /// it: its rows in an inode with no name left on them, and no warning on any stream. Widened + /// to 600 ms by a gate, a second process took the output there. + /// + /// The handle is kept now, from before the copy is staged through the move, so the interval + /// is one nothing can claim the output in. Measured over the whole of it: the staging file + /// appearing is what says the promotion has begun, and from that instant until it answers, + /// every claim on the output is refused. flock belongs to the open file description, so a + /// second open from this very process is refused exactly as another process's is - which is + /// also why the promotion never asks for the output again on the far side of the move. + /// + [Fact] + public void Nothing_can_claim_the_output_between_the_standing_claim_and_the_move() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + using var promoting = new ManualResetEventSlim(false); + Thread? racer = null; + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"theirs\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + var counted = WriteTempOfAtLeast(temp, 64L * 1024 * 1024); + var adopt = $"{output}.adopt"; + + // Spins on the output for as long as the promotion is staging or moving, and comes + // back with whether the path was ever anybody's to take. It starts asking when the + // staged file appears, which is the first instant a promotion has reached. + var claimed = false; + racer = new Thread(() => + { + while (!File.Exists(adopt) && !promoting.IsSet) Thread.SpinWait(64); + while (!promoting.IsSet) + { + try + { + using var mine = new FileStream(output, FileMode.Open, + FileAccess.ReadWrite, FileShare.None); + // A claim granted has to be a claim on the output, and one taken here + // is not always: the open and the lock behind it are two syscalls, and + // a thread descheduled between them comes back holding the inode the + // rename has since unnamed - a file at no path, whose lock the + // promotion let go on its way out. The staged file still standing and + // the claimed length still the path's are what say otherwise. + if (!File.Exists(adopt) || mine.Length != new FileInfo(output).Length) + continue; + claimed = true; + return; + } + catch (IOException) { } + catch (UnauthorizedAccessException) { } + } + }); + racer.Start(); + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var answer = Invoke(run, output, Path.GetFileName(temp), counted); + promoting.Set(); + Assert.True(racer.Join(TimeSpan.FromSeconds(30)), "the racing claim never finished"); + + Assert.Equal("Taken", answer); + Assert.False(claimed, + "the output was claimable between the standing claim and the rename"); + + // And the run that promoted has it: the rows are readable once it lets go. + run.Dispose(); + Assert.Equal(counted, new FileInfo(output).Length); + Assert.Empty(Directory.GetFiles(dir, "*.tmp")); + Assert.False(File.Exists(adopt)); + } + finally + { + promoting.Set(); + racer?.Join(TimeSpan.FromSeconds(30)); + Directory.Delete(dir, true); + } + } + + /// + /// A promotion that stops on the output staged the whole temp beside it first and then took + /// the copy back - 172 ms and a moved directory mtime for a 256 MiB temp, for a run that + /// replaces nothing. The output is asked for before any of it now, so a stop costs the + /// directory beside it nothing at all: no entry appears under the staging name, and the + /// directory's own mtime is where the run found it. + /// + [Fact] + public void A_held_output_stops_the_promotion_before_a_byte_is_staged() + { + var dir = NewDir(); + using var stopped = new ManualResetEventSlim(false); + Thread? watcher = null; + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"theirs\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + var counted = WriteTempOfAtLeast(temp, 64L * 1024 * 1024); + var adopt = $"{output}.adopt"; + var tempBefore = new FileInfo(temp).Length; + + // The run that has the output: appending, sharing reads, the way a resumed run holds + // it. Its claim is what the promotion stops on. + using var live = new StreamWriter( + new FileStream(output, FileMode.Open, FileAccess.ReadWrite, FileShare.Read)) + { AutoFlush = true }; + live.BaseStream.Seek(0, SeekOrigin.End); + + // Whether the staging name is ever on disk. A create and a delete inside one tick + // would leave the directory's mtime moved and nothing to see in it, so the entry is + // watched for as well as measured after. + var sawStaging = false; + watcher = new Thread(() => + { + while (!stopped.IsSet) + { + if (File.Exists(adopt)) { sawStaging = true; return; } + Thread.SpinWait(64); + } + }); + watcher.Start(); + + var directoryBefore = Directory.GetLastWriteTimeUtc(dir); + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var answer = Invoke(run, output, Path.GetFileName(temp), counted); + stopped.Set(); + Assert.True(watcher.Join(TimeSpan.FromSeconds(30)), "the staging watch never finished"); + + Assert.Equal("OutputHeld", answer); + Assert.False(sawStaging, "the stop staged a copy of the temp beside the output"); + Assert.Equal(directoryBefore, Directory.GetLastWriteTimeUtc(dir)); + Assert.False(File.Exists(adopt)); + Assert.Equal(tempBefore, new FileInfo(temp).Length); + + // The same inode, not a new file wearing the name. + live.WriteLine("{\"id\":\"theirs-2\"}"); + Assert.Equal(["{\"id\":\"theirs\"}", "{\"id\":\"theirs-2\"}"], ReadSharedLines(output)); + } + finally + { + stopped.Set(); + watcher?.Join(TimeSpan.FromSeconds(30)); + Directory.Delete(dir, true); + } + } + + /// + /// Which errno the claim reads as another run holding the file. .NET backs FileShare.None on + /// Unix with a non-blocking flock, and a refused LOCK_EX comes back EWOULDBLOCK - 35 on + /// macOS, 11 on Linux, where the name EAGAIN shares that value. Read as a union of the two, + /// each platform also read its own EDEADLK as a sharing violation: 11 is EDEADLK on macOS + /// and 35 is EDEADLK on Linux, so a deadlock the kernel had just refused to enter was + /// reported as an export to wait for. Each number is read on the platform that uses it. + /// + [Fact] + public void The_sharing_errno_is_the_one_this_platform_uses() + { + var isSharingViolation = typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( + "IsSharingViolation", BindingFlags.Static | BindingFlags.NonPublic)!; + bool Reads(int hResult) => + (bool)isSharingViolation.Invoke(null, [new IOException("errno", hResult)])!; + + Assert.Equal(OperatingSystem.IsMacOS(), Reads(35)); + Assert.Equal(OperatingSystem.IsLinux(), Reads(11)); + + // Windows carries the win32 code in the low half of an HRESULT, and those two are read + // wherever they turn up. + Assert.True(Reads(unchecked((int)0x80070020))); + Assert.True(Reads(unchecked((int)0x80070021))); + + // And the failures that are not a second run at all: ELOOP, ENOTSUP for a socket, and + // the win32-shaped HRESULT .NET maps an over-long path to by hand. + Assert.False(Reads(62)); + Assert.False(Reads(45)); + Assert.False(Reads(unchecked((int)0x800700CE))); + } + + /// + /// The staging failure a promotion answers with, beside its answer: the reason a caller + /// quotes in the stop it writes. + /// + private static (string Answer, string Reason) InvokeForStaging( + Mgx.Cmdlets.Base.MgxCmdletBase run, string outputPath, string tempFileName, long dataLength) + { + var args = Slots(Promote, outputPath, tempFileName, dataLength); + var answer = Promote.Invoke(run, args)!.ToString()!; + return (answer, args[4]?.ToString() ?? ""); + } + + /// + private static (string Answer, string Reason) InvokeCanForStaging( + string outputPath, string tempFileName, long dataLength) + { + var args = Slots(CanPromote, outputPath, tempFileName, dataLength); + var answer = CanPromote.Invoke(null, args)!.ToString()!; + return (answer, args[4]?.ToString() ?? ""); + } + + private static void Mkfifo(string path) + { + using var mkfifo = System.Diagnostics.Process.Start("/usr/bin/mkfifo", [path]); + Assert.NotNull(mkfifo); + mkfifo.WaitForExit(); + Assert.Equal(0, mkfifo.ExitCode); + } + + private static void Symlink(string target, string link) + { + using var ln = System.Diagnostics.Process.Start("/bin/ln", ["-s", target, link]); + Assert.NotNull(ln); + ln.WaitForExit(); + Assert.Equal(0, ln.ExitCode); + } + + /// + /// A second directory entry for a file's own inode. Not a symlink: nothing on the entry says + /// it is an alias, and unlinking it takes the name and leaves the file. + /// + private static void HardLink(string target, string link) + { + using var ln = System.Diagnostics.Process.Start("/bin/ln", [target, link]); + Assert.NotNull(ln); + ln.WaitForExit(); + Assert.Equal(0, ln.ExitCode); + } + + /// + /// A FIFO at the staging name. The copy was staged with FileMode.Create and + /// FileAccess.Write, which is O_WRONLY, and open(2) on a FIFO with no reader blocks until + /// one arrives - so the promotion never returned, holding the temp's claim and the output's + /// lock for as long as the process lived, with StopProcessing unable to reach it and a + /// second run refused on the output. The name is swept before it is created now, and a FIFO + /// standing there is one of the kinds that goes. + /// + [Fact] + public void Sweeps_a_fifo_off_the_staging_name_and_promotes_without_blocking() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var adopt = $"{output}.adopt"; + Mkfifo(adopt); + + // On its own thread with a watchdog: the failure this pins is a promotion that does + // not come back at all, and a test that called it inline would hang the suite. + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + string? answer = null; + var promoting = new Thread(() => + { + try { answer = Invoke(run, output, Path.GetFileName(temp), counted); } + catch (Exception ex) { answer = $"threw {ex.GetType().Name}"; } + }) { IsBackground = true }; + promoting.Start(); + Assert.True(promoting.Join(TimeSpan.FromSeconds(15)), + "the promotion never returned: the staging open is waiting for a reader on the FIFO"); + + Assert.Equal("Taken", answer); + run.Dispose(); + + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + Assert.False(File.Exists(adopt)); + Assert.Empty(Directory.GetFiles(dir, "*.tmp")); + + // A regular file at the output path, not the pipe that stood beside it: a seekable + // handle is what says so. + using var landed = new FileStream(output, FileMode.Open, FileAccess.Read); + Assert.True(landed.CanSeek); + } + finally + { + try { File.Delete($"{Path.Combine(dir, "out.jsonl")}.adopt"); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A symlink at the staging name, pointing at a file outside the output's directory. The + /// staging followed it: the copy went into the link's target, truncating a file no export + /// had ever written, and the rename then moved the LINK onto the output path - so the + /// output was a symlink to somebody's file and both passes answered as though the promotion + /// had landed. The sweep takes the link off the name, and never what it pointed at. + /// + [Fact] + public void Sweeps_a_symlink_off_the_staging_name_and_leaves_its_target_alone() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + var elsewhere = NewDir(); + try + { + var victim = Path.Combine(elsewhere, "precious.conf"); + var victimBytes = "KEEP-THIS-EXACTLY\n"u8.ToArray(); + File.WriteAllBytes(victim, victimBytes); + + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var adopt = $"{output}.adopt"; + Symlink(victim, adopt); + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + Assert.Equal("Taken", Invoke(run, output, Path.GetFileName(temp), counted)); + run.Dispose(); + + Assert.Equal(victimBytes, File.ReadAllBytes(victim)); + Assert.True(File.Exists(victim), "the sweep deleted what the link pointed at"); + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + Assert.Null(new FileInfo(output).LinkTarget); + Assert.False(File.Exists(adopt)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + try { Directory.Delete(elsewhere, true); } catch { } + } + } + + /// + /// A dangling link at the staging name - the kind the sweep answered Absent about and left + /// standing, and the create then followed to a name in a directory that may not even be + /// there. Read off the entry rather than through an open, so a link with nothing behind it + /// is still a link. + /// + [Fact] + public void Sweeps_a_dangling_link_off_the_staging_name() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var adopt = $"{output}.adopt"; + Symlink(Path.Combine(dir, "no-such-file"), adopt); + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + Assert.Equal("Taken", Invoke(run, output, Path.GetFileName(temp), counted)); + run.Dispose(); + + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + Assert.False(File.Exists(Path.Combine(dir, "no-such-file")), + "the create made the file the dangling link pointed at"); + Assert.False(File.Exists(adopt)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// A directory at the staging name, which is the one kind the sweep does not remove: it is + /// not this run's to delete and could hold anything. Both passes stop there, and both say a + /// directory - the applying pass built its reason from the raw exception, so the same disk + /// state read "Access to the path ... is denied" in the run and "a directory stands at the + /// path" in the preview. + /// + [Fact] + public void A_directory_at_the_staging_name_stops_both_passes_saying_a_directory() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + var outputBytes = "{\"id\":\"old\"}\n"u8.ToArray(); + File.WriteAllBytes(output, outputBytes); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + var adopt = $"{output}.adopt"; + Directory.CreateDirectory(adopt); + File.WriteAllText(Path.Combine(adopt, "somebody-elses"), "keep\n"); + + var preview = InvokeCanForStaging(output, Path.GetFileName(temp), counted); + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + + Assert.Equal("StagingFailed", preview.Answer); + Assert.Equal("StagingFailed", applied.Answer); + Assert.Contains("a directory stands at the path", preview.Reason); + Assert.Equal(preview.Reason, applied.Reason); + + // The one wording carries no sentence-ending period of its own: the caller quotes + // it inside a sentence it punctuates itself. + Assert.DoesNotContain("denied", applied.Reason); + Assert.False(applied.Reason.TrimEnd().EndsWith('.')); + + // Nothing moved: not the output, not the temp, and not what the directory held. + Assert.Equal(outputBytes, File.ReadAllBytes(output)); + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + Assert.True(Directory.Exists(adopt)); + Assert.Equal("keep\n", File.ReadAllText(Path.Combine(adopt, "somebody-elses"))); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// A second run's handle on the staged copy. The discard was guarded on this run having + /// opened the name for writing, and FileMode.Create opened whatever stood there - so a + /// promotion that stopped unlinked the copy another run was holding across its own rename, + /// leaving that run writing into a file at no path. The create refuses a name that already + /// exists now, so the promotion stops before it has anything to discard. + /// + [Fact] + public void A_held_staging_name_stops_the_promotion_and_keeps_the_holder_s_file() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var adopt = $"{output}.adopt"; + + // The other run, holding the copy it has just written the way a promotion holds it + // across the rename. + using var holder = new FileStream(adopt, FileMode.CreateNew, FileAccess.ReadWrite, + FileShare.None); + holder.Write("{\"id\":\"theirs\"}\n"u8); + holder.Flush(); + var holderBytes = holder.Length; + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + + Assert.Equal("StagingFailed", applied.Answer); + Assert.Contains("another run has the staged copy open", applied.Reason); + + // The holder still has a file, at the name it wrote it under, with its own bytes. + Assert.True(File.Exists(adopt), "the stop unlinked the copy another run was holding"); + Assert.Equal(holderBytes, holder.Length); + + // Read back through the handle, since it is the only thing that can read the file: + // the hold is FileShare.None, which is what a promotion holds its copy under. + var theirs = new byte[holderBytes]; + holder.Position = 0; + Assert.Equal((int)holderBytes, holder.Read(theirs)); + Assert.Equal("{\"id\":\"theirs\"}\n"u8.ToArray(), theirs); + Assert.Equal(["{\"id\":\"old\"}"], File.ReadAllLines(output)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The other route that stages under the same name: the adoption of a temp left by a run + /// whose checkpoint predates the recorded name and length. It wrote its copy with + /// StreamWriter(path, append: false), which truncates whatever stands at the name and + /// follows a link there like every other open of a path. + /// + [Fact] + public void Adoption_sweeps_a_symlinked_staging_name_and_leaves_its_target_alone() + { + if (OperatingSystem.IsWindows()) return; + + var adopt = typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( + "TryAdoptOrphanedTemp", BindingFlags.Static | BindingFlags.NonPublic)!; + + var dir = NewDir(); + var elsewhere = NewDir(); + try + { + var victim = Path.Combine(elsewhere, "precious.conf"); + var victimBytes = "KEEP-THIS-EXACTLY\n"u8.ToArray(); + File.WriteAllBytes(victim, victimBytes); + + // No output at all, which is the only state adoption runs in. + var output = Path.Combine(dir, "out.jsonl"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + Symlink(victim, $"{output}.adopt"); + + var slots = new object?[adopt.GetParameters().Length]; + slots[0] = output; + slots[1] = 2L; + Assert.True((bool)adopt.Invoke(null, slots)!); + + Assert.Equal(victimBytes, File.ReadAllBytes(victim)); + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + Assert.Null(new FileInfo(output).LinkTarget); + Assert.False(File.Exists($"{output}.adopt")); + Assert.False(File.Exists(temp)); + } + finally + { + try { Directory.Delete(dir, true); } catch { } + try { Directory.Delete(elsewhere, true); } catch { } + } + } + + private static void Chmod(string mode, string path) + { + using var chmod = System.Diagnostics.Process.Start("/bin/chmod", [mode, path]); + Assert.NotNull(chmod); + chmod.WaitForExit(); + Assert.Equal(0, chmod.ExitCode); + } + + /// + /// An output directory this account may read and not write. The staging is a create in that + /// directory and a real run stops on it, and the preview only asked whether something + /// already stood at the staging name: it found nothing, answered that the run would recover + /// the items, and the run then stopped on the write the gate had just promised. The preview + /// makes the create too, at no length, and both passes fail on the same open with the same + /// reason. + /// + [Fact] + public void A_read_only_output_directory_stops_both_passes_on_the_create() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + var outputBytes = "{\"id\":\"old\"}\n"u8.ToArray(); + File.WriteAllBytes(output, outputBytes); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + var adopt = $"{output}.adopt"; + var entriesBefore = Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray(); + Chmod("555", dir); + + var preview = InvokeCanForStaging(output, Path.GetFileName(temp), counted); + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + + Assert.Equal("StagingFailed", preview.Answer); + Assert.Equal("StagingFailed", applied.Answer); + Assert.Equal(preview.Reason, applied.Reason); + Assert.Contains(adopt, applied.Reason); + + Chmod("755", dir); + Assert.Equal(outputBytes, File.ReadAllBytes(output)); + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + Assert.False(File.Exists(adopt)); + Assert.Equal(entriesBefore, Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray()); + } + finally + { + try { Chmod("755", dir); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The reason both passes carry has no sentence-ending period of its own, whichever site + /// built it: the caller quotes it inside a sentence and punctuates it there, and a + /// permission failure's runtime text ends in one - so the stop read "...is denied.. The + /// resume checkpoint..." wherever it was not taken off. One refinement reads the runtime's + /// words now, for the claim on a file and for the create a promotion stages with alike. + /// + [Fact] + public void The_staging_reason_carries_no_period_of_its_own() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + Chmod("555", dir); + + var preview = InvokeCanForStaging(output, Path.GetFileName(temp), counted); + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + Chmod("755", dir); + + // The record prints as "StagingFailure { Path = ..., Reason = ... }", so the reason + // is read out of it rather than off the end of the whole line. + const string opening = "Reason = "; + foreach (var (pass, printed) in new[] + { ("preview", preview.Reason), ("run", applied.Reason) }) + { + var at = printed.LastIndexOf(opening, StringComparison.Ordinal); + Assert.True(at >= 0, $"the {pass} carried no staging reason: {printed}"); + var reason = printed[(at + opening.Length)..].TrimEnd('}', ' '); + Assert.NotEmpty(reason); + Assert.False(reason.EndsWith('.'), + $"the {pass}'s staging reason ends in a period of its own: '{reason}'"); + } + } + finally + { + try { Chmod("755", dir); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// What the preview leaves at the staging name where the promotion would have gone through: + /// nothing. It makes the create a run makes - which is what tells it a directory it cannot + /// write in from one it can - and takes the entry straight back, so the directory holds + /// what it held and the two files the checkpoint stands for are untouched. + /// + [Fact] + public void The_preview_leaves_no_staged_copy_behind() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + var outputBytes = "{\"id\":\"old\"}\n"u8.ToArray(); + File.WriteAllBytes(output, outputBytes); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + var adopt = $"{output}.adopt"; + var entriesBefore = Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray(); + + Assert.Equal("Taken", InvokeCan(output, Path.GetFileName(temp), counted)); + + Assert.False(File.Exists(adopt)); + Assert.Equal(entriesBefore, Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray()); + Assert.Equal(outputBytes, File.ReadAllBytes(output)); + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + + // And the promotion the preview described still lands afterwards. + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + Assert.Equal("Taken", Invoke(run, output, Path.GetFileName(temp), counted)); + run.Dispose(); + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// A hard link at the staging name, standing for the output the promotion is about to + /// replace. Nothing on the entry says it is an alias - a hard link is a name, not a link - + /// so what the sweep asks about is the output's own inode, and the run had already claimed + /// that inode by the time it asked: a second open file description over a locked file is + /// refused even from this process, so the alias answered "another run has the staged copy + /// open" and the promotion stopped on a name nothing but itself was holding. The sweep runs + /// before either claim now, so the alias answers its claim and comes off the name - in the + /// run, which is the pass that clears the name; the preview names it and leaves it there. + /// + [Fact] + public void Sweeps_a_hard_link_to_the_output_off_the_staging_name_and_promotes() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + var outputBytes = "{\"id\":\"old\"}\n"u8.ToArray(); + File.WriteAllBytes(output, outputBytes); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + var adopt = $"{output}.adopt"; + + HardLink(output, adopt); + Assert.Equal("Taken", InvokeCan(output, Path.GetFileName(temp), counted)); + + // The preview holds neither file, so the alias answers a claim rather than reading + // as a staged copy another run has open - and it is left standing, since a preview + // removes nothing. What it names is where it was either way. + Assert.True(File.Exists(adopt), "the preview removed what a run would remove"); + Assert.Equal(outputBytes, File.ReadAllBytes(output)); + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + + Assert.Equal("Taken", applied.Answer); + Assert.Equal("", applied.Reason); + Assert.False(File.Exists(adopt)); + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + Assert.Empty(Directory.GetFiles(dir, "*.tmp")); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The same alias, one file over: a hard link at the staging name for the temp the + /// checkpoint names. The claim on the temp is the first one a promotion takes and the one it + /// holds longest, so this is the shape the old order refused hardest - and the temp itself + /// is never what the sweep removes, only the extra name. + /// + [Fact] + public void Sweeps_a_hard_link_to_the_temp_off_the_staging_name_and_promotes() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + var outputBytes = "{\"id\":\"old\"}\n"u8.ToArray(); + File.WriteAllBytes(output, outputBytes); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + var adopt = $"{output}.adopt"; + + HardLink(temp, adopt); + Assert.Equal("Taken", InvokeCan(output, Path.GetFileName(temp), counted)); + + Assert.True(File.Exists(adopt), "the preview removed what a run would remove"); + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + Assert.Equal(outputBytes, File.ReadAllBytes(output)); + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + + Assert.Equal("Taken", applied.Answer); + Assert.Equal("", applied.Reason); + Assert.False(File.Exists(adopt)); + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + Assert.Empty(Directory.GetFiles(dir, "*.tmp")); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// Two promotions over one output, released together, with a copy left at the staging name + /// by a run killed between the write and the rename. The sweep asked for that copy, let the + /// claim go, and unlinked the name afterwards - and what the other promotion does in between + /// is sweep the same copy, create its own at the name and hold it, so the stale unlink took + /// the second run's staged file out from under its handle and that run stopped on a name + /// that was no longer there. The claim is held across the unlink now, which refuses the + /// second sweep outright: whichever run gets there second either lands its own promotion or + /// stops with its temp where it was, and neither loses a file to the other. + /// + [Fact] + public void Two_promotions_over_one_leftover_never_unlink_each_others_files() + { + if (OperatingSystem.IsWindows()) return; + + for (var iteration = 0; iteration < 40; iteration++) + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var tempA = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + var tempB = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(tempA, "{\"id\":\"a1\"}\n{\"id\":\"a2\"}\n"); + File.WriteAllText(tempB, "{\"id\":\"b1\"}\n{\"id\":\"b2\"}\n"); + var countedA = new FileInfo(tempA).Length; + var countedB = new FileInfo(tempB).Length; + var adopt = $"{output}.adopt"; + File.WriteAllText(adopt, "LEFTOVER-FROM-AN-INTERRUPTED-PROMOTION\n"); + + var runA = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var runB = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + (string Answer, string Reason) a = default; + (string Answer, string Reason) b = default; + using var released = new Barrier(2); + var threads = new[] + { + new Thread(() => + { + released.SignalAndWait(); + a = InvokeForStaging(runA, output, Path.GetFileName(tempA), countedA); + }) { IsBackground = true }, + new Thread(() => + { + released.SignalAndWait(); + b = InvokeForStaging(runB, output, Path.GetFileName(tempB), countedB); + }) { IsBackground = true }, + }; + try + { + foreach (var t in threads) t.Start(); + foreach (var t in threads) + Assert.True(t.Join(TimeSpan.FromSeconds(30)), + "a promotion never returned"); + } + finally + { + runA.Dispose(); + runB.Dispose(); + } + + // One landed. Neither answer is a file that vanished: the run that stopped says + // so about a name another run holds or an output another run has, never about a + // copy of its own that is no longer there. + Assert.Single(new[] { a.Answer, b.Answer }.Where(x => x == "Taken")); + Assert.DoesNotContain("Could not find file", a.Reason); + Assert.DoesNotContain("Could not find file", b.Reason); + + // The output holds one run's rows and not a weave of both, the run that stopped + // still has the temp it would come back to, and the leftover is gone. + var landed = a.Answer == "Taken"; + Assert.Equal( + landed + ? ["{\"id\":\"a1\"}", "{\"id\":\"a2\"}"] + : ["{\"id\":\"b1\"}", "{\"id\":\"b2\"}"], + File.ReadAllLines(output)); + Assert.True(File.Exists(landed ? tempB : tempA), + "the promotion that stopped lost the temp it was to come back to"); + Assert.False(File.Exists(adopt)); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + } + + /// + /// A directory standing at the staging name where the create is what meets it, which is the + /// instant between the sweep and the create. The sweep calls that entry a directory; a + /// create refuses a name that already exists with the runtime's "already exists" whatever + /// kind of entry wears it, so one disk state came back in two wordings depending on which of + /// the two instructions reached it - and "already exists" reads as a file in the way, which + /// a caller answers by removing it. The refinement asks the filesystem on the create's route + /// too. + /// + [Fact] + public void A_directory_the_create_meets_reads_as_a_directory_and_not_as_a_file_in_the_way() + { + var reasonFor = typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( + "ReasonForUnopenable", BindingFlags.Static | BindingFlags.NonPublic)!; + + var dir = NewDir(); + try + { + var adopt = Path.Combine(dir, "out.jsonl.adopt"); + Directory.CreateDirectory(adopt); + + // The create a promotion stages its copy with, against the entry that appeared under + // it. The runtime's own words for it name no directory on either platform, and they + // are not the same words or even the same exception: a create meets a directory as a + // name that already exists on Unix and as an access failure on Windows. Which of the + // two the platform raises is not the question - both reach the refinement through + // the promotion's own catch, and what it makes of them is. + var raised = Assert.ThrowsAny(() => new FileStream( + adopt, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None)); + Assert.True(raised is IOException or UnauthorizedAccessException, + $"the create raised {raised.GetType()}, which the promotion does not catch"); + Assert.DoesNotContain("directory", raised.Message, StringComparison.OrdinalIgnoreCase); + + var slots = new object?[reasonFor.GetParameters().Length]; + slots[0] = adopt; + slots[1] = raised; + // Raised by a create, where the refinement is told which open it came from. + for (var i = 2; i < slots.Length; i++) slots[i] = true; + var refined = reasonFor.Invoke(null, slots)!.ToString()!; + + Assert.Contains("a directory stands at the path", refined); + Assert.DoesNotContain("already exists", refined); + + // And a failure from any other open still carries the runtime's words: the question + // the create's refusal cannot answer for itself is not asked of every exception. + slots[1] = new IOException("Too many levels of symbolic links."); + for (var i = 2; i < slots.Length; i++) slots[i] = false; + Assert.Contains("Too many levels of symbolic links", + reasonFor.Invoke(null, slots)!.ToString()!); + } + finally { try { Directory.Delete(dir, true); } catch { } } + } + + /// + /// The claim a promotion takes on the output the move will replace, over an output that is + /// not there yet - which is the ordinary end of a fresh run interrupted into a temp. It + /// opened FileMode.Open and answered Absent with no handle at all, so nothing was held + /// through the staging or the rename: two runs with different temps over one output path both + /// got past this line, both renamed a copy onto it, and the first one's rows ended in an + /// unlinked inode with its own handle appending into a file at no path. The claim makes the + /// file in order to hold it now, and a second promotion is refused on it the way it is + /// refused on an output that was already there. + /// + [Fact] + public void The_claim_on_an_absent_output_makes_the_file_and_refuses_a_second_promotion() + { + if (OperatingSystem.IsWindows()) return; + + var claimStanding = typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( + "ClaimStandingOutput", BindingFlags.Static | BindingFlags.NonPublic)!; + + var dir = NewDir(); + FileStream? held = null; + try + { + // No output at all, and a temp for each of the two runs. + var output = Path.Combine(dir, "out.jsonl"); + var theirs = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(theirs, "{\"id\":\"t1\"}\n{\"id\":\"t2\"}\n"); + var theirsCounted = new FileInfo(theirs).Length; + var theirsBytes = File.ReadAllBytes(theirs); + var mine = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(mine, "{\"id\":\"m1\"}\n{\"id\":\"m2\"}\n"); + var mineCounted = new FileInfo(mine).Length; + Assert.False(File.Exists(output)); + + // The first run, stopped where its claim is taken: what it holds from before the + // staging through the rename, and nothing else of the promotion. + var slots = new object?[claimStanding.GetParameters().Length]; + slots[0] = output; + var answer = claimStanding.Invoke(null, slots)!.ToString()!; + held = (FileStream?)slots[1]; + var created = slots.Length > 3 && slots[3] is true; + + Assert.Equal("Taken", answer); + Assert.True(created, "the claim on an absent output reported making nothing"); + Assert.NotNull(held); + Assert.True(File.Exists(output), "the claim on an absent output made no file to hold"); + Assert.Equal(0, new FileInfo(output).Length); + + // The second run, over the same output path with its own temp, while the first is + // between its claim and its move. Refused on the output, with both of its files + // where it found them. + using (var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection()) + { + Assert.Equal("OutputHeld", Invoke(run, output, Path.GetFileName(theirs), + theirsCounted)); + } + + Assert.Equal(theirsBytes, File.ReadAllBytes(theirs)); + Assert.Equal(0, new FileInfo(output).Length); + Assert.False(File.Exists($"{output}.adopt")); + + // And once the first run lets go, a promotion lands in the file the claim made: the + // rename replaces it, so nothing of the empty one is left to remove. + held.Dispose(); + held = null; + using (var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection()) + { + Assert.Equal("Taken", Invoke(run, output, Path.GetFileName(mine), mineCounted)); + } + + Assert.Equal(["{\"id\":\"m1\"}", "{\"id\":\"m2\"}"], File.ReadAllLines(output)); + Assert.False(File.Exists(mine)); + } + finally + { + held?.Dispose(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// An output that is not there in a directory this account may read and not write. The claim + /// cannot make the file, and what refused it is the directory and not the file it would name + /// - so the answer is left to the staging create, in that same directory, which is the + /// question both passes ask and answer in one wording. Nothing of either pass is left at the + /// output path. + /// + [Fact] + public void A_read_only_directory_with_no_output_stops_both_passes_on_the_create() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + var entriesBefore = Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray(); + Chmod("555", dir); + + var preview = InvokeCanForStaging(output, Path.GetFileName(temp), counted); + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + Chmod("755", dir); + + Assert.Equal("StagingFailed", preview.Answer); + Assert.Equal("StagingFailed", applied.Answer); + Assert.Equal(preview.Reason, applied.Reason); + Assert.Contains($"{output}.adopt", applied.Reason); + + // The temp is where it was, no output was left standing at the path, and the + // directory holds exactly the entries it held. + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + Assert.False(File.Exists(output)); + Assert.Equal(entriesBefore, Directory.GetFiles(dir).Select(Path.GetFileName) + .OrderBy(n => n, StringComparer.Ordinal).ToArray()); + } + finally + { + try { Chmod("755", dir); } catch { } + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The interleaving two promotions over one output reach on Unix, at the instant that makes + /// it: the claim on the output is granted several syscalls after the open that resolved the + /// name - .NET takes the lock behind FileShare.None after open(2) - and a second run that + /// renames its own copy onto the name in between leaves this one holding an inode with no + /// name on it. Read as a claim, that is a promotion renaming its copy over a file another + /// run has just promoted, and both runs answering that they took the output. The seam below + /// is that instant, fired on every attempt, so the name never settles: the answer is the + /// held stop, and nothing at the path is replaced. + /// + [Fact] + public void An_output_replaced_under_the_claim_is_not_an_output_this_run_took() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + + // The second run's rename, landing in the instant between this run's claim being + // granted and the check that the name still reaches what it granted. A fresh file + // each time, so which one stands at the path says how many attempts were made. + var renames = 0; + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = claimed => + { + if (!string.Equals(claimed, output, StringComparison.Ordinal)) return; + renames++; + var theirs = Path.Combine(dir, $"theirs-{renames}.jsonl"); + File.WriteAllText(theirs, $"{{\"id\":\"theirs-{renames}\"}}\n"); + File.Move(theirs, output, overwrite: true); + }; + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + + // What the other run left at the path is whole, this run's temp is where it was, + // and no copy was staged beside either - the answer is the hold it is, and the + // claim was asked three times to reach it. + Assert.Equal("OutputHeld", applied.Answer); + Assert.Equal(["{\"id\":\"theirs-3\"}"], File.ReadAllLines(output)); + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + Assert.False(File.Exists($"{output}.adopt")); + Assert.Equal(3, renames); + } + finally + { + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = null; + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The same instant in the sweep of the staging name, which is where the interleaving costs + /// the OTHER run its file: this run's claim on the leftover is granted after a second run + /// has taken that leftover off the name and created its own staged copy there, and the + /// unlink that follows a claim asks the directory - so it takes the second run's copy, and + /// that run's rename ends on a name that is no longer there. The sweep answers the hold it + /// met now, and the copy the other run is holding stays where it put it. + /// + [Fact] + public void A_staging_name_taken_under_the_sweep_s_claim_is_not_the_sweep_s_to_unlink() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + FileStream? theirs = null; + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + var adopt = $"{output}.adopt"; + File.WriteAllText(adopt, "LEFTOVER-FROM-AN-INTERRUPTED-PROMOTION\n"); + + // The second run, in that instant: it swept the same leftover, created its staged + // copy at the name and holds it there through the rename it is about to make. + var swept = 0; + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = claimed => + { + if (!string.Equals(claimed, adopt, StringComparison.Ordinal)) return; + if (swept++ > 0) return; + File.Delete(adopt); + theirs = new FileStream(adopt, FileMode.CreateNew, FileAccess.ReadWrite, + FileShare.None); + theirs.Write("{\"id\":\"theirs\"}\n"u8); + theirs.Flush(); + }; + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + + // The name is the other run's, which is the answer a name a second run holds always + // had here - and this run stopped on it with its own temp untouched. + Assert.Equal("StagingFailed", applied.Answer); + Assert.Contains("another run has the staged copy open", applied.Reason); + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + Assert.Equal(1, swept); + + // And the rename that copy is being held for lands, because the copy is still there. + File.Move(adopt, output, overwrite: true); + theirs!.Dispose(); + theirs = null; + Assert.Equal(["{\"id\":\"theirs\"}"], File.ReadAllLines(output)); + } + finally + { + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = null; + theirs?.Dispose(); + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// The unlink at the end of a promotion that landed, which is the same question one file + /// later: the temp goes because this run read it, and what goes is the file this run holds - + /// not whatever wears its name by then. A second run that took the name in the interval + /// keeps its file. + /// + [Fact] + public void A_temp_replaced_at_its_name_is_not_the_temp_a_promotion_unlinks() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var moved = Path.Combine(dir, "moved-away.tmp"); + + // Two claims are taken on the temp's name in one promotion: the one it is read + // through, and the check in front of the unlink that spends it. The second run + // arrives between them. + var claims = 0; + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = claimed => + { + if (!string.Equals(claimed, temp, StringComparison.Ordinal)) return; + if (++claims != 2) return; + File.Move(temp, moved); + File.WriteAllText(temp, "{\"id\":\"theirs\"}\n"); + }; + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var answer = Invoke(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + + // The rows are in the output, and the file standing at the temp's name is the other + // run's, whole. The temp this promotion actually spent is where that run moved it, + // for the sweep that reaches it next. + Assert.Equal("Taken", answer); + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + Assert.Equal(["{\"id\":\"theirs\"}"], File.ReadAllLines(temp)); + Assert.True(File.Exists(moved)); + Assert.Equal(2, claims); + } + finally + { + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = null; + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// And the empty output a promotion makes in order to hold it, taken back where the + /// promotion stops. That unlink is the same shape as the two above: what goes is the file + /// this claim made, and a second run standing at the name by then keeps what it put there. + /// + [Fact] + public void An_output_the_claim_made_is_not_taken_back_from_a_second_run() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + // Nothing at the output path, so the claim makes the file in order to hold it. + var output = Path.Combine(dir, "out.jsonl"); + var adopt = $"{output}.adopt"; + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + + var claims = 0; + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = claimed => + { + if (!string.Equals(claimed, output, StringComparison.Ordinal)) return; + if (++claims == 1) + { + // A directory at the staging name, in the instant after the sweep passed + // it: the create the copy is staged with meets it, and the promotion stops + // with the empty file it made still standing at the output path. + Directory.CreateDirectory(adopt); + return; + } + if (claims != 2) return; + + // And the second run, in the instant before that file is taken back: it is + // gone from the name and theirs stands there instead. + File.Delete(output); + File.WriteAllText(output, "{\"id\":\"theirs\"}\n"); + }; + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + + Assert.Equal(["{\"id\":\"theirs\"}"], File.ReadAllLines(output)); + Assert.Equal("StagingFailed", applied.Answer); + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + Assert.Equal(2, claims); + } + finally + { + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = null; + try { Directory.Delete(dir, true); } catch { } + } + } + + /// + /// A claim granted, and then a question about it the platform cannot answer: the output's + /// parent directory stops being searchable in the instant between the two, so the stat of + /// the name comes back EACCES - which is not the name resolving to nothing, and is not read + /// as "still there". A claim that cannot be verified is not a claim, so the caller is handed + /// a refusal in the shape it already reads every refusal in: the file could not be opened, + /// the reason says why, and the handle comes back with it for the caller to let go of the + /// way it lets go of one it kept. A resumed run asks that way, and keeps nothing. + /// + [Fact] + public void A_claim_the_platform_cannot_verify_is_refused_with_the_handle_handed_back() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var contents = File.ReadAllBytes(output); + + // The premise: a directory with no search bit answers for no name under it. Where + // the stat still answers, the account is root, a mode says nothing, and there is + // nothing here to measure. + SetMode(dir, 0); + var unsearchable = !File.Exists(output); + SetMode(dir, Searchable); + if (!unsearchable) return; + + var looks = 0; + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = claimed => + { + if (!string.Equals(claimed, output, StringComparison.Ordinal)) return; + looks++; + SetMode(dir, 0); + }; + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var args = new object?[] { output, (long)contents.Length, null }; + var answer = Resume.Invoke(run, args)!.ToString()!; + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = null; + SetMode(dir, Searchable); + + // The answer a run words its stop out of, and the reason it quotes in it: the call, + // the errno, and what could not be finished with them. + Assert.Equal("CannotBeOpened", answer); + Assert.Equal(1, looks); + Assert.Contains("cannot be verified", args[2]!.ToString()!); + + // The handle went with the answer, on the caller's own path. Asked for again the way + // every later claim asks for it, the file is granted at once - where a failure + // carried out of the claim with the stream still open left an exclusive lock on the + // caller's own output until the finalizer got around to it, with nothing + // deterministic able to say when. + using var next = new FileStream(output, FileMode.Open, FileAccess.ReadWrite, + FileShare.None); + Assert.Equal(contents.Length, next.Length); + } + finally + { + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = null; + try { SetMode(dir, Searchable); Directory.Delete(dir, true); } + catch { } + } + } + + /// + /// The same failed question at the one caller that lets its claim go in a finally. A + /// promotion over an output that is not there yet makes the file in order to hold it, and + /// takes that file away again on every answer but Taken - under the claim that made it, so + /// the name goes only while this run is still what stands at it. That take-back is armed by + /// the handle and by the claim's own answer about what it made, and a refusal that clears + /// either of them disarms it: the finally then holds nothing, unlinks nothing, and a file + /// with no rows in it that no run holds is left standing at the caller's -OutputFile. + /// + /// The parent answers again for the second look, which is the one the take-back is made + /// under: the failure being transient is what makes the empty file outlive it. + /// + /// + [Fact] + public void An_output_made_to_hold_a_claim_that_cannot_be_verified_is_taken_back() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + + // The premise, as above: a directory with no search bit answers for no name under + // it, and an account it does not refuse is one there is nothing here to measure on. + SetMode(dir, 0); + var unsearchable = !File.Exists(temp); + SetMode(dir, Searchable); + if (!unsearchable) return; + + var looks = 0; + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = claimed => + { + if (!string.Equals(claimed, output, StringComparison.Ordinal)) return; + + // The claim on the output, which cannot be verified; then the finally's look at + // the file it made, which can. + SetMode(dir, ++looks == 1 ? 0 : Searchable); + }; + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var args = Slots(Promote, output, Path.GetFileName(temp), counted); + var answer = Promote.Invoke(run, args)!.ToString()!; + run.Dispose(); + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = null; + SetMode(dir, Searchable); + + // Nothing of this run's making left at either name, and the temp holding every item + // the checkpoint counts exactly as it was found - so the run that comes back + // promotes the same bytes to the same length over the same output. + Assert.False(File.Exists(output), + "the empty output the claim made is left standing at the path"); + Assert.False(File.Exists($"{output}.adopt")); + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + + // Two looks: the claim's, which could not be answered, and the finally's, which is + // the one the take-back was made under. + Assert.Equal(2, looks); + Assert.Equal("OutputUnopenable", answer); + Assert.Contains("cannot be verified", args[3]!.ToString()!); + } + finally + { + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = null; + try { SetMode(dir, Searchable); Directory.Delete(dir, true); } + catch { } + } + } + + /// + /// The output leaving the name in the instant the claim on it was granted, and the directory + /// refusing a new file in the same one: the first look found a file, so the claim opened + /// rather than created; the file left the name, so the claim was dropped and asked again; + /// and the attempt that answers is a create the directory refused. Which of those two the + /// answer stands for decides who words the stop. A create the directory refused is the + /// staging create's own answer a few lines later, in that same directory, and the preview + /// asks it too - so the claim leaves the wording there rather than naming the output in a + /// sentence only the applying pass can reach. + /// + [Fact] + public void A_create_refused_at_an_output_that_went_is_left_to_the_staging_create_s_words() + { + if (OperatingSystem.IsWindows()) return; + + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText(output, "{\"id\":\"old\"}\n"); + var adopt = $"{output}.adopt"; + var temp = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllText(temp, "{\"id\":\"a\"}\n{\"id\":\"b\"}\n"); + var counted = new FileInfo(temp).Length; + var tempBytes = File.ReadAllBytes(temp); + + // The premise: a directory this account may not write refuses a new file in it. + // Where it takes one, the account is root and there is nothing here to measure. + var probe = Path.Combine(dir, "probe"); + SetMode(dir, ReadAndSearch); + var refused = false; + try { File.WriteAllText(probe, "x"); } + catch (UnauthorizedAccessException) { refused = true; } + SetMode(dir, Searchable); + if (!refused) + { + File.Delete(probe); + return; + } + + var looks = 0; + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = claimed => + { + if (!string.Equals(claimed, output, StringComparison.Ordinal)) return; + if (looks++ > 0) return; + + // The file this claim was granted over, off the name - and the directory closed + // behind it, so the attempt that follows finds nothing at the name, asks for + // the create, and is refused it. + File.Delete(output); + SetMode(dir, ReadAndSearch); + }; + + using var run = new Mgx.Cmdlets.Cmdlets.Export.ExportMgxCollection(); + var applied = InvokeForStaging(run, output, Path.GetFileName(temp), counted); + run.Dispose(); + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = null; + + // The stop is the staging create's, in its own words, over the name it was making. + Assert.Equal("StagingFailed", applied.Answer); + Assert.Equal(1, looks); + Assert.Contains(adopt, applied.Reason); + Assert.Contains("is denied", applied.Reason); + + // And it is the wording the preview reaches over the same disk state, which is the + // whole reason the claim does not answer this one itself. + var preview = InvokeCanForStaging(output, Path.GetFileName(temp), counted); + Assert.Equal("StagingFailed", preview.Answer); + Assert.Equal(preview.Reason, applied.Reason); + + // Nothing was left at either name: no empty output of this run's making, and the + // temp holding the items exactly as it was found. + Assert.False(File.Exists(output)); + Assert.False(File.Exists(adopt)); + Assert.Equal(tempBytes, File.ReadAllBytes(temp)); + } + finally + { + Mgx.Engine.Pagination.HeldEntry.BeforeTheClaimIsVerified = null; + try { SetMode(dir, Searchable); Directory.Delete(dir, true); } + catch { } + } + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/OpaqueLinkTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/OpaqueLinkTests.cs new file mode 100644 index 0000000..22b12ca --- /dev/null +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/OpaqueLinkTests.cs @@ -0,0 +1,54 @@ +using Mgx.Engine.Pagination; + +namespace Mgx.IntegrationTests; + +/// +/// A service-issued link is opaque: validation may refuse it, but a link that passes is +/// followed byte-identically - no re-encoding, no normalization, no re-parsing. A repaired +/// link is a different link, and Graph's skiptokens do not survive repair. +/// (Corpus: GraphSDK-2488, opaque service links.) +/// +public class OpaqueLinkTests +{ + private static readonly Uri GraphHost = new("https://graph.microsoft.com/v1.0/users"); + + [Fact] + public void A_validated_link_is_the_original_string_byte_for_byte() + { + foreach (var link in HostileInputs.OpaqueLinks) + { + var validated = NextLinkValidator.Validate(link, GraphHost); + Assert.NotNull(validated); + Assert.Same(link, validated); // the same string instance - no rebuild happened + } + } + + [Fact] + public void ValidateOrThrow_returns_the_original_string_too() + { + foreach (var link in HostileInputs.OpaqueLinks) + Assert.Same(link, NextLinkValidator.ValidateOrThrow(link, GraphHost)); + } + + [Fact] + public void A_checkpoint_round_trips_its_link_byte_for_byte() + { + foreach (var link in HostileInputs.OpaqueLinks) + { + var path = Path.Combine(Path.GetTempPath(), $"mgx-oplink-{Guid.NewGuid():N}.json"); + try + { + new PaginationCheckpoint + { + Resource = "https://graph.microsoft.com/v1.0/users", + NextLink = link, + ItemsCollected = 1, + }.Save(path); + var loaded = PaginationCheckpoint.Load(path); + Assert.NotNull(loaded); + Assert.Equal(link, loaded!.NextLink); + } + finally { File.Delete(path); } + } + } +} diff --git a/tests/Mgx.IntegrationTests/Engine/Pagination/OrphanAdoptionTests.cs b/tests/Mgx.IntegrationTests/Engine/Pagination/OrphanAdoptionTests.cs index 63a9ad2..03ad736 100644 --- a/tests/Mgx.IntegrationTests/Engine/Pagination/OrphanAdoptionTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/Pagination/OrphanAdoptionTests.cs @@ -4,10 +4,10 @@ namespace Mgx.IntegrationTests.Engine.Pagination; /// /// Orphan-temp adoption after a JSONL run is interrupted, for checkpoints that predate the -/// recorded temp name and length. With only a line count and a glob to go on, adoption is -/// safe solely when no output exists; against an existing output it must refuse and leave the -/// caller to re-enumerate, since nothing ties the newest temp to the run the checkpoint is -/// actually about. +/// recorded temp name and length. With only a line count and the newest file a run could have +/// named - "{output}.{32-hex}.tmp" - to go on, adoption is safe solely when no output exists; +/// against an existing output it must refuse and leave the caller to re-enumerate, since +/// nothing ties that temp to the run the checkpoint is actually about. /// public class OrphanAdoptionTests { @@ -15,8 +15,28 @@ public class OrphanAdoptionTests typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( "TryAdoptOrphanedTemp", BindingFlags.Static | BindingFlags.NonPublic)!; + private static readonly MethodInfo CanAdopt = + typeof(Mgx.Cmdlets.Base.MgxCmdletBase).GetMethod( + "CanAdoptOrphanedTemp", BindingFlags.Static | BindingFlags.NonPublic)!; + + /// + /// The two arguments, followed by an empty slot for each out parameter the method carries - + /// the temp it picked at 2, the staging failure at 3, what the sweep removed at 4. Sized off + /// the method, since reflection wants a slot for every one of them. + /// + private static object?[] Slots(MethodInfo method, string outputPath, long itemCount) + { + var slots = new object?[method.GetParameters().Length]; + slots[0] = outputPath; + slots[1] = itemCount; + return slots; + } + private static bool Invoke(string outputPath, long itemCount) => - (bool)Adopt.Invoke(null, [outputPath, itemCount])!; + (bool)Adopt.Invoke(null, Slots(Adopt, outputPath, itemCount))!; + + private static bool InvokeCan(string outputPath, long itemCount) => + (bool)CanAdopt.Invoke(null, Slots(CanAdopt, outputPath, itemCount))!; private static string NewDir() { @@ -32,7 +52,7 @@ public void Adopts_the_temp_when_no_output_exists_yet() try { var output = Path.Combine(dir, "out.jsonl"); - File.WriteAllLines($"{output}.abc.tmp", ["{\"id\":\"a\"}", "{\"id\":\"b\"}", "{\"id\":\"c\"}"]); + File.WriteAllLines($"{output}.{Guid.NewGuid():N}.tmp", ["{\"id\":\"a\"}", "{\"id\":\"b\"}", "{\"id\":\"c\"}"]); Assert.True(Invoke(output, 2)); @@ -54,7 +74,7 @@ public void Refuses_when_an_output_already_exists() File.WriteAllLines(output, ["{\"id\":\"run1-1\"}", "{\"id\":\"run1-2\"}"]); // Run 2 crashed: a temp is on disk, but nothing says it is run 2's rather than a // survivor of some unrelated enumeration against the same output name. - File.WriteAllLines($"{output}.def.tmp", ["{\"id\":\"run2-1\"}", "{\"id\":\"run2-2\"}", "{\"id\":\"unflushed\"}"]); + File.WriteAllLines($"{output}.{Guid.NewGuid():N}.tmp", ["{\"id\":\"run2-1\"}", "{\"id\":\"run2-2\"}", "{\"id\":\"unflushed\"}"]); Assert.False(Invoke(output, 2)); @@ -74,7 +94,7 @@ public void Refuses_when_the_temp_holds_less_than_the_checkpoint_promises() { var output = Path.Combine(dir, "out.jsonl"); File.WriteAllLines(output, ["{\"id\":\"kept\"}"]); - File.WriteAllLines($"{output}.ghi.tmp", ["{\"id\":\"only-one\"}"]); + File.WriteAllLines($"{output}.{Guid.NewGuid():N}.tmp", ["{\"id\":\"only-one\"}"]); Assert.False(Invoke(output, 5)); @@ -85,6 +105,131 @@ public void Refuses_when_the_temp_holds_less_than_the_checkpoint_promises() finally { Directory.Delete(dir, true); } } + /// + /// A count is a count of line STARTS unless something says otherwise, and ReadLine returns + /// an unterminated tail as a line like any other - so a temp cut inside the row the + /// checkpoint counts handed the fragment over and it was written into the output as an + /// item, with the rest of the enumeration appended behind it. mgx's own runs flush before + /// they count, so this is a temp truncated under one rather than one it left. + /// + [Fact] + public void Refuses_when_the_counted_row_was_cut_short() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText($"{output}.{Guid.NewGuid():N}.tmp", + "{\"id\":\"a\"}\n{\"id\":\"b\"}\n{\"id\":\"to"); + + Assert.False(Invoke(output, 3)); + + Assert.False(File.Exists(output)); + Assert.False(File.Exists($"{output}.adopt")); + } + finally { Directory.Delete(dir, true); } + } + + /// + /// The state an interrupted run does leave: rows written after the last flush are torn or + /// missing, and the checkpoint counts none of them. Refusing here would cost a + /// re-enumeration for nothing. + /// + [Fact] + public void Adopts_when_only_the_uncounted_row_was_cut_short() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + File.WriteAllText($"{output}.{Guid.NewGuid():N}.tmp", + "{\"id\":\"a\"}\n{\"id\":\"b\"}\n{\"id\":\"to"); + + Assert.True(Invoke(output, 2)); + + Assert.Equal(["{\"id\":\"a\"}", "{\"id\":\"b\"}"], File.ReadAllLines(output)); + Assert.Empty(Directory.GetFiles(dir, "*.tmp")); + } + finally { Directory.Delete(dir, true); } + } + + /// + /// "Orphan" is an assumption about a file this run did not write, and a second export + /// against the same output holds one matching the same glob right now. Adoption copies what + /// it picks into the output and then unlinks it, so it took the live run's rows as its own + /// and left that run writing into an unlinked inode - the failure the stale-temp sweep asks + /// CanTakeExclusively about before it deletes anything. Adoption has to ask the same + /// question, and a file it cannot claim is not an orphan. + /// + [Fact] + public void Refuses_a_temp_a_live_run_still_holds() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + var live = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + + // Opened the way an export opens its own temp: new StreamWriter(path, append). + using (var writer = new StreamWriter(live, append: false)) + { + writer.WriteLine("{\"id\":\"other-1\"}"); + writer.WriteLine("{\"id\":\"other-2\"}"); + writer.Flush(); + + Assert.False(InvokeCan(output, 2)); + Assert.False(Invoke(output, 2)); + + Assert.False(File.Exists(output)); + Assert.False(File.Exists($"{output}.adopt")); + Assert.True(File.Exists(live), "a running export's temp was consumed as an orphan"); + + // The run that owns it goes on writing into the file it still has. + writer.WriteLine("{\"id\":\"other-3\"}"); + } + + Assert.Equal( + ["{\"id\":\"other-1\"}", "{\"id\":\"other-2\"}", "{\"id\":\"other-3\"}"], + File.ReadAllLines(live)); + } + finally { Directory.Delete(dir, true); } + } + + /// + /// The newest temp is the one a live run is writing into, which is exactly the file + /// adoption must not take - so passing over it cannot mean giving up. The dead run's temp + /// underneath is the orphan recovery exists for, and it is still the one to adopt. + /// + [Fact] + public void Adopts_the_orphan_underneath_a_temp_a_live_run_holds() + { + var dir = NewDir(); + try + { + var output = Path.Combine(dir, "out.jsonl"); + + // A dead run's temp, aged so the live one below is unambiguously newer. + var orphan = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + File.WriteAllLines(orphan, ["{\"id\":\"dead-1\"}", "{\"id\":\"dead-2\"}"]); + File.SetLastWriteTimeUtc(orphan, DateTime.UtcNow.AddMinutes(-5)); + + var live = Path.Combine(dir, $"out.jsonl.{Guid.NewGuid():N}.tmp"); + using (var writer = new StreamWriter(live, append: false)) + { + writer.WriteLine("{\"id\":\"live-1\"}"); + writer.WriteLine("{\"id\":\"live-2\"}"); + writer.Flush(); + + Assert.True(Invoke(output, 2)); + + Assert.Equal(["{\"id\":\"dead-1\"}", "{\"id\":\"dead-2\"}"], File.ReadAllLines(output)); + Assert.False(File.Exists(orphan)); + Assert.True(File.Exists(live), "a running export's temp was consumed as an orphan"); + } + } + finally { Directory.Delete(dir, true); } + } + [Fact] public void Does_nothing_when_there_is_no_temp() { diff --git a/tests/Mgx.IntegrationTests/Engine/SdkVersionTests.cs b/tests/Mgx.IntegrationTests/Engine/SdkVersionTests.cs index 80dd1c4..f7068b5 100644 --- a/tests/Mgx.IntegrationTests/Engine/SdkVersionTests.cs +++ b/tests/Mgx.IntegrationTests/Engine/SdkVersionTests.cs @@ -28,18 +28,32 @@ private static string ManifestVersion() return match.Groups[1].Value; } - /// Walk up from the test binaries until the repository-relative path exists. + /// The file that marks the repository root. + private const string RepositoryMarker = "Mgx.slnx"; + + /// + /// Resolve a repository-relative file under the repository root - the first ancestor of the + /// test binaries holding Mgx.slnx. Nothing above that root is consulted, so a module folder + /// beside a worktree checked out under it cannot supply the manifest this pins against. + /// private static string FindRepositoryFile(string relativePath) { var dir = new DirectoryInfo(AppContext.BaseDirectory); while (dir != null) { - var candidate = Path.Combine(dir.FullName, relativePath); - if (File.Exists(candidate)) return candidate; + if (File.Exists(Path.Combine(dir.FullName, RepositoryMarker))) + { + var candidate = Path.Combine(dir.FullName, relativePath); + if (File.Exists(candidate)) return candidate; + + throw new FileNotFoundException( + $"Could not locate '{relativePath}' under repository root {dir.FullName}"); + } dir = dir.Parent; } - throw new FileNotFoundException($"Could not locate '{relativePath}' above {AppContext.BaseDirectory}"); + throw new DirectoryNotFoundException( + $"Could not locate {RepositoryMarker} above {AppContext.BaseDirectory}"); } [Fact] diff --git a/tests/Mgx.IntegrationTests/Fakes/FakeGraphSession.cs b/tests/Mgx.IntegrationTests/Fakes/FakeGraphSession.cs index 1940408..04473f3 100644 --- a/tests/Mgx.IntegrationTests/Fakes/FakeGraphSession.cs +++ b/tests/Mgx.IntegrationTests/Fakes/FakeGraphSession.cs @@ -1,142 +1,42 @@ -using System.Reflection; -using Mgx.Cmdlets.Base; - -namespace Mgx.IntegrationTests.Fakes; - -/// -/// The AuthContext shape MgxCmdletBase fingerprints. Only the members it reads are present. -/// -public sealed class FakeAuthContext -{ - public string? TenantId { get; set; } = "11111111-1111-1111-1111-111111111111"; - public string? ClientId { get; set; } = "22222222-2222-2222-2222-222222222222"; - public string? AuthType { get; set; } = "Delegated"; - public string? Account { get; set; } = "user@contoso.com"; - public string? Authority { get; set; } = "https://login.microsoftonline.com/common"; - public string[]? Scopes { get; set; } = ["User.Read.All"]; -} - -public sealed class FakeGraphEnvironment -{ - public string? GraphEndpoint { get; set; } = "https://graph.microsoft.com"; - public string? AzureADEndpoint { get; set; } = "https://login.microsoftonline.com"; -} +namespace Microsoft.Graph.PowerShell.Authentication; /// -/// Stands in for Microsoft.Graph.PowerShell.Authentication.GraphSession, found through -/// rather than by name. +/// Stands in for the Graph SDK's GraphSession singleton. MgxCmdletBase.FindType resolves that +/// type by full name across the loaded assemblies, so a type of this name declared here is what +/// the injection paths find in a process where the SDK is absent. +/// +/// Everything mgx reads off the real session is duck-typed reflection - Instance, AuthContext, +/// GraphHttpClient, Environment - so matching the member names is the whole contract. +/// +/// Instance is null until a test arms it through Mgx.IntegrationTests.GraphSessionScope, and the +/// scope detaches it again on dispose. Nothing else in the suite sees a session. /// -public sealed class FakeGraphSession +internal sealed class GraphSession { - public static FakeGraphSession? Instance { get; set; } + private static GraphSession? s_instance; - public object? AuthContext { get; set; } = new FakeAuthContext(); + public static GraphSession? Instance => s_instance; - public object? Environment { get; set; } = new FakeGraphEnvironment(); + public object? AuthContext { get; set; } public HttpClient? GraphHttpClient { get; set; } -} -/// -/// Stands in for the SDK's AuthenticationHelpers, whose provider the bridge only passes on to -/// the handler constructor. -/// -public static class FakeAuthenticationHelpers -{ - public static Task GetAuthenticationProviderAsync(object authContext) => - Task.FromResult(new object()); -} + public object? Environment { get; set; } -/// -/// Stands in for the SDK's AuthenticationHandler, answering rather than delegating to keep the -/// built client off the network. -/// -public sealed class FakeAuthenticationHandler : DelegatingHandler -{ - public FakeAuthenticationHandler(object authProvider, HttpMessageHandler inner) + internal static GraphSession Attach() { - AuthProvider = authProvider; - InnerHandler = inner; + var session = new GraphSession(); + s_instance = session; + return session; } - public object AuthProvider { get; } - - /// Requests the built client actually sent, in order. - public static List Sent { get; } = []; - - protected override Task SendAsync( - HttpRequestMessage request, CancellationToken cancellationToken) - { - lock (Sent) Sent.Add(request.RequestUri?.ToString() ?? string.Empty); - return Task.FromResult(new HttpResponseMessage(System.Net.HttpStatusCode.OK) - { - Content = new StringContent( - "{ \"value\": [] }", System.Text.Encoding.UTF8, "application/json") - }); - } + internal static void Detach() => s_instance = null; } -/// -/// Installs the fake Graph SDK types behind MgxCmdletBase's type resolver for the life of the -/// instance. Names it does not fake fall through to the real assemblies in the process. -/// -public sealed class GraphSdkStub : IDisposable +/// The members mgx reads off GraphSession.Environment, and nothing more. +internal sealed class FakeGraphEnvironment { - private const string SessionType = "Microsoft.Graph.PowerShell.Authentication.GraphSession"; - private const string HelpersType = - "Microsoft.Graph.PowerShell.Authentication.Core.Utilities.AuthenticationHelpers"; - private const string HandlerType = - "Microsoft.Graph.PowerShell.Authentication.Handlers.AuthenticationHandler"; - - private readonly bool _withAuthHelpers; - private readonly bool _sdkPresent; - - public GraphSdkStub(bool connected = true, bool withAuthHelpers = true, bool sdkPresent = true) - { - _withAuthHelpers = withAuthHelpers; - _sdkPresent = sdkPresent; - Session = new FakeGraphSession(); - if (!connected) - Session.AuthContext = null; - FakeGraphSession.Instance = sdkPresent ? Session : null; - MgxCmdletBase.s_typeResolverForTests = Resolve; - } - - /// A process with no Graph SDK in it at all. - public static GraphSdkStub WithoutTheSdk() => new(sdkPresent: false); + public string GraphEndpoint { get; set; } = "https://graph.microsoft.com"; - public FakeGraphSession Session { get; } - - /// Drops GraphSession.Instance while the fake types stay resolvable. - public void ClearInstance() => FakeGraphSession.Instance = null; - - private Type? Resolve(string fullName) - { - if (!_sdkPresent && (fullName == SessionType || fullName == HelpersType || fullName == HandlerType)) - return null; - - return fullName switch - { - SessionType => typeof(FakeGraphSession), - HelpersType => _withAuthHelpers ? typeof(FakeAuthenticationHelpers) : null, - HandlerType => _withAuthHelpers ? typeof(FakeAuthenticationHandler) : null, - _ => RealType(fullName) - }; - } - - private static Type? RealType(string fullName) => - AppDomain.CurrentDomain.GetAssemblies() - .SelectMany(a => - { - try { return a.GetTypes(); } - catch (ReflectionTypeLoadException) { return []; } - }) - .FirstOrDefault(t => t.FullName == fullName); - - public void Dispose() - { - MgxCmdletBase.s_typeResolverForTests = null; - FakeGraphSession.Instance = null; - lock (FakeAuthenticationHandler.Sent) FakeAuthenticationHandler.Sent.Clear(); - } + public string AzureADEndpoint { get; set; } = "https://login.microsoftonline.com"; } diff --git a/tests/Mgx.IntegrationTests/Fakes/GraphSessionScope.cs b/tests/Mgx.IntegrationTests/Fakes/GraphSessionScope.cs new file mode 100644 index 0000000..ad0e12d --- /dev/null +++ b/tests/Mgx.IntegrationTests/Fakes/GraphSessionScope.cs @@ -0,0 +1,56 @@ +using Mgx.Cmdlets.Base; +using Mgx.Cmdlets.Cmdlets.Configuration; +using Mgx.Engine.Http; +using Microsoft.Graph.PowerShell.Authentication; + +namespace Mgx.IntegrationTests; + +/// +/// Arms the fake GraphSession for the life of the scope and puts the process back the way it was +/// on dispose: no session instance, no injection state, no cached mgx client, a reset pipeline +/// factory. Without the restore a later test would find a session where the suite assumes none. +/// +internal sealed class GraphSessionScope : IDisposable +{ + internal GraphSession Session { get; } + + private readonly ResilientGraphClientOptions _previousOptions; + private readonly string _previousEndpoint; + + private GraphSessionScope(HttpClient? graphHttpClient, object? authContext) + { + // A scenario that drives the real GetClient sets both of these - the options because it + // chooses what the limiter under test does, the endpoint because GetClient reads it off + // the session it is about to arm. Neither belongs to the tests that run after it. + _previousOptions = MgxCmdletBase.s_clientOptions; + _previousEndpoint = MgxCmdletBase.s_graphEndpoint; + + MgxCmdletBase.ResetHttpClient(); + Session = GraphSession.Attach(); + Session.GraphHttpClient = graphHttpClient; + Session.AuthContext = authContext; + Session.Environment = new FakeGraphEnvironment(); + } + + internal static GraphSessionScope Arm(HttpClient? graphHttpClient = null, object? authContext = null) => + new(graphHttpClient, authContext); + + /// An AuthContext-shaped object. BuildAuthFingerprint reads these members by name. + internal static object AuthContextFor(string tenantId, string clientId) => new + { + TenantId = tenantId, + ClientId = clientId, + AuthType = "AppOnly", + Environment = "Global" + }; + + public void Dispose() + { + GraphSession.Detach(); + EnableMgxResilience.ReleaseInjection(); + MgxCmdletBase.s_clientOptions = _previousOptions; + MgxCmdletBase.s_graphEndpoint = _previousEndpoint; + MgxCmdletBase.ResetHttpClient(); + ResiliencePipelineFactory.Reset(); + } +} diff --git a/tests/Mgx.IntegrationTests/Fakes/MockHttpHandler.cs b/tests/Mgx.IntegrationTests/Fakes/MockHttpHandler.cs index fa8a174..b7cc260 100644 --- a/tests/Mgx.IntegrationTests/Fakes/MockHttpHandler.cs +++ b/tests/Mgx.IntegrationTests/Fakes/MockHttpHandler.cs @@ -1,28 +1,46 @@ -using System.Diagnostics; +using System.Diagnostics; using System.Net; using System.Text; namespace Mgx.IntegrationTests; /// -/// Serializes test classes that share ResiliencePipelineFactory and MgxCmdletBase static state. -/// DisableParallelization also keeps this collection from running alongside the Resilience -/// collection, which mutates the same statics. +/// Serializes test classes that share ResiliencePipelineFactory static state. +/// Without this, xUnit runs classes in parallel and Reset() calls from one +/// class can corrupt circuit breaker/pipeline state in another. /// -[CollectionDefinition("Pipeline", DisableParallelization = true)] +[CollectionDefinition("Pipeline")] public class PipelineCollection; +/// +/// A request as it looked on the wire at send time. Captured eagerly because the +/// client may dispose the request content once the call completes, so reading +/// Requests[n].Content after the fact is unreliable. +/// +public sealed record CapturedRequest( + HttpMethod Method, + string Uri, + IReadOnlyDictionary Headers, + IReadOnlyDictionary ContentHeaders, + byte[]? Body) +{ + public string? BodyText => Body == null ? null : Encoding.UTF8.GetString(Body); +} + /// /// Mock HTTP handler that returns configurable responses. /// Tracks request count for verifying retry behavior. /// public class MockHttpHandler : HttpMessageHandler { - private readonly Queue _responses = new(); + private readonly Queue _responses = new(); + private readonly MockRuleSet _rules = new(); private readonly List _requests = []; private readonly List _requestTicks = []; + private readonly List _captured = []; + private readonly List _served = []; private readonly object _lock = new(); - private MockResponse? _defaultResponse; + private MockReply? _defaultResponse; public int RequestCount { @@ -42,19 +60,90 @@ public List ArrivalGapsMs lock (_lock) { return [.. _requestTicks.Zip(_requestTicks.Skip(1), - (a, b) => (b - a) * 1000.0 / Stopwatch.Frequency)]; + (first, second) => (second - first) * 1000.0 / Stopwatch.Frequency)]; } } } - public void QueueResponse(HttpStatusCode statusCode, string? body = null, Dictionary? headers = null) + /// + /// Requests buffered at send time: method, URI, headers, and body bytes. + /// Survives the client disposing the originals. + /// + public List CapturedRequests + { + get { lock (_lock) { return [.. _captured]; } } + } + + /// + /// The status codes handed back, in order. A test whose assertion is that nothing was + /// reported needs some way to know the wire reported something to begin with. An answer + /// that never reached the caller is not here: one thrown instead of answered, and one held + /// past the attempt timeout that canceled the send. + /// + public List ServedStatusCodes + { + get { lock (_lock) { return [.. _served]; } } + } + + /// + /// Program a response for the requests a predicate picks out, rather than for a position in + /// the queue. A fault can then be aimed at page 3, at one fan-out item, or at the second + /// attempt of a request, and it lands there however the requests interleave. + /// Precedence: a keyed entry that matches this request wins and consumes no queue entry; an + /// unmatched request takes the queue; an empty queue takes the default response. + /// + /// Among the keyed entries the first one that matches, in programming order, answers - so a + /// narrower entry has to be programmed before a broader one, and a fault registered after a + /// generator that also matches its request never answers at all. An entry that never + /// answered is named in , and a test that aims a fault should + /// assert that list is empty: nothing else about the run says the fault was not injected. + /// + /// is what those assertions call this entry, and defaults to its + /// position in programming order. + /// + public MockResponseRule When(Func predicate, string? name = null) + => _rules.When(this, predicate, name); + + /// Every keyed entry, with the requests it matched and the ones it answered. + public IReadOnlyList ProgrammedRules + { + get { lock (_lock) { return _rules.ProgrammedRules; } } + } + + /// + /// The keyed entries that never answered a request, by name. A fault an earlier entry + /// shadowed leaves no other trace: the operation completes exactly as it would have with + /// nothing programmed. + /// + public IReadOnlyList UnansweredRules + { + get { lock (_lock) { return _rules.UnansweredRules; } } + } + + public void QueueResponse(HttpStatusCode statusCode, string? body = null, Dictionary? headers = null, string contentType = "application/json") + { + _responses.Enqueue(new MockReply { StatusCode = statusCode, Body = body, Headers = headers, ContentType = contentType }); + } + + /// + /// Queue a response with a raw byte body, for binary and non-UTF8 cases. + /// + public void QueueBytes(HttpStatusCode statusCode, byte[] body, string contentType, Dictionary? headers = null) + { + _responses.Enqueue(new MockReply { StatusCode = statusCode, BodyBytes = body, Headers = headers, ContentType = contentType }); + } + + /// + /// Queue a response with no content at all - a 204, or a 200 with a zero-length body. + /// + public void QueueEmpty(HttpStatusCode statusCode, Dictionary? headers = null) { - _responses.Enqueue(new MockResponse(statusCode, body, headers, null)); + _responses.Enqueue(new MockReply { StatusCode = statusCode, Headers = headers }); } public void SetDefaultResponse(HttpStatusCode statusCode, string? body = null, Dictionary? headers = null) { - _defaultResponse = new MockResponse(statusCode, body, headers, null); + _defaultResponse = new MockReply { StatusCode = statusCode, Body = body, Headers = headers, ContentType = "application/json" }; } /// @@ -73,38 +162,54 @@ public void QueueFailuresThenSuccess(int failCount, HttpStatusCode failStatus, s /// public void QueueException(Exception exception) { - _responses.Enqueue(new MockResponse(HttpStatusCode.OK, null, null, exception)); + _responses.Enqueue(new MockReply { Exception = exception }); } - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { - MockResponse mock; + byte[]? bodyBytes = null; + if (request.Content != null) + bodyBytes = await request.Content.ReadAsByteArrayAsync(cancellationToken); + + var captured = new CapturedRequest( + request.Method, + request.RequestUri?.OriginalString ?? string.Empty, + request.Headers.ToDictionary(h => h.Key, h => h.Value.ToArray()), + request.Content?.Headers.ToDictionary(h => h.Key, h => h.Value.ToArray()) + ?? new Dictionary(), + bodyBytes); + + var keyed = new MockRequest(request, bodyBytes); + + MockReply mock; lock (_lock) { _requests.Add(request); _requestTicks.Add(Stopwatch.GetTimestamp()); - mock = _responses.Count > 0 ? _responses.Dequeue() : (_defaultResponse ?? new MockResponse(HttpStatusCode.OK, null, null, null)); + _captured.Add(captured); + // Keyed first, then the queue, then the default. A keyed answer leaves the queue + // where it stands, so a test can aim one fault and script the rest in order. + mock = _rules.Resolve(keyed) + ?? (_responses.Count > 0 ? _responses.Dequeue() : _defaultResponse ?? new MockReply()); } + // On the send's own token: an answer held past the attempt timeout has to end as a + // timeout, not as a mock that never returns. + if (mock.Delay > TimeSpan.Zero) + await Task.Delay(mock.Delay, cancellationToken); + if (mock.Exception != null) - return Task.FromException(mock.Exception); + throw mock.Exception; - // Real transports (SocketsHttpHandler) set RequestMessage on the response; consumers - // like the pacer's OnRetry hook read the request URI off it. Mirror that here. - var response = new HttpResponseMessage(mock.StatusCode) { RequestMessage = request }; - if (mock.Body != null) - response.Content = new StringContent(mock.Body, Encoding.UTF8, "application/json"); + var response = mock.CreateResponse(keyed); + // Recorded here rather than where the answer was chosen, so the list says what the + // handler handed back: an answer held past an attempt timeout is never handed back at + // all, a thrown one never has a status, and two held answers come back in an order the + // one they resolved in does not predict. + lock (_lock) _served.Add(response.StatusCode); - if (mock.Headers != null) - { - foreach (var (key, value) in mock.Headers) - response.Headers.TryAddWithoutValidation(key, value); - } - - return Task.FromResult(response); + return response; } - - private record MockResponse(HttpStatusCode StatusCode, string? Body, Dictionary? Headers, Exception? Exception); } public static class TestData diff --git a/tests/Mgx.IntegrationTests/Fakes/MockResponseRules.cs b/tests/Mgx.IntegrationTests/Fakes/MockResponseRules.cs new file mode 100644 index 0000000..4851110 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Fakes/MockResponseRules.cs @@ -0,0 +1,311 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; + +namespace Mgx.IntegrationTests; + +/// +/// One request as a rule predicate sees it: the message itself, plus the body bytes the +/// handler has already read. A predicate over a $batch envelope needs those bytes, and it +/// cannot await them - the handler reads the content once, so a predicate that read it again +/// would be reading a stream the client is entitled to have disposed. +/// +public sealed class MockRequest(HttpRequestMessage message, byte[]? body) +{ + /// The request as sent. Everything below is shorthand onto it. + public HttpRequestMessage Message => message; + + public HttpMethod Method => message.Method; + + public Uri? RequestUri => message.RequestUri; + + /// The URI as the caller wrote it, absolute or relative. + public string Uri => message.RequestUri?.OriginalString ?? string.Empty; + + public HttpRequestHeaders Headers => message.Headers; + + /// The request body, or null when the request carried no content. + public byte[]? Body => body; + + public string? BodyText => body == null ? null : Encoding.UTF8.GetString(body); +} + +/// +/// What a handler answers one request with, whether the entry came from the queue or from a +/// request-keyed rule. Both fakes render it the same way, so a queued response and a keyed +/// response are indistinguishable on the wire. +/// +public sealed record MockReply +{ + public HttpStatusCode StatusCode { get; init; } = HttpStatusCode.OK; + + public string? Body { get; init; } + + public byte[]? BodyBytes { get; init; } + + public string? ContentType { get; init; } + + public Dictionary? Headers { get; init; } + + /// Thrown instead of answering. Nothing is recorded as served. + public Exception? Exception { get; init; } + + /// + /// Held before the answer is handed back, so an attempt timeout has something to fire on. + /// The wait takes the send's own token, which is what makes it a timeout rather than a hang. + /// + public TimeSpan Delay { get; init; } + + /// + /// The body delivers or and then stops without + /// closing, leaving the body-read timeout as the only thing that ends the read. + /// + public bool StallBody { get; init; } + + /// + /// Computes the whole response from the request, for answers too numerous to enumerate in + /// advance. Takes precedence over every field here except . + /// + public Func? Factory { get; init; } + + internal HttpResponseMessage CreateResponse(MockRequest request) + { + if (Factory != null) + { + var computed = Factory(request); + computed.RequestMessage ??= request.Message; + return computed; + } + + // Real transports (SocketsHttpHandler) set RequestMessage on the response; consumers + // like the pacer's OnRetry hook read the request URI off it. Mirror that here. + var response = new HttpResponseMessage(StatusCode) { RequestMessage = request.Message }; + if (StallBody) + { + var stalling = new StallingContent(BodyBytes ?? Encoding.UTF8.GetBytes(Body ?? "{")); + stalling.Headers.TryAddWithoutValidation("Content-Type", ContentType ?? "application/json"); + response.Content = stalling; + } + else if (BodyBytes != null) + { + response.Content = new ByteArrayContent(BodyBytes); + if (ContentType != null) + response.Content.Headers.TryAddWithoutValidation("Content-Type", ContentType); + } + else if (Body != null) + { + response.Content = new StringContent(Body, Encoding.UTF8, ContentType ?? "application/json"); + } + + if (Headers != null) + { + foreach (var (key, value) in Headers) + response.Headers.TryAddWithoutValidation(key, value); + } + + return response; + } +} + +/// One request-keyed entry: which requests it answers, and with what. +internal sealed class MockRule +{ + public required Func Predicate { get; init; } + + /// + /// What an assertion calls this entry. A predicate cannot be printed, so the name is the + /// only thing a failure message can carry; null until the rule set fills in its position. + /// + public string? Name { get; set; } + + /// Null answers every matching request; N answers only the Nth, 1-based. + public int? Attempt { get; set; } + + public MockReply Reply { get; set; } = new(); + + /// How many requests have matched this entry's predicate so far. + public int Matches { get; set; } + + /// How many of those this entry was the one to answer. + public int Answers { get; set; } +} + +/// +/// One programmed entry as an assertion sees it: what it was called, how many requests matched +/// its predicate, and how many of those it answered. A matched entry that answered nothing was +/// shadowed by an earlier one, or scoped to an attempt that never came. +/// +public sealed record MockRuleReport(string Name, int Matches, int Answers) +{ + public override string ToString() => $"{Name} (matched {Matches}, answered {Answers})"; +} + +/// +/// One request-keyed entry under construction. OnAttempt and AfterDelay narrow +/// it and must come before the answer; a Respond or Throw call arms it and hands +/// back the handler, so entries chain. +/// +public sealed class MockResponseRule +{ + private readonly MockRuleSet _rules; + private readonly MockRule _rule; + private readonly TOwner _owner; + private TimeSpan _delay; + + internal MockResponseRule(MockRuleSet rules, MockRule rule, TOwner owner) + { + _rules = rules; + _rule = rule; + _owner = owner; + } + + /// + /// Answer only the th request matching this entry's predicate, + /// 1-based. The counting is per entry, so the requests another entry answered still count + /// here and nowhere else; every other attempt falls through to whatever answers next - a + /// later entry, the queue, or the default. + /// + public MockResponseRule OnAttempt(int attempt) + { + _rule.Attempt = attempt; + return this; + } + + /// Hold the answer this long before handing it back. + public MockResponseRule AfterDelay(TimeSpan delay) + { + _delay = delay; + return this; + } + + /// Answer with a status and an optional body. + public TOwner Respond( + HttpStatusCode statusCode, + string? body = null, + Dictionary? headers = null, + string contentType = "application/json") + => Arm(new MockReply + { + StatusCode = statusCode, + Body = body, + Headers = headers, + ContentType = contentType + }); + + /// Answer with a raw byte body, for binary and non-UTF8 cases. + public TOwner RespondBytes( + HttpStatusCode statusCode, + byte[] body, + string contentType, + Dictionary? headers = null) + => Arm(new MockReply + { + StatusCode = statusCode, + BodyBytes = body, + ContentType = contentType, + Headers = headers + }); + + /// Answer with no content at all - a 204, or a 200 with a zero-length body. + public TOwner RespondEmpty(HttpStatusCode statusCode, Dictionary? headers = null) + => Arm(new MockReply { StatusCode = statusCode, Headers = headers }); + + /// + /// Compute the answer from the request. What a page-generating fake is built on: 100,000 + /// items are a function of the skip token, not a hundred thousand queued responses. + /// + public TOwner Respond(Func factory) + => Arm(new MockReply { Factory = factory }); + + /// + /// Answer with headers and a body that delivers and then stops + /// without closing, so the body-read timeout is the only thing that ends the read. + /// + public TOwner RespondStalled( + HttpStatusCode statusCode = HttpStatusCode.OK, + string prefix = "{", + string contentType = "application/json", + Dictionary? headers = null) + => Arm(new MockReply + { + StatusCode = statusCode, + Body = prefix, + ContentType = contentType, + Headers = headers, + StallBody = true + }); + + /// Throw instead of answering, for the transport failures that never reach a status. + public TOwner Throw(Exception exception) => Arm(new MockReply { Exception = exception }); + + private TOwner Arm(MockReply reply) + { + _rule.Reply = reply with { Delay = _delay }; + _rules.Add(_rule); + return _owner; + } +} + +/// +/// The request-keyed entries a handler answers from, in the order they were programmed. Not +/// synchronized on its own: a handler resolves under the same lock it already holds for its +/// queue, so one request takes exactly one answer. +/// +public sealed class MockRuleSet +{ + private readonly List _rules = []; + + /// Nothing is programmed, so a handler can skip the work of preparing a match. + public bool IsEmpty => _rules.Count == 0; + + internal void Add(MockRule rule) + { + rule.Name ??= $"#{_rules.Count + 1}"; + _rules.Add(rule); + } + + /// + /// Start an entry keyed on . is what + /// assertions and failure messages call it, and defaults to its position in programming + /// order, 1-based, as #1. + /// + public MockResponseRule When( + TOwner owner, Func predicate, string? name = null) + => new(this, new MockRule { Predicate = predicate, Name = name }, owner); + + /// + /// Every entry in programming order, with the requests that matched it and the ones it + /// answered. Read after the operation under test has finished: the counts are live. + /// + public IReadOnlyList ProgrammedRules => + [.. _rules.Select(rule => new MockRuleReport(rule.Name!, rule.Matches, rule.Answers))]; + + /// + /// The entries that never answered a request, by name. An entry a broader one programmed + /// before it shadows is here and nowhere else: the operation it was aimed at completes, + /// every item arrives, and no other observation says the fault was never injected. + /// + public IReadOnlyList UnansweredRules => + [.. _rules.Where(rule => rule.Answers == 0).Select(rule => rule.Name!)]; + + /// + /// The first entry whose predicate matches and whose attempt scope selects this request, or + /// null when none does. Every matching entry's counter advances whether or not it is the one + /// that answers, so an entry's attempt index means the Nth request matching that entry and + /// nothing else - one entry's faults never renumber another's. + /// + internal MockReply? Resolve(MockRequest request) + { + MockReply? chosen = null; + foreach (var rule in _rules) + { + if (!rule.Predicate(request)) continue; + var matched = ++rule.Matches; + if (chosen != null) continue; + if (rule.Attempt is int scoped && scoped != matched) continue; + rule.Answers++; + chosen = rule.Reply; + } + return chosen; + } +} diff --git a/tests/Mgx.IntegrationTests/Fakes/PagedCollection.cs b/tests/Mgx.IntegrationTests/Fakes/PagedCollection.cs new file mode 100644 index 0000000..bdb297f --- /dev/null +++ b/tests/Mgx.IntegrationTests/Fakes/PagedCollection.cs @@ -0,0 +1,147 @@ +using System.Net; +using System.Text; +using Mgx.IntegrationTests.Fakes; + +namespace Mgx.IntegrationTests; + +/// +/// A Graph collection that exists only as a function of the request: every page is computed +/// from the skip token the previous page's nextLink carried, so a hundred thousand items are +/// one programmed entry rather than a hundred thousand queued responses. Nothing is enumerated +/// in advance, and nothing here is mutable, so the handler can answer from it under its lock or +/// outside it. +/// +/// It is registered through the request-keyed programming and takes exactly the precedence +/// every other entry does: a fault aimed at one page is an entry registered BEFORE this one - +/// the first matching entry answers - and the pages either side of it still come from here. +/// +/// +public sealed class PagedCollection +{ + /// What this generator's nextLinks carry, and what reads. + private const string SkipToken = "$skiptoken=page"; + + private readonly string _idPrefix; + + /// Items in the whole collection. + /// Items on every page but the last. + /// The collection's URL with no query string. Every request whose + /// URI is this, or this followed by a query, is one of these pages. + /// The shape Graph returns when the collection divides + /// evenly: the last full page still carries a nextLink, and following it yields an empty + /// page. False ends the collection with the last full page's absent nextLink instead, and + /// true is refused on a total that does not divide evenly. + /// Item ids are this followed by the item's 1-based position. + public PagedCollection( + int total, + int pageSize, + string resourceUrl = "https://graph.microsoft.com/v1.0/users", + bool trailingEmptyPage = false, + string idPrefix = "user") + { + ArgumentOutOfRangeException.ThrowIfNegative(total); + ArgumentOutOfRangeException.ThrowIfLessThan(pageSize, 1); + // Graph appends its empty page behind a full one. A collection that does not divide + // evenly ends at a short page, and a nextLink behind a short page is a shape the service + // never returns - served here it would make a boundary test pass against a wire no + // caller will ever meet. + if (trailingEmptyPage && total % pageSize != 0) + throw new ArgumentException( + $"A trailing empty page is the shape Graph returns when the collection divides " + + $"evenly; {total} items served {pageSize} at a time ends at a short page.", + nameof(trailingEmptyPage)); + + Total = total; + PageSize = pageSize; + ResourceUrl = resourceUrl; + _idPrefix = idPrefix; + + // An empty collection is still one page: a request has to be answered with something. + var full = Math.Max(1, (total + pageSize - 1) / pageSize); + PageCount = trailingEmptyPage ? full + 1 : full; + } + + public int Total { get; } + + public int PageSize { get; } + + public string ResourceUrl { get; } + + /// Pages a complete enumeration asks for, and so the requests it costs. + public int PageCount { get; } + + /// Where an enumeration starts, page size and all. + public string InitialUrl => $"{ResourceUrl}?$top={PageSize}"; + + /// The link page 's predecessor carries; is 2 or more. + public string NextLinkTo(int page) => $"{ResourceUrl}?{SkipToken}{page}"; + + /// + /// Which page a request asks for, 1-based: the number in the skip token, or 1 for the + /// initial URL, which carries none. + /// + public static int PageOf(string uri) + { + var token = uri.IndexOf(SkipToken, StringComparison.Ordinal); + if (token < 0) return 1; + + var start = token + SkipToken.Length; + var end = start; + while (end < uri.Length && char.IsAsciiDigit(uri[end])) end++; + return end > start ? int.Parse(uri[start..end]) : 1; + } + + /// Whether addresses this collection rather than some other resource. + private bool Addresses(string uri) => + uri.StartsWith(ResourceUrl, StringComparison.Ordinal) + && (uri.Length == ResourceUrl.Length || uri[ResourceUrl.Length] == '?'); + + /// The predicate a handler entry is keyed on. + public bool Matches(MockRequest request) => Addresses(request.Uri); + + /// + /// Page as a Graph collection response. A page past the end is a + /// thrown exception rather than an empty body: the iterator asking for one is the defect + /// these tests exist to catch, and an obliging answer would hide it. + /// + public string BodyFor(int page) + { + if (page < 1 || page > PageCount) + throw new InvalidOperationException( + $"Page {page} was requested from a collection of {Total} items served " + + $"{PageSize} at a time, which is {PageCount} page(s)."); + + var already = (page - 1) * PageSize; + var count = Math.Clamp(Total - already, 0, PageSize); + + var json = new StringBuilder(count * 24 + 128).Append("{\"value\":["); + for (var i = 0; i < count; i++) + { + if (i > 0) json.Append(','); + json.Append("{\"id\":\"").Append(_idPrefix).Append(already + i + 1).Append("\"}"); + } + json.Append(']'); + if (page < PageCount) + json.Append(",\"@odata.nextLink\":\"").Append(NextLinkTo(page + 1)).Append('"'); + return json.Append('}').ToString(); + } + + /// The answer to one request, for a handler entry armed with a factory. + public HttpResponseMessage Answer(MockRequest request) => + new(HttpStatusCode.OK) + { + Content = new StringContent( + BodyFor(PageOf(request.Uri)), Encoding.UTF8, "application/json") + }; + + /// + /// Program to serve this collection, and hand the handler back so + /// further entries chain. Entries registered first win, so a fault on one page goes on + /// before this call. + /// + public MockHttpHandler ServeOn(MockHttpHandler handler) => handler.When(Matches).Respond(Answer); + + /// + public StubHttpMessageHandler ServeOn(StubHttpMessageHandler handler) => + handler.When(Matches).Respond(Answer); +} diff --git a/tests/Mgx.IntegrationTests/Fakes/RecordingHost.cs b/tests/Mgx.IntegrationTests/Fakes/RecordingHost.cs new file mode 100644 index 0000000..8d4ea82 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Fakes/RecordingHost.cs @@ -0,0 +1,88 @@ +using System.Globalization; +using System.Management.Automation; +using System.Management.Automation.Host; +using System.Management.Automation.Runspaces; +using System.Reflection; +using System.Security; + +namespace Mgx.IntegrationTests; + +/// +/// A PSHost that keeps every line written to it. A ShouldProcess gate writes its "What if:" line +/// to the host rather than to one of the streams a PowerShell object exposes, so a test that +/// reads what a preview actually said needs a host of its own to read it off. +/// +internal sealed class RecordingHost : PSHost +{ + private readonly Guid _id = Guid.NewGuid(); + + public RecordingHostUI Recorder { get; } = new(); + + public override string Name => "MgxRecordingHost"; + public override Version Version => new(1, 0); + public override Guid InstanceId => _id; + public override PSHostUserInterface UI => Recorder; + public override CultureInfo CurrentCulture => CultureInfo.InvariantCulture; + public override CultureInfo CurrentUICulture => CultureInfo.InvariantCulture; + public override void EnterNestedPrompt() { } + public override void ExitNestedPrompt() { } + public override void NotifyBeginApplication() { } + public override void NotifyEndApplication() { } + public override void SetShouldExit(int exitCode) { } + + /// + /// Imports into a runspace over one of these hosts, runs the + /// command adds, and returns the lines the host was given. + /// + internal static List Preview(Assembly module, Action build) + { + var host = new RecordingHost(); + using var runspace = RunspaceFactory.CreateRunspace(host); + runspace.Open(); + using var ps = PowerShell.Create(); + ps.Runspace = runspace; + ps.AddCommand("Import-Module").AddParameter("Assembly", module); + ps.Invoke(); + ps.Commands.Clear(); + + build(ps); + ps.Invoke(); + + Assert.Empty(ps.Streams.Error.Select(e => e.FullyQualifiedErrorId)); + return host.Recorder.Lines; + } +} + +/// Every write, in the order it arrived, and nothing else. +internal sealed class RecordingHostUI : PSHostUserInterface +{ + public List Lines { get; } = []; + + public override PSHostRawUserInterface? RawUI => null; + public override void Write(string value) => Lines.Add(value); + public override void Write(ConsoleColor f, ConsoleColor b, string value) => Lines.Add(value); + public override void WriteLine(string value) => Lines.Add(value); + public override void WriteErrorLine(string value) => Lines.Add(value); + public override void WriteDebugLine(string value) => Lines.Add(value); + public override void WriteVerboseLine(string value) => Lines.Add(value); + public override void WriteWarningLine(string value) => Lines.Add(value); + public override void WriteProgress(long sourceId, ProgressRecord record) { } + public override string ReadLine() => string.Empty; + public override SecureString ReadLineAsSecureString() => new(); + + public override Dictionary Prompt( + string caption, string message, + System.Collections.ObjectModel.Collection descriptions) => []; + + public override int PromptForChoice( + string caption, string message, + System.Collections.ObjectModel.Collection choices, + int defaultChoice) => defaultChoice; + + public override PSCredential PromptForCredential( + string caption, string message, string userName, string targetName) => PSCredential.Empty; + + public override PSCredential PromptForCredential( + string caption, string message, string userName, string targetName, + PSCredentialTypes allowedCredentialTypes, PSCredentialUIOptions options) => PSCredential.Empty; +} diff --git a/tests/Mgx.IntegrationTests/Fakes/StallingContent.cs b/tests/Mgx.IntegrationTests/Fakes/StallingContent.cs new file mode 100644 index 0000000..fcdea0c --- /dev/null +++ b/tests/Mgx.IntegrationTests/Fakes/StallingContent.cs @@ -0,0 +1,93 @@ +using System.Net; +using System.Text; + +namespace Mgx.IntegrationTests; + +/// +/// A response body that delivers a prefix and then stops, without closing the stream, and ends +/// only when the read's own token is canceled. Requests go out with +/// HttpCompletionOption.ResponseHeadersRead, so neither HttpClient.Timeout nor the pipeline's +/// attempt timeout covers the content read at all - the engine's BodyReadTimeout is the only +/// thing that bounds it, and this is what makes that bound observable in-process. +/// +public sealed class StallingContent : HttpContent +{ + private readonly byte[] _prefix; + private readonly long _declaredLength; + + /// Delivered before the stall, so the body looks like one that started. + /// What Content-Length claims, so a reader keeps waiting for the rest. + public StallingContent(string prefix = "{", long declaredLength = 4096) + : this(Encoding.UTF8.GetBytes(prefix), declaredLength) + { + } + + public StallingContent(byte[] prefix, long declaredLength = 4096) + { + _prefix = prefix; + _declaredLength = declaredLength; + } + + protected override async Task SerializeToStreamAsync(Stream stream, TransportContext? context) + => await SerializeToStreamAsync(stream, context, CancellationToken.None); + + protected override async Task SerializeToStreamAsync( + Stream stream, TransportContext? context, CancellationToken cancellationToken) + { + await stream.WriteAsync(_prefix, cancellationToken); + await stream.FlushAsync(cancellationToken); + await Task.Delay(Timeout.Infinite, cancellationToken); + } + + /// + /// The stall as a reader on the content stream sees it. HttpContent's own implementation of + /// this buffers the whole body first and drops the cancellation token doing it, so a caller + /// that reads the body as a stream - a collection page, a $batch envelope, a content + /// download - waits on this content forever instead of hitting its read timeout. A real + /// response hands the stream over as soon as the headers are in and stalls on the read; + /// this does the same, on the read's own token. + /// + protected override Task CreateContentReadStreamAsync() + => Task.FromResult(new StallingStream(_prefix)); + + protected override bool TryComputeLength(out long length) + { + length = _declaredLength; + return true; + } + + private sealed class StallingStream(byte[] prefix) : Stream + { + private int _delivered; + + public override async ValueTask ReadAsync( + Memory buffer, CancellationToken cancellationToken = default) + { + if (_delivered < prefix.Length) + { + var count = Math.Min(buffer.Length, prefix.Length - _delivered); + prefix.AsSpan(_delivered, count).CopyTo(buffer.Span); + _delivered += count; + return count; + } + + await Task.Delay(Timeout.Infinite, cancellationToken); + return 0; + } + + public override Task ReadAsync( + byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => ReadAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => _delivered; set => throw new NotSupportedException(); } + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + } +} diff --git a/tests/Mgx.IntegrationTests/Fakes/StubHttpMessageHandler.cs b/tests/Mgx.IntegrationTests/Fakes/StubHttpMessageHandler.cs index ec57887..13b3df9 100644 --- a/tests/Mgx.IntegrationTests/Fakes/StubHttpMessageHandler.cs +++ b/tests/Mgx.IntegrationTests/Fakes/StubHttpMessageHandler.cs @@ -11,6 +11,7 @@ namespace Mgx.IntegrationTests.Fakes; public sealed class StubHttpMessageHandler : HttpMessageHandler { private readonly Queue> _responses = new(); + private readonly MockRuleSet _rules = new(); private Func? _last; private readonly Lock _sync = new(); @@ -22,6 +23,41 @@ public int RequestCount get { lock (_sync) return Requests.Count; } } + /// + /// Program a response for the requests a predicate picks out, rather than for a position in + /// the script, with the same precedence uses: a keyed entry + /// that matches this request wins and consumes no scripted response; an unmatched request + /// takes the script, whose last entry repeats as it always has. + /// + /// Among the keyed entries the first one that matches, in programming order, answers - so a + /// narrower entry has to be programmed before a broader one, and a fault registered after a + /// generator that also matches its request never answers at all. An entry that never + /// answered is named in , and a test that aims a fault should + /// assert that list is empty: nothing else about the run says the fault was not injected. + /// + /// is what those assertions call this entry, and defaults to its + /// position in programming order. + /// + public MockResponseRule When( + Func predicate, string? name = null) + => _rules.When(this, predicate, name); + + /// Every keyed entry, with the requests it matched and the ones it answered. + public IReadOnlyList ProgrammedRules + { + get { lock (_sync) return _rules.ProgrammedRules; } + } + + /// + /// The keyed entries that never answered a request, by name. A fault an earlier entry + /// shadowed leaves no other trace: the operation completes exactly as it would have with + /// nothing programmed. + /// + public IReadOnlyList UnansweredRules + { + get { lock (_sync) return _rules.UnansweredRules; } + } + public StubHttpMessageHandler Enqueue(Func factory) { _responses.Enqueue(factory); @@ -56,26 +92,48 @@ public StubHttpMessageHandler EnqueueRepeated(int count, Func SendAsync( + protected override async Task SendAsync( HttpRequestMessage request, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - Func factory; + // Only read when something is programmed to look: a predicate over a $batch envelope + // needs the bytes, and a handler with no keyed entries touches the request exactly as + // it did before there were any. + byte[]? bodyBytes = null; + if (!_rules.IsEmpty && request.Content != null) + bodyBytes = await request.Content.ReadAsByteArrayAsync(cancellationToken); + var keyed = new MockRequest(request, bodyBytes); + + MockReply? reply; + Func? factory = null; lock (_sync) { // Record method/uri rather than the message: HttpClient disposes the // request once the call completes, so the object is not safe to keep. Requests.Add((request.Method, request.RequestUri?.ToString() ?? string.Empty)); - if (_responses.Count > 0) - _last = _responses.Dequeue(); + reply = _rules.Resolve(keyed); + if (reply == null) + { + if (_responses.Count > 0) + _last = _responses.Dequeue(); - factory = _last ?? throw new InvalidOperationException( - "StubHttpMessageHandler received a request but no response was scripted."); + factory = _last ?? throw new InvalidOperationException( + "StubHttpMessageHandler received a request but no response was scripted."); + } } - var response = factory(request); - return Task.FromResult(response); + if (reply == null) + return factory!(request); + + // On the send's own token, so an answer held past the attempt timeout ends as a timeout. + if (reply.Delay > TimeSpan.Zero) + await Task.Delay(reply.Delay, cancellationToken); + + if (reply.Exception != null) + throw reply.Exception; + + return reply.CreateResponse(keyed); } } diff --git a/tests/Mgx.IntegrationTests/Infrastructure/MgxTestHost.cs b/tests/Mgx.IntegrationTests/Infrastructure/MgxTestHost.cs index de924bb..d9fb46f 100644 --- a/tests/Mgx.IntegrationTests/Infrastructure/MgxTestHost.cs +++ b/tests/Mgx.IntegrationTests/Infrastructure/MgxTestHost.cs @@ -1,5 +1,6 @@ -using System.Collections; +using System.Collections; using System.Management.Automation; +using System.Threading; using System.Management.Automation.Runspaces; using System.Reflection; using Mgx.Cmdlets.Base; @@ -73,7 +74,9 @@ public MgxTestHost(StubHttpMessageHandler handler, string graphEndpoint = "https MgxCmdletBase.SetClientOptions(options ?? FastOptions); MgxCmdletBase.s_graphEndpoint = graphEndpoint; // Cleared for the tests that drive the real client build through a stubbed Graph SDK - MgxCmdletBase.s_testTransportFactory = useTestTransport ? () => _httpClient : null; + MgxCmdletBase.s_testTransportOwned = useTestTransport; + MgxCmdletBase.s_testTransport = useTestTransport ? _httpClient : null; + Interlocked.Increment(ref MgxCmdletBase.s_testTransportEpoch); } public MgxResult Run(Action build) @@ -108,7 +111,9 @@ [.. ps.Streams.Verbose.Select(v => v.Message)], public void Dispose() { - MgxCmdletBase.s_testTransportFactory = null; + MgxCmdletBase.s_testTransportOwned = false; + MgxCmdletBase.s_testTransport = null; + Interlocked.Increment(ref MgxCmdletBase.s_testTransportEpoch); MgxCmdletBase.s_graphEndpoint = "https://graph.microsoft.com"; MgxCmdletBase.SetClientOptions(ResilientGraphClientOptions.Default); ResiliencePipelineFactory.Reset(); diff --git a/tests/Mgx.IntegrationTests/Serialization/PSWrapperEquivalenceTests.cs b/tests/Mgx.IntegrationTests/Serialization/PSWrapperEquivalenceTests.cs new file mode 100644 index 0000000..e909175 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Serialization/PSWrapperEquivalenceTests.cs @@ -0,0 +1,87 @@ +using System.Collections; +using System.Management.Automation; +using Mgx.Cmdlets.Cmdlets; + +namespace Mgx.IntegrationTests; + +/// +/// The acceptance criterion for body serialization: a PowerShell wrapper never alters the +/// wire JSON. Serialize(x) must be byte-identical to Serialize(PSObject.AsPSObject(x)), +/// with the wrapping at any depth. (GraphSDK-3654 PSObject wrappers, GraphSDK-3361 +/// scalar/array.) +/// +public class PSWrapperEquivalenceTests +{ + public static TheoryData Values() + { + var d = new TheoryData + { + { "bool", true }, + { "int", 42 }, + { "long-max", long.MaxValue }, + { "double", 3.5d }, + { "decimal", 79228162514264337593543950335m }, + { "guid", Guid.Parse("11111111-2222-3333-4444-555555555555") }, + { "datetime-utc", new DateTime(2026, 1, 5, 10, 0, 0, DateTimeKind.Utc) }, + { "datetime-unspecified", new DateTime(2026, 1, 5, 10, 0, 0, DateTimeKind.Unspecified) }, + { "datetimeoffset", new DateTimeOffset(2026, 1, 5, 10, 0, 0, TimeSpan.FromHours(2)) }, + { "timespan", new TimeSpan(1, 30, 0) }, + { "enum", DayOfWeek.Monday }, + { "string-unicode", "Müller & Söhne 日本語 😀" }, + { "string-quotes", "it's \"quoted\"" }, + { "bytes", new byte[] { 1, 2, 3 } }, + { "array", new object[] { 1, "two", true } }, + { "single-element-array", new object[] { "alone" } }, + { "empty-array", Array.Empty() }, + }; + return d; + } + + private static string Serialize(object body) => InvokeMgxRequest.SerializeBody(body); + + [Theory] + [MemberData(nameof(Values))] + public void Wrapping_at_depth_0_changes_nothing(string label, object value) + { + _ = label; + var direct = Serialize(new Hashtable { ["v"] = value }); + var wrapped = Serialize(new Hashtable { ["v"] = PSObject.AsPSObject(value) }); + Assert.Equal(direct, wrapped); + } + + [Theory] + [MemberData(nameof(Values))] + public void Wrapping_at_depth_1_changes_nothing(string label, object value) + { + _ = label; + var direct = Serialize(new Hashtable { ["outer"] = new Hashtable { ["v"] = value } }); + var wrapped = Serialize(PSObject.AsPSObject( + new Hashtable { ["outer"] = PSObject.AsPSObject(new Hashtable { ["v"] = PSObject.AsPSObject(value) }) })); + Assert.Equal(direct, wrapped); + } + + [Theory] + [MemberData(nameof(Values))] + public void Wrapping_at_depth_3_changes_nothing(string label, object value) + { + _ = label; + object Nest(object v) => new Hashtable { ["a"] = new Hashtable { ["b"] = new Hashtable { ["c"] = v } } }; + var direct = Serialize(Nest(value)); + var wrapped = Serialize(Nest(PSObject.AsPSObject(value))); + Assert.Equal(direct, wrapped); + } + + [Fact] + public void An_array_of_wrapped_elements_matches_the_unwrapped_array() + { + var direct = Serialize(new Hashtable { ["members"] = new object[] { "a", 1, true } }); + var wrapped = Serialize(new Hashtable + { + ["members"] = new object[] + { + PSObject.AsPSObject("a"), PSObject.AsPSObject(1), PSObject.AsPSObject(true) + } + }); + Assert.Equal(direct, wrapped); + } +} diff --git a/tests/Mgx.IntegrationTests/Serialization/WireBodyTests.cs b/tests/Mgx.IntegrationTests/Serialization/WireBodyTests.cs new file mode 100644 index 0000000..10814d4 --- /dev/null +++ b/tests/Mgx.IntegrationTests/Serialization/WireBodyTests.cs @@ -0,0 +1,313 @@ +using System.Collections; +using System.Globalization; +using System.Net; +using System.Text.Json.Serialization; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// The JSON that actually goes on the wire for a -Body argument, asserted byte for byte - +/// not whether serialization merely succeeds. (Corpus: GraphSDK-3361 scalar/array, +/// GraphSDK-3654 PSObject wrappers, M365DSC-5306/7175 malformed bodies.) +/// +[Collection("Pipeline")] +public class WireBodyTests +{ + /// Runs a POST built by the given script and returns the wire body text. + private static string WireBodyFor(string bodyExpression) + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.NoContent); + using (MgxTransportScope.Inject(wire)) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript($"Invoke-MgxRequest -Uri /widgets -Method POST -Confirm:$false -Body $({bodyExpression})"); + ps.Invoke(); + + var errors = string.Join(" | ", ps.Streams.Error.Select(e => e.Exception.Message)); + var captured = wire.CapturedRequests; + Assert.True(captured.Count > 0, $"no request sent; errors: {errors}"); + return captured[0].BodyText!; + } + } + + [Fact] + public void An_enum_serializes_as_its_camel_case_name() + => Assert.Equal("""{"day":"monday"}""", WireBodyFor("@{ day = [System.DayOfWeek]::Monday }")); + + [Fact] + public void A_flags_enum_combination_joins_its_names_without_a_space() + => Assert.Equal("""{"opt":"ignoreCase,multiline"}""", + WireBodyFor("@{ opt = [System.Text.RegularExpressions.RegexOptions]'IgnoreCase, Multiline' }")); + + [Fact] + public void A_single_flag_serializes_as_its_camel_case_name() + => Assert.Equal("""{"opt":"multiline"}""", + WireBodyFor("@{ opt = [System.Text.RegularExpressions.RegexOptions]::Multiline }")); + + /// Read and Fetch share a value; Enum.ToString and STJ pick different names for it. + [Flags] + private enum AliasedFlags { Read = 1, Fetch = 1, Write = 2 } + + private enum AliasedPlain { Read = 1, Fetch = 1, Write = 2 } + + [Flags] + private enum WireFlags { None = 0, Read = 1, Write = 2 } + + /// Read carries a wire name Enum.ToString cannot see. + [Flags] + private enum RenamedFlags + { + [JsonStringEnumMemberName("read-only")] Read = 1, + Write = 2, + } + + /// BC covers B and C, so 7 has a composite member and a bit left over. + [Flags] + private enum CompositeFlags + { + A = 1, + B = 2, + C = 4, + [JsonStringEnumMemberName("b-c")] BC = 6, + } + + private enum PlatformEnum { IOS, Android } + + private sealed class FlagsKeyedHolder + { + public Dictionary Scores { get; } = new() { [WireFlags.Read | WireFlags.Write] = 1 }; + } + + private sealed class RenamedKeyedHolder + { + public Dictionary Scores { get; } = new() { [RenamedFlags.Read | RenamedFlags.Write] = 1 }; + } + + private static string Serialize(object body) + => Mgx.Cmdlets.Cmdlets.InvokeMgxRequest.SerializeBody(body); + + [Fact] + public void An_aliased_flag_gets_the_same_name_as_the_same_enum_without_flags() + { + // Resolving a single name from Enum.ToString would silently rename it: the two + // members share a value, and Enum.ToString and STJ's name cache disagree on which + // one wins. Only the multi-name form belongs to the flags path. + var withFlags = Serialize(new Hashtable { ["v"] = AliasedFlags.Read }); + var withoutFlags = Serialize(new Hashtable { ["v"] = AliasedPlain.Read }); + Assert.Equal(withoutFlags, withFlags); + } + + [Fact] + public void A_flags_value_with_no_names_stays_a_number_under_any_culture() + { + var original = CultureInfo.CurrentCulture; + try + { + // sv-SE writes a negative sign as U+2212, so a number recognized by inspecting + // Enum.ToString would be quoted as a name instead and Graph would refuse it. + CultureInfo.CurrentCulture = new CultureInfo("sv-SE"); + Assert.Equal("""{"v":-3}""", Serialize(new Hashtable { ["v"] = (WireFlags)(-3) })); + } + finally { CultureInfo.CurrentCulture = original; } + } + + [Fact] + public void A_flags_enum_used_as_a_dictionary_key_joins_its_names_too() + => Assert.Equal("""{"thing":{"Scores":{"read,write":1}}}""", + Serialize(new Hashtable { ["thing"] = new FlagsKeyedHolder() })); + + [Fact] + public void An_aliased_flag_in_a_combination_gets_the_name_it_gets_alone() + // Enum.ToString calls this value Fetch and STJ calls it Read, so a combination read + // from Enum.ToString sent "fetch,write" where the member alone was sent "read". + => Assert.Equal("""{"v":"read,write"}""", + Serialize(new Hashtable { ["v"] = AliasedFlags.Read | AliasedFlags.Write })); + + [Fact] + public void A_renamed_flag_keeps_its_wire_name_in_a_combination() + => Assert.Equal("""{"v":"read-only,write"}""", + Serialize(new Hashtable { ["v"] = RenamedFlags.Read | RenamedFlags.Write })); + + [Fact] + public void A_renamed_combination_names_a_dictionary_key_the_same_way() + => Assert.Equal("""{"thing":{"Scores":{"read-only,write":1}}}""", + Serialize(new Hashtable { ["thing"] = new RenamedKeyedHolder() })); + + [Fact] + public void A_member_covering_several_bits_wins_over_the_bits() + // 7 is A|B|C, and BC is a member: components are taken largest first, so BC is chosen + // over B and C, and written after A because the names go out in ascending value order. + => Assert.Equal("""{"v":"a,b-c"}""", Serialize(new Hashtable { ["v"] = (CompositeFlags)7 })); + + [Fact] + public void A_composite_member_on_its_own_stays_a_single_name() + => Assert.Equal("""{"v":"b-c"}""", Serialize(new Hashtable { ["v"] = CompositeFlags.BC })); + + [Fact] + public void A_leading_acronym_is_lower_cased_whole() + // The -Body help documents this: Graph spells the same value "iOS", so a caller who + // needs that has to pass the string. Pinned so the help and the wire cannot drift. + => Assert.Equal("""{"v":"ios"}""", Serialize(new Hashtable { ["v"] = PlatformEnum.IOS })); + + [Fact] + public void A_byte_array_serializes_as_base64() + => Assert.Equal("""{"key":"AQID"}""", WireBodyFor("@{ key = [byte[]](1,2,3) }")); + + [Fact] + public void A_timespan_serializes_as_an_iso_duration() + => Assert.Equal("""{"d":"PT1H30M"}""", WireBodyFor("@{ d = [TimeSpan]::new(1,30,0) }")); + + [Fact] + public void A_kindless_datetime_is_pinned_to_utc() + => Assert.Equal("""{"at":"2026-01-05T10:00:00Z"}""", + WireBodyFor("@{ at = [DateTime]::new(2026,1,5,10,0,0) }")); + + [Fact] + public void Non_ascii_text_is_sent_as_utf8_not_escape_sequences() + => Assert.Equal("""{"name":"Müller & Söhne "}""", + WireBodyFor("@{ name = 'Müller & Söhne ' }")); + + [Fact] + public void A_single_element_array_stays_an_array() + => Assert.Equal("""{"members":["a"]}""", WireBodyFor("@{ members = @('a') }")); + + [Fact] + public void A_script_property_and_an_alias_property_serialize_like_convertto_json() + { + var body = WireBodyFor(""" + $o = [PSCustomObject]@{ displayName = 'x' } + $o | Add-Member -MemberType ScriptProperty -Name mailNickname -Value { 'nick-' + $this.displayName } -PassThru | + Add-Member -MemberType AliasProperty -Name alias -Value displayName -PassThru + """); + var doc = System.Text.Json.JsonDocument.Parse(body); + Assert.Equal("x", doc.RootElement.GetProperty("displayName").GetString()); + Assert.Equal("nick-x", doc.RootElement.GetProperty("mailNickname").GetString()); + Assert.Equal("x", doc.RootElement.GetProperty("alias").GetString()); + } + + [Fact] + public void A_wrapped_and_an_unwrapped_hashtable_produce_identical_wire_json() + { + var direct = WireBodyFor("@{ a = 1; nested = @{ b = $true } }"); + var wrapped = WireBodyFor("[PSObject]@{ a = 1; nested = @{ b = $true } }"); + Assert.Equal(direct, wrapped); + } + + /// Runs a POST and returns (errors, requestCount) - for bodies that must refuse. + private static (List Errors, int Requests) RefusalFor(string bodyExpression) + { + var wire = new MockHttpHandler(); + wire.SetDefaultResponse(HttpStatusCode.NoContent); + using (MgxTransportScope.Inject(wire)) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript($"Invoke-MgxRequest -Uri /widgets -Method POST -Confirm:$false -Body $({bodyExpression})"); + ps.Invoke(); + return (ps.Streams.Error.ToList(), wire.CapturedRequests.Count); + } + } + + [Fact] + public void A_secure_string_in_a_body_refuses_naming_the_property_path() + { + var (errors, requests) = RefusalFor("@{ profile = @{ password = (ConvertTo-SecureString 'x' -AsPlainText -Force) } }"); + Assert.Equal(0, requests); + var error = Assert.Single(errors); + Assert.StartsWith("InvalidBodyValue", error.FullyQualifiedErrorId); + Assert.Contains("-Body.profile.password", error.Exception.Message); + Assert.Contains("SecureString", error.Exception.Message); + } + + [Fact] + public void A_script_block_in_a_body_refuses() + { + var (errors, requests) = RefusalFor("@{ handler = { Get-Date } }"); + Assert.Equal(0, requests); + var error = Assert.Single(errors); + Assert.StartsWith("InvalidBodyValue", error.FullyQualifiedErrorId); + Assert.Contains("ScriptBlock", error.Exception.Message); + } + + [Fact] + public void NaN_in_a_body_refuses_naming_the_property_path() + { + var (errors, requests) = RefusalFor("@{ metrics = @{ score = [double]::NaN } }"); + Assert.Equal(0, requests); + var error = Assert.Single(errors); + Assert.StartsWith("InvalidBodyValue", error.FullyQualifiedErrorId); + Assert.Contains("-Body.metrics.score", error.Exception.Message); + } + + [Fact] + public void A_self_referencing_body_is_a_clean_error_not_a_stack_overflow() + { + var (errors, requests) = RefusalFor("$h = @{}; $h.self = $h; $h"); + Assert.Equal(0, requests); + var error = Assert.Single(errors); + Assert.StartsWith("InvalidBodyValue", error.FullyQualifiedErrorId); + Assert.Contains("self-referencing", error.Exception.Message); + } + + [Fact] + public void Scalar_wire_forms_hold_for_the_remaining_corpus_types() + { + // Exact wire values for the types the wrapper-equivalence theory covers only + // relationally: identical-on-both-sides is not the same as correct. + Assert.Equal("""{"v":79228162514264337593543950335}""", + WireBodyFor("@{ v = [decimal]::MaxValue }")); + Assert.Equal("""{"v":"11111111-2222-3333-4444-555555555555"}""", + WireBodyFor("@{ v = [Guid]'11111111-2222-3333-4444-555555555555' }")); + Assert.Equal("""{"v":9223372036854775807}""", + WireBodyFor("@{ v = [long]::MaxValue }")); + Assert.Equal("""{"v":"2026-01-05T10:00:00+02:00"}""", + WireBodyFor("@{ v = [DateTimeOffset]::new(2026,1,5,10,0,0,[TimeSpan]::FromHours(2)) }")); + Assert.Equal("""{"v":[]}""", WireBodyFor("@{ v = @() }")); + } + + [Fact] + public void A_refused_batch_body_fails_that_item_and_sends_the_rest() + { + var wire = new MockHttpHandler(); + wire.QueueResponse(HttpStatusCode.OK, + """{ "responses": [ { "id": "1", "status": 204 } ] }"""); + using (MgxTransportScope.Inject(wire)) + { + using var ps = System.Management.Automation.PowerShell.Create(); + ps.AddCommand("Import-Module") + .AddParameter("Assembly", typeof(Mgx.Cmdlets.Cmdlets.InvokeMgxRequest).Assembly); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript("function Get-MgContext { [PSCustomObject]@{ TenantId = 'test-tenant-00000000-0000-0000-0000-000000000000' } }"); + ps.Invoke(); + ps.Commands.Clear(); + ps.AddScript(""" + @( + @{ Url = '/users/u1'; Method = 'PATCH'; Body = @{ s = (ConvertTo-SecureString 'x' -AsPlainText -Force) } } + @{ Url = '/users/u2'; Method = 'PATCH'; Body = @{ displayName = 'ok' } } + ) | Invoke-MgxBatchRequest -Confirm:$false + """); + ps.Invoke(); + + var error = Assert.Single(ps.Streams.Error, + e => e.FullyQualifiedErrorId.StartsWith("InvalidBatchItemBody")); + Assert.Contains("SecureString", error.Exception.Message); + Assert.Single(wire.CapturedRequests); // the good item still went + } + } +} diff --git a/tests/Mgx.IntegrationTests/TestSetup/HostileInputs.cs b/tests/Mgx.IntegrationTests/TestSetup/HostileInputs.cs new file mode 100644 index 0000000..cea7c53 --- /dev/null +++ b/tests/Mgx.IntegrationTests/TestSetup/HostileInputs.cs @@ -0,0 +1,48 @@ +namespace Mgx.IntegrationTests; + +/// +/// The shared hostile-input corpus. Each group generalizes a real failure; the issue +/// numbers are provenance, never scope (M365DSC and Graph SDK issue trackers). +/// +public static class HostileInputs +{ + /// OData filter values that break naive encoding. + /// (GraphSDK-2709/2942 query corruption, GraphSDK-1947 '#'.) + public static readonly string[] FilterValues = + [ + "startsWith(displayName,'O''Brien')", // escaped apostrophe + "displayName eq 'A & B'", + "displayName eq 'C#'", + "displayName eq '50% off'", + "displayName eq 'a+b'", + "displayName eq 'x/y'", + "displayName eq '?'", + "displayName eq 'Müller'", + "displayName eq '日本語'", + "displayName eq '😀'", // surrogate pair + "mail eq 'a b@contoso.com'", + ]; + + /// Path segments that must survive into the request path. + /// (GraphSDK-1947, M365DSC-5354.) + public static readonly string[] PathSegments = + [ + "a#b.txt", + "a b.txt", + "100%.pdf", + "it's.txt", + "naïve.txt", + "01ABCDEF!ABC:XYZ", // drive-item id shape + "contoso.sharepoint.com,9f0e,ab12", // site id shape + ]; + + /// Service-issued links that must be followed byte-identically. + /// (GraphSDK-2488 nextLink handling.) + public static readonly string[] OpaqueLinks = + [ + "https://graph.microsoft.com/v1.0/users?$skiptoken=X%27443700...%27&$top=100", + "https://graph.microsoft.com/v1.0/users?$skiptoken=a+b=c==&$select=id", + "https://graph.microsoft.com/v1.0/users?$skiptoken=pre%20encoded%23frag", + "https://graph.microsoft.com/v1.0/drives/b!YZCE_x-y/items?$skiptoken=1", + ]; +} diff --git a/tests/Mgx.IntegrationTests/TestSetup/MgxTransportScope.cs b/tests/Mgx.IntegrationTests/TestSetup/MgxTransportScope.cs new file mode 100644 index 0000000..0b633b2 --- /dev/null +++ b/tests/Mgx.IntegrationTests/TestSetup/MgxTransportScope.cs @@ -0,0 +1,112 @@ +using Mgx.Cmdlets.Base; +using Mgx.Engine.Http; + +namespace Mgx.IntegrationTests; + +/// +/// Points MgxCmdletBase at a mock transport for the life of the scope and puts every static it +/// touched back the way it found it, so a later test in the Pipeline collection that drives a +/// cmdlet without injecting cannot inherit this one's wire. +/// +/// The seam sits above the auth probe in GetClient, so a test shell needs no Graph session and no +/// matching fingerprint. The fields are internal and assigned directly rather than reflected: +/// renaming one is a compile error here instead of a NullReferenceException at run time. +/// +/// Every scope resets ResiliencePipelineFactory on entry and exit. Retry-count assertions read +/// the pipeline the factory hands out, so a pipeline built under another test's MaxRetryAttempts +/// would change what those counts mean. +/// +internal sealed class MgxTransportScope : IDisposable +{ + private readonly HttpClient? _createdClient; + private readonly HttpClient? _previousTransport; + private readonly bool _previousOwned; + private readonly ResilientGraphClientOptions _previousOptions; + private readonly string _previousEndpoint; + + private MgxTransportScope(HttpClient client, bool createdHere, bool owned, + ResilientGraphClientOptions options, string endpoint) + { + _createdClient = createdHere ? client : null; + _previousTransport = MgxCmdletBase.s_testTransport; + _previousOwned = MgxCmdletBase.s_testTransportOwned; + _previousOptions = MgxCmdletBase.s_clientOptions; + _previousEndpoint = MgxCmdletBase.s_graphEndpoint; + + ResiliencePipelineFactory.Reset(); + MgxCmdletBase.s_clientOptions = options; + MgxCmdletBase.s_graphEndpoint = endpoint; + Publish(client, owned); + } + + /// + /// Arms a transport: the transport itself, then the epoch that retires whatever preceded it. + /// GetClient takes the epoch, the transport, then the epoch again, and rereads on a change - + /// so the transport going first is what lets an unchanged epoch mean the two belong together. + /// Bumping first would leave a window in which the new epoch is live over the old transport, + /// which a reader cannot tell from a settled state. + /// + /// Entering and leaving both go through here, so the two directions cannot drift apart. + /// + /// + private void Publish(HttpClient? transport, bool owned) + { + MgxCmdletBase.s_testTransportOwned = owned; + MgxCmdletBase.s_testTransport = transport; + BetweenPublishWrites?.Invoke(); + Interlocked.Increment(ref MgxCmdletBase.s_testTransportEpoch); + } + + /// + /// Runs between the two writes above, which is the only point from which their order is + /// visible. Set by the one test that pins it; null for every other scope, so a test running + /// in another collection at the same time is unaffected. + /// + internal Action? BetweenPublishWrites { get; set; } + + /// The options a scope installs unless the caller passes its own. + internal static ResilientGraphClientOptions DefaultOptions => + new() { NoRateLimit = true, MaxRetryAttempts = 1 }; + + /// + /// Arms the seam with an HttpClient over . The scope owns that client + /// and disposes it on exit. + /// + internal static MgxTransportScope Inject( + HttpMessageHandler wire, + bool owned = false, + ResilientGraphClientOptions? options = null, + string endpoint = "https://graph.microsoft.com") => + new(new HttpClient(wire), createdHere: true, owned, options ?? DefaultOptions, endpoint); + + /// + /// Arms the seam with a caller-owned HttpClient, for a test that needs the same instance + /// across several scopes or disposes it itself. + /// + internal static MgxTransportScope Inject( + HttpClient client, + bool owned = false, + ResilientGraphClientOptions? options = null, + string endpoint = "https://graph.microsoft.com") => + new(client, createdHere: false, owned, options ?? DefaultOptions, endpoint); + + /// + /// Replaces the options the armed transport runs under, without rebuilding the transport. + /// Bumps the epoch, so a cmdlet instance holding a client from before the change rebuilds. + /// + internal static void SetOptions(ResilientGraphClientOptions options) + { + ResiliencePipelineFactory.Reset(); + MgxCmdletBase.s_clientOptions = options; + Interlocked.Increment(ref MgxCmdletBase.s_testTransportEpoch); + } + + public void Dispose() + { + Publish(_previousTransport, _previousOwned); + MgxCmdletBase.s_graphEndpoint = _previousEndpoint; + MgxCmdletBase.s_clientOptions = _previousOptions; + ResiliencePipelineFactory.Reset(); + _createdClient?.Dispose(); + } +} diff --git a/tests/Mgx.IntegrationTests/TestSetup/ModuleAssemblyClosureTests.cs b/tests/Mgx.IntegrationTests/TestSetup/ModuleAssemblyClosureTests.cs new file mode 100644 index 0000000..f79bacf --- /dev/null +++ b/tests/Mgx.IntegrationTests/TestSetup/ModuleAssemblyClosureTests.cs @@ -0,0 +1,361 @@ +using System.Reflection.Metadata; +using System.Reflection.PortableExecutable; +using System.Xml.Linq; + +namespace Mgx.IntegrationTests; + +/// +/// Everything the shipped assemblies reference has to be reachable from the module folder plus +/// the runtime the host already has. Nothing else is on offer: the ALC resolver serves +/// Dependencies/ only for assemblies found nowhere, so a reference that is neither ours, nor +/// staged, nor part of the framework src/ targets resolves to nothing and throws +/// FileNotFoundException at the first call that needs it - on a build agent, not here. +/// +/// The closure is computed by reading metadata, not by loading: start at the staged +/// Mgx.Cmdlets.dll and Mgx.Engine.dll and follow every assembly reference through whatever the +/// running runtime would resolve it to. That is deliberately the newest runtime rather than +/// net8.0's, because a host is free to be newer and the module has to survive it: PowerShell 7.6 +/// binds the engine's System.Text.Json reference to its own 10.0 copy, and that copy's own +/// references are as much a part of the module's closure as the engine's are. +/// +/// System.Management.Automation is where the walk stops. It is the host itself, it is never +/// shipped, and whatever it drags in is the host's to supply. +/// +/// Every path here resolves under the repository root - the directory holding Mgx.slnx - and +/// nothing above that root is ever consulted, so a module folder above the repository cannot be +/// measured in place of the staged one. +/// +public class ModuleAssemblyClosureTests +{ + /// The host. Present by definition, never staged, and not expanded. + private const string HostAssembly = "System.Management.Automation"; + + /// + /// A walk that stops early reports no unresolved references and reads exactly like a clean + /// one, so the size of the closure is asserted too. The floor carries slack because + /// references come and go; what it catches is a walk that has stopped walking. + /// + private const int ClosureFloor = 40; + + /// The file that marks the repository root. + private const string RepositoryMarker = "Mgx.slnx"; + + /// + /// Walk up from to the first ancestor containing Mgx.slnx + /// and return it as the repository root. Nothing above that directory is ever consulted, so + /// a decoy further up - a module folder beside a worktree checked out under it - cannot bind. + /// + private static string FindRepositoryRoot(string startDirectory) + { + var dir = new DirectoryInfo(startDirectory); + while (dir != null) + { + if (File.Exists(Path.Combine(dir.FullName, RepositoryMarker))) return dir.FullName; + dir = dir.Parent; + } + + throw new DirectoryNotFoundException( + $"Could not locate {RepositoryMarker} above {startDirectory}"); + } + + /// Resolve a repository-relative path under the root reached from . + private static string FindRepositoryPath(string relativePath, string startDirectory) + { + var root = FindRepositoryRoot(startDirectory); + var candidate = Path.Combine(root, relativePath); + if (File.Exists(candidate) || Directory.Exists(candidate)) return candidate; + + throw new FileNotFoundException( + $"Could not locate '{relativePath}' under repository root {root}"); + } + + /// FindRepositoryPath, starting from the test binaries. + private static string FindRepositoryPath(string relativePath) => + FindRepositoryPath(relativePath, AppContext.BaseDirectory); + + /// The staged module folder - the directory holding the module manifest. + private static string ModuleRoot() => + Path.GetDirectoryName(FindRepositoryPath( + Path.Combine("Modules", "M365DSC.mgx", "M365DSC.mgx.psd1")))!; + + /// + /// The staged Dependencies folder. Absent means the module has not been built, which is the + /// same demand the Pester harness makes: this test measures build output, so it fails rather + /// than skipping - a skipped test does not count toward the suite floor CI asserts on. + /// + private static string DependenciesDirectory() + { + var deps = Path.Combine(ModuleRoot(), "Dependencies"); + if (!Directory.Exists(deps)) + throw new DirectoryNotFoundException($"Dependencies folder not found at '{deps}'. Run ./build.ps1 first."); + + return deps; + } + + /// Simple assembly names of the DLLs in a folder, or an empty set if it is absent. + private static HashSet DllNamesIn(string directory) + { + var names = new HashSet(StringComparer.OrdinalIgnoreCase); + if (!Directory.Exists(directory)) return names; + + foreach (var file in Directory.EnumerateFiles(directory, "*.dll")) + names.Add(Path.GetFileNameWithoutExtension(file)); + + return names; + } + + /// The framework src/ compiles against, e.g. "net8.0". Read, not hardcoded, so a TFM bump moves this with it. + private static string ShippedTargetFramework() + { + var csproj = FindRepositoryPath(Path.Combine("src", "Mgx.Cmdlets", "Mgx.Cmdlets.csproj")); + var tfm = XDocument.Load(csproj).Descendants("TargetFramework").FirstOrDefault()?.Value; + + Assert.False(string.IsNullOrWhiteSpace(tfm), $"No in {csproj}"); + return tfm!; + } + + /// Directories that may hold a Microsoft.NETCore.App reference pack. + private static IEnumerable ReferencePackRoots() + { + // Restored as a package when the SDK building net8.0 is newer than 8.0 ... + var nugetRoot = Environment.GetEnvironmentVariable("NUGET_PACKAGES"); + if (string.IsNullOrEmpty(nugetRoot)) + nugetRoot = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".nuget", "packages"); + yield return Path.Combine(nugetRoot, "microsoft.netcore.app.ref"); + + // ... and bundled with the SDK when that SDK is installed alongside, which is what CI has. + var dotnetRoot = Environment.GetEnvironmentVariable("DOTNET_ROOT"); + if (string.IsNullOrEmpty(dotnetRoot)) + { + // /shared/Microsoft.NETCore.App// is where the core library lives. + var runtimeDirectory = RuntimeDirectory(); + dotnetRoot = Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(runtimeDirectory))); + } + + if (!string.IsNullOrEmpty(dotnetRoot)) + yield return Path.Combine(dotnetRoot, "packs", "Microsoft.NETCore.App.Ref"); + } + + /// + /// Simple names of every assembly the shared framework src/ targets provides, taken from that + /// framework's reference pack. This is the list a host is guaranteed to satisfy on its own; + /// anything outside it has to come from the module folder. + /// + private static HashSet SharedFrameworkAssemblyNames(string targetFramework) + { + var major = targetFramework["net".Length..].Split('.')[0] + "."; + var candidates = + from root in ReferencePackRoots() + where Directory.Exists(root) + from versionDirectory in Directory.EnumerateDirectories(root) + let name = Path.GetFileName(versionDirectory) + where name.StartsWith(major, StringComparison.Ordinal) + && Version.TryParse(name, out _) + let refDirectory = Path.Combine(versionDirectory, "ref", targetFramework) + where Directory.Exists(refDirectory) + orderby Version.Parse(name) + select refDirectory; + + var newest = candidates.LastOrDefault(); + Assert.True(newest is not null, + $"No Microsoft.NETCore.App reference pack for {targetFramework} under: " + + string.Join(", ", ReferencePackRoots()) + + ". The closure cannot be checked without the list of assemblies the framework supplies."); + + return DllNamesIn(newest!); + } + + /// Where the running shared framework's assemblies sit on disk. + private static string RuntimeDirectory() => Path.GetDirectoryName(typeof(object).Assembly.Location)!; + + /// + /// Assembly references recorded in a file's metadata. Reading rather than loading keeps the + /// walk honest about the staged copies: Mgx.Cmdlets, Polly.Core and the rest are already + /// loaded in this process from the test's own output folder, and loading them again from + /// module/ would just hand back the copy that is already there. + /// + private static IReadOnlyList ReferencesOf(string assemblyPath) + { + using var stream = File.OpenRead(assemblyPath); + using var pe = new PEReader(stream); + var metadata = pe.GetMetadataReader(); + + return [.. metadata.AssemblyReferences + .Select(handle => metadata.GetString(metadata.GetAssemblyReference(handle).Name))]; + } + + /// + /// The file a host would bind a reference to: the module folder first, then this process's own + /// folder, then the shared framework - app-local before framework, the order the runtime uses. + /// + private static string? ResolveReference(string simpleName, params string[] directories) + { + foreach (var directory in directories) + { + var candidate = Path.Combine(directory, $"{simpleName}.dll"); + if (File.Exists(candidate)) return candidate; + } + + return null; + } + + /// + /// The runtime's own implementation assemblies. They are in no reference pack because they + /// have no compile-time surface, and no package ships them - they exist wherever the runtime + /// does, so a host cannot be missing one. + /// + private static bool IsRuntimePrivate(string simpleName) => + simpleName.StartsWith("System.Private.", StringComparison.Ordinal); + + private sealed record Reached(string Name, string ReferencedBy, string? Path); + + /// + /// Breadth-first over assembly references from the two staged assemblies, recording where each + /// name was first reached from and which file it binds to. + /// + private static IReadOnlyList Closure() + { + var moduleRoot = ModuleRoot(); + var dependencies = DependenciesDirectory(); + var probeOrder = new[] { moduleRoot, dependencies, AppContext.BaseDirectory, RuntimeDirectory() }; + + var roots = new[] { "Mgx.Cmdlets", "Mgx.Engine" }; + var reached = new List(); + var seen = new HashSet(roots, StringComparer.OrdinalIgnoreCase); + var queue = new Queue<(string Name, string Path)>(); + + foreach (var root in roots) + { + var path = Path.Combine(moduleRoot, $"{root}.dll"); + Assert.True(File.Exists(path), $"'{path}' not found. Run ./build.ps1 first."); + queue.Enqueue((root, path)); + } + + while (queue.Count > 0) + { + var (from, path) = queue.Dequeue(); + foreach (var name in ReferencesOf(path)) + { + if (!seen.Add(name)) continue; + + var resolved = ResolveReference(name, probeOrder); + reached.Add(new Reached(name, from, resolved)); + + // The host and everything below it is the host's to supply. + if (name == HostAssembly) continue; + if (resolved is not null) queue.Enqueue((name, resolved)); + } + } + + return reached; + } + + [Fact] + public void Every_referenced_assembly_resolves_from_the_module_layout_or_the_framework() + { + var mgx = DllNamesIn(ModuleRoot()); + var staged = DllNamesIn(DependenciesDirectory()); + var framework = SharedFrameworkAssemblyNames(ShippedTargetFramework()); + var closure = Closure(); + + var unsatisfied = closure + .Where(r => !mgx.Contains(r.Name) + && !staged.Contains(r.Name) + && !framework.Contains(r.Name) + && !IsRuntimePrivate(r.Name) + && r.Name != HostAssembly) + .Select(r => $"{r.Name} (referenced by {r.ReferencedBy})") + .ToList(); + + Assert.True(unsatisfied.Count == 0, + "assemblies in the module's closure that a host is not guaranteed to have and the " + + "module does not ship - stage each into module/Dependencies/ (build.ps1 and " + + "Mgx.Cmdlets.csproj's StageModuleOutput target both list what is staged): " + + string.Join("; ", unsatisfied)); + + // Two ways the check above goes quiet: a walk that stops after the roots, and a reference + // that resolves to nothing so its own references are never read. Neither shows up as a + // failure on its own, so both are asserted. + var dead = closure + .Where(r => r.Path is null && r.Name != HostAssembly) + .Select(r => $"{r.Name} (referenced by {r.ReferencedBy})") + .ToList(); + + Assert.True(dead.Count == 0, + "references the walk could not bind to a file, so nothing below them was read: " + + string.Join("; ", dead)); + + Assert.True(closure.Count >= ClosureFloor, + $"the walk reached {closure.Count} assemblies - it is not following references"); + } + + [Fact] + public void The_walk_up_does_not_cross_the_repository_root() + { + // A decoy module/mgx.psd1 sits above the repository root here - the shape of a worktree + // checked out under a directory that holds a module folder of its own, where a walk-up + // taking the first ancestor with a module/mgx.psd1 under it measures the decoy's staging + // and reports on a module nobody built. Anchoring on Mgx.slnx means the decoy is never + // reached: the staged manifest resolves under the marked root, and a root without one + // resolves to nothing rather than to the decoy above it. + var sandbox = Path.Combine(Path.GetTempPath(), $"mgx-closure-root-test-{Guid.NewGuid():N}"); + var manifest = Path.Combine("module", "mgx.psd1"); + var repoRoot = Path.Combine(sandbox, "repo"); + var bareRoot = Path.Combine(sandbox, "bare"); + try + { + Directory.CreateDirectory(Path.Combine(sandbox, "module")); + File.WriteAllText(Path.Combine(sandbox, manifest), string.Empty); + + Directory.CreateDirectory(Path.Combine(repoRoot, "module")); + File.WriteAllText(Path.Combine(repoRoot, RepositoryMarker), string.Empty); + File.WriteAllText(Path.Combine(repoRoot, manifest), string.Empty); + var startDirectory = Path.Combine(repoRoot, "tests", "x", "bin", "Debug"); + Directory.CreateDirectory(startDirectory); + + Assert.Equal(Path.Combine(repoRoot, manifest), + FindRepositoryPath(manifest, startDirectory)); + + // The same tree with nothing staged under its root: the decoy above is still not an + // answer, and a walk that climbs past the marker to find one binds it here. + Directory.CreateDirectory(bareRoot); + File.WriteAllText(Path.Combine(bareRoot, RepositoryMarker), string.Empty); + var bareStart = Path.Combine(bareRoot, "tests", "x", "bin", "Debug"); + Directory.CreateDirectory(bareStart); + + try + { + var climbed = FindRepositoryPath(manifest, bareStart); + Assert.Fail($"a root with no '{manifest}' of its own resolved one at '{climbed}'"); + } + catch (FileNotFoundException) + { + // Nothing above the root was consulted, which is the whole of it. + } + } + finally + { + Directory.Delete(sandbox, recursive: true); + } + } + + [Fact] + public void The_staged_dependencies_are_carried_not_compiled_against() + { + // Dependencies/ holds two different kinds of file. Polly.Core and + // System.Threading.RateLimiting are ours: the engine compiles against them, and they are + // isolated there so a neighboring module's copy cannot win the bind. System.IO.Pipelines + // is nobody's: it is carried because the System.Text.Json a newer host supplies needs it + // and net8.0 does not include it. If it ever turned into a compile-time reference, the + // engine would be built against a surface the framework it targets does not have. + var moduleRoot = ModuleRoot(); + var referenced = ReferencesOf(Path.Combine(moduleRoot, "Mgx.Engine.dll")) + .Concat(ReferencesOf(Path.Combine(moduleRoot, "Mgx.Cmdlets.dll"))) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + Assert.DoesNotContain("System.IO.Pipelines", referenced); + Assert.Contains("Polly.Core", referenced); + Assert.Contains("System.Threading.RateLimiting", referenced); + } +} diff --git a/tests/Mgx.IntegrationTests/TestSetup/RegressionCorpusIndexTests.cs b/tests/Mgx.IntegrationTests/TestSetup/RegressionCorpusIndexTests.cs new file mode 100644 index 0000000..85a9d7a --- /dev/null +++ b/tests/Mgx.IntegrationTests/TestSetup/RegressionCorpusIndexTests.cs @@ -0,0 +1,207 @@ +using System.Text.RegularExpressions; + +namespace Mgx.IntegrationTests; + +/// +/// Provenance is only useful if it can be trusted, and a doc comment naming an issue number is +/// unverifiable on its own. Every issue id written anywhere under tests/ has to have a row in +/// the index saying what it guarantees and where. The index itself is private (tmp/CORPUS.md, +/// untracked), so a host without it - public CI - runs these as counted no-ops: the guard +/// binds wherever the index lives. Every path here resolves under the repository root - the +/// directory holding Mgx.slnx - and nothing above that root is ever consulted. +/// +/// The guard runs one way only. The reverse - every indexed id has a test - cannot hold while +/// the index carries rows that name the release bringing their test, which is the honest way to +/// record a gap. +/// +public class RegressionCorpusIndexTests +{ + private const string IndexPath = "tmp/CORPUS.md"; + private const string RepositoryMarker = "Mgx.slnx"; + + /// + /// Matches an issue id and any ids abbreviated onto it: "M365DSC-5306/7175" is two ids, and + /// a scan that read it as one would let the second go unindexed. + /// + private static readonly Regex IssueId = new( + @"\b(M365DSC|GraphSDK)-(\d+)((?:/\d+)*)", RegexOptions.Compiled); + + private static IEnumerable IdsIn(string text) + { + foreach (Match match in IssueId.Matches(text)) + { + var prefix = match.Groups[1].Value; + yield return $"{prefix}-{match.Groups[2].Value}"; + foreach (var abbreviated in match.Groups[3].Value.Split('/', StringSplitOptions.RemoveEmptyEntries)) + yield return $"{prefix}-{abbreviated}"; + } + } + + /// + /// Walk up from to the first ancestor containing + /// Mgx.slnx and return it as the repository root. Nothing above that directory is ever + /// consulted, so a decoy further up - a worktree's own tmp/, a stranger's tests/ - cannot + /// bind. + /// + private static string FindRepositoryRoot(string startDirectory) + { + var dir = new DirectoryInfo(startDirectory); + while (dir != null) + { + if (File.Exists(Path.Combine(dir.FullName, RepositoryMarker))) return dir.FullName; + dir = dir.Parent; + } + + throw new DirectoryNotFoundException( + $"Could not locate {RepositoryMarker} above {startDirectory}"); + } + + /// Resolve a repository-relative path under the root reached from . + private static string FindRepositoryPath(string relativePath, string startDirectory) + { + var root = FindRepositoryRoot(startDirectory); + var candidate = Path.Combine(root, relativePath); + if (File.Exists(candidate) || Directory.Exists(candidate)) return candidate; + + throw new DirectoryNotFoundException( + $"Could not locate '{relativePath}' under repository root {root}"); + } + + /// FindRepositoryPath, starting from the test binaries. + private static string FindRepositoryPath(string relativePath) => + FindRepositoryPath(relativePath, AppContext.BaseDirectory); + + /// FindRepositoryPath for a file allowed to be absent: null instead of a throw. + private static string? TryFindRepositoryPath(string relativePath, string startDirectory) + { + try { return FindRepositoryPath(relativePath, startDirectory); } + catch (DirectoryNotFoundException) { return null; } + } + + /// TryFindRepositoryPath, starting from the test binaries. + private static string? TryFindRepositoryPath(string relativePath) => + TryFindRepositoryPath(relativePath, AppContext.BaseDirectory); + + /// + /// Every source file under tests/, minus build output, the index itself, and this file - which + /// carries fabricated ids on purpose and would otherwise fail its own guard. + /// + private static IEnumerable CorpusSourceFiles() + { + var testsRoot = FindRepositoryPath("tests"); + // The index lives outside tests/ and needs no exclusion; this file carries fabricated + // ids on purpose and would otherwise fail its own guard. + var excluded = new[] + { + Path.GetFullPath(FindRepositoryPath( + Path.Combine("tests", "Mgx.IntegrationTests", "TestSetup", "RegressionCorpusIndexTests.cs"))) + }; + + return Directory.EnumerateFiles(testsRoot, "*", SearchOption.AllDirectories) + .Where(path => path.EndsWith(".cs", StringComparison.OrdinalIgnoreCase) + || path.EndsWith(".ps1", StringComparison.OrdinalIgnoreCase) + || path.EndsWith(".psm1", StringComparison.OrdinalIgnoreCase) + || path.EndsWith(".md", StringComparison.OrdinalIgnoreCase)) + .Where(path => !path.Contains($"{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}", + StringComparison.Ordinal) + && !path.Contains($"{Path.DirectorySeparatorChar}obj{Path.DirectorySeparatorChar}", + StringComparison.Ordinal)) + .Where(path => !excluded.Contains(Path.GetFullPath(path), StringComparer.Ordinal)); + } + + /// The ids a body of text claims, that the index does not carry. + private static IReadOnlyList Unindexed(string indexText, string claimingText) + { + var indexed = IdsIn(indexText).ToHashSet(StringComparer.Ordinal); + return [.. IdsIn(claimingText).Distinct(StringComparer.Ordinal).Where(id => !indexed.Contains(id))]; + } + + [Fact] + public void Every_issue_id_under_tests_has_a_row_in_the_index() + { + if (TryFindRepositoryPath(IndexPath) is not string indexFile) return; + var indexText = File.ReadAllText(indexFile); + + var missing = new SortedDictionary>(StringComparer.Ordinal); + var claimed = new HashSet(StringComparer.Ordinal); + var scanned = 0; + foreach (var file in CorpusSourceFiles()) + { + scanned++; + var text = File.ReadAllText(file); + claimed.UnionWith(IdsIn(text)); + + foreach (var id in Unindexed(indexText, text)) + { + if (!missing.TryGetValue(id, out var files)) missing[id] = files = []; + files.Add(Path.GetFileName(file)); + } + } + + // A scan that reaches nothing reports nothing missing, so the guard has to fail on its + // own silence first. A repository root resolved above the wrong directory, an extension + // filter that stops matching, an exclusion that widens - each of those disarms the check + // below while leaving it green. The floors carry slack because files and ids are added + // over time; what they catch is a scan that has stopped running, not one that has moved. + Assert.True(scanned >= 50, + $"the scan reached {scanned} file(s) under tests/ - it is not reading the suite"); + Assert.True(claimed.Count >= 12, + $"the scan found {claimed.Count} issue id(s) under tests/ - it is not reading provenance"); + + Assert.True(missing.Count == 0, + "issue ids claimed under tests/ with no row in " + IndexPath + ": " + + string.Join("; ", missing.Select(e => $"{e.Key} ({string.Join(", ", e.Value)})"))); + } + + [Fact] + public void A_fabricated_id_in_a_test_file_fails_the_guard() + { + // What the guard above is worth: an id nobody has indexed has to be reported, including + // one abbreviated onto an id that IS indexed. + if (TryFindRepositoryPath(IndexPath) is not string indexFile) return; + var indexText = File.ReadAllText(indexFile); + + Assert.Equal(["GraphSDK-9999"], + Unindexed(indexText, "/// (Corpus: GraphSDK-9999, an issue that does not exist.)")); + + Assert.Equal(["M365DSC-9999"], + Unindexed(indexText, "/// (Corpus: M365DSC-7273/9999, one real id and one invented.)")); + } + + [Fact] + public void The_index_is_where_the_suite_says_it_is() + { + // A stale path here would read as an absent index and quietly turn all three guards + // into no-ops on every host; a host that carries the index pins the path with this. + if (TryFindRepositoryPath("tmp") is not string) return; + Assert.True(TryFindRepositoryPath(IndexPath) is not null); + } + + [Fact] + public void The_walk_up_does_not_cross_the_repository_root() + { + // A decoy tmp/ sits above the repository root here - the exact shape of a worktree + // checked out under the system temp directory, where the walk-up used to resolve the + // system temp directory itself as the repository's tmp/. Anchoring on Mgx.slnx means + // the decoy is never reached: "tests" resolves under the marked root, and "tmp" - which + // exists only above the root - resolves to nothing. + var sandbox = Path.Combine(Path.GetTempPath(), $"mgx-corpus-root-test-{Guid.NewGuid():N}"); + var repoRoot = Path.Combine(sandbox, "repo"); + var startDirectory = Path.Combine(repoRoot, "tests", "x", "bin", "Debug"); + try + { + Directory.CreateDirectory(Path.Combine(sandbox, "tmp")); + Directory.CreateDirectory(repoRoot); + File.WriteAllText(Path.Combine(repoRoot, "Mgx.slnx"), string.Empty); + Directory.CreateDirectory(Path.Combine(repoRoot, "tests")); + Directory.CreateDirectory(startDirectory); + + Assert.Equal(Path.Combine(repoRoot, "tests"), FindRepositoryPath("tests", startDirectory)); + Assert.Null(TryFindRepositoryPath("tmp", startDirectory)); + } + finally + { + Directory.Delete(sandbox, recursive: true); + } + } +} diff --git a/tests/TestHarness.psm1 b/tests/TestHarness.psm1 index 6f684e9..4f51a20 100644 --- a/tests/TestHarness.psm1 +++ b/tests/TestHarness.psm1 @@ -1,4 +1,4 @@ -#Requires -Version 7.0 +#Requires -Version 7.0 <# .SYNOPSIS @@ -323,7 +323,18 @@ function Invoke-PesterTest $configuration.CodeCoverage.Enabled = $false Write-Host -Object 'Running all mgx PowerShell Unit Tests' - return Invoke-Pester -Configuration $configuration + $results = Invoke-Pester -Configuration $configuration + + foreach ($container in $results.Containers | Where-Object { -not $_.Passed }) + { + if ($container.Result -eq 'Failed' -and $container.Tests.Count -eq 0) + { + $name = if ($container.Item -is [System.IO.FileInfo]) { $container.Item.FullName } else { "$($container.Item)" } + Write-Warning -Message "The test file '$name' could not be run at all, so none of its tests were: $($container.ErrorRecord.Exception.Message)" + } + } + + return $results } <# diff --git a/tests/Unit/Benchmarks/Mgx.BenchBaseline.Tests.ps1 b/tests/Unit/Benchmarks/Mgx.BenchBaseline.Tests.ps1 new file mode 100644 index 0000000..d58ea94 --- /dev/null +++ b/tests/Unit/Benchmarks/Mgx.BenchBaseline.Tests.ps1 @@ -0,0 +1,882 @@ +#Requires -Modules Pester + +<# + Baseline promotion and comparison in tests/benchmarks/common.ps1. + + These cover the benchmark harness rather than the built module: no tenant, no Graph + session, and nothing read out of the real results/ directory - every fixture is written + under $TestDrive and every call is pointed at it. +#> + +BeforeAll { + $script:BenchRoot = Join-Path (Split-Path -Parent (Split-Path -Parent $PSScriptRoot)) 'benchmarks' + + # common.ps1 pins the thread to invariant culture so benchmark output never prints 39,2s. + # Right for a benchmark process, and not something to leave behind in a test run. + $script:CultureBefore = [System.Threading.Thread]::CurrentThread.CurrentCulture + . (Join-Path $script:BenchRoot 'common.ps1') + + function script:New-BenchEntry + { + param + ( + [Parameter(Mandatory)] [hashtable] $Result, + [datetime] $RecordedAt = (Get-Date) + ) + + return [pscustomobject]@{ + Result = [pscustomobject]$Result + Telemetry = $null + MgxVersion = '2.1.5' + SdkVersion = '2.34.0' + PSVersion = '7.6.4' + RecordedAt = $RecordedAt.ToString('o') + } + } + + function script:Write-BenchFixture + { + param + ( + [Parameter(Mandatory)] [string] $Directory, + [Parameter(Mandatory)] [string] $FileName, + [Parameter(Mandatory)] [object[]] $Entries + ) + + if (-not (Test-Path -Path $Directory)) + { + New-Item -ItemType Directory -Path $Directory -Force | Out-Null + } + ConvertTo-Json -InputObject @($Entries) -Depth 10 | + Set-Content -Path (Join-Path $Directory $FileName) + } + + # A 06 gauntlet result in the shape 06-fault-gauntlet.ps1 records. + function script:New-GauntletResult + { + param ([Parameter(Mandatory)] [int] $MgxMs) + + return @{ + mgx = [pscustomobject]@{ ElapsedMs = $MgxMs; Output = [pscustomobject]@{ ok = 1000; failed = 0 } } + sdkMgx = [pscustomobject]@{ ElapsedMs = 246715 } + } + } +} + +AfterAll { + [System.Threading.Thread]::CurrentThread.CurrentCulture = $script:CultureBefore +} + +Describe 'benchmark scenario table' { + It 'defines metrics for every benchmark run.ps1 executes' { + # Everything that writes or reads a baseline keys off this table. A scenario added to + # the suite without an entry here is pinned as nothing and read as unmeasured. + $runText = Get-Content -Path (Join-Path $script:BenchRoot 'run.ps1') -Raw + $suiteIds = [regex]::Matches($runText, "id\s*=\s*'(\d+)'") | ForEach-Object { $_.Groups[1].Value } + $suiteIds.Count | Should -BeGreaterThan 0 + + $known = (Get-BenchScenario).Id + foreach ($id in $suiteIds) + { + $known | Should -Contain $id -Because "run.ps1 runs benchmark $id" + } + } + + It 'names each scenario a headline metric first' { + foreach ($scenario in (Get-BenchScenario)) + { + $scenario.Metrics.Count | Should -BeGreaterThan 0 + $scenario.Metrics[0].Name | Should -Not -BeNullOrEmpty + } + } + + It 'skips an id it has no metrics for rather than throwing' { + Get-BenchScenario -Id '99' -WarningAction SilentlyContinue | Should -BeNullOrEmpty + } +} + +Describe 'Export-BenchBaseline' { + BeforeEach { + $script:Results = Join-Path $TestDrive 'results' + $script:Baseline = Join-Path $TestDrive 'baseline.json' + Remove-Item -Path $script:Results, $script:Baseline -Recurse -Force -ErrorAction SilentlyContinue + } + + It 'promotes the newest entry, its headline metric and its version block' { + Write-BenchFixture -Directory $script:Results -FileName '06-fault-gauntlet.json' -Entries @( + (New-BenchEntry -Result (New-GauntletResult -MgxMs 111111) -RecordedAt (Get-Date).AddDays(-2)), + (New-BenchEntry -Result (New-GauntletResult -MgxMs 134654)) + ) + + Export-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -Path $script:Baseline + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'06' + $pinned.Script | Should -Be '06-fault-gauntlet.ps1' + $pinned.Headline | Should -Be 'MgxFanoutMs' + $pinned.Metrics.MgxFanoutMs.Value | Should -Be 134654 + $pinned.Metrics.MgxFanoutMs.Unit | Should -Be 'ms' + $pinned.Metrics.MgxCompleted.Value | Should -Be 1000 + $pinned.MgxVersion | Should -Be '2.1.5' + $pinned.SdkVersion | Should -Be '2.34.0' + $pinned.PSVersion | Should -Be '7.6.4' + $pinned.RecordedAt | Should -Not -BeNullOrEmpty + } + + It 'leaves scenarios it was not asked about where they were' { + Write-BenchFixture -Directory $script:Results -FileName '06-fault-gauntlet.json' -Entries @( + (New-BenchEntry -Result (New-GauntletResult -MgxMs 134654)) + ) + Write-BenchFixture -Directory $script:Results -FileName '08-delta-sync.json' -Entries @( + (New-BenchEntry -Result @{ initial = [pscustomobject]@{ ElapsedMs = 145621; Output = [pscustomobject]@{ items = 130233 } } }) + ) + + Export-BenchBaseline -Benchmark '08' -ResultsDirectory $script:Results -Path $script:Baseline + Export-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -Path $script:Baseline + + $scenarios = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios + $scenarios.'08'.Metrics.InitialSyncMs.Value | Should -Be 145621 + $scenarios.'06'.Metrics.MgxFanoutMs.Value | Should -Be 134654 + } + + It 'pins no headline when the first metric of the scenario row is absent, rather than the next one' { + # 09's row names CheckpointedExportMs first. An entry with no 'checkpointed' node still + # carries four of its five metrics, and the next one that resolves is not a stand-in for + # the one it is read on - CheckpointOverheadPct is a percentage that crosses zero. + Write-BenchFixture -Directory $script:Results -FileName '09-kill-resume.json' -Entries @( + (New-BenchEntry -Result @{ + baseline = [pscustomobject]@{ ElapsedMs = 140322 } + resume = [pscustomobject]@{ ElapsedMs = 38104 } + verification = [pscustomobject]@{ overheadPct = -3.2; duplicateIds = 0 } + }) + ) + + Export-BenchBaseline -Benchmark '09' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningAction SilentlyContinue + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'09' + $pinned.Headline | Should -BeNullOrEmpty + $pinned.Metrics.BaselineExportMs.Value | Should -Be 140322 + $pinned.Metrics.CheckpointOverheadPct.Value | Should -Be -3.2 + + # And the comparison says so rather than reading whatever is first in the file. + $row = Compare-BenchBaseline -Benchmark '09' -ResultsDirectory $script:Results ` + -BaselinePath $script:Baseline + $row.Status | Should -Be 'not comparable' + $row.Note | Should -BeLike '*no headline*' + $row.Metric | Should -BeNullOrEmpty + $row.DeltaPct | Should -BeNullOrEmpty + } + + It 'tries a metric''s next source when the first one is not numeric' { + # 10's WallSeconds tries Arms.paced.Seconds before WallMs. An entry that recorded the + # scenario as inconclusive on its first source still carries a number on its second, + # and losing that to the first source's non-numeric value would cost WallSeconds its + # headline along with it. + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Mode = 'paced' + Arms = [pscustomobject]@{ paced = [pscustomobject]@{ Seconds = 'INCONCLUSIVE'; ThrottleRetries = 12 } } + WallMs = 39400 + }) + ) + + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningAction SilentlyContinue + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'10' + $pinned.Headline | Should -Be 'WallSeconds' + $pinned.Metrics.WallSeconds.Value | Should -Be 39.4 + $pinned.Metrics.WallSeconds.Path | Should -Be 'WallMs' + } + + It 'warns which metric was not recorded when none of its sources resolves' { + # WallSeconds tries Arms.paced.Seconds, then WallMs. An entry that leaves the first + # inconclusive and never writes the second must not be dropped in silence, and the + # warning it gets must name the value that was not a number rather than just listing + # every path tried - WallMs was never written, and a path with nothing at it is not + # part of the story. + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Mode = 'paced' + Arms = [pscustomobject]@{ + paced = [pscustomobject]@{ Seconds = 'INCONCLUSIVE'; ThrottleRetries = 12 } + unpaced = [pscustomobject]@{ Seconds = 40.3 } + } + }) + ) + + $warnings = $null + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningVariable warnings 3>$null + + "$warnings" | Should -BeLike "*10: 'WallSeconds' not recorded - 'Arms.paced.Seconds' holds 'INCONCLUSIVE', which is not a number*" + "$warnings" | Should -Not -BeLike '*WallMs*' + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'10' + $pinned.Headline | Should -BeNullOrEmpty + $pinned.Metrics.WallSeconds | Should -BeNullOrEmpty + $pinned.Metrics.ThrottleRetries.Value | Should -Be 12 + } + + It 'does not pin an empty string or a boolean as a measurement' { + # [double] coerces both without throwing - '' to 0, $true to 1 - so a source that + # never recorded a number could be pinned as though it had. Neither belongs in the + # baseline, and the warning must name what each source actually held. + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Mode = 'both' + Arms = [pscustomobject]@{ + paced = [pscustomobject]@{ Seconds = ''; ThrottleRetries = 3 } + unpaced = [pscustomobject]@{ Seconds = 40.3 } + } + WallMs = $true + }) + ) + + $warnings = $null + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningVariable warnings 3>$null + + "$warnings" | Should -BeLike "*'Arms.paced.Seconds' holds '', which is not a number*" + "$warnings" | Should -BeLike "*'WallMs' holds 'True', which is not a number*" + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'10' + $pinned.Headline | Should -BeNullOrEmpty + $pinned.Metrics.WallSeconds | Should -BeNullOrEmpty + $pinned.Metrics.UnpacedWallSeconds.Value | Should -Be 40.3 + $pinned.Metrics.ThrottleRetries.Value | Should -Be 3 + } + + It 'refuses "NaN", "Infinity" and "-Infinity" as measurements, named in the not-recorded warning' { + # NumberStyles.Float - the style [double]::TryParse is called with - parses these three + # strings as valid doubles: NaN, PositiveInfinity, NegativeInfinity. None is a + # measurement a baseline can hold a percentage delta against, so each must be refused + # and named rather than pinned. + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Mode = 'both' + Arms = [pscustomobject]@{ + paced = [pscustomobject]@{ Seconds = 'NaN'; ThrottleRetries = 3 } + unpaced = [pscustomobject]@{ Seconds = '-Infinity' } + } + WallMs = 'Infinity' + }) + ) + + $warnings = $null + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningVariable warnings 3>$null + + "$warnings" | Should -BeLike "*10: 'WallSeconds' not recorded - 'Arms.paced.Seconds' holds 'NaN', which is not a number; 'WallMs' holds 'Infinity', which is not a number.*" + "$warnings" | Should -BeLike "*10: 'UnpacedWallSeconds' not recorded - 'Arms.unpaced.Seconds' holds '-Infinity', which is not a number.*" + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'10' + $pinned.Headline | Should -BeNullOrEmpty + $pinned.Metrics.WallSeconds | Should -BeNullOrEmpty + $pinned.Metrics.UnpacedWallSeconds | Should -BeNullOrEmpty + $pinned.Metrics.ThrottleRetries.Value | Should -Be 3 + } + + It 'refuses a typed double NaN, not just the string it prints as' { + # ConvertTo-Json always re-quotes NaN and the infinities as strings, so the fixture + # helper above can only exercise the string arm of Test-BenchNumericValue. A result + # file written with JSON's non-standard bare NaN token round-trips through + # ConvertFrom-Json as a genuine [double] - the other arm - and that value must be + # refused too, not coerced into a measurement by the typed-value check alone. + New-Item -ItemType Directory -Path $script:Results -Force | Out-Null + $raw = @" +[ + { + "Result": { "Mode": "paced", "Arms": { "paced": { "Seconds": NaN, "ThrottleRetries": 12 } } }, + "Telemetry": null, + "MgxVersion": "2.1.5", + "SdkVersion": "2.34.0", + "PSVersion": "7.6.4", + "RecordedAt": "$((Get-Date).ToString('o'))" + } +] +"@ + Set-Content -Path (Join-Path $script:Results '10-pacing-under-real-throttling.json') -Value $raw + + $warnings = $null + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningVariable warnings 3>$null + + "$warnings" | Should -BeLike "*10: 'WallSeconds' not recorded - 'Arms.paced.Seconds' holds 'NaN', which is not a number*" + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'10' + $pinned.Headline | Should -BeNullOrEmpty + $pinned.Metrics.WallSeconds | Should -BeNullOrEmpty + $pinned.Metrics.ThrottleRetries.Value | Should -Be 12 + } + + It 'warns per rescued non-number and pins the metric from the source that resolved it' { + # A first source that recorded 'INCONCLUSIVE' before a second recorded the real figure + # used to fall back in silence. The rescue is now visible - one warning per source that + # was tried and skipped - and the metric still pins from the source that resolved it. + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Mode = 'paced' + Arms = [pscustomobject]@{ paced = [pscustomobject]@{ Seconds = 'INCONCLUSIVE'; ThrottleRetries = 12 } } + WallMs = 41200 + }) + ) + + $warnings = $null + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningVariable warnings 3>$null + + "$warnings" | Should -BeLike "*10: 'WallSeconds' - 'Arms.paced.Seconds' holds 'INCONCLUSIVE', which is not a number - trying the next source.*" + "$warnings" | Should -Not -BeLike '*not recorded*' + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'10' + $pinned.Headline | Should -Be 'WallSeconds' + $pinned.Metrics.WallSeconds.Value | Should -Be 41.2 + $pinned.Metrics.WallSeconds.Path | Should -Be 'WallMs' + } + + It 'does not warn about an optional source a single-arm entry never writes' { + # UnpacedWallSeconds' only source is Arms.unpaced.Seconds, written only when run.ps1 + # drives both arms. A standalone -Mode paced run never writes it - that is not a + # metric gone missing, it is one this entry was never going to carry, and every + # healthy single-arm run must promote in silence. + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Mode = 'paced' + Arms = [pscustomobject]@{ paced = [pscustomobject]@{ Seconds = 41.2; ThrottleRetries = 3 } } + }) + ) + + $warnings = $null + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningVariable warnings 3>$null + + $warnings | Should -BeNullOrEmpty + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'10' + $pinned.Headline | Should -Be 'WallSeconds' + $pinned.Metrics.UnpacedWallSeconds | Should -BeNullOrEmpty + $pinned.Metrics.WallSeconds.Value | Should -Be 41.2 + } + + It 'does not promote an entry older than the run that asked for it' { + # A benchmark that died before recording leaves the previous run's entry newest. + Write-BenchFixture -Directory $script:Results -FileName '06-fault-gauntlet.json' -Entries @( + (New-BenchEntry -Result (New-GauntletResult -MgxMs 134654) -RecordedAt (Get-Date).AddDays(-2)) + ) + + Export-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -Path $script:Baseline ` + -Since (Get-Date) -WarningAction SilentlyContinue + + Test-Path -Path $script:Baseline | Should -BeFalse + } + + It 'does not promote a run that recorded itself inconclusive, and names the scenario' { + # 10 records Conclusive false when its unpaced arm was never refused: the throttled + # regime it exists to measure was never reached, so the wall times beside the verdict + # are two unthrottled runs. They are numbers, and every later run would be read + # against them. + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Conclusive = $false + Arms = [pscustomobject]@{ + paced = [pscustomobject]@{ Seconds = 399.7; ThrottleRetries = 0 } + unpaced = [pscustomobject]@{ Seconds = 79.8 } + } + }) + ) + + $warnings = $null + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningVariable warnings 3>$null + + "$warnings" | Should -BeLike '*benchmark 10 recorded an inconclusive run (Conclusive is false) - not promoted*' + Test-Path -Path $script:Baseline | Should -BeFalse + } + + It 'quotes the reason an inconclusive entry carries, where it carries one' { + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Conclusive = $false + InconclusiveReason = 'the unpaced arm was never throttled' + Arms = [pscustomobject]@{ + paced = [pscustomobject]@{ Seconds = 399.7; ThrottleRetries = 0 } + unpaced = [pscustomobject]@{ Seconds = 79.8 } + } + }) + ) + + $warnings = $null + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningVariable warnings 3>$null + + "$warnings" | Should -BeLike '*(the unpaced arm was never throttled)*' + "$warnings" | Should -Not -BeLike '*Conclusive is false*' + } + + It 'promotes a run that recorded itself conclusive' { + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Conclusive = $true + Arms = [pscustomobject]@{ + paced = [pscustomobject]@{ Seconds = 61.4; ThrottleRetries = 3 } + unpaced = [pscustomobject]@{ Seconds = 88.2 } + } + }) + ) + + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'10' + $pinned.Headline | Should -Be 'WallSeconds' + $pinned.Metrics.WallSeconds.Value | Should -Be 61.4 + $pinned.Metrics.UnpacedWallSeconds.Value | Should -Be 88.2 + } + + It 'promotes a scenario that records no verdict at all' { + # The field is a claim a scenario may make, not one the promotion requires. Seven of + # the eight scenarios in the file never write it. + Write-BenchFixture -Directory $script:Results -FileName '06-fault-gauntlet.json' -Entries @( + (New-BenchEntry -Result (New-GauntletResult -MgxMs 134654)) + ) + + Export-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -Path $script:Baseline + + $pinned = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'06' + $pinned.Metrics.MgxFanoutMs.Value | Should -Be 134654 + } + + It 'leaves the row already pinned where it was when the next run is inconclusive' { + # A stale conclusive number is still a measurement, and the comparison reports its age. + # Dropping the row instead would report the scenario as unmeasured, which is the one + # thing the file exists to prevent. + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Conclusive = $true + Arms = [pscustomobject]@{ + paced = [pscustomobject]@{ Seconds = 61.4; ThrottleRetries = 3 } + unpaced = [pscustomobject]@{ Seconds = 88.2 } + } + }) + ) + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline + + # A later suite: 10 comes back inconclusive and 06 records normally, so the file is + # rewritten around the row that must not move. + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ + Conclusive = $false + Arms = [pscustomobject]@{ + paced = [pscustomobject]@{ Seconds = 399.7; ThrottleRetries = 0 } + unpaced = [pscustomobject]@{ Seconds = 79.8 } + } + }) + ) + Write-BenchFixture -Directory $script:Results -FileName '06-fault-gauntlet.json' -Entries @( + (New-BenchEntry -Result (New-GauntletResult -MgxMs 134654)) + ) + + $warnings = $null + Export-BenchBaseline -Benchmark '06', '10' -ResultsDirectory $script:Results -Path $script:Baseline ` + -WarningVariable warnings 3>$null + + $scenarios = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios + $scenarios.'10'.Metrics.WallSeconds.Value | Should -Be 61.4 + $scenarios.'10'.Metrics.UnpacedWallSeconds.Value | Should -Be 88.2 + $scenarios.'06'.Metrics.MgxFanoutMs.Value | Should -Be 134654 + "$warnings" | Should -BeLike '*the row already in the file is left as it was*' + } +} + +Describe 'Get-BenchScenarioMetric' { + It 'refuses a source whose scaled value overflows, and tries the next source' { + # None of the table's own Scale values can overflow a source that was itself finite - + # 10's only one, WallMs at 0.001, shrinks - so this is exercised directly against a + # scenario shaped like the real ones rather than through Export-BenchBaseline. A + # source read within double's range is not the same thing as a source whose scaled + # value is: '1e308' is a legitimate double, and Scale 1000 pushes it past + # [double]::MaxValue. + $scenario = [pscustomobject]@{ + Id = 'S' + Discriminator = $null + Metrics = @( + @{ Name = 'ScaledSeconds'; Unit = 's'; Sources = @(@{ Path = 'Raw'; Scale = 1000.0 }, @{ Path = 'Fallback' }) } + ) + } + $entry = [pscustomobject]@{ Result = [pscustomobject]@{ Raw = '1e308'; Fallback = 41.5 } } + + $warnings = $null + $read = Get-BenchScenarioMetric -Scenario $scenario -Entry $entry -WarningVariable warnings 3>$null + + "$warnings" | Should -BeLike "*S: 'ScaledSeconds' - 'Raw' holds '1e308', which scaled by 1000 is not finite - trying the next source.*" + $read.Metrics.ScaledSeconds.Value | Should -Be 41.5 + $read.Metrics.ScaledSeconds.Path | Should -Be 'Fallback' + } + + It 'warns and pins nothing when the only source overflows once scaled' { + $scenario = [pscustomobject]@{ + Id = 'S' + Discriminator = $null + Metrics = @( + @{ Name = 'ScaledSeconds'; Unit = 's'; Sources = @(@{ Path = 'Raw'; Scale = 1000.0 }) } + ) + } + $entry = [pscustomobject]@{ Result = [pscustomobject]@{ Raw = '1e308' } } + + $warnings = $null + $read = Get-BenchScenarioMetric -Scenario $scenario -Entry $entry -WarningVariable warnings 3>$null + + "$warnings" | Should -BeLike "*S: 'ScaledSeconds' not recorded - 'Raw' holds '1e308', which scaled by 1000 is not finite.*" + $read.Metrics.ScaledSeconds | Should -BeNullOrEmpty + } + + It 'names each source by path when two of them overflow the same metric' { + # Before the path was named here, two overflowing sources on the same metric printed + # the identical "'1e308' scaled by is not finite" for both, and the record could + # not say which one was which. Different scales on the same raw value pin that down. + $scenario = [pscustomobject]@{ + Id = 'S' + Discriminator = $null + Metrics = @( + @{ Name = 'ScaledSeconds'; Unit = 's'; Sources = @(@{ Path = 'RawA'; Scale = 1000.0 }, @{ Path = 'RawB'; Scale = 1e10 }) } + ) + } + $entry = [pscustomobject]@{ Result = [pscustomobject]@{ RawA = '1e308'; RawB = '1e308' } } + + $warnings = $null + $read = Get-BenchScenarioMetric -Scenario $scenario -Entry $entry -WarningVariable warnings 3>$null + + "$warnings" | Should -BeLike "*'RawA' holds '1e308', which scaled by 1000 is not finite*" + "$warnings" | Should -BeLike "*'RawB' holds '1e308', which scaled by 10000000000 is not finite*" + $read.Metrics.ScaledSeconds | Should -BeNullOrEmpty + } + + It 'shows a non-number raw value the way the comparison does, not PowerShell''s default array join' { + # Interpolating an array or object directly prints PowerShell's own join - '1 2 3' for + # @(1,2,3), space-separated and unrecognizable as the source. ConvertTo-BenchDisplayValue + # is what Compare-BenchBaseline already reads a non-number baseline value through; the + # record path's warning is read the same way rather than through raw interpolation. + $scenario = [pscustomobject]@{ + Id = 'S' + Discriminator = $null + Metrics = @( + @{ Name = 'Metric'; Unit = 'ms'; Sources = @(@{ Path = 'Raw' }) } + ) + } + $entry = [pscustomobject]@{ Result = [pscustomobject]@{ Raw = @(1, 2, 3) } } + + $warnings = $null + $read = Get-BenchScenarioMetric -Scenario $scenario -Entry $entry -WarningVariable warnings 3>$null + + # -BeLike reads a bracketed substring as a wildcard character class, not literal text, + # so the exact message is compared instead - it also pins the whole sentence rather + # than just the fragment that changed. + "$warnings" | Should -Be "S: 'Metric' not recorded - 'Raw' holds '[1,2,3]', which is not a number." + "$warnings" | Should -Not -BeLike '*1 2 3*' + $read.Metrics.Metric | Should -BeNullOrEmpty + } +} + +Describe 'Compare-BenchBaseline' { + BeforeEach { + $script:Results = Join-Path $TestDrive 'results' + $script:Baseline = Join-Path $TestDrive 'baseline.json' + Remove-Item -Path $script:Results, $script:Baseline -Recurse -Force -ErrorAction SilentlyContinue + + Write-BenchFixture -Directory $script:Results -FileName '06-fault-gauntlet.json' -Entries @( + (New-BenchEntry -Result (New-GauntletResult -MgxMs 134654)) + ) + Export-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -Path $script:Baseline + } + + It 'reports a delta on the headline metric' { + $row = Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -BaselinePath $script:Baseline + $row.Scenario | Should -Be '06' + $row.Metric | Should -Be 'MgxFanoutMs' + $row.Status | Should -Be 'compared' + $row.DeltaPct | Should -Be 0 + $row.Flagged | Should -BeFalse + } + + It 'flags a headline metric past the threshold without failing' { + Write-BenchFixture -Directory $script:Results -FileName '06-fault-gauntlet.json' -Entries @( + (New-BenchEntry -Result (New-GauntletResult -MgxMs 269308)) + ) + + $row = Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -BaselinePath $script:Baseline + $row.DeltaPct | Should -Be 100 + $row.Flagged | Should -BeTrue + $row.Note | Should -Be 'REGRESSED' + } + + It 'reports a scenario with no baseline entry as unmeasured rather than skipping it' { + Write-BenchFixture -Directory $script:Results -FileName '08-delta-sync.json' -Entries @( + (New-BenchEntry -Result @{ initial = [pscustomobject]@{ ElapsedMs = 145621 } }) + ) + + $rows = Compare-BenchBaseline -Benchmark '06', '08' -ResultsDirectory $script:Results -BaselinePath $script:Baseline + $rows.Count | Should -Be 2 + ($rows | Where-Object { $_.Scenario -eq '08' }).Status | Should -Be 'unmeasured' + } + + It 'reports a scenario that recorded nothing this run' { + $rows = Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results ` + -BaselinePath $script:Baseline -Since (Get-Date).AddDays(1) + $rows.Status | Should -Be 'no result' + } + + It 'says so and returns nothing when there is no baseline' { + $rows = Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results ` + -BaselinePath (Join-Path $TestDrive 'absent.json') + $rows | Should -BeNullOrEmpty + } + + It 'refuses to compare two entries read from different paths' { + # 10 records a two-arm summary through run.ps1 and a single arm when run standalone. + # Seconds against milliseconds would read as a 1000x regression. + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ Arms = [pscustomobject]@{ + paced = [pscustomobject]@{ Seconds = 39.4; ThrottleRetries = 12 } + unpaced = [pscustomobject]@{ Seconds = 40.3 } } }) + ) + Export-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -Path $script:Baseline + + Write-BenchFixture -Directory $script:Results -FileName '10-pacing-under-real-throttling.json' -Entries @( + (New-BenchEntry -Result @{ Mode = 'paced'; WallMs = 39400; ThrottleRetries = 12 }) + ) + + $row = Compare-BenchBaseline -Benchmark '10' -ResultsDirectory $script:Results -BaselinePath $script:Baseline + $row.Status | Should -Be 'not comparable' + $row.DeltaPct | Should -BeNullOrEmpty + } + + It 'refuses a negative baseline, naming the sign' { + # No headline in the table can go negative today, which is what the guard is for: it + # keeps that true of the next one added. Over a negative baseline the percentage delta + # inverts, so -9ms against -4.5ms would be reported as a 100% regression by a figure + # that in fact doubled in the same direction. + $negative = [pscustomobject]@{ + SchemaVersion = 1 + GeneratedAt = (Get-Date).ToString('o') + Scenarios = [pscustomobject]@{ + '06' = [pscustomobject]@{ + Scenario = '06' + Script = '06-fault-gauntlet.ps1' + Headline = 'MgxFanoutMs' + Metrics = [pscustomobject]@{ + MgxFanoutMs = [pscustomobject]@{ Value = -4.5; Unit = 'ms'; Path = 'mgx.ElapsedMs' } + } + } + } + } + ConvertTo-Json -InputObject $negative -Depth 10 | Set-Content -Path $script:Baseline + + Write-BenchFixture -Directory $script:Results -FileName '06-fault-gauntlet.json' -Entries @( + (New-BenchEntry -Result (New-GauntletResult -MgxMs (-9))) + ) + + $row = Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results ` + -BaselinePath $script:Baseline + $row.Status | Should -Be 'not comparable' + $row.Note | Should -BeLike '*negative*' + $row.DeltaPct | Should -BeNullOrEmpty + } + + It 'refuses to compare a paced arm against an unpaced baseline' { + Write-BenchFixture -Directory $script:Results -FileName '07-adaptive-pacing.json' -Entries @( + (New-BenchEntry -Result @{ Mode = 'unpaced'; ElapsedMs = 388108; Output = [pscustomobject]@{ failed = 0 } }) + ) + Export-BenchBaseline -Benchmark '07' -ResultsDirectory $script:Results -Path $script:Baseline + + Write-BenchFixture -Directory $script:Results -FileName '07-adaptive-pacing.json' -Entries @( + (New-BenchEntry -Result @{ Mode = 'paced'; ElapsedMs = 388108; Output = [pscustomobject]@{ failed = 0 } }) + ) + + $row = Compare-BenchBaseline -Benchmark '07' -ResultsDirectory $script:Results -BaselinePath $script:Baseline + $row.Status | Should -Be 'not comparable' + } + + It 'refuses a baseline holding the string "NaN" or "Infinity", naming it as a non-number' { + # ConvertTo-Json quotes a non-finite double as a string - "NaN" and "Infinity" among + # them - so a baseline.json promoted before Export-BenchBaseline refused a non-finite + # measurement still holds one of those tokens today, as a string rather than a typed + # double. Test-BenchNumericValue refuses a "NaN" string the same as it refuses any + # other non-numeric one, so this is caught by the new gate ahead of the cast, not by + # the finiteness check after it - a stale row like this must still be caught somewhere + # or the comparison reports it "compared" on a NaN. + $stale = [pscustomobject]@{ + SchemaVersion = 1 + GeneratedAt = (Get-Date).ToString('o') + Scenarios = [pscustomobject]@{ + '06' = [pscustomobject]@{ + Scenario = '06' + Script = '06-fault-gauntlet.ps1' + Headline = 'MgxFanoutMs' + Metrics = [pscustomobject]@{ + MgxFanoutMs = [pscustomobject]@{ Value = 'NaN'; Unit = 'ms'; Path = 'mgx.ElapsedMs' } + } + } + } + } + ConvertTo-Json -InputObject $stale -Depth 10 | Set-Content -Path $script:Baseline + + $row = Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -BaselinePath $script:Baseline + $row.Status | Should -Be 'not comparable' + $row.Note | Should -Be "not comparable - the baseline value is 'NaN', which is not a number" + $row.DeltaPct | Should -BeNullOrEmpty + + (Get-Content -Path $script:Baseline -Raw).Replace('"NaN"', '"Infinity"') | Set-Content -Path $script:Baseline + + $row2 = Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -BaselinePath $script:Baseline + $row2.Status | Should -Be 'not comparable' + $row2.Note | Should -Be "not comparable - the baseline value is 'Infinity', which is not a number" + $row2.DeltaPct | Should -BeNullOrEmpty + } + + It 'still reports a bare JSON NaN or Infinity baseline token as non-finite, not as a non-number' { + # A typed double is exempted from the not-a-number gate above even when it is NaN or an + # infinity, because the gate itself would otherwise refuse it as a non-number - the same + # refusal a "NaN" string earns - and this arm exists specifically to say something more + # precise than that about a value that did arrive as a genuine double. The only way a + # baseline.json value round-trips through ConvertFrom-Json as a typed double rather than + # a string is a bare, unquoted token, which is not standard JSON but is what a pre-guard + # Export-BenchBaseline could still have written raw. + New-Item -ItemType Directory -Path (Split-Path -Parent $script:Baseline) -Force | Out-Null + $raw = @" +{ + "SchemaVersion": 1, + "GeneratedAt": "$((Get-Date).ToString('o'))", + "Scenarios": { + "06": { + "Scenario": "06", + "Script": "06-fault-gauntlet.ps1", + "Headline": "MgxFanoutMs", + "Metrics": { "MgxFanoutMs": { "Value": NaN, "Unit": "ms", "Path": "mgx.ElapsedMs" } } + } + } +} +"@ + Set-Content -Path $script:Baseline -Value $raw + + $row = Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -BaselinePath $script:Baseline + $row.Status | Should -Be 'not comparable' + $row.Note | Should -Be 'not comparable - the baseline value is NaN' + $row.DeltaPct | Should -BeNullOrEmpty + } + + It 'reports "INCONCLUSIVE", a boolean and an array as not comparable, naming each, with a healthy row still compared' { + # A stale baseline.json can hold anything a prior run's Value field once did - an + # inconclusive string, a boolean the old [double] cast would have silently read as 1, + # or an array or object nothing in the table ever wrote but nothing stopped from being + # written by hand. Three different shapes on three different scenarios, and a fourth, + # healthy scenario in the same call must still compare rather than the whole request + # failing on the first broken row. + Write-BenchFixture -Directory $script:Results -FileName '01-list-users.json' -Entries @( + (New-BenchEntry -Result @{ mgx = [pscustomobject]@{ Median = [pscustomobject]@{ ElapsedMs = 4200; PeakWorkingSetMB = 128 } } }) + ) + Write-BenchFixture -Directory $script:Results -FileName '02-fanout-lookup.json' -Entries @( + (New-BenchEntry -Result @{ mgx = [pscustomobject]@{ Median = [pscustomobject]@{ ElapsedMs = 6100; PeakWorkingSetMB = 96 } } }) + ) + Write-BenchFixture -Directory $script:Results -FileName '03-user-report.json' -Entries @( + (New-BenchEntry -Result @{ mgx = [pscustomobject]@{ Median = [pscustomobject]@{ ElapsedMs = 8300; PeakWorkingSetMB = 64 } } }) + ) + + # '06' rides along from the Describe's own BeforeEach, which already promoted a healthy + # baseline and result for it - carried forward here as the fourth, unbroken row rather + # than lost when the other three are written into the same file. + $healthy06 = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios.'06' + + $stale = [pscustomobject]@{ + SchemaVersion = 1 + GeneratedAt = (Get-Date).ToString('o') + Scenarios = [pscustomobject]@{ + '01' = [pscustomobject]@{ + Scenario = '01'; Script = '01-list-users.ps1'; Headline = 'MgxListMs' + Metrics = [pscustomobject]@{ MgxListMs = [pscustomobject]@{ Value = 'INCONCLUSIVE'; Unit = 'ms'; Path = 'mgx.Median.ElapsedMs' } } + } + '02' = [pscustomobject]@{ + Scenario = '02'; Script = '02-fanout-lookup.ps1'; Headline = 'MgxFanoutMs' + Metrics = [pscustomobject]@{ MgxFanoutMs = [pscustomobject]@{ Value = $true; Unit = 'ms'; Path = 'mgx.Median.ElapsedMs' } } + } + '03' = [pscustomobject]@{ + Scenario = '03'; Script = '03-user-report.ps1'; Headline = 'MgxReportMs' + Metrics = [pscustomobject]@{ MgxReportMs = [pscustomobject]@{ Value = @(1, 2); Unit = 'ms'; Path = 'mgx.Median.ElapsedMs' } } + } + '06' = $healthy06 + } + } + ConvertTo-Json -InputObject $stale -Depth 10 | Set-Content -Path $script:Baseline + + $rows = Compare-BenchBaseline -Benchmark '01', '02', '03', '06' -ResultsDirectory $script:Results -BaselinePath $script:Baseline + + $byId = @{} + foreach ($row in $rows) { $byId[$row.Scenario] = $row } + + $byId['01'].Status | Should -Be 'not comparable' + $byId['01'].Note | Should -Be "not comparable - the baseline value is 'INCONCLUSIVE', which is not a number" + $byId['02'].Status | Should -Be 'not comparable' + $byId['02'].Note | Should -Be "not comparable - the baseline value is 'True', which is not a number" + $byId['03'].Status | Should -Be 'not comparable' + $byId['03'].Note | Should -Be "not comparable - the baseline value is '[1,2]', which is not a number" + + $byId['06'].Status | Should -Be 'compared' + $byId['06'].DeltaPct | Should -Be 0 + } + + It 'refuses a current value that is NaN, naming it' { + # Get-BenchScenarioMetric now refuses every path that could hand back a non-finite + # pinned value - a raw non-number before scaling, an overflow after it - so nothing + # this release ships can make a live run's Current arrive here non-finite. The guard + # below exists for the same reason the baseline one does: a value that arrives + # non-finite by some path not yet closed must be refused rather than compared, not + # silently trusted because today's callers happen not to produce one. Standing in for + # Get-BenchScenarioMetric is the only way to reach that guard, since nothing else can. + Mock -CommandName Get-BenchScenarioMetric -MockWith { + [pscustomobject]@{ + Metrics = [ordered]@{ + MgxFanoutMs = [pscustomobject]@{ Value = [double]::NaN; Unit = 'ms'; Path = 'mgx.ElapsedMs' } + } + Discriminator = $null + } + } + + $row = Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -BaselinePath $script:Baseline + $row.Status | Should -Be 'not comparable' + $row.Note | Should -Be "not comparable - this run's value is NaN" + $row.DeltaPct | Should -BeNullOrEmpty + } + + It 'refuses a baseline with no Scenarios, naming the file, instead of indexing into null' { + # $baseline.Scenarios.PSObject.Properties[...] used to be reached unguarded: a missing + # key and an explicit null both read back as $null, and indexing into that threw + # "Cannot index into a null array" without naming the file or which field was absent. + foreach ($text in @( + '{"SchemaVersion":1,"GeneratedAt":"2026-01-01T00:00:00Z"}', + '{"SchemaVersion":1,"GeneratedAt":"2026-01-01T00:00:00Z","Scenarios":null}' + )) { + Set-Content -Path $script:Baseline -Value $text + + $thrown = $null + try { Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -BaselinePath $script:Baseline } + catch { $thrown = $_ } + + $thrown.Exception.Message | Should -Be "Baseline '$script:Baseline' carries no Scenarios; record one with -RecordBaseline before comparing." + } + } + + It 'refuses a baseline whose SchemaVersion is not the one Export-BenchBaseline writes' { + $scenarios = (Get-Content -Path $script:Baseline -Raw | ConvertFrom-Json).Scenarios + $wrongSchema = [pscustomobject]@{ + SchemaVersion = 2 + GeneratedAt = (Get-Date).ToString('o') + Scenarios = $scenarios + } + ConvertTo-Json -InputObject $wrongSchema -Depth 10 | Set-Content -Path $script:Baseline + + $thrown = $null + try { Compare-BenchBaseline -Benchmark '06' -ResultsDirectory $script:Results -BaselinePath $script:Baseline } + catch { $thrown = $_ } + + $thrown.Exception.Message | Should -Be "Baseline '$script:Baseline' is schema 2; this suite reads schema 1." + } +} diff --git a/tests/Unit/Benchmarks/Mgx.BenchClampSummary.Tests.ps1 b/tests/Unit/Benchmarks/Mgx.BenchClampSummary.Tests.ps1 new file mode 100644 index 0000000..933ddae --- /dev/null +++ b/tests/Unit/Benchmarks/Mgx.BenchClampSummary.Tests.ps1 @@ -0,0 +1,145 @@ +#Requires -Modules Pester + +<# + Get-ClampSummary and the functions it composes (Get-ClampClockGap, Get-ClampBuckets, + Get-ClampOverall, Get-ClampHeadline) in tests/benchmarks/common.ps1 - the summary + 18-spo-latency-clamp.ps1 builds from its checkpointed rows.jsonl once a load finishes. + + No tenant and no live load: every case writes a synthetic rows.jsonl under $TestDrive and + calls Get-ClampSummary on its path directly, the same call the script makes. +#> + +BeforeAll { + $script:BenchRoot = Join-Path (Split-Path -Parent (Split-Path -Parent $PSScriptRoot)) 'benchmarks' + + # common.ps1 pins the thread to invariant culture so benchmark output never prints 39,2s. + # Right for a benchmark process, and not something to leave behind in a test run. + $script:CultureBefore = [System.Threading.Thread]::CurrentThread.CurrentCulture + . (Join-Path $script:BenchRoot 'common.ps1') + + # One row, in the shape 18 writes to its rows.jsonl. ElapsedMs and ErrorText are the two + # fields each case varies; the rest are fixed because nothing here reads them. + function script:Write-ClampRow { + param( + [Parameter(Mandatory)] [string] $Path, + [Parameter(Mandatory)] [datetime] $Timestamp, + [Parameter(Mandatory)] [int] $Status, + [object] $ElapsedMs = $null, + [string] $ErrorText = $null, + [int] $Worker = 1 + ) + $row = [pscustomobject]@{ + Timestamp = $Timestamp.ToString('o'); Worker = $Worker; Label = 'item'; Uri = '/drives/x/items/1' + Attempt = 1; Attempts = 1; Status = $Status; ElapsedMs = $ElapsedMs + CallElapsedMs = 100; PacingWaitMs = $null + ResourceUnit = $null; ThrottleLimitPercentage = $null; ThrottleScope = $null + RetryAfter = $null; RateLimitHeader = $false + Error = $ErrorText + } + Add-Content -Path $Path -Value (ConvertTo-Json -InputObject $row -Compress -Depth 4) + } +} + +AfterAll { + [System.Threading.Thread]::CurrentThread.CurrentCulture = $script:CultureBefore +} + +Describe 'Get-ClampSummary' { + BeforeEach { + $script:RowsPath = Join-Path $TestDrive "$([guid]::NewGuid()).jsonl" + $script:Started = [datetime]'2026-01-01T00:00:00Z' + } + + Context 'the clock gap' { + It 'refuses and names the gap when the wall clock outran the stopwatch by more than a bucket' { + # 20 minutes wall clock, a 5-minute Stopwatch - the 5757093 run's own shape, a + # machine sleep the Stopwatch does not advance through. + Write-ClampRow -Path $script:RowsPath -Timestamp $script:Started.AddMinutes(1) -Status 200 -ElapsedMs 100 + $ended = $script:Started.AddMinutes(20) + + $summary = Get-ClampSummary -RowsPath $script:RowsPath -StartedAt $script:Started -EndedAt $ended ` + -WallMs 300000 -Minutes 20 -BucketMinutes 5 + + $summary.Refused | Should -BeTrue + $summary.GapMinutes | Should -Be 15 + $summary.RefusalReason | Should -Be 'the machine was not awake for 15 minute(s) of the window; rows kept, summary withheld' + $summary.Buckets | Should -BeNullOrEmpty + $summary.Overall | Should -BeNullOrEmpty + } + + It 'does not refuse when the wall clock and the stopwatch agree within a bucket' { + Write-ClampRow -Path $script:RowsPath -Timestamp $script:Started.AddMinutes(1) -Status 200 -ElapsedMs 100 + $ended = $script:Started.AddMinutes(2) + + # A 1-second wall/Stopwatch drift - ordinary scheduling noise, not a sleep. + $summary = Get-ClampSummary -RowsPath $script:RowsPath -StartedAt $script:Started -EndedAt $ended ` + -WallMs 119000 -Minutes 2 -BucketMinutes 5 + + $summary.Refused | Should -BeFalse + $summary.Overall | Should -Not -BeNullOrEmpty + } + } + + Context 'buckets' { + It 'marks a bucket with no rows rather than omitting it, filling every index in the window' { + # Rows land in minute 1 (bucket 0) and minute 12 (bucket 2); bucket 1 (minutes + # 5-10) gets none, the way buckets 2 and 3 got none across the 5757093 run's sleep. + Write-ClampRow -Path $script:RowsPath -Timestamp $script:Started.AddMinutes(1) -Status 200 -ElapsedMs 100 + Write-ClampRow -Path $script:RowsPath -Timestamp $script:Started.AddMinutes(12) -Status 200 -ElapsedMs 100 + $ended = $script:Started.AddMinutes(15) + + $summary = Get-ClampSummary -RowsPath $script:RowsPath -StartedAt $script:Started -EndedAt $ended ` + -WallMs ([long]($ended - $script:Started).TotalMilliseconds) -Minutes 15 -BucketMinutes 5 + + $summary.Refused | Should -BeFalse + $summary.Buckets.Count | Should -Be 3 + $summary.Buckets[1].Bucket | Should -Be 1 + $summary.Buckets[1].Attempts | Should -Be 0 + $summary.Buckets[1].Note | Should -Be 'no rows' + $summary.Buckets[0].Note | Should -BeNullOrEmpty + $summary.Buckets[2].Note | Should -BeNullOrEmpty + } + } + + Context 'errors' { + It 'counts a Status-0 refusal in Errors and groups it by the text before its first colon' { + Write-ClampRow -Path $script:RowsPath -Timestamp $script:Started.AddMinutes(1) -Status 200 -ElapsedMs 100 + 1..3 | ForEach-Object { + Write-ClampRow -Path $script:RowsPath -Timestamp $script:Started.AddMinutes(1) -Status 0 ` + -ErrorText 'Circuit breaker tripped: too many failures caused Mgx to temporarily stop requests. Wait 15s.' + } + Write-ClampRow -Path $script:RowsPath -Timestamp $script:Started.AddMinutes(1) -Status 0 -ErrorText 'no response traced' + $ended = $script:Started.AddMinutes(2) + + $summary = Get-ClampSummary -RowsPath $script:RowsPath -StartedAt $script:Started -EndedAt $ended ` + -WallMs ([long]($ended - $script:Started).TotalMilliseconds) -Minutes 2 -BucketMinutes 5 + + $summary.Overall.Errors | Should -Be 4 + $summary.Buckets[0].Errors | Should -Be 4 + $summary.Overall.Refusals['Circuit breaker tripped'] | Should -Be 3 + $summary.Overall.Refusals['no response traced'] | Should -Be 1 + } + } + + Context 'the headline' { + It 'takes the headline from the bucket distribution, not a single late sample' { + # Bucket 0: nine calls at 200ms, then one late straggler at 1864ms - the shape of + # the 5757093 run's own last row, which PacingState quoted as though it were the + # headline. Nearest-rank p90 of ten values with one high outlier is still 200. + 1..9 | ForEach-Object { Write-ClampRow -Path $script:RowsPath -Timestamp $script:Started.AddMinutes(1) -Status 200 -ElapsedMs 200 } + Write-ClampRow -Path $script:RowsPath -Timestamp $script:Started.AddMinutes(1) -Status 200 -ElapsedMs 1864 + # Bucket 1: nine calls at 900ms - the worst bucket, and the one the headline names. + 1..9 | ForEach-Object { Write-ClampRow -Path $script:RowsPath -Timestamp $script:Started.AddMinutes(6) -Status 200 -ElapsedMs 900 } + $ended = $script:Started.AddMinutes(10) + + $summary = Get-ClampSummary -RowsPath $script:RowsPath -StartedAt $script:Started -EndedAt $ended ` + -WallMs ([long]($ended - $script:Started).TotalMilliseconds) -Minutes 10 -BucketMinutes 5 + + $summary.Buckets[0].P90ElapsedMs | Should -Be 200 + $summary.Headline.WorstBucketFromMinute | Should -Be 5 + $summary.Headline.WorstBucketP90ElapsedMs | Should -Be 900 + $summary.Headline.Sentence | Should -BeLike '*900*' + $summary.Headline.Sentence | Should -Not -BeLike '*1864*' + } + } +} diff --git a/tests/Unit/Benchmarks/Mgx.BenchLoadShape.Tests.ps1 b/tests/Unit/Benchmarks/Mgx.BenchLoadShape.Tests.ps1 new file mode 100644 index 0000000..18b7f8d --- /dev/null +++ b/tests/Unit/Benchmarks/Mgx.BenchLoadShape.Tests.ps1 @@ -0,0 +1,175 @@ +#Requires -Modules Pester + +<# + The load shape benchmark 10 issues, and whether it can reach the throttled regime, in + tests/benchmarks/common.ps1. + + The thresholds are passed in rather than read from results/, so these run the same + arithmetic on every machine whatever any local calibration says. The numbers used below + are the ones benchmark 13 measured against the seeded tenant: 22,954 RU of burst allowance + and 731 RU/s served while refusing. +#> + +BeforeAll { + $script:BenchRoot = Join-Path (Split-Path -Parent (Split-Path -Parent $PSScriptRoot)) 'benchmarks' + + $script:CultureBefore = [System.Threading.Thread]::CurrentThread.CurrentCulture + . (Join-Path $script:BenchRoot 'common.ps1') + + $script:Seeded = @{ BurstRu = 22954; SustainedRuPerSecond = 731 } + + function Get-Shape + { + param + ( + [int] $Count = 60000, + [int] $Pipelines = 12, + [int] $Concurrency = 128, + [int] $IdPool = 5000 + ) + + return Get-BenchPacingLoadShape -Count $Count -Pipelines $Pipelines ` + -Concurrency $Concurrency -IdPool $IdPool @script:Seeded + } + + # The defaults in the script's own param block. A default that stops being admissible is the + # failure this benchmark shipped once already, so the test reads them out of the file. + $script:ScriptDefaults = @{} + $text = Get-Content -Path (Join-Path $script:BenchRoot '10-pacing-under-real-throttling.ps1') -Raw + foreach ($name in 'Count', 'Pipelines', 'Concurrency', 'IdPool') + { + $match = [regex]::Match($text, "\[int\]\s*\`$$name\s*=\s*(\d+)") + if ($match.Success) { $script:ScriptDefaults[$name] = [int]$match.Groups[1].Value } + } +} + +AfterAll { + [System.Threading.Thread]::CurrentThread.CurrentCulture = $script:CultureBefore +} + +Describe 'Get-BenchPacingLoadShape' { + It 'issues distinct ids only, so the request count is Pipelines x IdsPerPipeline' { + # Invoke-MgxRequest deduplicates the ids it is piped: a slice that repeats one issues a + # single request for it, and the arm reports a volume it never sent. + $shape = Get-Shape -Count 60000 -Pipelines 12 -IdPool 5000 + $shape.IdsPerPipeline | Should -Be 5000 + $shape.TotalRequests | Should -Be 60000 + $shape.TotalRequests | Should -Be ($shape.IdsPerPipeline * $shape.Pipelines) + $shape.Clamped | Should -BeFalse + } + + It 'clamps a -Count the pool cannot cover and says it clamped' { + # 120,000 over 8 pipelines of a 5,000 id pool is 40,000 requests, whatever was asked for. + $shape = Get-Shape -Count 120000 -Pipelines 8 -IdPool 5000 + $shape.Requested | Should -Be 120000 + $shape.IdsPerPipeline | Should -Be 5000 + $shape.TotalRequests | Should -Be 40000 + $shape.Clamped | Should -BeTrue + } + + It 'divides a -Count under the ceiling evenly and does not clamp' { + $shape = Get-Shape -Count 24000 -Pipelines 12 -IdPool 5000 + $shape.IdsPerPipeline | Should -Be 2000 + $shape.TotalRequests | Should -Be 24000 + $shape.Clamped | Should -BeFalse + } + + It 'scales the projected draw with the pipelines and the concurrency' { + (Get-Shape -Pipelines 12 -Concurrency 128).ExpectedRuPerSecond | Should -Be 2160 + (Get-Shape -Pipelines 8 -Concurrency 128).ExpectedRuPerSecond | Should -Be 1440 + (Get-Shape -Pipelines 12 -Concurrency 64).ExpectedRuPerSecond | Should -Be 1080 + } + + It 'refuses a set whose rate cannot drain the allowance, whatever its volume' { + # The point the first version of the benchmark missed: no -Count exhausts a burst + # allowance that refills faster than the load draws it down. + $shape = Get-Shape -Count 500000 -Pipelines 2 -IdPool 250000 + $shape.TotalRequests | Should -Be 500000 + $shape.ExpectedRuPerSecond | Should -Be 360 + $shape.RateClears | Should -BeFalse + $shape.Admissible | Should -BeFalse + $shape.Reasons -join ' ' | Should -Match 'volume cannot buy a rate' + } + + It 'refuses a set that never spends past the allowance' { + $shape = Get-Shape -Count 12000 -Pipelines 12 -IdPool 5000 + $shape.RateClears | Should -BeTrue + $shape.VolumeClears | Should -BeFalse + $shape.Admissible | Should -BeFalse + $shape.ThrottledSeconds | Should -Be 0 + } + + It 'admits a set that clears both, and times the two phases' { + $shape = Get-Shape -Count 60000 -Pipelines 12 -IdPool 5000 + $shape.RateClears | Should -BeTrue + $shape.VolumeClears | Should -BeTrue + $shape.Admissible | Should -BeTrue + $shape.Reasons | Should -BeNullOrEmpty + + # 22,954 RU at 2,160 RU/s, then the remaining 37,046 at the 731 RU/s served while refusing. + $shape.BurstSeconds | Should -Be 10.6 + $shape.ThrottledSeconds | Should -Be 50.7 + $shape.EstimatedSeconds | Should -Be 61.3 + } +} + +Describe 'Get-BenchPipelineSlice' { + BeforeAll { + $script:Pool = 1..5000 | ForEach-Object { "id-$_" } + } + + It 'gives every pipeline its own count of ids, none of them repeated' { + # The invariant the arm's request count rests on. A repeated id inside one pipeline is + # deduplicated away by Invoke-MgxRequest and never sent. + $slices = Get-BenchPipelineSlice -Pool $script:Pool -Pipelines 12 -IdsPerPipeline 5000 + + $slices.Count | Should -Be 12 + foreach ($slice in $slices) + { + $slice.Count | Should -Be 5000 + @($slice | Sort-Object -Unique).Count | Should -Be 5000 + } + } + + It 'staggers where the pipelines start so they are not all on one object' { + $slices = Get-BenchPipelineSlice -Pool $script:Pool -Pipelines 12 -IdsPerPipeline 100 + + @($slices | ForEach-Object { $_[0] } | Sort-Object -Unique).Count | Should -Be 12 + } + + It 'holds the invariant for a short slice and for a pool smaller than the pipeline count' { + foreach ($case in @( + @{ Pipelines = 12; Ids = 2000 }, + @{ Pipelines = 3; Ids = 5000 }, + @{ Pipelines = 1; Ids = 1 })) + { + $slices = Get-BenchPipelineSlice -Pool $script:Pool -Pipelines $case.Pipelines -IdsPerPipeline $case.Ids + foreach ($slice in $slices) { @($slice | Sort-Object -Unique).Count | Should -Be $case.Ids } + } + + $tiny = Get-BenchPipelineSlice -Pool @('a', 'b', 'c') -Pipelines 8 -IdsPerPipeline 3 + $tiny.Count | Should -Be 8 + foreach ($slice in $tiny) { @($slice | Sort-Object -Unique).Count | Should -Be 3 } + } + + It 'refuses to hand out more distinct ids than the pool holds' { + { Get-BenchPipelineSlice -Pool @('a', 'b') -Pipelines 2 -IdsPerPipeline 3 } | + Should -Throw -ExpectedMessage '*deduplicates*' + } +} + +Describe 'benchmark 10 defaults' { + It 'reads four defaults out of the script' { + $script:ScriptDefaults.Keys.Count | Should -Be 4 + } + + It 'ships a default run that is admissible on the calibrated tenant' { + $shape = Get-BenchPacingLoadShape -Count $script:ScriptDefaults.Count ` + -Pipelines $script:ScriptDefaults.Pipelines ` + -Concurrency $script:ScriptDefaults.Concurrency ` + -IdPool $script:ScriptDefaults.IdPool @script:Seeded + + $shape.Clamped | Should -BeFalse -Because 'a default that asks for more than it can issue reports a volume it never sent' + $shape.Admissible | Should -BeTrue -Because 'run.ps1 passes no arguments, so the defaults are what gets measured' + } +} diff --git a/tests/Unit/Benchmarks/Mgx.BenchResult.Tests.ps1 b/tests/Unit/Benchmarks/Mgx.BenchResult.Tests.ps1 new file mode 100644 index 0000000..69c2940 --- /dev/null +++ b/tests/Unit/Benchmarks/Mgx.BenchResult.Tests.ps1 @@ -0,0 +1,373 @@ +#Requires -Modules Pester + +<# + What a recorded benchmark result says about the run that produced it, in + tests/benchmarks/common.ps1: the documented budget tiers, and the tenant and tree identity + Write-BenchResult stamps on every entry. + + No tenant and no Graph session: Get-MgContext, the count call and the telemetry snapshot + are redefined in the scope common.ps1 is dot-sourced into, which is where Write-BenchResult + resolves them from, and every entry is written under $TestDrive. +#> + +BeforeAll { + $script:BenchRoot = Join-Path (Split-Path -Parent (Split-Path -Parent $PSScriptRoot)) 'benchmarks' + + # common.ps1 pins the thread to invariant culture so benchmark output never prints 39,2s. + # Right for a benchmark process, and not something to leave behind in a test run. + $script:CultureBefore = [System.Threading.Thread]::CurrentThread.CurrentCulture + . (Join-Path $script:BenchRoot 'common.ps1') + + # The faked tenant and the faked counters, each with CmdletBinding because the callers + # below pass -ErrorAction. Defined after the dot-source and in the same scope, so these + # are the definitions the real code finds. + $script:FakeTenantId = '00000000-1111-2222-3333-444444444444' + $script:FakeUserCount = 19 + + function Get-MgContext + { + [CmdletBinding()] + param () + + if (-not $script:FakeSession) { return $null } + return [pscustomobject]@{ TenantId = $script:FakeTenantId; ClientId = 'probe'; AuthType = 'AppOnly' } + } + + function Get-BenchDirectoryObjectCount + { + [CmdletBinding()] + param () + + $script:CountCalls++ + return [long]$script:FakeUserCount + } + + # 1,750 units over a 5,000 ms wall clock is 350 RU/s exactly, so the recorded fraction + # names its own denominator: 1.0 against the S tier's 350, 0.438 against the 800 of a + # tenant whose tier is not known. + function Get-MgxTelemetry + { + [CmdletBinding()] + param () + + return [pscustomobject]@{ + ResourceUnitsConsumed = 1750 + Requests = 1750 + Succeeded = 1750 + Failed = 0 + ThrottleRetries = 0 + OtherRetries = 0 + AdaptivePacingWaitMs = 0 + AdaptivePacingActivations = 0 + RateLimiterWaitMs = 0 + LastThrottlePercentage = -1.0 + PacingState = 'directory: latency 100ms (1.0x of 100ms baseline)' + } + } + + function Write-BenchProbe + { + param + ( + [Parameter(Mandatory)] [string] $Name, + [switch] $Session, + [long] $UserCount = 19, + [long] $WallMs = 5000, + [string] $TenantId = '00000000-1111-2222-3333-444444444444', + [object] $Identity + ) + + $script:FakeSession = [bool]$Session + $script:FakeUserCount = $UserCount + $script:FakeTenantId = $TenantId + $script:CountCalls = 0 + Get-BenchTenantFact -Refresh | Out-Null + + # Passed on only when the caller passed one, so the probe covers both the supplied and + # the unsupplied path rather than turning $null into an identity of five nulls. + $supplied = @{} + if ($PSBoundParameters.ContainsKey('Identity')) { $supplied['Identity'] = $Identity } + + Write-BenchResult -Benchmark $Name -ResultsDirectory $TestDrive -WallMs $WallMs ` + -Result ([pscustomobject]@{ Probe = $Name }) @supplied | Out-Null + + return @(Get-Content -Path (Join-Path $TestDrive "$Name.json") -Raw | ConvertFrom-Json)[-1] + } +} + +AfterAll { + [System.Threading.Thread]::CurrentThread.CurrentCulture = $script:CultureBefore +} + +Describe 'Get-BenchDocumentedBudget' { + It 'puts users in tier at RU/s' -ForEach @( + @{ UserCount = 0; Tier = 'S'; RuPerSecond = 350 } + @{ UserCount = 19; Tier = 'S'; RuPerSecond = 350 } + @{ UserCount = 49; Tier = 'S'; RuPerSecond = 350 } + @{ UserCount = 50; Tier = 'M'; RuPerSecond = 500 } + @{ UserCount = 500; Tier = 'M'; RuPerSecond = 500 } + @{ UserCount = 501; Tier = 'L'; RuPerSecond = 800 } + @{ UserCount = 130233; Tier = 'L'; RuPerSecond = 800 } + ) { + $budget = Get-BenchDocumentedBudget -UserCount $UserCount + $budget.Tier | Should -Be $Tier + $budget.RuPerSecond | Should -Be $RuPerSecond + $budget.RuPer10Seconds | Should -Be ($RuPerSecond * 10) + $budget.UserCount | Should -Be $UserCount + $budget.Band | Should -Not -BeNullOrEmpty + } + + It 'answers null for a count it does not have' { + # An assumed tier is worse than none: it would put a denominator nobody measured into + # the ratio and label it documented. + Get-BenchDocumentedBudget -UserCount $null | Should -BeNullOrEmpty + Get-BenchDocumentedBudget -UserCount -1 | Should -BeNullOrEmpty + } +} + +Describe 'Write-BenchResult metadata' { + It 'records the tenant, its size and its documented budget' { + $entry = Write-BenchProbe -Name 'tenant' -Session -UserCount 19 + + $entry.TenantId | Should -Be $script:FakeTenantId + $entry.DirectoryObjectCount | Should -Be 19 + $entry.DocumentedBudgetRuPerSecond | Should -Be 350 + } + + It 'treats a session whose tenant id is not a GUID as no tenant' { + # 06 connects a mock Graph and names the tenant 'gauntlet-mock-tenant'. There is no + # directory behind that to count, and the benchmark has to stay off the network. + $entry = Write-BenchProbe -Name 'mocktenant' -Session -TenantId 'gauntlet-mock-tenant' + + $entry.TenantId | Should -BeNullOrEmpty + $entry.DirectoryObjectCount | Should -BeNullOrEmpty + $script:CountCalls | Should -Be 0 -Because 'a mock session must not be counted against a real tenant' + } + + It 'records those three as null rather than failing with no session' { + # 06 and the mock-server benchmarks never connect. They still have to record. + $entry = Write-BenchProbe -Name 'notenant' + + $entry.PSObject.Properties.Name | Should -Contain 'TenantId' + $entry.TenantId | Should -BeNullOrEmpty + $entry.DirectoryObjectCount | Should -BeNullOrEmpty + $entry.DocumentedBudgetRuPerSecond | Should -BeNullOrEmpty + $entry.Result.Probe | Should -Be 'notenant' + } + + It 'records the commit of the tree that ran, with a dirty marker' { + $entry = Write-BenchProbe -Name 'commit' + + # Null outside a checkout, which CI is not - so assert the shape wherever there is one. + if ($entry.Commit) + { + $entry.Commit | Should -Match '^[0-9a-f]{7,40}(-dirty)?$' + } + } + + It 'records the wall clock as metadata, not only inside the telemetry block' { + $entry = Write-BenchProbe -Name 'wall' -WallMs 5000 + $entry.WallMs | Should -Be 5000 + + $noWall = Write-BenchProbe -Name 'nowall' -WallMs 0 + $noWall.WallMs | Should -BeNullOrEmpty + } + + It 'measures the draw against the documented budget and says so' { + # 350 RU/s against the S tier's 350 RU/s is the whole budget. + $entry = Write-BenchProbe -Name 'documented' -Session -UserCount 19 + + $entry.Telemetry.RuPerSecond | Should -Be 350 + $entry.Telemetry.BudgetFraction | Should -Be 1 + $entry.Telemetry.BudgetBasis | Should -Be 'documented' + } + + It 'measures the same draw against a larger tenant as a smaller fraction' { + $entry = Write-BenchProbe -Name 'documented-large' -Session -UserCount 130233 + + $entry.Telemetry.BudgetFraction | Should -Be 0.438 + $entry.Telemetry.BudgetBasis | Should -Be 'documented' + } + + It 'falls back to 800 RU/s with no documented tier and labels the fallback' { + $entry = Write-BenchProbe -Name 'assumed' + + $entry.Telemetry.BudgetFraction | Should -Be 0.438 + $entry.Telemetry.BudgetBasis | Should -Be 'assumed-800' + } +} + +Describe 'Write-BenchResult -Identity' { + It 'stamps the entry with the identity it was handed, not this process''s' { + # The case this exists for: a comparison whose arms ran as child processes writes its + # combined entry from a parent that imported no local module and connected to nothing. + # A session is faked here so the process HAS a tenant of its own to leak - the entry + # must carry the arms' tenant, not that one. + $entry = Write-BenchProbe -Name 'identity' -Session -UserCount 19 -Identity ([pscustomobject]@{ + MgxVersion = '2.1.4' + SdkVersion = '2.34.0' + TenantId = 'e5011376-aaaa-bbbb-cccc-dddddddddddd' + DirectoryObjectCount = 100000 + DocumentedBudgetRuPerSecond = 800 + }) + + $entry.MgxVersion | Should -Be '2.1.4' + $entry.SdkVersion | Should -Be '2.34.0' + $entry.TenantId | Should -Be 'e5011376-aaaa-bbbb-cccc-dddddddddddd' + $entry.TenantId | Should -Not -Be $script:FakeTenantId + $entry.DirectoryObjectCount | Should -Be 100000 + $entry.DocumentedBudgetRuPerSecond | Should -Be 800 + } + + It 'takes a hashtable as readily as an object' { + $entry = Write-BenchProbe -Name 'identity-hash' -Identity @{ + MgxVersion = '2.1.4'; SdkVersion = '2.34.0'; TenantId = 'e5011376' + DirectoryObjectCount = 100000; DocumentedBudgetRuPerSecond = 800 + } + + $entry.MgxVersion | Should -Be '2.1.4' + $entry.DirectoryObjectCount | Should -Be 100000 + } + + It 'records a field the identity does not carry as null rather than falling back' { + # Get-BenchEntryIdentity answers null for a field the arms disagreed on. Substituting + # this process's value there would put back exactly what the parameter exists to keep + # out, and label a disagreement as a measurement. + $entry = Write-BenchProbe -Name 'identity-partial' -Session -UserCount 19 ` + -Identity ([pscustomobject]@{ MgxVersion = '2.1.4'; TenantId = $null }) + + $entry.MgxVersion | Should -Be '2.1.4' + $entry.TenantId | Should -BeNullOrEmpty + $entry.SdkVersion | Should -BeNullOrEmpty + $entry.DirectoryObjectCount | Should -BeNullOrEmpty + $entry.DocumentedBudgetRuPerSecond | Should -BeNullOrEmpty + } + + It 'leaves the run''s own fields to the process that wrote them' { + $entry = Write-BenchProbe -Name 'identity-run' -WallMs 5000 -Identity ([pscustomobject]@{ + MgxVersion = '2.1.4'; PSVersion = '1.0'; WallMs = 42 + }) + + $entry.PSVersion | Should -Be $PSVersionTable.PSVersion.ToString() + $entry.WallMs | Should -Be 5000 + $entry.RecordedAt | Should -Not -BeNullOrEmpty + } + + It 'reads this process with no identity supplied' { + $entry = Write-BenchProbe -Name 'identity-absent' -Session -UserCount 19 + + $entry.MgxVersion | Should -Be (Get-Module M365DSC.mgx -ErrorAction SilentlyContinue)?.Version?.ToString() + $entry.TenantId | Should -Be $script:FakeTenantId + $entry.DirectoryObjectCount | Should -Be 19 + $entry.DocumentedBudgetRuPerSecond | Should -Be 350 + } +} + +Describe 'Get-BenchEntryIdentity' { + BeforeAll { + function script:New-ArmEntry + { + param + ( + [string] $MgxVersion = '2.1.4', + [string] $SdkVersion = '2.34.0', + [object] $TenantId = 'e5011376-aaaa-bbbb-cccc-dddddddddddd', + [object] $DirectoryObjectCount = 100000, + [object] $DocumentedBudgetRuPerSecond = 800 + ) + + return [pscustomobject]@{ + MgxVersion = $MgxVersion + SdkVersion = $SdkVersion + TenantId = $TenantId + DirectoryObjectCount = $DirectoryObjectCount + DocumentedBudgetRuPerSecond = $DocumentedBudgetRuPerSecond + } + } + } + + It 'answers the five fields the arms agree on' { + $identity = Get-BenchEntryIdentity -Entries @((New-ArmEntry), (New-ArmEntry)) + + $identity.MgxVersion | Should -Be '2.1.4' + $identity.SdkVersion | Should -Be '2.34.0' + $identity.TenantId | Should -Be 'e5011376-aaaa-bbbb-cccc-dddddddddddd' + $identity.DirectoryObjectCount | Should -Be 100000 + $identity.DocumentedBudgetRuPerSecond | Should -Be 800 + } + + It 'counts a long and an int holding the same number as agreement' { + # The arms record through the same writer, but one of them reaching the count as a long + # and the other as an int is not a disagreement about which tenant ran. + $identity = Get-BenchEntryIdentity -Entries @( + (New-ArmEntry -DirectoryObjectCount ([long]100000)), + (New-ArmEntry -DirectoryObjectCount ([int]100000)) + ) + + $identity.DirectoryObjectCount | Should -Be 100000 + } + + It 'answers null for a field the arms disagree on, naming it and both values' { + $warnings = $null + $identity = Get-BenchEntryIdentity -WarningVariable warnings 3>$null -Entries @( + (New-ArmEntry -MgxVersion '2.1.4'), + (New-ArmEntry -MgxVersion '2.1.1') + ) + + $identity.MgxVersion | Should -BeNullOrEmpty + "$warnings" | Should -BeLike '*disagree on MgxVersion (2.1.4, 2.1.1)*' + $identity.TenantId | Should -Be 'e5011376-aaaa-bbbb-cccc-dddddddddddd' -Because 'a field they agree on is still answered' + } + + It 'names a field one arm recorded and the other did not as a disagreement' { + # An arm with no tenant and an arm with one did not run against the same thing, and + # taking the one that has a value would say they did. + $warnings = $null + $identity = Get-BenchEntryIdentity -WarningVariable warnings 3>$null -Entries @( + (New-ArmEntry), + (New-ArmEntry -TenantId $null) + ) + + $identity.TenantId | Should -BeNullOrEmpty + "$warnings" | Should -BeLike '*disagree on TenantId (e5011376-aaaa-bbbb-cccc-dddddddddddd, (none))*' + } + + It 'answers null without warning when neither arm recorded a field' { + $warnings = $null + $identity = Get-BenchEntryIdentity -WarningVariable warnings 3>$null -Entries @( + (New-ArmEntry -TenantId $null -DocumentedBudgetRuPerSecond $null), + (New-ArmEntry -TenantId $null -DocumentedBudgetRuPerSecond $null) + ) + + $identity.TenantId | Should -BeNullOrEmpty + $identity.DocumentedBudgetRuPerSecond | Should -BeNullOrEmpty + "$warnings" | Should -BeNullOrEmpty + } + + It 'reads a single arm''s entry as that arm''s identity' { + $identity = Get-BenchEntryIdentity -Entries @((New-ArmEntry -MgxVersion '2.1.5')) + + $identity.MgxVersion | Should -Be '2.1.5' + } +} + +Describe 'Export-BenchBaseline over a recorded entry' { + It 'still reads the version block off an entry carrying the new fields' { + # The promotion reads MgxVersion/SdkVersion/PSVersion/RecordedAt by name. The tenant + # fields extend that block; they must not have moved what was already in it. + $results = Join-Path $TestDrive 'promote' + Write-BenchResult -Benchmark '06-fault-gauntlet' -ResultsDirectory $results -Result ([pscustomobject]@{ + mgx = [pscustomobject]@{ ElapsedMs = 134654; Output = [pscustomobject]@{ ok = 1000; failed = 0 } } + sdkMgx = [pscustomobject]@{ ElapsedMs = 246715 } + }) | Out-Null + + $baseline = Join-Path $TestDrive 'promoted-baseline.json' + Export-BenchBaseline -Benchmark '06' -ResultsDirectory $results -Path $baseline + + $pinned = (Get-Content -Path $baseline -Raw | ConvertFrom-Json).Scenarios.'06' + $pinned.Headline | Should -Be 'MgxFanoutMs' + $pinned.Metrics.MgxFanoutMs.Value | Should -Be 134654 + $pinned.PSVersion | Should -Be $PSVersionTable.PSVersion.ToString() + $pinned.RecordedAt | Should -Not -BeNullOrEmpty + } +} diff --git a/tests/Unit/Benchmarks/Mgx.BenchWatchdog.Tests.ps1 b/tests/Unit/Benchmarks/Mgx.BenchWatchdog.Tests.ps1 new file mode 100644 index 0000000..ee2b6f8 --- /dev/null +++ b/tests/Unit/Benchmarks/Mgx.BenchWatchdog.Tests.ps1 @@ -0,0 +1,65 @@ +#Requires -Modules Pester + +<# + Invoke-WatchdoggedContender in tests/benchmarks/common.ps1. + + Each case spawns a real child pwsh process, so these run at the watchdog's own 10-second + poll granularity rather than instantly - the same cost 16 and 17 already pay to use it. +#> + +BeforeAll { + $script:BenchRoot = Join-Path (Split-Path -Parent (Split-Path -Parent $PSScriptRoot)) 'benchmarks' + + # common.ps1 pins the thread to invariant culture; not something to leave behind in a + # test run. + $script:CultureBefore = [System.Threading.Thread]::CurrentThread.CurrentCulture + . (Join-Path $script:BenchRoot 'common.ps1') +} + +AfterAll { + [System.Threading.Thread]::CurrentThread.CurrentCulture = $script:CultureBefore +} + +Describe 'Invoke-WatchdoggedContender' { + BeforeEach { + $script:ResultFile = Join-Path $TestDrive 'result.json' + $script:HeartbeatFile = Join-Path $TestDrive 'heartbeat' + Remove-Item -Path $script:ResultFile, $script:HeartbeatFile -ErrorAction SilentlyContinue + } + + It 'records a Crashed row naming the exit code when the child dies before its first heartbeat' { + # An import or connect failure inside the child's first poll: it exits before ever + # touching $HeartbeatFile, which is what used to fall back to $p.StartTime and read + # back null once the child had already exited - and used to come back Hung, which is + # not what happened: nothing here waited for progress that never came, the child quit + # on its own within the first poll. + $child = Join-Path $TestDrive 'dies-early.ps1' + Set-Content -Path $child -Value 'exit 3' + + $row = Invoke-WatchdoggedContender -Name 'dies-early' -ScriptPath $child ` + -ResultFile $script:ResultFile -HeartbeatFile $script:HeartbeatFile -StallSeconds 60 + + $row.Hung | Should -BeFalse + $row.Crashed | Should -BeTrue + $row.ExitCode | Should -Be 3 + $row.Output.note | Should -BeLike '*exited 3*' + } + + It 'reports Hung = false and Crashed = false for a child that heartbeats and then completes' { + $child = Join-Path $TestDrive 'completes.ps1' + Set-Content -Path $child -Value @' +param([string] $HeartbeatFile, [string] $ResultFile) +Set-Content -Path $HeartbeatFile -Value ([datetime]::UtcNow.Ticks) +[pscustomobject]@{ Name = 'completes'; ElapsedMs = 1 } | ConvertTo-Json | Set-Content -Path $ResultFile +exit 0 +'@ + + $row = Invoke-WatchdoggedContender -Name 'completes' -ScriptPath $child ` + -ArgumentList @('-HeartbeatFile', $script:HeartbeatFile, '-ResultFile', $script:ResultFile) ` + -ResultFile $script:ResultFile -HeartbeatFile $script:HeartbeatFile -StallSeconds 60 + + $row.Hung | Should -BeFalse + $row.Crashed | Should -BeFalse + $row.Name | Should -Be 'completes' + } +} diff --git a/tests/Unit/Benchmarks/Mgx.MockPlan.Tests.ps1 b/tests/Unit/Benchmarks/Mgx.MockPlan.Tests.ps1 new file mode 100644 index 0000000..15101a9 --- /dev/null +++ b/tests/Unit/Benchmarks/Mgx.MockPlan.Tests.ps1 @@ -0,0 +1,131 @@ +#Requires -Modules Pester + +<# + Set-Plan's window validation, served counting under a plan fault on the entity route, + and slowBody's chunks: 1 timing, all in tests/benchmarks/mock-graph-server.ps1. + + A real child process on a real socket, the same way 06 and the pathological gauntlets use + it: the plan screen, the disposition a fault hands back, and a trickled body are exactly + the things no in-process fake stands in for. +#> + +BeforeAll { + $script:BenchRoot = Join-Path (Split-Path -Parent (Split-Path -Parent $PSScriptRoot)) 'benchmarks' + + # common.ps1 pins the thread to invariant culture; not something to leave behind in a + # test run. + $script:CultureBefore = [System.Threading.Thread]::CurrentThread.CurrentCulture + . (Join-Path $script:BenchRoot 'common.ps1') + + $script:Server = Start-BenchMockServer -Port 9970 + $script:Base = $script:Server.BaseUrl + + # Posts a plan without Invoke-RestMethod's default throw-on-4xx, so a refusal is a value + # to assert on rather than an exception to catch. Read inside this function's own scope + # rather than relied on to cross into the caller's, so the answer does not depend on + # exactly how -StatusCodeVariable propagates. Defined here, in BeforeAll, rather than at + # the script's top level: Pester's discovery pass does not carry a plain top-level + # function into the It blocks that run afterward. + function Set-MockPlan([hashtable[]] $Rules) { + $body = @{ rules = $Rules } | ConvertTo-Json -Depth 6 -Compress + $response = Invoke-RestMethod "$script:Base/plan" -Method Post -ContentType 'application/json' ` + -Body $body -SkipHttpErrorCheck -StatusCodeVariable 'statusCode' + [pscustomobject]@{ StatusCode = $statusCode; Body = $response } + } +} + +AfterAll { + Stop-BenchMockServer $script:Server + [System.Threading.Thread]::CurrentThread.CurrentCulture = $script:CultureBefore +} + +Describe 'Set-Plan refuses a zero window or count' { + BeforeEach { + $null = Invoke-RestMethod "$script:Base/plan" -Method Post -ContentType 'application/json' -Body '{"rules":[]}' + } + + It 'refuses an explicit from: 0' { + $result = Set-MockPlan @(@{ route = '^/v1\.0/users/1$'; kind = 'outage'; from = 0 }) + $result.StatusCode | Should -Be 400 + $result.Body.error.code | Should -Be 'MockPlanRejected' + $result.Body.error.message | Should -Be "rule 1: 'from' must be at least 1, got 0" + } + + It 'refuses an explicit count: 0' { + $result = Set-MockPlan @(@{ route = '^/v1\.0/users/1$'; kind = 'outage'; count = 0 }) + $result.StatusCode | Should -Be 400 + $result.Body.error.message | Should -Be "rule 1: 'count' must be at least 1, got 0" + } + + It 'refuses an explicit to: 0' { + $result = Set-MockPlan @(@{ route = '^/v1\.0/users/1$'; kind = 'outage'; to = 0 }) + $result.StatusCode | Should -Be 400 + $result.Body.error.message | Should -Be "rule 1: 'to' must be at least 1, got 0" + } + + It 'refuses an explicit seconds: 0' { + $result = Set-MockPlan @(@{ route = '^/v1\.0/users/1$'; kind = 'delayedVisibility'; seconds = 0 }) + $result.StatusCode | Should -Be 400 + $result.Body.error.message | Should -Be "rule 1: 'seconds' must be at least 1, got 0" + } + + It 'names the failing rule by its 1-based position' { + $result = Set-MockPlan @( + @{ route = '^/v1\.0/users/2$'; kind = 'outage' } + @{ route = '^/v1\.0/users/3$'; kind = 'outage'; count = 0 } + ) + $result.StatusCode | Should -Be 400 + $result.Body.error.message | Should -Be "rule 2: 'count' must be at least 1, got 0" + } + + It 'still accepts a rule with no window fields at all, meaning no end' { + $result = Set-MockPlan @(@{ route = '^/v1\.0/users/9$'; kind = 'outage' }) + $result.StatusCode | Should -Be 200 + $result.Body.ok | Should -BeTrue + } + + It 'still accepts from and count set at their floor of 1' { + $result = Set-MockPlan @(@{ route = '^/v1\.0/users/9$'; kind = 'outage'; from = 1; count = 1 }) + $result.StatusCode | Should -Be 200 + $result.Body.ok | Should -BeTrue + } +} + +Describe 'served counts a plan-faulted entity read the way pagesServed counts a cut page' { + BeforeEach { + $null = Invoke-RestMethod "$script:Base/plan" -Method Post -ContentType 'application/json' -Body '{"rules":[]}' + $null = Invoke-RestMethod "$script:Base/reset" + } + + It 'leaves served unchanged while a throttleStorm rule is firing' { + $null = Set-MockPlan @(@{ route = '^/v1\.0/users/42$'; kind = 'throttleStorm'; count = 3 }) + $before = Invoke-RestMethod "$script:Base/stats" + 1..3 | ForEach-Object { Invoke-RestMethod "$script:Base/v1.0/users/42" -SkipHttpErrorCheck | Out-Null } + $after = Invoke-RestMethod "$script:Base/stats" + ($after.served - $before.served) | Should -Be 0 + ($after.throttleStorm - $before.throttleStorm) | Should -Be 3 + } + + It 'raises served by one once the window closes and the entity is answered whole' { + $null = Set-MockPlan @(@{ route = '^/v1\.0/users/43$'; kind = 'throttleStorm'; count = 3 }) + $before = Invoke-RestMethod "$script:Base/stats" + 1..4 | ForEach-Object { Invoke-RestMethod "$script:Base/v1.0/users/43" -SkipHttpErrorCheck | Out-Null } + $after = Invoke-RestMethod "$script:Base/stats" + ($after.served - $before.served) | Should -Be 1 + } +} + +Describe 'Send-Slow honors bodyMs with a single chunk' { + BeforeEach { + $null = Invoke-RestMethod "$script:Base/plan" -Method Post -ContentType 'application/json' -Body '{"rules":[]}' + } + + It 'takes about bodyMs, not an instant write, when chunks is 1' { + $null = Set-MockPlan @(@{ route = '^/v1\.0/users/777$'; kind = 'slowBody'; chunks = 1; bodyMs = 1500 }) + $sw = [System.Diagnostics.Stopwatch]::StartNew() + Invoke-RestMethod "$script:Base/v1.0/users/777" | Out-Null + $sw.Stop() + $sw.ElapsedMilliseconds | Should -BeGreaterThan 1300 + $sw.ElapsedMilliseconds | Should -BeLessThan 5000 + } +} diff --git a/tests/Unit/Mgx.Behaviour.Tests.ps1 b/tests/Unit/Mgx.Behaviour.Tests.ps1 index 0d9f7d9..7198a2e 100644 --- a/tests/Unit/Mgx.Behaviour.Tests.ps1 +++ b/tests/Unit/Mgx.Behaviour.Tests.ps1 @@ -1,12 +1,21 @@ -#Requires -Modules Pester +#Requires -Modules Pester <# - Behavioural tests for the compiled module surface. + The behavior of the built module as PowerShell sees it: the cmdlet and parameter contract, + and what the cmdlets decide before a request leaves the process. - Prerequisites: run build.ps1 first. + Run ./build.ps1 first, then the harness, which is what CI runs too: - Usage: - Invoke-Pester ./tests/Unit/Mgx.Behaviour.Tests.ps1 -Output Detailed + Import-Module ./tests/TestHarness.psm1; Invoke-TestHarness + + The harness excludes the Live tag, so a clone with no Graph session passes. What a cmdlet + does against a real tenant is proven in tests/Live, run on its own: + + Invoke-Pester -Path ./tests/Live + + The one Live-tagged block left here is the sustained auth run at the bottom: it needs an + hour of directory traffic to cross a token boundary, which is not something tests/Live can + ask of anyone running the suite. Pass -IncludeLive to the harness to reach it. #> BeforeAll { @@ -15,7 +24,7 @@ BeforeAll { } Describe 'Module Loading' { - It 'Should import the M365DSC.mgx module' { + It 'Should import Mgx module' { $module = Get-Module M365DSC.mgx $module | Should -Not -BeNullOrEmpty # Read expected version from manifest to avoid hardcoded values breaking on version bumps @@ -184,20 +193,6 @@ Describe 'Expand-MgxRelation Parameter Compatibility' { $err.Exception.Message | Should -Not -BeLike '*ConsistencyLevel*' } } - - # R3-11: Buffer size warning at 50k items - It 'Should warn when buffer exceeds 50k items' -Tag 'Slow' { - $objects = 1..50001 | ForEach-Object { [PSCustomObject]@{id = "id-$_"} } - $w = @() - try { - $objects | Expand-MgxRelation '/users/{id}/manager' -As Manager ` - -WarningVariable w 3>$null -ErrorAction Stop - } catch { - # Expected: auth error from EndProcessing. Warning fires in ProcessRecord before this. - } - $bufferWarning = $w | Where-Object { $_ -match 'buffer' -or $_ -match '50.000' } - $bufferWarning | Should -Not -BeNullOrEmpty - } } Describe 'Invoke-MgxRequest Parameter Compatibility' { @@ -546,7 +541,7 @@ Describe 'Disable-MgxResilience Parameter Compatibility' { } Describe 'Get-MgxResilience Parameter Compatibility' { - It 'Should declare the type it actually emits' { + It 'Should have the OutputType its help documents' { $outputType = (Get-Command Get-MgxResilience).OutputType $outputType.Name | Should -Contain 'Mgx.Cmdlets.Models.MgxResilienceOutput' } @@ -803,6 +798,10 @@ Describe 'Set-MgxOption Pipeline Parameters' { { Set-MgxOption -CircuitBreakerMinThroughput 0 } | Should -Throw } + It 'Should reject -CircuitBreakerMinThroughput 1' { + { Set-MgxOption -CircuitBreakerMinThroughput 1 } | Should -Throw + } + It 'Should reject -RateLimitQueueLimit -1' { { Set-MgxOption -RateLimitQueueLimit -1 } | Should -Throw } @@ -1339,21 +1338,11 @@ Describe 'NormalizeSelect (via reflection)' { } } -Describe 'Expand-MgxRelation Buffer Warning' { - It 'Source code contains 50k buffer warning guard' { - # The actual live test (piping 50k items) requires Graph connection and takes ~15 min - # due to fan-out HTTP calls. Verify the guard exists in source instead. - $source = Get-Content (Join-Path $PSScriptRoot '../../src/Mgx.Cmdlets/Cmdlets/Expand/ExpandMgxRelation.cs') -Raw - $source | Should -Match '50_000' - $source | Should -Match 'WriteWarning' - $source | Should -Match 'buffer|Buffer' - } -} - -Describe 'HttpClient Ownership Disposal Guard' { - # Tests the s_ownsHttpClient guard in MgxCmdletBase.ScheduleDelayedHttpClientDispose. - # When s_ownsHttpClient is false (SDK fallback path), ResetHttpClient must NOT dispose - # the HttpClient (it belongs to the SDK). When true, it must dispose after delay. +Describe 'HttpClient Reset Leaves A Held Client Alone' { + # A replaced transport is dropped, not closed. Every ResilientGraphClient built on it + # captured it in its constructor and keeps sending through it for as long as its + # enumeration runs, so closing it on a timer killed long-running cmdlets mid-stream; and a + # borrowed SDK client was never mgx's to close at all. ResetHttpClient must leave both. BeforeAll { # Ensure Mgx.Engine types are loaded (transitive dep of Mgx.Cmdlets) @@ -1391,7 +1380,6 @@ Describe 'HttpClient Ownership Disposal Guard' { $script:CmdletBaseType.GetField('s_clientOptions', $script:Flags).SetValue( $null, $script:ShortTimeoutOptions) - # ResetHttpClient -> ScheduleDelayedHttpClientDispose checks s_ownsHttpClient # ResetHttpClient is public static, so use 'Static,Public' (not $script:Flags which is NonPublic) $script:CmdletBaseType.GetMethod('ResetHttpClient', [System.Reflection.BindingFlags]'Static,Public').Invoke($null, $null) @@ -1403,7 +1391,7 @@ Describe 'HttpClient Ownership Disposal Guard' { $script:DisposedField.GetValue($httpClient) | Should -BeFalse } - It 'Should dispose HttpClient when s_ownsHttpClient is true' { + It 'Should NOT dispose an owned HttpClient a running cmdlet may still hold' { $httpClient = [System.Net.Http.HttpClient]::new() $script:CmdletBaseType.GetField('s_graphHttpClient', $script:Flags).SetValue($null, $httpClient) @@ -1414,11 +1402,11 @@ Describe 'HttpClient Ownership Disposal Guard' { $script:CmdletBaseType.GetMethod('ResetHttpClient', [System.Reflection.BindingFlags]'Static,Public').Invoke($null, $null) - # Wait longer than TotalTimeoutSeconds (1s) to let scheduled disposal fire + # Longer than the TotalTimeoutSeconds (1s) the disposal used to be scheduled on Start-Sleep -Seconds 3 - # HttpClient SHOULD be disposed (we own it) - $script:DisposedField.GetValue($httpClient) | Should -BeTrue + # Ownership says who may replace it, not how long anything still using it will live + $script:DisposedField.GetValue($httpClient) | Should -BeFalse } } @@ -1494,3 +1482,35 @@ finally { } } +# Sustained auth pipeline test. Exercises token refresh under load. +# MSAL's AuthenticationHandler refreshes tokens proactively (5 min before expiry). +# Azure AD tokens expire at 60-90 min, so crossing a token boundary requires ~65+ min. +# This test validates the auth pipeline survives sustained use; manual 2+ hour runs +# are recommended before enterprise deployment per panel review findings. +Describe 'Sustained Auth Pipeline' -Tag 'Live', 'LongRunning' { + It 'Export-MgxCollection survives sustained load without auth errors' { + $context = Get-MgContext -ErrorAction SilentlyContinue + if (-not $context) { + Set-ItResult -Skipped -Because 'Not connected to Microsoft Graph' + return + } + $outFile = [System.IO.Path]::GetTempFileName() + '.jsonl' + $cpFile = [System.IO.Path]::GetTempFileName() + '.json' + try { + $sw = [System.Diagnostics.Stopwatch]::StartNew() + + # Export audit logs (large collection, sustained HTTP traffic) + Export-MgxCollection /auditLogs/signIns -OutputFile $outFile ` + -CheckpointPath $cpFile -All -ErrorAction Stop + + $sw.Stop() + $lineCount = (Get-Content $outFile -Raw).Split("`n").Where({ $_ }).Count + + # Should have exported records without auth failure + $lineCount | Should -BeGreaterThan 0 + Write-Host "Exported $lineCount sign-in records in $([math]::Round($sw.Elapsed.TotalMinutes, 1)) minutes" + } finally { + Remove-Item $outFile, $cpFile -Force -ErrorAction SilentlyContinue + } + } +} diff --git a/tests/Unit/Mgx.ErrorContract.Tests.ps1 b/tests/Unit/Mgx.ErrorContract.Tests.ps1 new file mode 100644 index 0000000..4af4b16 --- /dev/null +++ b/tests/Unit/Mgx.ErrorContract.Tests.ps1 @@ -0,0 +1,100 @@ +#Requires -Modules Pester + +<# + Pins what about_Mgx_Errors documents, as far as it reaches without a transport: + pre-flight failures terminate with stable ids, land in -ErrorVariable, and set $? + false. Per-item wire behavior is pinned by the xUnit suite, which fakes HTTP - + ErrorActionMatrixTests drives all four preferences across the six failure contexts. +#> + +BeforeAll { + Import-Module -Name (Join-Path (Split-Path -Parent $PSScriptRoot) 'TestHarness.psm1') -Force + $script:Paths = Get-MgxTestPath + Import-Module -Name $script:Paths.ManifestPath -Force -ErrorAction Stop +} + +Describe 'about_Mgx_Errors' { + It 'ships and documents both halves of the contract' { + $topic = Get-Help about_Mgx_Errors -ErrorAction Stop | Out-String + $topic | Should -Match 'TERMINATING ERRORS' + $topic | Should -Match 'NON-TERMINATING ERRORS' + $topic | Should -Match 'CANCELLATION' + } +} + +# Carried as text rather than as scriptblocks so each case can be driven twice: once inside a +# try, which is what proves it terminates, and once bare with -ErrorVariable appended, which is +# the only way to read $? and the collected record off the statement that terminated. +$script:PreFlightCases = @( + @{ Name = 'an absolute Uri on Invoke-MgxRequest' + Command = "Invoke-MgxRequest -Uri 'https://graph.microsoft.com/v1.0/users'" + ErrorId = 'AbsoluteUriNotAllowed' } + @{ Name = 'an absolute Uri on Export-MgxCollection' + Command = "Export-MgxCollection -Uri 'https://graph.microsoft.com/v1.0/users' -OutputFile ([IO.Path]::GetTempFileName())" + ErrorId = 'AbsoluteUriNotAllowed' } + @{ Name = 'an absolute Uri on Sync-MgxDelta' + Command = "Sync-MgxDelta -Uri 'https://graph.microsoft.com/v1.0/users/delta' -DeltaPath ([IO.Path]::GetTempFileName())" + ErrorId = 'AbsoluteUriNotAllowed' } + @{ Name = '-Search without ConsistencyLevel' + Command = "Invoke-MgxRequest -Uri '/users' -Search 'displayName:x'" + ErrorId = 'ConsistencyLevelRequired' } + @{ Name = 'a -Uri without {id} on Expand-MgxRelation' + Command = "[pscustomobject]@{ id = 'x' } | Expand-MgxRelation -Uri '/users/x/manager' -As manager" + ErrorId = 'MissingIdPlaceholder' } +) + +Describe 'Pre-flight failures terminate' { + It ' terminates with ' -TestCases $script:PreFlightCases { + # A terminating error is catchable; a non-terminating one is not. Catching it IS + # the assertion that it terminates. + $caught = $null + try { & ([scriptblock]::Create($Command)) } catch { $caught = $_ } + $caught | Should -Not -BeNullOrEmpty + $caught.FullyQualifiedErrorId | Should -Match "^$ErrorId" + } + + It ' sets $? false and lands in -ErrorVariable' -TestCases $script:PreFlightCases { + # ThrowTerminatingError ends the statement, not the script, so $? and -ErrorVariable are + # readable from the statement after it - which a catch would have hidden. Both reads have + # to happen in the session that ran the command, so the probe runs in a fresh runspace: + # under Pester the same error takes the whole test body with it, and $? outside the probe + # would report the wrapper rather than the cmdlet. + $probe = [powershell]::Create() + try { + $script = @' +Import-Module -Name '__MANIFEST__' -Force +$ev = $null +__COMMAND__ -ErrorVariable ev 2>$null +[pscustomobject]@{ Ok = $?; Collected = @($ev) } +'@ + $null = $probe.AddScript( + $script.Replace('__MANIFEST__', $script:Paths.ManifestPath).Replace('__COMMAND__', $Command)) + $result = $probe.Invoke()[0] + } + finally { $probe.Dispose() } + + $result.Ok | Should -BeFalse + $result.Collected.Count | Should -Be 1 + + # A terminating error reaches -ErrorVariable as the CmdletInvocationException carrying the + # record, not as the record itself. + $collected = $result.Collected[0] + $record = if ($collected -is [System.Management.Automation.ErrorRecord]) { $collected } + else { $collected.ErrorRecord } + $record.FullyQualifiedErrorId | Should -Match "^$ErrorId" + } +} + +Describe 'about_Mgx_Errors vocabulary' { + It 'names every category the presentation map can produce' { + # A text check only: the categories themselves are asserted where records are + # actually produced (the xUnit wire suites). This holds the DOCUMENT to the + # vocabulary, not the code to the document. + $topic = Get-Help about_Mgx_Errors | Out-String + foreach ($category in 'ObjectNotFound', 'PermissionDenied', 'AuthenticationError', + 'LimitsExceeded', 'ResourceUnavailable', 'ConnectionError', + 'InvalidArgument', 'ResourceExists') { + $topic | Should -Match $category + } + } +} diff --git a/tests/Unit/Mgx.Harness.Tests.ps1 b/tests/Unit/Mgx.Harness.Tests.ps1 new file mode 100644 index 0000000..568a999 --- /dev/null +++ b/tests/Unit/Mgx.Harness.Tests.ps1 @@ -0,0 +1,147 @@ +#Requires -Modules Pester + +<# + Invoke-TestHarness's own result contract, from tests/TestHarness.psm1: a container that + cannot run at all - a parse error, a top-level throw - must not read as a clean pass. + Pester already reports this on the object it hands back (Result, FailedContainersCount); + these pin that it keeps doing so, and that the harness's own warning names the file. + + Each case drives Invoke-TestHarness in a child pwsh process rather than in-process: + Invoke-TestHarness re-imports Pester with -Force on every call, and Pester does not + support that while a run it started is still on the call stack - a second such call inside + this container's own run hangs rather than fails, so the nested run is given a process of + its own. +#> + +BeforeAll { + $script:HarnessModule = Join-Path (Split-Path -Parent $PSScriptRoot) 'TestHarness.psm1' + + # Written once for the container and driven with the arguments below: a scenario is just + # the fixture files plus the path this runs Invoke-TestHarness against. + $script:ChildRunner = Join-Path $TestDrive 'Invoke-HarnessChild.ps1' + Set-Content -Path $script:ChildRunner -Value @' +param( + [Parameter(Mandatory)] [string] $HarnessModule, + [Parameter(Mandatory)] [string] $TestPath, + [Parameter(Mandatory)] [string] $TestResultsFile, + [Parameter(Mandatory)] [string] $ResultFile +) +Import-Module -Name $HarnessModule -Force +$warnings = $null +$result = Invoke-TestHarness -TestPath $TestPath -TestResultsFile $TestResultsFile ` + -IgnoreCodeCoverage -SkipDotNetTests -WarningVariable warnings -WarningAction SilentlyContinue +[pscustomobject]@{ + Result = $result.Pester.Result + FailedCount = $result.Pester.FailedCount + FailedContainersCount = $result.Pester.FailedContainersCount + PassedCount = $result.Pester.PassedCount + Warnings = @($warnings | ForEach-Object { $_.ToString() }) +} | ConvertTo-Json -Depth 5 | Set-Content -Path $ResultFile -Encoding utf8 +'@ + + function Invoke-HarnessChild + { + <# + .SYNOPSIS + Runs Invoke-TestHarness -TestPath $TestPath in its own pwsh process and + returns the JSON result it wrote, so the harness's re-import of Pester never + runs nested inside this run. + #> + [CmdletBinding()] + param ([Parameter(Mandatory)] [string] $TestPath) + + $stamp = [Guid]::NewGuid().ToString('N') + $resultFile = Join-Path $TestDrive "result-$stamp.json" + $testResultsFile = Join-Path $TestDrive "nunit-$stamp.xml" + $stdout = Join-Path $TestDrive "stdout-$stamp.log" + $stderr = Join-Path $TestDrive "stderr-$stamp.log" + + $arguments = @( + '-NoProfile', '-NonInteractive', '-File', $script:ChildRunner, + '-HarnessModule', $script:HarnessModule, + '-TestPath', $TestPath, + '-TestResultsFile', $testResultsFile, + '-ResultFile', $resultFile + ) + $process = Start-Process -FilePath 'pwsh' -ArgumentList $arguments -PassThru -NoNewWindow ` + -RedirectStandardOutput $stdout -RedirectStandardError $stderr + if (-not $process.WaitForExit(60000)) + { + try { $process.Kill() } catch { } + throw "Invoke-TestHarness child process did not exit within 60s for '$TestPath'." + } + if (-not (Test-Path -Path $resultFile)) + { + $errorText = Get-Content -Path $stderr -Raw -ErrorAction SilentlyContinue + throw "Invoke-TestHarness child process (exit $($process.ExitCode)) wrote no result for '$TestPath'. Stderr: $errorText" + } + return Get-Content -Path $resultFile -Raw | ConvertFrom-Json + } +} + +Describe 'Invoke-TestHarness result contract' { + Context 'a test path with a parse error' { + BeforeAll { + $script:Dir = Join-Path $TestDrive 'parse-error' + New-Item -ItemType Directory -Path $script:Dir -Force | Out-Null + Set-Content -Path (Join-Path $script:Dir 'Ok.Tests.ps1') -Value 'Describe "ok" { It "passes" { $true | Should -BeTrue } }' + # Unclosed block: Pester's own discovery fails to parse the file, the same way a + # dropped brace or a moved helper does. + Set-Content -Path (Join-Path $script:Dir 'Broken.Tests.ps1') -Value 'Describe "broken" {' + $script:Result = Invoke-HarnessChild -TestPath $script:Dir + } + + It 'reports Result Failed' { + $script:Result.Result | Should -Be 'Failed' + } + + It 'counts the container as failed, not a test' { + $script:Result.FailedContainersCount | Should -Be 1 + $script:Result.FailedCount | Should -Be 0 + } + + It 'warns naming the broken file' { + ($script:Result.Warnings -join "`n") | Should -Match ([Regex]::Escape('Broken.Tests.ps1')) + } + } + + Context 'a test path with a top-level throw' { + BeforeAll { + $script:Dir = Join-Path $TestDrive 'top-level-throw' + New-Item -ItemType Directory -Path $script:Dir -Force | Out-Null + Set-Content -Path (Join-Path $script:Dir 'Ok.Tests.ps1') -Value 'Describe "ok" { It "passes" { $true | Should -BeTrue } }' + Set-Content -Path (Join-Path $script:Dir 'Broken.Tests.ps1') -Value 'throw "dot-sourced helper is gone"' + $script:Result = Invoke-HarnessChild -TestPath $script:Dir + } + + It 'reports Result Failed' { + $script:Result.Result | Should -Be 'Failed' + } + + It 'counts the container as failed, not a test' { + $script:Result.FailedContainersCount | Should -Be 1 + $script:Result.FailedCount | Should -Be 0 + } + + It 'warns naming the broken file' { + ($script:Result.Warnings -join "`n") | Should -Match ([Regex]::Escape('Broken.Tests.ps1')) + } + } + + Context 'a clean test path' { + BeforeAll { + $script:Dir = Join-Path $TestDrive 'clean' + New-Item -ItemType Directory -Path $script:Dir -Force | Out-Null + Set-Content -Path (Join-Path $script:Dir 'Ok.Tests.ps1') -Value 'Describe "ok" { It "passes" { $true | Should -BeTrue } }' + $script:Result = Invoke-HarnessChild -TestPath $script:Dir + } + + It 'reports Result Passed' { + $script:Result.Result | Should -Be 'Passed' + } + + It 'writes no warning' { + $script:Result.Warnings.Count | Should -Be 0 + } + } +} diff --git a/tests/Unit/Mgx.HelpFreshness.Tests.ps1 b/tests/Unit/Mgx.HelpFreshness.Tests.ps1 index 2799c43..8b4eaa9 100644 --- a/tests/Unit/Mgx.HelpFreshness.Tests.ps1 +++ b/tests/Unit/Mgx.HelpFreshness.Tests.ps1 @@ -8,9 +8,23 @@ rejects. The existing surface tests compare example COUNT and parameter PRESENCE, so none of it registered. - This compares the text. It is deliberately coarse - descriptions are reflowed and links are - rendered differently by the generator - so it checks that the MAML contains no parameter - description that the markdown has since changed, using a normalized comparison. + This compares the text. Every piece of prose is compared whole - the synopsis, the + description, each example's prose and every parameter description - normalized for what the + generator does to it: it reflows prose and it renders a markdown link as its text followed by + its target. Runs of whitespace then collapse to a single space on both sides, and MAML's + separate elements join with a space rather than nothing, so a real edit inside + the prose still tells the two sides apart - only where a paragraph happens to break is out of + the comparison, because a break reads the same as an ordinary space once joined. The one + further allowance is the space after a rendered link's closing ")": platyPS drops it there + when it folds the paragraph that follows a link into the same one. Normalize marks that one + ")" - on the markdown side where it renders the link, and on the MAML side where the link + already reads as its rendered "text (target)" - with a sentinel no help text contains, so + the fold finds exactly that position and nowhere else; an ordinary parenthetical's ")" is + never marked and keeps its space. Measured with New-ExternalHelp (0.14.2): a plain + list item compiles to its own , a wrapped continuation folds into the para of the + item it continues, and whether a break lands before the next bullet depends on whether the + line before it was indented - none of which is where a documentation edit lives, so that + shape is pinned separately, para for para, rather than compared here. #> BeforeAll { @@ -18,14 +32,214 @@ BeforeAll { $script:Maml = Join-Path $repo 'Modules/M365DSC.mgx/en-US/Mgx.Cmdlets.dll-Help.xml' $script:HelpDir = Join-Path $repo 'Modules/M365DSC.mgx/help' + # The mark Normalize leaves on a rendered link's closing ")" - a private-use code point no + # help text contains - so FoldLinkSpace can find that one position later and nothing else. + $script:LinkMark = [char]0xE000 + $script:LinkMarkPattern = [regex]::Escape($script:LinkMark) + function Normalize([string] $t) { if (-not $t) { return '' } - # markdown link -> its text; code ticks and whitespace collapse - $t = [regex]::Replace($t, '\[([^\]]+)\]\([^)]+\)', '$1') + # markdown link -> text followed by its target, the way the generator renders one; code + # ticks and whitespace collapse. The target stays in the comparison, so a link retargeted + # in the markdown is a difference and not a wash. The link's closing ")" is marked so + # FoldLinkSpace can find exactly that position later. + $t = [regex]::Replace($t, '\[([^\]]+)\]\(([^)]+)\)', ('$1 ($2)' + $script:LinkMark)) + # The MAML side never carries that markdown syntax - by the time its text reaches here + # platyPS has already rendered the link to "text (target)" - so the same mark goes on a + # parenthetical whose target is itself a URL, which is what every markdown link in + # module/help points at. The lookahead skips a ")" the line above already marked, so a + # link is never marked twice. + $t = $t -replace "(\(https?://[^)]*\))(?!$script:LinkMarkPattern)", ('$1' + $script:LinkMark) $t = $t -replace '`', '' ($t -replace '\s+', ' ').Trim() } + # Drops the whitespace after the ")" Normalize marked as a rendered link's, and the mark + # itself - the fold platyPS performs when it merges the paragraph after a link into the one + # the link is in. A ")" with no mark - every ordinary parenthetical - is untouched, so a real + # edit that drops its own following space still reads as a difference. + function FoldLinkSpace([string] $t) { + $t -replace "$script:LinkMarkPattern\s*", '' + } + + # An excerpt of $t from $from, marked where it is cut. + function Excerpt([string] $t, [int] $from, [int] $length = 80) { + if ($from -ge $t.Length) { return '' } + $lead = if ($from -gt 0) { '...' } else { '' } + $take = [Math]::Min($length, $t.Length - $from) + $trail = if ($t.Length - $from -gt $length) { '...' } else { '' } + "$lead$($t.Substring($from, $take))$trail" + } + + # Both sides from where they part company: an appended sentence differs only at the end, and + # a message quoting the two openings would print the same words twice. + function Divergence([string] $want, [string] $got) { + $shared = 0 + $common = [Math]::Min($want.Length, $got.Length) + while ($shared -lt $common -and $want[$shared] -eq $got[$shared]) { $shared++ } + $from = [Math]::Max(0, $shared - 20) + "markdown says '$(Excerpt $want $from)' MAML says '$(Excerpt $got $from)'" + } + + # A section's prose with whitespace collapsed to a single space, so a paragraph break folds + # the same way an ordinary space between words does and where the generator happens to break + # or fold one never enters the comparison - only the sequence of prose, marker and + # punctuation characters does. Where that break lands does not track markdown's blank lines: + # a plain list item gets its own , a wrapped continuation folds into the para of + # the item it continues, and a lead-in line folds into the first item that follows it - that + # shape is pinned separately, below, rather than depended on here. The one further fold is the + # space after a rendered link's ")": platyPS drops it there when the paragraph that follows a + # link joins the one the link is in. Normalize already marked that ")", so FoldLinkSpace finds + # only that position - an ordinary parenthetical was never marked and keeps its space. + function Paragraphs([string[]] $lines) { + FoldLinkSpace (Normalize ($lines -join ' ')) + } + + # The paragraphs a MAML node carries at $xpath, joined with a space and folded the way + # Paragraphs folds markdown's. local-name() throughout: the document declares three + # namespaces and the prefixes are not worth registering to read a . + function MamlParas($node, [string] $xpath) { + FoldLinkSpace ((@($node.SelectNodes($xpath) | ForEach-Object { Normalize $_.InnerText } | + Where-Object { $_ })) -join ' ') + } + + # The same paragraphs MamlParas joins, kept apart instead - for pinning the LIST a MAML node + # carries, para for para, rather than the one string MamlParas flattens them to. + function MamlParaList($node, [string] $xpath) { + @($node.SelectNodes($xpath) | ForEach-Object { Normalize $_.InnerText } | Where-Object { $_ } | + ForEach-Object { FoldLinkSpace $_ }) + } + + # The markdown's own list of paragraphs, grouped the way New-ExternalHelp (0.14.2) groups + # them into elements - which does not track markdown's blank lines. A bullet line + # ("-" or "*" then a space, however far indented - a nested item under another still opens + # one) opens a new paragraph, unless the line before it was an indented continuation, in which + # case the break that would otherwise land before a bullet does not. An indented line with no + # bullet marker is always a continuation and always joins the paragraph before it. A lead-in + # line - unindented prose before the first bullet has appeared - joins the paragraph of the + # first bullet after it. Each group is folded the way Paragraphs folds one, so the result + # lines up against MamlParaList's, position for position. + function MarkdownParaList([string[]] $lines) { + $groups = [System.Collections.Generic.List[System.Collections.Generic.List[string]]]::new() + $sawBullet = $false + $prevContinuation = $false + $prevLeadIn = $false + foreach ($line in $lines) { + $bullet = $line -match '^\s*[-*]\s' + $continuation = -not $bullet -and $line -match '^\s+\S' + $suppress = $continuation -or $prevContinuation -or ($prevLeadIn -and -not $sawBullet) + if ($groups.Count -eq 0 -or -not $suppress) { + $groups.Add([System.Collections.Generic.List[string]]::new()) + } + $groups[$groups.Count - 1].Add($line) + if ($bullet) { $sawBullet = $true } + $prevContinuation = $continuation + $prevLeadIn = -not $bullet -and -not $continuation + } + @($groups | ForEach-Object { Paragraphs $_ }) + } + + # A node carrying one per string in $paras, so a pinned MAML shape can + # be read back through MamlParas itself rather than a second copy of what it does. + function AsMamlParas([string[]] $paras) { + $xml = [xml]'' + foreach ($p in $paras) { + $el = $xml.CreateElement('para') + $el.InnerText = $p + [void]$xml.DocumentElement.AppendChild($el) + } + $xml.DocumentElement + } + + # The lines of one '## ' section, up to the next heading of any level. A line + # inside a ``` fence never ends a section: a fenced example's own '# comment' opens with + # the same character a heading does, so fence state is tracked and heading detection is + # suspended for as long as one is open - the fenced lines themselves are code, not prose, + # so they are dropped rather than carried into the section like MarkdownExamples drops them. + function SectionLines([string[]] $lines, [string] $heading) { + $out = [System.Collections.Generic.List[string]]::new() + $in = $false + $fenced = $false + foreach ($line in $lines) { + if ($line -match "^## $heading\s*$") { $in = $true; continue } + if (-not $in) { continue } + if ($line -match '^```') { $fenced = -not $fenced; continue } + if ($fenced) { continue } + if ($line -match '^#{1,3} ') { break } + $out.Add($line) + } + $out + } + + # Every '### Example' heading with its title and the prose around its code fences. Prose + # stops at the second fence, the way the generator's does: a fence count + # resets with each example, and once it reaches two, nothing further joins the prose - only + # the code between the fences was ever remarks, not whatever trails the second one. + function MarkdownExamples([string[]] $lines) { + $examples = [System.Collections.Generic.List[object]]::new() + $title = $null + $prose = [System.Collections.Generic.List[string]]::new() + $fenced = $false + $fences = 0 + foreach ($line in $lines) { + # The fence is read before anything else, and nothing inside one is read as markdown: + # a PowerShell comment opens with the '#' a heading opens with, and most of these + # examples carry one. + if ($line -match '^```') { + $fenced = -not $fenced + if ($fenced) { $fences++ } + continue + } + if ($fenced) { continue } + if ($line -match '^### Example') { + if ($title) { $examples.Add([pscustomobject]@{ Title = $title; Prose = Paragraphs $prose }) } + $title = Normalize ($line -replace '^### ', '') + $prose.Clear() + $fences = 0 + continue + } + if (-not $title) { continue } + if ($line -match '^#{1,3} ') { + $examples.Add([pscustomobject]@{ Title = $title; Prose = Paragraphs $prose }) + $title = $null + continue + } + if ($fences -ge 2) { continue } + $prose.Add($line) + } + if ($title) { $examples.Add([pscustomobject]@{ Title = $title; Prose = Paragraphs $prose }) } + $examples + } + + # Every '### ' entry under OUTPUTS, with the prose beneath it - the type from the + # heading and the prose from the lines under it, read the same way MarkdownExamples reads an + # example: entering and leaving on '## ' headings of its own, a '### ' line starting a new + # entry rather than ending the section, and a fence dropped rather than read as markdown. + function MarkdownOutputs([string[]] $lines) { + $outputs = [System.Collections.Generic.List[object]]::new() + $in = $false + $type = $null + $prose = [System.Collections.Generic.List[string]]::new() + $fenced = $false + foreach ($line in $lines) { + if ($line -match '^## OUTPUTS\s*$') { $in = $true; continue } + if (-not $in) { continue } + if ($line -match '^```') { $fenced = -not $fenced; continue } + if ($fenced) { continue } + if ($line -match '^## ') { break } + if ($line -match '^### (.+)$') { + if ($type) { $outputs.Add([pscustomobject]@{ Type = $type; Prose = Paragraphs $prose }) } + $type = $Matches[1].Trim() + $prose.Clear() + continue + } + if (-not $type) { continue } + $prose.Add($line) + } + if ($type) { $outputs.Add([pscustomobject]@{ Type = $type; Prose = Paragraphs $prose }) } + $outputs + } + $script:MamlXml = if (Test-Path $script:Maml) { [xml](Get-Content $script:Maml -Raw) } else { $null } } @@ -41,6 +255,8 @@ Describe 'Compiled help matches its markdown source' { @{ Text = 'ODataType'; Why = '2.0.0 replaced PSObjects with hashtables; @odata.type is a key now' } @{ Text = 'instead of PSObjects'; Why = '-Raw returns raw JSON instead of hashtables' } @{ Text = 'ephemeral and deleted on success'; Why = 'Sync-MgxDelta gained -CheckpointPath in 2.1.0' } + @{ Text = 'belongs to a chunk that was refused'; Why = 'after a chunk failure the batch-level retry pass is skipped for every candidate, including ones a cleanly answered chunk left failing' } + @{ Text = 'its request went out and may have been applied'; Why = 'a refusal status also covers a POST an open circuit or a rate limiter stopped before it left' } ) $raw = Get-Content $script:Maml -Raw $hits = $stale | Where-Object { $raw -match [regex]::Escape($_.Text) } @@ -50,9 +266,13 @@ Describe 'Compiled help matches its markdown source' { It 'carries the synopsis, description, outputs and examples its markdown gives' { # The parameter check below was the whole guard, so a SYNOPSIS saying the wrong thing - # entirely, a replaced OUTPUTS section, or a renamed example all shipped green. Sections - # are compared by their opening sentence: the generator reflows prose, so a prefix is - # what can be compared honestly. + # entirely, a replaced OUTPUTS section, or a renamed example all shipped green. Comparing + # a section by its opening sentence closed only half of that: the opening survives a + # rewrite of everything under it, so a DESCRIPTION whose remaining paragraphs said + # "deletes every user in the tenant" passed, and so did an example whose prose described + # something its code does not do, and so did a mutated OUTPUTS type. Prose is compared + # whole, including each OUTPUTS entry's - a cmdlet can document more than one return type, + # and each carries its own type and its own prose. $mismatches = [System.Collections.Generic.List[string]]::new() foreach ($md in Get-ChildItem $script:HelpDir -Filter '*.md') { @@ -62,51 +282,52 @@ Describe 'Compiled help matches its markdown source' { Where-Object { $_.details.name.Trim() -eq $cmdlet } | Select-Object -First 1 if (-not $node) { $mismatches.Add("$cmdlet : absent from MAML"); continue } - function SectionText([string[]] $l, [string] $heading) { - $out = [System.Collections.Generic.List[string]]::new() - $in = $false - foreach ($line in $l) { - if ($line -match "^## $heading\s*$") { $in = $true; continue } - if ($in -and $line -match '^#{1,3} ') { break } - if ($in -and $line.Trim()) { $out.Add($line) } - } - Normalize ($out -join ' ') - } - function Head([string] $t, [int] $n = 50) { - if ($t.Length -gt $n) { $t.Substring(0, $n) } else { $t } - } - - $wantSyn = SectionText $lines 'SYNOPSIS' - $gotSyn = Normalize $node.details.description.para - if ($wantSyn -and $gotSyn -notlike "*$(Head $wantSyn)*") { - $mismatches.Add("$cmdlet SYNOPSIS : markdown '$(Head $wantSyn)...' MAML '$(Head $gotSyn)...'") - } - - $wantOut = SectionText $lines 'OUTPUTS' - $gotOut = Normalize (($node.returnValues.returnValue.type.name | ForEach-Object { $_ }) -join ' ') - if ($wantOut -and $gotOut) { - # OUTPUTS opens with the type name under a ### heading; compare that token. - $wantType = ($lines | Where-Object { $_ -match '^### ' } | ForEach-Object { $_ } | - Select-Object -First 0) - $typeLine = $null - $inOut = $false - foreach ($line in $lines) { - if ($line -match '^## OUTPUTS\s*$') { $inOut = $true; continue } - if ($inOut -and $line -match '^## ') { break } - if ($inOut -and $line -match '^### (.+)$') { $typeLine = $Matches[1].Trim(); break } + $wantSyn = Paragraphs (SectionLines $lines 'SYNOPSIS') + $gotSyn = MamlParas $node "*[local-name()='details']/*[local-name()='description']/*[local-name()='para']" + if ($wantSyn -ne $gotSyn) { + $mismatches.Add("$cmdlet SYNOPSIS : $(Divergence $wantSyn $gotSyn)") + } + + $wantDesc = Paragraphs (SectionLines $lines 'DESCRIPTION') + $gotDesc = MamlParas $node "*[local-name()='description']/*[local-name()='para']" + if ($wantDesc -ne $gotDesc) { + $mismatches.Add("$cmdlet DESCRIPTION : $(Divergence $wantDesc $gotDesc)") + } + + $mdOutputs = @(MarkdownOutputs $lines) + $mamlOutputs = @($node.SelectNodes("*[local-name()='returnValues']/*[local-name()='returnValue']")) + for ($i = 0; $i -lt [Math]::Min($mdOutputs.Count, $mamlOutputs.Count); $i++) { + $wantType = Normalize $mdOutputs[$i].Type + $gotType = Normalize $mamlOutputs[$i].SelectSingleNode("*[local-name()='type']/*[local-name()='name']").InnerText + if ($wantType -ne $gotType) { + $mismatches.Add("$cmdlet OUTPUTS $($i + 1) type : markdown '$wantType' MAML '$gotType'") } - if ($typeLine -and $gotOut -notlike "*$typeLine*") { - $mismatches.Add("$cmdlet OUTPUTS : markdown '$typeLine' MAML '$(Head $gotOut 60)'") + + $wantOutProse = $mdOutputs[$i].Prose + $gotOutProse = MamlParas $mamlOutputs[$i] "*[local-name()='description']/*[local-name()='para']" + if ($wantOutProse -ne $gotOutProse) { + $mismatches.Add("$cmdlet OUTPUTS $($i + 1) prose : $(Divergence $wantOutProse $gotOutProse)") } } + if ($mdOutputs.Count -ne $mamlOutputs.Count) { + $mismatches.Add("$cmdlet OUTPUTS : markdown names $($mdOutputs.Count) type(s), MAML carries $($mamlOutputs.Count)") + } - $mdExamples = @($lines | Where-Object { $_ -match '^### Example' }) - $mamlExamples = @($node.examples.example) + $mdExamples = @(MarkdownExamples $lines) + $mamlExamples = @($node.SelectNodes("*[local-name()='examples']/*[local-name()='example']")) for ($i = 0; $i -lt [Math]::Min($mdExamples.Count, $mamlExamples.Count); $i++) { - $want = Normalize ($mdExamples[$i] -replace '^### ', '') - $got = Normalize $mamlExamples[$i].title - if ($got -notlike "*$want*") { - $mismatches.Add("$cmdlet example $($i + 1) : markdown '$want' MAML '$got'") + # The generator pads a title out to a fixed width with dashes, so the title is + # the one thing here compared by containment rather than whole. + $wantTitle = $mdExamples[$i].Title + $gotTitle = Normalize $mamlExamples[$i].SelectSingleNode("*[local-name()='title']").InnerText + if ($gotTitle -notlike "*$wantTitle*") { + $mismatches.Add("$cmdlet example $($i + 1) : markdown '$wantTitle' MAML '$gotTitle'") + } + + $wantProse = $mdExamples[$i].Prose + $gotProse = MamlParas $mamlExamples[$i] "*[local-name()='remarks']/*[local-name()='para']" + if ($wantProse -ne $gotProse) { + $mismatches.Add("$cmdlet example $($i + 1) prose : $(Divergence $wantProse $gotProse)") } } } @@ -114,7 +335,78 @@ Describe 'Compiled help matches its markdown source' { ($mismatches -join "`n") | Should -BeNullOrEmpty -Because 'run ./build.ps1 to regenerate the compiled help' } + It 'reads a list the way the generator compiles it, seams and all' { + # No compared section carries a list today, so nothing above exercises this. Each shape + # pins the para LIST New-ExternalHelp (0.14.2) actually compiled its markdown to on a + # scratch help file: a plain item gets its own para, a wrapped continuation folds into + # the para of the item it continues, and a lead-in line folds into the first item after + # it - a different break position per shape. MarkdownParaList rebuilds that same list from + # the markdown side independently, so the two are compared para for para - count and each + # para's text - rather than joined into one string where a fixture that merged or split a + # shape's paras would still read as a match. + $shapes = @( + @{ Name = 'plain two items'; Markdown = @('- Item one', '- Item two') + Maml = @('- Item one', '- Item two') } + @{ Name = 'plain three items'; Markdown = @('- Item one', '- Item two', '- Item three') + Maml = @('- Item one', '- Item two', '- Item three') } + @{ Name = '* items'; Markdown = @('* Item one', '* Item two') + Maml = @('* Item one', '* Item two') } + @{ Name = 'a nested item' + Markdown = @('- Item one', '- Item two', ' - Nested under two') + Maml = @('- Item one', '- Item two', ' - Nested under two') } + @{ Name = 'a wrapped item' + Markdown = @('- Item one', ' continued here.', '- Item two') + Maml = @('- Item one continued here. - Item two') } + @{ Name = 'a wrapped first item' + Markdown = @('- Item one', ' continued here.', '- Item two', '- Item three') + Maml = @('- Item one continued here. - Item two', '- Item three') } + @{ Name = 'a lead-in line' + Markdown = @('Lead prose.', '- Item one', '- Item two') + Maml = @('Lead prose. - Item one', '- Item two') } + ) + + foreach ($shape in $shapes) { + $want = @(MarkdownParaList $shape.Markdown) + $got = @(MamlParaList (AsMamlParas $shape.Maml) "*[local-name()='para']") + $got.Count | Should -Be $want.Count -Because "the $($shape.Name) shape should compile to $($want.Count) para(s)" + for ($i = 0; $i -lt $want.Count; $i++) { + $got[$i] | Should -Be $want[$i] -Because "the $($shape.Name) shape's para $($i + 1) should match para for para" + } + } + } + + It 'still catches a changed word once the seams are out of the comparison' { + # Stripping whitespace must not blind the comparison to the prose itself - only to where + # the generator chose to break it. + $want = Paragraphs @('- Item one', '- Item two') + $got = MamlParas (AsMamlParas @('- Item one', '- Item deux')) "*[local-name()='para']" + $want | Should -Not -Be $got + } + + It 'still catches a parenthetical that lost its own space' { + # module/help/Enable-MgxResilience.md, measured: nothing about "(Get-MgUser, Get-MgGroup, + # etc.)" tells platyPS to fold the sentence after it into the same paragraph, so a MAML + # that lost the space there is a real drift, not the one platyPS itself produces. + $want = Paragraphs @('After calling this cmdlet, all SDK cmdlets (Get-MgUser, Get-MgGroup, etc.) automatically gain resilience with zero script changes.') + $got = MamlParas (AsMamlParas @('After calling this cmdlet, all SDK cmdlets (Get-MgUser, Get-MgGroup, etc.)automatically gain resilience with zero script changes.')) "*[local-name()='para']" + $want | Should -Not -Be $got -Because 'only a rendered link''s own ")" may lose its following space' + } + + It 'tolerates the space a rendered link''s fold actually drops' { + # module/help/Invoke-MgxBatchRequest.md DESCRIPTION, measured with New-ExternalHelp + # (0.14.2): the paragraph after a link folds into the link's own with no + # space at the seam, so the markdown's two paragraphs and the MAML's one must still + # compare equal. + $want = Paragraphs @('Source: [Combine multiple HTTP requests using JSON batching](https://learn.microsoft.com/en-us/graph/json-batching)', '', 'Pipeline input can be string URLs.') + $got = MamlParas (AsMamlParas @('Source: Combine multiple HTTP requests using JSON batching (https://learn.microsoft.com/en-us/graph/json-batching)Pipeline input can be string URLs.')) "*[local-name()='para']" + $want | Should -Be $got -Because "platyPS's own fold at a link boundary is not a drift" + } + It 'documents every parameter with the description its markdown gives' { + # Whole descriptions, not their openings. A comparison of the first 60 characters passes + # every edit that appends to a parameter which already said something true - a widened + # range, a sentence naming what a gate covers, a sentence naming what a redaction reaches + # - and those are most of what a help edit is. $mismatches = [System.Collections.Generic.List[string]]::new() foreach ($md in Get-ChildItem $script:HelpDir -Filter '*.md') { @@ -127,7 +419,11 @@ Describe 'Compiled help matches its markdown source' { $param = $Matches[1] $desc = [System.Collections.Generic.List[string]]::new() for ($j = $i + 1; $j -lt $lines.Count; $j++) { - if ($lines[$j] -match '^```' -or $lines[$j] -match '^### ' -or $lines[$j] -match '^## ') { break } + # The yaml block ends the description. A fence that is not it belongs to the + # description and is compared with it: the generator renders a fenced block + # verbatim and eats the asterisks of anything written into prose, so a + # description naming a literal like ***REDACTED*** has to fence it. + if ($lines[$j] -match '^```yaml' -or $lines[$j] -match '^### ' -or $lines[$j] -match '^## ') { break } if ($lines[$j].Trim()) { $desc.Add($lines[$j]) } } $want = Normalize ($desc -join ' ') @@ -141,16 +437,151 @@ Describe 'Compiled help matches its markdown source' { if (-not $node) { $mismatches.Add("$cmdlet -$param : absent from MAML"); continue } $got = Normalize (($node.description.para | ForEach-Object { $_ }) -join ' ') - # Coarse: the MAML must open with what the markdown says. - $head = if ($want.Length -gt 60) { $want.Substring(0, 60) } else { $want } - if ($got -notlike "*$head*") { - $mismatches.Add("$cmdlet -$param : markdown says '$head...' MAML says '$($got.Substring(0,[Math]::Min(60,$got.Length)))...'") + if ($got -ne $want) { + $mismatches.Add("$cmdlet -$param : $(Divergence $want $got)") } } } ($mismatches -join "`n") | Should -BeNullOrEmpty -Because 'run ./build.ps1 to regenerate the compiled help' } + + It 'shows every parameter of the cmdlet in its SYNTAX block' { + # The checks above compare sections and parameter descriptions, and a parameter can have + # a section of its own while the SYNTAX block does not list it - which is the block a + # reader looks at first, and the only place that says which parameters go together. + # -BatchChunkConcurrency had a full section and a range the cmdlet enforces, and was + # missing from the line above it. + # + # -ProgressAction is off both sides. PowerShell 7.4 made it a common parameter, so + # Get-Command reports it among those, while platyPS 0.14.2 does not know it and writes it + # into the block as an ordinary one: comparing it would say every block is wrong. + $repo = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent + Import-Module "$repo/Modules/M365DSC.mgx/M365DSC.mgx.psd1" -Force + $common = [System.Management.Automation.PSCmdlet]::CommonParameters + $mismatches = [System.Collections.Generic.List[string]]::new() + + foreach ($md in Get-ChildItem $script:HelpDir -Filter '*.md') { + $cmdInfo = Get-Command -Name $md.BaseName -ErrorAction Stop + $cmdInfo | Should -Not -BeNullOrEmpty -Because "$($md.BaseName) must resolve for this comparison to mean anything" + + # Each fenced block under ## SYNTAX, keyed by the parameter set the heading above it + # names. A single-set cmdlet has no heading, and Get-Command calls that set + # __AllParameterSets. + $blocks = @{} + $inSyntax = $false + $inFence = $false + $setName = $null + $buffer = $null + foreach ($line in Get-Content $md.FullName) { + if ($line -match '^## SYNTAX\s*$') { $inSyntax = $true; continue } + if ($inSyntax -and $line -match '^## ') { break } + if (-not $inSyntax) { continue } + if ($line -match '^### (.+)$') { + $setName = ($Matches[1] -replace '\s*\(Default\)\s*$', '').Trim() + continue + } + if ($line -match '^```') { + if ($inFence) { + $key = if ($setName) { $setName } else { '__AllParameterSets' } + $blocks[$key] = $buffer -join ' ' + $inFence = $false + $buffer = $null + } + else { $inFence = $true; $buffer = @() } + continue + } + if ($inFence) { $buffer += $line } + } + + if ($blocks.Count -eq 0) { + $mismatches.Add("$($md.BaseName) : no SYNTAX block") + continue + } + + foreach ($key in $blocks.Keys) { + $set = $cmdInfo.ParameterSets | Where-Object { $_.Name -eq $key } + if (-not $set) { + $mismatches.Add("$($md.BaseName) SYNTAX [$key] : the cmdlet has no such parameter set (it has $(($cmdInfo.ParameterSets.Name) -join ', '))") + continue + } + + # A '-' that opens a parameter follows a '[' or a space, which is what keeps the + # one in the cmdlet's own name out of the list. + $shown = @([regex]::Matches($blocks[$key], '(?:^|[\s\[])-([A-Za-z]\w*)') | + ForEach-Object { $_.Groups[1].Value } | + Where-Object { $_ -ne 'ProgressAction' } | Sort-Object -Unique) + $takes = @($set.Parameters.Name | + Where-Object { $_ -notin $common -and $_ -ne 'ProgressAction' } | + Sort-Object -Unique) + + $absent = @($takes | Where-Object { $_ -notin $shown }) + $extra = @($shown | Where-Object { $_ -notin $takes }) + if ($absent) { $mismatches.Add("$($md.BaseName) SYNTAX [$key] : absent $($absent -join ', ')") } + if ($extra) { $mismatches.Add("$($md.BaseName) SYNTAX [$key] : names $($extra -join ', '), which the cmdlet does not take") } + } + } + + ($mismatches -join "`n") | Should -BeNullOrEmpty -Because 'a SYNTAX block should list the parameters its cmdlet takes' + } + + It 'gives every parameter one yaml block naming the type its cmdlet declares' { + # The block under a parameter's heading is the only place the markdown states the type, + # the parameter sets, the default and the pipeline binding, and platyPS builds both the + # compiled parameter entry and the SYNTAX line out of it. A section with no block leaves + # Get-Help reporting no type at all - -ConsistencyLevel had none, and the block that + # belonged to it sat under -ThrottlePriority's, where nothing reads it - and a section + # with two makes the second dead text. The checks above compare prose, so neither showed. + # + # Type: is the .NET type's short name, what Type.Name gives: Int32, String, + # SwitchParameter, ActionPreference, Hashtable, and an array keeps its brackets, String[]. + # Never the namespace-qualified name. + $repo = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent + Import-Module "$repo/Modules/M365DSC.mgx/M365DSC.mgx.psd1" -Force + $mismatches = [System.Collections.Generic.List[string]]::new() + + foreach ($md in Get-ChildItem $script:HelpDir -Filter '*.md') { + $cmdInfo = Get-Command -Name $md.BaseName -ErrorAction Stop + $lines = Get-Content $md.FullName + + for ($i = 0; $i -lt $lines.Count; $i++) { + if ($lines[$i] -notmatch '^### -(\w+)$') { continue } + $param = $Matches[1] + + # Only a ```yaml fence opens one. A description may carry a fence of its own - + # the generator eats the asterisks of a literal written into prose - and its + # closing fence is bare, so counting openings keeps the two apart. + $blocks = 0 + $inYaml = $false + $types = [System.Collections.Generic.List[string]]::new() + for ($j = $i + 1; $j -lt $lines.Count; $j++) { + if ($lines[$j] -match '^### ' -or $lines[$j] -match '^## ') { break } + if ($lines[$j] -match '^```yaml\s*$') { $blocks++; $inYaml = $true; continue } + if ($inYaml -and $lines[$j] -match '^```\s*$') { $inYaml = $false; continue } + if ($inYaml -and $lines[$j] -match '^Type:\s*(.+)$') { $types.Add($Matches[1].Trim()) } + } + + if ($blocks -ne 1) { + $mismatches.Add("$($md.BaseName) -$param : $blocks yaml blocks under the heading, not 1") + continue + } + if (-not $cmdInfo.Parameters.ContainsKey($param)) { + $mismatches.Add("$($md.BaseName) -$param : the cmdlet takes no such parameter") + continue + } + + $declared = $cmdInfo.Parameters[$param].ParameterType.Name + if ($types.Count -ne 1) { + $mismatches.Add("$($md.BaseName) -$param : the yaml block names no Type; the cmdlet declares $declared") + } + elseif ($types[0] -ne $declared) { + $mismatches.Add("$($md.BaseName) -$param : the yaml block says Type: $($types[0]), the cmdlet declares $declared") + } + } + } + + ($mismatches -join "`n") | Should -BeNullOrEmpty -Because "a parameter's yaml block should name the type its cmdlet declares" + } } Describe 'Documented output types match the cmdlets' { diff --git a/tests/benchmarks/06-fault-gauntlet.ps1 b/tests/benchmarks/06-fault-gauntlet.ps1 index 3e17847..ba02c14 100644 --- a/tests/benchmarks/06-fault-gauntlet.ps1 +++ b/tests/benchmarks/06-fault-gauntlet.ps1 @@ -8,6 +8,10 @@ # 4. mgx - ids | Invoke-MgxRequest '/users/{id}' (fan-out, full resilience stack) # The server's /reset is called between contenders so each faces the identical schedule. # Fault profile: 15% of ids throttle once (429, Retry-After 1s); 3% fail twice with 503. +# +# The server also takes a programmable fault plan over POST /plan. 06 posts none, so what +# it measures is the default schedule above and nothing else; a scenario wanting a plan +# gets its own script rather than growing this one. param( [int] $N = 1000, [int] $Port = 8787 @@ -23,34 +27,14 @@ Import-Module Microsoft.Graph.Users # claiming no pipeline input. Real Graph ids are strings, so this matches actual usage. $ids = @(1..$N | ForEach-Object { "$_" }) -# Find a free port (a crashed prior run can leave a zombie listener behind) -$Port = ($Port..($Port + 20)) | Where-Object { - -not (Test-Connection -TargetName localhost -TcpPort $_ -TimeoutSeconds 1 -Quiet) -} | Select-Object -First 1 -if (-not $Port) { throw 'no free port found in range' } -$base = "http://localhost:$Port" - -# --- Start mock server as a child process --- -$server = Start-Process pwsh -PassThru ` - -ArgumentList '-NoProfile', '-File', (Join-Path $PSScriptRoot 'mock-graph-server.ps1'), '-Port', $Port +# --- Start mock server as a child process, on a port nothing else is holding --- +$server = Start-BenchMockServer -Port $Port +$base = $server.BaseUrl try { - $up = $false - foreach ($i in 1..50) { - try { $null = Invoke-RestMethod "$base/ping" -TimeoutSec 1; $up = $true; break } catch { Start-Sleep -Milliseconds 200 } - } - if (-not $up) { throw "mock server did not come up on port $Port" } - Write-Host "mock server up (pid $($server.Id))" - # --- Point the whole Graph stack at the mock --- - if (-not (Get-MgEnvironment -Name MgxBench -ErrorAction SilentlyContinue)) { - Add-MgEnvironment -Name MgxBench -GraphEndpoint $base -AzureADEndpoint 'https://login.microsoftonline.com' | Out-Null - } - Disconnect-MgGraph -ErrorAction SilentlyContinue | Out-Null - Connect-MgGraph -Environment MgxBench ` - -AccessToken (ConvertTo-SecureString 'mock-token-not-validated' -AsPlainText -Force) -NoWelcome - # -AccessToken auth leaves AuthContext.TenantId empty, and Mgx's auth fingerprint - # treats an empty TenantId as "not connected" (1.0.4 identity fix). Give it a value. - [Microsoft.Graph.PowerShell.Authentication.GraphSession]::Instance.AuthContext.TenantId = 'gauntlet-mock-tenant' + # This run's port, every run: the environment is written into the user profile, so one + # left standing from a previous run names a port nothing is listening on now. + Connect-BenchMockGraph -GraphEndpoint $base function Reset-Server { $null = Invoke-RestMethod "$base/reset" } function Get-ServerStats { Invoke-RestMethod "$base/stats" } @@ -121,5 +105,6 @@ try { } finally { Disconnect-MgGraph -ErrorAction SilentlyContinue | Out-Null - if ($server -and -not $server.HasExited) { $server.Kill() } + Remove-BenchGraphEnvironment + Stop-BenchMockServer $server } diff --git a/tests/benchmarks/10-pacing-under-real-throttling.ps1 b/tests/benchmarks/10-pacing-under-real-throttling.ps1 index e323a8a..5e98a5f 100644 --- a/tests/benchmarks/10-pacing-under-real-throttling.ps1 +++ b/tests/benchmarks/10-pacing-under-real-throttling.ps1 @@ -5,47 +5,281 @@ # independent of send rate and backing off has no upside by construction. Real Graph throttles # HARDER the faster you push, which is the whole premise of pacing. # -# This drives the SAME read workload twice against a real tenant - once paced (2.1 default), -# once with -NoAdaptivePacing - hard enough to cross the resource-unit budget and provoke -# genuine 429s. Reads only: throttling is not write-specific, so this needs no write consent. +# The first version of this benchmark did not answer it either, and looked like it had. It drove +# 8,000 reads at concurrency 128, spent 8,009 RU at ~180 RU/s, and recorded zero throttle retries +# in BOTH arms. Benchmark 13 explains why: this tenant hands out ~23,000 RU of burst allowance +# before the first refusal and keeps serving ~730 RU/s while refusing, so that run was a third of +# the burst at a quarter of the rate. It could not have been throttled. A clean result there says +# nothing about pacing - it says the workload was small. # -# Each round runs in a FRESH pwsh process because the adapted rate is static process state, -# with a cooldown between so the second round does not inherit a drained budget. +# So this version states its admissibility, takes the thresholds from 13's recorded result rather +# than a constant someone has to remember to update, and records whether the run actually entered +# the throttled regime. If the unpaced arm was never refused there was nothing for pacing to +# avoid, and the comparison is reported as inconclusive rather than as a win for whichever arm +# happened to finish first. +# +# Reaching the regime needs more in flight than one pipeline holds: Invoke-MgxRequest caps +# -Concurrency at 128, and 128 in flight measured ~180 req/s - a quarter of the rate this tenant +# keeps serving at while refusing. Rate is the lever, and volume cannot substitute for it. The +# work is therefore split across N pipelines in parallel runspaces of ONE process. The pacer, the +# telemetry collector and the HTTP client are all process-wide statics, so those runspaces share +# a single pacer - which is the point: one workload bucket, N producers, the shape a real fan-out +# has. The roof is the transport's: 20 pooled connections multiplexing HTTP/2 streams hold +# roughly 2,000 requests in flight, and the 12 x 128 default sits under it. +# +# The ids handed to a pipeline have to be distinct. Invoke-MgxRequest deduplicates the ids it is +# piped, by design, so a slice that repeats an id issues one request for it - and a -Count above +# what the pool can cover was inert: the arm reported the volume it was asked for and issued a +# fraction of it. Each pipeline now draws IdsPerPipeline distinct ids off the shared pool, from a +# rotated start so the pipelines are not all on the same object at once, and the arm refuses to +# run if any slice holds a duplicate. Requests issued is then the number of ids handed out, and a +# -Count above Pipelines x IdPool is reported as clamped rather than quietly dropped. +# +# -Plan prints that shape and the admissibility arithmetic against 13's thresholds and exits +# without connecting, which is how a parameter set is checked before spending an hour on it. +# +# Requests are /users/{id}?$select=id, 1 RU each, the same shape 13 calibrated the thresholds +# with, so request counts and unit counts are the same number. +# +# Reads only: throttling is not write-specific, so this needs no write consent. +# +# CmdletBinding so an argument this script does not define is refused rather than collected into +# $args and ignored: a run that silently drops the parameter it was given is the same class of +# wrong answer as one that silently drops the requests it was given. +[CmdletBinding()] param( [ValidateSet('both', 'paced', 'unpaced')] [string] $Mode = 'both', - [int] $Count = 20000, - [int] $Concurrency = 60, - [int] $CooldownSeconds = 300 + # 12 x 5,000: 60,000 units at a projected ~2,160 RU/s, which is ~10s inside the seeded + # tenant's 23,000 RU allowance and ~51s past it at the 731 RU/s it serves while refusing. + [ValidateRange(1, [int]::MaxValue)] [int] $Count = 60000, + [ValidateRange(1, 512)] [int] $Pipelines = 12, + [ValidateRange(1, 128)] [int] $Concurrency = 128, + [ValidateRange(1, [int]::MaxValue)] [int] $IdPool = 5000, + [int] $CooldownSeconds = 300, + [switch] $Plan ) . "$PSScriptRoot/common.ps1" +function Get-ThrottleThreshold { + <# + The bar this benchmark has to clear, measured rather than assumed. Benchmark 13 holds a + fixed send rate until the tenant refuses and records both numbers that matter: the burst + allowance (units served before the first 429) and the rate the tenant keeps serving at + once it is refusing. Reading them back here means a tenant with a different budget + recalibrates by running 13, not by editing a constant in this file. + + Without a recorded 13, fall back to the documented 8,000 RU per 10s for tenants above + 500 users - and say which one is in force, because a threshold that came from a document + and one that came from this tenant deserve different confidence. + #> + $file = Join-Path $PSScriptRoot 'results/13-resource-unit-rate.json' + if (Test-Path $file) { + $rounds = @(Get-Content $file -Raw | ConvertFrom-Json | ForEach-Object { + $_.Result.PSObject.Properties | Where-Object { $_.Name -match '^rate\d+$' } | + ForEach-Object { $_.Value } + }) + $refused = @($rounds | Where-Object { $null -ne $_.First429Seconds -and $_.ServedRateAfter429 }) + if ($refused) { + return @{ + BurstRu = [int](($refused | Measure-Object RuBeforeFirst429 -Maximum).Maximum) + Sustained= [int](($refused | Measure-Object ServedRateAfter429 -Minimum).Minimum) + Source = 'benchmark 13, this tenant' + } + } + } + @{ BurstRu = 8000; Sustained = 800; Source = 'documented default (8,000 RU / 10s)' } +} + +$threshold = Get-ThrottleThreshold +$shape = Get-BenchPacingLoadShape -Count $Count -Pipelines $Pipelines -Concurrency $Concurrency ` + -IdPool $IdPool -BurstRu $threshold.BurstRu -SustainedRuPerSecond $threshold.Sustained + +function Show-LoadShape { + <# + What each arm will issue, and whether that can reach the regime - printed before + anything connects, because the answer is arithmetic and does not need a tenant. + #> + Write-Host ("throttling threshold from {0}: burst {1} RU, sustained {2} RU/s" -f ` + $threshold.Source, $threshold.BurstRu, $threshold.Sustained) + Write-Host '=== LOAD SHAPE (per arm) ===' + Write-Host (" distinct ids per pipeline {0,10} (pool of {1}, shared across pipelines)" -f ` + $shape.IdsPerPipeline, $shape.IdPool) + Write-Host (" pipelines {0,10} x concurrency {1} = {2} requests in flight" -f ` + $shape.Pipelines, $shape.Concurrency, $shape.InFlight) + Write-Host (" requests issued {0,10} = resource units (1 RU each)" -f $shape.TotalRequests) + if ($shape.Clamped) { + Write-Host (" CLAMPED: -Count {0} exceeds {1} pipelines x {2} pool ids; the arm issues {3}." -f ` + $shape.Requested, $shape.Pipelines, $shape.IdPool, $shape.TotalRequests) -ForegroundColor Yellow + } + Write-Host (" projected draw {0,10} RU/s (from 180 RU/s per pipeline at 128)" -f ` + $shape.ExpectedRuPerSecond) + Write-Host (" time inside the allowance {0,10}s then {1}s past it at {2} RU/s" -f ` + $shape.BurstSeconds, $shape.ThrottledSeconds, $shape.SustainedRuPerSecond) + Write-Host '' + if ($shape.Admissible) { + Write-Host ("ADMISSIBLE: {0} RU/s over the {1} RU/s ceiling, and {2} of {3} estimated seconds spent past the allowance." -f ` + $shape.ExpectedRuPerSecond, $shape.SustainedRuPerSecond, $shape.ThrottledSeconds, $shape.EstimatedSeconds) + } + else { + Write-Host 'INADMISSIBLE: this parameter set cannot reach the throttled regime.' -ForegroundColor Yellow + foreach ($reason in $shape.Reasons) { Write-Host " $reason" -ForegroundColor Yellow } + } +} + +if ($Plan) { + Show-LoadShape + Write-Host '' + Write-Host '-Plan: nothing was run and no tenant was contacted.' + return +} + if ($Mode -eq 'both') { + Show-LoadShape + Write-Host '' + + $resultFile = Join-Path $PSScriptRoot 'results/10-pacing-under-real-throttling.json' + $arms = [ordered]@{} + # The arms' own entries, kept whole beside the figures read out of them: this process starts + # the two rounds and reads their files, and never imports the local module or connects to + # anything, so the identity of the runs being compared is only in what they recorded. + $armEntries = @() + $roundIndex = 0 foreach ($m in 'paced', 'unpaced') { + $roundIndex++ + if ($roundIndex -gt 1) { + # The bucket refills at roughly the sustained rate, so a full burst allowance is back + # in ~30s. 300s is an order of magnitude over that, which is the margin worth paying + # to be sure arm two is not measuring arm one's drained budget. + Write-Host "cooldown ${CooldownSeconds}s (let the resource-unit budget refill)..." + Start-Sleep -Seconds $CooldownSeconds + } Write-Host "=== round: $m ===" -ForegroundColor Cyan + # Start-Process rather than capturing stdout: each arm runs for minutes, and a round + # whose output only appears once it has finished is a round you cannot tell from a hang. + # The arm's own JSON record carries everything compared below, so nothing needs parsing + # out of the console. + $before = @(if (Test-Path $resultFile) { Get-Content $resultFile -Raw | ConvertFrom-Json }).Count $p = Start-Process pwsh -PassThru -Wait -NoNewWindow -ArgumentList ` - '-NoProfile', '-File', $PSCommandPath, '-Mode', $m, '-Count', $Count, '-Concurrency', $Concurrency + '-NoProfile', '-File', $PSCommandPath, '-Mode', $m, '-Count', $Count, ` + '-Pipelines', $Pipelines, '-Concurrency', $Concurrency, '-IdPool', $IdPool if ($p.ExitCode -ne 0) { throw "round '$m' failed (exit $($p.ExitCode))" } - if ($m -eq 'paced') { - Write-Host "cooldown ${CooldownSeconds}s (let the resource-unit budget refill)..." - Start-Sleep -Seconds $CooldownSeconds + + $entries = @(Get-Content $resultFile -Raw | ConvertFrom-Json) + $entry = @($entries | Select-Object -Skip $before | Where-Object { $_.Result.Mode -eq $m } | Select-Object -Last 1)[0] + if (-not $entry) { throw "round '$m' recorded no result in $resultFile" } + $armEntries += $entry + $r = $entry.Result + $arms[$m] = @{ + Seconds = [math]::Round($r.WallMs / 1000, 1) + Ok = [int]$r.Ok + Failed = [int]$r.Failed + ThrottleRetries = [int]$r.ThrottleRetries + RetryDelayMs = [long]$r.RetryDelayMs + PacingWaitMs = [long]$r.PacingWaitMs + ResourceUnits = [long]$r.ResourceUnits + RuPerSecond = if ($r.WallMs -gt 0) { [math]::Round($r.ResourceUnits / ($r.WallMs / 1000), 1) } else { 0 } + ReachedRegime = [int]$r.ThrottleRetries -gt 0 + } + } + + Write-Host '' + Write-Host ("=== ADAPTIVE PACING AGAINST REAL THROTTLING ({0} reads, {1} x {2} in flight) ===" -f ` + $shape.TotalRequests, $shape.Pipelines, $shape.Concurrency) + Write-Host ('{0,-28} {1,8} {2,8} {3,8} {4,10} {5,12} {6,10}' -f ` + 'Arm', 'Wall', 'Ok', 'Failed', '429 retries', 'RetryDelay', 'RU/s') + foreach ($k in 'paced', 'unpaced') { + $a = $arms[$k] + $label = if ($k -eq 'paced') { 'Adaptive pacing on (default)' } else { '-NoAdaptivePacing' } + Write-Host ('{0,-28} {1,7:F1}s {2,8} {3,8} {4,10} {5,11:F1}s {6,10:F1}' -f ` + $label, $a.Seconds, $a.Ok, $a.Failed, $a.ThrottleRetries, ($a.RetryDelayMs / 1000), $a.RuPerSecond) + } + + # Admissibility, stated per arm rather than inferred from the wall times: what each arm + # actually drew against the rate this tenant was measured to keep serving at, and whether it + # was ever refused. A paced arm that is never refused is the outcome pacing exists to + # produce; an UNPACED arm that is never refused means the load never got there. + Write-Host '' + Write-Host ('=== ADMISSIBILITY (thresholds from {0}) ===' -f $threshold.Source) + foreach ($k in 'paced', 'unpaced') { + $a = $arms[$k] + Write-Host (' {0,-8} {1,8} RU at {2,7:F1} RU/s ({3,5:P0} of the {4} RU/s ceiling), {5} throttle retries - {6}' -f ` + $k, $a.ResourceUnits, $a.RuPerSecond, ($a.RuPerSecond / $threshold.Sustained), + $threshold.Sustained, $a.ThrottleRetries, + $(if ($a.ReachedRegime) { 'reached the throttled regime' } else { 'never refused' })) + } + + # The verdict, and the conditions under which there is not one. An unpaced arm that was never + # refused did not reach the regime this benchmark exists to measure, and the wall times above + # it are then a comparison of two unthrottled runs - which is benchmark 14's question, already + # answered there. Saying so is the difference between no result and a wrong one. + Write-Host '' + $conclusive = $arms.unpaced.ReachedRegime + if (-not $conclusive) { + Write-Host 'INCONCLUSIVE: the unpaced arm was never throttled, so there was nothing for pacing to avoid.' -ForegroundColor Yellow + Write-Host (" It spent {0} RU at {1} RU/s; the tenant serves {2} RU/s while refusing and allows {3} RU of burst." -f ` + $arms.unpaced.ResourceUnits, $arms.unpaced.RuPerSecond, $threshold.Sustained, $threshold.BurstRu) -ForegroundColor Yellow + Write-Host ' Raise -Pipelines (offer a higher rate) and -Count with it, then re-run; -Plan checks a set first.' -ForegroundColor Yellow + } + else { + $delta = $arms.unpaced.Seconds - $arms.paced.Seconds + $verdict = if ($delta -gt 0) { "pacing finished {0:F1}s sooner" -f $delta } + elseif ($delta -lt 0) { "pacing finished {0:F1}s later" -f (-$delta) } + else { 'both arms finished level' } + Write-Host ("Unpaced took {0} throttle retries and {1:F1}s honoring Retry-After; {2}." -f ` + $arms.unpaced.ThrottleRetries, ($arms.unpaced.RetryDelayMs / 1000), $verdict) + if ($arms.unpaced.Failed -gt $arms.paced.Failed) { + Write-Host ("Unpaced also lost {0} request(s) that pacing completed." -f ` + ($arms.unpaced.Failed - $arms.paced.Failed)) } } + + Write-BenchResult -Benchmark '10-pacing-under-real-throttling' ` + -Identity (Get-BenchEntryIdentity -Entries $armEntries) -Result ([pscustomobject]@{ + Count = $Count + Pipelines = $Pipelines + Concurrency = $Concurrency + Threshold = $threshold + # The load the arms were actually asked for and the arithmetic that said it could reach + # the regime, recorded beside the verdict: a comparison is only readable against the + # shape that produced it. + Shape = $shape + Conclusive = $conclusive + Arms = $arms + }) Write-Host 'both rounds complete - see results/10-pacing-under-real-throttling.json' return } +# --- single arm --------------------------------------------------------------------------- + +Show-LoadShape +Write-Host '' + Import-MgxLocal Connect-MgxBenchmark -# A fixed, pre-materialised id list. Bounded work only: no loop that can run away. -Write-Host "collecting $Count ids..." -$ids = Get-BenchUserIds -Count $Count -Write-Host " got $($ids.Count)" +# A bounded id pool, shared: the tenant does not need 60,000 users for the workload to draw +# 60,000 units, because each pipeline may issue the whole pool once. A fixed pre-materialized +# list keeps the work bounded with no loop that can run away. +Write-Host "collecting up to $IdPool ids..." +$pool = Get-BenchUserIds -Count $IdPool +Write-Host " got $($pool.Count) ids" -# The token-bucket limiter would cap us well under the RU budget and we would never reach -# the throttling regime this benchmark exists to measure. Lift it; the pacer (or its absence) -# is the variable under test. +# The pool can come back short - a tenant with fewer users than -IdPool, or fewer seeded ones - +# and the shape is then not the one printed above. Recompute it against what the tenant gave. +if ($pool.Count -lt $IdPool) { + $shape = Get-BenchPacingLoadShape -Count $Count -Pipelines $Pipelines -Concurrency $Concurrency ` + -IdPool $pool.Count -BurstRu $threshold.BurstRu -SustainedRuPerSecond $threshold.Sustained + Write-Host (" pool short of -IdPool $IdPool - the shape this arm will issue:") -ForegroundColor Yellow + Show-LoadShape + Write-Host '' +} + +# The token-bucket limiter would cap us well under the RU budget and we would never reach the +# throttling regime this benchmark exists to measure. Lift it; the pacer (or its absence) is the +# variable under test. Both settings are process-wide statics, so the parallel runspaces below +# inherit them without setting them again. Set-MgxOption -NoRateLimit if ($Mode -eq 'unpaced') { Set-MgxOption -NoAdaptivePacing @@ -54,29 +288,91 @@ if ($Mode -eq 'unpaced') { Write-Host ' adaptive pacing ON (2.1 default)' } +# Distinct ids per pipeline, and the slicer refuses rather than hand back a slice that repeats +# one: Invoke-MgxRequest deduplicates its pipeline ids, so a repeat is a request this arm would +# report and never send - which is how the published figures came to describe a regime the load +# never entered. +$script:slices = Get-BenchPipelineSlice -Pool $pool -Pipelines $Pipelines -IdsPerPipeline $shape.IdsPerPipeline +Write-Host (" $Pipelines pipelines x $($shape.IdsPerPipeline) distinct ids = $($shape.TotalRequests) requests") + +$script:commonPath = Join-Path $PSScriptRoot 'common.ps1' +$script:armConcurrency = $Concurrency +$script:modulePath = (Get-Module Mgx).Path + $result = Measure-BenchPass -Name "reads $Mode" -Script { - $items = @($ids | Invoke-MgxRequest '/users/{id}' -Property id,displayName ` - -Concurrency $Concurrency -ErrorVariable errs -ErrorAction SilentlyContinue) - @{ ok = $items.Count; failed = @($errs).Count } + $tallies = $script:slices | ForEach-Object -Parallel { + # A fresh runspace has no modules, but the assembly - and with it the pacer, the + # telemetry collector and the HTTP client - loads once per PROCESS. Importing here + # attaches this runspace to that shared state rather than creating a second copy, and + # Connect-MgxBenchmark returns immediately because the Graph session is static too; + # each runspace saying so is the confirmation that they share one pacer. + # + # Deliberately not Import-MgxLocal: it imports -Force, and eight runspaces force-loading + # the same binary module at once is a race for no gain. The parent already imported it, + # which fixed the Users-before-Mgx assembly order this only has to preserve. + . $using:commonPath + if (-not (Get-Module Microsoft.Graph.Users)) { Import-Module Microsoft.Graph.Users } + Import-Module $using:modulePath + Connect-MgxBenchmark | Out-Null + $slice = $_ + $items = @($slice | Invoke-MgxRequest '/users/{id}' -Property id ` + -Concurrency $using:armConcurrency -ErrorVariable errs -ErrorAction SilentlyContinue) + [pscustomobject]@{ Ok = $items.Count; Failed = @($errs).Count } + } -ThrottleLimit $Pipelines + + @{ + ok = (@($tallies) | Measure-Object Ok -Sum).Sum + failed = (@($tallies) | Measure-Object Failed -Sum).Sum + } } $t = Get-MgxTelemetry +$wallSeconds = $result.ElapsedMs / 1000 Write-Host '' Write-Host ("=== $Mode ===") -Write-Host (" wall {0,8:F1}s" -f ($result.ElapsedMs / 1000)) +Write-Host (" wall {0,8:F1}s" -f $wallSeconds) Write-Host (" completed {0,8} failed {1}" -f $result.Output.ok, $result.Output.failed) Write-Host (" throttle retries {0,8}" -f $t.ThrottleRetries) -Write-Host (" retry delay {0,8:F1}s (time spent honouring Retry-After)" -f ($t.RetryDelayMs / 1000)) +Write-Host (" retry delay {0,8:F1}s (time spent honoring Retry-After)" -f ($t.RetryDelayMs / 1000)) Write-Host (" pacing wait {0,8:F1}s over {1} activations" -f ($t.AdaptivePacingWaitMs / 1000), $t.AdaptivePacingActivations) -Write-Host (" resource units {0,8} ({1:F1} RU/s)" -f $t.ResourceUnitsConsumed, ($t.ResourceUnitsConsumed / ($result.ElapsedMs / 1000))) +Write-Host (" resource units {0,8} ({1:F1} RU/s)" -f $t.ResourceUnitsConsumed, ($t.ResourceUnitsConsumed / $wallSeconds)) Write-Host (" pacing state {0}" -f $t.PacingState) -# Write-BenchResult attaches the telemetry snapshot itself, including resource units. +# The admissibility check for this arm on its own, in the same terms the plan stated: what it +# drew against the rate the tenant was measured to keep serving at, and whether it was refused. +$drawn = if ($wallSeconds -gt 0) { $t.ResourceUnitsConsumed / $wallSeconds } else { 0 } +Write-Host (" admissibility {0,8:F0} RU/s of the {1} RU/s ceiling, {2} refusal(s) - {3}" -f ` + $drawn, $threshold.Sustained, $t.ThrottleRetries, + $(if ($t.ThrottleRetries -gt 0) { 'reached the throttled regime' } else { 'never refused' })) +Write-Host (" issued {0,8} of {1} planned" -f $result.Output.ok, $shape.TotalRequests) + +if ($Mode -eq 'unpaced' -and $t.ThrottleRetries -eq 0) { + Write-Host '' + Write-Host (" NOTE: no refusal at {0:F0} RU/s over {1} units, against a {2} RU/s ceiling and {3} RU of" -f ` + $drawn, $t.ResourceUnitsConsumed, $threshold.Sustained, $threshold.BurstRu) -ForegroundColor Yellow + Write-Host (" burst. This arm did not reach the throttled regime, so it sets no baseline for the") -ForegroundColor Yellow + Write-Host (" paced arm to beat. Raise -Pipelines; -Plan checks a set without spending a run on it.") -ForegroundColor Yellow +} + +# WallMs is read back by Write-BenchResult for the RU rate: telemetry's own elapsed total is the +# SUM of per-request durations and would understate the budget draw by the concurrency factor. +# ThrottleRetries, RetryDelayMs and ResourceUnits are duplicated out of telemetry into the +# result itself: the parent process reads this record back to compare the arms, and it never +# loads Mgx, so it has no telemetry of its own to read them from. Write-BenchResult -Benchmark '10-pacing-under-real-throttling' -Result ([pscustomobject]@{ - Mode = $Mode - Count = $ids.Count - Concurrency = $Concurrency - WallMs = $result.ElapsedMs - Ok = $result.Output.ok - Failed = $result.Output.failed -}) + Mode = $Mode + Count = $Count + Pipelines = $Pipelines + Concurrency = $Concurrency + # The shape this arm issued and the verdict on whether it reached the regime, so the entry + # is readable on its own rather than only through the summary the parent writes. + Shape = $shape + ReachedRegime = $t.ThrottleRetries -gt 0 + WallMs = $result.ElapsedMs + Ok = $result.Output.ok + Failed = $result.Output.failed + ThrottleRetries = $t.ThrottleRetries + RetryDelayMs = $t.RetryDelayMs + PacingWaitMs = $t.AdaptivePacingWaitMs + ResourceUnits = $t.ResourceUnitsConsumed +}) -WallMs ([long]$result.ElapsedMs) diff --git a/tests/benchmarks/13-resource-unit-rate.ps1 b/tests/benchmarks/13-resource-unit-rate.ps1 index 780b0c8..a8267ed 100644 --- a/tests/benchmarks/13-resource-unit-rate.ps1 +++ b/tests/benchmarks/13-resource-unit-rate.ps1 @@ -24,6 +24,21 @@ param( Import-MgxLocal Connect-MgxBenchmark +# What the tenant is DOCUMENTED to allow, read before the first round measures what it actually +# allows. Recorded per round beside the measured numbers: the burst allowance and the sustained +# ceiling only mean something against the budget the tier promises, and the tier is banded on +# the user count, so the same two numbers off two tenants are two different findings. +$tenant = Get-BenchTenantFact +$documented = $tenant.Budget +if ($documented) { + Write-Host ("tenant {0}: {1} users, documented tier {2} ({3}) = {4} RU/s ({5} RU / 10s)" -f ` + $tenant.TenantId, $documented.UserCount, $documented.Tier, $documented.Band, + $documented.RuPerSecond, $documented.RuPer10Seconds) +} +else { + Write-Host 'documented budget unknown (no user count) - rounds record it as null.' +} + # Enough ids to avoid hammering one object; requests cycle through them. $ids = Get-BenchUserIds -Count 5000 Write-Host "using $($ids.Count) user ids; $($Rates.Count) round(s) of ${Seconds}s" @@ -123,6 +138,10 @@ foreach ($rate in $Rates) { RuBeforeFirst429 = $okBeforeFirst429 # 1 RU each, so units == successes ServedRateAfter429 = $served RuHeaderSample = $ruHeaderSample + # The documented tier the two measured numbers above are to be read against. + DocumentedBudgetRuPerSecond = $documented?.RuPerSecond + DocumentedTier = $documented?.Tier + DocumentedTierBand = $documented?.Band } if ($null -eq $first429Seconds) { @@ -140,6 +159,10 @@ foreach ($rate in $Rates) { # --- Report ------------------------------------------------------------------------------- Write-Host '' Write-Host "=== RESOURCE-UNIT RATE (1-RU requests, no retry) ===" +if ($documented) { + Write-Host ("documented for this tenant: tier {0}, {1}, {2} RU/s" -f ` + $documented.Tier, $documented.Band, $documented.RuPerSecond) +} Write-Host ('{0,10} {1,10} {2,9} {3,11} {4,13} {5,15}' -f ` 'Target', 'Achieved', 'Sent', 'First 429', 'RU before it', 'Served after it') foreach ($k in $results.Keys) { diff --git a/tests/benchmarks/14-pacing-cold-cost.ps1 b/tests/benchmarks/14-pacing-cold-cost.ps1 index 43e544a..9c5d0da 100644 --- a/tests/benchmarks/14-pacing-cold-cost.ps1 +++ b/tests/benchmarks/14-pacing-cold-cost.ps1 @@ -54,4 +54,6 @@ $null = $ids | Invoke-MgxRequest -Uri '/users/{id}?$select=id' -Concurrency $Con $sw.Stop() $t = Get-MgxTelemetry -"RESULT|{0}|{1}|{2}" -f $sw.Elapsed.TotalSeconds, $t.Requests, $t.AdaptivePacingWaitMs +# The parent arm parses this back with [double], which is invariant; format it the same way +# or a comma decimal separator reads as a thousands group and the wall clock is 10000x out. +"RESULT|{0}|{1}|{2}" -f $sw.Elapsed.TotalSeconds.ToString([cultureinfo]::InvariantCulture), $t.Requests, $t.AdaptivePacingWaitMs diff --git a/tests/benchmarks/15-fanout-concurrency-scaling.ps1 b/tests/benchmarks/15-fanout-concurrency-scaling.ps1 new file mode 100644 index 0000000..10405c3 --- /dev/null +++ b/tests/benchmarks/15-fanout-concurrency-scaling.ps1 @@ -0,0 +1,233 @@ +# Benchmark 15: does raising -Concurrency actually put more requests on the wire? +# +# Benchmark 10 recorded 8,000 reads at -Concurrency 128 and achieved ~180 req/s, with the pacer's +# per-attempt latency baseline sitting at 632ms. The same tenant at -Concurrency 20 recorded a +# 94ms baseline. If 128 requests were genuinely in flight at 94ms each the run would have moved +# ~1,360 req/s, so something between the fan-out semaphore and the socket is not absorbing them - +# and until that is known, raising Invoke-MgxRequest's documented 1-128 range would buy nothing. +# +# The recorded latency cannot settle it on its own. RecordHttpTime wraps HttpClient.SendAsync +# (ResilientGraphClient.cs:245-248), and SendAsync includes waiting for a free connection out of +# the pool, so queueing time is INSIDE the number. Little's law over it is circular: it returns +# the offered concurrency by construction whether or not the transport is the constraint. +# +# So the question is settled by control rather than inference. Two raw HttpClients, identical to +# the module's transport in every respect except MaxConnectionsPerServer - 20 (what +# TransportDefaults ships) against 512 - offered the same concurrency over the same requests. If +# the pool is the ceiling, 512 pulls away and the gap IS the cost of the default. If they match, +# 20 connections were never the constraint and the service is simply slower under load, which +# makes the current range correct as it stands. +# +# The module ladder above it answers the user-facing half: what a caller gets for each step up +# in -Concurrency. Rate limiting and pacing are both off - either one would be measured instead +# of the transport. +# +# Reads only, ~1 RU each, a few thousand units total: this stays far inside the burst allowance +# benchmark 13 measured, so nothing here provokes throttling. +param( + [int[]] $Ladder = @(8, 20, 40, 64, 128), + [int] $Count = 400, + [int] $IdPool = 400, + [int] $RawOffered = 128, + [int[]] $RawPools = @(20, 512) +) + +. "$PSScriptRoot/common.ps1" +Import-MgxLocal +Connect-MgxBenchmark + +Write-Host "collecting up to $IdPool ids..." +$ids = Get-BenchUserIds -Count $IdPool +Write-Host " got $($ids.Count)" +$work = @(0..($Count - 1) | ForEach-Object { $ids[$_ % $ids.Count] }) + +# --- Part A: what a caller gets per step of -Concurrency ------------------------------------ + +# Both gates off. The token bucket caps request rate outright and the pacer holds a cold +# workload at 4 rps doubling per clean second; either would be the thing measured. +Set-MgxOption -NoRateLimit +Set-MgxOption -NoAdaptivePacing + +# A cold connection pool would charge its handshakes to whichever arm ran first. Warm it, then +# run the ladder up and back down: if the two directions disagree, the ladder is measuring drift +# rather than concurrency, and the report says so instead of averaging the disagreement away. +Write-Host 'warming the connection pool...' +$null = @($work | Select-Object -First 40 | Invoke-MgxRequest '/users/{id}' -Property id -Concurrency 8 ` + -ErrorAction SilentlyContinue) + +function Measure-Ladder { + param([int[]] $Steps, [string] $Direction) + $out = [ordered]@{} + foreach ($c in $Steps) { + $pass = Measure-BenchPass -Name "concurrency $c ($Direction)" -Script { + $items = @($work | Invoke-MgxRequest '/users/{id}' -Property id -Concurrency $c ` + -ErrorVariable errs -ErrorAction SilentlyContinue) + @{ ok = $items.Count; failed = @($errs).Count } + } + $seconds = $pass.ElapsedMs / 1000 + $requests = if ($pass.MgxTelemetry) { $pass.MgxTelemetry.Requests } else { $pass.Output.ok } + $out["$c"] = [pscustomobject]@{ + Concurrency = $c + Seconds = [math]::Round($seconds, 2) + Ok = $pass.Output.ok + Failed = $pass.Output.failed + RequestsPerSecond = if ($seconds -gt 0) { [math]::Round($pass.Output.ok / $seconds, 1) } else { 0 } + # Mean time inside SendAsync, which includes any wait for a free connection. + MeanHttpMs = if ($pass.MgxTelemetry -and $requests -gt 0) { + [math]::Round($pass.MgxTelemetry.HttpMs / $requests, 1) + } else { $null } + } + Write-Host (" concurrency {0,4}: {1,7:F2}s {2,7:F1} req/s mean http {3,7} ms" -f ` + $c, $out["$c"].Seconds, $out["$c"].RequestsPerSecond, $out["$c"].MeanHttpMs) + } + $out +} + +Write-Host '' +Write-Host "=== Part A: Invoke-MgxRequest -Concurrency ladder ($Count reads per step) ===" +$up = Measure-Ladder -Steps $Ladder -Direction 'up' +Write-Host ' --- and back down (ordering check) ---' +$down = Measure-Ladder -Steps ($Ladder[($Ladder.Count - 1)..0]) -Direction 'down' + +# --- Part B: the transport control ---------------------------------------------------------- + +function Invoke-RawFanOut { + <# + The module's transport with one variable changed. Everything here mirrors + MgxCmdletBase's handler and TransportDefaults - HTTP/2 preferred with fallback allowed, + the same connect timeout, lifetime and decompression - so MaxConnectionsPerServer is the + only difference between the two arms, and the difference between their numbers has + exactly one cause. + #> + param( + [Parameter(Mandatory)] [string] $Token, + [Parameter(Mandatory)] [string[]] $Work, + [Parameter(Mandatory)] [int] $Offered, + [Parameter(Mandatory)] [int] $PoolSize, + [int] $DeadlineSeconds = 180 + ) + $handler = [System.Net.Http.SocketsHttpHandler]::new() + $handler.MaxConnectionsPerServer = $PoolSize + $handler.EnableMultipleHttp2Connections = $true + $handler.ConnectTimeout = [TimeSpan]::FromSeconds(10) + $handler.PooledConnectionLifetime = [TimeSpan]::FromMinutes(2) + $handler.AutomaticDecompression = [System.Net.DecompressionMethods]::GZip -bor + [System.Net.DecompressionMethods]::Deflate -bor + [System.Net.DecompressionMethods]::Brotli + $client = [System.Net.Http.HttpClient]::new($handler) + $client.Timeout = [TimeSpan]::FromSeconds(120) + $client.DefaultRequestVersion= [System.Net.HttpVersion]::Version20 + $client.DefaultVersionPolicy = [System.Net.Http.HttpVersionPolicy]::RequestVersionOrLower + $client.DefaultRequestHeaders.Authorization = + [System.Net.Http.Headers.AuthenticationHeaderValue]::new('Bearer', $Token) + + # Warm the pool at this size before timing: connection setup belongs to neither arm. + try { ($client.GetAsync("https://graph.microsoft.com/v1.0/users/$($Work[0])?`$select=id")).GetAwaiter().GetResult().Dispose() } catch { } + + $pending = [System.Collections.Generic.List[System.Threading.Tasks.Task[System.Net.Http.HttpResponseMessage]]]::new() + $sent = 0; $ok = 0; $failed = 0; $version = $null + + # Harvest is inline rather than a scriptblock. Benchmark 13 can use one because its counters + # live at script scope; in here they are function locals, and a scriptblock assigning to + # $script:ok would tick a different variable and report zero completions forever. + # A deadline, because the loop's exit condition is "everything landed". HttpClient.Timeout + # would eventually fault a hung request, but an arm that spends two minutes per straggler is + # not a measurement either - it is a hang with a stopwatch attached. + $sw = [Diagnostics.Stopwatch]::StartNew() + while (($sent -lt $Work.Count -or $pending.Count -gt 0) -and $sw.Elapsed.TotalSeconds -lt $DeadlineSeconds) { + while ($pending.Count -lt $Offered -and $sent -lt $Work.Count) { + $pending.Add($client.GetAsync("https://graph.microsoft.com/v1.0/users/$($Work[$sent])?`$select=id")) + $sent++ + } + + $still = [System.Collections.Generic.List[System.Threading.Tasks.Task[System.Net.Http.HttpResponseMessage]]]::new() + foreach ($t in $pending) { + if (-not $t.IsCompleted) { $still.Add($t); continue } + if ($t.Status -ne 'RanToCompletion') { $failed++; continue } + $r = $t.Result + if ([int]$r.StatusCode -ge 200 -and [int]$r.StatusCode -lt 300) { + $ok++ + if ($null -eq $version) { $version = $r.Version.ToString() } + } else { $failed++ } + $r.Dispose() + } + $pending = $still + + if ($pending.Count -ge $Offered -or $sent -ge $Work.Count) { Start-Sleep -Milliseconds 5 } + } + $sw.Stop() + $abandoned = $pending.Count + ($Work.Count - $sent) + $client.Dispose() + + if ($abandoned -gt 0) { + Write-Host (" WARNING: pool $PoolSize hit the ${DeadlineSeconds}s deadline with $abandoned request(s) unaccounted for;") -ForegroundColor Yellow + Write-Host (' its rate is a floor, not a measurement.') -ForegroundColor Yellow + } + + [pscustomobject]@{ + PoolSize = $PoolSize + Offered = $Offered + Seconds = [math]::Round($sw.Elapsed.TotalSeconds, 2) + Ok = $ok + Failed = $failed + Abandoned = $abandoned + RequestsPerSecond = if ($sw.Elapsed.TotalSeconds -gt 0) { [math]::Round($ok / $sw.Elapsed.TotalSeconds, 1) } else { 0 } + NegotiatedVersion = $version + } +} + +Write-Host '' +Write-Host "=== Part B: same transport, pool size varied (offered concurrency $RawOffered) ===" +$token = Get-BenchAppToken +$raw = [ordered]@{} +foreach ($pool in $RawPools) { + $r = Invoke-RawFanOut -Token $token -Work $work -Offered $RawOffered -PoolSize $pool + $raw["$pool"] = $r + Write-Host (" pool {0,4}: {1,7:F2}s {2,7:F1} req/s ok {3} failed {4} negotiated HTTP/{5}" -f ` + $r.PoolSize, $r.Seconds, $r.RequestsPerSecond, $r.Ok, $r.Failed, $r.NegotiatedVersion) +} + +# --- Verdict --------------------------------------------------------------------------------- + +Write-Host '' +$default = $raw["$($RawPools[0])"] +$wide = $raw["$($RawPools[-1])"] +$ratio = if ($default.RequestsPerSecond -gt 0) { $wide.RequestsPerSecond / $default.RequestsPerSecond } else { 0 } + +if ($default.Abandoned -gt 0 -or $wide.Abandoned -gt 0) { + Write-Host 'INCONCLUSIVE: an arm did not finish inside its deadline, so the two rates are not comparable.' -ForegroundColor Yellow + Write-Host ' Re-run with a smaller -Count, or a longer deadline, before reading anything into the ratio.' -ForegroundColor Yellow +} + +if ($default.NegotiatedVersion -and $default.NegotiatedVersion -notlike '2*') { + Write-Host ("HTTP/{0} was negotiated, not HTTP/2. TransportDefaults sizes the pool at 20 on the" -f $default.NegotiatedVersion) -ForegroundColor Yellow + Write-Host ' assumption of multiplexing, so without it 20 connections is 20 requests in flight.' -ForegroundColor Yellow +} +if ($ratio -ge 1.5) { + Write-Host ("POOL-BOUND: {0} connections served {1:F1} req/s, {2} served {3:F1} - {4:F1}x." -f ` + $default.PoolSize, $default.RequestsPerSecond, $wide.PoolSize, $wide.RequestsPerSecond, $ratio) + Write-Host ' MaxConnectionsPerServer is the fan-out ceiling, not -Concurrency. Its comment sizes it' + Write-Host ' for "fan-out concurrency of 5" - the default, not the 128 the parameter allows.' +} +else { + Write-Host ("NOT POOL-BOUND: {0} connections and {1} are within {2:F1}x of each other." -f ` + $default.PoolSize, $wide.PoolSize, [math]::Max($ratio, 1 / [math]::Max($ratio, 0.001))) + Write-Host ' The service, not the connection pool, is what limits this fan-out. Raising either' + Write-Host ' MaxConnectionsPerServer or the -Concurrency range would buy nothing.' +} + +$first = $up["$($Ladder[0])"] +$last = $up["$($Ladder[-1])"] +$ladderGain = if ($first.RequestsPerSecond -gt 0) { $last.RequestsPerSecond / $first.RequestsPerSecond } else { 0 } +Write-Host ("Ladder: {0:F1}x concurrency ({1} -> {2}) returned {3:F1}x throughput." -f ` + ($Ladder[-1] / $Ladder[0]), $Ladder[0], $Ladder[-1], $ladderGain) + +Write-BenchResult -Benchmark '15-fanout-concurrency-scaling' -Result ([pscustomobject]@{ + LadderUp = $up + LadderDown = $down + RawByPool = $raw + PoolRatio = [math]::Round($ratio, 2) + PoolBound = ($ratio -ge 1.5) + NegotiatedVersion = $default.NegotiatedVersion + Count = $Count +}) diff --git a/tests/benchmarks/16-pathological-gauntlet.ps1 b/tests/benchmarks/16-pathological-gauntlet.ps1 new file mode 100644 index 0000000..14601e3 --- /dev/null +++ b/tests/benchmarks/16-pathological-gauntlet.ps1 @@ -0,0 +1,330 @@ +# Benchmark 16: the pathological class. No tenant required. +# +# 06 measures the mock's default schedule - a fault an entity meets once or twice and then +# never again, which is what ordinary Graph throttling looks like. This one measures the +# faults that do not go away by themselves. Each is posted to the mock as a fault plan +# (POST /plan) before its contenders run, and cleared after: +# +# storm every request 429s with Retry-After for -StormSeconds, then the storm ends +# outage every attempt 503s, for as long as the scenario lasts; the contender has to +# give up, and how long it takes to admit that is the measurement +# abort the collection's body is killed halfway through, connection dropped +# visibility a resource 404s for -VisibilitySeconds after it is first asked for and then +# appears - the read-after-create shape, where nothing in either stack retries +# a 404 and the caller is the one that has to poll +# +# The same four contenders as 06, in 06's order: a naive Invoke-RestMethod loop with no +# retry, the bare Graph SDK, the SDK with Enable-MgxResilience injected, and Mgx. Each runs +# in its own child pwsh under common.ps1's watchdog, which buys two things: a hang is killed +# and recorded as Hung instead of stalling the run (two of these scenarios exist to produce +# one), and "bare SDK" means a process Mgx was never loaded into, so its Telemetry is null +# because there is none rather than because it read zero. The server's /reset runs before +# every contender, so a storm or an outage lands on the same request ordinal for each of +# them, and its /stats delta is recorded per contender beside what the contender itself saw. +# +# The abort scenario asks for one large page rather than a long paged enumeration, because +# the mock speaks http and Mgx refuses to follow an @odata.nextLink that is not https +# (NextLinkValidator, an SSRF guard): a paged enumeration is a fault the Mgx arm would never +# reach, and a row reading "25 of 1000" against contenders that met the fault is worse than +# no row. One page every contender does read, cut in the middle, measures the same thing. +# +# Standalone: not in run.ps1, like 11-15. Next-link cycles are out of scope until the +# detection they exercise exists. +param( + [int] $N = 200, + [int] $Port = 8787, + # Storm and visibility are wall-clock windows, so they bound their scenario whatever the + # item count is. A window of 0 is not a short window, it is no window - the title would + # read "throttled for 0s" or "404s for 0s" and the plan would have nothing to arm. + [ValidateRange(1, [int]::MaxValue)] [int] $StormSeconds = 15, + [ValidateRange(1, [int]::MaxValue)] [int] $VisibilitySeconds = 8, + [int] $VisibilityBudgetSeconds = 40, + [int] $PollMs = 500, + # Under the outage every attempt fails and every failure costs a contender its whole + # retry budget, so that scenario is measured on a handful of items rather than on N. + [int] $OutageItems = 10, + # One page big enough that half of it is a body worth dying in the middle of. + [int] $PageSize = 1000, + [int] $StallSeconds = 120, + # child-mode plumbing (internal) + [ValidateSet('', 'storm', 'outage', 'abort', 'visibility')] [string] $Scenario = '', + [ValidateSet('', 'naive', 'sdk', 'sdkMgx', 'mgx')] [string] $Contender = '', + [string] $Base, + [string] $ResultFile, + [string] $HeartbeatFile +) + +. "$PSScriptRoot/common.ps1" + +# Named for the stack under the call, not for the call: the entity scenarios ask for one +# user at a time and the abort scenario enumerates a collection, and the row is read on +# which stack answered. +$labels = [ordered]@{ + naive = 'naive Invoke-RestMethod' + sdk = 'Graph SDK' + sdkMgx = 'Graph SDK + MgxResilience' + mgx = 'mgx' +} +# The id the visibility scenario reads. Nothing else asks for it, so the plan can govern +# that one path and leave every other route on the default schedule. +$visibilityId = '999' + +# ---------------- child mode: one contender against one scenario ---------------- +if ($Scenario -and $Contender) { + switch ($Contender) { + 'naive' { } # no Graph module at all: it is a REST loop + 'sdk' { Import-Module Microsoft.Graph.Users } + default { Import-MgxLocal } + } + function Beat { Set-Content -Path $HeartbeatFile -Value ([datetime]::UtcNow.Ticks) } + Beat + if ($Contender -ne 'naive') { Connect-BenchMockGraph -GraphEndpoint $Base } + if ($Contender -eq 'sdkMgx') { Enable-MgxResilience } + + # The heartbeat is touched every few items, and a failed item is an item: under the + # outage a contender makes steady progress without producing a single object, and a + # watchdog reading output alone would call that a hang. + $script:steps = 0 + function Step { $script:steps++; if ($script:steps % 5 -eq 0) { Beat } } + + # The record's id and category, not its message: the message carries this run's port and + # would differ between two runs of the same scenario. + function Get-ErrorFacts($record) { + if ($null -eq $record) { return $null } + [ordered]@{ + id = [string]$record.FullyQualifiedErrorId + category = [string]$record.CategoryInfo.Category + exception = $record.Exception.GetType().Name + inner = $(if ($record.Exception.InnerException) { $record.Exception.InnerException.GetType().Name } else { $null }) + } + } + + # One user at a time, the way each contender asks for one. + function Invoke-EntityWork([string[]] $Items) { + $ok = 0; $failed = 0; $first = $null + switch ($Contender) { + 'naive' { + foreach ($id in $Items) { + try { $null = Invoke-RestMethod "$Base/v1.0/users/$id" -TimeoutSec 30; $ok++ } + catch { $failed++; if (-not $first) { $first = $_ } } + Step + } + } + 'mgx' { + # 2>&1 rather than -ErrorVariable: the error records reach the pipeline as + # they happen, so a scenario where every item fails still beats. + $Items | Invoke-MgxRequest '/users/{id}' -ErrorAction Continue 2>&1 | ForEach-Object { + if ($_ -is [System.Management.Automation.ErrorRecord]) { + $failed++; if (-not $first) { $first = $_ } + } + else { $ok++ } + Step + } + } + default { + foreach ($id in $Items) { + $u = Get-MgUser -UserId "$id" -ErrorAction SilentlyContinue -ErrorVariable ev + if ($u) { $ok++ } else { $failed++; if (-not $first -and $ev) { $first = $ev[0] } } + Step + } + } + } + @{ ok = $ok; failed = $failed; error = (Get-ErrorFacts $first) } + } + + # The collection, page by page, until it ends or something cuts it off. + function Invoke-ListWork { + $ok = 0; $err = $null + switch ($Contender) { + 'naive' { + $uri = "$Base/v1.0/users" + try { + while ($uri) { + $page = Invoke-RestMethod $uri -TimeoutSec 30 + $ok += @($page.value).Count + Beat + $uri = $page.'@odata.nextLink' + } + } + catch { $err = $_ } + } + 'mgx' { + try { Invoke-MgxRequest '/users' -All -ErrorAction Stop | ForEach-Object { $ok++; Step } } + catch { $err = $_ } + } + default { + try { Get-MgUser -All -ErrorAction Stop | ForEach-Object { $ok++; Step } } + catch { $err = $_ } + } + } + @{ ok = $ok; failed = $(if ($err) { 1 } else { 0 }); error = (Get-ErrorFacts $err) } + } + + # Read-after-create. No layer in any of these stacks retries a 404, so what is measured + # is the caller's own polling: how many reads, and how long before the resource answers. + function Invoke-VisibilityWork { + $sw = [System.Diagnostics.Stopwatch]::StartNew() + $attempts = 0; $visibleMs = $null; $first = $null + while ($sw.Elapsed.TotalSeconds -lt $VisibilityBudgetSeconds) { + $attempts++ + $got = switch ($Contender) { + 'naive' { + try { $null = Invoke-RestMethod "$Base/v1.0/users/$visibilityId" -TimeoutSec 30; $true } + catch { if (-not $first) { $first = $_ }; $false } + } + 'mgx' { + $r = @($visibilityId | Invoke-MgxRequest '/users/{id}' -ErrorAction SilentlyContinue -ErrorVariable ev) + if (-not $first -and $ev) { $first = $ev[0] } + [bool]$r.Count + } + default { + $u = Get-MgUser -UserId $visibilityId -ErrorAction SilentlyContinue -ErrorVariable ev + if (-not $first -and $ev) { $first = $ev[0] } + [bool]$u + } + } + if ($got) { $visibleMs = $sw.ElapsedMilliseconds; break } + Start-Sleep -Milliseconds $PollMs + Beat + } + # $visibleMs is truthy-tested nowhere below: 0 is a legitimate elapsed reading (visible + # on the very first read) and PowerShell treats 0 as $false, which used to record a + # resource visible at 0ms the same as one that never became visible at all. + @{ ok = $(if ($null -ne $visibleMs) { 1 } else { 0 }); failed = $(if ($null -ne $visibleMs) { 0 } else { 1 }) + attempts = $attempts; visibleAfterMs = $visibleMs; error = (Get-ErrorFacts $first) } + } + + $ids = @(1..$N | ForEach-Object { "$_" }) + $outageIds = @($ids | Select-Object -First $OutageItems) + $pass = switch ($Scenario) { + 'storm' { Measure-BenchPass -Name $labels[$Contender] -Script { Invoke-EntityWork $ids } } + 'outage' { Measure-BenchPass -Name $labels[$Contender] -Script { Invoke-EntityWork $outageIds } } + 'abort' { Measure-BenchPass -Name $labels[$Contender] -Script { Invoke-ListWork } } + 'visibility' { Measure-BenchPass -Name $labels[$Contender] -Script { Invoke-VisibilityWork } } + } + if ($Contender -eq 'sdkMgx') { Disable-MgxResilience } + Disconnect-MgGraph -ErrorAction SilentlyContinue | Out-Null + $pass | ConvertTo-Json -Depth 8 | Set-Content $ResultFile + exit 0 +} + +# ---------------- parent mode ---------------- +Import-MgxLocal + +$server = Start-BenchMockServer -Port $Port +$base = $server.BaseUrl +$tmp = [System.IO.Path]::GetTempPath() +try { + function Set-Plan([hashtable] $Document, [string] $Label) { + $json = ConvertTo-Json -InputObject $Document -Depth 6 -Compress + $posted = @($Document.rules).Count + $response = Invoke-RestMethod "$base/plan" -Method Post -Body $json -ContentType 'application/json' + # The server answers with the rule count it actually armed. A mismatch means this + # scenario is about to run against a plan that is not the one just posted - worth + # stopping for, since every row after it would print as a result against no fault. + if ($response.rules -ne $posted) { + throw "plan for '$Label' posted $posted rule(s) but the server reports $($response.rules)" + } + } + function Reset-Server { $null = Invoke-RestMethod "$base/reset" } + function Get-ServerStats { Invoke-RestMethod "$base/stats" } + # What this contender cost the server, not what the server has counted since it started. + function Get-StatsDelta($Before, $After) { + $delta = [ordered]@{} + foreach ($property in $After.PSObject.Properties) { + $was = $Before.PSObject.Properties[$property.Name] + $delta[$property.Name] = $property.Value - $(if ($was) { $was.Value } else { 0 }) + } + [pscustomobject]$delta + } + function Show-Row([string] $Id, $Row) { + $o = $Row.Output + $telemetry = if ($Row.MgxTelemetry) { + 'req {0}, retries {1}' -f $Row.MgxTelemetry.Requests, + ($Row.MgxTelemetry.ThrottleRetries + $Row.MgxTelemetry.OtherRetries) + } else { 'no telemetry' } + # A crashed contender's ElapsedMs is watchdog poll noise (up to 10s), not a + # measurement - it never got the chance to be one - so the wall-time slot names the + # exit instead of printing a number nobody asked for. + $wall = if ($Row.Crashed) { "EXITED $($Row.ExitCode) without a result" } else { '{0:F1}s' -f ($Row.ElapsedMs / 1000) } + $note = if ($Row.Hung) { 'HUNG (killed by watchdog)' } + elseif ($null -ne $o.visibleAfterMs) { 'visible after {0:F1}s in {1} reads' -f ($o.visibleAfterMs / 1000), $o.attempts } + elseif ($null -ne $o.attempts) { 'never visible in {0} reads' -f $o.attempts } + elseif ($o.error) { '{0} / {1}' -f $o.error.exception, $o.error.id } + else { '' } + Write-Host ("{0,-10} {1,-26} completed {2,5} / failed {3,5} wall {4,8} {5,-20} {6}" -f ` + $Id, $Row.Name, $o.ok, $o.failed, $wall, $telemetry, $note) + } + + $scenarios = [ordered]@{ + storm = @{ + Title = "429 storm: every request throttled for ${StormSeconds}s, Retry-After 2, then clean ($N ids)" + Plan = @{ rules = @(@{ route = '^/(v1\.0|beta)/users'; kind = 'throttleStorm' + scope = 'server'; seconds = $StormSeconds; retryAfter = 2 }) } + } + outage = @{ + Title = "never-recovering 5xx: every attempt 503, no window ($OutageItems ids)" + Plan = @{ rules = @(@{ route = '^/(v1\.0|beta)/users'; kind = 'outage'; retryAfter = 1 }) } + } + abort = @{ + Title = "connection death mid-body: one page of $PageSize users, killed halfway through the body" + Plan = @{ pages = 1; pageSize = $PageSize + rules = @(@{ route = '^/(v1\.0|beta)/users$'; kind = 'abort' + from = 1; count = 1 }) } + } + visibility = @{ + Title = "delayed visibility: /users/$visibilityId 404s for ${VisibilitySeconds}s, then appears" + Plan = @{ rules = @(@{ route = ('^/(v1\.0|beta)/users/{0}$' -f $visibilityId) + kind = 'delayedVisibility'; seconds = $VisibilitySeconds }) } + } + } + + $results = [ordered]@{} + foreach ($id in $scenarios.Keys) { + $definition = $scenarios[$id] + Write-Host '' + Write-Host ("### {0}: {1}" -f $id, $definition.Title) + Set-Plan $definition.Plan $id + $results[$id] = [ordered]@{} + foreach ($arm in $labels.Keys) { + Reset-Server + $before = Get-ServerStats + $row = Invoke-WatchdoggedContender -Name $labels[$arm] -ScriptPath $PSCommandPath ` + -ArgumentList @( + '-Scenario', $id, '-Contender', $arm, '-Base', $base, + '-N', $N, '-OutageItems', $OutageItems, + '-VisibilityBudgetSeconds', $VisibilityBudgetSeconds, '-PollMs', $PollMs, + '-ResultFile', (Join-Path $tmp "bench16-$id-$arm.json"), + '-HeartbeatFile', (Join-Path $tmp "bench16-$id-$arm.beat")) ` + -ResultFile (Join-Path $tmp "bench16-$id-$arm.json") ` + -HeartbeatFile (Join-Path $tmp "bench16-$id-$arm.beat") ` + -StallSeconds $StallSeconds + $row | Add-Member -NotePropertyName ServerStats -NotePropertyValue (Get-StatsDelta $before (Get-ServerStats)) -Force + $results[$id][$arm] = $row + Show-Row $id $row + } + # Cleared between scenarios so the next one's plan is the only thing in force. + Set-Plan @{ rules = @() } "$id (clear)" + } + + Write-Host '' + Write-Host ("=== PATHOLOGICAL GAUNTLET (N={0}) ===" -f $N) + foreach ($id in $results.Keys) { + foreach ($arm in $results[$id].Keys) { Show-Row $id $results[$id][$arm] } + } + Write-Host '' + Write-Host 'server-side truth: the /stats delta each contender left behind' + foreach ($id in $results.Keys) { + foreach ($arm in $results[$id].Keys) { + $s = $results[$id][$arm].ServerStats + Write-Host ("{0,-10} {1,-26} served {2,5} pages {3,4} 429 {4,4} 503 {5,4} abort {6,3} 404 {7,4} errors {8,3}" -f ` + $id, $labels[$arm], $s.served, $s.pagesServed, $s.throttleStorm, $s.outage, + $s.abort, $s.delayedVisibility, $s.serverErrors) + } + } + Write-BenchResult -Benchmark '16-pathological-gauntlet' -Result $results +} +finally { + Remove-BenchGraphEnvironment + Stop-BenchMockServer $server +} diff --git a/tests/benchmarks/17-pathological-environment.ps1 b/tests/benchmarks/17-pathological-environment.ps1 new file mode 100644 index 0000000..490ec90 --- /dev/null +++ b/tests/benchmarks/17-pathological-environment.ps1 @@ -0,0 +1,336 @@ +# Benchmark 17: the pathological faults that act on the session rather than on the socket. +# No tenant required. +# +# 16's faults are properties of the server. These two are properties of the process the +# operation is running in, and they arrive while it is running: +# +# auth a second runspace calls Connect-MgGraph again, with a different access token +# and a different tenant id, while a read is in flight. GraphSession.Instance is +# a process-wide singleton, so the runspace boundary does not protect anything: +# what the operation holds is replaced under it. +# modules a second runspace imports Az.Accounts and then PnP.PowerShell while a read is +# in flight. Module load and assembly resolution are irreversible in a process, +# and Az.Accounts brings its own System.Text.Json. +# +# This records what happens today. Recovering from a session replaced mid-operation is 2.3.0 +# work, so there is no pass or fail here and nothing is asserted: the rows are the before +# picture that work will be measured against. Every row says "recorded". +# +# Contenders: mgx ($ids | Invoke-MgxRequest '/users/{id}', one cmdlet invocation that spans +# the perturbation) and the bare Graph SDK (Get-MgUser per id, a sequence of invocations). +# The naive Invoke-RestMethod loop is not a contender here: it holds no session and imports +# nothing, so neither perturbation has anything to act on. +# +# The read is a paced sequence of entity reads rather than a paged enumeration, for the same +# reason 16's abort scenario is one page: the mock speaks http, and Mgx refuses an +# @odata.nextLink that is not https, so a paged enumeration is not a shape both contenders +# can be measured on. What is needed here is an operation that is still running when the +# perturbation lands, and 200 reads paced at -PaceMs give that for both. +# +# The timeline is the consumer's: an item counts as received when the pipeline handed it +# over, which for the fan-out can lag its own HTTP by the width of the fan. +# +# Standalone: not in run.ps1. +param( + [int] $N = 200, + [int] $Port = 8787, + # 200 reads at 100ms is a twenty-second operation, which leaves room for a perturbation + # to land in the middle of it and for the read to carry on afterwards. + [int] $PaceMs = 100, + # Counted from the read's first item, not from the start of the child. + [int] $PerturbAfterMs = 5000, + [int] $StallSeconds = 180, + # child-mode plumbing (internal) + [ValidateSet('', 'auth', 'modules')] [string] $Scenario = '', + [ValidateSet('', 'sdk', 'mgx')] [string] $Contender = '', + [string] $Base, + [string] $ResultFile, + [string] $HeartbeatFile +) + +. "$PSScriptRoot/common.ps1" + +$labels = [ordered]@{ + sdk = 'Graph SDK' + mgx = 'mgx' +} + +# ---------------- child mode: one contender under one perturbation ---------------- +if ($Scenario -and $Contender) { + if ($Contender -eq 'sdk') { Import-Module Microsoft.Graph.Users } else { Import-MgxLocal } + function Beat { Set-Content -Path $HeartbeatFile -Value ([datetime]::UtcNow.Ticks) } + Beat + Connect-BenchMockGraph -GraphEndpoint $Base + + $script:steps = 0 + function Step { $script:steps++; if ($script:steps % 10 -eq 0) { Beat } } + + function Get-ErrorFacts($record) { + if ($null -eq $record) { return $null } + [ordered]@{ + id = [string]$record.FullyQualifiedErrorId + category = [string]$record.CategoryInfo.Category + exception = $record.Exception.GetType().Name + inner = $(if ($record.Exception.InnerException) { $record.Exception.InnerException.GetType().Name } else { $null }) + } + } + + function Get-SessionFacts { + $context = Get-MgContext + [ordered]@{ + tenantId = [string]$context.TenantId + authType = [string]$context.AuthType + scopes = @($context.Scopes).Count + } + } + + # Every id, one read at a time, paced so the operation is still running when the + # perturbation lands. Each item is stamped as the pipeline hands it over, which is what + # makes "before, during and after" answerable afterwards. + function Invoke-PacedRead { + $received = [System.Collections.Generic.List[object]]::new() + $first = $null + $startedAt = [datetime]::UtcNow + switch ($Contender) { + 'mgx' { + $ids | Invoke-MgxRequest '/users/{id}' -ErrorAction Continue 2>&1 | ForEach-Object { + if ($_ -is [System.Management.Automation.ErrorRecord]) { + if (-not $first) { $first = $_ } + } + else { + $received.Add([pscustomobject]@{ id = [string]$_.id; at = [datetime]::UtcNow }) + if (-not $sync.ReadStarted) { $sync.ReadStarted = [datetime]::UtcNow } + } + Step + Start-Sleep -Milliseconds $PaceMs + } + } + default { + foreach ($id in $ids) { + $user = Get-MgUser -UserId $id -ErrorAction SilentlyContinue -ErrorVariable ev + if ($user) { + $received.Add([pscustomobject]@{ id = [string]$user.Id; at = [datetime]::UtcNow }) + if (-not $sync.ReadStarted) { $sync.ReadStarted = [datetime]::UtcNow } + } + elseif (-not $first -and $ev) { $first = $ev[0] } + Step + Start-Sleep -Milliseconds $PaceMs + } + } + } + @{ startedAt = $startedAt; endedAt = [datetime]::UtcNow; received = $received + error = (Get-ErrorFacts $first) } + } + + # The perturbation runs in a second runspace of THIS process, because that is what makes + # it a perturbation: a child process would share neither the session singleton nor the + # assemblies. It waits for the read's first item and then for -PerturbAfterMs, so it + # lands the same distance into the read whichever contender is running - the two do not + # take the same time to hand over their first item, and a delay counted from anywhere + # else lands before the mgx arm has read anything. + $perturbations = @{ + auth = @' +param($DelayMs, $Token, $TenantId, $AuthModule) +$log = [ordered]@{ action = 'Connect-MgGraph again, different access token and tenant'; steps = @() } +while (-not $sync.ReadStarted) { Start-Sleep -Milliseconds 50 } +Start-Sleep -Milliseconds $DelayMs +$log.startedAt = [datetime]::UtcNow +try { + # By path, not by name. A new runspace has no modules, and the name resolves to the + # newest Microsoft.Graph.Authentication installed - which is not necessarily the one + # this process already loaded, and loading a second version of it throws instead of + # perturbing anything. The path is the one the read is running against. + Import-Module $AuthModule -ErrorAction Stop + Connect-MgGraph -Environment MgxBench -NoWelcome -ErrorAction Stop ` + -AccessToken (ConvertTo-SecureString $Token -AsPlainText -Force) + [Microsoft.Graph.PowerShell.Authentication.GraphSession]::Instance.AuthContext.TenantId = $TenantId + $log.steps += "connected as $TenantId" +} +catch { $log.steps += "connect failed: $($_.Exception.Message)" } +$log.finishedAt = [datetime]::UtcNow +[pscustomobject]$log +'@ + modules = @' +param($DelayMs) +$log = [ordered]@{ action = 'Import-Module Az.Accounts, then PnP.PowerShell'; steps = @() } +while (-not $sync.ReadStarted) { Start-Sleep -Milliseconds 50 } +Start-Sleep -Milliseconds $DelayMs +$log.startedAt = [datetime]::UtcNow +foreach ($module in 'Az.Accounts', 'PnP.PowerShell') { + try { + Import-Module $module -ErrorAction Stop + $log.steps += "$module imported at $([datetime]::UtcNow.ToString('HH:mm:ss.fff'))" + } + catch { $log.steps += "$module failed: $($_.Exception.Message)" } +} +$log.finishedAt = [datetime]::UtcNow +[pscustomobject]$log +'@ + } + + $ids = @(1..$N | ForEach-Object { "$_" }) + $sessionBefore = Get-SessionFacts + + # Shared across both runspaces of this process: the read stamps it when its first item + # arrives, and the perturbation blocks on it. + $sync = [hashtable]::Synchronized(@{ ReadStarted = $null }) + $runspace = [runspacefactory]::CreateRunspace() + $runspace.Open() + $runspace.SessionStateProxy.SetVariable('sync', $sync) + $shell = [powershell]::Create() + $shell.Runspace = $runspace + $null = $shell.AddScript($perturbations[$Scenario]) + $null = if ($Scenario -eq 'auth') { + $shell.AddParameters(@{ DelayMs = $PerturbAfterMs + Token = 'replacement-token-not-validated' + TenantId = 'gauntlet-replacement-tenant' + AuthModule = (Get-Module Microsoft.Graph.Authentication | + Select-Object -First 1).Path }) + } else { + $shell.AddParameters(@{ DelayMs = $PerturbAfterMs }) + } + $handle = $shell.BeginInvoke() + + $pass = Measure-BenchPass -Name $labels[$Contender] -Script { Invoke-PacedRead } + + $perturbation = $null + try { $perturbation = @($shell.EndInvoke($handle))[0] } + catch { $perturbation = [pscustomobject]@{ action = 'perturbation runspace'; steps = @("threw: $($_.Exception.Message)") } } + $runspaceErrors = @($shell.Streams.Error | + ForEach-Object { '{0}: {1}' -f $_.CategoryInfo.Category, $_.Exception.Message } | + Where-Object { $_ -notmatch '^\w+: *$' }) + if ($runspaceErrors.Count) { + $perturbation | Add-Member -NotePropertyName runspaceErrors -NotePropertyValue $runspaceErrors -Force + } + $shell.Dispose(); $runspace.Close(); $runspace.Dispose() + + # What the operation got, against what the mock holds. The ids are the mock's own, so a + # loss and a duplicate are both countable without a tenant. + $out = $pass.Output + $received = @($out.received) + $distinct = @($received.id | Sort-Object -Unique) + $windowStart = $perturbation.startedAt + $windowEnd = $perturbation.finishedAt + $summary = [ordered]@{ + items = $received.Count + distinctItems = $distinct.Count + duplicates = $received.Count - $distinct.Count + lost = $N - $distinct.Count + itemsBefore = @($received | Where-Object { $windowStart -and $_.at -lt $windowStart }).Count + itemsDuring = @($received | Where-Object { $windowStart -and $windowEnd -and $_.at -ge $windowStart -and $_.at -le $windowEnd }).Count + itemsAfter = @($received | Where-Object { $windowEnd -and $_.at -gt $windowEnd }).Count + firstId = $(if ($received.Count) { $received[0].id } else { $null }) + lastId = $(if ($received.Count) { $received[-1].id } else { $null }) + readStartedAt = $out.startedAt.ToString('o') + readEndedAt = $out.endedAt.ToString('o') + error = $out.error + sessionBefore = $sessionBefore + sessionAfter = (Get-SessionFacts) + perturbation = $perturbation + # The proof that this row measured anything: the perturbation has to have started + # after the read did and before it ended. + perturbedInFlight = [bool]($windowStart -and $windowStart -ge $out.startedAt -and $windowStart -le $out.endedAt) + } + # The per-item list is what the summary was derived from; it is not what anyone reads. + $pass.Output = $summary + + # No teardown: the process exits here, and the parent takes the Graph environment out. + # A session this scenario deliberately replaced is not worth disconnecting cleanly, and + # a throw on the way out would lose the row. + $pass | ConvertTo-Json -Depth 8 | Set-Content $ResultFile + exit 0 +} + +# ---------------- parent mode ---------------- +Import-MgxLocal + +$server = Start-BenchMockServer -Port $Port +$base = $server.BaseUrl +$tmp = [System.IO.Path]::GetTempPath() +try { + function Show-Record([string] $Id, $Row) { + $o = $Row.Output + if ($Row.Hung) { + Write-Host ("{0,-8} {1,-11} HUNG (killed by watchdog) after {2:F1}s" -f $Id, $Row.Name, ($Row.ElapsedMs / 1000)) + return + } + if ($Row.Crashed) { + # ElapsedMs here is watchdog poll noise, not a measurement, so it is left out + # rather than printed as though it were a wall time. + Write-Host ("{0,-8} {1,-11} EXITED {2} without a result" -f $Id, $Row.Name, $Row.ExitCode) + return + } + $telemetry = if ($Row.MgxTelemetry) { + 'req {0}, retries {1}' -f $Row.MgxTelemetry.Requests, + ($Row.MgxTelemetry.ThrottleRetries + $Row.MgxTelemetry.OtherRetries) + } else { 'no telemetry' } + Write-Host ("{0,-8} {1,-11} recorded {2,4} items ({3} before / {4} during / {5} after the perturbation), {6} duplicated, {7} lost, wall {8:F1}s, {9}" -f ` + $Id, $Row.Name, $o.items, $o.itemsBefore, $o.itemsDuring, $o.itemsAfter, + $o.duplicates, $o.lost, ($Row.ElapsedMs / 1000), $telemetry) + Write-Host ("{0,-8} {1,-11} perturbation landed in flight: {2} - {3}" -f ` + $Id, $Row.Name, $o.perturbedInFlight, $o.perturbation.action) + foreach ($step in @($o.perturbation.steps)) { + Write-Host ("{0,-8} {1,-11} step: {2}" -f $Id, $Row.Name, $step) + } + foreach ($runspaceError in @($o.perturbation.runspaceErrors | Where-Object { $_ })) { + Write-Host ("{0,-8} {1,-11} runspace error: {2}" -f $Id, $Row.Name, $runspaceError) + } + $surfaced = if ($o.error) { '{0} / {1}' -f $o.error.exception, $o.error.id } else { 'none' } + Write-Host ("{0,-8} {1,-11} error surfaced: {2}; session tenant {3} -> {4}" -f ` + $Id, $Row.Name, $surfaced, $o.sessionBefore.tenantId, $o.sessionAfter.tenantId) + } + + # A rule whose window never opens governs the entity route without ever firing, which + # takes the route off the mock's default 429/503 schedule: the perturbation is then the + # only thing that can go wrong, and a lost item is the perturbation's doing. + $plan = @{ rules = @(@{ route = '^/(v1\.0|beta)/users/'; kind = 'outage'; from = 1000000 }) } + $planResponse = Invoke-RestMethod "$base/plan" -Method Post -ContentType 'application/json' ` + -Body (ConvertTo-Json -InputObject $plan -Depth 6 -Compress) + # The server answers with the rule count it actually armed; a mismatch means the entity + # route is not off the default schedule the way this whole file assumes it is. + $postedRules = @($plan.rules).Count + if ($planResponse.rules -ne $postedRules) { + throw "plan setup posted $postedRules rule(s) but the server reports $($planResponse.rules)" + } + + Write-Host '' + Write-Host 'The naive Invoke-RestMethod loop is not a contender here: it holds no session and' + Write-Host 'imports nothing, so neither perturbation has anything of its to act on.' + + $scenarios = [ordered]@{ + auth = 'a second runspace connects again, with a different token and tenant, mid-read' + modules = 'a second runspace imports Az.Accounts and PnP.PowerShell mid-read' + } + + $results = [ordered]@{} + foreach ($id in $scenarios.Keys) { + Write-Host '' + Write-Host ("### {0}: {1}" -f $id, $scenarios[$id]) + $results[$id] = [ordered]@{} + foreach ($arm in $labels.Keys) { + $null = Invoke-RestMethod "$base/reset" + $row = Invoke-WatchdoggedContender -Name $labels[$arm] -ScriptPath $PSCommandPath ` + -ArgumentList @( + '-Scenario', $id, '-Contender', $arm, '-Base', $base, + '-N', $N, '-PaceMs', $PaceMs, '-PerturbAfterMs', $PerturbAfterMs, + '-ResultFile', (Join-Path $tmp "bench17-$id-$arm.json"), + '-HeartbeatFile', (Join-Path $tmp "bench17-$id-$arm.beat")) ` + -ResultFile (Join-Path $tmp "bench17-$id-$arm.json") ` + -HeartbeatFile (Join-Path $tmp "bench17-$id-$arm.beat") ` + -StallSeconds $StallSeconds + $results[$id][$arm] = $row + Show-Record $id $row + } + } + + Write-Host '' + Write-Host ("=== PATHOLOGICAL ENVIRONMENT (N={0}, paced {1}ms, perturbed at {2}ms) ===" -f $N, $PaceMs, $PerturbAfterMs) + foreach ($id in $results.Keys) { + foreach ($arm in $results[$id].Keys) { Show-Record $id $results[$id][$arm] } + } + Write-BenchResult -Benchmark '17-pathological-environment' -Result $results +} +finally { + Remove-BenchGraphEnvironment + Stop-BenchMockServer $server +} diff --git a/tests/benchmarks/18-spo-latency-clamp.ps1 b/tests/benchmarks/18-spo-latency-clamp.ps1 new file mode 100644 index 0000000..b77d97b --- /dev/null +++ b/tests/benchmarks/18-spo-latency-clamp.ps1 @@ -0,0 +1,427 @@ +# Benchmark 18: does SPO clamp a drive workload by latency instead of by 429? +# +# After heavy traffic a SharePoint-backed drive has been seen to serve the same GET five times +# slower or worse for about an hour, with no 429 anywhere in the run and no throttle header on +# any response. Nothing in the pacer reacts to that: the AIMD cap is armed by a 429, the +# proximity damping by x-ms-throttle-limit-percentage, and a clamp shows neither. The pacer +# already keeps a slow EMA of drive latency and reports the ratio in Get-MgxTelemetry's +# PacingState - telemetry only, deliberately not a pacing input (AdaptiveRequestPacer +# RecordLatency). Whether it should become one is a measurement, and this is the measurement. +# +# So the recorded metric is NOT throughput. It is the latency distribution over the window, +# per five-minute bucket, beside the throttle signals present or absent on each call: the +# resource unit, the throttle-limit percentage, the throttle scope, Retry-After, and whether +# any RateLimit-* header appeared at all. A clamp is a run whose late buckets sit several times +# the early ones on p90 while the throttle-signal columns stay empty. The absence of the +# signals is half the finding, which is why every call records them as absent rather than the +# summary reporting only the ones that showed up. +# +# What is recorded per call, and from where. Two figures, never conflated: +# - ElapsedMs is the HTTP round trip, read from the request tracer's own response line +# ("[Mgx] Response: 200 OK in 41 ms"). This is the latency the clamp is about. +# - CallElapsedMs is the whole Invoke-MgxRequest call, which includes the pacer's proactive +# wait and any Retry-After sleep. PacingWaitMs, from the pacer's verbose line, says how +# much of the gap is mgx's own. Reading a paced call's wall time as SPO latency would +# manufacture a clamp out of the client's own back-off. +# The headers come from the same trace: the tracer keeps Retry-After, x-ms-resource-unit and +# every RateLimit-* and x-ms-throttle-* header, which is exactly the set this measurement +# needs. It is on when $DebugPreference is Continue, which the workers set - the -Debug switch +# itself asks ShouldContinue and would stall a headless run. No raw HttpClient probe is needed +# and none is taken: the subject is mgx under pacing, not a bare socket, and a parallel probe +# would be a different request on a different connection answered under a different queue. +# Get-MgxTelemetry supplies the run-level counters (resource units, throttle retries, pacing +# wait, PacingState with the drive bucket's latency ratio), sampled once per bucket - those are +# process-wide totals, so they describe the window and never one call. +# +# Read-only throughout: GET only, against the drive the operator names. There is no default +# drive, because a default is how a benchmark ends up pointed at production. The load itself is +# heavy by design - $Concurrency workers issuing back-to-back single-item GETs for $Minutes - +# and heavy read traffic against a drive is what the clamp is reported to follow, so it draws +# on the tenant's budget and may leave the drive slow for other callers for the following hour. +# That is the point of the experiment and the reason it wants an uncontended machine, a window +# longer than the hour the clamp is said to last, and a tenant nobody is working in. +# +# Rows checkpoint to results/18-spo-latency-clamp.rows.jsonl as they arrive, flushed per line, +# so a run killed at minute 50 leaves the fifty minutes it collected. The summary is written +# only by a completed window: a partial summary in the results log would be promoted as a +# baseline by the next -RecordBaseline, and half a window is not a distribution. +# +# The short smoke, which must run end to end before a long run is worth starting: +# ./18-spo-latency-clamp.ps1 -DriveId -Minutes 2 -Concurrency 2 ` +# -PrecedingActivity 'smoke, drive idle' +# It writes rows and a summary like the long form, on the same paths. +param( + # The drive to measure. One of the two, and no default: -DriveId b!xxxx, or the same drive + # as a URI in the /drives/ shape. Anything else is refused rather than guessed at. + [string] $DriveId, + [string] $DriveUri, + # Longer than the hour the clamp is reported to hold, so a default run outlasts it. + [int] $Minutes = 75, + [int] $Concurrency = 8, + [int] $BucketMinutes = 5, + # How many item ids to cycle through. One item repeatedly would measure that item's cache. + [int] $ItemPoolSize = 200, + # What the drive had just been through, in the operator's words ("2h export at c=64", + # "idle overnight"). The distribution means nothing without it, so it is recorded verbatim. + [string] $PrecedingActivity = '' +) + +. "$PSScriptRoot/common.ps1" + +# Get-ClampPercentile and the rest of the summary - buckets, totals, refusals-by-kind, the +# headline - live in common.ps1, dot-sourced above: they read the checkpointed rows file back +# rather than trust a live accumulator, and a Pester test calls the identical functions over a +# synthetic rows.jsonl. See the comment above Get-ClampSummary in common.ps1. + +# --- Preconditions: refuse, never measure nothing quietly ----------------------------------- + +if ($Minutes -le 0) { throw "18-spo-latency-clamp: -Minutes is $Minutes; a window has to be at least a minute long." } +if ($Concurrency -le 0) { throw "18-spo-latency-clamp: -Concurrency is $Concurrency; a paced load needs at least one worker." } +if ($BucketMinutes -le 0) { throw "18-spo-latency-clamp: -BucketMinutes is $BucketMinutes; a bucket has to span at least a minute." } +if ($ItemPoolSize -le 0) { throw "18-spo-latency-clamp: -ItemPoolSize is $ItemPoolSize; the load needs at least one item to read." } + +$drivePath = $null +if ($DriveId) { + if ($DriveId -notmatch '^[^/?#\s]+$') { + throw "18-spo-latency-clamp: -DriveId '$DriveId' is not a drive id. Pass the id alone, or pass the whole thing as -DriveUri /drives/." + } + $drivePath = "/drives/$DriveId" +} +if ($DriveUri) { + if ($DriveUri -notmatch '^/drives/[^/?#\s]+$') { + throw "18-spo-latency-clamp: -DriveUri '$DriveUri' is not in the /drives/ shape." + } + if ($drivePath -and $drivePath -ne $DriveUri) { + throw "18-spo-latency-clamp: -DriveId and -DriveUri name different drives ('$drivePath' and '$DriveUri'). Pass one." + } + $drivePath = $DriveUri +} +if (-not $drivePath) { + throw '18-spo-latency-clamp: name the drive to measure - -DriveId or -DriveUri /drives/. There is no default: a default drive is how a read benchmark ends up pointed at production.' +} + +Import-MgxLocal +Connect-MgxBenchmark + +# One probe GET. Every reason this run cannot happen is named here, before anything is +# measured: a drive that does not answer, a tenant with no SharePoint at all, a session whose +# token carries no Files/Sites read. +$probe = $null +$probeSw = [Diagnostics.Stopwatch]::StartNew() +try { + $probe = Invoke-MgxRequest -Uri $drivePath -Property id, driveType, name -ErrorAction Stop +} +catch { + $ex = $_.Exception + $status = 0 + if ($null -ne $ex.StatusCode) { try { $status = [int]$ex.StatusCode } catch { $status = 0 } } + $text = "$($ex.Message)" + + # The documented shape of a tenant with no SharePoint: every sites/drives call answers + # BadRequest, and only the message says why. No permission grant changes it. + if ($text -match 'SPO license') { + throw "18-spo-latency-clamp: the tenant reports no SPO license, so $drivePath cannot be measured on it - '$text'. This benchmark needs a tenant with SharePoint; no role grant substitutes for the license." + } + if ($status -eq 401 -or $status -eq 403) { + throw "18-spo-latency-clamp: the session cannot read $drivePath (HTTP $status - '$text'). Connect with Files.Read.All or Sites.Read.All and try again." + } + throw "18-spo-latency-clamp: $drivePath did not answer a probe GET$(if ($status) { " (HTTP $status)" }) - '$text'. Nothing was measured." +} +$probeSw.Stop() +if (-not $probe -or -not $probe.id) { + throw "18-spo-latency-clamp: $drivePath answered a probe GET without a drive id, so there is no drive there to measure." +} +Write-Host ("drive {0} ({1}) answered a probe in {2} ms" -f $probe.id, ($probe.driveType ?? 'unknown type'), $probeSw.ElapsedMilliseconds) + +# The item pool. One item read over and over measures a cache; a pool the size of a page does +# not. A drive with no children at root is still measurable - /root is a drive item too - so +# an empty listing narrows the load rather than refusing it. +$items = @() +try { + $items = @(Invoke-MgxRequest -Uri "$drivePath/root/children" -Property id -Top $ItemPoolSize -ErrorAction Stop | + ForEach-Object { $_.id } | Where-Object { $_ }) +} +catch { + Write-Warning "listing $drivePath/root/children failed ('$($_.Exception.Message)') - the load falls back to /root." +} +$targets = if ($items.Count -gt 0) { + @($items | Select-Object -First $ItemPoolSize | ForEach-Object { [pscustomobject]@{ Label = 'item'; Uri = "$drivePath/items/$_" } }) +} else { + @([pscustomobject]@{ Label = 'root'; Uri = "$drivePath/root" }) +} +Write-Host ("load reads {0} target(s) in rotation, {1} worker(s), for {2} minute(s)" -f $targets.Count, $Concurrency, $Minutes) +if ($PrecedingActivity) { Write-Host "preceding activity, as stated: $PrecedingActivity" } +else { Write-Warning 'no -PrecedingActivity given; the distribution is uninterpretable without knowing what the drive had just been through.' } + +# --- The load ------------------------------------------------------------------------------- + +$resultsDir = Join-Path $PSScriptRoot 'results' +if (-not (Test-Path $resultsDir)) { New-Item -ItemType Directory -Path $resultsDir -Force | Out-Null } +$rowsPath = Join-Path $resultsDir '18-spo-latency-clamp.rows.jsonl' + +$modulePath = Join-Path $PSScriptRoot '../../Modules/M365DSC.mgx/M365DSC.mgx.psd1' +if (-not (Test-Path $modulePath)) { $modulePath = 'Mgx' } +$startedAt = Get-Date +$deadline = [datetime]::UtcNow.AddMinutes($Minutes) + +# A rolling figure for the live progress line only - the real summary is read back from +# $rowsPath by Get-ClampSummary once the load below finishes, never from this list. +$liveLatencies = [System.Collections.Generic.List[double]]::new() +$attempts = 0 +$pacingSamples = @() +$lastBucket = -1 + +# AutoFlush, append: the file is the checkpoint. A run that dies at minute 50 has minute 50 on +# disk, not a buffer that never reached it. +$writer = [System.IO.StreamWriter]::new($rowsPath, $true, [System.Text.UTF8Encoding]::new($false)) +$writer.AutoFlush = $true +$loadSw = [Diagnostics.Stopwatch]::StartNew() +try { + 1..$Concurrency | ForEach-Object -ThrottleLimit $Concurrency -Parallel { + $worker = $_ + $deadline = $using:deadline + $targets = $using:targets + $modulePath = $using:modulePath + + # Staggered: eight runspaces importing the same binary module in the same millisecond is + # a race nobody needs, and the ramp is seconds against a window of an hour or more. + Start-Sleep -Milliseconds (($worker - 1) * 400) + $ErrorActionPreference = 'Stop' + Import-Module $modulePath -ErrorAction Stop + # The request tracer, which is where the response headers come from, is on whenever + # DebugPreference is not silent. The -Debug switch asks ShouldContinue on a compiled + # cmdlet and would block a headless run; the preference does not. + $DebugPreference = 'Continue' + $VerbosePreference = 'Continue' + + $index = 0 + while ([datetime]::UtcNow -lt $deadline) { + $target = $targets[$index % $targets.Count] + $index++ + $sentAt = (Get-Date).ToString('o') + $sw = [Diagnostics.Stopwatch]::StartNew() + $streams = $null + $failure = $null + try { + $streams = Invoke-MgxRequest -Uri $target.Uri -Property id -ErrorAction Stop 5>&1 4>&1 + } + catch { + $failure = "$($_.Exception.Message)" + } + $sw.Stop() + + # One row per attempt on the wire: a call retried past a 429 met the service twice, + # and the second meeting has its own latency and its own headers. + $traces = @() + $pacingWaitMs = $null + foreach ($record in @($streams)) { + if ($record -is [System.Management.Automation.VerboseRecord]) { + if ("$($record.Message)" -match 'Adaptive pacing: waited (\d+)ms') { + $pacingWaitMs = [int]$Matches[1] + } + continue + } + if ($record -isnot [System.Management.Automation.DebugRecord]) { continue } + $lines = "$($record.Message)" -split "`r?`n" + if ($lines[0] -notmatch '^\[Mgx\] Response: (\d+) .* in (\d+) ms') { continue } + $trace = [ordered]@{ + Status = [int]$Matches[1] + ElapsedMs = [int]$Matches[2] + ResourceUnit = $null + ThrottleLimitPercentage = $null + ThrottleScope = $null + RetryAfter = $null + RateLimitHeader = $false + } + foreach ($line in $lines) { + # The body follows the header block and is not indented; a four-space indent + # is a header and nothing else is. + if ($line -match '^ Body \(') { break } + if ($line -notmatch '^ ([^:]+):\s*(.*)$') { continue } + $name = $Matches[1].Trim(); $value = $Matches[2].Trim() + switch -Regex ($name) { + '^(?i)x-ms-resource-unit$' { $trace.ResourceUnit = $value } + '^(?i)x-ms-throttle-limit-percentage$' { $trace.ThrottleLimitPercentage = $value } + '^(?i)x-ms-throttle-scope$' { $trace.ThrottleScope = $value } + '^(?i)Retry-After$' { $trace.RetryAfter = $value } + '^(?i)RateLimit-' { $trace.RateLimitHeader = $true } + } + } + $traces += , $trace + } + + if ($traces.Count -eq 0) { + # No response reached the tracer: a transport failure, or a refusal raised before + # the request went out. Recorded, never dropped - a window with a hole in it is + # not a distribution, and a silent drop is how one gets a hole. + [pscustomobject]@{ + Timestamp = $sentAt; Worker = $worker; Label = $target.Label; Uri = $target.Uri + Attempt = 1; Attempts = 1; Status = 0; ElapsedMs = $null + CallElapsedMs = [int]$sw.ElapsedMilliseconds; PacingWaitMs = $pacingWaitMs + ResourceUnit = $null; ThrottleLimitPercentage = $null; ThrottleScope = $null + RetryAfter = $null; RateLimitHeader = $false + Error = $(if ($failure) { $failure } else { 'no response traced' }) + } + continue + } + + $n = 0 + foreach ($trace in $traces) { + $n++ + [pscustomobject]@{ + Timestamp = $sentAt; Worker = $worker; Label = $target.Label; Uri = $target.Uri + Attempt = $n; Attempts = $traces.Count; Status = $trace.Status; ElapsedMs = $trace.ElapsedMs + CallElapsedMs = [int]$sw.ElapsedMilliseconds; PacingWaitMs = $pacingWaitMs + ResourceUnit = $trace.ResourceUnit + ThrottleLimitPercentage = $trace.ThrottleLimitPercentage + ThrottleScope = $trace.ThrottleScope + RetryAfter = $trace.RetryAfter + RateLimitHeader = $trace.RateLimitHeader + Error = $(if ($failure -and $n -eq $traces.Count) { $failure } else { $null }) + } + } + } + } | ForEach-Object { + $row = $_ + $writer.WriteLine((ConvertTo-Json -InputObject $row -Compress -Depth 4)) + $attempts++ + if ($null -ne $row.ElapsedMs -and $row.Status -ge 200 -and $row.Status -lt 300) { + $liveLatencies.Add([double]$row.ElapsedMs) + } + + # Clamped with a comparison, not [math]::Max: Max(0, ) binds the int overload + # and rounds the minute away before the floor sees it, which halves the first bucket + # and invents a last one. + $minutesIn = ((Get-Date $row.Timestamp) - $startedAt).TotalMinutes + if ($minutesIn -lt 0) { $minutesIn = 0 } + $index = [int][math]::Floor($minutesIn / $BucketMinutes) + + # Once per bucket, not once per row: these are process-wide totals, and the point of + # sampling them on the clock is to see the drive bucket's latency ratio move. Forward + # only: rows arrive from every worker at once and their timestamps interleave across a + # boundary, so a late row from the previous bucket must not re-sample. The p90 printed + # here is $liveLatencies' rolling figure, for this progress line only - the summary + # Write-BenchResult records below is read back from $rowsPath, not from it. + if ($index -gt $lastBucket) { + $lastBucket = $index + $t = $null + try { $t = Get-MgxTelemetry -ErrorAction Stop } catch { $t = $null } + if ($t) { + $pacingSamples += [pscustomobject]@{ + AtMinute = $index * $BucketMinutes + Requests = $t.Requests + ResourceUnits = $t.ResourceUnitsConsumed + ThrottleRetries = $t.ThrottleRetries + PacingWaitMs = $t.AdaptivePacingWaitMs + PacingActivations = $t.AdaptivePacingActivations + LastThrottlePercentage = $t.LastThrottlePercentage + DrivePacingState = @($t.PacingState -split ';' | Where-Object { $_ -match '^\s*drive:' } | + ForEach-Object { $_.Trim() })[0] + } + } + Write-Host (" minute {0,4}: {1} attempt(s) so far, p90 {2} ms" -f ` + ($index * $BucketMinutes), $attempts, (Get-ClampPercentile -Values $liveLatencies -Percentile 90)) + } + } +} +finally { + $writer.Flush() + $writer.Dispose() + $loadSw.Stop() +} +Write-Host ("rows written to {0} ({1} attempt row(s))" -f $rowsPath, $attempts) + +# --- Report --------------------------------------------------------------------------------- +# +# Read back from $rowsPath rather than carried over from the loop above: the checkpoint is the +# one artifact a killed run still has, and a summary built from anything else can disagree with +# what re-running Get-ClampSummary against the same file would say. It is also what the Pester +# cases over this benchmark call, so the table below and the assertions over it read the same +# computation, not two implementations of it. + +$endedAt = Get-Date +$wallMs = [long]$loadSw.ElapsedMilliseconds + +Write-Host '' +Write-Host '=== SPO LATENCY CLAMP (paced drive reads, read-only) ===' +Write-Host ("drive {0}, {1} worker(s), {2} target(s)" -f $drivePath, $Concurrency, $targets.Count) +# Both clocks, always: a Stopwatch that lost time to a suspended machine only shows up next to +# the wall clock it should agree with, never in either figure alone. +Write-Host ("window measured: {0:F1} minute(s) wall clock (StartedAt to EndedAt), {1:F1} minute(s) stopwatch" -f ` + ($endedAt - $startedAt).TotalMinutes, ($wallMs / 60000.0)) +Write-Host ("preceding activity: {0}" -f $(if ($PrecedingActivity) { $PrecedingActivity } else { 'NOT STATED' })) + +$summary = Get-ClampSummary -RowsPath $rowsPath -StartedAt $startedAt -EndedAt $endedAt ` + -WallMs $wallMs -Minutes $Minutes -BucketMinutes $BucketMinutes + +if ($summary.Refused) { + # Named in $summary.RefusalReason, printed verbatim: rows are already safe on disk (written + # above, independent of this check), but a distribution computed across a gap the machine + # was not awake for is not a distribution, and nothing is appended to the results log for a + # later -RecordBaseline to pick up. + Write-Host $summary.RefusalReason + exit 1 +} + +$bucketRows = $summary.Buckets +$overall = $summary.Overall + +Write-Host ('{0,7} {1,8} {2,8} {3,8} {4,8} {5,6} {6,7} {7,7} {8,7} {9,7} {10,7}' -f ` + 'minute', 'calls', 'p50', 'p90', 'p99', '429s', 'errors', 'pct', 'scope', 'retry', 'ratelim') +foreach ($b in $bucketRows) { + $mark = if ($b.Note) { " <- $($b.Note)" } else { '' } + Write-Host ('{0,7} {1,8} {2,7}ms {3,7}ms {4,7}ms {5,6} {6,7} {7,7} {8,7} {9,7} {10,7}{11}' -f ` + $b.FromMinute, $b.Calls, ($b.P50ElapsedMs ?? '-'), ($b.P90ElapsedMs ?? '-'), ($b.P99ElapsedMs ?? '-'), + $b.Throttled429, $b.Errors, $b.ThrottlePercentageSeen, $b.ThrottleScopeSeen, $b.RetryAfterSeen, $b.RateLimitSeen, $mark) +} +Write-Host '' +Write-Host $summary.Headline.Sentence +if ($overall.Refusals.Count -gt 0) { + Write-Host 'refusals by kind:' + foreach ($kind in $overall.Refusals.Keys) { + Write-Host (" {0}: {1}" -f $kind, $overall.Refusals[$kind]) + } +} +if ($overall.Throttled429 -eq 0 -and $overall.ThrottlePercentageSeen -eq 0 -and $overall.ThrottleScopeSeen -eq 0 -and $overall.RateLimitSeen -eq 0) { + Write-Host 'no 429 and no throttle-signal header anywhere in the window: whatever the latency did, nothing in the response said so.' +} +else { + Write-Host ("throttle signals present: {0} 429(s), {1} percentage header(s), {2} scope header(s), {3} RateLimit-* header(s)" -f ` + $overall.Throttled429, $overall.ThrottlePercentageSeen, $overall.ThrottleScopeSeen, $overall.RateLimitSeen) +} + +$telemetry = $null +try { $telemetry = Get-MgxTelemetry -ErrorAction Stop } catch { $telemetry = $null } +# Labeled, and never the headline above: PacingState is the pacer's one most-recent latency +# sample, which is exactly the figure that used to be read as this benchmark's verdict. +if ($telemetry?.PacingState) { Write-Host "last pacer sample: $($telemetry.PacingState)" } +foreach ($sample in $pacingSamples) { + if ($sample.DrivePacingState) { Write-Host (" minute {0,4}: {1}" -f $sample.AtMinute, $sample.DrivePacingState) } +} + +$result = [ordered]@{ + DriveUri = $drivePath + DriveType = $probe.driveType + Minutes = $Minutes + Concurrency = $Concurrency + BucketMinutes = $BucketMinutes + Targets = $targets.Count + StartedAt = $startedAt.ToString('o') + EndedAt = $endedAt.ToString('o') + PrecedingActivity = $PrecedingActivity + # Recorded, not assumed: an unpaced run measures a different thing and must not be read + # against a paced baseline without the difference being visible in the entry. + AdaptivePacing = -not (Get-MgxOption).NoAdaptivePacing + RowsFile = '18-spo-latency-clamp.rows.jsonl' + ProbeElapsedMs = [int]$probeSw.ElapsedMilliseconds + Overall = $overall + # The verdict, taken from the buckets - see Get-ClampHeadline in common.ps1. PacingState + # below is the pacer's own last snapshot, kept for diagnosis and never read as this. + Headline = $summary.Headline.Sentence + Buckets = $bucketRows + PacingSamples = $pacingSamples + PacingState = $telemetry?.PacingState + WallMs = $wallMs +} + +Write-BenchResult -Benchmark '18-spo-latency-clamp' -Result $result -WallMs $wallMs diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md index 6d04c1a..fe2c1f2 100644 --- a/tests/benchmarks/README.md +++ b/tests/benchmarks/README.md @@ -1,6 +1,6 @@ # Mgx benchmark suite -Reproduces every number in the main README's Benchmarks section. Each script is self-contained; `run.ps1` runs the full suite and regenerates the README tables. +Reproduces every number in the main README's Benchmarks section. Each script is self-contained; `run.ps1` runs the full suite. The main README's tables are written by hand from what the scripts print - nothing here generates them. ## Resource units @@ -20,9 +20,12 @@ there is no session telemetry to read - expected, not a failure. `~/.mgx-bench/app.json` (`{ tenantId, appId, clientSecret }`) — the app needs `User.ReadWrite.All`, `Group.ReadWrite.All`, `Directory.ReadWrite.All`, `AuditLog.Read.All` - A seeded test tenant (~100k users, ~15k groups) for the tenant benchmarks. - `06-fault-gauntlet.ps1` needs **no tenant** — it runs against a local mock. + `06-fault-gauntlet.ps1`, `16-pathological-gauntlet.ps1` and + `17-pathological-environment.ps1` need **no tenant** — they run against a local mock. + 17 also needs `Az.Accounts` and `PnP.PowerShell` installed: importing them mid-read is + the perturbation. -Never point these scripts at a production tenant. Benchmarks 04 and 07 create and delete objects; 07 deliberately provokes throttling. +Never point these scripts at a production tenant. Benchmarks 04 and 07 create and delete objects; 07 and 10 deliberately provoke throttling. ## The benchmarks @@ -41,9 +44,26 @@ Never point these scripts at a production tenant. Benchmarks 04 and 07 create an | 11 | `11-throttle-accuracy.ps1` | Under throttling, retrieval hinges on honoring `Retry-After` | yes | | 12 | `12-delta-replay.ps1` | Delta enumerations repeat objects; replay factor vs ground truth | yes | | 13 | `13-resource-unit-rate.ps1` | Sustained RU ceiling vs burst allowance, at held send rates | yes | +| 14 | `14-pacing-cold-cost.ps1` | What pacing costs a short run that finishes during the ramp | yes | +| 15 | `15-fanout-concurrency-scaling.ps1` | What `-Concurrency` buys, and whether the connection pool is the ceiling | yes | +| 16 | `16-pathological-gauntlet.ps1` | Faults that never clear: storm, outage, death mid-body, delayed visibility | no | +| 17 | `17-pathological-environment.ps1` | What a read does when the session is replaced, or a competing module loaded, under it | no | +| 18 | `18-spo-latency-clamp.ps1` | Whether SPO stretches drive latency with no 429 and no throttle header - latency-as-pacing-input evidence | yes, with SPO | -11-13 are not in `run.ps1`: each deliberately drives the tenant to 429s, so they run standalone -(10 also throttles on purpose, which is why `run.ps1` puts it last). +11-18 are not in `run.ps1`. 11-13 each deliberately drive the tenant to 429s, so they run +standalone (10 also throttles on purpose, which is why `run.ps1` puts it last). 14 and 15 are +cheap and answer one question each; they are read on their own rather than as part of a sweep. +16 and 17 need no tenant, but they are minutes of deliberate pathology and record what each +stack does rather than a number the README quotes, so they are read on their own too. 17 has +no pass or fail at all: recovering from a session replaced mid-operation is 2.3.0 work, and +what it records is the before picture. +18 is read-only and still not cheap: it holds sustained paced reads against one drive for over +an hour. It needs the drive named on the command line - `-DriveId` or `-DriveUri /drives/`, +with no default - a tenant with a SharePoint license, an uncontended machine, and a window +longer than the hour the clamp is reported to last; it refuses to start rather than measure +nothing when any of those is missing. Its per-call rows land in +`results/18-spo-latency-clamp.rows.jsonl` as they are collected, and `-RecordBaseline` promotes +it only when it is named, since `run.ps1` never runs it. ## Methodology @@ -55,4 +75,7 @@ Never point these scripts at a production tenant. Benchmarks 04 and 07 create an - Memory numbers are peak working set sampled at 200ms plus managed-heap delta; streaming claims are measured with streaming consumers (piped, never assigned). - Results append to `results/.json` with Mgx/SDK/PS versions and a - timestamp; the README tables are generated from the latest entries. + timestamp. That directory stays local; `run.ps1 -RecordBaseline` promotes the + latest entries into the committed `baseline.json`, and `-CompareBaseline` + reads a later run against it. The main README's tables are written by hand + from the same entries. diff --git a/tests/benchmarks/baseline.json b/tests/benchmarks/baseline.json new file mode 100644 index 0000000..cbd8d46 --- /dev/null +++ b/tests/benchmarks/baseline.json @@ -0,0 +1,227 @@ +{ + "SchemaVersion": 1, + "GeneratedAt": "2026-09-10T17:11:39.2385100+02:00", + "Scenarios": { + "01": { + "Scenario": "01", + "Script": "01-list-users.ps1", + "Headline": "MgxListMs", + "Discriminator": null, + "Metrics": { + "MgxListMs": { + "Value": 56924.0, + "Unit": "ms", + "Path": "mgx.Median.ElapsedMs" + }, + "MgxTimeToFirstMs": { + "Value": 663.0, + "Unit": "ms", + "Path": "ttfrMgx.Median.ElapsedMs" + }, + "MgxPeakWorkingSetMB": { + "Value": 314.6, + "Unit": "MB", + "Path": "mgx.Median.PeakWorkingSetMB" + } + }, + "MgxVersion": "2.1.4", + "SdkVersion": "2.34.0", + "PSVersion": "7.6.4", + "RecordedAt": "2026-09-10T13:56:49.347805+02:00", + "PromotedAt": "2026-09-10T15:50:28.117482+02:00" + }, + "02": { + "Scenario": "02", + "Script": "02-fanout-lookup.ps1", + "Headline": "MgxFanoutMs", + "Discriminator": null, + "Metrics": { + "MgxFanoutMs": { + "Value": 127303.0, + "Unit": "ms", + "Path": "mgx.Median.ElapsedMs" + }, + "MgxPeakWorkingSetMB": { + "Value": 330.1, + "Unit": "MB", + "Path": "mgx.Median.PeakWorkingSetMB" + } + }, + "MgxVersion": "2.1.4", + "SdkVersion": "2.34.0", + "PSVersion": "7.6.4", + "RecordedAt": "2026-09-10T14:40:49.214531+02:00", + "PromotedAt": "2026-09-10T15:50:28.122807+02:00" + }, + "04": { + "Scenario": "04", + "Script": "04-batch-create.ps1", + "Headline": "MgxBatchCreateMs", + "Discriminator": null, + "Metrics": { + "MgxBatchCreateMs": { + "Value": 503002.0, + "Unit": "ms", + "Path": "mgxCreate.ElapsedMs" + }, + "MgxBatchPatchMs": { + "Value": 49333.0, + "Unit": "ms", + "Path": "batchPatch.ElapsedMs" + }, + "MgxFanoutPatchMs": { + "Value": 28851.0, + "Unit": "ms", + "Path": "fanoutPatch.ElapsedMs" + } + }, + "MgxVersion": "2.1.4", + "SdkVersion": "2.34.0", + "PSVersion": "7.6.4", + "RecordedAt": "2026-09-10T17:06:39.0267+02:00", + "PromotedAt": "2026-09-10T17:11:39.2330620+02:00" + }, + "05": { + "Scenario": "05", + "Script": "05-memory-export.ps1", + "Headline": "MgxExportPeakWorkingSetMB", + "Discriminator": null, + "Metrics": { + "MgxExportPeakWorkingSetMB": { + "Value": 311.6, + "Unit": "MB", + "Path": "mgxExport.PeakWorkingSetMB" + }, + "MgxExportMs": { + "Value": 47303.0, + "Unit": "ms", + "Path": "mgxExport.ElapsedMs" + }, + "MgxExportManagedHeapDeltaMB": { + "Value": 24.8, + "Unit": "MB", + "Path": "mgxExport.ManagedHeapDeltaMB" + } + }, + "MgxVersion": "2.1.4", + "SdkVersion": "2.34.0", + "PSVersion": "7.6.4", + "RecordedAt": "2026-09-10T13:59:39.157651+02:00", + "PromotedAt": "2026-09-10T15:50:28.121054+02:00" + }, + "08": { + "Scenario": "08", + "Script": "08-delta-sync.ps1", + "Headline": "InitialSyncMs", + "Discriminator": null, + "Metrics": { + "InitialSyncMs": { + "Value": 255781.0, + "Unit": "ms", + "Path": "initial.ElapsedMs" + }, + "IncrementalSyncMs": { + "Value": 452.0, + "Unit": "ms", + "Path": "incremental.ElapsedMs" + }, + "SteadySyncMs": { + "Value": 147.0, + "Unit": "ms", + "Path": "steady.ElapsedMs" + }, + "InitialItems": { + "Value": 140261.0, + "Unit": "", + "Path": "initial.Output.items" + } + }, + "MgxVersion": "2.1.4", + "SdkVersion": "2.34.0", + "PSVersion": "7.6.4", + "RecordedAt": "2026-09-10T14:54:54.564289+02:00", + "PromotedAt": "2026-09-10T15:50:28.124234+02:00" + }, + "09": { + "Scenario": "09", + "Script": "09-kill-resume.ps1", + "Headline": "CheckpointedExportMs", + "Discriminator": null, + "Metrics": { + "CheckpointedExportMs": { + "Value": 48406.0, + "Unit": "ms", + "Path": "checkpointed.ElapsedMs" + }, + "BaselineExportMs": { + "Value": 45995.0, + "Unit": "ms", + "Path": "baseline.ElapsedMs" + }, + "ResumeMs": { + "Value": 26640.0, + "Unit": "ms", + "Path": "resume.ElapsedMs" + }, + "CheckpointOverheadPct": { + "Value": 5.2, + "Unit": "%", + "Path": "verification.overheadPct" + }, + "DuplicateIds": { + "Value": 0.0, + "Unit": "", + "Path": "verification.duplicateIds" + } + }, + "MgxVersion": "2.1.4", + "SdkVersion": "2.34.0", + "PSVersion": "7.6.4", + "RecordedAt": "2026-09-10T14:57:23.266194+02:00", + "PromotedAt": "2026-09-10T15:50:28.126139+02:00" + }, + "18": { + "Scenario": "18", + "Script": "18-spo-latency-clamp.ps1", + "Headline": "P90ElapsedMs", + "Discriminator": "live suite content tests about 25 minutes earlier; nothing else of ours on this tenant since", + "Metrics": { + "P90ElapsedMs": { + "Value": 225.0, + "Unit": "ms", + "Path": "Overall.P90ElapsedMs" + }, + "P50ElapsedMs": { + "Value": 171.0, + "Unit": "ms", + "Path": "Overall.P50ElapsedMs" + }, + "P99ElapsedMs": { + "Value": 517.0, + "Unit": "ms", + "Path": "Overall.P99ElapsedMs" + }, + "P90CallElapsedMs": { + "Value": 295.0, + "Unit": "ms", + "Path": "Overall.P90CallElapsedMs" + }, + "ClampRatio": { + "Value": 1.0, + "Unit": "x", + "Path": "Overall.ClampRatio" + }, + "Throttled429": { + "Value": 203.0, + "Unit": "", + "Path": "Overall.Throttled429" + } + }, + "MgxVersion": "2.1.4", + "SdkVersion": "2.34.0", + "PSVersion": "7.6.4", + "RecordedAt": "2026-09-10T14:31:37.737793+02:00", + "PromotedAt": "2026-09-10T15:11:15.980556+02:00" + } + } +} diff --git a/tests/benchmarks/common.ps1 b/tests/benchmarks/common.ps1 index fe31389..a48acfe 100644 --- a/tests/benchmarks/common.ps1 +++ b/tests/benchmarks/common.ps1 @@ -1,6 +1,9 @@ # Shared plumbing for the Mgx benchmark suite. Dot-source from each benchmark script: # . "$PSScriptRoot/common.ps1" -# Auth resolution order: existing Graph session, then $env:MGX_BENCH_APP, then ~/.mgx-bench/app.json. +# Auth resolution order: existing Graph session, then AZURE_* certificate variables, then +# AZURE_* secret variables, then $env:MGX_BENCH_APP / ~/.mgx-bench/app.json. With both a +# certificate path and a secret set, the certificate wins - and a certificate path that +# does not exist throws rather than falling through, so a stale path masks the secret. $ErrorActionPreference = 'Stop' @@ -33,10 +36,20 @@ function Connect-MgxBenchmark { return } + # A secret in the environment, for an app that has no certificate uploaded. Same three + # variables minus the certificate, and like the certificate path it leaves nothing at rest. + if ($env:AZURE_TENANT_ID -and $env:AZURE_CLIENT_ID -and $env:AZURE_CLIENT_SECRET) { + $cred = [pscredential]::new($env:AZURE_CLIENT_ID, + (ConvertTo-SecureString $env:AZURE_CLIENT_SECRET -AsPlainText -Force)) + Connect-MgGraph -TenantId $env:AZURE_TENANT_ID -ClientSecretCredential $cred -NoWelcome + Write-Host "Connected app-only by secret ($($env:AZURE_CLIENT_ID))" + return + } + $credPath = if ($env:MGX_BENCH_APP) { $env:MGX_BENCH_APP } else { Join-Path $HOME '.mgx-bench/app.json' } if (-not (Test-Path $credPath)) { - throw "No Graph session, no AZURE_* certificate variables, and no app credentials at '$credPath'." + throw "No Graph session, no AZURE_* certificate or secret variables, and no app credentials at '$credPath'." } $cfg = Get-Content $credPath -Raw | ConvertFrom-Json $cred = [pscredential]::new($cfg.appId, (ConvertTo-SecureString $cfg.clientSecret -AsPlainText -Force)) @@ -88,6 +101,20 @@ function Get-BenchAppToken { } } + if ($env:AZURE_TENANT_ID -and $env:AZURE_CLIENT_ID -and $env:AZURE_CLIENT_SECRET) { + foreach ($attempt in 1..3) { + try { + $resp = Invoke-RestMethod -Method POST -TimeoutSec 30 ` + -Uri "https://login.microsoftonline.com/$($env:AZURE_TENANT_ID)/oauth2/v2.0/token" ` + -Body @{ grant_type = 'client_credentials'; client_id = $env:AZURE_CLIENT_ID + client_secret = $env:AZURE_CLIENT_SECRET + scope = 'https://graph.microsoft.com/.default' } + return $resp.access_token + } + catch { if ($attempt -eq 3) { throw }; Start-Sleep -Seconds 5 } + } + } + $credPath = if ($env:MGX_BENCH_APP) { $env:MGX_BENCH_APP } else { Join-Path $HOME '.mgx-bench/app.json' } if (-not (Test-Path $credPath)) { throw "No app credentials at '$credPath' (set MGX_BENCH_APP)." } @@ -163,7 +190,90 @@ function Import-MgxLocal { # Local build first (repo checkout), gallery module as fallback. $local = Join-Path $PSScriptRoot '../../Modules/M365DSC.mgx/M365DSC.mgx.psd1' if (Test-Path $local) { Import-Module $local -Force } - else { Import-Module M365DSC.mgx -Force } + else { Import-Module Mgx -Force } +} + +# The gauntlets point the whole Graph stack at a local mock through a named Graph +# environment. Add-MgEnvironment writes that name into the user profile, so an environment +# one run adds outlives it: the next run finds MgxBench already there, keeps whatever +# GraphEndpoint the first run wrote, and aims its SDK contenders at a port this run's mock +# does not hold - which reports no completions, or, when an orphaned listener still holds +# that port, counts answered by a server this run never started. Point it at this run's +# endpoint every time, and take it out again when the run ends. +function Set-BenchGraphEnvironment { + param( + [Parameter(Mandatory)] [string] $GraphEndpoint, + [string] $Name = 'MgxBench' + ) + Remove-BenchGraphEnvironment -Name $Name + Add-MgEnvironment -Name $Name -GraphEndpoint $GraphEndpoint ` + -AzureADEndpoint 'https://login.microsoftonline.com' | Out-Null + $Name +} + +function Remove-BenchGraphEnvironment { + param([string] $Name = 'MgxBench') + # Remove-MgEnvironment raises "Environment not found" as a terminating error that + # -ErrorAction does not downgrade, so the name is looked up first: a finally block has to + # run on a machine where the environment was never added, and on the second call of a run + # that already removed it. + if (Get-MgEnvironment -Name $Name -ErrorAction SilentlyContinue) { + Remove-MgEnvironment -Name $Name | Out-Null + } +} + +# Starts mock-graph-server.ps1 as a child process on the first free port at or above $Port, +# waits for it to answer /ping, and returns the process, the port and the base URL. The port +# is searched rather than assumed: a crashed run can leave a listener behind, and a second +# gauntlet may be running on the same machine. The caller stops it in its finally. +function Start-BenchMockServer { + param( + [int] $Port = 8787, + [int] $Range = 20, + [int] $ReadySeconds = 10 + ) + $free = ($Port..($Port + $Range)) | Where-Object { + -not (Test-Connection -TargetName localhost -TcpPort $_ -TimeoutSeconds 1 -Quiet) + } | Select-Object -First 1 + if (-not $free) { throw 'no free port found in range' } + $base = "http://localhost:$free" + $process = Start-Process pwsh -PassThru ` + -ArgumentList '-NoProfile', '-File', (Join-Path $PSScriptRoot 'mock-graph-server.ps1'), '-Port', $free + $up = $false + foreach ($i in 1..($ReadySeconds * 5)) { + try { $null = Invoke-RestMethod "$base/ping" -TimeoutSec 1; $up = $true; break } + catch { Start-Sleep -Milliseconds 200 } + } + if (-not $up) { + if (-not $process.HasExited) { $process.Kill() } + throw "mock server did not come up on port $free" + } + Write-Host "mock server up (pid $($process.Id))" + [pscustomobject]@{ Process = $process; Port = $free; BaseUrl = $base } +} + +# By pid, and only ever by pid: another gauntlet's server is the same executable under the +# same name, and a pattern kill takes it with this one. +function Stop-BenchMockServer { + param([object] $Server) + if ($Server -and $Server.Process -and -not $Server.Process.HasExited) { $Server.Process.Kill() } +} + +# Points the whole Graph stack at a mock endpoint: the SDK's own cmdlets, and through the +# session the clients Mgx builds. The token is a string the mock does not read. The tenant id +# it is given is read - by Mgx: -AccessToken auth leaves AuthContext.TenantId empty, and the +# auth fingerprint treats an empty TenantId as "not connected" (1.0.4 identity fix). +function Connect-BenchMockGraph { + param( + [Parameter(Mandatory)] [string] $GraphEndpoint, + [string] $Token = 'mock-token-not-validated', + [string] $TenantId = 'gauntlet-mock-tenant' + ) + $null = Set-BenchGraphEnvironment -GraphEndpoint $GraphEndpoint + Disconnect-MgGraph -ErrorAction SilentlyContinue | Out-Null + Connect-MgGraph -Environment MgxBench ` + -AccessToken (ConvertTo-SecureString $Token -AsPlainText -Force) -NoWelcome + [Microsoft.Graph.PowerShell.Authentication.GraphSession]::Instance.AuthContext.TenantId = $TenantId } # Runs one measured pass: telemetry snapshot around $Script, wall time, peak working set @@ -288,14 +398,19 @@ function Invoke-WatchdoggedContender { $sw = [System.Diagnostics.Stopwatch]::StartNew() $p = Start-Process pwsh -PassThru -NoNewWindow ` -ArgumentList (@('-NoProfile', '-File', $ScriptPath) + $ArgumentList) + # Captured now, not read off $p later: Process.StartTime reads back null on Unix once the + # child has exited, and a child that dies before its first Beat (an import or connect + # failure inside the first poll) has usually done exactly that by the time the fallback + # below is needed. + $launchedAt = Get-Date while (-not $p.HasExited) { Start-Sleep -Seconds 10 - $lastBeat = if (Test-Path $HeartbeatFile) { (Get-Item $HeartbeatFile).LastWriteTime } else { $p.StartTime } + $lastBeat = if (Test-Path $HeartbeatFile) { (Get-Item $HeartbeatFile).LastWriteTime } else { $launchedAt } if (((Get-Date) - $lastBeat).TotalSeconds -gt $StallSeconds) { $p.Kill() Write-Host (" WATCHDOG: '{0}' no progress for {1}s - killed after {2:F0}s total" -f $Name, $StallSeconds, $sw.Elapsed.TotalSeconds) return [pscustomobject]@{ - Name = $Name; Hung = $true; ElapsedMs = [long]$sw.ElapsedMilliseconds + Name = $Name; Hung = $true; Crashed = $false; ElapsedMs = [long]$sw.ElapsedMilliseconds Output = [pscustomobject]@{ note = "HUNG - no progress for ${StallSeconds}s, killed by watchdog" } } } @@ -303,14 +418,380 @@ function Invoke-WatchdoggedContender { if (Test-Path $ResultFile) { $r = Get-Content $ResultFile -Raw | ConvertFrom-Json $r | Add-Member -NotePropertyName Hung -NotePropertyValue $false -Force + $r | Add-Member -NotePropertyName Crashed -NotePropertyValue $false -Force return $r } + # Exited on its own, within the poll granularity above, before ever touching + # $HeartbeatFile or $ResultFile - an import or connect failure, not a stall. Hung stays + # false: nothing here was killed for want of progress, so the elapsed time is watchdog + # poll noise (up to 10s) rather than a measurement, and Crashed is what callers should + # branch on instead of reading Hung as a catch-all for "no result". [pscustomobject]@{ - Name = $Name; Hung = $true; ElapsedMs = [long]$sw.ElapsedMilliseconds - Output = [pscustomobject]@{ note = "child exited $($p.ExitCode) without writing a result" } + Name = $Name; Hung = $false; Crashed = $true; ExitCode = $p.ExitCode + ElapsedMs = [long]$sw.ElapsedMilliseconds + Output = [pscustomobject]@{ note = "exited $($p.ExitCode) without a result" } } } +function Get-BenchPacingLoadShape { + <# + .SYNOPSIS + The load a pacing arm will issue, and whether it can reach the throttled regime. + + .DESCRIPTION + Invoke-MgxRequest deduplicates the ids it is piped, by design, so a slice that repeats + an id issues one request for it: the request count of an arm is the count of DISTINCT + ids handed to each pipeline, never the volume asked for. A pipeline can be handed the + shared pool at most once, so an arm can issue at most Pipelines x IdPool requests and + anything above that is clamped - reported, because an arm that believes it issued more + than it did describes a regime it never entered. + + Admissibility has two halves, and a run needs both: + + rate the offered draw must exceed the rate the tenant keeps serving at while it + refuses, or the burst allowance never drains and nothing is ever refused. + Volume cannot buy this: it is set by how much is in flight. + volume the run must then spend long enough past the allowance for the throttled + phase to be at least as long as the unthrottled one, or the comparison is + mostly a comparison of two unthrottled runs. + + The rate here is a projection, from the 180 RU/s that 128 requests in flight measured + against the seeded tenant; the arms report what they actually drew, and the verdict is + taken from that. One process can hold roughly 2,000 requests in flight - the transport + pools 20 connections and multiplexes HTTP/2 streams over them - so a pipeline count + whose product with the concurrency passes that will not offer the rate projected here. + + .PARAMETER BurstRu + Resource units the tenant serves before its first refusal. + + .PARAMETER SustainedRuPerSecond + The rate it keeps serving at once it is refusing. + + .PARAMETER RuPerSecondPerPipeline + Measured draw of one pipeline at concurrency 128. Scaled linearly with -Concurrency. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [ValidateRange(1, [int]::MaxValue)] [int] $Count, + [Parameter(Mandatory)] [ValidateRange(1, 512)] [int] $Pipelines, + [Parameter(Mandatory)] [ValidateRange(1, 128)] [int] $Concurrency, + [Parameter(Mandatory)] [ValidateRange(1, [int]::MaxValue)] [int] $IdPool, + [Parameter(Mandatory)] [ValidateRange(1, [int]::MaxValue)] [int] $BurstRu, + [Parameter(Mandatory)] [ValidateRange(1, [int]::MaxValue)] [int] $SustainedRuPerSecond, + [double] $RuPerSecondPerPipeline = 180 + ) + + $idsPerPipeline = [math]::Min([int][math]::Ceiling($Count / $Pipelines), $IdPool) + $total = $idsPerPipeline * $Pipelines + $inFlight = $Pipelines * $Concurrency + $expectedRate = [math]::Round($Pipelines * $RuPerSecondPerPipeline * ($Concurrency / 128), 0) + + $burstSeconds = [math]::Round($BurstRu / $expectedRate, 1) + $throttledSeconds = [math]::Round([math]::Max(0, $total - $BurstRu) / $SustainedRuPerSecond, 1) + + $rateClears = $expectedRate -gt $SustainedRuPerSecond + $volumeClears = $total -gt $BurstRu -and $throttledSeconds -ge $burstSeconds + + $reasons = @() + if (-not $rateClears) { + $reasons += ("offered draw {0} RU/s does not exceed the {1} RU/s the tenant serves while refusing - raise -Pipelines (volume cannot buy a rate)" -f ` + $expectedRate, $SustainedRuPerSecond) + } + if (-not $volumeClears) { + $reasons += ("{0} requests leaves {1}s in the throttled regime against {2}s inside the {3} RU burst allowance - raise -Count, and -IdPool or -Pipelines with it" -f ` + $total, $throttledSeconds, $burstSeconds, $BurstRu) + } + + [pscustomobject]@{ + Requested = $Count + Pipelines = $Pipelines + Concurrency = $Concurrency + IdPool = $IdPool + IdsPerPipeline = $idsPerPipeline + TotalRequests = $total + Clamped = $total -lt $Count + InFlight = $inFlight + ExpectedRuPerSecond = $expectedRate + BurstRu = $BurstRu + SustainedRuPerSecond = $SustainedRuPerSecond + BurstSeconds = $burstSeconds + ThrottledSeconds = $throttledSeconds + EstimatedSeconds = [math]::Round($burstSeconds + $throttledSeconds, 1) + RateClears = $rateClears + VolumeClears = $volumeClears + Admissible = $rateClears -and $volumeClears + Reasons = $reasons + } +} + +function Get-BenchPipelineSlice { + <# + .SYNOPSIS + The ids each pipeline of a fan-out arm issues: distinct within a pipeline, staggered + across them. + + .DESCRIPTION + Every pipeline may issue the whole shared pool once, and each starts at its own rotated + offset so the pipelines are not all pulling the same object at the same moment. The walk + takes fewer steps than the pool is long, so it never wraps onto its own start and no + slice can hold a repeat - which is the invariant the arm's request count rests on, and + which this checks before handing the slices back rather than trusting the arithmetic. + + .OUTPUTS + One list of ids per pipeline, in pipeline order. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string[]] $Pool, + [Parameter(Mandatory)] [ValidateRange(1, 512)] [int] $Pipelines, + [Parameter(Mandatory)] [ValidateRange(1, [int]::MaxValue)] [int] $IdsPerPipeline + ) + + if ($Pool.Count -eq 0) { throw 'no ids to slice across the pipelines.' } + if ($IdsPerPipeline -gt $Pool.Count) { + throw ("a pipeline cannot issue $IdsPerPipeline distinct ids from a pool of $($Pool.Count); " + + 'Invoke-MgxRequest deduplicates its pipeline ids, so the repeats would never be sent.') + } + + $slices = [System.Collections.Generic.List[object]]::new() + $stride = [math]::Max(1, [int][math]::Floor($Pool.Count / $Pipelines)) + for ($i = 0; $i -lt $Pipelines; $i++) { + $slice = [System.Collections.Generic.List[string]]::new() + $start = ($i * $stride) % $Pool.Count + for ($j = 0; $j -lt $IdsPerPipeline; $j++) { $slice.Add($Pool[($start + $j) % $Pool.Count]) } + + $distinct = @($slice | Sort-Object -Unique).Count + if ($distinct -ne $slice.Count) { + throw ("pipeline $i was given $($slice.Count - $distinct) duplicate id(s) of $($slice.Count); " + + 'the arm would issue fewer requests than it reports.') + } + $slices.Add($slice) + } + , $slices +} + +# --- Which tenant, and which tree, produced a number ----------------------------------------- +# +# Mgx, SDK and PowerShell versions plus a timestamp do not identify a measurement. Two runs +# against two tenants are the same entry with different numbers in it, so a cross-tenant claim +# cannot be audited from results/; and every working tree that has not bumped the version +# carries the same version string, so a slow number cannot be attributed to the code or to the +# tenant after the fact. Everything below answers null rather than throwing when its source is +# absent: the no-tenant benchmarks must still record. + +function Get-BenchDocumentedBudget { + <# + .SYNOPSIS + Graph's documented resource-unit budget for an application+tenant pair, by tenant size. + + .DESCRIPTION + The budget is banded on the tenant's user count rather than fixed: + + S under 50 users 3,500 ResourceUnits / 10s 350 RU/s + M 50 to 500 users 5,000 ResourceUnits / 10s 500 RU/s + L above 500 users 8,000 ResourceUnits / 10s 800 RU/s + + Source: https://learn.microsoft.com/en-us/graph/throttling-limits - "Identity and access + service limits", whose application+tenant pair row carries those three quotas and whose + note reads "The tenant sizes are defined as follows: S - under 50 users, M - between 50 + and 500 users, and L - above 500 users." + + One function, because the denominator a result records its draw against and the tier a + calibration run reports have to be the same number or the two artifacts disagree about + what a tenant is allowed. + + .PARAMETER UserCount + The tenant's user count. Null (unknown) yields null: an assumed tier is worse than none. + #> + [CmdletBinding()] + param([object] $UserCount) + + if ($null -eq $UserCount) { return $null } + $users = [long]$UserCount + if ($users -lt 0) { return $null } + + $tier = if ($users -lt 50) { @{ Tier = 'S'; Band = 'under 50 users'; RuPer10Seconds = 3500 } } + elseif ($users -le 500) { @{ Tier = 'M'; Band = '50 to 500 users'; RuPer10Seconds = 5000 } } + else { @{ Tier = 'L'; Band = 'above 500 users'; RuPer10Seconds = 8000 } } + + [pscustomobject]@{ + Tier = $tier.Tier + Band = $tier.Band + UserCount = $users + RuPer10Seconds = $tier.RuPer10Seconds + RuPerSecond = $tier.RuPer10Seconds / 10 + } +} + +function Get-BenchDirectoryObjectCount { + <# + .SYNOPSIS + The tenant's user count, in one request. + + .DESCRIPTION + $count=true on a $top=1 page carries @odata.count for the whole collection, which is the + cheapest exact count Graph offers - the /users/$count segment needs the same + ConsistencyLevel header and answers in text/plain, which the SDK does not surface. + + Through the SDK's own cmdlet rather than Invoke-MgxRequest: this runs as a result is + being written, sometimes moments after a benchmark that deliberately drove the tenant + into refusing, and it must not sit in mgx's retry pipeline waiting out a Retry-After + that belongs to no measurement. The uri is relative, so it resolves against whatever + endpoint the session was connected to rather than assuming the worldwide cloud. + #> + [CmdletBinding()] + param() + + $response = Invoke-MgGraphRequest -Method GET -OutputType HashTable -ErrorAction Stop ` + -Uri 'v1.0/users?$top=1&$count=true' ` + -Headers @{ ConsistencyLevel = 'eventual' } + $count = $response['@odata.count'] + if ($null -eq $count) { return $null } + [long]$count +} + +function Get-BenchTenantFact { + <# + .SYNOPSIS + Which tenant a benchmark ran against, how big it is, and what it is budgeted at. + + .DESCRIPTION + Computed once per process and cached: every benchmark records at least one result and + several record many, and none of them should pay for the same count twice. + + All three are null when there is no Graph session - the ordinary state of the no-tenant + benchmarks, and of the parent process of a benchmark whose arms connect in children. + A tenant that answers the context but not the count records the tenant and null for the + rest, because a partial fact is still an auditable one. + + An Entra tenant id is a GUID, and a session whose id is not one is not a directory any + of this describes: 06 connects a mock Graph under a made-up tenant name and must stay + off the network, so a non-GUID id records as no tenant rather than as a count request + nobody asked for. + + .PARAMETER Refresh + Recompute rather than answer from the cache. + #> + [CmdletBinding()] + param([switch] $Refresh) + + if ($script:BenchTenantFact -and -not $Refresh) { return $script:BenchTenantFact } + + $fact = [pscustomobject]@{ + TenantId = $null + DirectoryObjectCount = $null + Budget = $null + } + try { + $ctx = Get-MgContext -ErrorAction Stop + $tenantGuid = [guid]::Empty + if ($ctx -and $ctx.TenantId -and [guid]::TryParse($ctx.TenantId, [ref] $tenantGuid)) { + $fact.TenantId = $ctx.TenantId + $fact.DirectoryObjectCount = Get-BenchDirectoryObjectCount + $fact.Budget = Get-BenchDocumentedBudget -UserCount $fact.DirectoryObjectCount + } + } + catch { + # No Microsoft.Graph.Authentication, no session, or a tenant that would not answer the + # count. A recorded null says "not known"; a thrown error would lose the whole result. + Write-Verbose "tenant facts unavailable: $($_.Exception.Message)" + } + $script:BenchTenantFact = $fact + $fact +} + +function Get-BenchCommit { + <# + .SYNOPSIS + The short commit of the tree the suite ran from, with a dirty marker. + + .DESCRIPTION + The marker carries more than the hash does: a number measured over uncommitted edits is + not reproducible from the hash alone, and saying so is the difference between an entry + that can be re-run and one that only looks like it can. Null outside a git checkout. + #> + [CmdletBinding()] + param() + + if ($script:BenchCommitRead) { return $script:BenchCommit } + $script:BenchCommitRead = $true + $script:BenchCommit = $null + try { + $hash = & git -C $PSScriptRoot rev-parse --short HEAD 2>$null + if ($hash) { + $commit = "$hash".Trim() + if (@(& git -C $PSScriptRoot status --porcelain 2>$null).Count -gt 0) { + $commit = "$commit-dirty" + } + $script:BenchCommit = $commit + } + } + catch { + # No git, or not a checkout. Recording nothing is right; guessing is not. + $script:BenchCommit = $null + } + $script:BenchCommit +} + +# The five fields of a recorded entry that say WHERE its numbers came from: which module and +# which SDK ran, against which tenant, how big that tenant is, and what the documentation +# budgets a tenant that size at. Everything else in an entry describes the run itself. +$script:BenchIdentityField = @( + 'MgxVersion' + 'SdkVersion' + 'TenantId' + 'DirectoryObjectCount' + 'DocumentedBudgetRuPerSecond' +) + +# One identity field off one entry. Get-BenchResultValue resolves a path through PSObject +# properties, which is every entry read back out of a results file but not a hashtable a caller +# assembled by hand, so a dictionary is read as one. A blank string is read as no value: an arm +# that recorded an empty tenant did not record a tenant, and pairing that against an arm that +# did is a disagreement, not a name to print with nothing after it. +function Get-BenchIdentityField { + param( + [object] $Node, + [Parameter(Mandatory)] [string] $Name + ) + $value = if ($Node -is [System.Collections.IDictionary]) { $Node[$Name] } + else { Get-BenchResultValue -Node $Node -Path $Name } + if ($value -is [string] -and [string]::IsNullOrWhiteSpace($value)) { return $null } + $value +} + +# Reads that identity off the entries the arms of a comparison recorded, for a caller writing a +# combined entry from a process that did none of the measuring itself. +# +# Arms that disagree on a field have no shared answer to give, so the field is recorded as null +# and the disagreement is named: two arms measured against two tenants, or under two module +# versions, are a comparison of two different things, and the entry has to say so rather than +# pick whichever arm was read first. Values are compared as they print, so a count recorded as +# a long by one arm and an int by the other still agrees. +function Get-BenchEntryIdentity { + [CmdletBinding()] + param( + [Parameter(Mandatory)] [object[]] $Entries + ) + $identity = [ordered]@{} + foreach ($name in $script:BenchIdentityField) { + $seen = [ordered]@{} + foreach ($entry in $Entries) { + $value = Get-BenchIdentityField -Node $entry -Name $name + $key = if ($null -eq $value) { '(none)' } else { [string]$value } + if (-not $seen.Contains($key)) { $seen[$key] = $value } + } + if ($seen.Count -le 1) { + $identity[$name] = @($seen.Values)[0] + } + else { + $identity[$name] = $null + Write-Warning "the runs disagree on ${name} ($((@($seen.Keys)) -join ', ')) - recorded as null." + } + } + [pscustomobject]$identity +} + # Appends a result object to results/.json (one JSON doc per file, array of entries). function Write-BenchResult { param( @@ -318,10 +799,24 @@ function Write-BenchResult { [Parameter(Mandatory)] [object] $Result, # The run's wall clock. Only needed for the RU rate below; picked up from $Result.WallMs # when the caller already records one. - [long] $WallMs = 0 + [long] $WallMs = 0, + # Where the per-run log lives. Named for the same reason Export-BenchBaseline and + # Compare-BenchBaseline name theirs: so a test can be pointed somewhere that is not the + # suite's own measurements. + [string] $ResultsDirectory = (Join-Path $PSScriptRoot 'results'), + # Who measured, for a benchmark whose arms ran somewhere other than this process. The + # five fields below are read from this process by default, which is right wherever this + # process did the work; a comparison that starts its arms as child runs records from a + # parent that imported no local module and connected to no tenant, and the default then + # describes the parent - whichever copy of Mgx happened to autoload there, and no tenant + # at all. Get-BenchEntryIdentity reads this off the arm entries. Supplied, it is taken + # whole and a field it does not carry is recorded as null, because falling back to this + # process for that one field is exactly the substitution the parameter exists to stop. + # Everything else in the entry still describes the process that writes it. + [object] $Identity ) - $dir = Join-Path $PSScriptRoot 'results' - if (-not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir | Out-Null } + $dir = $ResultsDirectory + if (-not (Test-Path $dir)) { New-Item -ItemType Directory -Path $dir -Force | Out-Null } $file = Join-Path $dir "$Benchmark.json" $entries = @() if (Test-Path $file) { $entries = @(Get-Content $file -Raw | ConvertFrom-Json) } @@ -332,8 +827,21 @@ function Write-BenchResult { $wall = if ($WallMs -gt 0) { $WallMs } elseif ($Result.WallMs -and [long]$Result.WallMs -gt 0) { [long]$Result.WallMs } else { 0 } - try { - $t = Get-MgxTelemetry -ErrorAction Stop + # The counters are snapshotted before the tenant facts below, which cost one request the + # first time a process records: what gets recorded has to describe the benchmark and + # nothing else. + $t = $null + try { $t = Get-MgxTelemetry -ErrorAction Stop } + catch { + # A benchmark that does not load Mgx (the bare-SDK comparison arms) has no telemetry. + # That is expected; record its absence rather than failing the run. + $t = $null + } + + $fact = Get-BenchTenantFact + $documentedBudget = $fact.Budget?.RuPerSecond + + if ($t) { $telemetry = [pscustomobject]@{ ResourceUnits = $t.ResourceUnitsConsumed TotalRequests = $t.Requests @@ -379,26 +887,51 @@ function Write-BenchResult { RuPerSecond = $(if ($wall -gt 0) { [math]::Round($t.ResourceUnitsConsumed / ($wall / 1000), 1) } else { $null }) - # The documented budget is 8,000 RU per 10s per application+tenant pair for tenants - # above 500 users, i.e. 800 RU/s. Recorded as a ratio so a run that approaches the - # ceiling is obvious without re-deriving the arithmetic each time. + # Recorded as a ratio so a run that approaches the ceiling is obvious without + # re-deriving the arithmetic each time. Against this tenant's documented budget + # where the tier is known, and against the 800 RU/s of a large tenant where it is + # not - BudgetBasis says which, because the same ratio over two denominators is two + # different claims. BudgetFraction = $(if ($wall -gt 0) { - [math]::Round(($t.ResourceUnitsConsumed / ($wall / 1000)) / 800, 3) + $denominator = if ($null -ne $documentedBudget) { $documentedBudget } else { 800 } + [math]::Round(($t.ResourceUnitsConsumed / ($wall / 1000)) / $denominator, 3) } else { $null }) + BudgetBasis = $(if ($wall -le 0) { $null } + elseif ($null -ne $documentedBudget) { 'documented' } + else { 'assumed-800' }) } } - catch { - # A benchmark that does not load Mgx (the bare-SDK comparison arms) has no telemetry. - # That is expected; record its absence rather than failing the run. - $telemetry = $null + + # The telemetry block above keeps this process's own denominator whatever -Identity says: + # its numerator is this process's counters, and a draw measured here against a budget + # documented elsewhere is neither run's number. + $stamp = $Identity + if (-not $PSBoundParameters.ContainsKey('Identity')) { + $stamp = [pscustomobject]@{ + MgxVersion = (Get-Module M365DSC.mgx -ErrorAction SilentlyContinue)?.Version?.ToString() + SdkVersion = (Get-Module Microsoft.Graph.Authentication -ErrorAction SilentlyContinue)?.Version?.ToString() + TenantId = $fact.TenantId + DirectoryObjectCount = $fact.DirectoryObjectCount + DocumentedBudgetRuPerSecond = $documentedBudget + } } $meta = [pscustomobject]@{ Result = $Result Telemetry = $telemetry - MgxVersion = (Get-Module M365DSC.mgx -ErrorAction SilentlyContinue)?.Version?.ToString() - SdkVersion = (Get-Module Microsoft.Graph.Authentication -ErrorAction SilentlyContinue)?.Version?.ToString() + MgxVersion = Get-BenchIdentityField -Node $stamp -Name 'MgxVersion' + SdkVersion = Get-BenchIdentityField -Node $stamp -Name 'SdkVersion' PSVersion = $PSVersionTable.PSVersion.ToString() + # Which tenant produced the number, how big it is, and what the documentation budgets a + # tenant that size at. Null together when there was no session behind the numbers. + TenantId = Get-BenchIdentityField -Node $stamp -Name 'TenantId' + DirectoryObjectCount = Get-BenchIdentityField -Node $stamp -Name 'DirectoryObjectCount' + DocumentedBudgetRuPerSecond = Get-BenchIdentityField -Node $stamp -Name 'DocumentedBudgetRuPerSecond' + # Which tree produced it. Versions are identical across working trees. + Commit = Get-BenchCommit + # First-class, not only inside the telemetry block: a scenario that loads no Mgx records + # no telemetry at all, and its wall clock is still the figure it is read on. + WallMs = $(if ($wall -gt 0) { $wall } else { $null }) RecordedAt = (Get-Date).ToString('o') } # -InputObject (not pipeline) so a single-element array still serializes as a JSON array @@ -406,3 +939,1029 @@ function Write-BenchResult { ConvertTo-Json -InputObject $all -Depth 12 | Set-Content -Path $file Write-Host " result appended to $file" } + +# --- The baseline snapshot ------------------------------------------------------------------ +# +# results/.json is the append-only per-run log: every run adds an entry and nothing +# is ever promoted out of it, so a number measured on a laptop in August is indistinguishable +# from the one a release was signed off against - and .gitignore keeps that whole directory +# local, so nobody else sees either. baseline.json beside it is the pinned snapshot: the newest +# entry of each named scenario, reduced to the figures that scenario exists to produce, and +# committed, so a later run has something to be read against. +# +# One table names those figures. The FIRST metric of a scenario is its headline - the figure +# that scenario is read on; the rest ride along so the committed file says what the run looked +# like without anyone re-deriving it from a log. Everything that writes or reads a baseline goes +# through this table, so a scenario cannot be pinned on one figure and read on another. +# +# Every headline is lower-is-better and strictly positive. That is deliberate: a percentage +# delta on a figure that can cross zero flips its own sign - 09's checkpoint overhead comes out +# slightly negative on a quiet tenant - so such a figure is recorded and never made a headline. +function Get-BenchScenario { + <# + .SYNOPSIS + The benchmark scenarios that can be promoted into baseline.json, and the metrics of each. + + .DESCRIPTION + Each scenario carries the script that produces it and an ordered list of metrics. A + metric names its unit and one or more Sources: dotted paths into the recorded Result + object, tried in order, the first that resolves winning. Scenario 10 needs the list - + run through run.ps1 it records a two-arm summary, run standalone with -Mode it records + one arm - and a Scale brings a path recorded in another unit onto the metric's own. + + Discriminator, where a scenario has one, is a path whose value is recorded beside the + metrics. 07 and 10 each record a Mode, and a number off the paced arm is not the same + number as one off the unpaced arm: reading one against the other is a wrong answer + rather than a missing one. + + .PARAMETER Id + Scenario ids to return, in the order given. Omit for all of them. An id with no + definition is skipped with a warning rather than throwing: a promotion at the end of a + two-hour suite must not lose the scenarios it does know because of one it does not. + #> + [CmdletBinding()] + param([string[]] $Id) + + $table = [ordered]@{ + '01' = @{ + Script = '01-list-users.ps1' + Metrics = @( + @{ Name = 'MgxListMs'; Unit = 'ms'; Sources = @(@{ Path = 'mgx.Median.ElapsedMs' }) } + @{ Name = 'MgxTimeToFirstMs'; Unit = 'ms'; Sources = @(@{ Path = 'ttfrMgx.Median.ElapsedMs' }) } + @{ Name = 'MgxPeakWorkingSetMB'; Unit = 'MB'; Sources = @(@{ Path = 'mgx.Median.PeakWorkingSetMB' }) } + ) + } + '02' = @{ + Script = '02-fanout-lookup.ps1' + Metrics = @( + @{ Name = 'MgxFanoutMs'; Unit = 'ms'; Sources = @(@{ Path = 'mgx.Median.ElapsedMs' }) } + @{ Name = 'MgxPeakWorkingSetMB'; Unit = 'MB'; Sources = @(@{ Path = 'mgx.Median.PeakWorkingSetMB' }) } + ) + } + '03' = @{ + Script = '03-user-report.ps1' + Metrics = @( + @{ Name = 'MgxReportMs'; Unit = 'ms'; Sources = @(@{ Path = 'mgx.Median.ElapsedMs' }) } + @{ Name = 'MgxPeakWorkingSetMB'; Unit = 'MB'; Sources = @(@{ Path = 'mgx.Median.PeakWorkingSetMB' }) } + ) + } + '04' = @{ + Script = '04-batch-create.ps1' + Metrics = @( + @{ Name = 'MgxBatchCreateMs'; Unit = 'ms'; Sources = @(@{ Path = 'mgxCreate.ElapsedMs' }) } + @{ Name = 'MgxBatchPatchMs'; Unit = 'ms'; Sources = @(@{ Path = 'batchPatch.ElapsedMs' }) } + @{ Name = 'MgxFanoutPatchMs'; Unit = 'ms'; Sources = @(@{ Path = 'fanoutPatch.ElapsedMs' }) } + ) + } + '05' = @{ + # Memory is the claim here, not speed, so peak working set is the headline and the + # wall time rides along. The managed-heap delta is the figure the README quotes for + # the streaming arms, and it sits near zero by design - too near to carry a + # percentage, which is why it is recorded rather than diffed. + Script = '05-memory-export.ps1' + Metrics = @( + @{ Name = 'MgxExportPeakWorkingSetMB'; Unit = 'MB'; Sources = @(@{ Path = 'mgxExport.PeakWorkingSetMB' }) } + @{ Name = 'MgxExportMs'; Unit = 'ms'; Sources = @(@{ Path = 'mgxExport.ElapsedMs' }) } + @{ Name = 'MgxExportManagedHeapDeltaMB'; Unit = 'MB'; Sources = @(@{ Path = 'mgxExport.ManagedHeapDeltaMB' }) } + ) + } + '06' = @{ + Script = '06-fault-gauntlet.ps1' + Metrics = @( + @{ Name = 'MgxFanoutMs'; Unit = 'ms'; Sources = @(@{ Path = 'mgx.ElapsedMs' }) } + @{ Name = 'SdkMgxMs'; Unit = 'ms'; Sources = @(@{ Path = 'sdkMgx.ElapsedMs' }) } + @{ Name = 'MgxCompleted'; Unit = ''; Sources = @(@{ Path = 'mgx.Output.ok' }) } + @{ Name = 'MgxFailed'; Unit = ''; Sources = @(@{ Path = 'mgx.Output.failed' }) } + ) + } + '07' = @{ + Script = '07-adaptive-pacing.ps1' + Discriminator = 'Mode' + Metrics = @( + @{ Name = 'ArmMs'; Unit = 'ms'; Sources = @(@{ Path = 'ElapsedMs' }) } + @{ Name = 'ArmFailed'; Unit = ''; Sources = @(@{ Path = 'Output.failed' }) } + ) + } + '08' = @{ + Script = '08-delta-sync.ps1' + Metrics = @( + @{ Name = 'InitialSyncMs'; Unit = 'ms'; Sources = @(@{ Path = 'initial.ElapsedMs' }) } + @{ Name = 'IncrementalSyncMs'; Unit = 'ms'; Sources = @(@{ Path = 'incremental.ElapsedMs' }) } + @{ Name = 'SteadySyncMs'; Unit = 'ms'; Sources = @(@{ Path = 'steady.ElapsedMs' }) } + @{ Name = 'InitialItems'; Unit = ''; Sources = @(@{ Path = 'initial.Output.items' }) } + ) + } + '09' = @{ + # The checkpointed export is the headline: it is what checkpointing costs, in the + # same units as everything else. The overhead percentage the script derives from it + # reads better and is recorded, but it can land either side of zero on a quiet + # tenant, so it is not what gets diffed. + Script = '09-kill-resume.ps1' + Metrics = @( + @{ Name = 'CheckpointedExportMs'; Unit = 'ms'; Sources = @(@{ Path = 'checkpointed.ElapsedMs' }) } + @{ Name = 'BaselineExportMs'; Unit = 'ms'; Sources = @(@{ Path = 'baseline.ElapsedMs' }) } + @{ Name = 'ResumeMs'; Unit = 'ms'; Sources = @(@{ Path = 'resume.ElapsedMs' }) } + @{ Name = 'CheckpointOverheadPct'; Unit = '%'; Sources = @(@{ Path = 'verification.overheadPct' }) } + @{ Name = 'DuplicateIds'; Unit = ''; Sources = @(@{ Path = 'verification.duplicateIds' }) } + ) + } + '10' = @{ + Script = '10-pacing-under-real-throttling.ps1' + Discriminator = 'Mode' + Metrics = @( + @{ Name = 'WallSeconds'; Unit = 's'; Sources = @(@{ Path = 'Arms.paced.Seconds' }, @{ Path = 'WallMs'; Scale = 0.001 }) } + @{ Name = 'UnpacedWallSeconds'; Unit = 's'; Sources = @(@{ Path = 'Arms.unpaced.Seconds' }) } + @{ Name = 'ThrottleRetries'; Unit = ''; Sources = @(@{ Path = 'Arms.paced.ThrottleRetries' }, @{ Path = 'ThrottleRetries' }) } + ) + } + '18' = @{ + # A latency distribution, so the headline is the whole window's p90 of the HTTP + # round trip - lower is better and it cannot reach zero. The clamp ratio the script + # derives from the five-minute buckets is the figure the claim is stated in, and it + # rides along rather than leading: a window that never stretched reports 1.0, which + # is the good outcome and would read as an improvement against anything pinned. + # P90CallElapsedMs is the same window measured at the caller, pacer waits included; + # it is here so the two are never mistaken for one another. + # + # Discriminated on the preceding activity rather than on the drive: the whole point + # of the measurement is that the same drive answers differently depending on what it + # has just been through, so two runs are comparable only where the operator states + # the same history for both. The drive itself would be the other candidate and is + # deliberately not used - baseline.json is committed, and a drive id in it would put + # a tenant's identifiers in the repository, which is why the live suite passes one + # through MGX_LIVE_CONTENT_URI instead. The entry's own Result records the drive. + Script = '18-spo-latency-clamp.ps1' + Discriminator = 'PrecedingActivity' + Metrics = @( + @{ Name = 'P90ElapsedMs'; Unit = 'ms'; Sources = @(@{ Path = 'Overall.P90ElapsedMs' }) } + @{ Name = 'P50ElapsedMs'; Unit = 'ms'; Sources = @(@{ Path = 'Overall.P50ElapsedMs' }) } + @{ Name = 'P99ElapsedMs'; Unit = 'ms'; Sources = @(@{ Path = 'Overall.P99ElapsedMs' }) } + @{ Name = 'P90CallElapsedMs'; Unit = 'ms'; Sources = @(@{ Path = 'Overall.P90CallElapsedMs' }) } + @{ Name = 'ClampRatio'; Unit = 'x'; Sources = @(@{ Path = 'Overall.ClampRatio' }) } + @{ Name = 'Throttled429'; Unit = ''; Sources = @(@{ Path = 'Overall.Throttled429' }) } + ) + } + } + + $wanted = if ($Id) { $Id } else { @($table.Keys) } + foreach ($key in $wanted) { + if (-not $table.Contains($key)) { + Write-Warning "benchmark '$key' has no baseline metrics in Get-BenchScenario - skipped." + continue + } + $s = $table[$key] + [pscustomobject]@{ + Id = $key + Script = $s.Script + # Write-BenchResult names the file after the benchmark, and every benchmark passes + # its own script name without the extension. + ResultFile = [System.IO.Path]::GetFileNameWithoutExtension($s.Script) + '.json' + Discriminator = $s.Discriminator + Metrics = $s.Metrics + } + } +} + +# Walks a dotted path into a recorded Result object. Returns $null for any segment that is not +# there, which is how a metric reports itself absent rather than throwing halfway through a +# promotion. +function Get-BenchResultValue { + param( + [object] $Node, + [Parameter(Mandatory)] [string] $Path + ) + foreach ($segment in ($Path -split '\.')) { + if ($null -eq $Node) { return $null } + $property = $Node.PSObject.Properties[$segment] + if (-not $property) { return $null } + $Node = $property.Value + } + $Node +} + +# The newest entry of one benchmark's results file, or $null. -Since rejects an entry older than +# the run that asked for it: a benchmark that died before recording leaves the previous run's +# entry newest, and promoting or comparing that reports a stale number as a fresh one. +function Get-BenchLatestResult { + param( + [Parameter(Mandatory)] [string] $ResultFile, + [datetime] $Since = [datetime]::MinValue + ) + if (-not (Test-Path $ResultFile)) { return $null } + $entries = @(Get-Content $ResultFile -Raw | ConvertFrom-Json) + if ($entries.Count -eq 0) { return $null } + $entry = $entries[-1] + + # ConvertFrom-Json turns a round-trip timestamp into a DateTime on its own; a file written + # by another host may still hand back the string it was written as. + $recorded = $entry.RecordedAt + if ($recorded -isnot [datetime]) { + try { + $recorded = [datetime]::Parse([string]$recorded, [cultureinfo]::InvariantCulture, + [System.Globalization.DateTimeStyles]::RoundtripKind) + } + catch { $recorded = [datetime]::MinValue } + } + if ($recorded -lt $Since) { return $null } + $entry +} + +# A value is a measurement only when it is a finite number: one of PowerShell's numeric types, +# or a string that parses as one under the invariant culture the benchmark process is pinned +# to. [double] coerces almost anything without throwing - '' becomes 0, $true becomes 1 - so a +# source that never recorded a number would otherwise be pinned as though it had. NaN and the +# infinities pass both tests unless they are refused explicitly: a typed double or single can +# already hold one mid-computation, and NumberStyles.Float parses the strings "NaN", +# "Infinity" and "-Infinity" as valid doubles, so each arm is checked for +# [double]::IsFinite before the value is accepted. +function Test-BenchNumericValue([object] $Value) { + if ($Value -is [int] -or $Value -is [long] -or $Value -is [decimal]) { + return $true + } + if ($Value -is [double] -or $Value -is [single]) { + return [double]::IsFinite([double]$Value) + } + $parsed = 0.0 + $Value -is [string] -and [double]::TryParse($Value, [System.Globalization.NumberStyles]::Float, + [System.Globalization.CultureInfo]::InvariantCulture, [ref] $parsed) -and [double]::IsFinite($parsed) +} + +# Names what a value not accepted by Test-BenchNumericValue actually held, for a warning or a +# comparison note to quote. A string or a boolean prints the way PowerShell already prints it +# elsewhere in this file ('INCONCLUSIVE', 'True'). An array, an object, or JSON null read back +# off a stale baseline.json has no such print of its own worth trusting - $null and '' would +# both print as nothing, and an array joins its elements with spaces - so each renders the way +# ConvertTo-Json -Compress would, the same shape the file was written in. +function ConvertTo-BenchDisplayValue([object] $Value) { + if ($null -eq $Value -or $Value -is [array] -or $Value -is [System.Collections.IDictionary] -or $Value -is [pscustomobject]) { + return ConvertTo-Json -InputObject $Value -Compress -Depth 5 + } + "$Value" +} + +# Reads one scenario's metrics out of a recorded entry: an ordered map of +# name -> @{ Value; Unit; Path }, plus the discriminator value where the scenario has one. +function Get-BenchScenarioMetric { + param( + [Parameter(Mandatory)] [object] $Scenario, + [Parameter(Mandatory)] [object] $Entry + ) + $metrics = [ordered]@{} + foreach ($m in $Scenario.Metrics) { + $sources = @($m.Sources) + $resolved = $false + $notNumbers = [System.Collections.Generic.List[object]]::new() + for ($i = 0; $i -lt $sources.Count; $i++) { + $source = $sources[$i] + $value = Get-BenchResultValue -Node $Entry.Result -Path $source.Path + if ($null -eq $value) { continue } + if (-not (Test-BenchNumericValue $value)) { + $notNumbers.Add([pscustomobject]@{ + Description = "'$($source.Path)' holds '$(ConvertTo-BenchDisplayValue $value)', which is not a number" + }) + continue + } + $scale = if ($source.ContainsKey('Scale')) { [double]$source.Scale } else { 1.0 } + $number = if ($value -is [string]) { [double]::Parse($value, [System.Globalization.CultureInfo]::InvariantCulture) } else { [double]$value } + $scaled = $number * $scale + # A source that read within double's range can still overflow once Scale is + # applied - the table's own Scale (10's WallMs, 0.001) shrinks, so this cannot + # happen through it today, but a larger Scale would push a source that was itself + # finite past it. What must be finite is the multiply's result, not the raw + # value, so it is checked after scaling rather than before - and a source that + # fails here is a source that held a non-number just as surely as 'INCONCLUSIVE' + # is: the walk tries the next one and says so. Named the same way its sibling + # above names a raw non-number, so two overflowing sources on the same metric + # can still be told apart by path. + if (-not [double]::IsFinite($scaled)) { + $notNumbers.Add([pscustomobject]@{ + Description = "'$($source.Path)' holds '$(ConvertTo-BenchDisplayValue $value)', which scaled by $scale is not finite" + }) + continue + } + $metrics[$m.Name] = [pscustomobject]@{ + Value = [math]::Round($scaled, 3) + Unit = $m.Unit + Path = $source.Path + } + $resolved = $true + break + } + if ($resolved) { + # Every source ahead of the one that resolved held something that was not a + # number - a first source that recorded 'INCONCLUSIVE' before a second recorded + # the real figure, say. The next source covering for it should not read as + # silence, so each earns its own warning naming what it held. + foreach ($n in $notNumbers) { + Write-Warning "$($Scenario.Id): '$($m.Name)' - $($n.Description) - trying the next source." + } + } + # A source with nothing at its path was never written - 10's single-arm entries never + # write the other arm's, and that is not a metric gone missing, it is one this entry + # was never going to carry. Only a source that held something, and held a value that + # is not a number, earns a warning - naming what it held. + elseif ($notNumbers.Count -gt 0) { + $held = ($notNumbers | ForEach-Object { $_.Description }) -join '; ' + Write-Warning "$($Scenario.Id): '$($m.Name)' not recorded - $held." + } + } + $discriminator = $null + if ($Scenario.Discriminator) { + $discriminator = Get-BenchResultValue -Node $Entry.Result -Path $Scenario.Discriminator + } + [pscustomobject]@{ Metrics = $metrics; Discriminator = $discriminator } +} + +function Export-BenchBaseline { + <# + .SYNOPSIS + Promotes the newest results entry of each named benchmark into baseline.json. + + .DESCRIPTION + Writes one record per scenario: its id, the script that produced it, the metrics + Get-BenchScenario names for it, and the Mgx/SDK/PowerShell versions and timestamp + Write-BenchResult already recorded alongside the numbers. Scenarios already in the file + and not named here are left as they are, so promoting one scenario does not discard the + six the release baseline is otherwise made of. + + A scenario whose entry records Result.Conclusive false is reported and left out: the run + itself says it did not measure what the scenario exists to measure, and whatever row was + already pinned for it stays. Where such an entry also carries Result.InconclusiveReason, + the warning quotes it. + + .PARAMETER Benchmark + Scenario ids to promote. + + .PARAMETER Since + Ignore entries recorded before this. run.ps1 passes the moment the suite started, so a + benchmark that recorded nothing is reported rather than promoted from a previous run. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string[]] $Benchmark, + [string] $ResultsDirectory = (Join-Path $PSScriptRoot 'results'), + [string] $Path = (Join-Path $PSScriptRoot 'baseline.json'), + [datetime] $Since = [datetime]::MinValue + ) + + $scenarios = [ordered]@{} + if (Test-Path $Path) { + $existing = Get-Content $Path -Raw | ConvertFrom-Json + if ($existing.Scenarios) { + foreach ($p in $existing.Scenarios.PSObject.Properties) { $scenarios[$p.Name] = $p.Value } + } + } + + $promoted = 0 + foreach ($scenario in (Get-BenchScenario -Id $Benchmark)) { + $file = Join-Path $ResultsDirectory $scenario.ResultFile + $entry = Get-BenchLatestResult -ResultFile $file -Since $Since + if (-not $entry) { + Write-Warning "benchmark $($scenario.Id) recorded no result in $file - not promoted." + continue + } + # A run that says it measured nothing is not a baseline. 10 is the shape of it: it + # records Result.Conclusive false when its unpaced arm was never refused, which means + # the throttled regime it exists to compare against was never reached - and the wall + # times it did record are then two unthrottled runs wearing a throttling benchmark's + # name. Promoting that pins a number every later run is read against, so the row it + # would overwrite is kept instead: a stale conclusive figure is still a measurement, + # and none at all is what the comparison already reports as unmeasured. A scenario that + # records no verdict at all promotes as before - the field is a claim, not a + # requirement. + $conclusive = Get-BenchResultValue -Node $entry.Result -Path 'Conclusive' + if ($null -ne $conclusive -and -not $conclusive) { + $why = Get-BenchResultValue -Node $entry.Result -Path 'InconclusiveReason' + if ([string]::IsNullOrWhiteSpace([string]$why)) { $why = 'Conclusive is false' } + $kept = if ($scenarios.Contains($scenario.Id)) { + '; the row already in the file is left as it was' + } else { '' } + Write-Warning "benchmark $($scenario.Id) recorded an inconclusive run ($why) - not promoted$kept." + continue + } + $read = Get-BenchScenarioMetric -Scenario $scenario -Entry $entry + if ($read.Metrics.Count -eq 0) { + Write-Warning "benchmark $($scenario.Id) recorded an entry carrying none of its metrics - not promoted." + continue + } + # The headline is the first metric of the scenario's own row, and only where that + # metric resolved. Taking the first metric that HAPPENED to resolve promotes whichever + # one came next instead - a different figure, in a different unit, read as though it + # were the one the scenario is pinned on. 09 is the shape of it: with its leading + # sources absent from an entry, the next resolvable metric is CheckpointOverheadPct, a + # percentage that lands either side of zero on a quiet tenant. An entry that did not + # record its own headline is pinned with the metrics it did record and no headline, so + # the comparison reports it rather than reading something else in its place. + $tableHeadline = @($scenario.Metrics)[0].Name + $headline = if ($read.Metrics.Contains($tableHeadline)) { $tableHeadline } else { $null } + $scenarios[$scenario.Id] = [pscustomobject]@{ + Scenario = $scenario.Id + Script = $scenario.Script + Headline = $headline + Discriminator = $read.Discriminator + Metrics = $read.Metrics + MgxVersion = $entry.MgxVersion + SdkVersion = $entry.SdkVersion + PSVersion = $entry.PSVersion + RecordedAt = $entry.RecordedAt + PromotedAt = (Get-Date).ToString('o') + } + $promoted++ + if ($headline) { + Write-Host (" {0}: {1} = {2}{3}" -f $scenario.Id, $headline, + $read.Metrics[$headline].Value, $read.Metrics[$headline].Unit) + } + else { + Write-Warning "benchmark $($scenario.Id) recorded no '$tableHeadline' - pinned with its other metrics and no headline; the comparison will report it as not comparable." + } + } + + if ($promoted -eq 0) { + Write-Warning "nothing promoted; $Path left as it was." + return + } + + $sorted = [ordered]@{} + foreach ($key in (@($scenarios.Keys) | Sort-Object)) { $sorted[$key] = $scenarios[$key] } + $document = [pscustomobject]@{ + SchemaVersion = 1 + GeneratedAt = (Get-Date).ToString('o') + Scenarios = $sorted + } + ConvertTo-Json -InputObject $document -Depth 10 | Set-Content -Path $Path + Write-Host " baseline written to $Path ($promoted promoted, $($sorted.Count) scenario(s) in the file)" +} + +function Compare-BenchBaseline { + <# + .SYNOPSIS + Reads a finished run against baseline.json and prints a delta per scenario. + + .DESCRIPTION + Reports; never fails the run. The tenant's own throttling ceiling has moved 40% between + consecutive days, so a hard gate on these numbers is noise wearing a verdict's clothes. + A scenario with no baseline entry prints as unmeasured, and one whose fresh entry is + missing or not comparable prints as that - a silent skip is how a regression hides. + + Emits one row object per scenario as well as printing them, so a caller can assert. + + .PARAMETER ThresholdPercent + How far the headline metric may move before the row is flagged. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string[]] $Benchmark, + [string] $ResultsDirectory = (Join-Path $PSScriptRoot 'results'), + [string] $BaselinePath = (Join-Path $PSScriptRoot 'baseline.json'), + [datetime] $Since = [datetime]::MinValue, + # Proposed, not derived: 50%. The tenant's throttling ceiling has been seen to move 40% + # between consecutive days, so anything tighter flags the weather. 50% sits just above + # that band, which makes a flag mean "further than the worst environmental swing anyone + # has recorded here" - a weak claim, but a true one. Once baseline.json has been + # re-recorded a few times, a per-scenario band derived from those runs replaces it. + [double] $ThresholdPercent = 50 + ) + + if (-not (Test-Path $BaselinePath)) { + Write-Host '' + Write-Host '=== BASELINE COMPARISON ===' + Write-Host "no baseline at $BaselinePath - nothing to read this run against." + Write-Host 'record one with ./run.ps1 -RecordBaseline when a run is worth pinning.' + return + } + + $baseline = Get-Content $BaselinePath -Raw | ConvertFrom-Json + if (-not $baseline.Scenarios) { + throw "Baseline '$BaselinePath' carries no Scenarios; record one with -RecordBaseline before comparing." + } + if ($baseline.SchemaVersion -ne 1) { + throw "Baseline '$BaselinePath' is schema $($baseline.SchemaVersion); this suite reads schema 1." + } + Write-Host '' + Write-Host ('=== BASELINE COMPARISON (flagged past {0:F0}%) ===' -f $ThresholdPercent) + Write-Host ('{0,-4} {1,-28} {2,14} {3,14} {4,9} {5}' -f 'id', 'metric', 'baseline', 'this run', 'delta', 'note') + + $rows = @() + foreach ($scenario in (Get-BenchScenario -Id $Benchmark)) { + $pinnedProperty = $baseline.Scenarios.PSObject.Properties[$scenario.Id] + $pinned = if ($pinnedProperty) { $pinnedProperty.Value } else { $null } + $entry = Get-BenchLatestResult -ResultFile (Join-Path $ResultsDirectory $scenario.ResultFile) -Since $Since + $fresh = if ($entry) { Get-BenchScenarioMetric -Scenario $scenario -Entry $entry } else { $null } + + $row = [pscustomobject]@{ + Scenario = $scenario.Id + Metric = $null + Unit = $null + Baseline = $null + Current = $null + DeltaPct = $null + Status = $null + Flagged = $false + Note = $null + } + + if (-not $pinned) { + $row.Status = 'unmeasured' + $row.Note = 'unmeasured - no baseline entry' + } + elseif (-not $fresh) { + $row.Status = 'no result' + $row.Note = 'no result recorded by this run' + } + elseif (-not $pinned.Headline) { + # Pinned without one: the promoted run did not record the metric its scenario is + # read on. The metrics beside it are not substitutes - each is a different figure - + # so the row says there is nothing to compare instead of comparing something else. + $row.Status = 'not comparable' + $row.Note = 'not comparable - the baseline entry has no headline, so its run never recorded the metric this scenario is read on' + } + else { + $row.Metric = $pinned.Headline + $baselineProperty = $pinned.Metrics.PSObject.Properties[$pinned.Headline] + $b = if ($baselineProperty) { $baselineProperty.Value } else { $null } + $f = $fresh.Metrics[$pinned.Headline] + + if (-not $b -or -not $f) { + $row.Status = 'unmeasured' + $row.Note = "unmeasured - '$($pinned.Headline)' is not in both" + } + elseif ($b.Path -ne $f.Path -or $b.Unit -ne $f.Unit) { + $row.Status = 'not comparable' + $row.Note = "not comparable - baseline read $($b.Path), this run $($f.Path)" + } + elseif ($scenario.Discriminator -and + [string]$pinned.Discriminator -ne [string]$fresh.Discriminator) { + $row.Status = 'not comparable' + $row.Note = "not comparable - $($scenario.Discriminator) was '$($pinned.Discriminator)', this run '$($fresh.Discriminator)'" + } + else { + $row.Unit = $b.Unit + # [double] coerces almost anything without throwing - '' becomes 0, $true + # becomes 1 - so a stale baseline.json holding something that was never a + # number would cast into a plausible-looking figure instead of being refused. + # Test-BenchNumericValue is asked first, on the raw value, before either side + # is cast - the same gate a raw source reading already goes through in + # Get-BenchScenarioMetric. It also refuses NaN and the infinities outright, so + # a value already typed [double] or [single] is exempted from it here: a bare + # JSON NaN or Infinity token round-trips through ConvertFrom-Json as a genuine + # typed double, and that is a number for the cast and the finiteness check + # below to refuse, not one for this gate to call not a number at all. + $bIsTyped = $b.Value -is [double] -or $b.Value -is [single] + $fIsTyped = $f.Value -is [double] -or $f.Value -is [single] + if (-not $bIsTyped -and -not (Test-BenchNumericValue $b.Value)) { + $row.Status = 'not comparable' + $row.Note = "not comparable - the baseline value is '$(ConvertTo-BenchDisplayValue $b.Value)', which is not a number" + } + elseif (-not $fIsTyped -and -not (Test-BenchNumericValue $f.Value)) { + $row.Status = 'not comparable' + $row.Note = "not comparable - this run's value is '$(ConvertTo-BenchDisplayValue $f.Value)', which is not a number" + } + else { + # [double]$b.Value already reads "NaN", "Infinity" and "-Infinity" as the + # doubles they name - PowerShell's own numeric conversion, not + # [double]::Parse, accepts those three tokens under any culture - so a + # stale baseline.json recorded before Export-BenchBaseline refused a + # non-finite measurement, or a bare JSON NaN or Infinity token the gate + # above let through untested, casts here to a genuine non-finite double + # rather than throwing. Nothing pins a non-finite Current today, but the + # same cast is used for it, so both are checked before either reaches the + # arithmetic below. + $row.Baseline = [double]$b.Value + $row.Current = [double]$f.Value + if (-not [double]::IsFinite($row.Baseline)) { + $row.Status = 'not comparable' + $row.Note = "not comparable - the baseline value is $($row.Baseline)" + } + elseif (-not [double]::IsFinite($row.Current)) { + $row.Status = 'not comparable' + $row.Note = "not comparable - this run's value is $($row.Current)" + } + elseif ([math]::Abs($row.Baseline) -lt 1e-9) { + $row.Status = 'not comparable' + $row.Note = 'not comparable - the baseline value is zero' + } + elseif ($row.Baseline -lt 0) { + # A percentage delta over a negative denominator inverts its own sign: a + # figure that moved further below the baseline comes out positive and reads + # as the bad direction, and one that improved reads as a gain. Every + # headline in the table is meant to be strictly positive, so a negative one + # is a figure that was never fit to be a headline - which is a thing to say, + # not a delta to compute. + $row.Status = 'not comparable' + $row.Note = "not comparable - the baseline value is negative ($($row.Baseline)$($row.Unit)); a percentage delta over it would invert its own sign" + } + else { + $row.DeltaPct = [math]::Round((($row.Current - $row.Baseline) / $row.Baseline) * 100, 1) + $row.Status = 'compared' + # Lower is better for every headline in the table, so a positive delta is the + # bad direction. A move the other way is flagged too: a scenario that halves + # has usually measured less work rather than done the same work faster. + if ($row.DeltaPct -gt $ThresholdPercent) { + $row.Flagged = $true + $row.Note = 'REGRESSED' + } + elseif ($row.DeltaPct -lt -$ThresholdPercent) { + $row.Flagged = $true + $row.Note = 'IMPROVED - check it measured the same work' + } + } + } + } + } + + $show = { + param($value, $unit) + if ($null -eq $value) { '-' } else { '{0:F1}{1}' -f $value, $unit } + } + $color = if ($row.Flagged) { 'Yellow' } elseif ($row.Status -ne 'compared') { 'DarkGray' } else { 'Gray' } + Write-Host ('{0,-4} {1,-28} {2,14} {3,14} {4,9} {5}' -f ` + $row.Scenario, + $(if ($row.Metric) { $row.Metric } else { '-' }), + (& $show $row.Baseline $row.Unit), + (& $show $row.Current $row.Unit), + $(if ($null -ne $row.DeltaPct) { '{0:+0.0;-0.0;0.0}%' -f $row.DeltaPct } else { '-' }), + $(if ($row.Note) { $row.Note } else { $row.Status })) -ForegroundColor $color + + $rows += $row + } + + Write-Host 'reported, not gated: these numbers move with the tenant, so nothing here fails the run.' + $rows +} + +# --- Benchmark 18: the SPO latency-clamp summary, read from its checkpointed rows ------------- +# +# 18-spo-latency-clamp.ps1 used to build its buckets and totals while the load ran, from +# whatever rows happened to be in memory at each step. Measured against the run recorded +# 2026-09-10 (StartedAt 13:16:34, EndedAt 14:31:36, a 75-minute window with a 15-minute machine +# sleep in it): the Stopwatch the header timed itself with does not advance while suspended, so +# it printed "59.6 minute(s)" for a window that ran 75; the two buckets the sleep spanned were +# never created at all, because a live accumulator only ever holds a key for a minute a row +# actually arrived in; the printed table had no Errors column, so the 1,005 rows the sleep's +# breaker trip wrote were invisible next to the 429 count everyone was already looking at; and +# the result's PacingState is the pacer's one most-recent latency sample - that run's own last +# row happened to be a slow one, so the figure sitting beside Overall.P90ElapsedMs and +# ClampRatio read as the verdict when it was one call out of 5,919 in its bucket. +# +# The functions below read the rows file back rather than trust an accumulator that was live +# through all of that: every row the run checkpointed is on disk before the summary is built, +# so a bucket, a count or a headline computed from it is the same whether it runs while the load +# is finishing or an hour later against a copy. Get-ClampSummary is what the script calls once +# its load completes, and what a Pester test calls against a synthetic rows.jsonl it wrote +# itself - the same computation either way, rather than a live one and a tested one that can +# drift apart. + +function Get-ClampPercentile { + <# + .SYNOPSIS + Nearest-rank percentile: the value the tenant actually served, never an interpolation. + #> + [CmdletBinding()] + param([object[]] $Values, [double] $Percentile) + if (-not $Values -or @($Values).Count -eq 0) { return $null } + $sorted = @($Values | Sort-Object) + $rank = [int][math]::Ceiling(($Percentile / 100.0) * $sorted.Count) + if ($rank -lt 1) { $rank = 1 } + if ($rank -gt $sorted.Count) { $rank = $sorted.Count } + [math]::Round([double]$sorted[$rank - 1], 1) +} + +function Get-ClampClockGap { + <# + .SYNOPSIS + Whether the wall clock outran the run's Stopwatch by more than a bucket - a suspended + machine, not a pacer wait or a breaker hold, because a Stopwatch can only fail to + advance by not running. + + .PARAMETER StartedAt + .PARAMETER EndedAt + Get-Date at the start and end of the load, wall time. + + .PARAMETER WallMs + The load's own Stopwatch, in milliseconds - ElapsedMilliseconds, read after Stop(). + + .PARAMETER BucketMinutes + The run's bucket width, and the gap's own tolerance: a wall/Stopwatch drift inside one + bucket is ordinary scheduling noise, not a sleep. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [datetime] $StartedAt, + [Parameter(Mandatory)] [datetime] $EndedAt, + [Parameter(Mandatory)] [long] $WallMs, + [Parameter(Mandatory)] [ValidateRange(1, [int]::MaxValue)] [int] $BucketMinutes + ) + + $wallSpanMs = ($EndedAt - $StartedAt).TotalMilliseconds + $gapMs = $wallSpanMs - $WallMs + $gapMinutes = [math]::Round($gapMs / 60000.0, 1) + $refused = $gapMs -gt ($BucketMinutes * 60000.0) + + [pscustomobject]@{ + Refused = $refused + GapMinutes = $gapMinutes + Reason = $(if ($refused) { + "the machine was not awake for $gapMinutes minute(s) of the window; rows kept, summary withheld" + } else { $null }) + } +} + +function Get-ClampRows { + <# + .SYNOPSIS + Parses 18's checkpointed rows.jsonl - one call attempt per line - back into objects. + #> + [CmdletBinding()] + param([Parameter(Mandatory)] [string] $Path) + + if (-not (Test-Path $Path)) { throw "Get-ClampRows: no rows file at '$Path'." } + @(Get-Content -Path $Path | Where-Object { $_ } | ForEach-Object { ConvertFrom-Json -InputObject $_ }) +} + +function Get-ClampRefusalKind { + <# + .SYNOPSIS + Groups a refusal message to what comes before its first colon - two "Circuit breaker + tripped: too many failures..." trips with different wait times group under the same + "Circuit breaker tripped" - or the whole message when it carries no colon at all, such + as "no response traced". + #> + [CmdletBinding()] + param([AllowNull()] [AllowEmptyString()] [string] $Message) + + if (-not $Message) { return $null } + (($Message -split ':', 2)[0]).Trim() +} + +function Get-ClampBuckets { + <# + .SYNOPSIS + 18's per-bucket table: every index from 0 through Ceiling(Minutes/BucketMinutes)-1, so + a bucket the run never wrote a row to is reported rather than silently missing. + + .DESCRIPTION + A row's own minute-offset from StartedAt decides its bucket, exactly as the live run + computes it - so a bucket built here from the checkpoint agrees with the one the run + would have shown if its accumulator had covered the whole window. The index range is + computed from Minutes and BucketMinutes rather than from the rows themselves: the rows + say which buckets got traffic, not how many the run was asked to cover, and the gap + between the two is exactly what a marked "no rows" bucket exists to say. + #> + [CmdletBinding()] + param( + [AllowNull()] [object[]] $Rows, + [Parameter(Mandatory)] [datetime] $StartedAt, + [Parameter(Mandatory)] [ValidateRange(1, [int]::MaxValue)] [int] $Minutes, + [Parameter(Mandatory)] [ValidateRange(1, [int]::MaxValue)] [int] $BucketMinutes + ) + + $buckets = [ordered]@{} + foreach ($row in @($Rows)) { + $minutesIn = ((Get-Date $row.Timestamp) - $StartedAt).TotalMinutes + if ($minutesIn -lt 0) { $minutesIn = 0 } + $index = [int][math]::Floor($minutesIn / $BucketMinutes) + $key = "b$index" + if (-not $buckets.Contains($key)) { + $buckets[$key] = [ordered]@{ + Bucket = $index; FromMinute = $index * $BucketMinutes; Calls = 0; Attempts = 0 + Ok = 0; Throttled429 = 0; Errors = 0 + ThrottlePercentageSeen = 0; ThrottleScopeSeen = 0; RetryAfterSeen = 0; RateLimitSeen = 0 + Latencies = [System.Collections.Generic.List[double]]::new() + Note = $null + } + } + $b = $buckets[$key] + $b.Attempts++ + if ($row.Attempt -eq 1) { $b.Calls++ } + if ($row.Status -ge 200 -and $row.Status -lt 300) { $b.Ok++ } + elseif ($row.Status -eq 429) { $b.Throttled429++ } + if ($row.Status -eq 0 -or $row.Error) { $b.Errors++ } + if ($row.ThrottleLimitPercentage) { $b.ThrottlePercentageSeen++ } + if ($row.ThrottleScope) { $b.ThrottleScopeSeen++ } + if ($row.RetryAfter) { $b.RetryAfterSeen++ } + if ($row.RateLimitHeader) { $b.RateLimitSeen++ } + if ($null -ne $row.ElapsedMs -and $row.Status -ge 200 -and $row.Status -lt 300) { + $b.Latencies.Add([double]$row.ElapsedMs) + } + } + + $bucketCount = [int][math]::Ceiling($Minutes / [double]$BucketMinutes) + for ($index = 0; $index -lt $bucketCount; $index++) { + $key = "b$index" + if (-not $buckets.Contains($key)) { + $buckets[$key] = [ordered]@{ + Bucket = $index; FromMinute = $index * $BucketMinutes; Calls = 0; Attempts = 0 + Ok = 0; Throttled429 = 0; Errors = 0 + ThrottlePercentageSeen = 0; ThrottleScopeSeen = 0; RetryAfterSeen = 0; RateLimitSeen = 0 + Latencies = [System.Collections.Generic.List[double]]::new() + Note = 'no rows' + } + } + } + + @( + foreach ($key in $buckets.Keys) { + $b = $buckets[$key] + [pscustomobject]@{ + Bucket = $b.Bucket + FromMinute = $b.FromMinute + Calls = $b.Calls + Attempts = $b.Attempts + Ok = $b.Ok + Throttled429 = $b.Throttled429 + Errors = $b.Errors + P50ElapsedMs = Get-ClampPercentile -Values $b.Latencies -Percentile 50 + P90ElapsedMs = Get-ClampPercentile -Values $b.Latencies -Percentile 90 + P99ElapsedMs = Get-ClampPercentile -Values $b.Latencies -Percentile 99 + ThrottlePercentageSeen = $b.ThrottlePercentageSeen + ThrottleScopeSeen = $b.ThrottleScopeSeen + RetryAfterSeen = $b.RetryAfterSeen + RateLimitSeen = $b.RateLimitSeen + Note = $b.Note + } + } + ) | Sort-Object Bucket +} + +function Get-ClampOverall { + <# + .SYNOPSIS + 18's whole-window totals: outcome counts, percentiles, throttle-signal counts, the + clamp ratio, and refusals grouped by kind. + + .PARAMETER Buckets + Get-ClampBuckets' own output, for the clamp ratio - worst bucket P90 over the first + bucket's - which has to read the same buckets the table beside it prints. + #> + [CmdletBinding()] + param( + [AllowNull()] [object[]] $Rows, + [Parameter(Mandatory)] [AllowNull()] [object[]] $Buckets + ) + + $rowList = @($Rows) + $latencies = [System.Collections.Generic.List[double]]::new() + $callLatencies = [System.Collections.Generic.List[double]]::new() + $perWorker = @{} + $callCount = 0; $okCount = 0; $throttled = 0; $errorCount = 0 + $sawPercentage = 0; $sawScope = 0; $sawRetryAfter = 0; $sawRateLimit = 0 + $refusals = [ordered]@{} + + foreach ($row in $rowList) { + if ($row.Attempt -eq 1) { $callCount++ } + if ($row.Status -ge 200 -and $row.Status -lt 300) { $okCount++ } + elseif ($row.Status -eq 429) { $throttled++ } + if ($row.Status -eq 0 -or $row.Error) { + $errorCount++ + $kind = Get-ClampRefusalKind -Message $row.Error + if ($kind) { + if (-not $refusals.Contains($kind)) { $refusals[$kind] = 0 } + $refusals[$kind]++ + } + } + if ($row.ThrottleLimitPercentage) { $sawPercentage++ } + if ($row.ThrottleScope) { $sawScope++ } + if ($row.RetryAfter) { $sawRetryAfter++ } + if ($row.RateLimitHeader) { $sawRateLimit++ } + if (-not $perWorker.ContainsKey($row.Worker)) { $perWorker[$row.Worker] = 0 } + $perWorker[$row.Worker]++ + if ($null -ne $row.ElapsedMs -and $row.Status -ge 200 -and $row.Status -lt 300) { + $latencies.Add([double]$row.ElapsedMs) + } + if ($row.Attempt -eq 1 -and $null -ne $row.CallElapsedMs) { $callLatencies.Add([double]$row.CallElapsedMs) } + } + + $bucketList = @($Buckets) + $firstP90 = @($bucketList | Where-Object { $null -ne $_.P90ElapsedMs })[0]?.P90ElapsedMs + $worstP90 = ($bucketList | Where-Object { $null -ne $_.P90ElapsedMs } | + Measure-Object -Property P90ElapsedMs -Maximum).Maximum + $clampRatio = if ($firstP90 -and $firstP90 -gt 0 -and $null -ne $worstP90) { + [math]::Round($worstP90 / $firstP90, 2) + } else { $null } + + [ordered]@{ + Calls = $callCount + Attempts = $rowList.Count + Ok = $okCount + Throttled429 = $throttled + Errors = $errorCount + Workers = $perWorker.Count + P50ElapsedMs = Get-ClampPercentile -Values $latencies -Percentile 50 + P90ElapsedMs = Get-ClampPercentile -Values $latencies -Percentile 90 + P99ElapsedMs = Get-ClampPercentile -Values $latencies -Percentile 99 + P90CallElapsedMs = Get-ClampPercentile -Values $callLatencies -Percentile 90 + ThrottlePercentageSeen = $sawPercentage + ThrottleScopeSeen = $sawScope + RetryAfterSeen = $sawRetryAfter + RateLimitSeen = $sawRateLimit + ClampRatio = $clampRatio + Refusals = $refusals + } +} + +function Get-ClampHeadline { + <# + .SYNOPSIS + The one-line verdict, read from the buckets: whole-window p50/p90 and the worst + bucket's own p90, never a single live sample. + + .DESCRIPTION + Get-MgxTelemetry's PacingState carries the pacer's most recent latency sample - useful + for watching the pacer live, and exactly one call wide. Printed beside a headline that + is actually labeled as one, it stops reading as the run's verdict by default; this + function is that headline, and the caller prints PacingState only as "last pacer + sample", never in its place. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [object] $Overall, + [AllowNull()] [object[]] $Buckets + ) + + $worstBucket = @($Buckets) | Where-Object { $null -ne $_.P90ElapsedMs } | + Sort-Object P90ElapsedMs -Descending | Select-Object -First 1 + + $sentence = "whole window: p50 {0}ms, p90 {1}ms over {2} answered attempt(s); worst bucket (minute {3}) p90 {4}ms" -f ` + ($Overall.P50ElapsedMs ?? '-'), ($Overall.P90ElapsedMs ?? '-'), $Overall.Ok, ` + $(if ($worstBucket) { $worstBucket.FromMinute } else { '-' }), ` + $(if ($worstBucket) { $worstBucket.P90ElapsedMs } else { '-' }) + if ($null -ne $Overall.ClampRatio) { + $sentence += (" ({0:F2}x the first bucket's)" -f $Overall.ClampRatio) + } + + [pscustomobject]@{ + P50ElapsedMs = $Overall.P50ElapsedMs + P90ElapsedMs = $Overall.P90ElapsedMs + WorstBucketFromMinute = $(if ($worstBucket) { $worstBucket.FromMinute } else { $null }) + WorstBucketP90ElapsedMs = $(if ($worstBucket) { $worstBucket.P90ElapsedMs } else { $null }) + ClampRatio = $Overall.ClampRatio + Sentence = $sentence + } +} + +function Get-ClampSummary { + <# + .SYNOPSIS + 18's whole summary, built from its checkpointed rows file: the clock-gap refusal, every + bucket, the whole-window totals and refusals-by-kind, and the headline. + + .DESCRIPTION + Callable on a rows file path alone, with the run's own clocks and window - the same call + the script makes once its load finishes, and the one a test makes against a synthetic + rows.jsonl it wrote itself. When StartedAt/EndedAt and the Stopwatch disagree by more + than a bucket (Get-ClampClockGap), the rows stay checkpointed on disk, but Buckets, + Overall and Headline all come back $null: a summary built across a gap the machine was + not awake for is not a distribution, and the caller is expected to print + Refused/RefusalReason and stop rather than read the rest of this object. + + .PARAMETER RowsPath + 18's checkpointed rows.jsonl. + + .PARAMETER StartedAt + .PARAMETER EndedAt + Wall time (Get-Date) at the start and end of the load. + + .PARAMETER WallMs + The load's own Stopwatch, in milliseconds. + + .PARAMETER Minutes + .PARAMETER BucketMinutes + The window and bucket width the run was asked for - fixes the bucket count independent + of which buckets the rows happen to touch. + #> + [CmdletBinding()] + param( + [Parameter(Mandatory)] [string] $RowsPath, + [Parameter(Mandatory)] [datetime] $StartedAt, + [Parameter(Mandatory)] [datetime] $EndedAt, + [Parameter(Mandatory)] [long] $WallMs, + [Parameter(Mandatory)] [ValidateRange(1, [int]::MaxValue)] [int] $Minutes, + [Parameter(Mandatory)] [ValidateRange(1, [int]::MaxValue)] [int] $BucketMinutes + ) + + $gap = Get-ClampClockGap -StartedAt $StartedAt -EndedAt $EndedAt -WallMs $WallMs -BucketMinutes $BucketMinutes + if ($gap.Refused) { + return [pscustomobject]@{ + Refused = $true + RefusalReason = $gap.Reason + GapMinutes = $gap.GapMinutes + Buckets = $null + Overall = $null + Headline = $null + } + } + + $rows = Get-ClampRows -Path $RowsPath + $buckets = Get-ClampBuckets -Rows $rows -StartedAt $StartedAt -Minutes $Minutes -BucketMinutes $BucketMinutes + $overall = Get-ClampOverall -Rows $rows -Buckets $buckets + $headline = Get-ClampHeadline -Overall $overall -Buckets $buckets + + [pscustomobject]@{ + Refused = $false + RefusalReason = $null + GapMinutes = $gap.GapMinutes + Buckets = $buckets + Overall = $overall + Headline = $headline + } +} diff --git a/tests/benchmarks/mock-graph-server.ps1 b/tests/benchmarks/mock-graph-server.ps1 index 7f7cce3..ed7fb07 100644 --- a/tests/benchmarks/mock-graph-server.ps1 +++ b/tests/benchmarks/mock-graph-server.ps1 @@ -1,23 +1,136 @@ -# Minimal mock Graph endpoint for the fault-injection gauntlet (06-fault-gauntlet.ps1). -# Serves GET /v1.0/users/{id} with deterministic transient faults keyed on the entity id, -# so every contender faces the identical fault schedule regardless of request order: -# id % 100 0-14 -> first attempt gets 429 (Retry-After: 1), then success -# id % 100 15-17 -> first two attempts get 503 (Retry-After: 1), then success -# otherwise -> success -# Control endpoints: GET /ping (liveness), GET /reset (clear attempt state between -# contenders), GET /stats (server-side truth: requests served, faults injected). +# Mock Graph endpoint for the benchmark gauntlets. Started and stopped by the benchmark +# that needs it (06-fault-gauntlet.ps1 and the pathological gauntlets after it). +# +# Division of labor. Two kinds of fault live here and nowhere else: +# - faults every contender must meet identically. The gauntlets compare mgx against the +# Graph SDK and against a bare Invoke-RestMethod loop, so a fault injected inside mgx +# is not a fault the other two can face; it has to be a property of the server. +# - faults that need a real socket: a connection dropped mid-body, a body trickled over +# seconds, a body that stops mid-document under an honest Content-Length. +# Everything asserted in-process belongs to the xUnit fakes instead - the handlers, sessions +# and canned collections under tests/Mgx.IntegrationTests/Fakes, which answer a request +# without a socket. Those are cheaper, they run in the unit suite, and they can see the +# request objects. Do not grow this server for a fault one of them already holds. +# +# Routes: +# GET /(v1.0|beta)/users/{id} The entity. With no plan in force it carries the default +# schedule, keyed on the entity id so request order cannot change what a contender +# meets: +# id % 100 0-14 -> first attempt gets 429 (Retry-After: 1), then success +# id % 100 15-17 -> first two attempts get 503 (Retry-After: 1), then success +# otherwise -> success +# GET /(v1.0|beta)/users The collection. One page of mock users by default; with +# "pages" in the plan, that many pages joined by a real @odata.nextLink the client +# has to follow, items shaped like the entity route's. +# anything else A benign empty page. An unknown route is not a fault, so +# a benchmark that pages one measures nothing rather than failing - which is why a +# scenario needing a real collection asks for one of the routes above by name. A +# plan rule can still target it; the empty page is the body it works on. +# +# Control endpoints. Matched before any route, so a plan whose pattern matches every path +# cannot lock the server out: +# GET /ping liveness +# GET /reset clears attempt counters and rule clocks; KEEPS the plan, because 06 +# resets between contenders and every contender must meet the same plan +# GET /stats server-side truth (below) +# GET /plan the plan in force +# POST /plan sets it, replacing whatever was there; {"rules":[]} clears it. Screened +# where it is written, not on the request that would have met it: the document's +# own top-level fields must be ones the document can have, it must be an object +# whose 'rules' is an array, every rule's field names must be ones a rule can +# have, the route must compile as a regex, the kind must be one of the seven, +# the scope must be path or server, and every numeric field must be a number of +# at least zero. A plan failing any of those is refused 400 MockPlanRejected +# naming the field, and the plan already in force stays in force. +# +# The plan. POST /plan takes: +# { +# "pages": 3, # list routes serve this many pages (default 1) +# "pageSize": 5, # items per list page (default 25) +# "rules": [ { +# "route": "^/v1\\.0/users$", # .NET regex over the URL path; the query is not +# # part of it, so every page of one enumeration +# # is another attempt on the same route +# "kind": "abort", # one of the kinds below +# "scope": "path", # count attempts per path (default), or "server" +# # for one counter across every matching path - +# # what a token expiry or a tenant-wide storm is +# "from": 3, # first attempt it fires on (default 1) +# "count": 1, # how many attempts it fires for (default: no end) +# "to": 3, # the same thing said as a last attempt +# "seconds": 10, # and fire only this long after the rule's first +# # matching request; combines with the attempt +# # window rather than replacing it +# "bytes": 64, # abort, truncatedJson: body bytes to send +# "bodyMs": 2000, # slowBody: how long the body takes +# "chunks": 10, # slowBody: how many pieces to trickle it in +# "retryAfter": 1 # throttleStorm, outage: the header value +# } ] +# } +# The first rule whose route matches governs the request, and a route the plan governs does +# not also get the default schedule: outside a rule's window it simply succeeds. Kinds: +# abort headers and part of the body, then the connection dies +# slowBody the whole body, trickled over bodyMs +# truncatedJson a complete response whose JSON stops mid-document +# expiredToken 401 with WWW-Authenticate, the shape an expired token shows +# delayedVisibility 404 while the window is open, then the resource appears +# throttleStorm 429 with Retry-After for every request in the window +# outage 503 for every attempt; with no window it never recovers +# +# /stats reports the three fields it always did - served, a body the entity route served +# whole, and faulted429/faulted503, which stay the default schedule's own tally - then one +# counter per kind under the kind's own name, then pagesServed, a list page whose body went +# out whole, and serverErrors, a request this server itself failed to answer. Every counter +# is raised after its response has been written, so /stats never affirms a fault that never +# reached the wire. A plan fault counts under its kind only - not served, and not +# pagesServed - so a run's default-schedule numbers stay comparable across releases. +# +# The accept loop is single-threaded, as it always has been: one request is answered +# before the next is read. A slowBody rule therefore holds up whatever else is in flight, +# which is the honest shape of a stalled response on a shared connection - but it means a +# scenario should trickle one body, not two hundred. param( [int] $Port = 8787 ) $ErrorActionPreference = 'Stop' +# Retry-After and duration values are rendered into headers and JSON, so the process must +# not be able to print 1,5 for 1.5 on a comma-decimal machine. +[System.Threading.Thread]::CurrentThread.CurrentCulture = [System.Globalization.CultureInfo]::InvariantCulture + $listener = [System.Net.HttpListener]::new() $listener.Prefixes.Add("http://localhost:$Port/") $listener.Start() Write-Host "mock graph listening on http://localhost:$Port/" -$attempts = @{} -$stats = @{ served = 0; faulted429 = 0; faulted503 = 0 } +$attempts = @{} # default schedule: entity id -> attempts +$ruleState = @{} # plan: "|" -> @{ n; first } +$plan = @{ pages = 1; pageSize = 25; rules = @() } +$kinds = 'abort', 'slowBody', 'truncatedJson', 'expiredToken', + 'delayedVisibility', 'throttleStorm', 'outage' +$stats = [ordered]@{ + served = 0; faulted429 = 0; faulted503 = 0 + abort = 0; slowBody = 0; truncatedJson = 0; expiredToken = 0 + delayedVisibility = 0; throttleStorm = 0; outage = 0 + pagesServed = 0; serverErrors = 0 +} + +# The numeric plan fields, and what each must convert to. Every one of them is cast where it +# is used, deep inside a response a client is already waiting on, so they are screened here +# instead - see Test-PlanNumber. +$ruleNumbers = [ordered]@{ + from = 'int'; count = 'int'; to = 'int'; bytes = 'int' + bodyMs = 'int'; chunks = 'int'; retryAfter = 'int'; seconds = 'double' +} +# Every field a rule may carry: the numeric ones above, plus the three structural fields no +# kind owns. Built from $ruleNumbers rather than listed again by hand, so a numeric field +# added there is automatically a known field here too. +$ruleFieldNames = @($ruleNumbers.Keys) + @('route', 'kind', 'scope') +# The plan document's own top-level fields. A name outside this list - "pagez" for +# "pageSize" - is not an error Get-Field can see: it just looks like a field the document +# does not set, and the default is used in its place. Screened the same way a rule's own +# fields are, so a typo here fails the same way a typo there does. +$planFieldNames = @('rules', 'pages', 'pageSize') function Send-Json($ctx, [int]$status, [string]$json, [hashtable]$headers) { $ctx.Response.StatusCode = $status @@ -29,6 +142,292 @@ function Send-Json($ctx, [int]$status, [string]$json, [hashtable]$headers) { $ctx.Response.Close() } +# The honest length is announced and only part of the body delivered, then the connection +# is killed: the client sees the transport end, not a status code. +function Send-Aborted($ctx, [string]$json, [int]$bytes) { + $full = [System.Text.Encoding]::UTF8.GetBytes($json) + if ($bytes -le 0 -or $bytes -ge $full.Length) { $bytes = [math]::Max(1, [int]($full.Length / 2)) } + $ctx.Response.StatusCode = 200 + $ctx.Response.ContentType = 'application/json' + $ctx.Response.ContentLength64 = $full.Length + $ctx.Response.OutputStream.Write($full, 0, $bytes) + $ctx.Response.OutputStream.Flush() + $ctx.Response.Abort() +} + +# A complete, well-formed HTTP response - Content-Length matches what is sent, the +# connection closes cleanly - whose JSON stops mid-document. The client fails parsing, +# not reading. +function Send-Truncated($ctx, [string]$json, [int]$bytes) { + $full = [System.Text.Encoding]::UTF8.GetBytes($json) + if ($bytes -le 0 -or $bytes -ge $full.Length) { $bytes = [math]::Max(1, [int]($full.Length / 2)) } + $ctx.Response.StatusCode = 200 + $ctx.Response.ContentType = 'application/json' + $ctx.Response.ContentLength64 = $bytes + $ctx.Response.OutputStream.Write($full, 0, $bytes) + $ctx.Response.Close() +} + +# The whole body, delivered in $chunks pieces spaced to spend about $ms total between the +# first and the last. One chunk is still a body the client waited $ms for, but the loop below +# only sleeps in the gap between writes - and there is no gap when there is one write - so +# that case sleeps before it instead. +function Send-Slow($ctx, [string]$json, [int]$ms, [int]$chunks) { + $full = [System.Text.Encoding]::UTF8.GetBytes($json) + if ($chunks -lt 1) { $chunks = 10 } + if ($chunks -gt $full.Length) { $chunks = $full.Length } + $ctx.Response.StatusCode = 200 + $ctx.Response.ContentType = 'application/json' + $ctx.Response.ContentLength64 = $full.Length + if ($chunks -eq 1) { + if ($ms -gt 0) { Start-Sleep -Milliseconds $ms } + $ctx.Response.OutputStream.Write($full, 0, $full.Length) + $ctx.Response.OutputStream.Flush() + $ctx.Response.Close() + return + } + $size = [math]::Max(1, [int][math]::Ceiling($full.Length / $chunks)) + $gap = [math]::Max(1, [int]($ms / [math]::Max(1, $chunks - 1))) + $off = 0 + while ($off -lt $full.Length) { + $n = [math]::Min($size, $full.Length - $off) + $ctx.Response.OutputStream.Write($full, $off, $n) + $ctx.Response.OutputStream.Flush() + $off += $n + if ($off -lt $full.Length) { Start-Sleep -Milliseconds $gap } + } + $ctx.Response.Close() +} + +# [ordered], not a plain hashtable: a hashtable hands ConvertTo-Json its keys in whatever +# order that process happened to hash them, so the same id answered by two servers is two +# different bodies - and a server whose point is determinism cannot serve those. +function New-MockUser([int]$id) { + [ordered]@{ + id = "00000000-0000-0000-0000-{0:D12}" -f $id + displayName = "Mock User $id" + userPrincipalName = "mock.u$id@mock.local" + mail = "mock.u$id@mock.local" + department = 'Mockestration' + jobTitle = 'Test Subject' + } +} + +function Get-ListPage([string]$version, [string]$origin, [int]$page) { + $size = if ($plan.pageSize -gt 0) { [int]$plan.pageSize } else { 25 } + $pages = if ($plan.pages -gt 0) { [int]$plan.pages } else { 1 } + if ($page -lt 1) { $page = 1 } + $first = ($page - 1) * $size + 1 + $doc = [ordered]@{ + '@odata.context' = "$origin/$version/`$metadata#users" + value = @($first..($first + $size - 1) | ForEach-Object { New-MockUser $_ }) + } + if ($page -lt $pages) { + $doc['@odata.nextLink'] = "$origin/$version/users?`$skiptoken=p$($page + 1)" + } + ConvertTo-Json -InputObject $doc -Depth 5 -Compress +} + +function Get-Field($obj, [string]$name, $default) { + if ($null -eq $obj) { return $default } + $p = $obj.PSObject.Properties[$name] + if ($p -and $null -ne $p.Value) { return $p.Value } + $default +} + +# A numeric plan field that is not a number throws where it is cast, which is halfway +# through answering a request: the client is left holding an empty 200 and the counter for a +# fault that never went out. Screened here, the same typo is a 400 naming the field. +function Test-PlanNumber($container, [string]$name, [string]$type, [string]$where) { + $value = Get-Field $container $name $null + if ($null -eq $value) { return } + try { $number = if ($type -eq 'double') { [double]$value } else { [int]$value } } + catch { throw "$where '$name' must be a number, not '$value'" } + if ($number -lt 0 -or -not [double]::IsFinite([double]$number)) { + throw "$where '$name' must be a number of at least 0, not '$value'" + } +} + +# from, to, count and seconds describe a firing window, and Test-RuleFires treats a zero on +# any of them as no bound rather than a bound at zero: an explicit from: 0 clears the lower +# bound instead of matching nothing, and to/count/seconds: 0 clears the upper bound the same +# way leaving the field out does - so a rule meant to fire briefly fires forever instead. +# Called after Test-PlanNumber has already confirmed the value is a number of at least 0, so +# the only thing left to refuse here is exactly 0. +function Test-PlanWindowField($container, [string]$name, [int]$ruleNumber) { + $value = Get-Field $container $name $null + if ($null -eq $value -or [double]$value -ge 1) { return } + throw "rule ${ruleNumber}: '$name' must be at least 1, got $value" +} + +# True when this attempt falls inside the rule's window. The attempt window always +# applies; a "seconds" duration narrows it further, measured from the rule's first +# matching request so a scenario's clock starts when its traffic does, not when the plan +# was posted. +function Test-RuleFires([int]$index, $rule, [string]$path) { + $key = if ([string](Get-Field $rule 'scope' 'path') -eq 'server') { "$index|*" } else { "$index|$path" } + if (-not $ruleState.ContainsKey($key)) { $ruleState[$key] = @{ n = 0; first = [DateTime]::UtcNow } } + $st = $ruleState[$key] + $st.n++ + + $from = [int](Get-Field $rule 'from' 1) + $to = [int](Get-Field $rule 'to' 0) + if ($to -le 0) { + $count = [int](Get-Field $rule 'count' 0) + if ($count -gt 0) { $to = $from + $count - 1 } + } + if ($st.n -lt $from) { return $false } + if ($to -gt 0 -and $st.n -gt $to) { return $false } + + $seconds = [double](Get-Field $rule 'seconds' 0) + if ($seconds -gt 0 -and ([DateTime]::UtcNow - $st.first).TotalSeconds -ge $seconds) { return $false } + $true +} + +# Returns 'whole' when the route's own body went out complete (trickled counts: all of it +# arrives), 'partial' when a fault cut it short, 'error' when a status fault replaced it, and +# $null when no rule governs this path - in which case the caller carries on with the default +# schedule. A caller counting a served body - pagesServed, served - counts 'whole' and +# nothing else; see Test-DispositionServed. +function Invoke-PlanRule($ctx, [string]$path, [string]$body) { + if ($plan.rules.Count -eq 0) { return $null } + $index = -1 + for ($i = 0; $i -lt $plan.rules.Count; $i++) { + if ($path -match [string]$plan.rules[$i].route) { $index = $i; break } + } + if ($index -lt 0) { return $null } + + $rule = $plan.rules[$index] + if (-not (Test-RuleFires $index $rule $path)) { Send-Json $ctx 200 $body $null; return 'whole' } + + $retry = [string][int](Get-Field $rule 'retryAfter' 1) + switch ([string]$rule.kind) { + 'abort' { + Send-Aborted $ctx $body ([int](Get-Field $rule 'bytes' 0)) + $stats.abort++ + return 'partial' + } + 'slowBody' { + Send-Slow $ctx $body ([int](Get-Field $rule 'bodyMs' 1000)) ([int](Get-Field $rule 'chunks' 10)) + $stats.slowBody++ + return 'whole' + } + 'truncatedJson' { + Send-Truncated $ctx $body ([int](Get-Field $rule 'bytes' 0)) + $stats.truncatedJson++ + return 'partial' + } + 'expiredToken' { + Send-Json $ctx 401 '{"error":{"code":"InvalidAuthenticationToken","message":"Access token has expired."}}' ` + @{ 'WWW-Authenticate' = 'Bearer realm="", error="invalid_token", error_description="The access token has expired."' } + $stats.expiredToken++ + return 'error' + } + 'delayedVisibility' { + Send-Json $ctx 404 '{"error":{"code":"Request_ResourceNotFound","message":"Resource does not exist or one of its queried reference-property objects are not present."}}' $null + $stats.delayedVisibility++ + return 'error' + } + 'throttleStorm' { + Send-Json $ctx 429 '{"error":{"code":"TooManyRequests","message":"mock throttle storm"}}' @{ 'Retry-After' = $retry } + $stats.throttleStorm++ + return 'error' + } + 'outage' { + Send-Json $ctx 503 '{"error":{"code":"ServiceUnavailable","message":"mock outage"}}' @{ 'Retry-After' = $retry } + $stats.outage++ + return 'error' + } + default { + # Unreachable while Set-Plan screens kinds, and loud rather than benign if that + # ever stops being true: a scenario must not report success against a fault + # that never fired. + Send-Json $ctx 500 (@{ error = @{ code = 'MockPlanError' + message = "unknown fault kind '$($rule.kind)'" } } | + ConvertTo-Json -Compress -Depth 3) $null + return 'error' + } + } +} + +# The predicate every disposition-counting caller shares: only 'whole' is a body this route +# actually served. 'partial' and 'error' reached the wire too, but under a fault's own shape, +# and are counted under that fault's kind instead - never here as well. +function Test-DispositionServed([string]$disposition) { + $disposition -eq 'whole' +} + +function Set-Plan($ctx) { + $raw = [System.IO.StreamReader]::new($ctx.Request.InputStream, [System.Text.Encoding]::UTF8).ReadToEnd() + try { + # -NoEnumerate, or a one-element array document - [{"rules":[]}] - comes back out of + # ConvertFrom-Json as that one element, not the array it was: PSCustomObject passes the + # shape check below on the object the array held, and a document that is not the + # documented shape is accepted as though it were. + $new = if ($raw.Trim()) { $raw | ConvertFrom-Json -NoEnumerate } else { $null } + # A document that is not an object at all (an array, a bare string, a number) has no + # properties to read 'rules' off, and Get-Field's empty default for a missing property + # is indistinguishable from {"rules":[]} - the documented way to clear the plan on + # purpose. So the shape is checked directly, on the object graph ConvertFrom-Json + # already built, before anything reads a field out of it. + if ($new -isnot [System.Management.Automation.PSCustomObject]) { + $shape = if ($null -eq $new) { 'empty' } else { $new.GetType().Name } + throw "plan document must be a JSON object with a 'rules' array, not $shape" + } + # $rulesProp.Value is read by direct member access everywhere below, never assigned + # through an if/else branch: passing an array through a script block's trailing + # expression - even just to pick a default - re-enumerates it onto the pipeline, and a + # one-rule plan comes back out the other side as that one rule instead of an array of + # it. @() on the way into the foreach would paper over the read, not fix it. + $rulesProp = $new.PSObject.Properties['rules'] + if (-not $rulesProp -or $rulesProp.Value -isnot [array]) { + $shape = if (-not $rulesProp -or $null -eq $rulesProp.Value) { 'missing' } else { $rulesProp.Value.GetType().Name } + throw "plan 'rules' must be an array, not $shape" + } + $rules = $rulesProp.Value + # Screened here, not on the first request that would have matched: a scenario whose + # plan does not mean what it says must fail where it is written. Nothing below is + # replaced until the whole document has passed, so a refused plan leaves the one in + # force untouched rather than half of it. + foreach ($name in $new.PSObject.Properties.Name) { + if ($name -notin $script:planFieldNames) { + throw "plan has an unknown field '$name'; expected one of $($script:planFieldNames -join ', ')" + } + } + Test-PlanNumber $new 'pages' 'int' 'plan' + Test-PlanNumber $new 'pageSize' 'int' 'plan' + for ($i = 0; $i -lt $rules.Count; $i++) { + $r = $rules[$i] + if (-not $r.route -or -not $r.kind) { throw 'every rule needs a route and a kind' } + $null = [regex]::new([string]$r.route) + if ([string]$r.kind -notin $script:kinds) { + throw "unknown fault kind '$($r.kind)'; expected one of $($script:kinds -join ', ')" + } + foreach ($name in $r.PSObject.Properties.Name) { + if ($name -notin $script:ruleFieldNames) { + throw "rule '$($r.kind)' has an unknown field '$name'; expected one of $($script:ruleFieldNames -join ', ')" + } + } + $scope = [string](Get-Field $r 'scope' 'path') + if ($scope -notin 'path', 'server') { throw "scope must be path or server, not '$scope'" } + foreach ($field in $script:ruleNumbers.Keys) { + Test-PlanNumber $r $field $script:ruleNumbers[$field] "rule '$($r.kind)'" + } + foreach ($field in 'from', 'to', 'count', 'seconds') { + Test-PlanWindowField $r $field ($i + 1) + } + } + $plan.rules = $rules + $plan.pages = [int](Get-Field $new 'pages' 1) + $plan.pageSize = [int](Get-Field $new 'pageSize' 25) + $ruleState.Clear() + Send-Json $ctx 200 (@{ ok = $true; rules = $rules.Count } | ConvertTo-Json -Compress) $null + } + catch { + Send-Json $ctx 400 (@{ error = @{ code = 'MockPlanRejected'; message = $_.Exception.Message } } | ConvertTo-Json -Compress) $null + } +} + while ($listener.IsListening) { $ctx = $listener.GetContext() $path = $ctx.Request.Url.AbsolutePath @@ -36,47 +435,74 @@ while ($listener.IsListening) { try { switch -Regex ($path) { '^/ping$' { Send-Json $ctx 200 '{"ok":true}' $null; continue } - '^/reset$' { $attempts.Clear(); Send-Json $ctx 200 '{"reset":true}' $null; continue } - '^/stats$' { - Send-Json $ctx 200 (@{ served = $stats.served; faulted429 = $stats.faulted429; faulted503 = $stats.faulted503 } | ConvertTo-Json -Compress) $null + '^/reset$' { $attempts.Clear(); $ruleState.Clear(); Send-Json $ctx 200 '{"reset":true}' $null; continue } + '^/stats$' { Send-Json $ctx 200 (ConvertTo-Json -InputObject $stats -Compress) $null; continue } + '^/plan$' { + if ($ctx.Request.HttpMethod -in 'POST', 'PUT') { Set-Plan $ctx } + else { Send-Json $ctx 200 (ConvertTo-Json -InputObject $plan -Depth 5 -Compress) $null } continue } '^/(v1\.0|beta)/users/(?[^/?]+)$' { - $stats.served++ $id = [int]($Matches['id'] -replace '\D', '0') + $user = New-MockUser $id | ConvertTo-Json -Compress + $disposition = Invoke-PlanRule $ctx $path $user + if ($null -ne $disposition) { + # A plan fault does not count as served, same predicate as the list + # route's pagesServed: only a 'whole' disposition is a body this route + # actually served. + if (Test-DispositionServed $disposition) { $stats.served++ } + continue + } + if (-not $attempts.ContainsKey($id)) { $attempts[$id] = 0 } $attempts[$id]++ $n = $attempts[$id] $bucket = $id % 100 if ($bucket -lt 15 -and $n -eq 1) { - $stats.faulted429++ Send-Json $ctx 429 '{"error":{"code":"TooManyRequests","message":"mock throttle"}}' @{ 'Retry-After' = '1' } + $stats.faulted429++ } elseif ($bucket -ge 15 -and $bucket -lt 18 -and $n -le 2) { - $stats.faulted503++ Send-Json $ctx 503 '{"error":{"code":"ServiceUnavailable","message":"mock outage"}}' @{ 'Retry-After' = '1' } + $stats.faulted503++ } else { - $user = @{ - id = "00000000-0000-0000-0000-{0:D12}" -f $id - displayName = "Mock User $id" - userPrincipalName = "mock.u$id@mock.local" - mail = "mock.u$id@mock.local" - department = 'Mockestration' - jobTitle = 'Test Subject' - } | ConvertTo-Json -Compress Send-Json $ctx 200 $user $null } + $stats.served++ + continue + } + '^/(v1\.0|beta)/users$' { + # The list route counts under pagesServed and nowhere else: served is the + # entity route's tally, and a 2.1.4 run has to stay comparable with this one. + $version = $Matches[1] + $origin = $ctx.Request.Url.GetLeftPart([System.UriPartial]::Authority) + $page = if ($ctx.Request.Url.Query -match '\$skiptoken=p(\d+)') { [int]$Matches[1] } else { 1 } + $body = Get-ListPage $version $origin $page + $disposition = Invoke-PlanRule $ctx $path $body + if ($null -eq $disposition) { Send-Json $ctx 200 $body $null; $disposition = 'whole' } + # A page cut off mid-body is not a page served; it is counted by the kind + # that cut it. + if (Test-DispositionServed $disposition) { $stats.pagesServed++ } continue } default { - # Anything else (org probe, list endpoints): benign empty page - Send-Json $ctx 200 '{"value":[]}' $null + # Anything else (org probe, unimplemented endpoints): benign empty page + $empty = '{"value":[]}' + if ($null -eq (Invoke-PlanRule $ctx $path $empty)) { Send-Json $ctx 200 $empty $null } } } } catch { - try { $ctx.Response.Close() } catch { } + # A closed response nothing was written to reaches the client as 200 with an empty + # body, which a benchmark reads as an answer. A harness bug says so instead. + $stats.serverErrors++ + try { + Send-Json $ctx 500 (@{ error = @{ code = 'MockServerError' + message = $_.Exception.Message } } | + ConvertTo-Json -Compress -Depth 3) $null + } + catch { try { $ctx.Response.Close() } catch { } } } } diff --git a/tests/benchmarks/run.ps1 b/tests/benchmarks/run.ps1 index a292695..f922b42 100644 --- a/tests/benchmarks/run.ps1 +++ b/tests/benchmarks/run.ps1 @@ -7,13 +7,22 @@ # ./run.ps1 # everything except the slow ones # ./run.ps1 -IncludeSlow # everything, including 04 full baselines (~2h) # ./run.ps1 -Only 01,05 # just those benchmarks +# ./run.ps1 -RecordBaseline # pin what the run produced into baseline.json +# ./run.ps1 -CompareBaseline # read the run against the pinned baseline.json param( [string[]] $Only, [switch] $IncludeSlow, - [int] $CooldownSeconds = 300 + [int] $CooldownSeconds = 300, + [switch] $RecordBaseline, + [switch] $CompareBaseline ) $ErrorActionPreference = 'Stop' +# Export-BenchBaseline, Compare-BenchBaseline and the table of headline metrics they share live +# here. It is also what forces invariant culture, so the delta lines below print 12.5% and +# never 12,5%. +. (Join-Path $PSScriptRoot 'common.ps1') + $logDir = Join-Path $PSScriptRoot 'results/logs' if (-not (Test-Path $logDir)) { New-Item -ItemType Directory -Path $logDir -Force | Out-Null } @@ -33,14 +42,38 @@ $suite = @( @{ id = '10'; script = '10-pacing-under-real-throttling.ps1'; args = @(); cooldown = $false } ) +$promoteOnly = @() if ($Only) { # pwsh -File passes "05,02" as one literal string - split so both call styles work $Only = @($Only | ForEach-Object { $_ -split ',' } | Where-Object { $_ }) $suite = @($suite | Where-Object { $Only -contains $_.id }) + + # An id Get-BenchScenario knows the metrics for but that never runs inside this suite - + # 18, a live-tenant scenario driven standalone through -DriveId/-DriveUri - is not the + # same thing as an id nobody has ever heard of. Without -RecordBaseline it is refused by + # name rather than folded into the generic "no benchmarks matched"; with it, it is + # accepted and promoted below from its own latest recorded result, never run here. + $matchedIds = @($suite | ForEach-Object { $_.id }) + $unmatched = @($Only | Where-Object { $matchedIds -notcontains $_ }) + if ($unmatched.Count -gt 0) { + $standalone = @(Get-BenchScenario -Id $unmatched -WarningAction SilentlyContinue) + $promoteOnly = @($standalone | ForEach-Object { $_.Id }) + if ($promoteOnly.Count -gt 0 -and -not $RecordBaseline) { + $sentences = $standalone | ForEach-Object { + "$($_.Id) is not in the suite; run tests/benchmarks/$($_.Script) directly, then promote it with -RecordBaseline -Only $($_.Id)." + } + throw ($sentences -join ' ') + } + } } -if ($suite.Count -eq 0) { throw "no benchmarks matched -Only $($Only -join ',')" } +if ($suite.Count -eq 0 -and $promoteOnly.Count -eq 0) { throw "no benchmarks matched -Only $($Only -join ',')" } $stamp = Get-Date -Format 'yyyyMMdd-HHmmss' +# Both the promotion and the comparison read results/.json back, which is an +# append-only log: without a floor they would read the newest entry a benchmark ever wrote, +# and a benchmark that died before recording would be pinned or compared from a run that +# happened last month. +$runStart = Get-Date $outcomes = @() foreach ($b in $suite) { $log = Join-Path $logDir "$($b.id)-$stamp.log" @@ -62,3 +95,30 @@ Write-Host '' Write-Host '=== SUITE COMPLETE ===' $outcomes | Format-Table -AutoSize Write-Host "per-benchmark results in $(Join-Path $PSScriptRoot 'results')" + +$ran = @($suite | ForEach-Object { $_.id }) + +if ($RecordBaseline) { + # The seven scenarios the release baseline is made of. A full run also executes 03, 06 and + # 07, which are read on their own merits rather than pinned - naming one with -Only pins it + # anyway, which is how the no-tenant gauntlet gets into a baseline on a laptop. + $baselineDefault = @('01', '02', '04', '05', '08', '09', '10') + $promote = if ($Only) { $ran } else { @($ran | Where-Object { $baselineDefault -contains $_ }) } + Write-Host '' + Write-Host '=== RECORDING BASELINE ===' + if ($promote.Count -eq 0 -and $promoteOnly.Count -eq 0) { + Write-Warning "none of the benchmarks this run executed are baseline scenarios ($($baselineDefault -join ', ')) - nothing promoted." + } + else { + if ($promote.Count -gt 0) { Export-BenchBaseline -Benchmark $promote -Since $runStart } + # $promoteOnly never ran in this invocation, so there is no fresh entry to floor with + # -Since $runStart - it promotes from whatever its own standalone run last recorded, + # however long before this call that was. + if ($promoteOnly.Count -gt 0) { Export-BenchBaseline -Benchmark $promoteOnly } + } +} + +if ($CompareBaseline) { + # Reports, never gates: exit code is the suite's, not the comparison's. + $null = Compare-BenchBaseline -Benchmark $ran -Since $runStart +} diff --git a/tests/benchmarks/tenant-resilience-comparison.ps1 b/tests/benchmarks/tenant-resilience-comparison.ps1 deleted file mode 100644 index f2f55ee..0000000 --- a/tests/benchmarks/tenant-resilience-comparison.ps1 +++ /dev/null @@ -1,102 +0,0 @@ -# Benchmark: Microsoft.Graph SDK vs Mgx resilience vs Invoke-MgxRequest. -# -# Compares wall-clock time and memory for the same operation across three -# approaches. Run this on your own tenant to see real numbers. -# -# Requirements: Connect-MgGraph -Scopes "User.Read.All" - -Import-Module Microsoft.Graph.Users -Import-Module M365DSC.mgx - -$property = "id,displayName,mail,department,jobTitle,accountEnabled" -$selectArr = @("id","displayName","mail","department","jobTitle","accountEnabled") - -Write-Host "Benchmarking against your tenant..." -Write-Host "" - -# --- 1. Bare SDK: Get-MgUser -All --- -Write-Host "1) Get-MgUser -All (bare SDK, no resilience)" -[GC]::Collect() -$memBefore = [GC]::GetTotalMemory($true) -$sw = [System.Diagnostics.Stopwatch]::StartNew() - -$sdkUsers = Get-MgUser -All -Property $selectArr -$sw.Stop() - -$memAfter = [GC]::GetTotalMemory($false) -$sdkCount = $sdkUsers.Count -$sdkMs = $sw.ElapsedMilliseconds -$sdkMem = [math]::Round(($memAfter - $memBefore) / 1MB, 1) -Write-Host " $sdkCount users in ${sdkMs}ms (~${sdkMem} MB)" -$sdkUsers = $null - -# --- 2. SDK + Enable-MgxResilience --- -Write-Host "" -Write-Host "2) Get-MgUser -All (with Enable-MgxResilience)" -Enable-MgxResilience -[GC]::Collect() -$memBefore = [GC]::GetTotalMemory($true) -$sw = [System.Diagnostics.Stopwatch]::StartNew() - -$resilientUsers = Get-MgUser -All -Property $selectArr -$sw.Stop() - -$memAfter = [GC]::GetTotalMemory($false) -$resilientCount = $resilientUsers.Count -$resilientMs = $sw.ElapsedMilliseconds -$resilientMem = [math]::Round(($memAfter - $memBefore) / 1MB, 1) -Write-Host " $resilientCount users in ${resilientMs}ms (~${resilientMem} MB)" -Write-Host " (retry + circuit breaker + rate limiting active)" -Disable-MgxResilience -$resilientUsers = $null - -# --- 3. Invoke-MgxRequest -All (streaming pagination) --- -Write-Host "" -Write-Host "3) Invoke-MgxRequest -All (streaming pagination)" -[GC]::Collect() -$memBefore = [GC]::GetTotalMemory($true) -$sw = [System.Diagnostics.Stopwatch]::StartNew() - -$mgxUsers = Invoke-MgxRequest /users -All -Property $selectArr -$sw.Stop() - -$memAfter = [GC]::GetTotalMemory($false) -$mgxCount = $mgxUsers.Count -$mgxMs = $sw.ElapsedMilliseconds -$mgxMem = [math]::Round(($memAfter - $memBefore) / 1MB, 1) -Write-Host " $mgxCount users in ${mgxMs}ms (~${mgxMem} MB)" -$mgxUsers = $null - -# --- 4. Export-MgxCollection (constant memory, disk streaming) --- -Write-Host "" -Write-Host "4) Export-MgxCollection (JSONL to disk, constant memory)" -$outFile = Join-Path ([System.IO.Path]::GetTempPath()) "mgx-bench.jsonl" -[GC]::Collect() -$memBefore = [GC]::GetTotalMemory($true) -$sw = [System.Diagnostics.Stopwatch]::StartNew() - -$exportResult = Export-MgxCollection /users -OutputFile $outFile -All -Property $selectArr -$sw.Stop() - -$memAfter = [GC]::GetTotalMemory($false) -$exportMs = $sw.ElapsedMilliseconds -$exportMem = [math]::Round(($memAfter - $memBefore) / 1MB, 1) -Write-Host " $($exportResult.ItemCount) users in ${exportMs}ms (~${exportMem} MB)" -Remove-Item $outFile -ErrorAction SilentlyContinue - -# --- Summary --- -Write-Host "" -Write-Host "=== Summary ===" -Write-Host "" - -$results = @( - [PSCustomObject]@{ Approach = "Get-MgUser (bare SDK)"; Users = $sdkCount; TimeMs = $sdkMs; MemMB = $sdkMem; Resilience = "None" } - [PSCustomObject]@{ Approach = "Get-MgUser + MgxResilience"; Users = $resilientCount; TimeMs = $resilientMs; MemMB = $resilientMem; Resilience = "Retry + CB + RL" } - [PSCustomObject]@{ Approach = "Invoke-MgxRequest -All"; Users = $mgxCount; TimeMs = $mgxMs; MemMB = $mgxMem; Resilience = "Full stack" } - [PSCustomObject]@{ Approach = "Export-MgxCollection (JSONL)"; Users = $exportResult.ItemCount; TimeMs = $exportMs; MemMB = $exportMem; Resilience = "Full stack" } -) -$results | Format-Table -AutoSize - -# Show telemetry for the Mgx operations -Write-Host "Telemetry (Mgx operations only):" -Get-MgxTelemetry