diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml new file mode 100644 index 0000000..5a054f1 --- /dev/null +++ b/.github/workflows/performance.yml @@ -0,0 +1,24 @@ +name: Performance guard + +on: + pull_request: + branches: [ "main" ] + push: + branches: [ "main" ] + workflow_dispatch: + +permissions: + contents: read + +jobs: + release-benchmark: + name: Release 10k regression gate + runs-on: macos-latest + timeout-minutes: 20 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run two-strike performance gate + run: bash script/ci_performance_gate.sh diff --git a/Sources/StorageScopeBenchmark/main.swift b/Sources/StorageScopeBenchmark/main.swift index 55851b6..3df84dc 100644 --- a/Sources/StorageScopeBenchmark/main.swift +++ b/Sources/StorageScopeBenchmark/main.swift @@ -6,6 +6,7 @@ struct BenchmarkArguments { var useSyntheticFixture = false var keepFixture = false var showFullPath = false + var streamingCallbacks = false /// User-requested file count for the synthetic fixture. 0 means "use the v0.5.0 curated /// 7-file default". >0 builds the scaled generator (`depth` directory levels deep, /// `duplicateRatio` of items emitted as content-identical duplicates). @@ -21,12 +22,13 @@ func usage() -> String { Usage: StorageScopeBenchmark [--show-full-path] StorageScopeBenchmark --synthetic [--keep-fixture] [--show-full-path] - StorageScopeBenchmark --synthetic --items [--depth ] [--duplicates <0..1>] [--keep-fixture] + StorageScopeBenchmark --synthetic --items [--depth ] [--duplicates <0..1>] [--streaming] [--keep-fixture] Scaled fixtures (--items, --depth, --duplicates) build a synthetic tree of N files distributed across up to depth directory levels, with an optional fraction of duplicates (pairs sharing identical content). Useful for capturing v0.5.x perf baselines at 10k / 100k / 500k items. + Pass --streaming to install app-like progress and partial-snapshot callbacks. Examples: swift run StorageScopeBenchmark --synthetic --items 100000 --depth 8 --duplicates 0.2 @@ -55,6 +57,8 @@ func parseArguments(_ rawArguments: [String]) throws -> BenchmarkArguments { arguments.keepFixture = true case "--show-full-path": arguments.showFullPath = true + case "--streaming": + arguments.streamingCallbacks = true case "--items": let raw = try consumeNextValue(flag: value) guard let parsed = Int(raw), parsed >= 0 else { @@ -142,7 +146,8 @@ do { let report = try ScanBenchmarkRunner().run( rootURL: rootURL, - showFullPath: arguments.showFullPath + showFullPath: arguments.showFullPath, + streamingCallbacks: arguments.streamingCallbacks ) print(report.text) } catch { @@ -151,4 +156,4 @@ do { Data("StorageScopeBenchmark failed: \(message)\n\n\(usage())\n".utf8) ) exit(2) -} \ No newline at end of file +} diff --git a/Sources/StorageScopeCore/Models/StorageScan.swift b/Sources/StorageScopeCore/Models/StorageScan.swift index 499927d..8a8715c 100644 --- a/Sources/StorageScopeCore/Models/StorageScan.swift +++ b/Sources/StorageScopeCore/Models/StorageScan.swift @@ -25,8 +25,11 @@ public struct StorageScan: Sendable { public let duplicateCandidateItemLimit: Int public let duplicateCandidateItemsRetained: Int public let duplicateCandidateItemsConsidered: Int + public let duplicateCandidateEvictionCount: Int public let duplicateCandidateLimitReached: Bool + public let snapshotBuildCount: Int public let duplicateVerificationDuration: TimeInterval + public let duplicateVerificationBytesRead: Int64 public let enumerateDuration: TimeInterval public let cleanupCandidates: [CleanupCandidate] public let isPartial: Bool @@ -51,8 +54,11 @@ public struct StorageScan: Sendable { duplicateCandidateItemLimit: Int = 0, duplicateCandidateItemsRetained: Int = 0, duplicateCandidateItemsConsidered: Int = 0, + duplicateCandidateEvictionCount: Int = 0, duplicateCandidateLimitReached: Bool = false, + snapshotBuildCount: Int = 0, duplicateVerificationDuration: TimeInterval = 0, + duplicateVerificationBytesRead: Int64 = 0, enumerateDuration: TimeInterval = 0, cleanupCandidates: [CleanupCandidate], isPartial: Bool = false @@ -75,8 +81,11 @@ public struct StorageScan: Sendable { self.duplicateCandidateItemLimit = duplicateCandidateItemLimit self.duplicateCandidateItemsRetained = duplicateCandidateItemsRetained self.duplicateCandidateItemsConsidered = duplicateCandidateItemsConsidered + self.duplicateCandidateEvictionCount = duplicateCandidateEvictionCount self.duplicateCandidateLimitReached = duplicateCandidateLimitReached + self.snapshotBuildCount = snapshotBuildCount self.duplicateVerificationDuration = duplicateVerificationDuration + self.duplicateVerificationBytesRead = duplicateVerificationBytesRead self.enumerateDuration = enumerateDuration self.cleanupCandidates = cleanupCandidates self.isPartial = isPartial diff --git a/Sources/StorageScopeCore/Services/FileSystemScanner.swift b/Sources/StorageScopeCore/Services/FileSystemScanner.swift index 82351f9..a87fba9 100644 --- a/Sources/StorageScopeCore/Services/FileSystemScanner.swift +++ b/Sources/StorageScopeCore/Services/FileSystemScanner.swift @@ -100,6 +100,8 @@ public final class FileSystemScanner { let cores = ProcessInfo.processInfo.processorCount return min(6, max(2, cores / 2)) }() + private static let duplicatePrefixByteCount = 64 * 1_024 + /// os_signpost surface for Instruments. Subsystem mirrors the bundle identifier prefix; /// category ties Scanner-only work together so it can be filtered from app-side spans. @@ -161,6 +163,7 @@ public final class FileSystemScanner { cancellation: cancellation ) let duplicateVerificationDuration = Date().timeIntervalSince(duplicateVerificationStartedAt) + let duplicateVerificationBytesRead = accumulator.duplicateVerificationBytesRead let finishedAt = Date() os_signpost(.end, log: Self.log, name: "scan", signpostID: Self.signpostID, @@ -185,8 +188,11 @@ public final class FileSystemScanner { duplicateCandidateItemLimit: accumulator.duplicateCandidateItemLimit, duplicateCandidateItemsRetained: accumulator.duplicateCandidateItemsRetained, duplicateCandidateItemsConsidered: accumulator.duplicateCandidateItemsConsidered, + duplicateCandidateEvictionCount: accumulator.duplicateCandidateEvictionCount, duplicateCandidateLimitReached: accumulator.duplicateCandidateLimitReached, + snapshotBuildCount: accumulator.snapshotBuildCount, duplicateVerificationDuration: duplicateVerificationDuration, + duplicateVerificationBytesRead: duplicateVerificationBytesRead, enumerateDuration: enumerateDuration, cleanupCandidates: accumulator.cleanupCandidates( rootID: rootItem.id, @@ -223,36 +229,13 @@ public final class FileSystemScanner { let ioSemaphore = DispatchSemaphore(value: Self.hashConcurrency) let cacheLock = NSLock() - var hashedItems: [HashedStorageItem] = [] - hashedItems.reserveCapacity(group.items.count) - - for item in group.items { - // Cooperative cancellation: re-check before opening I/O for the next file so a - // cancelled scan doesn't keep burning file descriptors or queuing reads behind - // the ioSemaphore. Cancellation aborts the whole on-demand verify because the - // caller asked us to stop. - try cancellation?.check() - - if let hashed = try hashedFormItem( - item, - ioSemaphore: ioSemaphore, - cacheLock: cacheLock, - cancellation: cancellation - ) { - hashedItems.append(hashed) - } - } - - let groupedByHash = Dictionary(grouping: hashedItems, by: \.checksum) - return groupedByHash.compactMap { checksum, hashedItems in - let items = hashedItems.map(\.item).sorted { $0.url.path < $1.url.path } - return items.count > 1 ? VerifiedDuplicateGroup(checksum: checksum, byteSize: group.byteSize, items: items) : nil - }.sorted { lhs, rhs in - if lhs.reclaimableBytes == rhs.reclaimableBytes { - return lhs.byteSize > rhs.byteSize - } - return lhs.reclaimableBytes > rhs.reclaimableBytes - } + return try verifiedDuplicateGroups( + in: group, + ioSemaphore: ioSemaphore, + cacheLock: cacheLock, + recordBytesRead: nil, + cancellation: cancellation + ) } private func scanItem( @@ -293,11 +276,8 @@ public final class FileSystemScanner { ) } - accumulator.recordVisit(path: url.path) - if values?.isSymbolicLink == true { let size = Int64(values?.fileSize ?? 0) - accumulator.recordBytes(size) let item = StorageItem( url: url, kind: .alias, @@ -309,7 +289,7 @@ public final class FileSystemScanner { isReadable: values?.isReadable ?? true, fileExtension: url.pathExtension.nonEmptyLowercased ) - accumulator.recordItem(item) + accumulator.recordScannedItem(item, countedBytes: size, path: url.path) return item } @@ -319,7 +299,6 @@ public final class FileSystemScanner { guard isDirectory else { let logicalSize = Int64(values?.fileSize ?? 0) let allocatedSize = Int64(values?.totalFileAllocatedSize ?? values?.fileAllocatedSize ?? values?.fileSize ?? 0) - accumulator.recordBytes(max(logicalSize, allocatedSize)) let item = StorageItem( url: url, @@ -332,10 +311,12 @@ public final class FileSystemScanner { isReadable: values?.isReadable ?? true, fileExtension: url.pathExtension.nonEmptyLowercased ) - accumulator.recordItem(item) + accumulator.recordScannedItem(item, countedBytes: max(logicalSize, allocatedSize), path: url.path) return item } + accumulator.recordVisit(path: url.path) + do { let directoryOptions: FileManager.DirectoryEnumerationOptions = options.includeHidden ? [] : [.skipsHiddenFiles] let childURLs = try fileManager.contentsOfDirectory( @@ -355,12 +336,6 @@ public final class FileSystemScanner { var didCancel = false DispatchQueue.concurrentPerform(iterations: childURLs.count) { index in - // Each iteration bridges several Foundation objects (URLResourceValues, - // NSNumber signpost objects, Date/NSString instances inside scanItem). With - // no pool, concurrentPerform's calling thread never gets an autorelease drain - // point until the whole call returns, so these accumulate for the duration of - // a potentially huge directory enumeration. Draining per-iteration bounds peak - // RSS instead of letting it grow with the directory's child count. autoreleasepool { if cancellation?.isCancelled ?? false { return @@ -378,10 +353,6 @@ public final class FileSystemScanner { try cancellation?.check() if options.excludeEnabled, Self.isExcluded(childURLs[index], options: options) { - // Skip entirely: don't stat, count, or recurse. Leave the slot nil - // so the `for case let child?` filter below treats it as absent, - // matching the established "skip this child" pattern used for - // per-child errors elsewhere in this loop. return } @@ -398,9 +369,6 @@ public final class FileSystemScanner { didCancel = true cancellationLock.unlock() } catch { - // Unreachable: scanItem only throws FileSystemScannerError.cancelled; - // ingestion/directory-enumeration failures are caught internally and - // surface as .inaccessible StorageItems. Belt-and-braces guard. let unreachable = StorageItem( url: childURLs[index], kind: .inaccessible, @@ -450,7 +418,6 @@ public final class FileSystemScanner { } catch FileSystemScannerError.cancelled { throw FileSystemScannerError.cancelled } catch { - accumulator.recordInaccessible(path: url.path) let item = StorageItem( url: url, kind: .inaccessible, @@ -462,6 +429,7 @@ public final class FileSystemScanner { isReadable: false, fileExtension: nil ) + accumulator.recordInaccessible(path: url.path) accumulator.recordItem(item) return item } @@ -545,32 +513,13 @@ public final class FileSystemScanner { try cancellation?.check() let sizeGroup = verificationGroups[groupIndex] - var hashedItems: [HashedStorageItem] = [] - hashedItems.reserveCapacity(sizeGroup.items.count) - - for item in sizeGroup.items { - // Per-item cancellation probe keeps an outlier slow hash from orphaning - // the rest of a group's work after the user cancels. - if cancellation?.isCancelled ?? false { - markCancelled() - return - } - - if let hashed = try hashedFormItem( - item, - ioSemaphore: ioSemaphore, - cacheLock: cacheLock, - cancellation: cancellation - ) { - hashedItems.append(hashed) - } - } - - let groupedByHash = Dictionary(grouping: hashedItems, by: \.checksum) - let verifiedForSize = groupedByHash.compactMap { checksum, hashedItems -> VerifiedDuplicateGroup? in - let items = hashedItems.map(\.item).sorted { $0.url.path < $1.url.path } - return items.count > 1 ? VerifiedDuplicateGroup(checksum: checksum, byteSize: sizeGroup.byteSize, items: items) : nil - } + let verifiedForSize = try verifiedDuplicateGroups( + in: sizeGroup, + ioSemaphore: ioSemaphore, + cacheLock: cacheLock, + recordBytesRead: accumulator.recordDuplicateVerificationBytes, + cancellation: cancellation + ) guard !verifiedForSize.isEmpty else { return @@ -600,6 +549,103 @@ public final class FileSystemScanner { } } + private func verifiedDuplicateGroups( + in sizeGroup: DuplicateSizeGroup, + ioSemaphore: DispatchSemaphore, + cacheLock: NSLock, + recordBytesRead: ((Int) -> Void)?, + cancellation: ScanCancellation? + ) throws -> [VerifiedDuplicateGroup] { + var cachedFullHashesByPath: [String: String] = [:] + cachedFullHashesByPath.reserveCapacity(sizeGroup.items.count) + + for item in sizeGroup.items { + try cancellation?.check() + let cacheKey = DuplicateHashCache.LookupKey(item: item) + if let cached = hashCache?.checksum(for: cacheKey) { + cachedFullHashesByPath[cacheKey.path] = cached + } + } + + if cachedFullHashesByPath.count == sizeGroup.items.count { + let hashedItems = sizeGroup.items.compactMap { item -> HashedStorageItem? in + let path = DuplicateHashCache.LookupKey(item: item).path + guard let checksum = cachedFullHashesByPath[path] else { return nil } + return HashedStorageItem(checksum: checksum, item: item) + } + return verifiedDuplicateGroups(from: hashedItems, byteSize: sizeGroup.byteSize) + } + + var prefixHashedItems: [PrefixHashedStorageItem] = [] + prefixHashedItems.reserveCapacity(sizeGroup.items.count) + + for item in sizeGroup.items { + if cancellation?.isCancelled ?? false { + throw FileSystemScannerError.cancelled + } + + if let prefixed = try prefixHashedFormItem( + item, + ioSemaphore: ioSemaphore, + recordBytesRead: recordBytesRead, + cancellation: cancellation + ) { + prefixHashedItems.append(prefixed) + } + } + + let groupedByPrefix = Dictionary(grouping: prefixHashedItems, by: \.prefixChecksum) + var fullyHashedItems: [HashedStorageItem] = [] + + for prefixGroup in groupedByPrefix.values where prefixGroup.count > 1 { + for prefixed in prefixGroup { + let cacheKey = DuplicateHashCache.LookupKey(item: prefixed.item) + if let cached = cachedFullHashesByPath[cacheKey.path] { + fullyHashedItems.append(HashedStorageItem(checksum: cached, item: prefixed.item)) + continue + } + + if prefixed.isCompleteFile { + if let hashCache { + cacheLock.lock() + hashCache.record(cacheKey, checksum: prefixed.prefixChecksum) + cacheLock.unlock() + } + fullyHashedItems.append(HashedStorageItem(checksum: prefixed.prefixChecksum, item: prefixed.item)) + continue + } + + if let hashed = try hashedFormItem( + prefixed.item, + ioSemaphore: ioSemaphore, + cacheLock: cacheLock, + recordBytesRead: recordBytesRead, + cancellation: cancellation + ) { + fullyHashedItems.append(hashed) + } + } + } + + return verifiedDuplicateGroups(from: fullyHashedItems, byteSize: sizeGroup.byteSize) + } + + private func verifiedDuplicateGroups( + from hashedItems: [HashedStorageItem], + byteSize: Int64 + ) -> [VerifiedDuplicateGroup] { + let groupedByHash = Dictionary(grouping: hashedItems, by: \.checksum) + return groupedByHash.compactMap { checksum, hashedItems -> VerifiedDuplicateGroup? in + let items = hashedItems.map(\.item).sorted { $0.url.path < $1.url.path } + return items.count > 1 ? VerifiedDuplicateGroup(checksum: checksum, byteSize: byteSize, items: items) : nil + }.sorted { lhs, rhs in + if lhs.reclaimableBytes == rhs.reclaimableBytes { + return lhs.byteSize > rhs.byteSize + } + return lhs.reclaimableBytes > rhs.reclaimableBytes + } + } + private func duplicateGroupsWithinVerificationBudget( _ sizeGroups: [DuplicateSizeGroup], options: ScanOptions @@ -639,7 +685,11 @@ public final class FileSystemScanner { return plannedGroups } - private func sha256Checksum(for url: URL, cancellation: ScanCancellation?) throws -> String { + private func sha256Checksum( + for url: URL, + recordBytesRead: ((Int) -> Void)?, + cancellation: ScanCancellation? + ) throws -> String { // Probe cancellation before opening so a cancelled batch doesn't keep paying the // cost of `FileHandle(forReadingFrom:)` for files that no one will ever read. try cancellation?.check() @@ -666,12 +716,80 @@ public final class FileSystemScanner { if data.isEmpty { break } + recordBytesRead?(data.count) hasher.update(data: data) } return hasher.finalize().hexEncodedString() } + private func prefixChecksum( + for url: URL, + maxBytes: Int, + recordBytesRead: ((Int) -> Void)?, + cancellation: ScanCancellation? + ) throws -> (checksum: String, bytesRead: Int) { + try cancellation?.check() + + let handle = try FileHandle(forReadingFrom: url) + defer { + do { + try handle.close() + } catch { + os_signpost(.event, log: Self.log, name: "handle_close_failed", + "path=%{public}@ reason=%{public}@", url.path, "\(error)") + } + } + + var remaining = max(0, maxBytes) + var bytesRead = 0 + var hasher = SHA256() + while remaining > 0 { + try cancellation?.check() + let readSize = min(remaining, 1_048_576) + let data = try handle.read(upToCount: readSize) ?? Data() + if data.isEmpty { + break + } + bytesRead += data.count + remaining -= data.count + recordBytesRead?(data.count) + hasher.update(data: data) + } + + return (hasher.finalize().hexEncodedString(), bytesRead) + } + + private func prefixHashedFormItem( + _ item: StorageItem, + ioSemaphore: DispatchSemaphore, + recordBytesRead: ((Int) -> Void)?, + cancellation: ScanCancellation? + ) throws -> PrefixHashedStorageItem? { + ioSemaphore.wait() + defer { ioSemaphore.signal() } + + do { + let result = try prefixChecksum( + for: item.url, + maxBytes: Self.duplicatePrefixByteCount, + recordBytesRead: recordBytesRead, + cancellation: cancellation + ) + return PrefixHashedStorageItem( + prefixChecksum: result.checksum, + bytesRead: result.bytesRead, + item: item + ) + } catch FileSystemScannerError.cancelled { + throw FileSystemScannerError.cancelled + } catch { + os_signpost(.event, log: Self.log, name: "hash_prefix_skip", + "path=%{public}@ reason=%{public}@", item.url.path, "\(error)") + return nil + } + } + /// Hashes one item, consulting the persisted `hashCache` fast-path before falling back /// to a full `sha256Checksum` read. I/O is throttled through `ioSemaphore`; cache writes /// are serialised through `cacheLock`. Returns `nil` for per-file read failures (logged @@ -681,6 +799,7 @@ public final class FileSystemScanner { _ item: StorageItem, ioSemaphore: DispatchSemaphore, cacheLock: NSLock, + recordBytesRead: ((Int) -> Void)?, cancellation: ScanCancellation? ) throws -> HashedStorageItem? { let cacheKey = DuplicateHashCache.LookupKey(item: item) @@ -693,7 +812,11 @@ public final class FileSystemScanner { defer { ioSemaphore.signal() } do { - let checksum = try sha256Checksum(for: item.url, cancellation: cancellation) + let checksum = try sha256Checksum( + for: item.url, + recordBytesRead: recordBytesRead, + cancellation: cancellation + ) if let hashCache { cacheLock.lock() defer { cacheLock.unlock() } @@ -715,6 +838,16 @@ private struct HashedStorageItem { let item: StorageItem } +private struct PrefixHashedStorageItem { + let prefixChecksum: String + let bytesRead: Int + let item: StorageItem + + var isCompleteFile: Bool { + Int64(bytesRead) >= item.byteSize + } +} + private struct DirectoryScanSummary { private let retainedCandidateLimit: Int private var retainedCandidateItems: [StorageItem] = [] @@ -752,7 +885,7 @@ private struct DirectoryScanSummary { Array( items.sorted { lhs, rhs in if lhs.displaySize == rhs.displaySize { - return lhs.name.localizedStandardCompare(rhs.name) == .orderedAscending + return lhs.name < rhs.name } return lhs.displaySize > rhs.displaySize } @@ -774,6 +907,82 @@ private extension CleanupCandidate.Confidence { } } +struct DuplicateCandidateRetention { + private let limit: Int + private var candidatesBySize: [Int64: [StorageItem]] = [:] + private var smallestRetainedSize: Int64? + + private(set) var retainedCount = 0 + private(set) var consideredCount = 0 + private(set) var evictionCount = 0 + private(set) var limitReached = false + + init(limit: Int) { + self.limit = max(0, limit) + } + + var sizeGroups: [DuplicateSizeGroup] { + candidatesBySize + .compactMap { byteSize, items in + items.count > 1 ? DuplicateSizeGroup(byteSize: byteSize, items: items.sorted { $0.url.path < $1.url.path }) : nil + } + .sorted { lhs, rhs in + if lhs.totalBytes == rhs.totalBytes { + return lhs.byteSize > rhs.byteSize + } + return lhs.totalBytes > rhs.totalBytes + } + } + + mutating func record(_ item: StorageItem) { + consideredCount += 1 + guard limit > 0 else { + limitReached = true + return + } + + if retainedCount >= limit, + let smallestRetainedSize, + item.byteSize < smallestRetainedSize { + limitReached = true + return + } + + candidatesBySize[item.byteSize, default: []].append(item) + retainedCount += 1 + if smallestRetainedSize.map({ item.byteSize < $0 }) ?? true { + smallestRetainedSize = item.byteSize + } + + guard retainedCount > limit else { + return + } + + limitReached = true + evictSmallestRetained() + } + + private mutating func evictSmallestRetained() { + guard let smallestByteSize = smallestRetainedSize ?? candidatesBySize.keys.min(), + var items = candidatesBySize[smallestByteSize], + !items.isEmpty else { + smallestRetainedSize = candidatesBySize.keys.min() + return + } + + items.removeLast() + retainedCount -= 1 + evictionCount += 1 + + if items.isEmpty { + candidatesBySize.removeValue(forKey: smallestByteSize) + smallestRetainedSize = candidatesBySize.keys.min() + } else { + candidatesBySize[smallestByteSize] = items + } + } +} + private final class ScanAccumulator { /// Shares the FileSystemScanner signpost subsystem/category so cleanup-candidate /// build shows up alongside the scan/enumerate/verify spans when profiling. @@ -794,20 +1003,18 @@ private final class ScanAccumulator { private var lastProgressDate = Date.distantPast private let oldFileCutoff: Date private var retainedItemCount: Int - private var largestFileItems: [StorageItem] = [] - private var largestFolderItems: [StorageItem] = [] - private var oldLargeFileItems: [StorageItem] = [] + private var largestFileItems: RankedItemRetention + private var largestFolderItems: RankedItemRetention + private var oldLargeFileItems: RankedItemRetention private var fileTypeStats: [String: FileTypeAccumulator] = [:] - private var duplicateCandidatesBySize: [Int64: [StorageItem]] = [:] - private var duplicateCandidateItemCount = 0 - private var duplicateCandidateConsideredCount = 0 - private var smallestDuplicateCandidateByteSize: Int64? - private(set) var duplicateCandidateLimitReached = false + private var duplicateCandidateRetention: DuplicateCandidateRetention private var cleanupCandidatesByID: [String: CleanupCandidate] = [:] var scannedItemCount = 0 var inaccessibleItemCount = 0 var totalBytes: Int64 = 0 + private(set) var duplicateVerificationBytesRead: Int64 = 0 + private(set) var snapshotBuildCount = 0 /// Guards every mutable field above. Held briefly during directory enumeration's /// record*() calls; the user `progress` and `onSnapshot` closures are both invoked @@ -832,6 +1039,10 @@ private final class ScanAccumulator { self.rootURL = rootURL self.startedAt = startedAt self.retainedItemCount = 1 + self.largestFileItems = RankedItemRetention(limit: options.maxRankedResults) + self.largestFolderItems = RankedItemRetention(limit: options.maxRankedResults) + self.oldLargeFileItems = RankedItemRetention(limit: options.maxRankedResults) + self.duplicateCandidateRetention = DuplicateCandidateRetention(limit: options.maxDuplicateCandidateItems) self.oldFileCutoff = Calendar.current.date( byAdding: .day, value: -options.oldFileAgeDays, @@ -846,12 +1057,6 @@ private final class ScanAccumulator { emitProgressLocked(path: path) } - func recordBytes(_ bytes: Int64) { - lock.lock() - defer { lock.unlock() } - totalBytes += bytes - } - func recordInaccessible(path: String) { lock.lock() defer { lock.unlock() } @@ -859,21 +1064,40 @@ private final class ScanAccumulator { emitProgressLocked(path: path) } + func recordDuplicateVerificationBytes(_ bytes: Int) { + guard bytes > 0 else { return } + lock.lock() + duplicateVerificationBytesRead += Int64(bytes) + lock.unlock() + } + func recordPhase(path: String, phase: ScanPhase = .enumerating) { lock.lock() defer { lock.unlock() } emitProgressLocked(path: path, force: true, phase: phase) } + func recordScannedItem(_ item: StorageItem, countedBytes: Int64, path: String) { + lock.lock() + defer { lock.unlock() } + scannedItemCount += 1 + totalBytes += countedBytes + recordItemLocked(item) + emitProgressLocked(path: path) + } + func recordItem(_ item: StorageItem) { lock.lock() defer { lock.unlock() } + recordItemLocked(item) + } + + private func recordItemLocked(_ item: StorageItem) { switch item.kind { case .file: recordFileLocked(item) case .folder, .package: - largestFolderItems.append(item) - trimRankedItems(&largestFolderItems, limit: options.maxRankedResults) { $0.displaySize > $1.displaySize } + largestFolderItems.record(item) case .alias, .inaccessible, .other: break } @@ -933,17 +1157,15 @@ private final class ScanAccumulator { } var largestFiles: [StorageItem] { - sortedRankedItems(largestFileItems, limit: options.maxRankedResults) { $0.displaySize > $1.displaySize } + largestFileItems.sortedItems } func largestFolders(excluding rootID: String) -> [StorageItem] { - sortedRankedItems(largestFolderItems.filter { $0.id != rootID }, limit: options.maxRankedResults) { - $0.displaySize > $1.displaySize - } + largestFolderItems.sortedItems.filter { $0.id != rootID } } var oldLargeFiles: [StorageItem] { - sortedRankedItems(oldLargeFileItems, limit: options.maxRankedResults) { $0.displaySize > $1.displaySize } + oldLargeFileItems.sortedItems } var typeBreakdown: [FileTypeStat] { @@ -986,16 +1208,7 @@ private final class ScanAccumulator { } var duplicateSizeGroups: [DuplicateSizeGroup] { - duplicateCandidatesBySize - .compactMap { byteSize, items in - items.count > 1 ? DuplicateSizeGroup(byteSize: byteSize, items: items.sorted { $0.url.path < $1.url.path }) : nil - } - .sorted { lhs, rhs in - if lhs.totalBytes == rhs.totalBytes { - return lhs.byteSize > rhs.byteSize - } - return lhs.totalBytes > rhs.totalBytes - } + duplicateCandidateRetention.sizeGroups } var duplicateCandidateItemLimit: Int { @@ -1003,11 +1216,19 @@ private final class ScanAccumulator { } var duplicateCandidateItemsRetained: Int { - duplicateCandidateItemCount + duplicateCandidateRetention.retainedCount } var duplicateCandidateItemsConsidered: Int { - duplicateCandidateConsideredCount + duplicateCandidateRetention.consideredCount + } + + var duplicateCandidateEvictionCount: Int { + duplicateCandidateRetention.evictionCount + } + + var duplicateCandidateLimitReached: Bool { + duplicateCandidateRetention.limitReached } func cleanupCandidates( @@ -1058,7 +1279,7 @@ private final class ScanAccumulator { private func emitProgressLocked(path: String, force: Bool = false, phase: ScanPhase = .enumerating) { let now = Date() - guard force || scannedItemCount == 1 || scannedItemCount.isMultiple(of: 25) || now.timeIntervalSince(lastProgressDate) > 0.35 else { + guard force || scannedItemCount == 1 || now.timeIntervalSince(lastProgressDate) > 0.35 else { return } @@ -1069,6 +1290,7 @@ private final class ScanAccumulator { // snapshot — a one-tick lag that leaves cancel-preserved partial results stale by // a full throttle interval. if let onSnapshot { + snapshotBuildCount += 1 onSnapshot(snapshotLocked()) } progress?(ScanProgress(scannedItemCount: scannedItemCount, totalBytes: totalBytes, currentPath: path, phase: phase)) @@ -1100,17 +1322,19 @@ private final class ScanAccumulator { scannedItemCount: scannedItemCount, inaccessibleItemCount: inaccessibleItemCount, totalBytes: totalBytes, - largestFiles: sortedRankedItems(largestFileItems, limit: options.maxRankedResults) { $0.displaySize > $1.displaySize }, - largestFolders: sortedRankedItems(largestFolderItems, limit: options.maxRankedResults) { $0.displaySize > $1.displaySize }, - oldLargeFiles: sortedRankedItems(oldLargeFileItems, limit: options.maxRankedResults) { $0.displaySize > $1.displaySize }, + largestFiles: largestFileItems.sortedItems, + largestFolders: largestFolderItems.sortedItems, + oldLargeFiles: oldLargeFileItems.sortedItems, typeBreakdown: typeBreakdown, categoryBreakdown: categoryBreakdown, duplicateSizeGroups: [], verifiedDuplicateGroups: [], duplicateCandidateItemLimit: duplicateCandidateItemLimit, - duplicateCandidateItemsRetained: duplicateCandidateItemCount, - duplicateCandidateItemsConsidered: duplicateCandidateConsideredCount, + duplicateCandidateItemsRetained: duplicateCandidateRetention.retainedCount, + duplicateCandidateItemsConsidered: duplicateCandidateRetention.consideredCount, + duplicateCandidateEvictionCount: duplicateCandidateRetention.evictionCount, duplicateCandidateLimitReached: duplicateCandidateLimitReached, + snapshotBuildCount: snapshotBuildCount, duplicateVerificationDuration: 0, enumerateDuration: Date().timeIntervalSince(startedAt), cleanupCandidates: [], @@ -1127,14 +1351,12 @@ private final class ScanAccumulator { } private func recordFileLocked(_ item: StorageItem) { - largestFileItems.append(item) - trimRankedItems(&largestFileItems, limit: options.maxRankedResults) { $0.displaySize > $1.displaySize } + largestFileItems.record(item) if let modifiedAt = item.modifiedAt, item.displaySize >= options.largeFileThreshold, modifiedAt <= oldFileCutoff { - oldLargeFileItems.append(item) - trimRankedItems(&oldLargeFileItems, limit: options.maxRankedResults) { $0.displaySize > $1.displaySize } + oldLargeFileItems.record(item) } let typeLabel: String @@ -1151,58 +1373,7 @@ private final class ScanAccumulator { fileTypeStats[typeLabel] = typeStat if item.byteSize >= options.duplicateCandidateThreshold { - recordDuplicateCandidateLocked(item) - } - } - - private func recordDuplicateCandidateLocked(_ item: StorageItem) { - duplicateCandidateConsideredCount += 1 - let candidateLimit = max(0, options.maxDuplicateCandidateItems) - guard candidateLimit > 0 else { - duplicateCandidateLimitReached = true - return - } - - if duplicateCandidateItemCount >= candidateLimit, - let smallestDuplicateCandidateByteSize, - item.byteSize < smallestDuplicateCandidateByteSize { - duplicateCandidateLimitReached = true - return - } - - duplicateCandidatesBySize[item.byteSize, default: []].append(item) - duplicateCandidateItemCount += 1 - if smallestDuplicateCandidateByteSize.map({ item.byteSize < $0 }) ?? true { - smallestDuplicateCandidateByteSize = item.byteSize - } - - guard duplicateCandidateItemCount > candidateLimit else { - return - } - - duplicateCandidateLimitReached = true - removeSmallestDuplicateCandidateLocked() - } - - private func removeSmallestDuplicateCandidateLocked() { - guard let smallestByteSize = smallestDuplicateCandidateByteSize ?? duplicateCandidatesBySize.keys.min(), - var items = duplicateCandidatesBySize[smallestByteSize], - !items.isEmpty else { - smallestDuplicateCandidateByteSize = duplicateCandidatesBySize.keys.min() - return - } - - let removalIndex = items.indices.max { lhs, rhs in - items[lhs].url.path.localizedStandardCompare(items[rhs].url.path) == .orderedAscending - } ?? items.startIndex - items.remove(at: removalIndex) - duplicateCandidateItemCount -= 1 - - if items.isEmpty { - duplicateCandidatesBySize.removeValue(forKey: smallestByteSize) - smallestDuplicateCandidateByteSize = duplicateCandidatesBySize.keys.min() - } else { - duplicateCandidatesBySize[smallestByteSize] = items + duplicateCandidateRetention.record(item) } } @@ -1301,40 +1472,79 @@ private final class ScanAccumulator { } } - private func trimRankedItems( - _ items: inout [StorageItem], - limit: Int, - by areInIncreasingPriorityOrder: (StorageItem, StorageItem) -> Bool - ) { - let boundedLimit = max(0, limit) - guard boundedLimit > 0 else { - items.removeAll(keepingCapacity: false) +} + +/// Keeps only the highest-priority storage items without repeatedly sorting the +/// entire append buffer under `ScanAccumulator`'s shared lock. The heap root is +/// always the least desirable retained item, so a late larger item replaces it +/// in O(log limit). Final display ordering remains deterministic: size descending, +/// then path ascending for equal-size peers. +struct RankedItemRetention { + private let limit: Int + private var heap: [StorageItem] = [] + + init(limit: Int) { + self.limit = max(0, limit) + heap.reserveCapacity(self.limit) + } + + var count: Int { heap.count } + + var sortedItems: [StorageItem] { + heap.sorted { Self.isBetter($0, than: $1) } + } + + mutating func record(_ item: StorageItem) { + guard limit > 0 else { return } + + if heap.count < limit { + heap.append(item) + siftUp(from: heap.count - 1) return } - guard items.count > boundedLimit * 4 else { + guard let worst = heap.first, Self.isBetter(item, than: worst) else { return } + heap[0] = item + siftDown(from: 0) + } + + private static func isBetter(_ lhs: StorageItem, than rhs: StorageItem) -> Bool { + if lhs.displaySize != rhs.displaySize { + return lhs.displaySize > rhs.displaySize + } + return lhs.url.path < rhs.url.path + } - items = sortedRankedItems(items, limit: boundedLimit, by: areInIncreasingPriorityOrder) - } - - private func sortedRankedItems( - _ items: [StorageItem], - limit: Int, - by areInIncreasingPriorityOrder: (StorageItem, StorageItem) -> Bool - ) -> [StorageItem] { - Array(items.sorted { lhs, rhs in - if areInIncreasingPriorityOrder(lhs, rhs) { return true } - if areInIncreasingPriorityOrder(rhs, lhs) { return false } - // Tie on the priority Comparator (equal displaySize). Fall back to URL path so - // two equally-sized items land in a deterministic order regardless of the order - // `DispatchQueue.concurrentPerform` appended them to largestFileItems. Without - // this, `sorted(by:)` isn't guaranteed stable and two scans of the same fixture - // produce different largestFiles orderings — caught by - // `parallelEnumerationIsDeterministicAcrossRuns`. - return lhs.url.path < rhs.url.path - }.prefix(max(0, limit))) + private static func isWorse(_ lhs: StorageItem, than rhs: StorageItem) -> Bool { + isBetter(rhs, than: lhs) + } + + private mutating func siftUp(from startIndex: Int) { + var child = startIndex + while child > 0 { + let parent = (child - 1) / 2 + guard Self.isWorse(heap[child], than: heap[parent]) else { return } + heap.swapAt(child, parent) + child = parent + } + } + + private mutating func siftDown(from startIndex: Int) { + var parent = startIndex + while true { + let left = parent * 2 + 1 + guard left < heap.count else { return } + let right = left + 1 + var worseChild = left + if right < heap.count, Self.isWorse(heap[right], than: heap[left]) { + worseChild = right + } + guard Self.isWorse(heap[worseChild], than: heap[parent]) else { return } + heap.swapAt(parent, worseChild) + parent = worseChild + } } } diff --git a/Sources/StorageScopeCore/Services/ScanBenchmark.swift b/Sources/StorageScopeCore/Services/ScanBenchmark.swift index ac1e75c..d3988bf 100644 --- a/Sources/StorageScopeCore/Services/ScanBenchmark.swift +++ b/Sources/StorageScopeCore/Services/ScanBenchmark.swift @@ -3,6 +3,8 @@ import os public struct ScanBenchmarkReport: Hashable, Sendable { public let scopeLabel: String + public let buildConfiguration: String + public let streamingCallbacksEnabled: Bool public let duration: TimeInterval public let scannedItemCount: Int public let inaccessibleItemCount: Int @@ -11,9 +13,12 @@ public struct ScanBenchmarkReport: Hashable, Sendable { public let largestFolderCount: Int public let duplicateCandidateItemsConsidered: Int public let duplicateCandidateItemsRetained: Int + public let duplicateCandidateEvictionCount: Int public let duplicateCandidateLimitReached: Bool + public let snapshotBuildCount: Int public let verifiedDuplicateGroupCount: Int public let duplicateVerificationDuration: TimeInterval + public let duplicateVerificationBytesRead: Int64 public let enumerateDuration: TimeInterval public let verifyDuration: TimeInterval public let persistDuration: TimeInterval @@ -30,10 +35,13 @@ public struct ScanBenchmarkReport: Hashable, Sendable { public init( scopeLabel: String, scan: StorageScan, + streamingCallbacksEnabled: Bool = false, persistDuration: TimeInterval = 0, peakMemoryBytes: UInt64? = Self.currentPeakResidentMemoryBytes() ) { self.scopeLabel = scopeLabel + self.buildConfiguration = Self.currentBuildConfiguration() + self.streamingCallbacksEnabled = streamingCallbacksEnabled self.duration = scan.finishedAt.timeIntervalSince(scan.startedAt) self.scannedItemCount = scan.scannedItemCount self.inaccessibleItemCount = scan.inaccessibleItemCount @@ -42,9 +50,12 @@ public struct ScanBenchmarkReport: Hashable, Sendable { self.largestFolderCount = scan.largestFolders.count self.duplicateCandidateItemsConsidered = scan.duplicateCandidateItemsConsidered self.duplicateCandidateItemsRetained = scan.duplicateCandidateItemsRetained + self.duplicateCandidateEvictionCount = scan.duplicateCandidateEvictionCount self.duplicateCandidateLimitReached = scan.duplicateCandidateLimitReached + self.snapshotBuildCount = scan.snapshotBuildCount self.verifiedDuplicateGroupCount = scan.verifiedDuplicateGroups.count self.duplicateVerificationDuration = scan.duplicateVerificationDuration + self.duplicateVerificationBytesRead = scan.duplicateVerificationBytesRead self.enumerateDuration = scan.enumerateDuration self.verifyDuration = scan.duplicateVerificationDuration self.persistDuration = persistDuration @@ -56,6 +67,8 @@ public struct ScanBenchmarkReport: Hashable, Sendable { [ "StorageScope benchmark", "Scope: \(scopeLabel)", + "Build configuration: \(buildConfiguration)", + "Streaming callbacks: \(streamingCallbacksEnabled ? "enabled" : "disabled")", "Duration: \(Self.seconds(duration))", "Items scanned: \(scannedItemCount.formatted())", "Inaccessible: \(inaccessibleItemCount.formatted())", @@ -63,9 +76,12 @@ public struct ScanBenchmarkReport: Hashable, Sendable { "Largest files retained: \(largestFileCount.formatted())", "Largest folders retained: \(largestFolderCount.formatted())", "Duplicate candidates: \(duplicateCandidateItemsRetained.formatted()) retained / \(duplicateCandidateItemsConsidered.formatted()) considered", + "Duplicate evictions: \(duplicateCandidateEvictionCount.formatted())", "Duplicate cap reached: \(duplicateCandidateLimitReached ? "yes" : "no")", + "Snapshots built: \(snapshotBuildCount.formatted())", "Verified duplicate groups: \(verifiedDuplicateGroupCount.formatted())", "Duplicate verification: \(Self.seconds(duplicateVerificationDuration))", + "Duplicate verification bytes read: \(Self.bytes(duplicateVerificationBytesRead))", "Enumerate duration: \(Self.seconds(enumerateDuration))", "Verify duration: \(Self.seconds(verifyDuration))", "Persist duration: \(Self.seconds(persistDuration))", @@ -96,6 +112,19 @@ public struct ScanBenchmarkReport: Hashable, Sendable { ByteCountFormatter.string(fromByteCount: Int64(clamping: value), countStyle: .memory) } + public static func currentBuildConfiguration() -> String { + if let override = ProcessInfo.processInfo.environment["STORAGESCOPE_BENCHMARK_CONFIGURATION"], + !override.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + return override + } + + #if DEBUG + return "debug" + #else + return "release" + #endif + } + public static func currentPeakResidentMemoryBytes() -> UInt64? { var usage = rusage() guard getrusage(RUSAGE_SELF, &usage) == 0 else { @@ -123,8 +152,32 @@ public struct ScanBenchmarkRunner { self.hashCache = hashCache } - public func run(rootURL: URL, options: ScanOptions = .benchmarkDefaults(), showFullPath: Bool = false) throws -> ScanBenchmarkReport { - let scan = try scanner.scan(root: rootURL, options: options) + public func run( + rootURL: URL, + options: ScanOptions = .benchmarkDefaults(), + showFullPath: Bool = false, + streamingCallbacks: Bool = false + ) throws -> ScanBenchmarkReport { + let scan: StorageScan + if streamingCallbacks { + let snapshotLock = NSLock() + var latestSnapshot: StorageScan? + scan = try scanner.scan( + root: rootURL, + options: options, + progress: { _ in }, + onSnapshot: { snapshot in + snapshotLock.lock() + latestSnapshot = snapshot + snapshotLock.unlock() + } + ) + // Keep the staged snapshot observably live through the scan call, matching + // ScanStore's retained partial-result behavior without printing private data. + withExtendedLifetime(latestSnapshot) {} + } else { + scan = try scanner.scan(root: rootURL, options: options) + } // Gate the persist-phase timing on a real cache being attached. When `hashCache` // is nil, `hashCache?.persist()` is a no-op, so any non-zero persistDuration here @@ -142,6 +195,7 @@ public struct ScanBenchmarkRunner { return ScanBenchmarkReport( scopeLabel: ScanBenchmarkReport.scopeLabel(for: rootURL, showFullPath: showFullPath), scan: scan, + streamingCallbacksEnabled: streamingCallbacks, persistDuration: persistDuration ) } diff --git a/Tests/StorageScopeCoreTests/FileSystemScannerTests.swift b/Tests/StorageScopeCoreTests/FileSystemScannerTests.swift index 6f0b2b0..753ce3b 100644 --- a/Tests/StorageScopeCoreTests/FileSystemScannerTests.swift +++ b/Tests/StorageScopeCoreTests/FileSystemScannerTests.swift @@ -4,6 +4,27 @@ import Testing @Suite("FileSystemScanner") struct FileSystemScannerTests { + @Test("ranked retention keeps late larger items without exceeding its bound") + func rankedRetentionKeepsLateLargerItems() { + var retention = RankedItemRetention(limit: 3) + for (name, size) in [("a", 10), ("b", 20), ("c", 30), ("d", 40), ("e", 5)] { + retention.record(rankedItem(name: name, size: Int64(size))) + } + + #expect(retention.count == 3) + #expect(retention.sortedItems.map(\.name) == ["d", "c", "b"]) + } + + @Test("ranked retention uses path order as a deterministic size tie-break") + func rankedRetentionUsesDeterministicPathTieBreak() { + var retention = RankedItemRetention(limit: 2) + for name in ["z", "b", "a", "m"] { + retention.record(rankedItem(name: name, size: 100)) + } + + #expect(retention.sortedItems.map(\.name) == ["a", "b"]) + } + @Test("ranks largest files and folders") func scanRanksLargestFilesAndFolders() throws { let temporaryRoot = try makeTemporaryRoot() @@ -28,6 +49,19 @@ struct FileSystemScannerTests { #expect(scan.totalBytes >= 16_000) } + private func rankedItem(name: String, size: Int64) -> StorageItem { + StorageItem( + url: URL(fileURLWithPath: "/tmp/ranked-retention/\(name)"), + kind: .file, + byteSize: size, + allocatedSize: size, + modifiedAt: nil, + immediateChildCount: 0, + descendantCount: 0, + isReadable: true + ) + } + @Test("categorizes file type breakdown for scan review") func typeBreakdownIncludesStorageCategories() throws { let temporaryRoot = try makeTemporaryRoot() @@ -124,6 +158,46 @@ struct FileSystemScannerTests { ).isEmpty) } + @Test("duplicate verification records bytes read") + func duplicateVerificationRecordsBytesRead() throws { + let temporaryRoot = try makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: temporaryRoot) } + + try Data("same-content".utf8).write(to: temporaryRoot.appendingPathComponent("copy-a.txt")) + try Data("same-content".utf8).write(to: temporaryRoot.appendingPathComponent("copy-b.txt")) + + let scan = try FileSystemScanner().scan( + root: temporaryRoot, + options: ScanOptions(duplicateCandidateThreshold: 1) + ) + + #expect(scan.verifiedDuplicateGroups.count == 1) + #expect(scan.duplicateVerificationBytesRead == 24) + } + + @Test("duplicate verification avoids full reads for prefix-mismatched same-size files") + func duplicateVerificationSkipsFullHashForPrefixMismatches() throws { + let temporaryRoot = try makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: temporaryRoot) } + + let fileSize = 1_048_576 + for index in 0..<4 { + var data = Data(repeating: UInt8(index), count: fileSize) + data[0] = UInt8(index + 1) + try data.write(to: temporaryRoot.appendingPathComponent("unique-\(index).bin")) + } + + let scan = try FileSystemScanner().scan( + root: temporaryRoot, + options: ScanOptions(duplicateCandidateThreshold: 1) + ) + + #expect(scan.duplicateSizeGroups.count == 1) + #expect(scan.verifiedDuplicateGroups.isEmpty) + #expect(scan.duplicateVerificationBytesRead == Int64(4 * 64 * 1_024)) + #expect(scan.duplicateVerificationBytesRead <= scan.duplicateSizeGroups[0].totalBytes / 10) + } + @Test("caps automatic duplicate content verification") func duplicateVerificationHonorsByteBudget() throws { let temporaryRoot = try makeTemporaryRoot() @@ -663,6 +737,7 @@ struct FileSystemScannerTests { #expect(scan.duplicateCandidateItemLimit == 12) #expect(scan.duplicateCandidateItemsRetained == 12) #expect(scan.duplicateCandidateItemsConsidered == 14) + #expect(scan.duplicateCandidateEvictionCount == 2) #expect(scan.duplicateCandidateLimitReached) } @@ -697,6 +772,25 @@ struct FileSystemScannerTests { #expect(scan.duplicateCandidateLimitReached) } + @Test("duplicate candidate cap rejects smaller candidates without eviction churn") + func duplicateCandidateCapRejectsSmallerCandidatesBeforeEviction() throws { + var candidates = DuplicateCandidateRetention(limit: 12) + for index in 0..<12 { + candidates.record(fileItem(named: String(format: "large-%03d.bin", index), bytes: 8_192)) + } + + for index in 0..<100 { + candidates.record(fileItem(named: String(format: "small-%03d.bin", index), bytes: 2_048)) + } + + #expect(candidates.consideredCount == 112) + #expect(candidates.retainedCount == 12) + #expect(candidates.evictionCount == 0) + #expect(candidates.limitReached) + #expect(candidates.sizeGroups.map(\.byteSize) == [8_192]) + #expect(candidates.sizeGroups.first?.items.count == 12) + } + @Test("duplicate candidates include files pruned from retained tree") func duplicateCandidatesSurviveRetainedTreePruning() throws { let temporaryRoot = try makeTemporaryRoot() @@ -975,15 +1069,34 @@ struct FileSystemScannerTests { #expect(text.contains("StorageScope benchmark")) #expect(text.contains("Scope: \(temporaryRoot.lastPathComponent)")) + #expect(text.contains("Build configuration:")) #expect(text.contains("Items scanned:")) #expect(text.contains("Duplicate candidates:")) + #expect(text.contains("Duplicate evictions:")) #expect(text.contains("Duplicate verification:")) + #expect(text.contains("Snapshots built:")) #expect(text.contains("Results are local only.")) #expect(!text.contains(temporaryRoot.deletingLastPathComponent().path)) #expect(!text.localizedCaseInsensitiveContains("http://")) #expect(!text.localizedCaseInsensitiveContains("https://")) } + @Test("streaming benchmark mode exercises app-like snapshot callbacks") + func streamingBenchmarkModeBuildsSnapshots() throws { + let temporaryRoot = try makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: temporaryRoot) } + try writeFile(named: "streaming.dat", bytes: 4_096, in: temporaryRoot) + + let report = try ScanBenchmarkRunner().run( + rootURL: temporaryRoot, + streamingCallbacks: true + ) + + #expect(report.streamingCallbacksEnabled) + #expect(report.snapshotBuildCount > 0) + #expect(report.text.contains("Streaming callbacks: enabled")) + } + @Test("synthetic benchmark fixture creates expected candidate classes") func syntheticBenchmarkFixtureCreatesExpectedCandidateClasses() throws { let root = try SyntheticBenchmarkFixture.create() @@ -1030,6 +1143,43 @@ struct FileSystemScannerTests { #expect(process.terminationStatus == 0) } + @Test("benchmark script runs release builds by default") + func benchmarkScriptRunsReleaseBuildsByDefault() throws { + let repoRoot = try repositoryRoot() + let script = try String(contentsOf: repoRoot.appendingPathComponent("script/benchmark_scan.sh"), encoding: .utf8) + + #expect(script.contains("CONFIGURATION=\"release\"")) + #expect(script.contains("swift run -c \"$CONFIGURATION\" StorageScopeBenchmark")) + #expect(script.contains("--debug")) + } + + @Test("scanner records item bytes in a single accumulator call") + func scannerRecordsItemBytesInSingleAccumulatorCall() throws { + let repoRoot = try repositoryRoot() + let source = try String( + contentsOf: repoRoot.appendingPathComponent("Sources/StorageScopeCore/Services/FileSystemScanner.swift"), + encoding: .utf8 + ) + + #expect(!source.contains("func recordBytes")) + #expect(!source.contains("accumulator.recordBytes(")) + } + + @Test("duplicate candidate eviction avoids path comparisons") + func duplicateCandidateEvictionAvoidsPathComparisons() throws { + let repoRoot = try repositoryRoot() + let source = try String( + contentsOf: repoRoot.appendingPathComponent("Sources/StorageScopeCore/Services/FileSystemScanner.swift"), + encoding: .utf8 + ) + let evictionSource = try #require(source.range(of: "private mutating func evictSmallestRetained()")) + let evictionEnd = try #require(source[evictionSource.upperBound...].range(of: " }\n}\n")?.lowerBound) + let duplicateEvictionSource = String(source[evictionSource.lowerBound..= 0) #expect(report.verifyDuration >= 0) #expect(report.verifyDuration == report.duplicateVerificationDuration) + #expect(report.duplicateVerificationBytesRead >= 0) // No cache wired up: persistDuration is a no-op plus two Date() readouts, which // can register sub-microsecond timing noise instead of an exact 0. #expect(abs(report.persistDuration) < 0.001) @@ -1050,6 +1201,7 @@ struct FileSystemScannerTests { let text = report.text #expect(text.contains("Enumerate duration:")) #expect(text.contains("Verify duration:")) + #expect(text.contains("Duplicate verification bytes read:")) #expect(text.contains("Persist duration:")) #expect(text.contains("Phase total (enum+verify+persist):")) } @@ -1533,6 +1685,26 @@ struct FileSystemScannerTests { #expect(nonEmptySnapshot?.isPartial == true) } + @Test("snapshot cadence is time throttled, not every 25 items") + func snapshotCadenceIsTimeThrottled() throws { + let temporaryRoot = try makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: temporaryRoot) } + + for index in 0..<100 { + try writeFile(named: "file-\(index).bin", bytes: 1_000 + index, in: temporaryRoot) + } + + var snapshotCount = 0 + let scan = try FileSystemScanner().scan( + root: temporaryRoot, + options: ScanOptions(largeFileThreshold: 1, duplicateCandidateThreshold: 10_000), + onSnapshot: { _ in snapshotCount += 1 } + ) + + #expect(snapshotCount <= 2) + #expect(scan.snapshotBuildCount == snapshotCount) + } + @Test("pause blocks scan progress and resume continues to completion") func pauseBlocksProgressAndResumeCompletes() throws { let temporaryRoot = try makeTemporaryRoot() @@ -1723,6 +1895,21 @@ struct FileSystemScannerTests { ) } + private func fileItem(named name: String, bytes: Int64) -> StorageItem { + StorageItem( + url: URL(fileURLWithPath: "/tmp/\(name)"), + name: name, + kind: .file, + byteSize: bytes, + allocatedSize: bytes, + modifiedAt: nil, + immediateChildCount: 0, + descendantCount: 0, + isReadable: true, + fileExtension: URL(fileURLWithPath: name).pathExtension + ) + } + private func storageScan( rootURL: URL, scannedItemCount: Int, diff --git a/docs/perf-baselines/v0.7.1-2026-07-11/100k.txt b/docs/perf-baselines/v0.7.1-2026-07-11/100k.txt new file mode 100644 index 0000000..70d5480 --- /dev/null +++ b/docs/perf-baselines/v0.7.1-2026-07-11/100k.txt @@ -0,0 +1,13 @@ +StorageScope synthetic release benchmark, 102,775 scanned items + +Measured hot-path tranche before ranked heaps: +run 1: 8.67s, 32.7 MB peak RSS +run 2: 8.50s, 32.2 MB peak RSS +run 3: 8.53s, 32.4 MB peak RSS +median: 8.53s + +Final bounded-heap result: +run 1: 7.42s, 30.5 MB peak RSS +run 2: 7.42s, 30.4 MB peak RSS +run 3: 7.80s, 30.5 MB peak RSS +median: 7.42s diff --git a/docs/perf-baselines/v0.7.1-2026-07-11/10k.txt b/docs/perf-baselines/v0.7.1-2026-07-11/10k.txt new file mode 100644 index 0000000..a8fdb46 --- /dev/null +++ b/docs/perf-baselines/v0.7.1-2026-07-11/10k.txt @@ -0,0 +1,19 @@ +StorageScope synthetic release benchmark, 10,353 scanned items + +Merged main (690227d): +run 1: 4.67s, 29.9 MB peak RSS +run 2: 4.70s, 29.6 MB peak RSS +run 3: 4.75s, 29.9 MB peak RSS +median: 4.70s + +Measured hot-path tranche before ranked heaps: +run 1: 0.57s, 29.4 MB peak RSS +run 2: 0.56s, 29.0 MB peak RSS +run 3: 0.57s, 29.7 MB peak RSS +median: 0.57s + +Final bounded-heap result: +run 1: 0.50s, 28.5 MB peak RSS +run 2: 0.49s, 28.7 MB peak RSS +run 3: 0.49s, 28.6 MB peak RSS +median: 0.49s diff --git a/docs/perf-baselines/v0.7.1-2026-07-11/README.md b/docs/perf-baselines/v0.7.1-2026-07-11/README.md new file mode 100644 index 0000000..b7f7da1 --- /dev/null +++ b/docs/perf-baselines/v0.7.1-2026-07-11/README.md @@ -0,0 +1,69 @@ +# StorageScope performance baseline — 2026-07-11 + +Machine: Apple M1, 16 GB RAM +Configuration: SwiftPM release (`-c release`) +Fixture regime: synthetic files written immediately before scanning, warm filesystem cache +Branch base: `origin/main` at `690227d` + +All medians below use three scans of the same kept fixture. Fixture creation is excluded from scan duration. + +| Stage | 10k median | 100k median | 100k peak RSS | +| --- | ---: | ---: | ---: | +| Merged `main` baseline | 4.70s | not rerun; the 10k profile already reproduced the eviction convoy | 29.9 MB at 10k | +| Duplicate-retention, snapshot, and verification tranche | 0.57s | 8.53s | 32.4 MB | +| Bounded ranked-item heaps | **0.49s** | **7.42s** | **30.5 MB** | + +The complete tranche makes the freshly measured 10k case approximately 9.6x faster than merged `main`. Replacing append/resort/truncate ranked lists with deterministic bounded heaps improves the already-optimized 10k median by approximately 14% and the 100k median by approximately 13%. + +## Release profiles + +The merged-main 10k capture (`/tmp/storagescope-perf/20260711-170503`) recorded: + +- `__psynch_mutexwait`: 15,251 top-of-stack samples +- `CFStringCompareWithOptionsAndLocale`: 742 samples +- `CanonicalFileURLStringToFileSystemRepresentation`: 365 samples + +The final 100k capture (`/tmp/storagescope-perf/20260711-171724`) recorded: + +- `__psynch_mutexwait`: 11,896 top-of-stack samples +- `stat`: 217 samples +- `initializeWithCopy for StorageItem`: 168 samples +- `CanonicalFileURLStringToFileSystemRepresentation`: 43 samples + +The remaining dominant cost is still the single `ScanAccumulator` lock. A fixed-worker/batched-accumulator redesign remains the next structural phase; this tranche deliberately does not claim that exit criterion is complete. + +## Duplicate-verification guard + +`./script/benchmark_scan.sh /tmp/storagescope-prefix-mismatch-bench` read 262 KB while rejecting 4.2 MB of same-size, prefix-mismatched candidates. It produced no false verified groups. + +## App-like streaming regime + +The benchmark now supports `--streaming`, which installs progress and partial-snapshot callbacks without involving UI automation. On the same kept 10k fixture: + +- streaming callbacks: 0.50s median, 4 snapshots built +- callbacks disabled: 0.52s median, 0 snapshots built + +Streaming is already within the plan's 15% app-versus-benchmark target. Moving snapshot construction outside the accumulator lock is therefore not justified by this fixture today. + +## Rejected experiments + +- Accumulator batches of 64 regressed the 10k median to 0.58s by suppressing filesystem parallelism. +- Accumulator batches of 8 improved 10k to 0.42s but produced a 9.88s 100k median with runs spanning 7.41–10.77s. The scheduling variance is unacceptable, so batching was fully reverted. +- Replacing `NSLock` with `os_unfair_lock` left the 10k median unchanged at 0.57s and was reverted. +- Moving per-item classification outside the lock improved only about 5%, below the 10% acceptance gate, and was reverted. +- Verification-time hard-link probing passed correctness tests but regressed the 10k median and raised peak RSS to roughly 34 MB. It was reverted; hard-link identity belongs in a future metadata-prefetch design. +- An eight-way sharded accumulator passed 68 focused scanner tests, but improved the 10k median only from 0.49s to 0.46s and the 100k run only from 7.42s to 7.35s. Its per-shard duplicate retention also raised the first 100k peak-memory result to 51.3 MB, so it was rejected. +- Lazy cleanup-candidate name/path normalization measured 0.55s against a same-session 0.57s control median, about 3.5%. Static `Set` lookups for the small extension tables were also noisy or slower. Both variants missed the 10% gate and were rejected. + +## CI regression guard + +`script/ci_performance_gate.sh` builds the release benchmark, creates one kept 10k fixture, runs it twice, and validates both reports with `script/check_performance_gate.py`. Duration fails only when both runs exceed the broad hosted-runner ceiling; release mode, item count, memory, snapshot, and duplicate-counter invariants are checked on every run. + +## Validation + +- 214 tests across 20 suites passed. +- `./script/build_and_run.sh --verify` built, signed, and launched the app bundle. +- `./script/public_upload_audit.sh` passed. +- `bash -n script/benchmark_scan.sh` passed. +- `git diff --check` passed. +- `script/ci_performance_gate.sh` passed locally at 0.49s/29.2 MB and 0.53s/28.5 MB. diff --git a/script/benchmark_scan.sh b/script/benchmark_scan.sh index 8742b00..abeda40 100755 --- a/script/benchmark_scan.sh +++ b/script/benchmark_scan.sh @@ -4,4 +4,19 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT_DIR" -swift run StorageScopeBenchmark "$@" +CONFIGURATION="release" +ARGS=() + +while (($#)); do + case "$1" in + --debug) + CONFIGURATION="debug" + ;; + *) + ARGS+=("$1") + ;; + esac + shift +done + +STORAGESCOPE_BENCHMARK_CONFIGURATION="$CONFIGURATION" swift run -c "$CONFIGURATION" StorageScopeBenchmark "${ARGS[@]}" diff --git a/script/check_performance_gate.py b/script/check_performance_gate.py new file mode 100755 index 0000000..f134909 --- /dev/null +++ b/script/check_performance_gate.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Validate two StorageScope release benchmark reports without overfitting CI noise.""" + +from __future__ import annotations + +import argparse +import re +from pathlib import Path + + +def field(text: str, label: str) -> str: + match = re.search(rf"^{re.escape(label)}:\s*(.+)$", text, re.MULTILINE) + if not match: + raise ValueError(f"missing benchmark field: {label}") + return match.group(1).strip() + + +def integer(value: str) -> int: + return int(value.replace(",", "")) + + +def memory_megabytes(value: str) -> float: + match = re.fullmatch(r"([0-9.]+)\s*([KMGT])B", value) + if not match: + raise ValueError(f"unsupported memory value: {value}") + amount = float(match.group(1)) + scale = {"K": 1 / 1024, "M": 1, "G": 1024, "T": 1024 * 1024} + return amount * scale[match.group(2)] + + +def parse_report(path: Path) -> dict[str, float | int | str]: + text = path.read_text(encoding="utf-8") + retained, considered = re.fullmatch( + r"([0-9,]+) retained / ([0-9,]+) considered", + field(text, "Duplicate candidates"), + ).groups() + return { + "configuration": field(text, "Build configuration"), + "duration": float(field(text, "Duration").removesuffix("s")), + "items": integer(field(text, "Items scanned")), + "peak_memory_mb": memory_megabytes(field(text, "Peak memory")), + "snapshots": integer(field(text, "Snapshots built")), + "retained": integer(retained), + "considered": integer(considered), + "evictions": integer(field(text, "Duplicate evictions")), + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("reports", nargs=2, type=Path) + parser.add_argument("--minimum-items", type=int, default=10_000) + parser.add_argument("--maximum-duration", type=float, default=5.0) + parser.add_argument("--maximum-memory-mb", type=float, default=150.0) + args = parser.parse_args() + + reports = [parse_report(path) for path in args.reports] + failures: list[str] = [] + + for index, report in enumerate(reports, start=1): + if report["configuration"] != "release": + failures.append(f"run {index}: configuration was {report['configuration']}, not release") + if report["items"] < args.minimum_items: + failures.append(f"run {index}: scanned only {report['items']} items") + if report["peak_memory_mb"] > args.maximum_memory_mb: + failures.append( + f"run {index}: peak memory {report['peak_memory_mb']:.1f} MB exceeds " + f"{args.maximum_memory_mb:.1f} MB" + ) + if report["snapshots"] != 0: + failures.append(f"run {index}: benchmark unexpectedly built snapshots") + if report["retained"] > report["considered"]: + failures.append(f"run {index}: retained candidates exceed considered candidates") + maximum_possible_evictions = report["considered"] - report["retained"] + if report["evictions"] > maximum_possible_evictions: + failures.append(f"run {index}: duplicate eviction counter is internally inconsistent") + + durations = [float(report["duration"]) for report in reports] + if all(duration > args.maximum_duration for duration in durations): + failures.append( + "both release runs exceeded the duration ceiling: " + + ", ".join(f"{duration:.2f}s" for duration in durations) + ) + + print( + "StorageScope performance gate: " + + ", ".join( + f"run {index}={report['duration']:.2f}s/{report['peak_memory_mb']:.1f}MB" + for index, report in enumerate(reports, start=1) + ) + ) + if failures: + for failure in failures: + print(f"FAIL: {failure}") + return 1 + print("performance gate passed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/script/ci_performance_gate.sh b/script/ci_performance_gate.sh new file mode 100755 index 0000000..772593d --- /dev/null +++ b/script/ci_performance_gate.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +OUTPUT_DIR="$(mktemp -d "${TMPDIR:-/tmp}/storagescope-perf-gate.XXXXXX")" +FIXTURE="" +cleanup() { + [[ -z "$FIXTURE" ]] || rm -rf "$FIXTURE" + rm -rf "$OUTPUT_DIR" +} +trap cleanup EXIT + +swift build -c release --product StorageScopeBenchmark +BIN="$(swift build -c release --show-bin-path)/StorageScopeBenchmark" + +"$BIN" --synthetic --items 10000 --depth 5 --duplicates 0.1 --keep-fixture > "$OUTPUT_DIR/fixture.txt" +FIXTURE="$(sed -n 's/^Synthetic fixture: //p' "$OUTPUT_DIR/fixture.txt" | head -1)" +[[ -d "$FIXTURE" ]] || { echo "failed to create benchmark fixture" >&2; exit 1; } + +"$BIN" "$FIXTURE" > "$OUTPUT_DIR/run1.txt" +"$BIN" "$FIXTURE" > "$OUTPUT_DIR/run2.txt" + +python3 script/check_performance_gate.py "$OUTPUT_DIR/run1.txt" "$OUTPUT_DIR/run2.txt"