diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 65b45514730..91a3a9a4b81 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -65,3 +65,25 @@ jobs: S3_EMULATOR_URL: http://127.0.0.1:9090 run: | dotnet test --project ./test/OrchardCore.Tests.Integration/OrchardCore.Tests.Integration.csproj -c Release --no-build + test_azure_blob: + name: Integration Tests - Azure Blob Storage (Azurite) + if: github.event_name == 'workflow_dispatch' || + github.event_name == 'push' || + github.event.review.state == 'APPROVED' || + github.event.review.state == 'CHANGES_REQUESTED' + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: ./.github/actions/setup-dotnet + - name: Build + run: | + dotnet build -c Release ./test/OrchardCore.Tests.Integration/OrchardCore.Tests.Integration.csproj /p:NuGetAudit=false + - name: Start Azurite + run: | + docker run -d --name azurite -p 10000:10000 ghcr.io/skrypt/azurite-adls-gen2:latest azurite --blobHost 0.0.0.0 --skipApiVersionCheck --enableHierarchicalNamespace=true + - name: Azure Blob Integration Tests + env: + AZURITE_CONNECTION_STRING: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" + run: | + dotnet test --project ./test/OrchardCore.Tests.Integration/OrchardCore.Tests.Integration.csproj -c Release --no-build + diff --git a/Directory.Packages.props b/Directory.Packages.props index 5e014c8e3df..455bc66f25c 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -15,6 +15,7 @@ + diff --git a/src/OrchardCore.Modules/OrchardCore.Media.Azure/Services/MediaBlobContainerTenantEvents.cs b/src/OrchardCore.Modules/OrchardCore.Media.Azure/Services/MediaBlobContainerTenantEvents.cs index 6f844a8bbf4..a3ebe9746d0 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media.Azure/Services/MediaBlobContainerTenantEvents.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media.Azure/Services/MediaBlobContainerTenantEvents.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.Options; using OrchardCore.Environment.Shell; using OrchardCore.Environment.Shell.Removing; +using OrchardCore.FileStorage.AzureBlob; using OrchardCore.Modules; namespace OrchardCore.Media.Azure.Services; @@ -14,6 +15,7 @@ public sealed class MediaBlobContainerTenantEvents : ModularTenantEvents { private readonly MediaBlobStorageOptions _options; private readonly ShellSettings _shellSettings; + private readonly BlobFileStore _blobFileStore; private readonly ILogger _logger; internal readonly IStringLocalizer S; @@ -21,18 +23,22 @@ public sealed class MediaBlobContainerTenantEvents : ModularTenantEvents public MediaBlobContainerTenantEvents( IOptions options, ShellSettings shellSettings, + BlobFileStore blobFileStore, IStringLocalizer localizer, ILogger logger ) { _options = options.Value; _shellSettings = shellSettings; + _blobFileStore = blobFileStore; S = localizer; _logger = logger; } public override async Task ActivatingAsync() { + await _blobFileStore.EnsureCapabilitiesAsync(); + // Only create container if options are valid. if (_shellSettings.IsUninitialized() || !_options.IsConfigured() || !_options.CreateContainer) { diff --git a/src/OrchardCore.Modules/OrchardCore.Media.Azure/Services/MediaBlobStorageOptionsConfiguration.cs b/src/OrchardCore.Modules/OrchardCore.Media.Azure/Services/MediaBlobStorageOptionsConfiguration.cs index c75ba02d3b4..a2e7d36ebc7 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media.Azure/Services/MediaBlobStorageOptionsConfiguration.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media.Azure/Services/MediaBlobStorageOptionsConfiguration.cs @@ -27,5 +27,6 @@ protected override void FurtherConfigure(MediaBlobStorageOptions rawOptions, Med { options.RemoveContainer = rawOptions.RemoveContainer; options.RemoveFilesFromBasePath = rawOptions.RemoveFilesFromBasePath; + options.UseHierarchicalNamespace = rawOptions.UseHierarchicalNamespace; } } diff --git a/src/OrchardCore.Modules/OrchardCore.Media.Azure/Startup.cs b/src/OrchardCore.Modules/OrchardCore.Media.Azure/Startup.cs index 903d45f22f3..98c6c1c4148 100644 --- a/src/OrchardCore.Modules/OrchardCore.Media.Azure/Startup.cs +++ b/src/OrchardCore.Modules/OrchardCore.Media.Azure/Startup.cs @@ -84,20 +84,27 @@ public override void ConfigureServices(IServiceCollection services) services.AddSingleton(serviceProvider => serviceProvider.GetRequiredService()); + // Register the blob file store as a singleton so it can be injected for async initialization. + services.AddSingleton(serviceProvider => + { + var blobStorageOptions = serviceProvider.GetRequiredService>().Value; + var clock = serviceProvider.GetRequiredService(); + var contentTypeProvider = serviceProvider.GetRequiredService(); + var blobLogger = serviceProvider.GetRequiredService>(); + + return new BlobFileStore(blobStorageOptions, clock, contentTypeProvider, blobLogger); + }); + // Replace the default media file store with a blob file store. services.Replace(ServiceDescriptor.Singleton(serviceProvider => { - var blobStorageOptions = serviceProvider.GetRequiredService>().Value; - var shellOptions = serviceProvider.GetRequiredService>(); + var fileStore = serviceProvider.GetRequiredService(); var shellSettings = serviceProvider.GetRequiredService(); var mediaOptions = serviceProvider.GetRequiredService>().Value; - var clock = serviceProvider.GetRequiredService(); - var contentTypeProvider = serviceProvider.GetRequiredService(); var mediaEventHandlers = serviceProvider.GetServices(); var mediaCreatingEventHandlers = serviceProvider.GetServices(); var logger = serviceProvider.GetRequiredService>(); - var fileStore = new BlobFileStore(blobStorageOptions, clock, contentTypeProvider); var mediaUrlBase = "/" + fileStore.Combine(shellSettings.RequestUrlPrefix, mediaOptions.AssetsRequestPath); var originalPathBase = serviceProvider.GetRequiredService().HttpContext diff --git a/src/OrchardCore/OrchardCore.FileStorage.Abstractions/IFileStore.cs b/src/OrchardCore/OrchardCore.FileStorage.Abstractions/IFileStore.cs index 3092f9bf6f1..062f8f6135f 100644 --- a/src/OrchardCore/OrchardCore.FileStorage.Abstractions/IFileStore.cs +++ b/src/OrchardCore/OrchardCore.FileStorage.Abstractions/IFileStore.cs @@ -41,6 +41,46 @@ public interface IFileStore /// IAsyncEnumerable GetDirectoryContentAsync(string path = null, bool includeSubDirectories = false); + /// + /// Enumerates only the files (not directories) in a given directory within the file store. + /// + /// The path of the directory to enumerate, or null to enumerate the root of the file store. + /// The list of files in the given directory. + /// + /// Default implementation filters . Implementations + /// may override this to avoid enumerating directories for better performance. + /// + async IAsyncEnumerable GetFilesAsync(string path = null) + { + await foreach (var entry in GetDirectoryContentAsync(path)) + { + if (!entry.IsDirectory) + { + yield return entry; + } + } + } + + /// + /// Enumerates only the subdirectories in a given directory within the file store. + /// + /// The path of the directory to enumerate, or null to enumerate the root of the file store. + /// The list of subdirectories in the given directory. + /// + /// Default implementation filters . Implementations + /// may override this to avoid enumerating files for better performance. + /// + async IAsyncEnumerable GetDirectoriesAsync(string path = null) + { + await foreach (var entry in GetDirectoryContentAsync(path)) + { + if (entry.IsDirectory) + { + yield return entry; + } + } + } + /// /// Creates a directory in the file store if it doesn't already exist. /// @@ -109,7 +149,7 @@ public interface IFileStore Task CreateFileFromStreamAsync(string path, Stream inputStream, bool overwrite = false); /// - /// Calculates the free space available in this file store. + /// Calculates the free space available in this file store. /// /// The usable space in bytes, or if the space is unlimited. Task GetPermittedStorageAsync() => Task.FromResult(null); diff --git a/src/OrchardCore/OrchardCore.FileStorage.AzureBlob/BlobFileStore.cs b/src/OrchardCore/OrchardCore.FileStorage.AzureBlob/BlobFileStore.cs index 56395297ab1..67c7d85032c 100644 --- a/src/OrchardCore/OrchardCore.FileStorage.AzureBlob/BlobFileStore.cs +++ b/src/OrchardCore/OrchardCore.FileStorage.AzureBlob/BlobFileStore.cs @@ -3,7 +3,9 @@ using Azure; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; +using Azure.Storage.Files.DataLake; using Microsoft.AspNetCore.StaticFiles; +using Microsoft.Extensions.Logging; using OrchardCore.Modules; namespace OrchardCore.FileStorage.AzureBlob; @@ -12,24 +14,25 @@ namespace OrchardCore.FileStorage.AzureBlob; /// Provides an implementation that targets an underlying Azure Blob Storage account. /// /// -/// Azure Blob Storage has very different semantics for directories compared to a local file system, and -/// some special consideration is required for make this provider conform to the semantics of the -/// interface and behave in an expected way. +/// This store supports both flat-namespace (Gen1) and hierarchical-namespace / ADLS Gen2 storage accounts. +/// When HNS is detected (or forced via configuration), operations use native DataLake APIs for +/// atomic moves, real directories, and efficient listing. Otherwise, standard blob operations are used +/// with virtual directory semantics. /// -/// Directories have no physical manifestation in blob storage; we can obtain a reference to them, but +/// Directories have no physical manifestation in flat blob storage; we can obtain a reference to them, but /// that reference can be created regardless of whether the directory exists, and it can only be used /// as a scoping container to operate on blobs within that directory namespace. /// -/// As a consequence, this provider generally behaves as if any given directory always exists. To -/// simulate "creating" a directory (which cannot technically be done in blob storage) this provider creates -/// a marker file inside the directory, which makes the directory "exist" and appear when listing contents -/// subsequently. This marker file is ignored (excluded) when listing directory contents. +/// As a consequence, in flat-namespace mode this provider generally behaves as if any given directory always +/// exists. To simulate "creating" a directory this provider creates a marker file inside the directory, +/// which makes the directory "exist" and appear when listing contents subsequently. This marker file is +/// ignored (excluded) when listing directory contents. /// /// Note that the Blob Container is not created automatically, and existence of the Container is not verified. /// /// Create the Blob Container before enabling a Blob File Store. /// -/// Azure Blog Storage will create the BasePath inside the container during the upload of the first file. +/// Azure Blob Storage will create the BasePath inside the container during the upload of the first file. /// public class BlobFileStore : IFileStore { @@ -39,19 +42,32 @@ public class BlobFileStore : IFileStore private readonly BlobStorageOptions _options; private readonly IClock _clock; private readonly BlobContainerClient _blobContainer; + private readonly BlobServiceClient _blobServiceClient; + private readonly DataLakeFileSystemClient _dataLakeFileSystemClient; private readonly IContentTypeProvider _contentTypeProvider; + private readonly ILogger _logger; + private readonly bool? _useHierarchicalNamespaceOverride; + private readonly SemaphoreSlim _capabilitiesLock = new(1, 1); + private bool? _hnsEnabled; private readonly string _basePrefix; public BlobFileStore( BlobStorageOptions options, IClock clock, - IContentTypeProvider contentTypeProvider) + IContentTypeProvider contentTypeProvider, + ILogger logger = null) { _options = options; _clock = clock; _contentTypeProvider = contentTypeProvider; + _logger = logger; _blobContainer = new BlobContainerClient(_options.ConnectionString, _options.ContainerName); + _blobServiceClient = new BlobServiceClient(_options.ConnectionString); + _useHierarchicalNamespaceOverride = options.UseHierarchicalNamespace; + + var dataLakeServiceClient = new DataLakeServiceClient(_options.ConnectionString); + _dataLakeFileSystemClient = dataLakeServiceClient.GetFileSystemClient(_options.ContainerName); if (!string.IsNullOrEmpty(_options.BasePath)) { @@ -59,6 +75,85 @@ public BlobFileStore( } } + /// + /// Probes the storage account to determine whether Hierarchical Namespace (HNS) is enabled. + /// Must be called once at startup. + /// + public async Task EnsureCapabilitiesAsync() + { + if (_hnsEnabled.HasValue) + { + return; + } + + await _capabilitiesLock.WaitAsync(); + try + { + if (_hnsEnabled.HasValue) + { + return; + } + + try + { + var accountInfo = await _blobServiceClient.GetAccountInfoAsync(); + var detectedHns = accountInfo.Value.IsHierarchicalNamespaceEnabled; + + if (_useHierarchicalNamespaceOverride.HasValue && _useHierarchicalNamespaceOverride.Value != detectedHns) + { + if (_useHierarchicalNamespaceOverride.Value) + { + // Claiming Gen2 on a Gen1 account — DataLake API calls will fail at runtime. + throw new FileStoreException( + "'UseHierarchicalNamespace' is set to 'true' but the storage account does not have " + + "Hierarchical Namespace enabled. Correct the configuration or use a Gen2 storage account."); + } + + // Override=false on a Gen2 account is safe but suboptimal. + _logger?.LogWarning( + "'UseHierarchicalNamespace' is set to 'false' but the storage account has Hierarchical Namespace enabled. " + + "Flat-namespace operations will be used, which means moves are not atomic and directory operations are less efficient. " + + "Remove the setting to use native Gen2 operations."); + } + + _hnsEnabled = _useHierarchicalNamespaceOverride ?? detectedHns; + if (_hnsEnabled.Value) + { + _logger?.LogInformation("Azure Blob Storage Hierarchical Namespace (ADLS Gen2) detected. Using native directory and atomic move operations."); + } + else + { + _logger?.LogInformation("Azure Blob Storage flat namespace detected. Using standard blob operations with virtual directories."); + } + } + catch (FileStoreException) + { + throw; + } + catch (Exception ex) when (_useHierarchicalNamespaceOverride.HasValue) + { + // GetAccountInfo failed (e.g. container-scoped SAS token) but an explicit override + // is configured, so trust it and proceed. + _hnsEnabled = _useHierarchicalNamespaceOverride.Value; + _logger?.LogWarning(ex, + "Unable to validate the Azure Blob Storage account type. " + + "Proceeding with 'UseHierarchicalNamespace' set to '{HnsEnabled}' from configuration.", + _hnsEnabled.Value); + } + catch (Exception ex) + { + throw new FileStoreException( + "Unable to determine the Azure Blob Storage account type (Gen1 flat namespace or Gen2 Hierarchical Namespace). " + + "This is required to select the correct storage operations. " + + "If you are using a container-scoped SAS token, set 'UseHierarchicalNamespace' explicitly in your configuration.", ex); + } + } + finally + { + _capabilitiesLock.Release(); + } + } + public async Task GetFileInfoAsync(string path) { try @@ -82,6 +177,35 @@ public async Task GetFileInfoAsync(string path) public async Task GetDirectoryInfoAsync(string path) { + if (_hnsEnabled == true) + { + try + { + if (path == string.Empty) + { + return new BlobDirectory(path, _clock.UtcNow); + } + + var prefix = this.Combine(_basePrefix, path); + var directoryClient = _dataLakeFileSystemClient.GetDirectoryClient(prefix); + + if (await directoryClient.ExistsAsync()) + { + return new BlobDirectory(path, _clock.UtcNow); + } + + return null; + } + catch (RequestFailedException ex) when (ex.Status == 404) + { + return null; + } + catch (Exception ex) + { + throw new FileStoreException($"Cannot get directory info with path '{path}'.", ex); + } + } + try { if (path == string.Empty) @@ -143,18 +267,30 @@ private async IAsyncEnumerable GetDirectoryContentByHierarchyAs } folderPath = folderPath.Trim('/'); - yield return new BlobDirectory(folderPath, _clock.UtcNow); - } - else - { - var itemName = Path.GetFileName(WebUtility.UrlDecode(blob.Blob.Name)).Trim('/'); - // Ignore directory marker files. - if (itemName != DirectoryMarkerFileName) + if (blob.Blob is not null && blob.Blob.Properties is not null) { - var itemPath = this.Combine(path?.Trim('/'), itemName); - yield return new BlobFile(itemPath, blob.Blob.Properties.ContentLength, blob.Blob.Properties.LastModified); + yield return new BlobDirectory( + folderPath, + blob.Blob.Properties.LastModified.HasValue + ? blob.Blob.Properties.LastModified.Value.DateTime + : _clock.UtcNow); } + else + { + yield return new BlobDirectory(folderPath, _clock.UtcNow); + } + + continue; + } + + var itemName = Path.GetFileName(WebUtility.UrlDecode(blob.Blob.Name)).Trim('/'); + + // Ignore directory marker files. + if (!string.Equals(itemName, DirectoryMarkerFileName, StringComparison.Ordinal)) + { + var itemPath = this.Combine(path?.Trim('/'), itemName); + yield return new BlobFile(itemPath, blob.Blob.Properties.ContentLength, blob.Blob.Properties.LastModified); } } } @@ -163,14 +299,62 @@ private async IAsyncEnumerable GetDirectoryContentFlatAsync(str { path = this.NormalizePath(path); - // Folders are considered case sensitive in blob storage. - var directories = new HashSet(); - var prefix = this.Combine(_basePrefix, path); prefix = NormalizePrefix(prefix); - var page = _blobContainer.GetBlobsAsync(BlobTraits.Metadata, BlobStates.None, prefix, CancellationToken.None); - await foreach (var blob in page) + if (_hnsEnabled == true) + { + var page = _blobContainer.GetBlobsAsync(BlobTraits.Metadata, BlobStates.None, prefix, CancellationToken.None); + await foreach (var blob in page) + { + var name = blob.Name; + + if (blob.Metadata.TryGetValue("hdi_isfolder", out var value) && + value.Equals("true", StringComparison.OrdinalIgnoreCase)) + { + var directoryName = name; + if (!string.IsNullOrEmpty(_basePrefix)) + { + directoryName = directoryName[(_basePrefix.Length - 1)..]; + } + + if (blob.Properties is not null) + { + yield return new BlobDirectory( + directoryName, + blob.Properties.LastModified.HasValue + ? blob.Properties.LastModified.Value.DateTime + : _clock.UtcNow); + } + else + { + yield return new BlobDirectory(directoryName, _clock.UtcNow); + } + + continue; + } + + if (name.EndsWith(DirectoryMarkerFileName, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (!string.IsNullOrEmpty(_basePrefix)) + { + name = name[(_basePrefix.Length - 1)..]; + } + + yield return new BlobFile(name, blob.Properties.ContentLength, blob.Properties.LastModified); + } + + yield break; + } + + // Flat namespace: infer directory hierarchy from blob paths. + var directories = new HashSet(); + + var flatPage = _blobContainer.GetBlobsAsync(BlobTraits.Metadata, BlobStates.None, prefix, CancellationToken.None); + await foreach (var blob in flatPage) { var name = WebUtility.UrlDecode(blob.Name); @@ -209,6 +393,23 @@ private async IAsyncEnumerable GetDirectoryContentFlatAsync(str public async Task TryCreateDirectoryAsync(string path) { + if (_hnsEnabled == true) + { + try + { + var prefix = this.Combine(_basePrefix, path); + var directoryClient = _dataLakeFileSystemClient.GetDirectoryClient(prefix); + var response = await directoryClient.CreateIfNotExistsAsync(); + + // CreateIfNotExistsAsync returns null if it already existed. + return response is not null; + } + catch (Exception ex) + { + throw new FileStoreException($"Cannot create directory '{path}'.", ex); + } + } + // Since directories are only created implicitly when creating blobs, we // simply pretend like we created the directory, unless there is already // a blob with the same path. @@ -255,6 +456,40 @@ public async Task TryDeleteFileAsync(string path) public async Task TryDeleteDirectoryAsync(string path) { + if (_hnsEnabled == true) + { + try + { + if (string.IsNullOrEmpty(path)) + { + throw new FileStoreException("Cannot delete the root directory."); + } + + var prefix = this.Combine(_basePrefix, path); + var directoryClient = _dataLakeFileSystemClient.GetDirectoryClient(prefix); + + if (!await directoryClient.ExistsAsync()) + { + return false; + } + + await directoryClient.DeleteAsync(recursive: true); + return true; + } + catch (FileStoreException) + { + throw; + } + catch (RequestFailedException ex) when (ex.Status == 404) + { + return false; + } + catch (Exception ex) + { + throw new FileStoreException($"Cannot delete directory '{path}'.", ex); + } + } + try { if (string.IsNullOrEmpty(path)) @@ -288,6 +523,24 @@ public async Task TryDeleteDirectoryAsync(string path) public async Task MoveFileAsync(string oldPath, string newPath) { + if (_hnsEnabled == true) + { + try + { + var oldFullPath = this.Combine(_basePrefix, oldPath); + var newFullPath = this.Combine(_basePrefix, newPath); + + var fileClient = _dataLakeFileSystemClient.GetFileClient(oldFullPath); + await fileClient.RenameAsync(newFullPath); + } + catch (Exception ex) + { + throw new FileStoreException($"Cannot move file '{oldPath}' to '{newPath}'.", ex); + } + + return; + } + try { await CopyFileAsync(oldPath, newPath); diff --git a/src/OrchardCore/OrchardCore.FileStorage.AzureBlob/BlobStorageOptions.cs b/src/OrchardCore/OrchardCore.FileStorage.AzureBlob/BlobStorageOptions.cs index fd2d4f627fd..8588e12e971 100644 --- a/src/OrchardCore/OrchardCore.FileStorage.AzureBlob/BlobStorageOptions.cs +++ b/src/OrchardCore/OrchardCore.FileStorage.AzureBlob/BlobStorageOptions.cs @@ -17,6 +17,19 @@ public abstract class BlobStorageOptions /// public string BasePath { get; set; } = ""; + /// + /// Overrides auto-detection of Hierarchical Namespace (HNS / ADLS Gen2) support. + /// Leave null (default) to auto-detect via GetAccountInfo at startup. + /// Set to true or false to skip detection and force the behavior explicitly. + /// + /// + /// Auto-detection requires storage account-level permissions. If you are using a + /// container-scoped SAS token, detection will fail and startup will be blocked. + /// In that case, set this to true for a Gen2 (HNS-enabled) account, or + /// false for a Gen1 (flat namespace) account. + /// + public bool? UseHierarchicalNamespace { get; set; } + /// /// Returns a value indicating whether the basic state of the configuration is valid. /// diff --git a/src/OrchardCore/OrchardCore.FileStorage.AzureBlob/OrchardCore.FileStorage.AzureBlob.csproj b/src/OrchardCore/OrchardCore.FileStorage.AzureBlob/OrchardCore.FileStorage.AzureBlob.csproj index 71c90e93f7e..ececdce2685 100644 --- a/src/OrchardCore/OrchardCore.FileStorage.AzureBlob/OrchardCore.FileStorage.AzureBlob.csproj +++ b/src/OrchardCore/OrchardCore.FileStorage.AzureBlob/OrchardCore.FileStorage.AzureBlob.csproj @@ -16,6 +16,7 @@ + diff --git a/src/OrchardCore/OrchardCore.Media.Abstractions/Events/IMediaEventHandler.cs b/src/OrchardCore/OrchardCore.Media.Abstractions/Events/IMediaEventHandler.cs index cbea7e55b35..67a0d587d1a 100644 --- a/src/OrchardCore/OrchardCore.Media.Abstractions/Events/IMediaEventHandler.cs +++ b/src/OrchardCore/OrchardCore.Media.Abstractions/Events/IMediaEventHandler.cs @@ -13,5 +13,7 @@ public interface IMediaEventHandler Task MediaMovedAsync(MediaMoveContext context) => Task.CompletedTask; Task MediaCreatingDirectoryAsync(MediaCreatingContext context) => Task.CompletedTask; Task MediaCreatedDirectoryAsync(MediaCreatedContext context) => Task.CompletedTask; + Task MediaCreatedFileAsync(MediaCreatedContext context) => Task.CompletedTask; + Task MediaCopiedFileAsync(MediaMoveContext context) => Task.CompletedTask; Task MediaPermittedStorageAsync(MediaPermittedStorageContext context) => Task.CompletedTask; } diff --git a/src/OrchardCore/OrchardCore.Media.Core/DefaultMediaFileStore.cs b/src/OrchardCore/OrchardCore.Media.Core/DefaultMediaFileStore.cs index 48b01b34ded..46aa6f3d86a 100644 --- a/src/OrchardCore/OrchardCore.Media.Core/DefaultMediaFileStore.cs +++ b/src/OrchardCore/OrchardCore.Media.Core/DefaultMediaFileStore.cs @@ -57,6 +57,16 @@ public virtual IAsyncEnumerable GetDirectoryContentAsync(string return _fileStore.GetDirectoryContentAsync(path, includeSubDirectories); } + public virtual IAsyncEnumerable GetFilesAsync(string path = null) + { + return _fileStore.GetFilesAsync(path); + } + + public virtual IAsyncEnumerable GetDirectoriesAsync(string path = null) + { + return _fileStore.GetDirectoriesAsync(path); + } + public virtual async Task TryCreateDirectoryAsync(string path) { var creatingContext = new MediaCreatingContext @@ -146,6 +156,8 @@ public virtual async Task CopyFileAsync(string srcPath, string dstPath) } await _fileStore.CopyFileAsync(srcPath, dstPath); + + await _mediaEventHandlers.InvokeAsync((handler, ctx) => handler.MediaCopiedFileAsync(ctx), new MediaMoveContext { OldPath = srcPath, NewPath = dstPath }, _logger); } public virtual Task GetFileStreamAsync(string path) @@ -194,7 +206,11 @@ public virtual async Task CreateFileFromStreamAsync(string path, Stream await ValidateAvailableStorageAsync(outputStream.Length); - return await _fileStore.CreateFileFromStreamAsync(context.Path, outputStream, overwrite); + var result = await _fileStore.CreateFileFromStreamAsync(context.Path, outputStream, overwrite); + + await _mediaEventHandlers.InvokeAsync((handler, ctx) => handler.MediaCreatedFileAsync(ctx), new MediaCreatedContext { Path = result }, _logger); + + return result; } finally { @@ -206,7 +222,11 @@ public virtual async Task CreateFileFromStreamAsync(string path, Stream { await ValidateAvailableStorageAsync(inputStream.Length); - return await _fileStore.CreateFileFromStreamAsync(path, inputStream, overwrite); + var result = await _fileStore.CreateFileFromStreamAsync(path, inputStream, overwrite); + + await _mediaEventHandlers.InvokeAsync((handler, ctx) => handler.MediaCreatedFileAsync(ctx), new MediaCreatedContext { Path = result }, _logger); + + return result; } } diff --git a/test/OrchardCore.Tests.Integration/AzureBlob/BlobFileStoreGen1Tests.cs b/test/OrchardCore.Tests.Integration/AzureBlob/BlobFileStoreGen1Tests.cs new file mode 100644 index 00000000000..6ea301c36a5 --- /dev/null +++ b/test/OrchardCore.Tests.Integration/AzureBlob/BlobFileStoreGen1Tests.cs @@ -0,0 +1,27 @@ +using Xunit; + +namespace OrchardCore.Tests.Integration.AzureBlob; + +/// +/// Runs all tests with flat-namespace (Gen1) behavior. +/// +public sealed class BlobFileStoreGen1Tests : BlobFileStoreTestsBase +{ + protected override bool? UseHierarchicalNamespaceOverride => false; + + [AzuriteFact] + public async Task CreateDirectory_UsesMarkerFile() + { + await TryCreateDirectoryAsync("gen1-dir"); + + // Gen1 simulates directories with a marker blob — verify it exists. + var blobs = new List(); + await foreach (var blob in ContainerClient.GetBlobsAsync(Azure.Storage.Blobs.Models.BlobTraits.None, Azure.Storage.Blobs.Models.BlobStates.None, "gen1-dir/", CancellationToken.None)) + { + blobs.Add(blob.Name); + } + + Assert.Single(blobs); + Assert.EndsWith("OrchardCore.Media.txt", blobs[0]); + } +} diff --git a/test/OrchardCore.Tests.Integration/AzureBlob/BlobFileStoreGen2Tests.cs b/test/OrchardCore.Tests.Integration/AzureBlob/BlobFileStoreGen2Tests.cs new file mode 100644 index 00000000000..772c7dd73ac --- /dev/null +++ b/test/OrchardCore.Tests.Integration/AzureBlob/BlobFileStoreGen2Tests.cs @@ -0,0 +1,105 @@ +using Azure.Storage.Files.DataLake; +using OrchardCore.FileStorage; +using Xunit; + +namespace OrchardCore.Tests.Integration.AzureBlob; + +/// +/// Runs all tests with Gen2 (HNS) behavior. +/// Auto-detection via GetAccountInfo is used — no UseHierarchicalNamespace override. +/// +public sealed class BlobFileStoreGen2Tests : BlobFileStoreTestsBase +{ + protected override async Task CreateContainerAsync(string connectionString, string containerName) + => await new DataLakeServiceClient(connectionString) + .GetFileSystemClient(containerName) + .CreateIfNotExistsAsync(); + + [AzuriteFact] + public async Task CreateDirectory_UsesNoMarkerFile() + { + await TryCreateDirectoryAsync("gen2-dir"); + + // Gen2 creates real directory objects via the DataLake API — no marker blob. + var blobs = new List(); + await foreach (var blob in ContainerClient.GetBlobsAsync(Azure.Storage.Blobs.Models.BlobTraits.None, Azure.Storage.Blobs.Models.BlobStates.None, "gen2-dir/", CancellationToken.None)) + { + blobs.Add(blob.Name); + } + + Assert.Empty(blobs); + } + + [AzuriteFact] + public async Task CreateDirectory_Nested_CreatesIntermediateDirectories() + { + await TryCreateDirectoryAsync("a/b/c"); + + Assert.NotNull(await GetDirectoryInfoAsync("a/b/c")); + Assert.NotNull(await GetDirectoryInfoAsync("a/b")); + Assert.NotNull(await GetDirectoryInfoAsync("a")); + } + + [AzuriteFact] + public async Task GetDirectoryContent_Flat_IncludesGen2Directories() + { + await TryCreateDirectoryAsync("flat-gen2"); + await CreateTestFileAsync("flat-gen2/file.txt"); + await TryCreateDirectoryAsync("flat-gen2/subdir"); + await CreateTestFileAsync("flat-gen2/subdir/nested.txt"); + + var entries = new List(); + await foreach (var entry in GetDirectoryContentAsync("flat-gen2", includeSubDirectories: true)) + { + entries.Add(entry); + } + + // Gen2 flat listing should detect directories via hdi_isfolder metadata. + Assert.Contains(entries, e => e.IsDirectory && e.Name == "subdir"); + Assert.Contains(entries, e => !e.IsDirectory && e.Name == "file.txt"); + Assert.Contains(entries, e => !e.IsDirectory && e.Name == "nested.txt"); + } + + [AzuriteFact] + public async Task MoveFile_IsAtomic() + { + // Gen2 move uses DataLake RenameAsync which is an atomic server-side operation. + await CreateTestFileAsync("atomic-src.txt", "atomic"); + + await MoveFileAsync("atomic-src.txt", "atomic-dst.txt"); + + // Source should not exist and destination should have the content. + Assert.Null(await GetFileInfoAsync("atomic-src.txt")); + Assert.Equal("atomic", await ReadFileContentAsync("atomic-dst.txt")); + } + + [AzuriteFact] + public async Task GetDirectoryInfo_AfterDeletingDirectory_ReturnsNull() + { + await TryCreateDirectoryAsync("temp-dir"); + Assert.NotNull(await GetDirectoryInfoAsync("temp-dir")); + + await TryDeleteDirectoryAsync("temp-dir"); + + Assert.Null(await GetDirectoryInfoAsync("temp-dir")); + } + + [AzuriteFact] + public async Task CreateDirectory_EmptyDirectory_ExistsWithNoContent() + { + await TryCreateDirectoryAsync("empty-gen2-dir"); + + var info = await GetDirectoryInfoAsync("empty-gen2-dir"); + Assert.NotNull(info); + Assert.True(info.IsDirectory); + + var entries = new List(); + await foreach (var entry in GetDirectoryContentAsync("empty-gen2-dir")) + { + entries.Add(entry); + } + + // A real Gen2 directory should exist even with no files inside. + Assert.Empty(entries); + } +} diff --git a/test/OrchardCore.Tests.Integration/AzureBlob/BlobFileStoreTestsBase.cs b/test/OrchardCore.Tests.Integration/AzureBlob/BlobFileStoreTestsBase.cs new file mode 100644 index 00000000000..e7d7c538c7f --- /dev/null +++ b/test/OrchardCore.Tests.Integration/AzureBlob/BlobFileStoreTestsBase.cs @@ -0,0 +1,459 @@ +using Azure.Storage.Blobs; +using Microsoft.AspNetCore.StaticFiles; +using Moq; +using OrchardCore.FileStorage; +using OrchardCore.FileStorage.AzureBlob; +using OrchardCore.Modules; +using Xunit; + +namespace OrchardCore.Tests.Integration.AzureBlob; + +/// +/// Integration tests for that run against Azurite. +/// Subclasses set to exercise Gen1 (flat) or Gen2 (HNS) code paths. +/// +public abstract class BlobFileStoreTestsBase : IAsyncLifetime +{ + private const string ConnectionStringEnvVar = "AZURITE_CONNECTION_STRING"; + + /// + /// Override for . + /// null (default) lets auto-detect + /// via GetAccountInfo. Set to false to force flat-namespace (Gen1) behavior + /// against an HNS-enabled Azurite instance. + /// + protected virtual bool? UseHierarchicalNamespaceOverride => null; + + /// + /// Creates the test container. Gen1 uses the Blob SDK (flat namespace). + /// Gen2 subclasses override to use the DataLake SDK, which stamps the container as an HNS filesystem. + /// + protected virtual async Task CreateContainerAsync(string connectionString, string containerName) + => await new BlobContainerClient(connectionString, containerName).CreateIfNotExistsAsync(); + + private BlobFileStore _store; + private BlobContainerClient _containerClient; + private string _containerName; + + protected BlobContainerClient ContainerClient => _containerClient; + + public async ValueTask InitializeAsync() + { + var connectionString = System.Environment.GetEnvironmentVariable(ConnectionStringEnvVar); + _containerName = $"test-{Guid.NewGuid():N}"; + + var options = new TestBlobStorageOptions + { + ConnectionString = connectionString, + ContainerName = _containerName, + BasePath = "", + UseHierarchicalNamespace = UseHierarchicalNamespaceOverride, + }; + + await CreateContainerAsync(connectionString, _containerName); + _containerClient = new BlobContainerClient(connectionString, _containerName); + + var clock = Mock.Of(c => c.UtcNow == DateTime.UtcNow); + var contentTypeProvider = new FileExtensionContentTypeProvider(); + + _store = new BlobFileStore(options, clock, contentTypeProvider); + await _store.EnsureCapabilitiesAsync(); + } + + public async ValueTask DisposeAsync() + { + if (_containerClient is not null) + { + await _containerClient.DeleteIfExistsAsync(); + } + } + + protected async Task CreateTestFileAsync(string path, string content = "test content") + { + using var stream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(content)); + return await _store.CreateFileFromStreamAsync(path, stream); + } + + protected async Task ReadFileContentAsync(string path) + { + using var stream = await _store.GetFileStreamAsync(path); + using var reader = new StreamReader(stream); + return await reader.ReadToEndAsync(); + } + + protected Task TryCreateDirectoryAsync(string path) + => _store.TryCreateDirectoryAsync(path); + + protected Task TryDeleteDirectoryAsync(string path) + => _store.TryDeleteDirectoryAsync(path); + + protected Task GetDirectoryInfoAsync(string path) + => _store.GetDirectoryInfoAsync(path); + + protected Task GetFileInfoAsync(string path) + => _store.GetFileInfoAsync(path); + + protected IAsyncEnumerable GetDirectoryContentAsync(string path = null, bool includeSubDirectories = false) + => _store.GetDirectoryContentAsync(path, includeSubDirectories); + + protected Task MoveFileAsync(string oldPath, string newPath) + => _store.MoveFileAsync(oldPath, newPath); + + // -- File operations -- + + [AzuriteFact] + public async Task CreateFile_ReturnsPath() + { + var result = await CreateTestFileAsync("folder/file.txt"); + Assert.Equal("folder/file.txt", result); + } + + [AzuriteFact] + public async Task GetFileInfo_ReturnsCorrectMetadata() + { + var content = "hello world"; + await CreateTestFileAsync("info-test.txt", content); + + var info = await _store.GetFileInfoAsync("info-test.txt"); + + Assert.NotNull(info); + Assert.Equal("info-test.txt", info.Path); + Assert.Equal("info-test.txt", info.Name); + Assert.Equal(content.Length, info.Length); + Assert.False(info.IsDirectory); + } + + [AzuriteFact] + public async Task GetFileInfo_NonExistent_ReturnsNull() + { + var info = await _store.GetFileInfoAsync("does-not-exist.txt"); + Assert.Null(info); + } + + [AzuriteFact] + public async Task GetFileStream_ReturnsContent() + { + var expected = "stream content test"; + await CreateTestFileAsync("stream-test.txt", expected); + + var actual = await ReadFileContentAsync("stream-test.txt"); + + Assert.Equal(expected, actual); + } + + [AzuriteFact] + public async Task GetFileStream_NonExistent_Throws() + { + await Assert.ThrowsAsync( + () => _store.GetFileStreamAsync("no-such-file.txt")); + } + + [AzuriteFact] + public async Task DeleteFile_ReturnsTrue() + { + await CreateTestFileAsync("delete-me.txt"); + + var result = await _store.TryDeleteFileAsync("delete-me.txt"); + + Assert.True(result); + Assert.Null(await _store.GetFileInfoAsync("delete-me.txt")); + } + + [AzuriteFact] + public async Task DeleteFile_NonExistent_ReturnsFalse() + { + var result = await _store.TryDeleteFileAsync("ghost.txt"); + Assert.False(result); + } + + [AzuriteFact] + public async Task CopyFile_CreatesNewFile() + { + var content = "copy me"; + await CreateTestFileAsync("original.txt", content); + + await _store.CopyFileAsync("original.txt", "copied.txt"); + + Assert.Equal(content, await ReadFileContentAsync("original.txt")); + Assert.Equal(content, await ReadFileContentAsync("copied.txt")); + } + + [AzuriteFact] + public async Task CopyFile_SamePath_Throws() + { + await CreateTestFileAsync("same.txt"); + + await Assert.ThrowsAsync( + () => _store.CopyFileAsync("same.txt", "same.txt")); + } + + [AzuriteFact] + public async Task CreateFile_OverwriteTrue_Succeeds() + { + await CreateTestFileAsync("overwrite.txt", "v1"); + + using var stream = new MemoryStream("v2"u8.ToArray()); + await _store.CreateFileFromStreamAsync("overwrite.txt", stream, overwrite: true); + + var content = await ReadFileContentAsync("overwrite.txt"); + Assert.Equal("v2", content); + } + + [AzuriteFact] + public async Task CreateFile_OverwriteFalse_Throws() + { + await CreateTestFileAsync("no-overwrite.txt"); + + using var stream = new MemoryStream("v2"u8.ToArray()); + await Assert.ThrowsAsync( + () => _store.CreateFileFromStreamAsync("no-overwrite.txt", stream, overwrite: false)); + } + + // -- Directory operations -- + + [AzuriteFact] + public async Task GetDirectoryInfo_Root_ReturnsEntry() + { + var info = await _store.GetDirectoryInfoAsync(string.Empty); + + Assert.NotNull(info); + Assert.True(info.IsDirectory); + } + + [AzuriteFact] + public async Task GetDirectoryInfo_Existing_ReturnsEntry() + { + await _store.TryCreateDirectoryAsync("my-folder"); + + var info = await _store.GetDirectoryInfoAsync("my-folder"); + + Assert.NotNull(info); + Assert.True(info.IsDirectory); + Assert.Equal("my-folder", info.Path); + } + + [AzuriteFact] + public async Task GetDirectoryInfo_NonExistent_ReturnsNull() + { + var info = await _store.GetDirectoryInfoAsync("no-such-folder"); + + // Gen1 flat namespace: a directory "exists" if any blobs match the prefix. + // Since there are none, it returns null. + // Gen2 HNS: DataLake directory doesn't exist, returns null. + Assert.Null(info); + } + + [AzuriteFact] + public async Task CreateDirectory_NewDirectory_Succeeds() + { + var result = await _store.TryCreateDirectoryAsync("new-dir"); + + Assert.True(result); + + var info = await _store.GetDirectoryInfoAsync("new-dir"); + Assert.NotNull(info); + } + + [AzuriteFact] + public async Task DeleteDirectory_WithContents_DeletesAll() + { + await _store.TryCreateDirectoryAsync("dir-to-delete"); + await CreateTestFileAsync("dir-to-delete/file1.txt"); + await CreateTestFileAsync("dir-to-delete/file2.txt"); + + var result = await _store.TryDeleteDirectoryAsync("dir-to-delete"); + + Assert.True(result); + Assert.Null(await _store.GetFileInfoAsync("dir-to-delete/file1.txt")); + Assert.Null(await _store.GetFileInfoAsync("dir-to-delete/file2.txt")); + } + + [AzuriteFact] + public async Task DeleteDirectory_NonExistent_ReturnsFalse() + { + var result = await _store.TryDeleteDirectoryAsync("phantom-dir"); + Assert.False(result); + } + + [AzuriteFact] + public async Task DeleteDirectory_Root_Throws() + { + await Assert.ThrowsAsync( + () => _store.TryDeleteDirectoryAsync(string.Empty)); + } + + // -- Move -- + + [AzuriteFact] + public async Task MoveFile_MovesToNewPath() + { + var content = "move me"; + await CreateTestFileAsync("src.txt", content); + + await _store.MoveFileAsync("src.txt", "dst.txt"); + + Assert.Null(await _store.GetFileInfoAsync("src.txt")); + Assert.Equal(content, await ReadFileContentAsync("dst.txt")); + } + + [AzuriteFact] + public async Task MoveFile_AcrossDirectories() + { + await _store.TryCreateDirectoryAsync("dir-a"); + await _store.TryCreateDirectoryAsync("dir-b"); + await CreateTestFileAsync("dir-a/moved.txt", "data"); + + await _store.MoveFileAsync("dir-a/moved.txt", "dir-b/moved.txt"); + + Assert.Null(await _store.GetFileInfoAsync("dir-a/moved.txt")); + Assert.Equal("data", await ReadFileContentAsync("dir-b/moved.txt")); + } + + // -- Directory content listing -- + + [AzuriteFact] + public async Task GetDirectoryContent_ListsFilesAndDirs() + { + await CreateTestFileAsync("root-file.txt"); + await _store.TryCreateDirectoryAsync("sub-dir"); + await CreateTestFileAsync("sub-dir/nested.txt"); + + var entries = new List(); + await foreach (var entry in _store.GetDirectoryContentAsync()) + { + entries.Add(entry); + } + + Assert.Contains(entries, e => e.Name == "root-file.txt" && !e.IsDirectory); + Assert.Contains(entries, e => e.Name == "sub-dir" && e.IsDirectory); + } + + [AzuriteFact] + public async Task GetDirectoryContent_ExcludesMarkerFiles() + { + await _store.TryCreateDirectoryAsync("marker-test"); + + var entries = new List(); + await foreach (var entry in _store.GetDirectoryContentAsync("marker-test")) + { + entries.Add(entry); + } + + // The marker file (OrchardCore.Media.txt) used in Gen1 should never appear in listings. + Assert.DoesNotContain(entries, e => e.Name == "OrchardCore.Media.txt"); + } + + [AzuriteFact] + public async Task GetDirectoryContent_Flat_ListsNestedContent() + { + await CreateTestFileAsync("flat/a.txt"); + await CreateTestFileAsync("flat/sub/b.txt"); + + var entries = new List(); + await foreach (var entry in _store.GetDirectoryContentAsync("flat", includeSubDirectories: true)) + { + entries.Add(entry); + } + + Assert.Contains(entries, e => !e.IsDirectory && e.Name == "a.txt"); + Assert.Contains(entries, e => !e.IsDirectory && e.Name == "b.txt"); + } + + [AzuriteFact] + public async Task CreateDirectory_AlreadyExists_ReturnsFalse() + { + await _store.TryCreateDirectoryAsync("existing-dir"); + + // Creating the same directory again should indicate it already existed. + var result = await _store.TryCreateDirectoryAsync("existing-dir"); + + // Gen1 (flat namespace) always returns true — no real directory object to check against. + // Gen2 (HNS) returns false because the directory already exists as a first-class object. + if (UseHierarchicalNamespaceOverride == false) + { + Assert.True(result); + } + else + { + Assert.False(result); + } + } + + [AzuriteFact] + public async Task MoveFile_NonExistent_Throws() + { + await Assert.ThrowsAsync( + () => _store.MoveFileAsync("no-such-file.txt", "destination.txt")); + } + + [AzuriteFact] + public async Task DeleteDirectory_WithNestedSubdirectories_DeletesAll() + { + await _store.TryCreateDirectoryAsync("parent"); + await _store.TryCreateDirectoryAsync("parent/child"); + await CreateTestFileAsync("parent/top.txt"); + await CreateTestFileAsync("parent/child/deep.txt"); + + var result = await _store.TryDeleteDirectoryAsync("parent"); + + Assert.True(result); + Assert.Null(await _store.GetFileInfoAsync("parent/top.txt")); + Assert.Null(await _store.GetFileInfoAsync("parent/child/deep.txt")); + Assert.Null(await _store.GetDirectoryInfoAsync("parent")); + } + + [AzuriteFact] + public async Task GetDirectoryContent_Subdirectory_ListsOnlyDirectChildren() + { + await _store.TryCreateDirectoryAsync("listing"); + await CreateTestFileAsync("listing/file-a.txt"); + await _store.TryCreateDirectoryAsync("listing/inner"); + await CreateTestFileAsync("listing/inner/file-b.txt"); + + var entries = new List(); + await foreach (var entry in _store.GetDirectoryContentAsync("listing")) + { + entries.Add(entry); + } + + Assert.Contains(entries, e => e.Name == "file-a.txt" && !e.IsDirectory); + Assert.Contains(entries, e => e.Name == "inner" && e.IsDirectory); + // file-b.txt is nested inside "inner", should not appear at this level. + Assert.DoesNotContain(entries, e => e.Name == "file-b.txt"); + } + + [AzuriteFact] + public async Task MoveFile_PreservesContent() + { + var content = "preserve this content across move"; + await CreateTestFileAsync("move-preserve.txt", content); + + await _store.MoveFileAsync("move-preserve.txt", "moved-preserve.txt"); + + var actual = await ReadFileContentAsync("moved-preserve.txt"); + Assert.Equal(content, actual); + } +} + +/// +/// Concrete for testing (the base class is abstract). +/// +internal sealed class TestBlobStorageOptions : BlobStorageOptions; + +/// +/// Skips the test when AZURITE_CONNECTION_STRING is not set. +/// +[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] +internal sealed class AzuriteFactAttribute : FactAttribute +{ + public AzuriteFactAttribute( + [System.Runtime.CompilerServices.CallerFilePath] string sourceFilePath = null, + [System.Runtime.CompilerServices.CallerLineNumber] int sourceLineNumber = -1) + : base(sourceFilePath, sourceLineNumber) + { + if (string.IsNullOrEmpty(System.Environment.GetEnvironmentVariable("AZURITE_CONNECTION_STRING"))) + { + Skip = "Azurite is not configured. Set AZURITE_CONNECTION_STRING to run this test."; + } + } +} diff --git a/test/OrchardCore.Tests.Integration/OrchardCore.Tests.Integration.csproj b/test/OrchardCore.Tests.Integration/OrchardCore.Tests.Integration.csproj index 7c8203ca812..03be9dffba4 100644 --- a/test/OrchardCore.Tests.Integration/OrchardCore.Tests.Integration.csproj +++ b/test/OrchardCore.Tests.Integration/OrchardCore.Tests.Integration.csproj @@ -9,6 +9,7 @@ +