diff --git a/AGENTS.md b/AGENTS.md index 214100ef8c0..8348147e1e5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -391,6 +391,14 @@ npm run cms:test ## Common Extension Points +### Adding New Modules + +When a new module is added to the CMS, it must be registered in `src/OrchardCore/OrchardCore.Application.Cms.Core.Targets/OrchardCore.Application.Cms.Core.Targets.csproj` so it is discoverable by the application. Add a `ProjectReference` with `PrivateAssets="none"` and keep the references sorted alphabetically: + +```xml + +``` + ### Registering Services ```csharp diff --git a/OrchardCore.slnx b/OrchardCore.slnx index ddb4c52e87e..6052cb852bf 100644 --- a/OrchardCore.slnx +++ b/OrchardCore.slnx @@ -64,7 +64,7 @@ - + @@ -78,8 +78,10 @@ + + @@ -197,6 +199,10 @@ + + + + diff --git a/mkdocs.yml b/mkdocs.yml index d6dd9c0d0d3..ca35c793e0f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -250,6 +250,8 @@ nav: - Auto Setup: reference/modules/AutoSetup/README.md - Features: reference/modules/Features/README.md - Contents: reference/modules/Contents/README.md + - Content Transfer: reference/modules/ContentTransfer/README.md + - Data Pipelines: reference/modules/DataOrchestrator/README.md - Configuration: reference/modules/Configuration/README.md - Cors: reference/modules/Cors/README.md - Custom Settings: reference/modules/CustomSettings/README.md diff --git a/src/OrchardCore.Cms.Web/appsettings.Development.json b/src/OrchardCore.Cms.Web/appsettings.Development.json index a0a6bbea31c..20592d908fa 100644 --- a/src/OrchardCore.Cms.Web/appsettings.Development.json +++ b/src/OrchardCore.Cms.Web/appsettings.Development.json @@ -5,5 +5,10 @@ "YesSql": "Information", "Microsoft.Hosting.Lifetime": "Information" } + }, + "OrchardCore": { + "OrchardCore_Tenants": { + "TenantRemovalAllowed": true + } } } diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/AdminMenu.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/AdminMenu.cs new file mode 100644 index 00000000000..df0cc085116 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/AdminMenu.cs @@ -0,0 +1,43 @@ +using Microsoft.Extensions.Localization; +using OrchardCore.ContentTransfer.Controllers; +using OrchardCore.Mvc.Core.Utilities; +using OrchardCore.Navigation; + +namespace OrchardCore.ContentTransfer; + +public sealed class AdminMenu : AdminNavigationProvider +{ + private readonly IStringLocalizer S; + + public AdminMenu(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + protected override ValueTask BuildAsync(NavigationBuilder builder) + { + var adminControllerName = typeof(AdminController).ControllerName(); + + builder + .Add(S["Content"], content => content + .Add(S["Bulk Import"], S["Bulk Import"].PrefixPosition(), transfer => transfer + .Action(nameof(AdminController.List), adminControllerName, new + { + area = ContentTransferConstants.Feature.ModuleId, + }) + .Permission(ContentTransferPermissions.ListContentTransferEntries) + .LocalNav() + ) + .Add(S["Bulk Export"], S["Bulk Export"].PrefixPosition(), transfer => transfer + .Action(nameof(AdminController.Export), adminControllerName, new + { + area = ContentTransferConstants.Feature.ModuleId, + }) + .Permission(ContentTransferPermissions.ExportContentFromFile) + .LocalNav() + ) + ); + + return ValueTask.CompletedTask; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/BackgroundTasks/ExportFilesBackgroundTask.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/BackgroundTasks/ExportFilesBackgroundTask.cs new file mode 100644 index 00000000000..614807c92b2 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/BackgroundTasks/ExportFilesBackgroundTask.cs @@ -0,0 +1,389 @@ +using System.Data; +using DocumentFormat.OpenXml; +using DocumentFormat.OpenXml.Packaging; +using DocumentFormat.OpenXml.Spreadsheet; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Localization; +using Microsoft.Extensions.Options; +using OrchardCore.BackgroundTasks; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata; +using OrchardCore.ContentManagement.Records; +using OrchardCore.ContentTransfer.Indexes; +using OrchardCore.ContentTransfer.Models; +using OrchardCore.Entities; +using OrchardCore.Locking.Distributed; +using OrchardCore.Modules; +using YesSql; + +namespace OrchardCore.ContentTransfer.BackgroundTasks; + +[BackgroundTask( + Title = "Export Files Processor", + Schedule = "*/5 * * * *", + Description = "Regularly check for queued export requests and process them.", + LockTimeout = 3_000, LockExpiration = 30_000)] +public sealed class ExportFilesBackgroundTask : IBackgroundTask +{ + private static readonly TimeSpan _exportLockTimeout = TimeSpan.FromSeconds(1); + private static readonly TimeSpan _exportLockExpiration = TimeSpan.FromMinutes(30); + + public Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken cancellationToken) + => ProcessEntriesAsync(serviceProvider, cancellationToken); + + internal static async Task ProcessEntriesAsync(IServiceProvider serviceProvider, CancellationToken cancellationToken, string entryId = null) + { + var session = serviceProvider.GetRequiredService(); + var distributedLock = serviceProvider.GetRequiredService(); + var clock = serviceProvider.GetRequiredService(); + var contentImportOptions = serviceProvider.GetRequiredService>().Value; + var localizer = serviceProvider.GetRequiredService>(); + var fileStore = serviceProvider.GetRequiredService(); + var contentDefinitionManager = serviceProvider.GetRequiredService(); + var contentImportManager = serviceProvider.GetRequiredService(); + + var entries = await session.Query(x => + (x.Status == ContentTransferEntryStatus.New || x.Status == ContentTransferEntryStatus.Processing) + && x.Direction == ContentTransferDirection.Export + && (entryId == null || x.EntryId == entryId)) + .OrderBy(x => x.CreatedUtc) + .ListAsync(cancellationToken); + + var batchSize = contentImportOptions.ExportBatchSize < 1 + ? ContentImportOptions.DefaultExportBatchSize + : contentImportOptions.ExportBatchSize; + + foreach (var entry in entries) + { + if (cancellationToken.IsCancellationRequested) + { + break; + } + + (var locker, var locked) = await distributedLock.TryAcquireLockAsync( + GetExportLockKey(entry.EntryId), + _exportLockTimeout, + _exportLockExpiration); + + if (!locked) + { + continue; + } + + await using var acquiredLock = locker; + + var contentTypeDefinition = await contentDefinitionManager.GetTypeDefinitionAsync(entry.ContentType); + + if (contentTypeDefinition == null) + { + await SaveEntryWithErrorAsync(session, clock, entry, localizer["The content definition was removed."]); + continue; + } + + entry.Status = ContentTransferEntryStatus.Processing; + + var progressPart = entry.As(); + progressPart.Errors ??= []; + + try + { + var context = new ImportContentContext() + { + ContentItem = await serviceProvider.GetRequiredService().NewAsync(entry.ContentType), + ContentTypeDefinition = contentTypeDefinition, + }; + + var columns = await contentImportManager.GetColumnsAsync(context); + var exportColumns = columns.Where(x => x.Type != ImportColumnType.ImportOnly).ToList(); + + // Count total records for progress tracking. + var filters = entry.As(); + var hasFilters = filters != null; + + var totalCountQuery = BuildExportQuery( + session, entry.ContentType, hasFilters, + filters?.LatestOnly ?? false, + filters?.AllVersions ?? false, + filters?.CreatedFrom, filters?.CreatedTo, + filters?.ModifiedFrom, filters?.ModifiedTo, + filters?.Owners); + + var totalCount = await totalCountQuery.CountAsync(cancellationToken); + + progressPart.TotalRecords = totalCount; + + // Write Excel file directly to a temp file using pagination (avoids memory accumulation). + var fileName = entry.StoredFileName; + var tempFilePath = Path.GetTempFileName(); + + using var tempStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.None, bufferSize: 4096, FileOptions.DeleteOnClose); + using (var spreadsheetDocument = SpreadsheetDocument.Create(tempStream, SpreadsheetDocumentType.Workbook)) + { + var workbookPart = spreadsheetDocument.AddWorkbookPart(); + workbookPart.Workbook = new Workbook(); + + var worksheetPart = workbookPart.AddNewPart(); + worksheetPart.Worksheet = new Worksheet(new SheetData()); + + var sheets = workbookPart.Workbook.AppendChild(new Sheets()); + var sheet = new Sheet() + { + Id = workbookPart.GetIdOfPart(worksheetPart), + SheetId = 1, + Name = contentTypeDefinition.DisplayName?.Length > 31 + ? contentTypeDefinition.DisplayName[..31] + : contentTypeDefinition.DisplayName, + }; + sheets.Append(sheet); + + var sheetData = worksheetPart.Worksheet.GetFirstChild(); + + // Write header row. + var headerRow = new Row() { RowIndex = 1 }; + sheetData.Append(headerRow); + + uint columnIndex = 1; + var columnNames = new List(); + + foreach (var column in exportColumns) + { + var cell = new Cell() + { + CellReference = GetCellReference(columnIndex, 1), + DataType = CellValues.String, + CellValue = new CellValue(column.Name), + }; + headerRow.Append(cell); + columnNames.Add(column.Name); + columnIndex++; + } + + // Paginate content items and write each page directly. + uint rowIndex = 2; + var page = 0; + var contentManager = serviceProvider.GetRequiredService(); + + while (true) + { + if (cancellationToken.IsCancellationRequested) + { + break; + } + + var pageQuery = BuildExportQuery( + session, entry.ContentType, hasFilters, + filters?.LatestOnly ?? false, + filters?.AllVersions ?? false, + filters?.CreatedFrom, filters?.CreatedTo, + filters?.ModifiedFrom, filters?.ModifiedTo, + filters?.Owners); + + var contentItems = await pageQuery + .Skip(page * batchSize) + .Take(batchSize) + .ListAsync(cancellationToken); + + var items = contentItems.ToList(); + + if (items.Count == 0) + { + break; + } + + // Create a temporary DataTable for this batch only. + using var dataTable = new DataTable(); + + foreach (var colName in columnNames) + { + dataTable.Columns.Add(colName); + } + + foreach (var contentItem in items) + { + var mapContext = new ContentExportContext() + { + ContentItem = contentItem, + ContentTypeDefinition = contentTypeDefinition, + Row = dataTable.NewRow(), + }; + + await contentImportManager.ExportAsync(mapContext); + + // Write the row directly to the spreadsheet. + var row = new Row() { RowIndex = rowIndex }; + sheetData.Append(row); + + columnIndex = 1; + + foreach (var colName in columnNames) + { + var cellValue = mapContext.Row[colName]?.ToString() ?? string.Empty; + var cell = new Cell() + { + CellReference = GetCellReference(columnIndex, rowIndex), + DataType = CellValues.String, + CellValue = new CellValue(cellValue), + }; + row.Append(cell); + columnIndex++; + } + + rowIndex++; + progressPart.TotalProcessed++; + } + + // Save progress after each page. + progressPart.CurrentRow = (int)rowIndex - 2; + entry.ProcessSaveUtc = clock.UtcNow; + entry.Put(progressPart); + + session.Save(entry); + await session.SaveChangesAsync(cancellationToken); + + page++; + } + + workbookPart.Workbook.Save(); + } + + // Save the completed file to the file store. + tempStream.Seek(0, SeekOrigin.Begin); + await fileStore.CreateFileFromStreamAsync(fileName, tempStream, true); + + var nowUtc = clock.UtcNow; + entry.ProcessSaveUtc = nowUtc; + entry.CompletedUtc = nowUtc; + entry.Status = ContentTransferEntryStatus.Completed; + entry.Put(progressPart); + + session.Save(entry); + await session.SaveChangesAsync(cancellationToken); + + // Send notification if the Notifications module is enabled. + await TrySendExportNotificationAsync(serviceProvider, entry, contentTypeDefinition.DisplayName); + } + catch (Exception ex) + { + await SaveEntryWithErrorAsync(session, clock, entry, localizer["Error processing export: {0}", ex.Message]); + } + } + } + + private static async Task TrySendExportNotificationAsync( + IServiceProvider serviceProvider, + ContentTransferEntry entry, + string contentTypeName) + { + // Resolve IContentTransferNotificationHandler if available (registered when Notifications module is enabled). + var notificationHandler = serviceProvider.GetService(); + + if (notificationHandler != null) + { + await notificationHandler.NotifyExportCompletedAsync(entry, contentTypeName); + } + } + + private static async Task SaveEntryWithErrorAsync(ISession session, IClock clock, ContentTransferEntry entry, string error) + { + entry.Status = ContentTransferEntryStatus.Failed; + entry.Error = error; + entry.CompletedUtc = clock.UtcNow; + + session.Save(entry); + await session.SaveChangesAsync(); + } + + private static string GetCellReference(uint columnIndex, uint rowIndex) + { + var columnName = string.Empty; + var dividend = columnIndex; + + while (dividend > 0) + { + var modulo = (dividend - 1) % 26; + columnName = Convert.ToChar(65 + modulo) + columnName; + dividend = (dividend - modulo) / 26; + } + + return columnName + rowIndex; + } + + private static string GetExportLockKey(string entryId) + => $"ContentsTransfer_Export_{entryId}"; + + private static IQuery BuildExportQuery( + ISession session, + string contentTypeId, + bool hasFilters, + bool latestOnly, + bool allVersions, + DateTime? createdFrom, + DateTime? createdTo, + DateTime? modifiedFrom, + DateTime? modifiedTo, + string owners) + { + IQuery query; + + if (allVersions) + { + query = session.Query(x => x.ContentType == contentTypeId); + } + else if (latestOnly) + { + query = session.Query(x => x.ContentType == contentTypeId && x.Latest); + } + else + { + query = session.Query(x => x.ContentType == contentTypeId && x.Published); + } + + if (hasFilters) + { + if (createdFrom.HasValue) + { + query = query.Where(x => x.CreatedUtc >= createdFrom.Value); + } + + if (createdTo.HasValue) + { + query = query.Where(x => x.CreatedUtc <= createdTo.Value); + } + + if (modifiedFrom.HasValue) + { + query = query.Where(x => x.ModifiedUtc >= modifiedFrom.Value); + } + + if (modifiedTo.HasValue) + { + query = query.Where(x => x.ModifiedUtc <= modifiedTo.Value); + } + + if (!string.IsNullOrWhiteSpace(owners)) + { + var ownerList = owners.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + if (ownerList.Length == 1) + { + var owner = ownerList[0]; + query = query.Where(x => x.Owner == owner); + } + else if (ownerList.Length > 1) + { + var o0 = ownerList[0]; + var o1 = ownerList.ElementAtOrDefault(1) ?? o0; + var o2 = ownerList.ElementAtOrDefault(2) ?? o0; + var o3 = ownerList.ElementAtOrDefault(3) ?? o0; + var o4 = ownerList.ElementAtOrDefault(4) ?? o0; + + query = query.Where(x => + x.Owner == o0 || x.Owner == o1 || x.Owner == o2 || + x.Owner == o3 || x.Owner == o4); + } + } + } + + return query.OrderBy(x => x.CreatedUtc); + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/BackgroundTasks/ImportFilesBackgroundTask.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/BackgroundTasks/ImportFilesBackgroundTask.cs new file mode 100644 index 00000000000..b93a30503e4 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/BackgroundTasks/ImportFilesBackgroundTask.cs @@ -0,0 +1,656 @@ +using System.Data; +using DocumentFormat.OpenXml.Packaging; +using DocumentFormat.OpenXml.Spreadsheet; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Localization; +using Microsoft.Extensions.Options; +using OrchardCore.BackgroundTasks; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Handlers; +using OrchardCore.ContentManagement.Metadata; +using OrchardCore.ContentManagement.Metadata.Models; +using OrchardCore.ContentTransfer.Indexes; +using OrchardCore.ContentTransfer.Models; +using OrchardCore.Entities; +using OrchardCore.Locking.Distributed; +using OrchardCore.Modules; +using YesSql; + +namespace OrchardCore.ContentTransfer.BackgroundTasks; + +[BackgroundTask( + Title = "Imported Files Processor", + Schedule = "*/10 * * * *", + Description = "Regularly check for imported files and process them.", + LockTimeout = 3_000, + LockExpiration = 30_000)] +public sealed class ImportFilesBackgroundTask : IBackgroundTask +{ + private static readonly TimeSpan _importLockTimeout = TimeSpan.FromSeconds(1); + private static readonly TimeSpan _importLockExpiration = TimeSpan.FromMinutes(30); + + public Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken cancellationToken) + => ProcessEntriesAsync(serviceProvider, cancellationToken); + + internal static async Task ProcessEntriesAsync(IServiceProvider serviceProvider, CancellationToken cancellationToken, string entryId = null) + { + var session = serviceProvider.GetRequiredService(); + var distributedLock = serviceProvider.GetRequiredService(); + + var entries = await session.Query(x => + (x.Status == ContentTransferEntryStatus.New || x.Status == ContentTransferEntryStatus.Processing) + && x.Direction == ContentTransferDirection.Import + && (entryId == null || x.EntryId == entryId)) + .OrderBy(x => x.CreatedUtc) + .ListAsync(cancellationToken); + + foreach (var entry in entries) + { + if (cancellationToken.IsCancellationRequested) + { + break; + } + + (var locker, var locked) = await distributedLock.TryAcquireLockAsync( + GetImportLockKey(entry.EntryId), + _importLockTimeout, + _importLockExpiration); + + if (!locked) + { + continue; + } + + await using var acquiredLock = locker; + await using var scope = serviceProvider.CreateAsyncScope(); + await ProcessEntryAsync(scope.ServiceProvider, entry.EntryId, cancellationToken); + } + } + + private static async Task ProcessEntryAsync(IServiceProvider serviceProvider, string entryId, CancellationToken cancellationToken) + { + var session = serviceProvider.GetRequiredService(); + var clock = serviceProvider.GetRequiredService(); + var contentManager = serviceProvider.GetRequiredService(); + var contentImportOptions = serviceProvider.GetRequiredService>().Value; + var localizer = serviceProvider.GetRequiredService>(); + var fileStore = serviceProvider.GetRequiredService(); + var contentDefinitionManager = serviceProvider.GetRequiredService(); + var contentImportManager = serviceProvider.GetRequiredService(); + var entry = await session.Query(x => x.EntryId == entryId).FirstOrDefaultAsync(cancellationToken); + + if (entry == null || entry.Status == ContentTransferEntryStatus.Canceled || entry.Status == ContentTransferEntryStatus.CanceledWithImportedRecords) + { + return; + } + + var contentTypeDefinition = await contentDefinitionManager.GetTypeDefinitionAsync(entry.ContentType); + + if (contentTypeDefinition == null) + { + await SaveEntryWithErrorAsync(session, clock, entry, localizer["The content definition was removed."], cancellationToken); + return; + } + + var fileInfo = await fileStore.GetFileInfoAsync(entry.StoredFileName); + + if (fileInfo == null || fileInfo.Length == 0) + { + await SaveEntryWithErrorAsync(session, clock, entry, localizer["The import file no longer exists."], cancellationToken); + return; + } + + var batchSize = contentImportOptions.ImportBatchSize < 1 + ? ContentImportOptions.DefaultImportBatchSize + : contentImportOptions.ImportBatchSize; + + entry.Status = ContentTransferEntryStatus.Processing; + entry.Error = null; + entry.CompletedUtc = null; + + var progressPart = entry.As() ?? new ImportFileProcessStatsPart(); + progressPart.Errors ??= []; + progressPart.ErrorMessages ??= []; + entry.Put(progressPart); + + session.Save(entry); + await session.SaveChangesAsync(cancellationToken); + + await using var fileStream = await fileStore.GetFileStreamAsync(fileInfo); + + try + { + await ProcessExcelFileInBatchesAsync( + serviceProvider, + fileStream, + entry, + progressPart, + contentTypeDefinition, + contentManager, + contentImportManager, + session, + clock, + localizer, + batchSize, + cancellationToken); + } + catch (Exception ex) + { + await SaveEntryWithErrorAsync(session, clock, entry, localizer["Error processing file: {0}", ex.Message], cancellationToken); + return; + } + + if (await IsImportCanceledAsync(serviceProvider, entry.EntryId, cancellationToken)) + { + entry.Status = GetCanceledStatus(progressPart); + entry.ProcessSaveUtc = clock.UtcNow; + entry.CompletedUtc = clock.UtcNow; + entry.Put(progressPart); + session.Save(entry); + await session.SaveChangesAsync(cancellationToken); + return; + } + + var nowUtc = clock.UtcNow; + entry.ProcessSaveUtc = nowUtc; + entry.CompletedUtc = nowUtc; + entry.Status = progressPart.Errors.Count > 0 + ? ContentTransferEntryStatus.CompletedWithErrors + : ContentTransferEntryStatus.Completed; + entry.Put(progressPart); + + session.Save(entry); + await session.SaveChangesAsync(cancellationToken); + } + + private static async Task ProcessExcelFileInBatchesAsync( + IServiceProvider serviceProvider, + Stream stream, + ContentTransferEntry entry, + ImportFileProcessStatsPart progressPart, + ContentManagement.Metadata.Models.ContentTypeDefinition contentTypeDefinition, + IContentManager contentManager, + IContentImportManager contentImportManager, + ISession session, + IClock clock, + IStringLocalizer localizer, + int batchSize, + CancellationToken cancellationToken) + { + using var spreadsheetDocument = SpreadsheetDocument.Open(stream, false); + var workbookPart = spreadsheetDocument.WorkbookPart; + + if (workbookPart == null) + { + throw new InvalidOperationException(localizer["Unable to read the uploaded file."]); + } + + var firstSheet = workbookPart.Workbook.Descendants().FirstOrDefault(); + + if (firstSheet == null) + { + throw new InvalidOperationException(localizer["Unable to find a tab in the file that contains data."]); + } + + var worksheetPart = (WorksheetPart)workbookPart.GetPartById(firstSheet.Id); + var sheetData = worksheetPart.Worksheet.GetFirstChild(); + + if (sheetData == null) + { + throw new InvalidOperationException(localizer["Unable to find a tab in the file that contains data."]); + } + + var sharedStringTable = workbookPart.GetPartsOfType() + .FirstOrDefault()?.SharedStringTable; + + var headerRow = sheetData.Elements().FirstOrDefault(); + + if (headerRow == null) + { + throw new InvalidOperationException(localizer["Unable to find a tab in the file that contains data."]); + } + + var dataTable = new DataTable(); + var columnNames = new List(); + + foreach (var cell in headerRow.Descendants()) + { + var columnName = GetCellValue(cell, sharedStringTable)?.Trim() ?? string.Empty; + var columnIndex = GetColumnIndexFromCellReference(cell.CellReference); + + if (string.IsNullOrEmpty(columnName)) + { + columnName = "Col " + (columnIndex + 1); + } + + columnNames.Add(columnName); + var occurrences = columnNames.Count(x => x.Equals(columnName, StringComparison.OrdinalIgnoreCase)); + + if (occurrences > 1) + { + columnName += " " + occurrences; + } + + dataTable.Columns.Add(new DataColumn(columnName)); + } + + progressPart.TotalRecords = Math.Max(sheetData.Elements().Count() - 1, 0); + + var indexOfKeyColumn = dataTable.Columns.IndexOf(nameof(ContentItem.ContentItemId)); + var indexOfVersionKeyColumn = dataTable.Columns.IndexOf(nameof(ContentItem.ContentItemVersionId)); + var newRecords = new Dictionary(); + var existingVersionRows = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var existingRows = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + var rowIndex = 1; + + foreach (var sheetRow in sheetData.Elements().Skip(1)) + { + if (cancellationToken.IsCancellationRequested) + { + break; + } + + if (await IsImportCanceledAsync(serviceProvider, entry.EntryId, cancellationToken)) + { + break; + } + + if (rowIndex <= progressPart.CurrentRow) + { + rowIndex++; + continue; + } + + progressPart.TotalProcessed++; + progressPart.CurrentRow = rowIndex; + + var dataRow = dataTable.NewRow(); + var isEmpty = true; + + foreach (var cell in sheetRow.Descendants()) + { + var colIndex = GetColumnIndexFromCellReference(cell.CellReference); + + if (colIndex < dataTable.Columns.Count) + { + var value = GetCellValue(cell, sharedStringTable); + dataRow[colIndex] = value; + + if (!string.IsNullOrWhiteSpace(value)) + { + isEmpty = false; + } + } + } + + if (!isEmpty) + { + if (contentTypeDefinition.IsVersionable() && indexOfVersionKeyColumn > -1) + { + var contentItemVersionId = dataRow[indexOfVersionKeyColumn]?.ToString()?.Trim(); + + if (!string.IsNullOrEmpty(contentItemVersionId)) + { + existingVersionRows[contentItemVersionId] = new KeyValuePair(rowIndex, dataRow); + } + else if (indexOfKeyColumn > -1) + { + var contentItemId = dataRow[indexOfKeyColumn]?.ToString()?.Trim(); + + if (!string.IsNullOrEmpty(contentItemId)) + { + existingRows[contentItemId] = new KeyValuePair(rowIndex, dataRow); + } + else + { + newRecords[rowIndex] = dataRow; + } + } + else + { + newRecords[rowIndex] = dataRow; + } + } + else if (indexOfKeyColumn > -1) + { + var contentItemId = dataRow[indexOfKeyColumn]?.ToString()?.Trim(); + + if (!string.IsNullOrEmpty(contentItemId)) + { + existingRows[contentItemId] = new KeyValuePair(rowIndex, dataRow); + } + else + { + newRecords[rowIndex] = dataRow; + } + } + else + { + newRecords[rowIndex] = dataRow; + } + } + + if (newRecords.Count + existingRows.Count + existingVersionRows.Count >= batchSize) + { + await ProcessBatchAsync( + serviceProvider, + entry, + dataTable, + newRecords, + existingVersionRows, + existingRows, + contentTypeDefinition, + contentManager, + contentImportManager, + progressPart, + session, + clock, + cancellationToken); + + newRecords.Clear(); + existingVersionRows.Clear(); + existingRows.Clear(); + dataTable.Rows.Clear(); + } + + rowIndex++; + } + + if (newRecords.Count + existingRows.Count + existingVersionRows.Count > 0) + { + await ProcessBatchAsync( + serviceProvider, + entry, + dataTable, + newRecords, + existingVersionRows, + existingRows, + contentTypeDefinition, + contentManager, + contentImportManager, + progressPart, + session, + clock, + cancellationToken); + } + + dataTable.Dispose(); + } + + private static async Task ProcessBatchAsync( + IServiceProvider serviceProvider, + ContentTransferEntry entry, + DataTable dataTable, + Dictionary newRecords, + Dictionary> existingVersionRows, + Dictionary> existingRows, + ContentManagement.Metadata.Models.ContentTypeDefinition contentTypeDefinition, + IContentManager contentManager, + IContentImportManager contentImportManager, + ImportFileProcessStatsPart progressPart, + ISession session, + IClock clock, + CancellationToken cancellationToken) + { + if (existingVersionRows.Count > 0) + { + foreach (var existingVersionRow in existingVersionRows) + { + if (await IsImportCanceledAsync(serviceProvider, entry.EntryId, cancellationToken)) + { + return; + } + + var contentItem = await contentManager.GetVersionAsync(existingVersionRow.Key); + var isNew = contentItem == null; + + if (isNew) + { + contentItem = await contentManager.NewAsync(entry.ContentType); + } + + await ProcessRowAsync( + entry, + contentItem, + existingVersionRow.Value.Key, + existingVersionRow.Value.Value, + dataTable.Columns, + contentTypeDefinition, + contentManager, + contentImportManager, + progressPart, + isNew); + } + } + + if (existingRows.Count > 0) + { + var existingContentItems = (await contentManager.GetAsync(existingRows.Keys, VersionOptions.DraftRequired)) + .ToDictionary(x => x.ContentItemId); + + foreach (var existingRow in existingRows) + { + if (await IsImportCanceledAsync(serviceProvider, entry.EntryId, cancellationToken)) + { + return; + } + + var isNew = false; + + if (!existingContentItems.TryGetValue(existingRow.Key, out var contentItem)) + { + contentItem = await contentManager.NewAsync(entry.ContentType); + isNew = true; + } + + await ProcessRowAsync( + entry, + contentItem, + existingRow.Value.Key, + existingRow.Value.Value, + dataTable.Columns, + contentTypeDefinition, + contentManager, + contentImportManager, + progressPart, + isNew); + } + } + + foreach (var record in newRecords) + { + if (await IsImportCanceledAsync(serviceProvider, entry.EntryId, cancellationToken)) + { + return; + } + + await ProcessRowAsync( + entry, + await contentManager.NewAsync(entry.ContentType), + record.Key, + record.Value, + dataTable.Columns, + contentTypeDefinition, + contentManager, + contentImportManager, + progressPart, + true); + } + + entry.ProcessSaveUtc = clock.UtcNow; + entry.Put(progressPart); + + session.Save(entry); + await session.SaveChangesAsync(cancellationToken); + } + + private static async Task ProcessRowAsync( + ContentTransferEntry entry, + ContentItem contentItem, + int rowIndex, + DataRow row, + DataColumnCollection columns, + ContentManagement.Metadata.Models.ContentTypeDefinition contentTypeDefinition, + IContentManager contentManager, + IContentImportManager contentImportManager, + ImportFileProcessStatsPart progressPart, + bool isNew) + { + try + { + var mapContext = new ContentImportContext() + { + ContentItem = contentItem, + ContentTypeDefinition = contentTypeDefinition, + Columns = columns, + Row = row, + }; + + await contentImportManager.ImportAsync(mapContext); + + var validationResult = await contentManager.ValidateAsync(mapContext.ContentItem); + + if (!validationResult.Succeeded) + { + AddRowError(progressPart, rowIndex, FormatValidationErrors(validationResult)); + return; + } + + mapContext.ContentItem.Owner = entry.Owner; + mapContext.ContentItem.Author = entry.Author; + + if (isNew) + { + await contentManager.CreateAsync(mapContext.ContentItem, VersionOptions.DraftRequired); + } + else + { + await contentManager.UpdateAsync(mapContext.ContentItem); + } + + await contentManager.PublishAsync(mapContext.ContentItem); + progressPart.ImportedCount++; + + progressPart.Errors.Remove(rowIndex); + progressPart.ErrorMessages.Remove(rowIndex); + } + catch (Exception ex) + { + AddRowError(progressPart, rowIndex, ex.Message); + } + } + + private static void AddRowError(ImportFileProcessStatsPart progressPart, int rowIndex, string errorMessage) + { + progressPart.Errors ??= []; + progressPart.ErrorMessages ??= []; + + progressPart.Errors.Add(rowIndex); + progressPart.ErrorMessages[rowIndex] = string.IsNullOrWhiteSpace(errorMessage) + ? "The row failed to import." + : errorMessage; + } + + private static string FormatValidationErrors(ContentValidateResult validationResult) + { + var messages = validationResult.Errors + .Select(error => + { + if (error.MemberNames != null && error.MemberNames.Any()) + { + return $"{string.Join(", ", error.MemberNames)}: {error.ErrorMessage}"; + } + + return error.ErrorMessage; + }) + .Where(message => !string.IsNullOrWhiteSpace(message)) + .Distinct(StringComparer.Ordinal) + .ToArray(); + + return messages.Length > 0 + ? string.Join("; ", messages) + : "The row failed validation."; + } + + private static async Task SaveEntryWithErrorAsync(ISession session, IClock clock, ContentTransferEntry entry, string error, CancellationToken cancellationToken) + { + entry.Status = ContentTransferEntryStatus.Failed; + entry.Error = error; + entry.CompletedUtc = clock.UtcNow; + + session.Save(entry); + await session.SaveChangesAsync(cancellationToken); + } + + private static string GetImportLockKey(string entryId) + => $"ContentsTransfer_Import_{entryId}"; + + private static async Task IsImportCanceledAsync(IServiceProvider serviceProvider, string entryId, CancellationToken cancellationToken) + { + await using var scope = serviceProvider.CreateAsyncScope(); + var session = scope.ServiceProvider.GetRequiredService(); + var currentEntry = await session.Query(x => x.EntryId == entryId).FirstOrDefaultAsync(cancellationToken); + + return currentEntry?.Status == ContentTransferEntryStatus.Canceled + || currentEntry?.Status == ContentTransferEntryStatus.CanceledWithImportedRecords; + } + + private static ContentTransferEntryStatus GetCanceledStatus(ImportFileProcessStatsPart progressPart) + => (progressPart?.ImportedCount ?? 0) > 0 + ? ContentTransferEntryStatus.CanceledWithImportedRecords + : ContentTransferEntryStatus.Canceled; + + private static string GetCellValue(Cell cell, SharedStringTable sharedStringTable) + { + if (cell?.CellValue == null) + { + return string.Empty; + } + + var value = cell.CellValue.Text; + + if (cell.DataType != null && cell.DataType.Value == CellValues.SharedString) + { + if (sharedStringTable != null && int.TryParse(value, out var index)) + { + return sharedStringTable.ElementAt(index).InnerText; + } + } + + return value ?? string.Empty; + } + + private static int GetColumnIndexFromCellReference(string cellReference) + { + if (string.IsNullOrEmpty(cellReference)) + { + return 0; + } + + var columnLetters = string.Empty; + + foreach (var character in cellReference) + { + if (char.IsLetter(character)) + { + columnLetters += character; + } + else + { + break; + } + } + + var columnIndex = 0; + var multiplier = 1; + + for (var i = columnLetters.Length - 1; i >= 0; i--) + { + columnIndex += (columnLetters[i] - 'A' + 1) * multiplier; + multiplier *= 26; + } + + return columnIndex - 1; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Controllers/AdminController.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Controllers/AdminController.cs new file mode 100644 index 00000000000..85a644a48f4 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Controllers/AdminController.cs @@ -0,0 +1,1234 @@ +using System.Data; +using System.Security.Claims; +using DocumentFormat.OpenXml; +using DocumentFormat.OpenXml.Packaging; +using DocumentFormat.OpenXml.Spreadsheet; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Localization; +using Microsoft.AspNetCore.Mvc.Rendering; +using Microsoft.AspNetCore.Routing; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Localization; +using Microsoft.Extensions.Options; +using OrchardCore.Admin; +using OrchardCore.BackgroundJobs; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Display; +using OrchardCore.ContentManagement.Metadata; +using OrchardCore.ContentManagement.Records; +using OrchardCore.ContentTransfer.Indexes; +using OrchardCore.ContentTransfer.Models; +using OrchardCore.ContentTransfer.Services; +using OrchardCore.ContentTransfer.ViewModels; +using OrchardCore.DisplayManagement; +using OrchardCore.DisplayManagement.ModelBinding; +using OrchardCore.DisplayManagement.Notify; +using OrchardCore.Entities; +using OrchardCore.Modules; +using OrchardCore.Navigation; +using OrchardCore.Routing; +using YesSql; +using YesSql.Filters.Query; +using YesSql.Services; + +namespace OrchardCore.ContentTransfer.Controllers; + +public sealed class AdminController : Controller, IUpdateModel +{ + private readonly IAuthorizationService _authorizationService; + private readonly ISession _session; + + private readonly IDisplayManager _entryDisplayManager; + private readonly IContentTransferEntryAdminListQueryService _entriesAdminListQueryService; + private readonly IDisplayManager _entryOptionsDisplayManager; + private readonly INotifier _notifier; + private readonly IShapeFactory _shapeFactory; + private readonly PagerOptions _pagerOptions; + private readonly IClock _clock; + private readonly IContentTransferFileStore _contentTransferFileStore; + private readonly IContentManager _contentManager; + private readonly IDisplayManager _displayManager; + private readonly IUpdateModelAccessor _updateModelAccessor; + private readonly IContentImportManager _contentImportManager; + private readonly IContentItemDisplayManager _contentItemDisplayManager; + private readonly IContentDefinitionManager _contentDefinitionManager; + + private readonly IStringLocalizer S; + private readonly IHtmlLocalizer H; + + public AdminController( + IAuthorizationService authorizationService, + ISession session, + IShapeFactory shapeFactory, + IOptions pagerOptions, + IDisplayManager entryDisplayManager, + IContentTransferEntryAdminListQueryService entriesAdminListQueryService, + IDisplayManager entryOptionsDisplayManager, + INotifier notifier, + IStringLocalizer stringLocalizer, + IContentDefinitionManager contentDefinitionManager, + IHtmlLocalizer htmlLocalizer, + IContentTransferFileStore contentTransferFileStore, + IContentManager contentManager, + IDisplayManager displayManager, + IUpdateModelAccessor updateModelAccessor, + IContentImportManager contentImportManager, + IContentItemDisplayManager contentItemDisplayManager, + IClock clock) + { + _authorizationService = authorizationService; + _session = session; + _entryDisplayManager = entryDisplayManager; + _entriesAdminListQueryService = entriesAdminListQueryService; + _entryOptionsDisplayManager = entryOptionsDisplayManager; + _notifier = notifier; + S = stringLocalizer; + _contentDefinitionManager = contentDefinitionManager; + H = htmlLocalizer; + _contentTransferFileStore = contentTransferFileStore; + _contentManager = contentManager; + _displayManager = displayManager; + _updateModelAccessor = updateModelAccessor; + _contentImportManager = contentImportManager; + _contentItemDisplayManager = contentItemDisplayManager; + _shapeFactory = shapeFactory; + _pagerOptions = pagerOptions.Value; + _clock = clock; + } + + [Admin("content-transfer-entries", RouteName = "ListContentTransferEntries")] + public async Task List( + [ModelBinder(BinderType = typeof(ContentTransferEntryFilterEngineModelBinder), Name = "q")] QueryFilterResult queryFilterResult, + PagerParameters pagerParameters, + ListContentTransferEntryOptions options) + { + if (!await _authorizationService.AuthorizeAsync(HttpContext.User, ContentTransferPermissions.ListContentTransferEntries)) + { + return Forbid(); + } + + options.FilterResult = queryFilterResult; + await PopulateListOptionsAsync(options, ContentTransferDirection.Import); + + return View(await BuildListViewModelAsync(options, pagerParameters)); + } + + [HttpPost] + [ActionName(nameof(List))] + [FormValueRequired("submit.Filter")] + public async Task ListFilterPOST(ListContentTransferEntryOptions options) + { + return await FilterListAsync(nameof(List), options); + } + + [HttpPost] + [ActionName(nameof(List))] + [FormValueRequired("submit.BulkAction")] + public async Task ListPOST(ListContentTransferEntryOptions options, IEnumerable itemIds) + { + if (!await _authorizationService.AuthorizeAsync(HttpContext.User, ContentTransferPermissions.ListContentTransferEntries)) + { + return Forbid(); + } + + await ExecuteBulkActionAsync(itemIds, options.BulkAction, ContentTransferDirection.Import); + + return RedirectToAction(nameof(List)); + } + + public async Task Delete(string entryId, string returnUrl) + { + if (!await _authorizationService.AuthorizeAsync(HttpContext.User, ContentTransferPermissions.DeleteContentTransferEntries)) + { + return Forbid(); + } + + if (!string.IsNullOrWhiteSpace(entryId)) + { + var entry = await _session.Query(x => x.EntryId == entryId).FirstOrDefaultAsync(); + + if (entry != null) + { + if (!await DeleteEntryAsync(entry)) + { + await _notifier.ErrorAsync(H["The file for this transfer entry could not be deleted."]); + return RedirectTo(returnUrl); + } + + await _session.SaveChangesAsync(); + } + } + + return RedirectTo(returnUrl); + } + + [Admin("import/contents/{contentTypeId}", "ImportContentFromFile")] + public async Task Import(string contentTypeId) + { + var contentTypeDefinition = await _contentDefinitionManager.GetTypeDefinitionAsync(contentTypeId); + + if (contentTypeDefinition == null) + { + return NotFound(); + } + + var settings = contentTypeDefinition.GetSettings(); + + if (!settings.AllowBulkImport) + { + return BadRequest(); + } + + if (!await _authorizationService.AuthorizeAsync(User, ContentTransferPermissions.ImportContentFromFile, (object)contentTypeId)) + { + return Unauthorized(); + } + + var context = new ImportContentContext() + { + ContentItem = await _contentManager.NewAsync(contentTypeId), + ContentTypeDefinition = contentTypeDefinition, + }; + + var columns = await _contentImportManager.GetColumnsAsync(context); + + var importContent = new ImportContent() + { + ContentTypeId = contentTypeId, + ContentTypeName = contentTypeDefinition.Name, + }; + + var viewModel = new ContentImporterViewModel() + { + ContentTypeDefinition = contentTypeDefinition, + Content = await _displayManager.BuildEditorAsync(importContent, _updateModelAccessor.ModelUpdater, true, string.Empty, string.Empty), + Columns = columns.Where(x => x.Type != ImportColumnType.ExportOnly), + }; + + return View(viewModel); + } + + [HttpPost] + [ValidateAntiForgeryToken] + [ActionName(nameof(Import))] + [ContentTransferSizeLimit] + public async Task ImportPOST(string contentTypeId) + { + if (string.IsNullOrEmpty(contentTypeId)) + { + return NotFound(); + } + + if (!await _authorizationService.AuthorizeAsync(User, ContentTransferPermissions.ImportContentFromFile, (object)contentTypeId)) + { + return Unauthorized(); + } + + var contentTypeDefinition = await _contentDefinitionManager.GetTypeDefinitionAsync(contentTypeId); + + if (contentTypeDefinition == null) + { + return NotFound(); + } + + var settings = contentTypeDefinition.GetSettings(); + + if (!settings.AllowBulkImport) + { + return NotFound(); + } + + var importContent = new ImportContent() + { + ContentTypeId = contentTypeId, + ContentTypeName = contentTypeDefinition.Name, + }; + + var shape = await _displayManager.UpdateEditorAsync(importContent, _updateModelAccessor.ModelUpdater, false, string.Empty, string.Empty); + + if (ModelState.IsValid) + { + var extension = Path.GetExtension(importContent.File.FileName); + + // Create entry in the database + var fileName = Guid.NewGuid() + extension; + + var storedFileName = await _contentTransferFileStore.CreateFileFromStreamAsync(fileName, importContent.File.OpenReadStream(), false); + + var entry = new ContentTransferEntry() + { + EntryId = IdGenerator.GenerateId(), + ContentType = contentTypeId, + Owner = CurrentUserId(), + Author = User.Identity.Name, + UploadedFileName = importContent.File.FileName, + StoredFileName = storedFileName, + Status = ContentTransferEntryStatus.New, + Direction = ContentTransferDirection.Import, + CreatedUtc = _clock.UtcNow, + }; + + _session.Save(entry); + await _session.SaveChangesAsync(); + await TriggerImportProcessingAsync(entry.EntryId); + + await _notifier.SuccessAsync(H["The file was successfully added to the queue and will start processing shortly."]); + + return RedirectToAction(nameof(List)); + } + + var context = new ImportContentContext() + { + ContentItem = await _contentManager.NewAsync(contentTypeId), + ContentTypeDefinition = contentTypeDefinition, + }; + + var columns = await _contentImportManager.GetColumnsAsync(context); + + var viewModel = new ContentImporterViewModel() + { + ContentTypeDefinition = contentTypeDefinition, + Content = shape, + Columns = columns.Where(x => x.Type != ImportColumnType.ExportOnly), + }; + + return View(viewModel); + } + + [Admin("import/contents/{contentTypeId}/download-template", "ImportContentDownloadTemplateTemplate")] + public async Task DownloadTemplate(string contentTypeId) + { + if (string.IsNullOrEmpty(contentTypeId)) + { + return NotFound(); + } + + var contentTypeDefinition = await _contentDefinitionManager.GetTypeDefinitionAsync(contentTypeId); + + if (contentTypeDefinition == null) + { + return NotFound(); + } + + var settings = contentTypeDefinition.GetSettings(); + + if (!settings.AllowBulkImport) + { + return BadRequest(); + } + + if (!await _authorizationService.AuthorizeAsync(User, ContentTransferPermissions.ImportContentFromFile, (object)contentTypeId)) + { + return Unauthorized(); + } + + var context = new ImportContentContext() + { + ContentItem = await _contentManager.NewAsync(contentTypeId), + ContentTypeDefinition = contentTypeDefinition, + }; + + var columns = await _contentImportManager.GetColumnsAsync(context); + + var content = new MemoryStream(); + using (var spreadsheetDocument = SpreadsheetDocument.Create(content, SpreadsheetDocumentType.Workbook)) + { + var workbookPart = spreadsheetDocument.AddWorkbookPart(); + workbookPart.Workbook = new Workbook(); + + var worksheetPart = workbookPart.AddNewPart(); + worksheetPart.Worksheet = new Worksheet(new SheetData()); + + var sheets = workbookPart.Workbook.AppendChild(new Sheets()); + var sheet = new Sheet() + { + Id = workbookPart.GetIdOfPart(worksheetPart), + SheetId = 1, + Name = contentTypeDefinition.DisplayName?.Length > 31 + ? contentTypeDefinition.DisplayName[..31] + : contentTypeDefinition.DisplayName, + }; + sheets.Append(sheet); + + var sheetData = worksheetPart.Worksheet.GetFirstChild(); + var headerRow = new Row() { RowIndex = 1 }; + sheetData.Append(headerRow); + + uint columnIndex = 1; + foreach (var column in columns) + { + if (column.Type == ImportColumnType.ExportOnly) + { + continue; + } + + var cell = new Cell() + { + CellReference = GetCellReference(columnIndex, 1), + DataType = CellValues.String, + CellValue = new CellValue(column.Name), + }; + headerRow.Append(cell); + columnIndex++; + } + + workbookPart.Workbook.Save(); + } + + content.Seek(0, SeekOrigin.Begin); + + return new FileStreamResult(content, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") + { + FileDownloadName = $"{contentTypeDefinition.Name}_Template.xlsx", + }; + } + + [Admin("export/contents", "ExportContentToFile")] + public async Task Export( + [ModelBinder(BinderType = typeof(ContentTransferEntryFilterEngineModelBinder), Name = "q")] QueryFilterResult queryFilterResult, + PagerParameters pagerParameters, + ListContentTransferEntryOptions options) + { + if (!await _authorizationService.AuthorizeAsync(HttpContext.User, ContentTransferPermissions.ExportContentFromFile)) + { + return Forbid(); + } + + options.FilterResult = queryFilterResult; + await PopulateListOptionsAsync(options, ContentTransferDirection.Export, CurrentUserId()); + + return View(await BuildBulkExportViewModelAsync(options, pagerParameters)); + } + + [HttpPost] + [ActionName(nameof(Export))] + [FormValueRequired("submit.Filter")] + public async Task ExportFilterPOST(ListContentTransferEntryOptions options) + { + if (!await _authorizationService.AuthorizeAsync(HttpContext.User, ContentTransferPermissions.ExportContentFromFile)) + { + return Forbid(); + } + + return await FilterListAsync(nameof(Export), options); + } + + [HttpPost] + [ActionName(nameof(Export))] + [FormValueRequired("submit.BulkAction")] + public async Task ExportPOST(ListContentTransferEntryOptions options, IEnumerable itemIds) + { + if (!await _authorizationService.AuthorizeAsync(HttpContext.User, ContentTransferPermissions.ExportContentFromFile)) + { + return Forbid(); + } + + await ExecuteBulkActionAsync(itemIds, options.BulkAction, ContentTransferDirection.Export, CurrentUserId()); + + return RedirectToAction(nameof(Export)); + } + + [Admin("export/contents/download-file", "ExportContentDownloadFile")] + public async Task DownloadExport( + string contentTypeId, + bool partialExport = false, + DateTime? createdFrom = null, + DateTime? createdTo = null, + DateTime? modifiedFrom = null, + DateTime? modifiedTo = null, + string owners = null, + bool publishedOnly = true, + bool latestOnly = false, + bool allVersions = false) + { + var contentTypeDefinition = await _contentDefinitionManager.GetTypeDefinitionAsync(contentTypeId); + + if (contentTypeDefinition == null) + { + return NotFound(); + } + + var settings = contentTypeDefinition.GetSettings(); + + if (!settings.AllowBulkExport) + { + return BadRequest(); + } + + if (!await _authorizationService.AuthorizeAsync(User, ContentTransferPermissions.ExportContentFromFile, (object)contentTypeId)) + { + return Unauthorized(); + } + + var context = new ImportContentContext() + { + ContentItem = await _contentManager.NewAsync(contentTypeId), + ContentTypeDefinition = contentTypeDefinition, + }; + + var columns = await _contentImportManager.GetColumnsAsync(context); + var exportColumns = columns.Where(x => x.Type != ImportColumnType.ImportOnly).ToList(); + + // Build a filtered query for counting. + var countQuery = BuildExportQuery(contentTypeId, partialExport, latestOnly, allVersions, createdFrom, createdTo, modifiedFrom, modifiedTo, owners); + var totalCount = await countQuery.CountAsync(); + + var contentImportOptions = HttpContext.RequestServices.GetRequiredService>().Value; + var threshold = contentImportOptions.ExportQueueThreshold; + + if (totalCount > threshold) + { + // Queue the export for background processing. + var fileName = $"{contentTypeDefinition.Name}_Export_{Guid.NewGuid():N}.xlsx"; + + var entry = new ContentTransferEntry() + { + EntryId = IdGenerator.GenerateId(), + ContentType = contentTypeId, + Owner = CurrentUserId(), + Author = User.Identity.Name, + UploadedFileName = $"{contentTypeDefinition.Name}_Export.xlsx", + StoredFileName = fileName, + Status = ContentTransferEntryStatus.New, + Direction = ContentTransferDirection.Export, + CreatedUtc = _clock.UtcNow, + }; + + // Store the filters so the background task can apply them. + if (partialExport) + { + entry.Put(new ExportFilterPart + { + PublishedOnly = publishedOnly, + LatestOnly = latestOnly, + AllVersions = allVersions, + CreatedFrom = createdFrom, + CreatedTo = createdTo, + ModifiedFrom = modifiedFrom, + ModifiedTo = modifiedTo, + Owners = owners, + }); + } + + _session.Save(entry); + await _session.SaveChangesAsync(); + await TriggerExportProcessingAsync(entry.EntryId); + + await _notifier.InformationAsync(H["The export contains {0} records and has been queued for background processing. You can download it from Bulk Export when it is ready.", totalCount]); + + return RedirectToAction(nameof(Export)); + } + + // Immediate export: write directly to a temp file stream using pagination. + var batchSize = contentImportOptions.ExportBatchSize < 1 ? 200 : contentImportOptions.ExportBatchSize; + + var tempFilePath = Path.GetTempFileName(); + try + { + using (var fileStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.None)) + using (var spreadsheetDocument = SpreadsheetDocument.Create(fileStream, SpreadsheetDocumentType.Workbook)) + { + var workbookPart = spreadsheetDocument.AddWorkbookPart(); + workbookPart.Workbook = new Workbook(); + + var worksheetPart = workbookPart.AddNewPart(); + worksheetPart.Worksheet = new Worksheet(new SheetData()); + + var sheets = workbookPart.Workbook.AppendChild(new Sheets()); + var sheet = new Sheet() + { + Id = workbookPart.GetIdOfPart(worksheetPart), + SheetId = 1, + Name = contentTypeDefinition.DisplayName?.Length > 31 + ? contentTypeDefinition.DisplayName[..31] + : contentTypeDefinition.DisplayName, + }; + sheets.Append(sheet); + + var sheetData = worksheetPart.Worksheet.GetFirstChild(); + + // Write header row. + var headerRow = new Row() { RowIndex = 1 }; + sheetData.Append(headerRow); + + uint columnIndex = 1; + var columnNames = new List(); + + foreach (var column in exportColumns) + { + var cell = new Cell() + { + CellReference = GetCellReference(columnIndex, 1), + DataType = CellValues.String, + CellValue = new CellValue(column.Name), + }; + headerRow.Append(cell); + columnNames.Add(column.Name); + columnIndex++; + } + + // Paginate content items and write each page directly. + uint rowIndex = 2; + var page = 0; + + while (true) + { + var pageQuery = BuildExportQuery(contentTypeId, partialExport, latestOnly, allVersions, createdFrom, createdTo, modifiedFrom, modifiedTo, owners); + + var contentItems = await pageQuery + .Skip(page * batchSize) + .Take(batchSize) + .ListAsync(); + + var items = contentItems.ToList(); + + if (items.Count == 0) + { + break; + } + + // Create a temporary DataTable for this batch only. + using var dataTable = new DataTable(); + + foreach (var colName in columnNames) + { + dataTable.Columns.Add(colName); + } + + foreach (var contentItem in items) + { + var mapContext = new ContentExportContext() + { + ContentItem = contentItem, + ContentTypeDefinition = contentTypeDefinition, + Row = dataTable.NewRow(), + }; + + await _contentImportManager.ExportAsync(mapContext); + + var row = new Row() { RowIndex = rowIndex }; + sheetData.Append(row); + + columnIndex = 1; + + foreach (var colName in columnNames) + { + var cellValue = mapContext.Row[colName]?.ToString() ?? string.Empty; + var cell = new Cell() + { + CellReference = GetCellReference(columnIndex, rowIndex), + DataType = CellValues.String, + CellValue = new CellValue(cellValue), + }; + row.Append(cell); + columnIndex++; + } + + rowIndex++; + } + + page++; + } + + workbookPart.Workbook.Save(); + } + + // Read back from temp file for download (file-based, not memory-based). + var downloadStream = new FileStream(tempFilePath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 4096, options: FileOptions.DeleteOnClose); + + return new FileStreamResult(downloadStream, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") + { + FileDownloadName = $"{contentTypeDefinition.Name}_Export.xlsx", + }; + } + catch + { + if (System.IO.File.Exists(tempFilePath)) + { + System.IO.File.Delete(tempFilePath); + } + + throw; + } + } + + [Admin("export/dashboard", "ExportDashboard")] + public Task ExportDashboard( + [ModelBinder(BinderType = typeof(ContentTransferEntryFilterEngineModelBinder), Name = "q")] QueryFilterResult queryFilterResult, + PagerParameters pagerParameters, + ListContentTransferEntryOptions options) + { + return Export(queryFilterResult, pagerParameters, options); + } + + [Admin("import/entries/{entryId}/process", "ProcessImport")] + public async Task ProcessImport(string entryId, string returnUrl) + { + if (string.IsNullOrEmpty(entryId)) + { + return NotFound(); + } + + var entry = await _session.Query(x => + x.EntryId == entryId + && x.Direction == ContentTransferDirection.Import + && x.Owner == CurrentUserId()) + .FirstOrDefaultAsync(); + + if (entry == null) + { + return NotFound(); + } + + if (!await _authorizationService.AuthorizeAsync(User, ContentTransferPermissions.ImportContentFromFile, (object)entry.ContentType)) + { + return Forbid(); + } + + if (entry.Status != ContentTransferEntryStatus.New && entry.Status != ContentTransferEntryStatus.Processing) + { + await _notifier.WarningAsync(H["Only new or processing import files can be processed again."]); + return RedirectTo(returnUrl); + } + + await TriggerImportProcessingAsync(entry.EntryId); + await _notifier.SuccessAsync(H["The import file will be processed in the background shortly."]); + + return RedirectTo(returnUrl); + } + + [Admin("import/entries/{entryId}/cancel", "CancelImport")] + public async Task CancelImport(string entryId, string returnUrl) + { + if (string.IsNullOrEmpty(entryId)) + { + return NotFound(); + } + + var entry = await _session.Query(x => + x.EntryId == entryId + && x.Direction == ContentTransferDirection.Import + && x.Owner == CurrentUserId()) + .FirstOrDefaultAsync(); + + if (entry == null) + { + return NotFound(); + } + + if (!await _authorizationService.AuthorizeAsync(User, ContentTransferPermissions.ImportContentFromFile, (object)entry.ContentType)) + { + return Forbid(); + } + + if (entry.Status != ContentTransferEntryStatus.New && entry.Status != ContentTransferEntryStatus.Processing) + { + await _notifier.WarningAsync(H["Only new or processing import files can be canceled."]); + return RedirectTo(returnUrl); + } + + var progressPart = entry.As(); + var importedCount = progressPart?.ImportedCount ?? 0; + + entry.Status = importedCount > 0 + ? ContentTransferEntryStatus.CanceledWithImportedRecords + : ContentTransferEntryStatus.Canceled; + entry.ProcessSaveUtc = _clock.UtcNow; + entry.CompletedUtc = _clock.UtcNow; + + _session.Save(entry); + await _session.SaveChangesAsync(); + + await _notifier.SuccessAsync(importedCount > 0 + ? H["The import was canceled after some records had already been imported."] + : H["The import was canceled before any records were imported."]); + + return RedirectTo(returnUrl); + } + + [Admin("export/dashboard/{entryId}/download", "DownloadExportFile")] + public async Task DownloadExportFile(string entryId) + { + if (string.IsNullOrEmpty(entryId)) + { + return NotFound(); + } + + if (!await _authorizationService.AuthorizeAsync(HttpContext.User, ContentTransferPermissions.ExportContentFromFile)) + { + return Forbid(); + } + + var entry = await _session.Query(x => + x.EntryId == entryId + && x.Direction == ContentTransferDirection.Export + && x.Owner == CurrentUserId()) + .FirstOrDefaultAsync(); + + if (entry == null || entry.Status != ContentTransferEntryStatus.Completed) + { + return NotFound(); + } + + var fileInfo = await _contentTransferFileStore.GetFileInfoAsync(entry.StoredFileName); + + if (fileInfo == null || fileInfo.Length == 0) + { + await _notifier.ErrorAsync(H["The export file is no longer available."]); + return RedirectToAction(nameof(Export)); + } + + var stream = await _contentTransferFileStore.GetFileStreamAsync(fileInfo); + + return new FileStreamResult(stream, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") + { + FileDownloadName = entry.UploadedFileName ?? $"{entry.ContentType}_Export.xlsx", + }; + } + + [Admin("import/entries/{entryId}/download-errors", "DownloadErrors")] + public async Task DownloadErrors(string entryId) + { + if (string.IsNullOrEmpty(entryId)) + { + return NotFound(); + } + + if (!await _authorizationService.AuthorizeAsync(HttpContext.User, ContentTransferPermissions.ImportContentFromFile)) + { + return Forbid(); + } + + var entry = await _session.Query(x => + x.EntryId == entryId + && x.Direction == ContentTransferDirection.Import + && x.Owner == CurrentUserId()) + .FirstOrDefaultAsync(); + + if (entry == null) + { + return NotFound(); + } + + var statsPart = entry.As(); + + if (statsPart == null || statsPart.Errors == null || statsPart.Errors.Count == 0) + { + await _notifier.WarningAsync(H["No error records found for this entry."]); + return RedirectToAction(nameof(List)); + } + + var fileInfo = await _contentTransferFileStore.GetFileInfoAsync(entry.StoredFileName); + + if (fileInfo == null || fileInfo.Length == 0) + { + await _notifier.ErrorAsync(H["The original import file is no longer available."]); + return RedirectToAction(nameof(List)); + } + + await using var sourceStream = await _contentTransferFileStore.GetFileStreamAsync(fileInfo); + + using var sourceDoc = SpreadsheetDocument.Open(sourceStream, false); + var sourceWorkbookPart = sourceDoc.WorkbookPart; + var sourceSheet = sourceWorkbookPart.WorksheetParts.First().Worksheet; + var sourceSheetData = sourceSheet.GetFirstChild(); + var sharedStringTable = sourceWorkbookPart.GetPartsOfType().FirstOrDefault(); + + var tempFilePath = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid():N}.xlsx"); + var outputStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose | FileOptions.SequentialScan); + using (var destDoc = SpreadsheetDocument.Create(outputStream, SpreadsheetDocumentType.Workbook)) + { + var destWorkbookPart = destDoc.AddWorkbookPart(); + destWorkbookPart.Workbook = new Workbook(); + + if (sharedStringTable != null) + { + var destSharedStringPart = destWorkbookPart.AddNewPart(); + sharedStringTable.SharedStringTable.Save(destSharedStringPart); + } + + var destWorksheetPart = destWorkbookPart.AddNewPart(); + destWorksheetPart.Worksheet = new Worksheet(new SheetData()); + + var sheets = destDoc.WorkbookPart.Workbook.AppendChild(new Sheets()); + sheets.Append(new Sheet() + { + Id = destDoc.WorkbookPart.GetIdOfPart(destWorksheetPart), + SheetId = 1, + Name = "Errors", + }); + + var destSheetData = destWorksheetPart.Worksheet.GetFirstChild(); + statsPart.ErrorMessages ??= []; + + var sourceRowIndex = 0; + uint destRowIndex = 1; + + foreach (var sourceRow in sourceSheetData.Elements()) + { + if (sourceRowIndex == 0) + { + destSheetData.Append(CloneRowWithErrorMessage(sourceRow, destRowIndex, S["Errors"])); + destRowIndex++; + sourceRowIndex++; + continue; + } + + if (statsPart.Errors.Contains(sourceRowIndex)) + { + statsPart.ErrorMessages.TryGetValue(sourceRowIndex, out var errorMessage); + destSheetData.Append(CloneRowWithErrorMessage(sourceRow, destRowIndex, errorMessage)); + destRowIndex++; + } + + sourceRowIndex++; + } + + destWorkbookPart.Workbook.Save(); + } + + outputStream.Position = 0; + + return new FileStreamResult(outputStream, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") + { + FileDownloadName = $"{entry.ContentType}_Errors.xlsx", + }; + } + + private async Task PopulateListOptionsAsync(ListContentTransferEntryOptions options, ContentTransferDirection direction, string owner = null) + { + options.SearchText = options.FilterResult.ToString(); + options.OriginalSearchText = options.SearchText; + options.Direction = direction; + options.Owner = owner; + options.RouteValues.TryAdd("q", options.FilterResult.ToString()); + + options.Statuses = + [ + new(S["New"], nameof(ContentTransferEntryStatus.New)), + new(S["Processing"], nameof(ContentTransferEntryStatus.Processing)), + new(S["Completed"], nameof(ContentTransferEntryStatus.Completed)), + new(S["Completed With Errors"], nameof(ContentTransferEntryStatus.CompletedWithErrors)), + new(S["Canceled"], nameof(ContentTransferEntryStatus.Canceled)), + new(S["Canceled With Imported Records"], nameof(ContentTransferEntryStatus.CanceledWithImportedRecords)), + new(S["Failed"], nameof(ContentTransferEntryStatus.Failed)), + ]; + + options.Sorts = + [ + new(S["Recently created"], nameof(ContentTransferEntryOrder.Latest)), + new(S["Previously created"], nameof(ContentTransferEntryOrder.Oldest)), + ]; + + options.BulkActions = + [ + new(S["Remove"], nameof(ContentTransferEntryBulkAction.Remove)), + ]; + + options.ImportableTypes = direction == ContentTransferDirection.Import + ? await GetTransferableContentTypesAsync(ContentTransferDirection.Import) + : []; + options.ExportableTypes = direction == ContentTransferDirection.Export + ? await GetTransferableContentTypesAsync(ContentTransferDirection.Export) + : []; + } + + private async Task> GetTransferableContentTypesAsync(ContentTransferDirection direction) + { + var contentTypes = new List(); + + foreach (var contentTypeDefinition in await _contentDefinitionManager.ListTypeDefinitionsAsync()) + { + var settings = contentTypeDefinition.GetSettings(); + var isAllowed = direction == ContentTransferDirection.Import + ? settings.AllowBulkImport + : settings.AllowBulkExport; + var permission = direction == ContentTransferDirection.Import + ? ContentTransferPermissions.ImportContentFromFile + : ContentTransferPermissions.ExportContentFromFile; + + if (!isAllowed || !await _authorizationService.AuthorizeAsync(HttpContext.User, permission, (object)contentTypeDefinition.Name)) + { + continue; + } + + contentTypes.Add(new SelectListItem(contentTypeDefinition.DisplayName, contentTypeDefinition.Name)); + } + + return contentTypes; + } + + private async Task BuildListViewModelAsync(ListContentTransferEntryOptions options, PagerParameters pagerParameters) + { + var routeData = new RouteData(options.RouteValues); + var pager = new Pager(pagerParameters, _pagerOptions.GetPageSize()); + + var queryResult = await _entriesAdminListQueryService.QueryAsync(pager.Page, pager.PageSize, options, this); + var pagerShape = await _shapeFactory.PagerAsync(pager, queryResult.TotalCount, routeData); + + var summaries = new List(); + + foreach (var entry in queryResult.Entries) + { + dynamic shape = await _entryDisplayManager.BuildDisplayAsync(entry, this, "SummaryAdmin"); + shape.ContentTransferEntry = entry; + summaries.Add(shape); + } + + var startIndex = (pager.Page - 1) * pager.PageSize + 1; + options.StartIndex = queryResult.TotalCount == 0 ? 0 : startIndex; + options.EndIndex = queryResult.TotalCount == 0 ? 0 : startIndex + summaries.Count - 1; + options.EntriesCount = summaries.Count; + options.TotalItemCount = queryResult.TotalCount; + + var header = await _entryOptionsDisplayManager.BuildEditorAsync(options, this, false, string.Empty, string.Empty); + + return await _shapeFactory.CreateAsync("ContentTransferEntriesAdminList", viewModel => + { + viewModel.Options = options; + viewModel.Header = header; + viewModel.Entries = summaries; + viewModel.Pager = pagerShape; + }); + } + + private ContentExporterViewModel BuildContentExporterViewModel(IList exportableTypes) + => new() + { + ContentTypes = exportableTypes, + Extensions = + [ + new(S["Excel Workbook"], ".xlsx"), + ], + Extension = ".xlsx", + }; + + private async Task BuildBulkExportViewModelAsync(ListContentTransferEntryOptions options, PagerParameters pagerParameters) + { + return new BulkExportViewModel() + { + Exporter = BuildContentExporterViewModel(options.ExportableTypes), + List = await BuildListViewModelAsync(options, pagerParameters), + }; + } + + private async Task FilterListAsync(string actionName, ListContentTransferEntryOptions options) + { + if (!string.Equals(options.SearchText, options.OriginalSearchText, StringComparison.OrdinalIgnoreCase)) + { + return RedirectToAction(actionName, new RouteValueDictionary + { + { "q", options.SearchText }, + }); + } + + await _entryOptionsDisplayManager.UpdateEditorAsync(options, this, false, string.Empty, string.Empty); + options.RouteValues.TryAdd("q", options.FilterResult.ToString()); + + return RedirectToAction(actionName, options.RouteValues); + } + + private async Task ExecuteBulkActionAsync(IEnumerable itemIds, ContentTransferEntryBulkAction? bulkAction, ContentTransferDirection direction, string owner = null) + { + if (itemIds?.Any() != true) + { + return; + } + + var query = _session.Query(x => + x.EntryId.IsIn(itemIds) && x.Direction == direction); + + if (!string.IsNullOrWhiteSpace(owner)) + { + query = query.Where(x => x.Owner == owner); + } + + var entries = await query.ListAsync(); + + var deletedCount = 0; + var failedCount = 0; + + switch (bulkAction) + { + case ContentTransferEntryBulkAction.Remove: + foreach (var entry in entries) + { + if (await DeleteEntryAsync(entry)) + { + deletedCount++; + } + else + { + failedCount++; + } + } + + if (deletedCount > 0) + { + await _session.SaveChangesAsync(); + await _notifier.SuccessAsync(H["{0} {1} removed successfully.", deletedCount, H.Plural(deletedCount, "entry", "entries")]); + } + + if (failedCount > 0) + { + await _notifier.WarningAsync(H["{0} {1} could not be removed because the stored file could not be deleted.", failedCount, H.Plural(failedCount, "entry", "entries")]); + } + break; + } + } + + private async Task DeleteEntryAsync(ContentTransferEntry entry) + { + if (!string.IsNullOrWhiteSpace(entry.StoredFileName)) + { + var fileInfo = await _contentTransferFileStore.GetFileInfoAsync(entry.StoredFileName); + + if (fileInfo != null && !await _contentTransferFileStore.TryDeleteFileAsync(entry.StoredFileName)) + { + return false; + } + } + + _session.Delete(entry); + + return true; + } + + private static Task TriggerImportProcessingAsync(string entryId) + => HttpBackgroundJob.ExecuteAfterEndOfRequestAsync( + $"content-transfer-import-{entryId}", + entryId, + static (scope, id) => BackgroundTasks.ImportFilesBackgroundTask.ProcessEntriesAsync(scope.ServiceProvider, CancellationToken.None, id)); + + private static Task TriggerExportProcessingAsync(string entryId) + => HttpBackgroundJob.ExecuteAfterEndOfRequestAsync( + $"content-transfer-export-{entryId}", + entryId, + static (scope, id) => BackgroundTasks.ExportFilesBackgroundTask.ProcessEntriesAsync(scope.ServiceProvider, CancellationToken.None, id)); + + private static Row CloneRowWithErrorMessage(Row sourceRow, uint destinationRowIndex, string errorMessage) + { + var destinationRow = new Row() { RowIndex = destinationRowIndex }; + uint columnIndex = 1; + + foreach (var sourceCell in sourceRow.Elements()) + { + var clonedCell = (Cell)sourceCell.CloneNode(true); + clonedCell.CellReference = GetCellReference(columnIndex, destinationRowIndex); + destinationRow.Append(clonedCell); + columnIndex++; + } + + destinationRow.Append(new Cell() + { + CellReference = GetCellReference(columnIndex, destinationRowIndex), + DataType = CellValues.String, + CellValue = new CellValue(errorMessage ?? string.Empty), + }); + + return destinationRow; + } + + private static string GetCellReference(uint columnIndex, uint rowIndex) + { + var columnName = string.Empty; + var dividend = columnIndex; + + while (dividend > 0) + { + var modulo = (dividend - 1) % 26; + columnName = Convert.ToChar(65 + modulo) + columnName; + dividend = (dividend - modulo) / 26; + } + + return columnName + rowIndex; + } + + private IQuery BuildExportQuery( + string contentTypeId, + bool partialExport, + bool latestOnly, + bool allVersions, + DateTime? createdFrom, + DateTime? createdTo, + DateTime? modifiedFrom, + DateTime? modifiedTo, + string owners) + { + IQuery query; + + if (allVersions) + { + query = _session.Query(x => x.ContentType == contentTypeId); + } + else if (latestOnly) + { + query = _session.Query(x => x.ContentType == contentTypeId && x.Latest); + } + else + { + // Default: published only. + query = _session.Query(x => x.ContentType == contentTypeId && x.Published); + } + + if (partialExport) + { + if (createdFrom.HasValue) + { + query = query.Where(x => x.CreatedUtc >= createdFrom.Value); + } + + if (createdTo.HasValue) + { + query = query.Where(x => x.CreatedUtc <= createdTo.Value); + } + + if (modifiedFrom.HasValue) + { + query = query.Where(x => x.ModifiedUtc >= modifiedFrom.Value); + } + + if (modifiedTo.HasValue) + { + query = query.Where(x => x.ModifiedUtc <= modifiedTo.Value); + } + + if (!string.IsNullOrWhiteSpace(owners)) + { + var ownerList = owners.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + if (ownerList.Length == 1) + { + var owner = ownerList[0]; + query = query.Where(x => x.Owner == owner); + } + else if (ownerList.Length > 1) + { + // Capture into locals (safe for expression trees, supports up to 5 owners). + var o0 = ownerList[0]; + var o1 = ownerList.ElementAtOrDefault(1) ?? o0; + var o2 = ownerList.ElementAtOrDefault(2) ?? o0; + var o3 = ownerList.ElementAtOrDefault(3) ?? o0; + var o4 = ownerList.ElementAtOrDefault(4) ?? o0; + + query = query.Where(x => + x.Owner == o0 || x.Owner == o1 || x.Owner == o2 || + x.Owner == o3 || x.Owner == o4); + } + } + } + + return query.OrderBy(x => x.CreatedUtc); + } + + private string CurrentUserId() + => User.FindFirstValue(ClaimTypes.NameIdentifier); + + private IActionResult RedirectTo(string returnUrl) + { + return !string.IsNullOrEmpty(returnUrl) && Url.IsLocalUrl(returnUrl) + ? (IActionResult)this.LocalRedirect(returnUrl, true) + : RedirectToAction(nameof(List)); + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Drivers/ContentTransferEntryDisplayDriver.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Drivers/ContentTransferEntryDisplayDriver.cs new file mode 100644 index 00000000000..5499f78e7b4 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Drivers/ContentTransferEntryDisplayDriver.cs @@ -0,0 +1,18 @@ +using OrchardCore.ContentTransfer.ViewModels; +using OrchardCore.DisplayManagement.Handlers; +using OrchardCore.DisplayManagement.Views; + +namespace OrchardCore.ContentTransfer.Drivers; + +public sealed class ContentTransferEntryDisplayDriver : DisplayDriver +{ + public override Task DisplayAsync(ContentTransferEntry entry, BuildDisplayContext context) + { + return CombineAsync( + View("ContentTransferEntriesMeta_SummaryAdmin", new ContentTransferEntryViewModel(entry)).Location("SummaryAdmin", "Meta:20"), + View("ContentTransferEntriesActions_SummaryAdmin", new ContentTransferEntryViewModel(entry)).Location("SummaryAdmin", "Actions:5"), + View("ContentTransferEntriesButtonActions_SummaryAdmin", new ContentTransferEntryViewModel(entry)).Location("SummaryAdmin", "ActionsMenu:10"), + View("ContentTransferEntriesProgress_SummaryAdmin", new ContentTransferEntryViewModel(entry)).Location("SummaryAdmin", "Progress:5") + ); + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Drivers/ContentTypeTransferSettingsDisplayDriver.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Drivers/ContentTypeTransferSettingsDisplayDriver.cs new file mode 100644 index 00000000000..6c91dc6a69f --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Drivers/ContentTypeTransferSettingsDisplayDriver.cs @@ -0,0 +1,36 @@ +using OrchardCore.ContentManagement.Metadata.Models; +using OrchardCore.ContentTransfer.Models; +using OrchardCore.ContentTransfer.ViewModels; +using OrchardCore.ContentTypes.Editors; +using OrchardCore.DisplayManagement.Handlers; +using OrchardCore.DisplayManagement.Views; + +namespace OrchardCore.ContentTransfer.Drivers; + +public sealed class ContentTypeTransferSettingsDisplayDriver : ContentTypeDefinitionDisplayDriver +{ + public override IDisplayResult Edit(ContentTypeDefinition contentTypeDefinition, BuildEditorContext context) + { + return Initialize("ContentTypeTransferSettings_Edit", model => + { + var settings = contentTypeDefinition.GetSettings(); + model.AllowBulkImport = settings.AllowBulkImport; + model.AllowBulkExport = settings.AllowBulkExport; + }).Location("Content:5"); + } + + public override async Task UpdateAsync(ContentTypeDefinition contentTypeDefinition, UpdateTypeEditorContext context) + { + var model = new ContentTypeTransferSettingsViewModels(); + + await context.Updater.TryUpdateModelAsync(model, Prefix); + + context.Builder.WithSettings(new ContentTypeTransferSettings + { + AllowBulkImport = model.AllowBulkImport, + AllowBulkExport = model.AllowBulkExport, + }); + + return Edit(contentTypeDefinition, context); + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Drivers/ImportContentDisplayDriver.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Drivers/ImportContentDisplayDriver.cs new file mode 100644 index 00000000000..338423a635d --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Drivers/ImportContentDisplayDriver.cs @@ -0,0 +1,54 @@ +using Microsoft.Extensions.Localization; +using OrchardCore.ContentTransfer.Models; +using OrchardCore.ContentTransfer.ViewModels; +using OrchardCore.DisplayManagement.Handlers; +using OrchardCore.DisplayManagement.Views; +using OrchardCore.Mvc.ModelBinding; + +namespace OrchardCore.ContentTransfer.Drivers; + +public sealed class ImportContentDisplayDriver : DisplayDriver +{ + private readonly IStringLocalizer S; + + private static readonly HashSet _allowedExtensions = new(StringComparer.OrdinalIgnoreCase) + { + ".xlsx", + }; + + public ImportContentDisplayDriver(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + public override Task EditAsync(ImportContent model, BuildEditorContext context) + => Task.FromResult( + Initialize("ImportContentFile_Edit", viewModel => viewModel.File = model.File) + .Location("Content:1")); + + public override async Task UpdateAsync(ImportContent model, UpdateEditorContext context) + { + var viewModel = new ContentImportViewModel(); + + if (await context.Updater.TryUpdateModelAsync(viewModel, Prefix)) + { + if (viewModel.File?.Length == 0) + { + context.Updater.ModelState.AddModelError(Prefix, nameof(viewModel.File), S["File is required."]); + } + else + { + var extension = Path.GetExtension(viewModel.File.FileName); + + if (!_allowedExtensions.Contains(extension)) + { + context.Updater.ModelState.AddModelError(Prefix, nameof(viewModel.File), S["Only .xlsx files are supported."]); + } + + model.File = viewModel.File; + } + } + + return await EditAsync(model, context); + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Drivers/ListContentTransferEntryOptionsDisplayDriver.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Drivers/ListContentTransferEntryOptionsDisplayDriver.cs new file mode 100644 index 00000000000..8548f7ab5fa --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Drivers/ListContentTransferEntryOptionsDisplayDriver.cs @@ -0,0 +1,78 @@ +using OrchardCore.ContentTransfer.Models; +using OrchardCore.DisplayManagement.Handlers; +using OrchardCore.DisplayManagement.Views; + +namespace OrchardCore.ContentTransfer.Drivers; + +public sealed class ListContentTransferEntryOptionsDisplayDriver : DisplayDriver +{ + // Maintain the Options prefix for compatibility with binding. + protected override void BuildPrefix(ListContentTransferEntryOptions model, string htmlFieldPrefix) + { + Prefix = "Options"; + } + + public override Task DisplayAsync(ListContentTransferEntryOptions model, BuildDisplayContext context) + { + return CombineAsync( + Initialize("ListContentTransferEntriesAdminListBulkActions", m => BuildOptionsViewModel(m, model)) + .Location("BulkActions", "Content:10"), + View("ListContentTransferEntriesAdminFilters_Thumbnail__Status", model) + .Location("Thumbnail", "Content:30"), + View("ListContentTransferEntriesAdminFilters_Thumbnail__Sort", model) + .Location("Thumbnail", "Content:40") + ); + } + + public override Task EditAsync(ListContentTransferEntryOptions model, BuildEditorContext context) + { + model.FilterResult.MapTo(model); + + return CombineAsync( + Initialize("ListContentTransferEntriesAdminListBulkActions", m => BuildOptionsViewModel(m, model)) + .Location("BulkActions", "Content:10"), + Initialize("ListContentTransferEntriesAdminListSearch", m => BuildOptionsViewModel(m, model)) + .Location("Search:10"), + model.Direction == ContentTransferDirection.Export + ? Initialize("ContentTransferEntriesAdminListExport", m => BuildOptionsViewModel(m, model)) + .Location("Create:10") + : Initialize("ContentTransferEntriesAdminListImport", m => BuildOptionsViewModel(m, model)) + .Location("Create:10"), + Initialize("ListContentTransferEntriesAdminListActionBarButtons", m => BuildOptionsViewModel(m, model)) + .Location("ActionBarButtons:10"), + Initialize("ListContentTransferEntriesAdminListSummary", m => BuildOptionsViewModel(m, model)) + .Location("Summary:10"), + Initialize("ListContentTransferEntriesAdminListFilters", m => BuildOptionsViewModel(m, model)) + .Location("Actions:10.1"), + Initialize("ListContentTransferEntriesAdminList_Fields_BulkActions", m => BuildOptionsViewModel(m, model)) + .Location("Actions:10.1") + ); + } + + public override Task UpdateAsync(ListContentTransferEntryOptions model, UpdateEditorContext context) + { + // Map the incoming values from a form post to the filter result. + model.FilterResult.MapFrom(model); + + return EditAsync(model, context); + } + + private static void BuildOptionsViewModel(ListContentTransferEntryOptions m, ListContentTransferEntryOptions model) + { + m.Status = model.Status; + m.SearchText = model.SearchText; + m.OriginalSearchText = model.OriginalSearchText; + m.FilterResult = model.FilterResult; + m.Sorts = model.Sorts; + m.Statuses = model.Statuses; + m.BulkActions = model.BulkActions; + m.ImportableTypes = model.ImportableTypes; + m.ExportableTypes = model.ExportableTypes; + m.StartIndex = model.StartIndex; + m.EndIndex = model.EndIndex; + m.TotalItemCount = model.TotalItemCount; + m.OrderBy = model.OrderBy; + m.Direction = model.Direction; + m.Owner = model.Owner; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Manifest.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Manifest.cs new file mode 100644 index 00000000000..3056577575b --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Manifest.cs @@ -0,0 +1,10 @@ +using OrchardCore.Modules.Manifest; + +[assembly: Module( + Name = "Content Transfer", + Author = ManifestConstants.OrchardCoreTeam, + Website = ManifestConstants.OrchardCoreWebsite, + Version = ManifestConstants.OrchardCoreVersion, + Description = "Provides a way to import and export content from and to Excel files.", + Category = "Content Management" +)] diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Migrations/ContentTransferMigrations.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Migrations/ContentTransferMigrations.cs new file mode 100644 index 00000000000..5a40fc12cfd --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Migrations/ContentTransferMigrations.cs @@ -0,0 +1,57 @@ +using OrchardCore.ContentTransfer.Indexes; +using OrchardCore.Data.Migration; +using YesSql.Sql; + +namespace OrchardCore.ContentTransfer.Migrations; + +public sealed class ContentTransferMigrations : DataMigration +{ + public async Task CreateAsync() + { + await SchemaBuilder.CreateMapIndexTableAsync(table => table + .Column("EntryId", column => column.WithLength(26)) + .Column("Status", column => column.NotNull().WithLength(25)) + .Column("CreatedUtc", column => column.NotNull()) + .Column("ContentType", column => column.WithLength(255)) + .Column("Owner", column => column.WithLength(26)) + ); + + await SchemaBuilder.AlterIndexTableAsync(table => table + .CreateIndex("IDX_ContentTransferEntryIndex_DocumentId", + "DocumentId", + "EntryId", + "Status", + "CreatedUtc", + "ContentType", + "Owner") + ); + + await SchemaBuilder.AlterIndexTableAsync(table => table + .CreateIndex("IDX_ContentTransferEntryIndex_Status", + "Status", + "CreatedUtc", + "DocumentId", + "ContentType", + "Owner") + ); + + return 1; + } + + public async Task UpdateFrom1Async() + { + await SchemaBuilder.AlterIndexTableAsync(table => table + .AddColumn("Direction", column => column.NotNull().WithDefault(0)) + ); + + await SchemaBuilder.AlterIndexTableAsync(table => table + .CreateIndex("IDX_ContentTransferEntryIndex_Direction", + "Direction", + "Status", + "CreatedUtc", + "DocumentId") + ); + + return 2; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Models/ExportFilterPart.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Models/ExportFilterPart.cs new file mode 100644 index 00000000000..16c72f7fa94 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Models/ExportFilterPart.cs @@ -0,0 +1,24 @@ +namespace OrchardCore.ContentTransfer.Models; + +/// +/// Stores export filter criteria on a for queued exports. +/// This allows the background task to apply the same filters the user selected. +/// +public sealed class ExportFilterPart +{ + public bool PublishedOnly { get; set; } = true; + + public bool LatestOnly { get; set; } + + public bool AllVersions { get; set; } + + public DateTime? CreatedFrom { get; set; } + + public DateTime? CreatedTo { get; set; } + + public DateTime? ModifiedFrom { get; set; } + + public DateTime? ModifiedTo { get; set; } + + public string Owners { get; set; } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Models/ImportFileProcessStatsPart.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Models/ImportFileProcessStatsPart.cs new file mode 100644 index 00000000000..f6559750bb3 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Models/ImportFileProcessStatsPart.cs @@ -0,0 +1,37 @@ +using OrchardCore.ContentManagement; + +namespace OrchardCore.ContentTransfer.Models; + +public sealed class ImportFileProcessStatsPart : ContentPart +{ + /// + /// Current row being processed. + /// + public int CurrentRow { get; set; } + + /// + /// Total processes records. + /// + + public int TotalProcessed { get; set; } + + /// + /// Total successfully imported records. + /// + public int ImportedCount { get; set; } + + /// + /// The index of each row that failed validation. + /// + public HashSet Errors { get; set; } + + /// + /// The error message for each failed row. + /// + public Dictionary ErrorMessages { get; set; } + + /// + /// Total records available if the file. + /// + public int TotalRecords { get; set; } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/OrchardCore.ContentTransfer.csproj b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/OrchardCore.ContentTransfer.csproj new file mode 100644 index 00000000000..87145767d2a --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/OrchardCore.ContentTransfer.csproj @@ -0,0 +1,29 @@ + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/PermissionsProvider.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/PermissionsProvider.cs new file mode 100644 index 00000000000..9d07021b829 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/PermissionsProvider.cs @@ -0,0 +1,48 @@ +using OrchardCore.ContentManagement.Metadata; +using OrchardCore.Contents.Security; +using OrchardCore.Security.Permissions; + +namespace OrchardCore.ContentTransfer; + +public sealed class PermissionsProvider : IPermissionProvider +{ + private readonly IContentDefinitionManager _contentDefinitionManager; + + private readonly IEnumerable _allPermissions = + [ + ContentTransferPermissions.ListContentTransferEntries, + ContentTransferPermissions.DeleteContentTransferEntries, + ContentTransferPermissions.ImportContentFromFile, + ContentTransferPermissions.ExportContentFromFile, + ]; + + public PermissionsProvider(IContentDefinitionManager contentDefinitionManager) + { + _contentDefinitionManager = contentDefinitionManager; + } + + public async Task> GetPermissionsAsync() + { + var permissions = new List(_allPermissions); + + foreach (var contentTypeDefinition in await _contentDefinitionManager.LoadTypeDefinitionsAsync()) + { + permissions.Add(ContentTypePermissionsHelper.CreateDynamicPermission(ContentTransferPermissions.ImportContentFromFileOfType, contentTypeDefinition.Name)); + permissions.Add(ContentTypePermissionsHelper.CreateDynamicPermission(ContentTransferPermissions.ExportContentFromFileOfType, contentTypeDefinition.Name)); + } + + return permissions; + } + + public IEnumerable GetDefaultStereotypes() + { + return new[] + { + new PermissionStereotype + { + Name = OrchardCoreConstants.Roles.Administrator, + Permissions = _allPermissions, + }, + }; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Services/ContentTransferNotificationHandler.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Services/ContentTransferNotificationHandler.cs new file mode 100644 index 00000000000..99bceb486ea --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Services/ContentTransferNotificationHandler.cs @@ -0,0 +1,48 @@ +using Microsoft.Extensions.Localization; +using OrchardCore.Notifications; +using OrchardCore.Notifications.Models; +using OrchardCore.Users.Services; + +namespace OrchardCore.ContentTransfer.Services; + +public sealed class ContentTransferNotificationHandler : IContentTransferNotificationHandler +{ + private readonly INotificationService _notificationService; + private readonly IUserService _userService; + + internal readonly IStringLocalizer S; + + public ContentTransferNotificationHandler( + INotificationService notificationService, + IUserService userService, + IStringLocalizer stringLocalizer) + { + _notificationService = notificationService; + _userService = userService; + S = stringLocalizer; + } + + public async Task NotifyExportCompletedAsync(ContentTransferEntry entry, string contentTypeName) + { + if (string.IsNullOrEmpty(entry.Owner)) + { + return; + } + + var user = await _userService.GetUserByUniqueIdAsync(entry.Owner); + + if (user == null) + { + return; + } + + var message = new NotificationMessage() + { + Subject = S["Content Export Completed"], + Summary = S["Your export of '{0}' content items is ready for download.", contentTypeName], + TextBody = S["The export file '{0}' for content type '{1}' has been completed and is ready for download from Bulk Export.", entry.UploadedFileName, contentTypeName], + }; + + await _notificationService.SendAsync(user, message); + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Startup.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Startup.cs new file mode 100644 index 00000000000..d0ba1341343 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Startup.cs @@ -0,0 +1,222 @@ +using Microsoft.AspNetCore.Authorization; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using OrchardCore.Alias.Models; +using OrchardCore.ArchiveLater.Models; +using OrchardCore.Autoroute.Models; +using OrchardCore.BackgroundTasks; +using OrchardCore.ContentFields.Fields; +using OrchardCore.ContentTransfer.BackgroundTasks; +using OrchardCore.ContentTransfer.Drivers; +using OrchardCore.ContentTransfer.Handlers; +using OrchardCore.ContentTransfer.Handlers.Fields; +using OrchardCore.ContentTransfer.Indexes; +using OrchardCore.ContentTransfer.Migrations; +using OrchardCore.ContentTransfer.Models; +using OrchardCore.ContentTransfer.Services; +using OrchardCore.ContentTypes.Editors; +using OrchardCore.Data; +using OrchardCore.Data.Migration; +using OrchardCore.DisplayManagement.Handlers; +using OrchardCore.Environment.Shell; +using OrchardCore.Environment.Shell.Configuration; +using OrchardCore.FileStorage.FileSystem; +using OrchardCore.Html.Models; +using OrchardCore.Liquid.Models; +using OrchardCore.Markdown.Fields; +using OrchardCore.Markdown.Models; +using OrchardCore.Media.Fields; +using OrchardCore.Modules; +using OrchardCore.Navigation; +using OrchardCore.PublishLater.Models; +using OrchardCore.Security.Permissions; +using OrchardCore.Taxonomies.Fields; +using OrchardCore.Title.Models; +using YesSql.Filters.Query; + +namespace OrchardCore.ContentTransfer; + +public sealed class Startup : StartupBase +{ + private readonly IShellConfiguration _configuration; + + public Startup(IShellConfiguration configuration) + { + _configuration = configuration; + } + + public override void ConfigureServices(IServiceCollection services) + { + services.AddSingleton(serviceProvider => + { + var shellSettings = serviceProvider.GetRequiredService(); + var logger = serviceProvider.GetRequiredService>(); + var path = Path.Combine(ShellOptionConstants.DefaultAppDataPath, ShellOptionConstants.DefaultSitesPath, shellSettings.Name, "Temp"); + var fileStore = new FileSystemStore(path, logger); + + return new ContentTransferFileStore(fileStore); + }); + + services.AddDataMigration(); + services.AddIndexProvider(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped, ImportContentDisplayDriver>(); + + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.Configure(_configuration.GetSection("OrchardCore_ContentsTransfer")); + services.AddSingleton(); + services.AddSingleton(); + + services.AddScoped(); + services.AddScoped, ListContentTransferEntryOptionsDisplayDriver>(); + services.AddScoped, ContentTransferEntryDisplayDriver>(); + services.AddTransient(); + services.AddSingleton(sp => + { + var filterProviders = sp.GetServices(); + var builder = new QueryEngineBuilder(); + foreach (var provider in filterProviders) + { + provider.Build(builder); + } + + var parser = builder.Build(); + + return new DefaultContentTypeEntryAdminListFilterParser(parser); + }); + } +} + +[RequireFeatures("OrchardCore.Title")] +public sealed class TitleStartup : StartupBase +{ + public override void ConfigureServices(IServiceCollection services) + { + services.AddContentPartImportHandler(); + } +} + +[RequireFeatures("OrchardCore.Html")] +public sealed class HtmlBodyStartup : StartupBase +{ + public override void ConfigureServices(IServiceCollection services) + { + services.AddContentPartImportHandler(); + } +} + +[RequireFeatures("OrchardCore.Markdown")] +public sealed class MarkdownStartup : StartupBase +{ + public override void ConfigureServices(IServiceCollection services) + { + services.AddContentPartImportHandler(); + services.AddContentFieldImportHandler(); + } +} + +[RequireFeatures("OrchardCore.Alias")] +public sealed class AliasStartup : StartupBase +{ + public override void ConfigureServices(IServiceCollection services) + { + services.AddContentPartImportHandler(); + } +} + +[RequireFeatures("OrchardCore.ArchiveLater")] +public sealed class ArchiveLaterStartup : StartupBase +{ + public override void ConfigureServices(IServiceCollection services) + { + services.AddContentPartImportHandler(); + } +} + +[RequireFeatures("OrchardCore.Autoroute")] +public sealed class AutorouteStartup : StartupBase +{ + public override void ConfigureServices(IServiceCollection services) + { + services.AddContentPartImportHandler(); + } +} + +[RequireFeatures("OrchardCore.ContentFields")] +public sealed class ContentFieldsStartup : StartupBase +{ + public override void ConfigureServices(IServiceCollection services) + { + services.AddContentFieldImportHandler(); + services.AddContentFieldImportHandler(); + services.AddContentFieldImportHandler(); + services.AddContentFieldImportHandler(); + services.AddContentFieldImportHandler(); + services.AddContentFieldImportHandler(); + services.AddContentFieldImportHandler(); + services.AddContentFieldImportHandler(); + services.AddContentFieldImportHandler(); + services.AddContentFieldImportHandler(); + services.AddContentFieldImportHandler(); + services.AddContentFieldImportHandler(); + } +} + +[RequireFeatures("OrchardCore.Liquid")] +public sealed class LiquidStartup : StartupBase +{ + public override void ConfigureServices(IServiceCollection services) + { + services.AddContentPartImportHandler(); + } +} + +[RequireFeatures("OrchardCore.Media")] +public sealed class MediaStartup : StartupBase +{ + public override void ConfigureServices(IServiceCollection services) + { + services.AddContentFieldImportHandler(); + } +} + +[RequireFeatures("OrchardCore.PublishLater")] +public sealed class PublishLaterStartup : StartupBase +{ + public override void ConfigureServices(IServiceCollection services) + { + services.AddContentPartImportHandler(); + } +} + +[RequireFeatures("OrchardCore.Taxonomies")] +public sealed class TaxonomiesStartup : StartupBase +{ + public override void ConfigureServices(IServiceCollection services) + { + services.AddContentFieldImportHandler(); + } +} + +[RequireFeatures("OrchardCore.ContentLocalization")] +public sealed class ContentLocalizationStartup : StartupBase +{ + public override void ConfigureServices(IServiceCollection services) + { + services.AddContentFieldImportHandler(); + } +} + +[RequireFeatures("OrchardCore.Notifications")] +public sealed class NotificationsStartup : StartupBase +{ + public override void ConfigureServices(IServiceCollection services) + { + services.AddScoped(); + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/BulkExportViewModel.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/BulkExportViewModel.cs new file mode 100644 index 00000000000..19fb2e0d890 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/BulkExportViewModel.cs @@ -0,0 +1,10 @@ +using OrchardCore.DisplayManagement; + +namespace OrchardCore.ContentTransfer.ViewModels; + +public class BulkExportViewModel +{ + public ContentExporterViewModel Exporter { get; set; } + + public IShape List { get; set; } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/ContentExporterViewModel.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/ContentExporterViewModel.cs new file mode 100644 index 00000000000..7225d807364 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/ContentExporterViewModel.cs @@ -0,0 +1,42 @@ +using System.ComponentModel.DataAnnotations; +using Microsoft.AspNetCore.Mvc.ModelBinding; +using Microsoft.AspNetCore.Mvc.Rendering; +using OrchardCore.DisplayManagement; + +namespace OrchardCore.ContentTransfer.ViewModels; + +public class ContentExporterViewModel +{ + [Required] + public string Extension { get; set; } + + [Required] + public string ContentTypeId { get; set; } + + public bool PartialExport { get; set; } + + public DateTime? CreatedFrom { get; set; } + + public DateTime? CreatedTo { get; set; } + + public DateTime? ModifiedFrom { get; set; } + + public DateTime? ModifiedTo { get; set; } + + public string Owners { get; set; } + + public bool PublishedOnly { get; set; } = true; + + public bool LatestOnly { get; set; } + + public bool AllVersions { get; set; } + + [BindNever] + public IList ContentTypes { get; set; } + + [BindNever] + public IList Extensions { get; set; } + + [BindNever] + public IShape Content { get; set; } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/ContentImportViewModel.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/ContentImportViewModel.cs new file mode 100644 index 00000000000..72f2a9f64bf --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/ContentImportViewModel.cs @@ -0,0 +1,11 @@ +using System.ComponentModel.DataAnnotations; +using Microsoft.AspNetCore.Http; + +namespace OrchardCore.ContentTransfer.ViewModels; + +public class ContentImportViewModel +{ + [Required] + [DataType(DataType.Upload)] + public IFormFile File { get; set; } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/ContentImporterViewModel.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/ContentImporterViewModel.cs new file mode 100644 index 00000000000..70b57e4565c --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/ContentImporterViewModel.cs @@ -0,0 +1,13 @@ +using OrchardCore.ContentManagement.Metadata.Models; +using OrchardCore.DisplayManagement; + +namespace OrchardCore.ContentTransfer.ViewModels; + +public class ContentImporterViewModel +{ + public ContentTypeDefinition ContentTypeDefinition { get; set; } + + public IShape Content { get; set; } + + public IEnumerable Columns { get; set; } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/ContentTransferEntryViewModel.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/ContentTransferEntryViewModel.cs new file mode 100644 index 00000000000..2b342d2a54b --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/ContentTransferEntryViewModel.cs @@ -0,0 +1,17 @@ +using OrchardCore.DisplayManagement.Views; + +namespace OrchardCore.ContentTransfer.ViewModels; + +public class ContentTransferEntryViewModel : ShapeViewModel +{ + public ContentTransferEntry ContentTransferEntry { get; set; } + + public ContentTransferEntryViewModel() + { + } + + public ContentTransferEntryViewModel(ContentTransferEntry entry) + { + ContentTransferEntry = entry; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/ContentTypeTransferSettingsViewModels.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/ContentTypeTransferSettingsViewModels.cs new file mode 100644 index 00000000000..8edfabfbad6 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/ContentTypeTransferSettingsViewModels.cs @@ -0,0 +1,8 @@ +namespace OrchardCore.ContentTransfer.ViewModels; + +public class ContentTypeTransferSettingsViewModels +{ + public bool AllowBulkImport { get; set; } + + public bool AllowBulkExport { get; set; } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/ListContentTransferEntriesViewModel.cs b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/ListContentTransferEntriesViewModel.cs new file mode 100644 index 00000000000..297c4f18a5a --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/ViewModels/ListContentTransferEntriesViewModel.cs @@ -0,0 +1,18 @@ +using Microsoft.AspNetCore.Mvc.ModelBinding; +using OrchardCore.ContentTransfer.Models; + +namespace OrchardCore.ContentTransfer.ViewModels; + +public class ListContentTransferEntriesViewModel +{ + public ListContentTransferEntryOptions Options { get; set; } + + [BindNever] + public IList Entries { get; set; } + + [BindNever] + public dynamic Header { get; set; } + + [BindNever] + public dynamic Pager { get; set; } +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/Admin/Export.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/Admin/Export.cshtml new file mode 100644 index 00000000000..226276882d6 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/Admin/Export.cshtml @@ -0,0 +1,191 @@ +@using OrchardCore +@using OrchardCore.ContentTransfer + +@model BulkExportViewModel + +

@RenderTitleSegments(T["Bulk Export"])

 + +
+
+
@T["Export contents"]
+
+ @if (Model.Exporter.ContentTypes.Count == 0) + { +
+ @T["No content types are configured for bulk export. Enable bulk export in the content type settings."] +
+ } + else + { +
+
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+ +
+
+ + +
+
+ + +
+
+
+ +
+
+
@T["Filter Options"]
+ +
+ +
+ +
+
+
+ +
+ +
+
+ +
+ +
+ +
+
+
+ +
+ +
+
+ +
+ +
+ +
+
+ +
+
+
+ + +
+
+
+ +
+
+
+ + +
+
+
+ +
+
+
+ + +
+
+
+
+ +
+
+ +
+
+
+ } +
+
+
+ +
+ @await DisplayAsync(Model.List) +
+ +@if (Model.Exporter.ContentTypes.Count > 0) +{ + +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/Admin/ExportDashboard.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/Admin/ExportDashboard.cshtml new file mode 100644 index 00000000000..2a4ecf85c74 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/Admin/ExportDashboard.cshtml @@ -0,0 +1,61 @@ +@model ListContentTransferEntriesViewModel + +

@RenderTitleSegments(T["Export Dashboard"])

 + +
+
+
+
+
+ + +
+
+
+
+
+ +
    + @if (Model.Entries.Count > 0) + { + @foreach (var entry in Model.Entries) + { +
  • + @await DisplayAsync(entry) +
    • + } + } + else + { +
  • +
    + @T["No export requests found."] +
    +
    • + } +
+ +@await DisplayAsync(Model.Pager) + + diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/Admin/Import.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/Admin/Import.cshtml new file mode 100644 index 00000000000..d29de14d72d --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/Admin/Import.cshtml @@ -0,0 +1,93 @@ +@using Microsoft.Extensions.Options +@using OrchardCore.ContentTransfer + +@model ContentImporterViewModel + +@inject IOptions ContentImportOptions + + + +@if (!ViewContext.ModelState.IsValid) +{ + @Html.ValidationSummary(false, string.Empty, new { @class = "text-danger" }) +} + +
+
@T["Import {0} contents", Model.ContentTypeDefinition.DisplayName]
+
+
+ + @await DisplayAsync(Model.Content) +
+ + +
+
+
+
+ +
+
@T["File Requirements"]
+ +
+ +
    +
  • @T["The uploaded excel file must be in .xls, .xlsx or .csv format."]
    • + @if (ContentImportOptions.Value.MaxAllowedFileSizeInBytes > 0) + { +
  • @T["The uploaded excel file must be less than or equal to {0}MB in size.", ContentImportOptions.Value.GetMaxAllowedSizeInMb()]
    • + } +
  • @T["The uploaded excel file must contains a single tab and all formats should be cleared."]
    • +
  • @T["The very first row of the uploaded file must be the column names. The column names and their requirements are defined below."]
    • +
  • @T["The order of the column in the file is not important as long as it exists. Any column not defined below will be ignored."]
    • +
  • @T["Click 'Download Template' link to download a sample template of a file."] + + @T["Download Template"] + +
    • +
+ + + + + + + + + + + + + + + @foreach (var column in Model.Columns) + { + + + + + + } + + +
@T["Columns Requirements"]
@T["Column Name as it must appear on the first row on the file"]@T["Required?"]@T["Description"]
@column.Name@(column.IsRequired ? T["Yes"] : T["No"]) +

@column.Description

+ @if (column.ValidValues != null && column.ValidValues.Length > 0) + { +

+ @T["Valid values"] + + @foreach (var validValue in column.ValidValues) + { + @Html.Raw("\"" + validValue + "\" ") + } +

+ } +
+
+
diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/Admin/List.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/Admin/List.cshtml new file mode 100644 index 00000000000..3b54fd964be --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/Admin/List.cshtml @@ -0,0 +1,8 @@ +@using OrchardCore.DisplayManagement +@model IShape + +

@RenderTitleSegments(T["Bulk Import"])

 + +
+ @await DisplayAsync(Model) +
diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesActions_SummaryAdmin.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesActions_SummaryAdmin.cshtml new file mode 100644 index 00000000000..ea3257cc335 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesActions_SummaryAdmin.cshtml @@ -0,0 +1,14 @@ +@model OrchardCore.DisplayManagement.Views.ShapeViewModel +@using OrchardCore.ContentTransfer +@using OrchardCore.Entities + +@{ + var entry = Model.Value.ContentTransferEntry; +} + +@if (entry.Direction == ContentTransferDirection.Export && entry.Status == ContentTransferEntryStatus.Completed) +{ + + @T["Download"] + +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminList.Fields.BulkActions.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminList.Fields.BulkActions.cshtml new file mode 100644 index 00000000000..1b6697c0d4f --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminList.Fields.BulkActions.cshtml @@ -0,0 +1,16 @@ +@using OrchardCore.DisplayManagement +@model ListContentTransferEntryOptions + +@inject IDisplayManager ContentOptionsDisplayManager + + diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminList.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminList.cshtml new file mode 100644 index 00000000000..ed13175cb04 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminList.cshtml @@ -0,0 +1,132 @@ +@model ListContentTransferEntriesViewModel + + + + +@Html.HiddenFor(o => o.Options.BulkAction) + +
+
+
+ @if (Model.Header?.Search != null) + { +
+ @await DisplayAsync(Model.Header.Search) +
+ } + + @if (Model.Header?.Create != null) + { +
+ @await DisplayAsync(Model.Header.Create) +
+ } +
+
+
+ +
    + + @if (Model.Header?.Summary != null || Model.Header?.Actions != null) + { +
  • +
    + @if (Model.Header.Summary != null) + { +
    + @await DisplayAsync(Model.Header.Summary) +
    + } + @if (Model.Header.Actions != null) + { +
    + @await DisplayAsync(Model.Header.Actions) +
    + } +
    +
    • + } + @if (Model.Entries.Count > 0) + { + foreach (var entry in Model.Entries) + { +
  • + @await DisplayAsync(entry) +
    • + } + } + else + { +
  • +
    + @T["No items found."] +
    +
    • + } +
+ +@await DisplayAsync(Model.Pager) + + + diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminListActionBarButtons.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminListActionBarButtons.cshtml new file mode 100644 index 00000000000..6bdae199092 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminListActionBarButtons.cshtml @@ -0,0 +1,9 @@ +@model ListContentTransferEntryOptions +@using OrchardCore.ContentTransfer + +@if (Model.Direction == ContentTransferDirection.Export && Model.ExportableTypes?.Count > 0) +{ + +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminListExport.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminListExport.cshtml new file mode 100644 index 00000000000..0f59aeb6ebf --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminListExport.cshtml @@ -0,0 +1,8 @@ +@model ListContentTransferEntryOptions + +@if (Model.ExportableTypes?.Count > 0) +{ + +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminListFilters.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminListFilters.cshtml new file mode 100644 index 00000000000..dd1b718b9ab --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminListFilters.cshtml @@ -0,0 +1,8 @@ +@model ListContentTransferEntryOptions + +
+ + +
diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminListImport.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminListImport.cshtml new file mode 100644 index 00000000000..c46b6d95cf0 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminListImport.cshtml @@ -0,0 +1,32 @@ +@using OrchardCore.ContentTransfer +@model ListContentTransferEntryOptions + +@if (Model.ImportableTypes?.Count == 0) +{ + return; +} + +@if (Model.ImportableTypes.Count == 1) +{ + + @T["Import {0}", Model.ImportableTypes.First().Text] + +} +else +{ +
+ + +
+} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminListSearch.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminListSearch.cshtml new file mode 100644 index 00000000000..e61eae4cec7 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminListSearch.cshtml @@ -0,0 +1,8 @@ +@model ListContentTransferEntryOptions + + +
+ + + +
diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminListSummary.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminListSummary.cshtml new file mode 100644 index 00000000000..11f9579f7b1 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesAdminListSummary.cshtml @@ -0,0 +1,12 @@ +@using Microsoft.AspNetCore.Mvc.Localization + +@model ListContentTransferEntryOptions + +
+
+ + + + +
+
diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesButtonActions_SummaryAdmin.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesButtonActions_SummaryAdmin.cshtml new file mode 100644 index 00000000000..9d1cd7acde8 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesButtonActions_SummaryAdmin.cshtml @@ -0,0 +1,39 @@ +@using Microsoft.AspNetCore.Authorization +@using OrchardCore.ContentTransfer +@using OrchardCore.Entities + +@inject IAuthorizationService AuthorizationService + +@model OrchardCore.DisplayManagement.Views.ShapeViewModel + +@{ + var entry = Model.Value.ContentTransferEntry; + var statsPart = entry.As(); + var canDelete = await AuthorizationService.AuthorizeAsync(User, ContentTransferPermissions.DeleteContentTransferEntries); + var canProcessImport = await AuthorizationService.AuthorizeAsync(User, ContentTransferPermissions.ImportContentFromFile, (object)entry.ContentType); +} + +@if (entry.Direction == ContentTransferDirection.Import && statsPart?.Errors?.Count > 0) +{ + + @T["Download errors ({0})", statsPart.Errors.Count] + +} + +@if (entry.Direction == ContentTransferDirection.Import + && canProcessImport + && (entry.Status == ContentTransferEntryStatus.New || entry.Status == ContentTransferEntryStatus.Processing)) +{ + + @T["Process now"] + + + + @T["Cancel import"] + +} + +@if (canDelete) +{ + @T["Delete"] +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesMeta_SummaryAdmin.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesMeta_SummaryAdmin.cshtml new file mode 100644 index 00000000000..39fff337f35 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesMeta_SummaryAdmin.cshtml @@ -0,0 +1,94 @@ +@model OrchardCore.DisplayManagement.Views.ShapeViewModel + +@using OrchardCore.ContentManagement +@using System.Globalization +@using OrchardCore.Entities + +@{ + var entry = Model.Value.ContentTransferEntry; + var statsPart = entry.As(); + var errorCount = statsPart?.Errors?.Count ?? 0; + var successCount = statsPart?.ImportedCount ?? 0; +} + + + @entry.ContentType + + + + + + + +@{ + var statusClasses = entry.Status switch + { + ContentTransferEntryStatus.New => "text-secondary-emphasis", + ContentTransferEntryStatus.Processing => "text-primary", + ContentTransferEntryStatus.Completed => "text-success", + ContentTransferEntryStatus.CompletedWithErrors => "text-warning-emphasis", + ContentTransferEntryStatus.Canceled => "text-secondary", + ContentTransferEntryStatus.CanceledWithImportedRecords => "text-warning", + ContentTransferEntryStatus.Failed => "text-danger", + _ => "text-body", + }; +} + + + + @switch (entry.Status) + { + case ContentTransferEntryStatus.New: + @T["New"] + break; + case ContentTransferEntryStatus.Processing: + @T["Processing"] + break; + case ContentTransferEntryStatus.Completed: + @T["Completed"] + break; + case ContentTransferEntryStatus.CompletedWithErrors: + @T["Completed with errors"] + break; + case ContentTransferEntryStatus.Canceled: + @T["Canceled"] + break; + case ContentTransferEntryStatus.CanceledWithImportedRecords: + @T["Canceled with imported records"] + break; + case ContentTransferEntryStatus.Failed: + @T["Failed"] + break; + } + + +@if (!string.IsNullOrEmpty(entry.Author)) +{ + +} + +@if (statsPart?.TotalProcessed > 0) +{ + + @successCount + + + @if (errorCount > 0) + { + + @errorCount + + } +} + +@if (entry.ProcessSaveUtc.HasValue) +{ + + + + +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesProgress_SummaryAdmin.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesProgress_SummaryAdmin.cshtml new file mode 100644 index 00000000000..6a10bab41b6 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntriesProgress_SummaryAdmin.cshtml @@ -0,0 +1,31 @@ +@using OrchardCore.ContentTransfer +@using OrchardCore.ContentTransfer.Models +@using OrchardCore.Entities + +@model OrchardCore.DisplayManagement.Views.ShapeViewModel + +@{ + var entry = Model.Value.ContentTransferEntry; + var part = entry.As(); +} + +@if (entry.Status == ContentTransferEntryStatus.Processing && part?.TotalRecords > 0) +{ + var percent = (int)((double)part.TotalProcessed / part.TotalRecords * 100); + var errorCount = part.Errors?.Count ?? 0; + +
+
+
+
@(percent)%
+
+ + @T["{0} of {1} records processed", part.TotalProcessed, part.TotalRecords] + @if (errorCount > 0) + { + (@T["{0} errors", errorCount]) + } + +
+
+} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntry.SummaryAdmin.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntry.SummaryAdmin.cshtml new file mode 100644 index 00000000000..1bbf67d9756 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTransferEntry.SummaryAdmin.cshtml @@ -0,0 +1,95 @@ +@using Microsoft.AspNetCore.Authorization +@using OrchardCore.ContentTransfer +@using OrchardCore.Contents +@using OrchardCore.ContentManagement +@using Microsoft.AspNetCore.Html +@using OrchardCore.Entities + +@inject IAuthorizationService AuthorizationService + +@{ + ContentTransferEntry entry = Model.ContentTransferEntry; +} + +
+
+
+ @if (Model.Selectors != null) + { +
+ @await DisplayAsync(Model.Selectors) +
+ } +
+ + +
+
+
+
+

+ @entry.UploadedFileName +

+
+ + @if (Model.Header != null) + { +
+ @await DisplayAsync(Model.Header) +
+ } + @if (Model.Tags != null) + { +
+ @await DisplayAsync(Model.Tags) +
+ } + + @if (Model.Meta != null) + { +
+ @await DisplayAsync(Model.Meta) +
+ } +
+ + @if (Model.Progress != null) + { +
+
+ @await DisplayAsync(Model.Progress) +
+
+ } + +
+
+
+ @if (Model.Actions != null) + { + @await DisplayAsync(Model.Actions) + } + + @if (Model.ActionsMenu != null && Model.ActionsMenu.HasItems) + { +
+ +
+ @await DisplayAsync(Model.ActionsMenu) +
+
+ } +
+
+
+ +@if (Model.Content != null) +{ +
+
+ @await DisplayAsync(Model.Content) +
+
+} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTypeTransferSettings.Edit.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTypeTransferSettings.Edit.cshtml new file mode 100644 index 00000000000..5a9be4daaaa --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ContentTypeTransferSettings.Edit.cshtml @@ -0,0 +1,17 @@ +@model ContentTypeTransferSettingsViewModels + +
+
+ + + @T["Determines whether you wish to grant the user ability to perform bulk imports from a file."] +
+
+ +
+
+ + + @T["Determines whether you wish to grant the user ability to perform bulk export to a file."] +
+
diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ImportContent.Edit.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ImportContent.Edit.cshtml new file mode 100644 index 00000000000..be403296bd9 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ImportContent.Edit.cshtml @@ -0,0 +1,4 @@ +@if (Model.Content != null) +{ + @await DisplayAsync(Model.Content) +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ImportContentFile.Edit.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ImportContentFile.Edit.cshtml new file mode 100644 index 00000000000..53f6303e550 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ImportContentFile.Edit.cshtml @@ -0,0 +1,26 @@ +@model ContentImportViewModel + +
+ + + +
+ + diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ListContentTransferEntriesAdminList.Fields.BulkActions.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ListContentTransferEntriesAdminList.Fields.BulkActions.cshtml new file mode 100644 index 00000000000..08e449672a2 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ListContentTransferEntriesAdminList.Fields.BulkActions.cshtml @@ -0,0 +1,16 @@ +@using OrchardCore.DisplayManagement +@model ListContentTransferEntryOptions + +@inject IDisplayManager DisplayManager + + diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ListContentTransferEntriesAdminListActionBarButtons.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ListContentTransferEntriesAdminListActionBarButtons.cshtml new file mode 100644 index 00000000000..6bdae199092 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ListContentTransferEntriesAdminListActionBarButtons.cshtml @@ -0,0 +1,9 @@ +@model ListContentTransferEntryOptions +@using OrchardCore.ContentTransfer + +@if (Model.Direction == ContentTransferDirection.Export && Model.ExportableTypes?.Count > 0) +{ + +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ListContentTransferEntriesAdminListBulkActions.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ListContentTransferEntriesAdminListBulkActions.cshtml new file mode 100644 index 00000000000..f75aec1dffb --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ListContentTransferEntriesAdminListBulkActions.cshtml @@ -0,0 +1,6 @@ +@model ListContentTransferEntryOptions + +@foreach (var item in Model.BulkActions) +{ + @item.Text +} diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ListContentTransferEntriesAdminListFilters.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ListContentTransferEntriesAdminListFilters.cshtml new file mode 100644 index 00000000000..dd1b718b9ab --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ListContentTransferEntriesAdminListFilters.cshtml @@ -0,0 +1,8 @@ +@model ListContentTransferEntryOptions + +
+ + +
diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ListContentTransferEntriesAdminListSearch.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ListContentTransferEntriesAdminListSearch.cshtml new file mode 100644 index 00000000000..e61eae4cec7 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ListContentTransferEntriesAdminListSearch.cshtml @@ -0,0 +1,8 @@ +@model ListContentTransferEntryOptions + + +
+ + + +
diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ListContentTransferEntriesAdminListSummary.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ListContentTransferEntriesAdminListSummary.cshtml new file mode 100644 index 00000000000..45f91bdaee2 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/ListContentTransferEntriesAdminListSummary.cshtml @@ -0,0 +1,11 @@ +@using Microsoft.AspNetCore.Mvc.Localization +@model ListContentTransferEntryOptions + +
+
+ + + + +
+
diff --git a/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/_ViewImports.cshtml b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/_ViewImports.cshtml new file mode 100644 index 00000000000..d9548deabd2 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.ContentTransfer/Views/_ViewImports.cshtml @@ -0,0 +1,10 @@ +@using OrchardCore.ContentTransfer +@using OrchardCore.ContentTransfer.Models +@using OrchardCore.ContentTransfer.ViewModels +@using OrchardCore.Entities + +@inherits OrchardCore.DisplayManagement.Razor.RazorPage + +@addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers +@addTagHelper *, OrchardCore.DisplayManagement +@addTagHelper *, OrchardCore.ResourceManagement diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/ContentItemLoad.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/ContentItemLoad.cs new file mode 100644 index 00000000000..d184a509452 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/ContentItemLoad.cs @@ -0,0 +1,108 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using OrchardCore.ContentManagement; +using OrchardCore.DataOrchestrator.Models; + +namespace OrchardCore.DataOrchestrator.Activities; + +/// +/// Creates or updates Orchard Core content items from pipeline data. +/// +public sealed class ContentItemLoad : EtlLoadActivity +{ + public override string Name => nameof(ContentItemLoad); + + public override string DisplayText => "Content Item"; + + public string ContentType + { + get => GetProperty(); + set => SetProperty(value); + } + + public override IEnumerable GetPossibleOutcomes() + { + return [new EtlOutcome("Done"), new EtlOutcome("Failed")]; + } + + public override async Task ExecuteAsync(EtlExecutionContext context) + { + var contentType = ContentType; + + if (string.IsNullOrEmpty(contentType)) + { + return EtlActivityResult.Failure("ContentItem load requires a 'contentType' configuration."); + } + + var data = context.DataStream; + if (data == null) + { + return EtlActivityResult.Failure("No data stream available."); + } + + var contentManager = context.ServiceProvider.GetRequiredService(); + var logger = context.ServiceProvider.GetRequiredService>(); + + var loaded = 0; + var failed = 0; + + await foreach (var record in data.WithCancellation(context.CancellationToken)) + { + try + { + var contentItemId = record["ContentItemId"]?.GetValue(); + ContentItem item; + + if (!string.IsNullOrEmpty(contentItemId)) + { + item = await contentManager.GetAsync(contentItemId, VersionOptions.DraftRequired); + item ??= await contentManager.NewAsync(contentType); + } + else + { + item = await contentManager.NewAsync(contentType); + } + + var itemJson = JConvert.SerializeObject(item); + var itemNode = JsonNode.Parse(itemJson) as JsonObject; + + if (itemNode is not null) + { + foreach (var property in record) + { + if (property.Key is "ContentItemId" or "ContentType") + { + continue; + } + + itemNode[property.Key] = property.Value?.DeepClone(); + } + + var merged = JConvert.DeserializeObject(itemNode.ToJsonString()); + if (merged is not null) + { + await contentManager.CreateAsync(merged, VersionOptions.Draft); + await contentManager.PublishAsync(merged); + loaded++; + } + } + } + catch (Exception ex) + { + failed++; + logger.LogError(ex, "Failed to create/update content item in ETL pipeline."); + } + } + + if (logger.IsEnabled(LogLevel.Information)) + { + logger.LogInformation("ETL ContentItem load completed: {Loaded} loaded, {Failed} failed.", loaded, failed); + } + + return failed > 0 + ? EtlActivityResult.Failure($"{failed} record(s) failed to load.") + : Outcomes("Done"); + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/ContentItemSource.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/ContentItemSource.cs new file mode 100644 index 00000000000..158758622d7 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/ContentItemSource.cs @@ -0,0 +1,126 @@ +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.DependencyInjection; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Records; +using OrchardCore.DataOrchestrator.Models; +using YesSql; + +namespace OrchardCore.DataOrchestrator.Activities; + +/// +/// Extracts published content items of a specified content type. +/// +public sealed class ContentItemSource : EtlSourceActivity +{ + public const string PublishedVersionScope = "Published"; + public const string LatestVersionScope = "Latest"; + public const string AllVersionsScope = "AllVersions"; + + public override string Name => nameof(ContentItemSource); + + public override string DisplayText => "Content Items"; + + public string ContentType + { + get => GetProperty(); + set => SetProperty(value); + } + + public string VersionScope + { + get => GetProperty(() => PublishedVersionScope); + set => SetProperty(value); + } + + public string Owner + { + get => GetProperty(); + set => SetProperty(value); + } + + public DateTime? CreatedUtcFrom + { + get => GetProperty(); + set => SetProperty(value); + } + + public DateTime? CreatedUtcTo + { + get => GetProperty(); + set => SetProperty(value); + } + + public override IEnumerable GetPossibleOutcomes() + { + return [new EtlOutcome("Done")]; + } + + public override Task ExecuteAsync(EtlExecutionContext context) + { + context.DataStream = ExtractAsync( + context.ServiceProvider, + ContentType, + VersionScope, + Owner, + CreatedUtcFrom, + CreatedUtcTo, + context.CancellationToken); + + return Task.FromResult(Outcomes("Done")); + } + + private static async IAsyncEnumerable ExtractAsync( + IServiceProvider serviceProvider, + string contentType, + string versionScope, + string owner, + DateTime? createdUtcFrom, + DateTime? createdUtcTo, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (string.IsNullOrEmpty(contentType)) + { + yield break; + } + + var session = serviceProvider.GetRequiredService(); + var query = versionScope switch + { + LatestVersionScope => session.Query(x => x.ContentType == contentType && x.Latest), + AllVersionsScope => session.Query(x => x.ContentType == contentType), + _ => session.Query(x => x.ContentType == contentType && x.Published), + }; + + var items = await query.ListAsync(cancellationToken); + + foreach (var item in items) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (!string.IsNullOrWhiteSpace(owner) && !string.Equals(item.Owner, owner, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (createdUtcFrom.HasValue && item.CreatedUtc < createdUtcFrom.Value) + { + continue; + } + + if (createdUtcTo.HasValue && item.CreatedUtc > createdUtcTo.Value) + { + continue; + } + + var json = JConvert.SerializeObject(item); + var node = JsonNode.Parse(json); + + if (node is JsonObject obj) + { + yield return obj; + } + } + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/ExcelExportLoad.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/ExcelExportLoad.cs new file mode 100644 index 00000000000..7266438d306 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/ExcelExportLoad.cs @@ -0,0 +1,145 @@ +using System.Text.Json.Nodes; +using DocumentFormat.OpenXml; +using DocumentFormat.OpenXml.Packaging; +using DocumentFormat.OpenXml.Spreadsheet; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using OrchardCore.DataOrchestrator.Models; +using OrchardCore.FileStorage; +using OrchardCore.Media; + +namespace OrchardCore.DataOrchestrator.Activities; + +/// +/// Exports pipeline data to an Excel workbook. +/// +public sealed class ExcelExportLoad : EtlLoadActivity +{ + public override string Name => nameof(ExcelExportLoad); + + public override string DisplayText => "Excel Workbook Export"; + + public string FileName + { + get => GetProperty(() => "etl-export.xlsx"); + set => SetProperty(value); + } + + public string WorksheetName + { + get => GetProperty(() => "Data"); + set => SetProperty(value); + } + + public override IEnumerable GetPossibleOutcomes() + { + return [new EtlOutcome("Done"), new EtlOutcome("Failed")]; + } + + public override async Task ExecuteAsync(EtlExecutionContext context) + { + var data = context.DataStream; + if (data == null) + { + return EtlActivityResult.Failure("No data stream available."); + } + + var logger = context.ServiceProvider.GetRequiredService>(); + var mediaFileStore = context.ServiceProvider.GetService(); + + if (mediaFileStore is null) + { + return EtlActivityResult.Failure("No IMediaFileStore service is available for Excel export."); + } + + try + { + var rows = new List(); + var headers = new List(); + + await foreach (var record in data.WithCancellation(context.CancellationToken)) + { + rows.Add(record.DeepClone().AsObject()); + + foreach (var property in record) + { + if (!headers.Contains(property.Key, StringComparer.OrdinalIgnoreCase)) + { + headers.Add(property.Key); + } + } + } + + await using var stream = new MemoryStream(); + using (var document = SpreadsheetDocument.Create(stream, SpreadsheetDocumentType.Workbook, true)) + { + var workbookPart = document.AddWorkbookPart(); + workbookPart.Workbook = new Workbook(); + + var worksheetPart = workbookPart.AddNewPart(); + var sheetData = new SheetData(); + worksheetPart.Worksheet = new Worksheet(sheetData); + + var sheets = workbookPart.Workbook.AppendChild(new Sheets()); + sheets.Append(new Sheet + { + Id = workbookPart.GetIdOfPart(worksheetPart), + SheetId = 1, + Name = WorksheetName, + }); + + var headerRow = new Row(); + foreach (var header in headers) + { + headerRow.Append(CreateTextCell(header)); + } + + sheetData.Append(headerRow); + + foreach (var row in rows) + { + var sheetRow = new Row(); + foreach (var header in headers) + { + sheetRow.Append(CreateTextCell(row[header]?.ToString() ?? string.Empty)); + } + + sheetData.Append(sheetRow); + } + + workbookPart.Workbook.Save(); + } + + stream.Position = 0; + var fileName = mediaFileStore.NormalizePath(FileName); + await mediaFileStore.CreateFileFromStreamAsync(fileName, stream, overwrite: true); + + if (logger.IsEnabled(LogLevel.Information)) + { + logger.LogInformation("ETL Excel export wrote {Count} records to '{FileName}'.", rows.Count, fileName); + } + + return Outcomes("Done"); + } + catch (Exception ex) + { + var fileName = FileName; + + if (logger.IsEnabled(LogLevel.Error)) + { + logger.LogError(ex, "ETL Excel export failed for '{FileName}'.", fileName); + } + + return EtlActivityResult.Failure($"Excel export failed for '{fileName}': {ex.Message}"); + } + } + + private static Cell CreateTextCell(string value) + { + return new Cell + { + DataType = CellValues.InlineString, + InlineString = new InlineString(new Text(value ?? string.Empty)), + }; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/ExcelSource.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/ExcelSource.cs new file mode 100644 index 00000000000..638833a4a89 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/ExcelSource.cs @@ -0,0 +1,207 @@ +using System.Runtime.CompilerServices; +using System.Text.Json.Nodes; +using DocumentFormat.OpenXml; +using DocumentFormat.OpenXml.Packaging; +using DocumentFormat.OpenXml.Spreadsheet; +using Microsoft.Extensions.DependencyInjection; +using OrchardCore.DataOrchestrator.Models; +using OrchardCore.FileStorage; +using OrchardCore.Media; + +namespace OrchardCore.DataOrchestrator.Activities; + +/// +/// Extracts rows from an Excel workbook stored in the media library. +/// +public sealed class ExcelSource : EtlSourceActivity +{ + public override string Name => nameof(ExcelSource); + + public override string DisplayText => "Excel Workbook"; + + public string FilePath + { + get => GetProperty(); + set => SetProperty(value); + } + + public string WorksheetName + { + get => GetProperty(); + set => SetProperty(value); + } + + public bool HasHeaderRow + { + get => GetProperty(() => true); + set => SetProperty(value); + } + + public override IEnumerable GetPossibleOutcomes() + { + return [new EtlOutcome("Done")]; + } + + public override Task ExecuteAsync(EtlExecutionContext context) + { + context.DataStream = ExtractAsync(context.ServiceProvider, FilePath, WorksheetName, HasHeaderRow, context.CancellationToken); + + return Task.FromResult(Outcomes("Done")); + } + + private static async IAsyncEnumerable ExtractAsync( + IServiceProvider serviceProvider, + string filePath, + string worksheetName, + bool hasHeaderRow, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(filePath)) + { + yield break; + } + + var mediaFileStore = serviceProvider.GetRequiredService(); + var normalizedPath = mediaFileStore.NormalizePath(filePath); + var fileInfo = await mediaFileStore.GetFileInfoAsync(normalizedPath); + + if (fileInfo == null) + { + yield break; + } + + await using var stream = await mediaFileStore.GetFileStreamAsync(fileInfo); + using var spreadsheetDocument = SpreadsheetDocument.Open(stream, false); + var workbookPart = spreadsheetDocument.WorkbookPart; + + if (workbookPart?.Workbook == null) + { + yield break; + } + + var sharedStringTable = workbookPart.GetPartsOfType().FirstOrDefault()?.SharedStringTable; + var sheet = string.IsNullOrWhiteSpace(worksheetName) + ? workbookPart.Workbook.Descendants().FirstOrDefault() + : workbookPart.Workbook.Descendants().FirstOrDefault(x => string.Equals(x.Name?.Value, worksheetName, StringComparison.OrdinalIgnoreCase)); + + if (sheet == null) + { + yield break; + } + + var worksheetPart = (WorksheetPart)workbookPart.GetPartById(sheet.Id); + var sheetData = worksheetPart.Worksheet.GetFirstChild(); + + if (sheetData == null) + { + yield break; + } + + var rows = sheetData.Elements().ToList(); + + if (rows.Count == 0) + { + yield break; + } + + var headerValues = GetRowValues(rows[0], sharedStringTable); + var columnNames = BuildColumnNames(headerValues, hasHeaderRow); + var dataRows = hasHeaderRow ? rows.Skip(1) : rows; + + foreach (var row in dataRows) + { + cancellationToken.ThrowIfCancellationRequested(); + + var values = GetRowValues(row, sharedStringTable); + var record = new JsonObject(); + + for (var i = 0; i < columnNames.Count; i++) + { + record[columnNames[i]] = i < values.Count ? values[i] : string.Empty; + } + + yield return record; + } + } + + private static List BuildColumnNames(List headerValues, bool hasHeaderRow) + { + if (!hasHeaderRow) + { + return Enumerable.Range(1, headerValues.Count == 0 ? 1 : headerValues.Count) + .Select(index => $"Column{index}") + .ToList(); + } + + var columnNames = new List(); + + for (var i = 0; i < headerValues.Count; i++) + { + var header = string.IsNullOrWhiteSpace(headerValues[i]) ? $"Column{i + 1}" : headerValues[i].Trim(); + var uniqueHeader = header; + var suffix = 2; + + while (columnNames.Contains(uniqueHeader, StringComparer.OrdinalIgnoreCase)) + { + uniqueHeader = $"{header}_{suffix++}"; + } + + columnNames.Add(uniqueHeader); + } + + return columnNames.Count == 0 ? ["Column1"] : columnNames; + } + + private static List GetRowValues(Row row, SharedStringTable sharedStringTable) + { + var values = new List(); + + foreach (var cell in row.Elements()) + { + var columnIndex = GetColumnIndexFromCellReference(cell.CellReference); + + while (values.Count <= columnIndex) + { + values.Add(string.Empty); + } + + values[columnIndex] = GetCellValue(cell, sharedStringTable); + } + + return values; + } + + private static int GetColumnIndexFromCellReference(StringValue cellReference) + { + var reference = cellReference?.Value ?? string.Empty; + var columnName = new string(reference.TakeWhile(char.IsLetter).ToArray()); + var columnIndex = 0; + + foreach (var ch in columnName) + { + columnIndex *= 26; + columnIndex += ch - 'A' + 1; + } + + return Math.Max(columnIndex - 1, 0); + } + + private static string GetCellValue(Cell cell, SharedStringTable sharedStringTable) + { + if (cell.CellValue == null) + { + return string.Empty; + } + + var value = cell.CellValue.InnerText; + + if (cell.DataType?.Value == CellValues.SharedString && + int.TryParse(value, out var sharedStringIndex) && + sharedStringTable?.ElementAtOrDefault(sharedStringIndex) is SharedStringItem sharedStringItem) + { + return sharedStringItem.InnerText; + } + + return value; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/FieldMappingTransform.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/FieldMappingTransform.cs new file mode 100644 index 00000000000..6ae578e6f22 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/FieldMappingTransform.cs @@ -0,0 +1,115 @@ +using System.Runtime.CompilerServices; +using System.Text.Json.Nodes; +using OrchardCore.DataOrchestrator.Models; + +namespace OrchardCore.DataOrchestrator.Activities; + +/// +/// Maps and renames fields in records based on configurable source-to-target mappings. +/// +public sealed class FieldMappingTransform : EtlTransformActivity +{ + public override string Name => nameof(FieldMappingTransform); + + public override string DisplayText => "Field Mapping"; + + public string MappingsJson + { + get => GetProperty(); + set => SetProperty(value); + } + + public override IEnumerable GetPossibleOutcomes() + { + return [new EtlOutcome("Done")]; + } + + public override Task ExecuteAsync(EtlExecutionContext context) + { + var input = context.DataStream; + var mappingsJson = MappingsJson; + + context.DataStream = TransformAsync(input, mappingsJson, context.CancellationToken); + + return Task.FromResult(Outcomes("Done")); + } + + private static async IAsyncEnumerable TransformAsync( + IAsyncEnumerable input, + string mappingsJson, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (input == null) + { + yield break; + } + + List<(string Source, string Target)> mappings = []; + + if (!string.IsNullOrWhiteSpace(mappingsJson)) + { + try + { + var array = JsonNode.Parse(mappingsJson) as JsonArray; + if (array != null) + { + foreach (var m in array) + { + var source = m?["source"]?.GetValue(); + var target = m?["target"]?.GetValue(); + if (!string.IsNullOrEmpty(source) && !string.IsNullOrEmpty(target)) + { + mappings.Add((source, target)); + } + } + } + } + catch + { + // If mappings can't be parsed, pass through + } + } + + await foreach (var record in input.WithCancellation(cancellationToken)) + { + if (mappings.Count == 0) + { + yield return record; + continue; + } + + var result = new JsonObject(); + + foreach (var (source, target) in mappings) + { + var value = ResolveJsonPath(record, source); + if (value is not null) + { + result[target] = value.DeepClone(); + } + } + + yield return result; + } + } + + private static JsonNode ResolveJsonPath(JsonObject root, string path) + { + var segments = path.Split('.'); + JsonNode current = root; + + foreach (var segment in segments) + { + if (current is JsonObject obj && obj.TryGetPropertyValue(segment, out var next)) + { + current = next; + } + else + { + return null; + } + } + + return current; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/FilterTransform.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/FilterTransform.cs new file mode 100644 index 00000000000..fcd6e9baca4 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/FilterTransform.cs @@ -0,0 +1,141 @@ +using System.Runtime.CompilerServices; +using System.Text.Json.Nodes; +using OrchardCore.DataOrchestrator.Models; + +namespace OrchardCore.DataOrchestrator.Activities; + +/// +/// Filters records based on a configurable field condition. +/// +public sealed class FilterTransform : EtlTransformActivity +{ + public override string Name => nameof(FilterTransform); + + public override string DisplayText => "Filter"; + + public string Field + { + get => GetProperty(); + set => SetProperty(value); + } + + public string Operator + { + get => GetProperty(); + set => SetProperty(value); + } + + public string Value + { + get => GetProperty(); + set => SetProperty(value); + } + + public override IEnumerable GetPossibleOutcomes() + { + return [new EtlOutcome("Done")]; + } + + public override Task ExecuteAsync(EtlExecutionContext context) + { + var input = context.DataStream; + + context.DataStream = TransformAsync(input, Field, Operator, Value, context.CancellationToken); + + return Task.FromResult(Outcomes("Done")); + } + + private static async IAsyncEnumerable TransformAsync( + IAsyncEnumerable input, + string field, + string op, + string value, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (input == null) + { + yield break; + } + + if (string.IsNullOrEmpty(field) || string.IsNullOrEmpty(op)) + { + await foreach (var record in input.WithCancellation(cancellationToken)) + { + yield return record; + } + + yield break; + } + + await foreach (var record in input.WithCancellation(cancellationToken)) + { + var fieldValue = ResolveFieldValue(record, field); + + if (Evaluate(fieldValue, op, value)) + { + yield return record; + } + } + } + + private static string ResolveFieldValue(JsonObject record, string field) + { + var segments = field.Split('.'); + JsonNode current = record; + + foreach (var segment in segments) + { + if (current is JsonObject obj && obj.TryGetPropertyValue(segment, out var next)) + { + current = next; + } + else + { + return null; + } + } + + return current?.ToString(); + } + + private static bool Evaluate(string fieldValue, string op, string compareValue) + { + if (fieldValue is null) + { + return op.Equals("not_equals", StringComparison.OrdinalIgnoreCase) || + op.Equals("not_contains", StringComparison.OrdinalIgnoreCase); + } + + return op.ToUpperInvariant() switch + { + "EQUALS" => string.Equals(fieldValue, compareValue, StringComparison.OrdinalIgnoreCase), + "NOT_EQUALS" => !string.Equals(fieldValue, compareValue, StringComparison.OrdinalIgnoreCase), + "CONTAINS" => fieldValue.Contains(compareValue ?? string.Empty, StringComparison.OrdinalIgnoreCase), + "NOT_CONTAINS" => !fieldValue.Contains(compareValue ?? string.Empty, StringComparison.OrdinalIgnoreCase), + "STARTS_WITH" => fieldValue.StartsWith(compareValue ?? string.Empty, StringComparison.OrdinalIgnoreCase), + "ENDS_WITH" => fieldValue.EndsWith(compareValue ?? string.Empty, StringComparison.OrdinalIgnoreCase), + "GREATER_THAN" => CompareValues(fieldValue, compareValue) > 0, + "GREATER_THAN_OR_EQUAL" => CompareValues(fieldValue, compareValue) >= 0, + "LESS_THAN" => CompareValues(fieldValue, compareValue) < 0, + "LESS_THAN_OR_EQUAL" => CompareValues(fieldValue, compareValue) <= 0, + "IS_EMPTY" => string.IsNullOrWhiteSpace(fieldValue), + "IS_NOT_EMPTY" => !string.IsNullOrWhiteSpace(fieldValue), + _ => true, + }; + } + + private static int CompareValues(string fieldValue, string compareValue) + { + if (decimal.TryParse(fieldValue, out var fieldNumber) && decimal.TryParse(compareValue, out var compareNumber)) + { + return fieldNumber.CompareTo(compareNumber); + } + + if (DateTime.TryParse(fieldValue, out var fieldDate) && DateTime.TryParse(compareValue, out var compareDate)) + { + return fieldDate.CompareTo(compareDate); + } + + return string.Compare(fieldValue, compareValue, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/FormatValueTransform.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/FormatValueTransform.cs new file mode 100644 index 00000000000..c47046fc745 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/FormatValueTransform.cs @@ -0,0 +1,187 @@ +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Text.Json.Nodes; +using OrchardCore.DataOrchestrator.Models; + +namespace OrchardCore.DataOrchestrator.Activities; + +/// +/// Formats or converts a field value and writes it back to the current record. +/// +public sealed class FormatValueTransform : EtlTransformActivity +{ + public const string CurrencyFormat = "Currency"; + public const string NumberFormat = "Number"; + public const string DateTimeFormat = "DateTime"; + public const string ConvertUtcToTimeZoneFormat = "ConvertUtcToTimeZone"; + public const string UppercaseFormat = "Uppercase"; + public const string LowercaseFormat = "Lowercase"; + + public override string Name => nameof(FormatValueTransform); + + public override string DisplayText => "Format Value"; + + public string Field + { + get => GetProperty(); + set => SetProperty(value); + } + + public string OutputField + { + get => GetProperty(); + set => SetProperty(value); + } + + public string FormatType + { + get => GetProperty(() => DateTimeFormat); + set => SetProperty(value); + } + + public string FormatString + { + get => GetProperty(); + set => SetProperty(value); + } + + public string Culture + { + get => GetProperty(); + set => SetProperty(value); + } + + public string TimeZoneId + { + get => GetProperty(); + set => SetProperty(value); + } + + public override IEnumerable GetPossibleOutcomes() + { + return [new EtlOutcome("Done")]; + } + + public override Task ExecuteAsync(EtlExecutionContext context) + { + context.DataStream = TransformAsync( + context.DataStream, + Field, + OutputField, + FormatType, + FormatString, + Culture, + TimeZoneId, + context.CancellationToken); + + return Task.FromResult(Outcomes("Done")); + } + + private static async IAsyncEnumerable TransformAsync( + IAsyncEnumerable input, + string field, + string outputField, + string formatType, + string formatString, + string culture, + string timeZoneId, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (input == null) + { + yield break; + } + + await foreach (var record in input.WithCancellation(cancellationToken)) + { + if (string.IsNullOrWhiteSpace(field)) + { + yield return record; + continue; + } + + var sourceValue = ResolveFieldValue(record, field); + if (sourceValue == null) + { + yield return record; + continue; + } + + var cloned = record.DeepClone().AsObject(); + var destinationField = string.IsNullOrWhiteSpace(outputField) ? field : outputField; + SetFieldValue(cloned, destinationField, ApplyFormat(sourceValue, formatType, formatString, culture, timeZoneId)); + yield return cloned; + } + } + + private static string ApplyFormat(string sourceValue, string formatType, string formatString, string culture, string timeZoneId) + { + var cultureInfo = string.IsNullOrWhiteSpace(culture) + ? CultureInfo.InvariantCulture + : CultureInfo.GetCultureInfo(culture); + + return formatType switch + { + CurrencyFormat when decimal.TryParse(sourceValue, out var currencyValue) => currencyValue.ToString(formatString ?? "C", cultureInfo), + NumberFormat when decimal.TryParse(sourceValue, out var numberValue) => numberValue.ToString(formatString ?? "N", cultureInfo), + DateTimeFormat when DateTime.TryParse(sourceValue, out var dateTimeValue) => dateTimeValue.ToString(formatString ?? "u", cultureInfo), + ConvertUtcToTimeZoneFormat when DateTime.TryParse(sourceValue, out var utcValue) => ConvertUtcToTimeZone(utcValue, timeZoneId, formatString, cultureInfo), + UppercaseFormat => sourceValue.ToUpper(cultureInfo), + LowercaseFormat => sourceValue.ToLower(cultureInfo), + _ => sourceValue, + }; + } + + private static string ConvertUtcToTimeZone(DateTime utcValue, string timeZoneId, string formatString, CultureInfo cultureInfo) + { + if (string.IsNullOrWhiteSpace(timeZoneId)) + { + return utcValue.ToString(formatString ?? "u", cultureInfo); + } + + var timeZone = TimeZoneInfo.FindSystemTimeZoneById(timeZoneId); + var normalized = utcValue.Kind == DateTimeKind.Utc ? utcValue : DateTime.SpecifyKind(utcValue, DateTimeKind.Utc); + var converted = TimeZoneInfo.ConvertTimeFromUtc(normalized, timeZone); + + return converted.ToString(formatString ?? "G", cultureInfo); + } + + private static string ResolveFieldValue(JsonObject record, string field) + { + var segments = field.Split('.', StringSplitOptions.RemoveEmptyEntries); + JsonNode current = record; + + foreach (var segment in segments) + { + if (current is JsonObject obj && obj.TryGetPropertyValue(segment, out var next)) + { + current = next; + } + else + { + return null; + } + } + + return current?.ToString(); + } + + private static void SetFieldValue(JsonObject root, string field, string value) + { + var segments = field.Split('.', StringSplitOptions.RemoveEmptyEntries); + var current = root; + + for (var i = 0; i < segments.Length - 1; i++) + { + if (current[segments[i]] is not JsonObject next) + { + next = []; + current[segments[i]] = next; + } + + current = next; + } + + current[segments[^1]] = value; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/JoinDataSetsTransform.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/JoinDataSetsTransform.cs new file mode 100644 index 00000000000..4947c72e714 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/JoinDataSetsTransform.cs @@ -0,0 +1,188 @@ +using System.Runtime.CompilerServices; +using System.Text.Json.Nodes; +using OrchardCore.DataOrchestrator.Models; + +namespace OrchardCore.DataOrchestrator.Activities; + +/// +/// Joins the current stream with records produced by another source activity in the pipeline. +/// +public sealed class JoinDataSetsTransform : EtlTransformActivity +{ + public const string InnerJoin = "Inner"; + public const string LeftJoin = "Left"; + + public override string Name => nameof(JoinDataSetsTransform); + + public override string DisplayText => "Join Data Sets"; + + public string JoinSourceActivityId + { + get => GetProperty(); + set => SetProperty(value); + } + + public string LeftField + { + get => GetProperty(); + set => SetProperty(value); + } + + public string RightField + { + get => GetProperty(); + set => SetProperty(value); + } + + public string JoinType + { + get => GetProperty(() => LeftJoin); + set => SetProperty(value); + } + + public string RightPrefix + { + get => GetProperty(() => "Joined_"); + set => SetProperty(value); + } + + public override IEnumerable GetPossibleOutcomes() + { + return [new EtlOutcome("Done"), new EtlOutcome("Failed")]; + } + + public override async Task ExecuteAsync(EtlExecutionContext context) + { + if (context.DataStream == null) + { + return EtlActivityResult.Failure("No left-side data stream available."); + } + + if (string.IsNullOrWhiteSpace(JoinSourceActivityId)) + { + return EtlActivityResult.Failure("A join source activity must be selected."); + } + + var joinSourceRecord = context.Pipeline.Activities.FirstOrDefault(x => x.ActivityId == JoinSourceActivityId); + if (joinSourceRecord == null) + { + return EtlActivityResult.Failure("The selected join source activity could not be found."); + } + + var joinSourceActivity = context.ActivityLibrary.InstantiateActivity(joinSourceRecord.Name); + if (joinSourceActivity is not EtlSourceActivity) + { + return EtlActivityResult.Failure("The selected join activity must be a source activity."); + } + + joinSourceActivity.Properties = joinSourceRecord.Properties?.DeepClone() as JsonObject ?? []; + + var joinContext = context.Clone(); + var joinResult = await joinSourceActivity.ExecuteAsync(joinContext); + if (!joinResult.IsSuccess || joinContext.DataStream == null) + { + return EtlActivityResult.Failure(joinResult.ErrorMessage ?? "Unable to execute the selected join source."); + } + + context.DataStream = TransformAsync( + context.DataStream, + joinContext.DataStream, + LeftField, + RightField, + JoinType, + RightPrefix, + context.CancellationToken); + + return Outcomes("Done"); + } + + private static async IAsyncEnumerable TransformAsync( + IAsyncEnumerable leftStream, + IAsyncEnumerable rightStream, + string leftField, + string rightField, + string joinType, + string rightPrefix, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(leftField) || string.IsNullOrWhiteSpace(rightField)) + { + await foreach (var record in leftStream.WithCancellation(cancellationToken)) + { + yield return record; + } + + yield break; + } + + var rightLookup = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + await foreach (var record in rightStream.WithCancellation(cancellationToken)) + { + var key = ResolveFieldValue(record, rightField); + if (string.IsNullOrWhiteSpace(key)) + { + continue; + } + + if (!rightLookup.TryGetValue(key, out var matches)) + { + matches = []; + rightLookup[key] = matches; + } + + matches.Add(record.DeepClone().AsObject()); + } + + await foreach (var leftRecord in leftStream.WithCancellation(cancellationToken)) + { + var leftKey = ResolveFieldValue(leftRecord, leftField); + var leftClone = leftRecord.DeepClone().AsObject(); + + if (!string.IsNullOrWhiteSpace(leftKey) && rightLookup.TryGetValue(leftKey, out var rightMatches)) + { + foreach (var rightRecord in rightMatches) + { + yield return MergeRecords(leftClone, rightRecord, rightPrefix); + } + } + else if (string.Equals(joinType, LeftJoin, StringComparison.OrdinalIgnoreCase)) + { + yield return leftClone; + } + } + } + + private static JsonObject MergeRecords(JsonObject leftRecord, JsonObject rightRecord, string rightPrefix) + { + var merged = leftRecord.DeepClone().AsObject(); + var prefix = rightPrefix ?? string.Empty; + + foreach (var property in rightRecord) + { + merged[$"{prefix}{property.Key}"] = property.Value?.DeepClone(); + } + + return merged; + } + + private static string ResolveFieldValue(JsonObject record, string field) + { + var segments = field.Split('.', StringSplitOptions.RemoveEmptyEntries); + JsonNode current = record; + + foreach (var segment in segments) + { + if (current is JsonObject obj && obj.TryGetPropertyValue(segment, out var next)) + { + current = next; + } + else + { + return null; + } + } + + return current?.ToString(); + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/JsonExportLoad.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/JsonExportLoad.cs new file mode 100644 index 00000000000..73608ae1c7f --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/JsonExportLoad.cs @@ -0,0 +1,85 @@ +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using OrchardCore.DataOrchestrator.Models; +using OrchardCore.FileStorage; +using OrchardCore.Media; + +namespace OrchardCore.DataOrchestrator.Activities; + +/// +/// Exports pipeline data as a JSON array to a file in the media file store. +/// +public sealed class JsonExportLoad : EtlLoadActivity +{ + public override string Name => nameof(JsonExportLoad); + + public override string DisplayText => "JSON Export"; + + public string FileName + { + get => GetProperty(() => "etl-export.json"); + set => SetProperty(value); + } + + public override IEnumerable GetPossibleOutcomes() + { + return [new EtlOutcome("Done"), new EtlOutcome("Failed")]; + } + + public override async Task ExecuteAsync(EtlExecutionContext context) + { + var data = context.DataStream; + if (data == null) + { + return EtlActivityResult.Failure("No data stream available."); + } + + var logger = context.ServiceProvider.GetRequiredService>(); + var mediaFileStore = context.ServiceProvider.GetService(); + + if (mediaFileStore is null) + { + return EtlActivityResult.Failure("No IMediaFileStore service is available for JSON export."); + } + + try + { + var records = new JsonArray(); + var loaded = 0; + + await foreach (var record in data.WithCancellation(context.CancellationToken)) + { + records.Add(record.DeepClone()); + loaded++; + } + + var fileName = mediaFileStore.NormalizePath(FileName); + var jsonString = records.ToJsonString(new JsonSerializerOptions { WriteIndented = true }); + var bytes = Encoding.UTF8.GetBytes(jsonString); + + using var stream = new MemoryStream(bytes); + await mediaFileStore.CreateFileFromStreamAsync(fileName, stream, overwrite: true); + + if (logger.IsEnabled(LogLevel.Information)) + { + logger.LogInformation("ETL JSON export wrote {Count} records to '{FileName}'.", loaded, fileName); + } + + return Outcomes("Done"); + } + catch (Exception ex) + { + var fileName = FileName; + + if (logger.IsEnabled(LogLevel.Error)) + { + logger.LogError(ex, "ETL JSON export failed for '{FileName}'.", fileName); + } + + return EtlActivityResult.Failure($"JSON export failed for '{fileName}': {ex.Message}"); + } + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/JsonSource.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/JsonSource.cs new file mode 100644 index 00000000000..06a109c7ce2 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Activities/JsonSource.cs @@ -0,0 +1,67 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using OrchardCore.DataOrchestrator.Models; + +namespace OrchardCore.DataOrchestrator.Activities; + +/// +/// Extracts records from a JSON array provided in the configuration. +/// +public sealed class JsonSource : EtlSourceActivity +{ + public override string Name => nameof(JsonSource); + + public override string DisplayText => "JSON Data"; + + public string Data + { + get => GetProperty(); + set => SetProperty(value); + } + + public override IEnumerable GetPossibleOutcomes() + { + return [new EtlOutcome("Done")]; + } + + public override Task ExecuteAsync(EtlExecutionContext context) + { + context.DataStream = ExtractAsync(Data); + + return Task.FromResult(Outcomes("Done")); + } + +#pragma warning disable CS1998 // Async method lacks 'await' operators + private static async IAsyncEnumerable ExtractAsync(string data) +#pragma warning restore CS1998 + { + if (string.IsNullOrWhiteSpace(data)) + { + yield break; + } + + JsonNode parsed; + + try + { + parsed = JsonNode.Parse(data); + } + catch + { + yield break; + } + + if (parsed is not JsonArray array) + { + yield break; + } + + foreach (var item in array) + { + if (item is JsonObject obj) + { + yield return obj.Deserialize(); + } + } + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/AdminMenu.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/AdminMenu.cs new file mode 100644 index 00000000000..f21191d793f --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/AdminMenu.cs @@ -0,0 +1,54 @@ +using Microsoft.Extensions.Localization; +using OrchardCore.DataOrchestrator.Controllers; +using OrchardCore.Mvc.Core.Utilities; +using OrchardCore.Navigation; + +namespace OrchardCore.DataOrchestrator; + +public sealed class AdminMenu : AdminNavigationProvider +{ + private readonly IStringLocalizer S; + + public AdminMenu(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + protected override ValueTask BuildAsync(NavigationBuilder builder) + { + var controllerName = typeof(AdminController).ControllerName(); + + if (NavigationHelper.UseLegacyFormat()) + { + builder + .Add(S["Configuration"], configuration => configuration + .Add(S["Data"], S["Data"].PrefixPosition(), data => data + .Add(S["Data Pipelines"], S["Data Pipelines"].PrefixPosition(), pipelines => pipelines + .Action(nameof(AdminController.Index), controllerName, new + { + area = EtlConstants.Features.DataPipelines, + }) + .Permission(EtlPermissions.ViewEtlPipelines) + .LocalNav() + ) + ) + ); + + return ValueTask.CompletedTask; + } + + builder + .Add(S["Tools"], tools => tools + .Add(S["Data Pipelines"], S["Data Pipelines"].PrefixPosition(), pipelines => pipelines + .Action(nameof(AdminController.Index), controllerName, new + { + area = EtlConstants.Features.DataPipelines, + }) + .Permission(EtlPermissions.ViewEtlPipelines) + .LocalNav() + ) + ); + + return ValueTask.CompletedTask; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Assets.json b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Assets.json new file mode 100644 index 00000000000..cb3b66ec1b1 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Assets.json @@ -0,0 +1,15 @@ +[ + { + "action": "sass", + "name": "etl-pipeline-editor-css", + "source": "Assets/Styles/etl-pipeline-editor.scss", + "tags": ["etl", "css"] + }, + { + "action": "parcel", + "name": "etl-pipeline-editor-js", + "source": "Assets/Scripts/etl-pipeline-editor.ts", + "dest": "wwwroot/Scripts/", + "tags": ["etl", "js"] + } +] diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Assets/Lib/jsplumb/typings.d.ts b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Assets/Lib/jsplumb/typings.d.ts new file mode 100644 index 00000000000..395d6eef7e2 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Assets/Lib/jsplumb/typings.d.ts @@ -0,0 +1,179 @@ +declare var jsPlumb: jsPlumbInstance; + +interface jsPlumbInstance { + setRenderMode(renderMode: string): string; + bind(event: string, callback: (connection: any, e: any) => void): void; + unbind(event?: string): void; + ready(callback: () => void): void; + importDefaults(defaults: any): void; + Defaults: Defaults; + restoreDefaults(): void; + addClass(el: any, clazz: string): void; + addEndpoint(ep: any, source: EndpointOptions, sourceOptions?: SourceOptions): any; + removeClass(el: any, clazz: string): void; + hasClass(el: any, clazz: string): void; + draggable(el: any, options?: DragOptions): jsPlumbInstance; + draggable(ids: string[], options?: DragOptions): jsPlumbInstance; + connect(connection: ConnectParams, referenceParams?: ConnectParams): Connection; + makeSource(el: any, options: SourceOptions): void; + makeTarget(el: any, options: TargetOptions): void; + repaintEverything(): void; + detachEveryConnection(): void; + detachAllConnections(el: string): void; + removeAllEndpoints(el: string, recurse?: boolean): jsPlumbInstance; + removeAllEndpoints(el: Element, recurse?: boolean): jsPlumbInstance; + select(params: SelectParams): Connections; + getConnections(options?: any, flat?: any): any[]; + deleteEndpoint(uuid: string, doNotRepaintAfterwards?: boolean): jsPlumbInstance; + deleteEndpoint(endpoint: Endpoint, doNotRepaintAfterwards?: boolean): jsPlumbInstance; + deleteConnection(connection: Connection): jsPlumbInstance; + remove(element: any): jsPlumbInstance; + repaint(el: string): jsPlumbInstance; + repaint(el: Element): jsPlumbInstance; + getInstance(): jsPlumbInstance; + getInstance(defaults: Defaults): jsPlumbInstance; + getInstanceIndex(): number; + registerConnectionType(name: string, type: JSPConnectionType): jsPlumbInstance; + batch(func: Function): jsPlumbInstance; + getSelector(sel: string): any; + getEndpoint(uuid: string): Endpoint; + + SVG: string; + CANVAS: string; + VML: string; +} + +interface Defaults { + Anchor?: string; + Endpoint?: any[]; + PaintStyle?: PaintStyle; + HoverPaintStyle?: PaintStyle; + EndpointStyles?: EndpointOptions; + ConnectionsDetachable?: boolean; + ReattachConnections?: boolean; + ConnectionOverlays?: any[][]; + Container?: any; // string(selector or id) or element + DragOptions?: DragOptions; +} + +interface JSPConnectionType { + connector: string, + paintStyle: PaintStyle, + hoverPaintStyle: PaintStyle, + overlays: Array +} + +interface PaintStyle { + fill?: string; + radius?: number; + stroke?: string; + strokeWidth?: number; + joinstyle?: string; + outlineStroke?: string; + outlineWidth?: number; +} + +interface Overlay { + setLabel(label: string): void; +} + +interface ArrowOverlay extends Overlay { + location: number; + id: string; + length: number; + foldback: number; +} + +interface LabelOverlay extends Overlay { + label: string; + id: string; + location: number; +} + +interface Connections { + detach(): void; + length: number; +} + +interface ConnectParams { + source?: any; // string, element or endpoint + target?: any; // string, element or endpoint + detachable?: boolean; + deleteEndpointsOnDetach?: boolean; + endPoint?: string; + anchor?: string; + anchors?: any[]; + label?: string; + uuids?: Array; + editable?: boolean; +} + +interface DragOptions { + scope?: string; + cursor?: string; + zIndex?: number; + grid?: Array; + containment?: boolean; + start?: (params: any) => void; + stop?: (params: any) => void; +} + +interface SourceOptions { + parent?: string; + endpoint?: string; + anchor?: string; + connector?: any[]; + connectorStyle?: PaintStyle; + uuid?: string; + overlays?: Array; +} + +interface TargetOptions { + isTarget?: boolean; + maxConnections?: number; + uniqueEndpoint?: boolean; + deleteEndpointsOnDetach?: boolean; + endpoint?: any; + dropOptions?: DropOptions; + anchor?: any; +} + +interface EndpointOptions { + endpoint?: string; + anchor?: string; + paintStyle?: PaintStyle; + isSource?: boolean; + connector?: Array; + connectorStyle?: PaintStyle; + hoverPaintStyle?: PaintStyle; + connectorHoverStyle?: PaintStyle; + dragOptions?: DragOptions; + overlays?: Array; + connectorOverlays?: Array; + uuid?: string; + parameters?: any; +} + +interface DropOptions { + hoverClass: string; +} + +interface SelectParams { + scope?: string; + source: string; + target: string; +} + +interface Connection { + setDetachable(detachable: boolean): void; + setParameter(name: string, value: T): void; + endpoints: Endpoint[]; + getOverlay(name: string): Overlay; + getParameters(): any; + sourceId: string; + targetId: string; +} + +interface Endpoint { + getParameters(): any; +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Assets/Scripts/etl-pipeline-editor.ts b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Assets/Scripts/etl-pipeline-editor.ts new file mode 100644 index 00000000000..72f0f630cfc --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Assets/Scripts/etl-pipeline-editor.ts @@ -0,0 +1,352 @@ +/// +/// + +class EtlPipelineEditor { + private jsPlumbInstance: jsPlumbInstance; + private container: HTMLElement; + private pipeline: EtlPipeline.Pipeline; + private localId: string; + + constructor( + container: HTMLElement, + pipeline: EtlPipeline.Pipeline, + deletePrompt: string, + localId: string, + loadLocalState: boolean + ) { + this.container = container; + this.pipeline = pipeline; + this.localId = localId; + + if (loadLocalState) { + const saved = this.loadLocalState(); + if (saved) { + this.pipeline = saved; + } + } + + jsPlumb.ready(() => { + this.init(); + }); + } + + public saveCurrentState(): void { + this.saveLocalState(); + } + + private init() { + const plumber = jsPlumb.getInstance({ + DragOptions: { cursor: 'pointer', zIndex: 2000 }, + ConnectionOverlays: [ + ['Arrow', { width: 12, length: 12, location: -5 }] + ], + Container: this.container + }); + + this.jsPlumbInstance = plumber; + + const activityElements = this.container.querySelectorAll('.activity'); + activityElements.forEach((el) => { + this.setupActivity(plumber, el); + }); + + this.updateConnections(plumber); + this.updateCanvasHeight(); + requestAnimationFrame(() => plumber.repaintEverything()); + + plumber.bind('connection', () => { + this.saveLocalState(); + this.updateCanvasHeight(); + }); + + plumber.bind('connectionDetached', () => { + this.saveLocalState(); + }); + } + + private setupActivity(plumber: jsPlumbInstance, el: HTMLElement) { + const activityId = el.getAttribute('data-activity-id'); + const activity = this.pipeline.activities.find(a => a.id === activityId); + + if (!activity) return; + + plumber.draggable(el, { + grid: [10, 10], + containment: true, + stop: () => { + this.updateCanvasHeight(); + this.saveLocalState(); + } + }); + + let color = '#7ab02c'; + if (activity.isTransform) color = '#3a8acd'; + if (activity.isLoad) color = '#6c757d'; + + if (activity.outcomes) { + activity.outcomes.forEach((outcome: EtlPipeline.Outcome) => { + plumber.addEndpoint(el, { + endpoint: 'Dot', + anchor: 'Right' as any, + paintStyle: { fill: color, radius: 7 }, + isSource: true, + connector: ['Flowchart', { stub: [40, 60], gap: 0, cornerRadius: 5, alwaysRespectStubs: true }], + connectorStyle: { strokeWidth: 2, stroke: '#999999', joinstyle: 'round', outlineStroke: 'white', outlineWidth: 2 }, + hoverPaintStyle: { fill: '#216477', stroke: '#216477' }, + connectorHoverStyle: { strokeWidth: 3, stroke: '#216477' }, + uuid: `${activityId}-${outcome.name}`, + parameters: { outcome: outcome }, + overlays: [ + ['Label', { + location: [0.5, 1.5], + label: outcome.displayName, + cssClass: 'outcome-label', + visible: true + }] + ] + } as any); + }); + } + + plumber.makeTarget(el, { + dropOptions: { hoverClass: 'hover' }, + anchor: 'Left' as any, + endpoint: ['Blank', { radius: 8 }] as any + }); + + this.setupActivityActions(el, activity); + } + + private setupActivityActions(el: HTMLElement, activity: EtlPipeline.Activity) { + el.addEventListener('click', (e: MouseEvent) => { + const target = e.target as HTMLElement; + if (target.closest('.activity-commands')) return; + + const commands = el.querySelector('.activity-commands'); + if (commands) { + commands.classList.toggle('d-none'); + } + }); + + const deleteBtn = el.querySelector('.activity-delete-action'); + if (deleteBtn) { + deleteBtn.addEventListener('click', () => { + if (!confirm('Are you sure you want to delete this activity?')) return; + + this.pipeline.removedActivities = this.pipeline.removedActivities || []; + this.pipeline.removedActivities.push(activity.id); + + this.jsPlumbInstance.remove(el); + this.saveLocalState(); + }); + } + } + + private updateConnections(plumber: jsPlumbInstance) { + plumber.batch(() => { + this.pipeline.transitions.forEach((t: EtlPipeline.Transition) => { + const sourceEndpoint = plumber.getEndpoint(`${t.sourceActivityId}-${t.sourceOutcomeName}`); + const targetEl = this.container.querySelector(`[data-activity-id="${t.destinationActivityId}"]`); + + if (sourceEndpoint && targetEl) { + plumber.connect({ + source: sourceEndpoint, + target: targetEl as any, + type: 'basic' + } as any); + } + }); + }); + } + + private updateCanvasHeight() { + let maxY = 400; + this.container.querySelectorAll('.activity').forEach((el) => { + const bottom = el.offsetTop + el.offsetHeight + 50; + if (bottom > maxY) maxY = bottom; + }); + this.container.style.minHeight = maxY + 'px'; + } + + public getState(): EtlPipeline.Pipeline { + const activities: EtlPipeline.Activity[] = []; + + this.container.querySelectorAll('.activity').forEach((el) => { + const id = el.getAttribute('data-activity-id'); + const existing = this.pipeline.activities.find(a => a.id === id); + if (existing) { + activities.push({ + ...existing, + x: el.offsetLeft, + y: el.offsetTop, + isStart: el.getAttribute('data-activity-start') === 'true' + }); + } + }); + + const transitions: EtlPipeline.Transition[] = []; + const connections = this.jsPlumbInstance.getConnections() as any[]; + connections.forEach((conn: any) => { + const sourceEndpoint = conn.endpoints[0]; + const outcome = sourceEndpoint.getParameter('outcome'); + const targetEl = conn.target; + transitions.push({ + sourceActivityId: conn.source.getAttribute('data-activity-id'), + destinationActivityId: targetEl.getAttribute('data-activity-id'), + sourceOutcomeName: outcome ? outcome.name : 'Done' + }); + }); + + return { + ...this.pipeline, + activities: activities, + transitions: transitions, + removedActivities: this.pipeline.removedActivities || [] + }; + } + + public serialize(): string { + return JSON.stringify(this.getState()); + } + + private saveLocalState() { + try { + sessionStorage.setItem(`etl-pipeline-${this.localId}`, this.serialize()); + } catch { + // Storage full or unavailable + } + } + + private loadLocalState(): EtlPipeline.Pipeline | null { + try { + const json = sessionStorage.getItem(`etl-pipeline-${this.localId}`); + return json ? JSON.parse(json) : null; + } catch { + return null; + } + } +} + +function initEtlPipelineEditor(): void { + const canvas = document.querySelector('.etl-canvas'); + if (!canvas) return; + + const pipelineData = canvas.dataset.pipeline; + if (!pipelineData) return; + + const pipeline: EtlPipeline.Pipeline = JSON.parse(pipelineData); + pipeline.removedActivities = pipeline.removedActivities || []; + + const deletePrompt = canvas.dataset.deleteActivityPrompt || 'Are you sure?'; + const localId = canvas.dataset.localId || ''; + const loadLocalState = canvas.dataset.loadLocalState === 'true'; + + const editor = new EtlPipelineEditor( + canvas, + pipeline, + deletePrompt, + localId, + loadLocalState + ); + + // Serialize state into hidden input on form submit + const form = document.getElementById('pipelineEditorForm') as HTMLFormElement | null; + const stateInput = document.getElementById('pipelineStateInput') as HTMLInputElement | null; + const persistPipeline = async (): Promise => { + if (!form || !stateInput) { + return true; + } + + stateInput.value = editor.serialize(); + editor.saveCurrentState(); + + const formData = new FormData(form); + + try { + const response = await fetch(form.action, { + method: 'POST', + body: formData, + credentials: 'same-origin', + headers: { + 'X-Requested-With': 'XMLHttpRequest' + } + }); + + return response.ok; + } catch { + return false; + } + }; + + if (form && stateInput) { + form.addEventListener('submit', () => { + stateInput.value = editor.serialize(); + }); + } + + const statePreservingLinks = document.querySelectorAll('.activity-add-action, .activity-edit-action'); + statePreservingLinks.forEach((link) => { + link.addEventListener('click', async (event: Event) => { + event.preventDefault(); + const succeeded = await persistPipeline(); + if (succeeded && link instanceof HTMLAnchorElement) { + window.location.href = link.href; + } + }); + }); + + // Activity picker modal category filtering + const pickerModal = document.getElementById('activity-picker'); + if (pickerModal) { + let activeActivityType = 'all'; + + const applyActivityFilters = (): void => { + const searchInput = pickerModal.querySelector('.activity-search'); + const query = (searchInput?.value || '').toLowerCase(); + const cards = pickerModal.querySelectorAll('.activity-card'); + + cards.forEach((card) => { + const cardTitle = card.querySelector('.card-title'); + const text = cardTitle ? cardTitle.textContent!.toLowerCase() : ''; + const cardType = card.dataset.activityType || 'all'; + const matchesType = activeActivityType === 'all' || cardType === activeActivityType; + const matchesQuery = !query || text.indexOf(query) >= 0; + + card.style.display = matchesType && matchesQuery ? '' : 'none'; + }); + }; + + const setActiveType = (activityType: string): void => { + activeActivityType = activityType || 'all'; + applyActivityFilters(); + }; + + pickerModal.addEventListener('show.bs.modal', (e: Event) => { + const modalEvent = e as any; + const button = modalEvent.relatedTarget as HTMLElement; + const activityType = button?.dataset.activityType || 'all'; + const title = button?.dataset.pickerTitle || 'Available Activities'; + + const modalTitle = pickerModal.querySelector('.modal-title'); + if (modalTitle) { + modalTitle.textContent = title; + } + + const searchInput = pickerModal.querySelector('.activity-search'); + if (searchInput) { + searchInput.value = ''; + } + + setActiveType(activityType); + }); + + const searchInput = pickerModal.querySelector('.activity-search'); + if (searchInput) { + searchInput.addEventListener('input', () => { + applyActivityFilters(); + }); + } + } +} + +document.addEventListener('DOMContentLoaded', initEtlPipelineEditor); diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Assets/Scripts/etl-pipeline-models.ts b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Assets/Scripts/etl-pipeline-models.ts new file mode 100644 index 00000000000..b4c79315912 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Assets/Scripts/etl-pipeline-models.ts @@ -0,0 +1,32 @@ +namespace EtlPipeline { + export interface Pipeline { + id: number; + name: string; + activities: Array; + transitions: Array; + removedActivities: Array; + } + + export interface Activity { + id: string; + x: number; + y: number; + name: string; + isStart: boolean; + isSource: boolean; + isTransform: boolean; + isLoad: boolean; + outcomes: Array; + } + + export interface Outcome { + name: string; + displayName: string; + } + + export interface Transition { + sourceActivityId: string; + destinationActivityId: string; + sourceOutcomeName: string; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Assets/Styles/etl-pipeline-editor.scss b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Assets/Styles/etl-pipeline-editor.scss new file mode 100644 index 00000000000..379599e7657 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Assets/Styles/etl-pipeline-editor.scss @@ -0,0 +1,156 @@ +.etl-container { + border: 1px solid var(--bs-border-color); + border-radius: 0.25rem; + overflow: hidden; +} + +.etl-canvas-container { + overflow: auto; + max-height: 700px; +} + +.etl-canvas { + position: relative; + min-height: 400px; + overflow: auto; + background-color: var(--bs-body-bg); + background-image: + linear-gradient(var(--bs-border-color-translucent) 1px, transparent 1px), + linear-gradient(90deg, var(--bs-border-color-translucent) 1px, transparent 1px); + background-size: 20px 20px; + + .activity { + position: absolute; + border: 1px solid #346789; + border-radius: 4px; + background-color: var(--bs-secondary-bg); + padding: 1em; + min-width: 120px; + min-height: 60px; + max-width: 250px; + opacity: 0.9; + cursor: move; + z-index: 20; + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.12); + transition: box-shadow 0.2s, opacity 0.2s; + + &:hover { + box-shadow: 2px 2px 12px rgba(0, 0, 0, 0.3); + opacity: 1; + } + + &.activity-source { + border-color: #7ab02c; + border-width: 2px; + + header h4 i { + color: #7ab02c; + } + } + + &.activity-transform { + border-color: #3a8acd; + border-width: 2px; + + header h4 i { + color: #3a8acd; + } + } + + &.activity-load { + border-color: #6c757d; + border-width: 2px; + + header h4 i { + color: #6c757d; + } + } + + &.activity-start { + border-width: 3px; + font-weight: bold; + } + + header { + h4 { + font-size: 0.9rem; + margin-bottom: 0.25rem; + + i { + margin-right: 0.3rem; + } + } + } + + .activity-commands { + position: absolute; + top: -30px; + right: 0; + white-space: nowrap; + + .btn { + border-radius: 50%; + width: 24px; + height: 24px; + padding: 0; + font-size: 0.7rem; + line-height: 24px; + } + } + } + + .jtk-endpoint { + z-index: 21; + cursor: pointer; + } + + .connection-label { + z-index: 31; + border: 1px solid var(--bs-border-color); + padding: 0 0.5rem; + border-radius: 2px; + background-color: var(--bs-secondary-bg); + font-size: 0.75rem; + } + + .outcome-label { + z-index: 21; + font-size: 10px; + font-weight: 500; + color: var(--bs-body-color); + background-color: var(--bs-body-bg); + border: 1px solid var(--bs-border-color); + padding: 0.1rem 0.4rem; + border-radius: 2px; + white-space: nowrap; + } +} + +// Activity picker modal +.modal-activities { + .activity-picker-categories { + gap: 0.25rem; + } + + .activity-card { + margin-bottom: 0.5rem; + + .card { + transition: border-color 0.2s, box-shadow 0.2s, transform 0.2s; + + &:hover { + border-color: var(--bs-primary); + box-shadow: 0 0.5rem 1rem rgba(0, 0, 0, 0.08); + transform: translateY(-1px); + } + } + + .card-title { + font-size: 0.9rem; + } + + .card-text { + font-size: 0.8rem; + } + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Assets/package.json b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Assets/package.json new file mode 100644 index 00000000000..ce2d23ad3a4 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Assets/package.json @@ -0,0 +1,10 @@ +{ + "name": "@orchardcore/etl", + "version": "1.0.0", + "dependencies": { + "@popperjs/core": "2.11.8", + "@types/bootstrap": "5.2.7", + "bootstrap": "5.3.8", + "jsplumb": "2.15.6" + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/BackgroundTasks/EtlPipelineBackgroundTask.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/BackgroundTasks/EtlPipelineBackgroundTask.cs new file mode 100644 index 00000000000..d8eab2cff37 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/BackgroundTasks/EtlPipelineBackgroundTask.cs @@ -0,0 +1,48 @@ +using OrchardCore.BackgroundTasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using OrchardCore.DataOrchestrator.Services; + +namespace OrchardCore.DataOrchestrator.BackgroundTasks; + +[BackgroundTask( + Schedule = "*/10 * * * *", + Title = "Data Pipeline Execution", + Description = "Executes enabled data pipelines on their configured schedule.")] +public sealed class EtlPipelineBackgroundTask : IBackgroundTask +{ + public async Task DoWorkAsync(IServiceProvider serviceProvider, CancellationToken cancellationToken) + { + var pipelineService = serviceProvider.GetRequiredService(); + var executor = serviceProvider.GetRequiredService(); + var logger = serviceProvider.GetRequiredService>(); + + var pipelines = await pipelineService.ListEnabledAsync(); + + foreach (var pipeline in pipelines) + { + if (cancellationToken.IsCancellationRequested) + { + break; + } + + if (string.IsNullOrEmpty(pipeline.Schedule)) + { + continue; + } + + try + { + var log = await executor.ExecuteAsync(pipeline, cancellationToken: cancellationToken); + await pipelineService.SaveLogAsync(log); + } + catch (Exception ex) + { + if (logger.IsEnabled(LogLevel.Error)) + { + logger.LogError(ex, "Error executing ETL pipeline '{PipelineName}'.", pipeline.Name); + } + } + } + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Controllers/AdminController.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Controllers/AdminController.cs new file mode 100644 index 00000000000..26e570f7cf6 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Controllers/AdminController.cs @@ -0,0 +1,626 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Localization; +using Microsoft.Extensions.Localization; +using OrchardCore.Admin; +using OrchardCore.DataOrchestrator.Activities; +using OrchardCore.DataOrchestrator.Models; +using OrchardCore.DataOrchestrator.Services; +using OrchardCore.DataOrchestrator.ViewModels; +using OrchardCore.DisplayManagement.ModelBinding; + +namespace OrchardCore.DataOrchestrator.Controllers; + +[Admin] +public sealed class AdminController : Controller +{ + private const string AdminPath = "DataPipelines"; + + private static readonly JsonSerializerOptions _camelCaseOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + }; + + private readonly IEtlPipelineService _pipelineService; + private readonly IEtlPipelineExecutor _executor; + private readonly IEtlActivityLibrary _activityLibrary; + private readonly IEtlActivityDisplayManager _activityDisplayManager; + private readonly IAuthorizationService _authorizationService; + private readonly IUpdateModelAccessor _updateModelAccessor; + + internal readonly IStringLocalizer S; + internal readonly IHtmlLocalizer H; + + public AdminController( + IEtlPipelineService pipelineService, + IEtlPipelineExecutor executor, + IEtlActivityLibrary activityLibrary, + IEtlActivityDisplayManager activityDisplayManager, + IAuthorizationService authorizationService, + IUpdateModelAccessor updateModelAccessor, + IStringLocalizer stringLocalizer, + IHtmlLocalizer htmlLocalizer) + { + _pipelineService = pipelineService; + _executor = executor; + _activityLibrary = activityLibrary; + _activityDisplayManager = activityDisplayManager; + _authorizationService = authorizationService; + _updateModelAccessor = updateModelAccessor; + S = stringLocalizer; + H = htmlLocalizer; + } + + [Admin(AdminPath, "EtlPipelines")] + public async Task Index(string q = null) + { + if (!await _authorizationService.AuthorizeAsync(User, EtlPermissions.ViewEtlPipelines)) + { + return Forbid(); + } + + var pipelines = (await _pipelineService.ListAsync()) + .OrderBy(x => x.Name) + .AsEnumerable(); + + if (!string.IsNullOrWhiteSpace(q)) + { + pipelines = pipelines.Where(x => + (!string.IsNullOrWhiteSpace(x.Name) && x.Name.Contains(q, StringComparison.OrdinalIgnoreCase)) || + (!string.IsNullOrWhiteSpace(x.Description) && x.Description.Contains(q, StringComparison.OrdinalIgnoreCase))); + } + + var viewModel = new EtlPipelineListViewModel + { + Pipelines = pipelines.ToList(), + Search = q, + }; + + return View(viewModel); + } + + [Admin(AdminPath + "/Create", "EtlPipelineCreate")] + public async Task Create() + { + if (!await _authorizationService.AuthorizeAsync(User, EtlPermissions.ManageEtlPipelines)) + { + return Forbid(); + } + + var viewModel = new EtlPipelinePropertiesViewModel(); + + return View(viewModel); + } + + [HttpPost] + [Admin(AdminPath + "/Create", "EtlPipelineCreate")] + public async Task Create(EtlPipelinePropertiesViewModel viewModel) + { + if (!await _authorizationService.AuthorizeAsync(User, EtlPermissions.ManageEtlPipelines)) + { + return Forbid(); + } + + if (!ModelState.IsValid) + { + return View(viewModel); + } + + var pipeline = new EtlPipelineDefinition + { + PipelineId = IdGenerator.GenerateId(), + Name = viewModel.Name, + Description = viewModel.Description, + IsEnabled = viewModel.IsEnabled, + Schedule = viewModel.Schedule, + }; + + await _pipelineService.SaveAsync(pipeline); + + return RedirectToAction(nameof(Edit), new { id = pipeline.Id }); + } + + [Admin(AdminPath + "/Edit/{id}", "EtlPipelineEdit")] + public async Task Edit(long id, string localId) + { + if (!await _authorizationService.AuthorizeAsync(User, EtlPermissions.ManageEtlPipelines)) + { + return Forbid(); + } + + var pipeline = await _pipelineService.GetByDocumentIdAsync(id); + if (pipeline == null) + { + return NotFound(); + } + + var newLocalId = string.IsNullOrWhiteSpace(localId) ? Guid.NewGuid().ToString() : localId; + var availableActivities = _activityLibrary.ListActivities(); + + var activityThumbnailShapes = new List(); + foreach (var activity in availableActivities) + { + activityThumbnailShapes.Add(await BuildActivityDisplayAsync(activity, pipeline.Id, newLocalId, "Thumbnail")); + } + + var activityDesignShapes = new List(); + foreach (var activityRecord in pipeline.Activities) + { + var activity = _activityLibrary.InstantiateActivity(activityRecord.Name); + if (activity != null) + { + activity.Properties = activityRecord.Properties?.DeepClone() as JsonObject ?? []; + activityDesignShapes.Add(await BuildActivityDisplayAsync(activity, activityRecord, pipeline.Id, newLocalId, "Design")); + } + } + + var activitiesData = pipeline.Activities.Select(a => + { + var activity = _activityLibrary.GetActivityByName(a.Name); + return new + { + id = a.ActivityId, + x = a.X, + y = a.Y, + name = a.Name, + isStart = a.IsStart, + isSource = activity is EtlSourceActivity, + isTransform = activity is EtlTransformActivity, + isLoad = activity is EtlLoadActivity, + outcomes = activity?.GetPossibleOutcomes() + .Select(o => new { name = o.Name, displayName = o.DisplayName }) + .ToArray() ?? [], + }; + }).ToList(); + + var pipelineData = new + { + id = pipeline.Id, + name = pipeline.Name, + activities = activitiesData, + transitions = pipeline.Transitions.Select(t => new + { + sourceActivityId = t.SourceActivityId, + destinationActivityId = t.DestinationActivityId, + sourceOutcomeName = t.SourceOutcomeName, + }), + }; + + var viewModel = new EtlPipelineEditorViewModel + { + Pipeline = pipeline, + PipelineJson = JsonSerializer.Serialize(pipelineData, _camelCaseOptions), + ActivityThumbnailShapes = activityThumbnailShapes, + ActivityDesignShapes = activityDesignShapes, + ActivityCategories = _activityLibrary.ListCategories().ToList(), + LocalId = newLocalId, + LoadLocalState = !string.IsNullOrWhiteSpace(localId), + }; + + return View(viewModel); + } + + [HttpPost] + [Admin(AdminPath + "/Edit/{id}", "EtlPipelineEdit")] + public async Task Edit(EtlPipelineEditorUpdateModel model, string submitAction) + { + if (!await _authorizationService.AuthorizeAsync(User, EtlPermissions.ManageEtlPipelines)) + { + return Forbid(); + } + + var pipeline = await _pipelineService.GetByDocumentIdAsync(model.Id); + if (pipeline == null) + { + return NotFound(); + } + + var state = JsonNode.Parse(model.State); + var activities = state["activities"]?.AsArray(); + var transitions = state["transitions"]?.AsArray(); + + if (activities != null) + { + foreach (var activityState in activities) + { + var activityId = activityState["id"]?.GetValue(); + var existing = pipeline.Activities.FirstOrDefault(a => a.ActivityId == activityId); + if (existing != null) + { + existing.X = (int)Math.Round(Convert.ToDecimal(activityState["x"]?.GetValue() ?? 0)); + existing.Y = (int)Math.Round(Convert.ToDecimal(activityState["y"]?.GetValue() ?? 0)); + existing.IsStart = activityState["isStart"]?.GetValue() ?? false; + } + } + } + + if (transitions != null) + { + pipeline.Transitions.Clear(); + foreach (var transitionState in transitions) + { + pipeline.Transitions.Add(new EtlTransition + { + SourceActivityId = transitionState["sourceActivityId"]?.GetValue(), + DestinationActivityId = transitionState["destinationActivityId"]?.GetValue(), + SourceOutcomeName = transitionState["sourceOutcomeName"]?.GetValue(), + }); + } + } + + var removedActivities = state["removedActivities"]?.AsArray(); + if (removedActivities != null) + { + foreach (var removedId in removedActivities) + { + var actId = removedId?.GetValue(); + var activity = pipeline.Activities.FirstOrDefault(a => a.ActivityId == actId); + if (activity != null) + { + pipeline.Activities.Remove(activity); + } + } + } + + await _pipelineService.SaveAsync(pipeline); + + if (string.Equals(submitAction, "run", StringComparison.OrdinalIgnoreCase)) + { + if (!await _authorizationService.AuthorizeAsync(User, EtlPermissions.ExecuteEtlPipelines)) + { + return Forbid(); + } + + var log = await _executor.ExecuteAsync(pipeline); + await _pipelineService.SaveLogAsync(log); + + return RedirectToAction(nameof(Logs), new { id = pipeline.Id }); + } + + return RedirectToAction(nameof(Edit), new { id = pipeline.Id }); + } + + [Admin(AdminPath + "/EditProperties/{id}", "EtlPipelineEditProperties")] + public async Task EditProperties(long id) + { + if (!await _authorizationService.AuthorizeAsync(User, EtlPermissions.ManageEtlPipelines)) + { + return Forbid(); + } + + var pipeline = await _pipelineService.GetByDocumentIdAsync(id); + if (pipeline == null) + { + return NotFound(); + } + + var viewModel = new EtlPipelinePropertiesViewModel + { + Name = pipeline.Name, + Description = pipeline.Description, + IsEnabled = pipeline.IsEnabled, + Schedule = pipeline.Schedule, + }; + + return View(viewModel); + } + + [HttpPost] + [Admin(AdminPath + "/EditProperties/{id}", "EtlPipelineEditProperties")] + public async Task EditProperties(long id, EtlPipelinePropertiesViewModel viewModel) + { + if (!await _authorizationService.AuthorizeAsync(User, EtlPermissions.ManageEtlPipelines)) + { + return Forbid(); + } + + var pipeline = await _pipelineService.GetByDocumentIdAsync(id); + if (pipeline == null) + { + return NotFound(); + } + + if (!ModelState.IsValid) + { + return View(viewModel); + } + + pipeline.Name = viewModel.Name; + pipeline.Description = viewModel.Description; + pipeline.IsEnabled = viewModel.IsEnabled; + pipeline.Schedule = viewModel.Schedule; + + await _pipelineService.SaveAsync(pipeline); + + return RedirectToAction(nameof(Edit), new { id = pipeline.Id }); + } + + [HttpPost] + [Admin(AdminPath + "/Delete/{id}", "EtlPipelineDelete")] + public async Task Delete(long id) + { + if (!await _authorizationService.AuthorizeAsync(User, EtlPermissions.ManageEtlPipelines)) + { + return Forbid(); + } + + var pipeline = await _pipelineService.GetByDocumentIdAsync(id); + if (pipeline == null) + { + return NotFound(); + } + + await _pipelineService.DeleteAsync(pipeline.PipelineId); + + return RedirectToAction(nameof(Index)); + } + + [HttpPost] + [Admin(AdminPath + "/Execute/{id}", "EtlPipelineExecute")] + public async Task Execute(long id) + { + if (!await _authorizationService.AuthorizeAsync(User, EtlPermissions.ExecuteEtlPipelines)) + { + return Forbid(); + } + + var pipeline = await _pipelineService.GetByDocumentIdAsync(id); + if (pipeline == null) + { + return NotFound(); + } + + var log = await _executor.ExecuteAsync(pipeline); + await _pipelineService.SaveLogAsync(log); + + return RedirectToAction(nameof(Logs), new { id = pipeline.Id }); + } + + [Admin(AdminPath + "/Logs/{id}", "EtlPipelineLogs")] + public async Task Logs(long id) + { + if (!await _authorizationService.AuthorizeAsync(User, EtlPermissions.ViewEtlPipelines)) + { + return Forbid(); + } + + var pipeline = await _pipelineService.GetByDocumentIdAsync(id); + if (pipeline == null) + { + return NotFound(); + } + + var logs = await _pipelineService.GetLogsAsync(pipeline.PipelineId); + + var viewModel = new EtlExecutionLogListViewModel + { + Pipeline = pipeline, + Logs = logs.OrderByDescending(l => l.StartedUtc).ToList(), + }; + + return View(viewModel); + } + + [Admin(AdminPath + "/Activity/Create", "EtlActivityCreate")] + public async Task CreateActivity(long pipelineId, string activityName, string returnUrl) + { + if (!await _authorizationService.AuthorizeAsync(User, EtlPermissions.ManageEtlPipelines)) + { + return Forbid(); + } + + var activity = _activityLibrary.InstantiateActivity(activityName); + if (activity == null) + { + return NotFound(); + } + + var updater = _updateModelAccessor.ModelUpdater; + var shape = await _activityDisplayManager.BuildEditorAsync(activity, updater, isNew: true); + shape.Metadata.Type = "EtlActivity_Edit"; + + var viewModel = new EtlActivityEditorViewModel + { + PipelineId = pipelineId, + ActivityName = activityName, + Activity = activity, + Editor = shape, + ReturnUrl = returnUrl, + }; + + return View(viewModel); + } + + [HttpPost] + [Admin(AdminPath + "/Activity/Create", "EtlActivityCreate")] + public async Task CreateActivity(EtlActivityEditorPostModel model) + { + if (!await _authorizationService.AuthorizeAsync(User, EtlPermissions.ManageEtlPipelines)) + { + return Forbid(); + } + + var pipeline = await _pipelineService.GetByDocumentIdAsync(model.PipelineId); + if (pipeline == null) + { + return NotFound(); + } + + var activity = _activityLibrary.InstantiateActivity(model.ActivityName); + if (activity == null) + { + return NotFound(); + } + + var updater = _updateModelAccessor.ModelUpdater; + var shape = await _activityDisplayManager.UpdateEditorAsync(activity, updater, isNew: true); + + if (!ModelState.IsValid) + { + shape.Metadata.Type = "EtlActivity_Edit"; + var viewModel = new EtlActivityEditorViewModel + { + PipelineId = model.PipelineId, + ActivityName = model.ActivityName, + Activity = activity, + Editor = shape, + ReturnUrl = model.ReturnUrl, + }; + return View(viewModel); + } + + var activityRecord = new EtlActivityRecord + { + ActivityId = Guid.NewGuid().ToString("n"), + Name = model.ActivityName, + Properties = activity.Properties, + X = 100, + Y = 100 + pipeline.Activities.Count * 120, + IsStart = pipeline.Activities.Count == 0, + }; + + pipeline.Activities.Add(activityRecord); + await _pipelineService.SaveAsync(pipeline); + + if (Url.IsLocalUrl(model.ReturnUrl)) + { + return this.Redirect(model.ReturnUrl, true); + } + + return RedirectToAction(nameof(Edit), new { id = pipeline.Id }); + } + + [Admin(AdminPath + "/Activity/Edit", "EtlActivityEdit")] + public async Task EditActivity(long pipelineId, string activityId, string returnUrl) + { + if (!await _authorizationService.AuthorizeAsync(User, EtlPermissions.ManageEtlPipelines)) + { + return Forbid(); + } + + var pipeline = await _pipelineService.GetByDocumentIdAsync(pipelineId); + if (pipeline == null) + { + return NotFound(); + } + + var activityRecord = pipeline.Activities.FirstOrDefault(a => a.ActivityId == activityId); + if (activityRecord == null) + { + return NotFound(); + } + + var activity = _activityLibrary.InstantiateActivity(activityRecord.Name); + if (activity == null) + { + return NotFound(); + } + + activity.Properties = activityRecord.Properties?.DeepClone() as JsonObject ?? []; + var updater = _updateModelAccessor.ModelUpdater; + var shape = await _activityDisplayManager.BuildEditorAsync(activity, updater, isNew: false); + shape.Metadata.Type = "EtlActivity_Edit"; + + var viewModel = new EtlActivityEditorViewModel + { + PipelineId = pipelineId, + ActivityId = activityId, + ActivityName = activityRecord.Name, + Activity = activity, + Editor = shape, + ReturnUrl = returnUrl, + }; + + return View("CreateActivity", viewModel); + } + + [HttpPost] + [Admin(AdminPath + "/Activity/Edit", "EtlActivityEdit")] + public async Task EditActivity(EtlActivityEditorPostModel model) + { + if (!await _authorizationService.AuthorizeAsync(User, EtlPermissions.ManageEtlPipelines)) + { + return Forbid(); + } + + var pipeline = await _pipelineService.GetByDocumentIdAsync(model.PipelineId); + if (pipeline == null) + { + return NotFound(); + } + + var activityRecord = pipeline.Activities.FirstOrDefault(a => a.ActivityId == model.ActivityId); + if (activityRecord == null) + { + return NotFound(); + } + + var activity = _activityLibrary.InstantiateActivity(activityRecord.Name); + if (activity == null) + { + return NotFound(); + } + + activity.Properties = activityRecord.Properties?.DeepClone() as JsonObject ?? []; + var updater = _updateModelAccessor.ModelUpdater; + var shape = await _activityDisplayManager.UpdateEditorAsync(activity, updater, isNew: false); + + if (!ModelState.IsValid) + { + shape.Metadata.Type = "EtlActivity_Edit"; + var viewModel = new EtlActivityEditorViewModel + { + PipelineId = model.PipelineId, + ActivityId = model.ActivityId, + ActivityName = activityRecord.Name, + Activity = activity, + Editor = shape, + ReturnUrl = model.ReturnUrl, + }; + return View("CreateActivity", viewModel); + } + + activityRecord.Properties = activity.Properties; + await _pipelineService.SaveAsync(pipeline); + + if (Url.IsLocalUrl(model.ReturnUrl)) + { + return this.Redirect(model.ReturnUrl, true); + } + + return RedirectToAction(nameof(Edit), new { id = pipeline.Id }); + } + + private async Task BuildActivityDisplayAsync(IEtlActivity activity, long pipelineId, string localId, string displayType) + { + var activityShape = await _activityDisplayManager.BuildDisplayAsync(activity, _updateModelAccessor.ModelUpdater, displayType); + activityShape.Metadata.Type = $"EtlActivity_{displayType}"; + activityShape.Properties["Activity"] = activity; + activityShape.Properties["PipelineId"] = pipelineId; + activityShape.Properties["ReturnUrl"] = Url.Action(nameof(Edit), new + { + id = pipelineId, + localId, + }); + + return activityShape; + } + + private async Task BuildActivityDisplayAsync(IEtlActivity activity, EtlActivityRecord activityRecord, long pipelineId, string localId, string displayType) + { + var activityShape = await _activityDisplayManager.BuildDisplayAsync(activity, _updateModelAccessor.ModelUpdater, displayType); + activityShape.Metadata.Type = $"EtlActivity_{displayType}"; + activityShape.Properties["Activity"] = activity; + activityShape.Properties["ActivityRecord"] = activityRecord; + activityShape.Properties["PipelineId"] = pipelineId; + activityShape.Properties["ReturnUrl"] = Url.Action(nameof(Edit), new + { + id = pipelineId, + localId, + }); + + return activityShape; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/ContentItemLoadDisplayDriver.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/ContentItemLoadDisplayDriver.cs new file mode 100644 index 00000000000..4979999738a --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/ContentItemLoadDisplayDriver.cs @@ -0,0 +1,18 @@ +using OrchardCore.DataOrchestrator.Activities; +using OrchardCore.DataOrchestrator.Display; +using OrchardCore.DataOrchestrator.ViewModels; + +namespace OrchardCore.DataOrchestrator.Drivers; + +public sealed class ContentItemLoadDisplayDriver : EtlActivityDisplayDriver +{ + protected override void EditActivity(ContentItemLoad activity, ContentItemLoadViewModel model) + { + model.ContentType = activity.ContentType; + } + + protected override void UpdateActivity(ContentItemLoadViewModel model, ContentItemLoad activity) + { + activity.ContentType = model.ContentType; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/ContentItemSourceDisplayDriver.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/ContentItemSourceDisplayDriver.cs new file mode 100644 index 00000000000..de27ede1bc6 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/ContentItemSourceDisplayDriver.cs @@ -0,0 +1,38 @@ +using Microsoft.AspNetCore.Mvc.Rendering; +using OrchardCore.ContentManagement.Metadata; +using OrchardCore.DataOrchestrator.Activities; +using OrchardCore.DataOrchestrator.Display; +using OrchardCore.DataOrchestrator.ViewModels; + +namespace OrchardCore.DataOrchestrator.Drivers; + +public sealed class ContentItemSourceDisplayDriver : EtlActivityDisplayDriver +{ + private readonly IContentDefinitionManager _contentDefinitionManager; + + public ContentItemSourceDisplayDriver(IContentDefinitionManager contentDefinitionManager) + { + _contentDefinitionManager = contentDefinitionManager; + } + + protected override async ValueTask EditActivityAsync(ContentItemSource activity, ContentItemSourceViewModel model) + { + model.AvailableContentTypes = (await _contentDefinitionManager.ListTypeDefinitionsAsync()) + .Select(x => new SelectListItem { Text = x.DisplayName, Value = x.Name }) + .ToList(); + model.ContentType = activity.ContentType; + model.VersionScope = activity.VersionScope; + model.Owner = activity.Owner; + model.CreatedUtcFrom = activity.CreatedUtcFrom; + model.CreatedUtcTo = activity.CreatedUtcTo; + } + + protected override void UpdateActivity(ContentItemSourceViewModel model, ContentItemSource activity) + { + activity.ContentType = model.ContentType; + activity.VersionScope = model.VersionScope; + activity.Owner = model.Owner; + activity.CreatedUtcFrom = model.CreatedUtcFrom; + activity.CreatedUtcTo = model.CreatedUtcTo; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/ExcelExportLoadDisplayDriver.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/ExcelExportLoadDisplayDriver.cs new file mode 100644 index 00000000000..36df647e56d --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/ExcelExportLoadDisplayDriver.cs @@ -0,0 +1,20 @@ +using OrchardCore.DataOrchestrator.Activities; +using OrchardCore.DataOrchestrator.Display; +using OrchardCore.DataOrchestrator.ViewModels; + +namespace OrchardCore.DataOrchestrator.Drivers; + +public sealed class ExcelExportLoadDisplayDriver : EtlActivityDisplayDriver +{ + protected override void EditActivity(ExcelExportLoad activity, ExcelExportLoadViewModel model) + { + model.FileName = activity.FileName; + model.WorksheetName = activity.WorksheetName; + } + + protected override void UpdateActivity(ExcelExportLoadViewModel model, ExcelExportLoad activity) + { + activity.FileName = model.FileName; + activity.WorksheetName = model.WorksheetName; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/ExcelSourceDisplayDriver.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/ExcelSourceDisplayDriver.cs new file mode 100644 index 00000000000..4cb800b918b --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/ExcelSourceDisplayDriver.cs @@ -0,0 +1,22 @@ +using OrchardCore.DataOrchestrator.Activities; +using OrchardCore.DataOrchestrator.Display; +using OrchardCore.DataOrchestrator.ViewModels; + +namespace OrchardCore.DataOrchestrator.Drivers; + +public sealed class ExcelSourceDisplayDriver : EtlActivityDisplayDriver +{ + protected override void EditActivity(ExcelSource activity, ExcelSourceViewModel model) + { + model.FilePath = activity.FilePath; + model.WorksheetName = activity.WorksheetName; + model.HasHeaderRow = activity.HasHeaderRow; + } + + protected override void UpdateActivity(ExcelSourceViewModel model, ExcelSource activity) + { + activity.FilePath = model.FilePath; + activity.WorksheetName = model.WorksheetName; + activity.HasHeaderRow = model.HasHeaderRow; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/FieldMappingTransformDisplayDriver.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/FieldMappingTransformDisplayDriver.cs new file mode 100644 index 00000000000..84c0f12500f --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/FieldMappingTransformDisplayDriver.cs @@ -0,0 +1,266 @@ +using System.Text.Json.Nodes; +using DocumentFormat.OpenXml; +using DocumentFormat.OpenXml.Packaging; +using DocumentFormat.OpenXml.Spreadsheet; +using Microsoft.AspNetCore.Http; +using OrchardCore.ContentManagement.Metadata; +using OrchardCore.DataOrchestrator.Activities; +using OrchardCore.DataOrchestrator.Display; +using OrchardCore.DataOrchestrator.Services; +using OrchardCore.DataOrchestrator.ViewModels; +using OrchardCore.FileStorage; +using OrchardCore.Media; + +namespace OrchardCore.DataOrchestrator.Drivers; + +public sealed class FieldMappingTransformDisplayDriver : EtlActivityDisplayDriver +{ + private readonly IHttpContextAccessor _httpContextAccessor; + private readonly IEtlPipelineService _pipelineService; + private readonly IContentDefinitionManager _contentDefinitionManager; + private readonly IMediaFileStore _mediaFileStore; + + public FieldMappingTransformDisplayDriver( + IHttpContextAccessor httpContextAccessor, + IEtlPipelineService pipelineService, + IContentDefinitionManager contentDefinitionManager, + IMediaFileStore mediaFileStore) + { + _httpContextAccessor = httpContextAccessor; + _pipelineService = pipelineService; + _contentDefinitionManager = contentDefinitionManager; + _mediaFileStore = mediaFileStore; + } + + protected override async ValueTask EditActivityAsync(FieldMappingTransform activity, FieldMappingTransformViewModel model) + { + model.AvailableSourceFields = (await GetAvailableSourceFieldsAsync()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(x => x) + .ToList(); + model.MappingsJson = activity.MappingsJson; + } + + protected override void UpdateActivity(FieldMappingTransformViewModel model, FieldMappingTransform activity) + { + activity.MappingsJson = model.MappingsJson; + } + + private async Task> GetAvailableSourceFieldsAsync() + { + if (!long.TryParse(_httpContextAccessor.HttpContext?.Request.Query["pipelineId"], out var pipelineId)) + { + return []; + } + + var pipeline = await _pipelineService.GetByDocumentIdAsync(pipelineId); + if (pipeline == null) + { + return []; + } + + var fields = new List(); + + foreach (var activityRecord in pipeline.Activities) + { + switch (activityRecord.Name) + { + case nameof(ContentItemSource): + fields.AddRange(await GetContentItemFieldsAsync(activityRecord.Properties)); + break; + case nameof(JsonSource): + fields.AddRange(GetJsonFields(activityRecord.Properties)); + break; + case nameof(ExcelSource): + fields.AddRange(await GetExcelFieldsAsync(activityRecord.Properties)); + break; + } + } + + return fields; + } + + private async Task> GetContentItemFieldsAsync(JsonObject properties) + { + var fields = new List + { + "ContentItemId", + "ContentItemVersionId", + "ContentType", + "DisplayText", + "Owner", + "CreatedUtc", + "ModifiedUtc", + "PublishedUtc", + "Published", + "Latest", + }; + + var contentType = properties?["ContentType"]?.GetValue(); + if (string.IsNullOrWhiteSpace(contentType)) + { + return fields; + } + + var typeDefinition = await _contentDefinitionManager.GetTypeDefinitionAsync(contentType); + if (typeDefinition == null) + { + return fields; + } + + foreach (var part in typeDefinition.Parts) + { + fields.Add(part.Name); + + foreach (var field in part.PartDefinition.Fields) + { + fields.Add($"{part.Name}.{field.Name}"); + } + } + + return fields; + } + + private static List GetJsonFields(JsonObject properties) + { + var data = properties?["Data"]?.GetValue(); + if (string.IsNullOrWhiteSpace(data)) + { + return []; + } + + try + { + if (JsonNode.Parse(data) is not JsonArray array || array.Count == 0 || array[0] is not JsonObject firstObject) + { + return []; + } + + var fields = new List(); + CollectJsonPaths(firstObject, null, fields); + return fields; + } + catch + { + return []; + } + } + + private async Task> GetExcelFieldsAsync(JsonObject properties) + { + var filePath = properties?["FilePath"]?.GetValue(); + if (string.IsNullOrWhiteSpace(filePath)) + { + return []; + } + + var normalizedPath = _mediaFileStore.NormalizePath(filePath); + var fileInfo = await _mediaFileStore.GetFileInfoAsync(normalizedPath); + if (fileInfo == null) + { + return []; + } + + await using var stream = await _mediaFileStore.GetFileStreamAsync(fileInfo); + using var spreadsheetDocument = SpreadsheetDocument.Open(stream, false); + var workbookPart = spreadsheetDocument.WorkbookPart; + + if (workbookPart?.Workbook == null) + { + return []; + } + + var worksheetName = properties?["WorksheetName"]?.GetValue(); + var sheet = string.IsNullOrWhiteSpace(worksheetName) + ? workbookPart.Workbook.Descendants().FirstOrDefault() + : workbookPart.Workbook.Descendants().FirstOrDefault(x => string.Equals(x.Name?.Value, worksheetName, StringComparison.OrdinalIgnoreCase)); + + if (sheet == null) + { + return []; + } + + var worksheetPart = (WorksheetPart)workbookPart.GetPartById(sheet.Id); + var sheetData = worksheetPart.Worksheet.GetFirstChild(); + var headerRow = sheetData?.Elements().FirstOrDefault(); + if (headerRow == null) + { + return []; + } + + var sharedStringTable = workbookPart.GetPartsOfType().FirstOrDefault()?.SharedStringTable; + return GetRowValues(headerRow, sharedStringTable) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .Select(x => x.Trim()); + } + + private static void CollectJsonPaths(JsonObject obj, string prefix, IList fields) + { + foreach (var property in obj) + { + var path = string.IsNullOrWhiteSpace(prefix) ? property.Key : $"{prefix}.{property.Key}"; + + if (property.Value is JsonObject childObject) + { + CollectJsonPaths(childObject, path, fields); + } + else + { + fields.Add(path); + } + } + } + + private static List GetRowValues(Row row, SharedStringTable sharedStringTable) + { + var values = new List(); + + foreach (var cell in row.Elements()) + { + var columnIndex = GetColumnIndexFromCellReference(cell.CellReference); + + while (values.Count <= columnIndex) + { + values.Add(string.Empty); + } + + values[columnIndex] = GetCellValue(cell, sharedStringTable); + } + + return values; + } + + private static int GetColumnIndexFromCellReference(StringValue cellReference) + { + var reference = cellReference?.Value ?? string.Empty; + var columnName = new string(reference.TakeWhile(char.IsLetter).ToArray()); + var columnIndex = 0; + + foreach (var ch in columnName) + { + columnIndex *= 26; + columnIndex += ch - 'A' + 1; + } + + return Math.Max(columnIndex - 1, 0); + } + + private static string GetCellValue(Cell cell, SharedStringTable sharedStringTable) + { + if (cell.CellValue == null) + { + return string.Empty; + } + + var value = cell.CellValue.InnerText; + + if (cell.DataType?.Value == CellValues.SharedString && + int.TryParse(value, out var sharedStringIndex) && + sharedStringTable?.ElementAtOrDefault(sharedStringIndex) is SharedStringItem sharedStringItem) + { + return sharedStringItem.InnerText; + } + + return value; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/FilterTransformDisplayDriver.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/FilterTransformDisplayDriver.cs new file mode 100644 index 00000000000..a4df07613d0 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/FilterTransformDisplayDriver.cs @@ -0,0 +1,22 @@ +using OrchardCore.DataOrchestrator.Activities; +using OrchardCore.DataOrchestrator.Display; +using OrchardCore.DataOrchestrator.ViewModels; + +namespace OrchardCore.DataOrchestrator.Drivers; + +public sealed class FilterTransformDisplayDriver : EtlActivityDisplayDriver +{ + protected override void EditActivity(FilterTransform activity, FilterTransformViewModel model) + { + model.Field = activity.Field; + model.Operator = activity.Operator; + model.Value = activity.Value; + } + + protected override void UpdateActivity(FilterTransformViewModel model, FilterTransform activity) + { + activity.Field = model.Field; + activity.Operator = model.Operator; + activity.Value = model.Value; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/FormatValueTransformDisplayDriver.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/FormatValueTransformDisplayDriver.cs new file mode 100644 index 00000000000..968fa1ea6b1 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/FormatValueTransformDisplayDriver.cs @@ -0,0 +1,28 @@ +using OrchardCore.DataOrchestrator.Activities; +using OrchardCore.DataOrchestrator.Display; +using OrchardCore.DataOrchestrator.ViewModels; + +namespace OrchardCore.DataOrchestrator.Drivers; + +public sealed class FormatValueTransformDisplayDriver : EtlActivityDisplayDriver +{ + protected override void EditActivity(FormatValueTransform activity, FormatValueTransformViewModel model) + { + model.Field = activity.Field; + model.OutputField = activity.OutputField; + model.FormatType = activity.FormatType; + model.FormatString = activity.FormatString; + model.Culture = activity.Culture; + model.TimeZoneId = activity.TimeZoneId; + } + + protected override void UpdateActivity(FormatValueTransformViewModel model, FormatValueTransform activity) + { + activity.Field = model.Field; + activity.OutputField = model.OutputField; + activity.FormatType = model.FormatType; + activity.FormatString = model.FormatString; + activity.Culture = model.Culture; + activity.TimeZoneId = model.TimeZoneId; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/JoinDataSetsTransformDisplayDriver.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/JoinDataSetsTransformDisplayDriver.cs new file mode 100644 index 00000000000..af812d48eb6 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/JoinDataSetsTransformDisplayDriver.cs @@ -0,0 +1,58 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc.Rendering; +using OrchardCore.DataOrchestrator.Activities; +using OrchardCore.DataOrchestrator.Display; +using OrchardCore.DataOrchestrator.Services; +using OrchardCore.DataOrchestrator.ViewModels; + +namespace OrchardCore.DataOrchestrator.Drivers; + +public sealed class JoinDataSetsTransformDisplayDriver : EtlActivityDisplayDriver +{ + private readonly IHttpContextAccessor _httpContextAccessor; + private readonly IEtlPipelineService _pipelineService; + + public JoinDataSetsTransformDisplayDriver( + IHttpContextAccessor httpContextAccessor, + IEtlPipelineService pipelineService) + { + _httpContextAccessor = httpContextAccessor; + _pipelineService = pipelineService; + } + + protected override async ValueTask EditActivityAsync(JoinDataSetsTransform activity, JoinDataSetsTransformViewModel model) + { + if (long.TryParse(_httpContextAccessor.HttpContext?.Request.Query["pipelineId"], out var pipelineId)) + { + var pipeline = await _pipelineService.GetByDocumentIdAsync(pipelineId); + if (pipeline != null) + { + model.AvailableJoinSources = pipeline.Activities + .Where(x => string.Equals(x.Name, nameof(ContentItemSource), StringComparison.Ordinal) || + string.Equals(x.Name, nameof(JsonSource), StringComparison.Ordinal) || + string.Equals(x.Name, nameof(ExcelSource), StringComparison.Ordinal)) + .Select(x => new SelectListItem + { + Value = x.ActivityId, + Text = $"{x.Name} ({x.ActivityId[..Math.Min(8, x.ActivityId.Length)]})", + }) + .ToList(); + } + } + + model.JoinSourceActivityId = activity.JoinSourceActivityId; + model.LeftField = activity.LeftField; + model.RightField = activity.RightField; + model.JoinType = activity.JoinType; + model.RightPrefix = activity.RightPrefix; + } + + protected override void UpdateActivity(JoinDataSetsTransformViewModel model, JoinDataSetsTransform activity) + { + activity.JoinSourceActivityId = model.JoinSourceActivityId; + activity.LeftField = model.LeftField; + activity.RightField = model.RightField; + activity.JoinType = model.JoinType; + activity.RightPrefix = model.RightPrefix; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/JsonExportLoadDisplayDriver.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/JsonExportLoadDisplayDriver.cs new file mode 100644 index 00000000000..3345f254d66 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/JsonExportLoadDisplayDriver.cs @@ -0,0 +1,18 @@ +using OrchardCore.DataOrchestrator.Activities; +using OrchardCore.DataOrchestrator.Display; +using OrchardCore.DataOrchestrator.ViewModels; + +namespace OrchardCore.DataOrchestrator.Drivers; + +public sealed class JsonExportLoadDisplayDriver : EtlActivityDisplayDriver +{ + protected override void EditActivity(JsonExportLoad activity, JsonExportLoadViewModel model) + { + model.FileName = activity.FileName; + } + + protected override void UpdateActivity(JsonExportLoadViewModel model, JsonExportLoad activity) + { + activity.FileName = model.FileName; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/JsonSourceDisplayDriver.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/JsonSourceDisplayDriver.cs new file mode 100644 index 00000000000..2f4135ebb6f --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Drivers/JsonSourceDisplayDriver.cs @@ -0,0 +1,18 @@ +using OrchardCore.DataOrchestrator.Activities; +using OrchardCore.DataOrchestrator.Display; +using OrchardCore.DataOrchestrator.ViewModels; + +namespace OrchardCore.DataOrchestrator.Drivers; + +public sealed class JsonSourceDisplayDriver : EtlActivityDisplayDriver +{ + protected override void EditActivity(JsonSource activity, JsonSourceViewModel model) + { + model.Data = activity.Data; + } + + protected override void UpdateActivity(JsonSourceViewModel model, JsonSource activity) + { + activity.Data = model.Data; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Manifest.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Manifest.cs new file mode 100644 index 00000000000..0dbd636d03e --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Manifest.cs @@ -0,0 +1,10 @@ +using OrchardCore.Modules.Manifest; + +[assembly: Module( + Name = "Data Orchestrator", + Author = ManifestConstants.OrchardCoreTeam, + Website = ManifestConstants.OrchardCoreWebsite, + Version = ManifestConstants.OrchardCoreVersion, + Description = "Provides a flexible extract, transform, and load pipeline system for data processing.", + Category = "Data" +)] diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Migrations/DataOrchestratorIndexMigrations.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Migrations/DataOrchestratorIndexMigrations.cs new file mode 100644 index 00000000000..f8ab1c70106 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Migrations/DataOrchestratorIndexMigrations.cs @@ -0,0 +1,39 @@ +using OrchardCore.Data.Migration; +using OrchardCore.DataOrchestrator.Indexes; +using YesSql.Sql; + +namespace OrchardCore.DataOrchestrator.Migrations; + +public sealed class DataOrchestratorIndexMigrations : DataMigration +{ + public async Task CreateAsync() + { + await SchemaBuilder.CreateMapIndexTableAsync(table => table + .Column("PipelineId", col => col.WithLength(26)) + .Column("Name", col => col.WithLength(255)) + .Column("IsEnabled") + ); + + await SchemaBuilder.AlterIndexTableAsync(table => table + .CreateIndex("IDX_EtlPipelineIndex_DocumentId", + "DocumentId", + "PipelineId", + "IsEnabled") + ); + + await SchemaBuilder.CreateMapIndexTableAsync(table => table + .Column("PipelineId", col => col.WithLength(44)) + .Column("StartedUtc") + .Column("Status", col => col.WithLength(20)) + ); + + await SchemaBuilder.AlterIndexTableAsync(table => table + .CreateIndex("IDX_EtlExecutionLogIndex_DocumentId", + "DocumentId", + "PipelineId", + "StartedUtc") + ); + + return 1; + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/OrchardCore.DataOrchestrator.csproj b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/OrchardCore.DataOrchestrator.csproj new file mode 100644 index 00000000000..84775c23d89 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/OrchardCore.DataOrchestrator.csproj @@ -0,0 +1,36 @@ + + + + true + OrchardCore ETL + $(OCCMSDescription) + + Provides a flexible ETL (Extract, Transform, Load) pipeline system with visual editor. + $(PackageTags) OrchardCoreCMS ETL + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/PermissionsProvider.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/PermissionsProvider.cs new file mode 100644 index 00000000000..b10cad3f946 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/PermissionsProvider.cs @@ -0,0 +1,25 @@ +using OrchardCore.Security.Permissions; + +namespace OrchardCore.DataOrchestrator; + +public sealed class PermissionsProvider : IPermissionProvider +{ + private readonly IEnumerable _allPermissions = + [ + EtlPermissions.ManageEtlPipelines, + EtlPermissions.ExecuteEtlPipelines, + EtlPermissions.ViewEtlPipelines, + ]; + + public Task> GetPermissionsAsync() + => Task.FromResult(_allPermissions); + + public IEnumerable GetDefaultStereotypes() => + [ + new PermissionStereotype + { + Name = OrchardCoreConstants.Roles.Administrator, + Permissions = _allPermissions, + }, + ]; +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ResourceManagementOptionsConfiguration.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ResourceManagementOptionsConfiguration.cs new file mode 100644 index 00000000000..13e6b184d47 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ResourceManagementOptionsConfiguration.cs @@ -0,0 +1,60 @@ +using Microsoft.Extensions.Options; +using OrchardCore.ResourceManagement; + +namespace OrchardCore.DataOrchestrator; + +public sealed class ResourceManagementOptionsConfiguration : IConfigureOptions +{ + private static readonly ResourceManifest _manifest; + + static ResourceManagementOptionsConfiguration() + { + _manifest = new ResourceManifest(); + + _manifest + .DefineScript("jsplumb") + .SetUrl( + "~/OrchardCore.Workflows/Scripts/jsplumb.min.js", + "~/OrchardCore.Workflows/Scripts/jsplumb.js") + .SetCdn( + "https://cdnjs.cloudflare.com/ajax/libs/jsPlumb/2.15.5/js/jsplumb.min.js", + "https://cdnjs.cloudflare.com/ajax/libs/jsPlumb/2.15.5/js/jsplumb.js") + .SetCdnIntegrity( + "sha384-vJ4MOlEjImsRl4La5sTXZ1UBtJ8uOOqxl2+0gdjRB7oVF6AvTVZ3woqYbTJb7vaf", + "sha384-6qcVETlKUuSEc/QpsceL6BNiyEMBFSPE/uyfdRUvEfao8/K9lynY+r8nd/mwLGGh") + .SetVersion("2.15.5"); + + _manifest + .DefineStyle("jsplumbtoolkit-defaults") + .SetUrl( + "~/OrchardCore.Workflows/Styles/jsplumbtoolkit-defaults.min.css", + "~/OrchardCore.Workflows/Styles/jsplumbtoolkit-defaults.css") + .SetCdn( + "https://cdnjs.cloudflare.com/ajax/libs/jsPlumb/2.15.5/css/jsplumbtoolkit-defaults.min.css", + "https://cdnjs.cloudflare.com/ajax/libs/jsPlumb/2.15.5/css/jsplumbtoolkit-defaults.css") + .SetCdnIntegrity( + "sha384-4TTNOHwtAFYbq+UTSu/7Fj0xnqOabg7FYr9DkNtEKnmIx/YaACNiwhd2XZfO0A/u", + "sha384-Q0wOomiqdBpz2z6/yYA8b3gc8A9t7z7QjD14d1WABvXVHbRYBu/IGOv3SOR57anB") + .SetVersion("2.15.5"); + + _manifest + .DefineScript("etl-pipeline-editor") + .SetDependencies("jsplumb", "bootstrap") + .SetUrl( + "~/OrchardCore.DataOrchestrator/Scripts/etl-pipeline-editor.min.js", + "~/OrchardCore.DataOrchestrator/Scripts/etl-pipeline-editor.js") + .SetVersion("1.0.0"); + + _manifest + .DefineStyle("etl-pipeline-editor") + .SetUrl( + "~/OrchardCore.DataOrchestrator/Styles/etl-pipeline-editor.min.css", + "~/OrchardCore.DataOrchestrator/Styles/etl-pipeline-editor.css") + .SetVersion("1.0.0"); + } + + public void Configure(ResourceManagementOptions options) + { + options.ResourceManifests.Add(_manifest); + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Startup.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Startup.cs new file mode 100644 index 00000000000..b0da94f0275 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Startup.cs @@ -0,0 +1,59 @@ +using Microsoft.Extensions.DependencyInjection; +using OrchardCore.BackgroundTasks; +using OrchardCore.Data; +using OrchardCore.Data.Migration; +using OrchardCore.DataOrchestrator.Activities; +using OrchardCore.DataOrchestrator.BackgroundTasks; +using OrchardCore.DataOrchestrator.Drivers; +using OrchardCore.DataOrchestrator.Helpers; +using OrchardCore.DataOrchestrator.Indexes; +using OrchardCore.DataOrchestrator.Migrations; +using OrchardCore.DataOrchestrator.Services; +using OrchardCore.Modules; +using OrchardCore.Navigation; +using OrchardCore.Security.Permissions; + +namespace OrchardCore.DataOrchestrator; + +public sealed class Startup : StartupBase +{ + public override void ConfigureServices(IServiceCollection services) + { + // Core services. + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + + // Data persistence. + services.AddDataMigration(); + services.AddIndexProvider(); + services.AddIndexProvider(); + + // Permissions and navigation. + services.AddPermissionProvider(); + services.AddNavigationProvider(); + + // Background task. + services.AddSingleton(); + + // Resource management. + services.AddResourceConfiguration(); + + // Built-in source activities. + services.AddEtlActivity(); + services.AddEtlActivity(); + services.AddEtlActivity(); + + // Built-in transform activities. + services.AddEtlActivity(); + services.AddEtlActivity(); + services.AddEtlActivity(); + services.AddEtlActivity(); + + // Built-in load activities. + services.AddEtlActivity(); + services.AddEtlActivity(); + services.AddEtlActivity(); + } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/ContentItemLoadViewModel.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/ContentItemLoadViewModel.cs new file mode 100644 index 00000000000..1667428e9d0 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/ContentItemLoadViewModel.cs @@ -0,0 +1,6 @@ +namespace OrchardCore.DataOrchestrator.ViewModels; + +public class ContentItemLoadViewModel +{ + public string ContentType { get; set; } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/ContentItemSourceViewModel.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/ContentItemSourceViewModel.cs new file mode 100644 index 00000000000..deaef95a1e6 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/ContentItemSourceViewModel.cs @@ -0,0 +1,18 @@ +using Microsoft.AspNetCore.Mvc.Rendering; + +namespace OrchardCore.DataOrchestrator.ViewModels; + +public class ContentItemSourceViewModel +{ + public string ContentType { get; set; } + + public string VersionScope { get; set; } + + public string Owner { get; set; } + + public DateTime? CreatedUtcFrom { get; set; } + + public DateTime? CreatedUtcTo { get; set; } + + public IList AvailableContentTypes { get; set; } = []; +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlActivityEditorPostModel.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlActivityEditorPostModel.cs new file mode 100644 index 00000000000..7237fb34e6f --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlActivityEditorPostModel.cs @@ -0,0 +1,12 @@ +namespace OrchardCore.DataOrchestrator.ViewModels; + +public class EtlActivityEditorPostModel +{ + public long PipelineId { get; set; } + + public string ActivityId { get; set; } + + public string ActivityName { get; set; } + + public string ReturnUrl { get; set; } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlActivityEditorViewModel.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlActivityEditorViewModel.cs new file mode 100644 index 00000000000..7ffd5d773a2 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlActivityEditorViewModel.cs @@ -0,0 +1,18 @@ +using OrchardCore.DataOrchestrator.Activities; + +namespace OrchardCore.DataOrchestrator.ViewModels; + +public class EtlActivityEditorViewModel +{ + public long PipelineId { get; set; } + + public string ActivityId { get; set; } + + public string ActivityName { get; set; } + + public string ReturnUrl { get; set; } + + public IEtlActivity Activity { get; set; } + + public dynamic Editor { get; set; } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlExecutionLogListViewModel.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlExecutionLogListViewModel.cs new file mode 100644 index 00000000000..50062ca63db --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlExecutionLogListViewModel.cs @@ -0,0 +1,10 @@ +using OrchardCore.DataOrchestrator.Models; + +namespace OrchardCore.DataOrchestrator.ViewModels; + +public class EtlExecutionLogListViewModel +{ + public EtlPipelineDefinition Pipeline { get; set; } + + public IList Logs { get; set; } = []; +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlPipelineEditorUpdateModel.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlPipelineEditorUpdateModel.cs new file mode 100644 index 00000000000..a3407c2cb1a --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlPipelineEditorUpdateModel.cs @@ -0,0 +1,8 @@ +namespace OrchardCore.DataOrchestrator.ViewModels; + +public class EtlPipelineEditorUpdateModel +{ + public long Id { get; set; } + + public string State { get; set; } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlPipelineEditorViewModel.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlPipelineEditorViewModel.cs new file mode 100644 index 00000000000..996e333e6bd --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlPipelineEditorViewModel.cs @@ -0,0 +1,22 @@ +using OrchardCore.DataOrchestrator.Models; + +namespace OrchardCore.DataOrchestrator.ViewModels; + +public class EtlPipelineEditorViewModel +{ + public EtlPipelineDefinition Pipeline { get; set; } + + public string PipelineJson { get; set; } + + public IList ActivityThumbnailShapes { get; set; } = []; + + public IList ActivityDesignShapes { get; set; } = []; + + public IList ActivityCategories { get; set; } = []; + + public string LocalId { get; set; } + + public bool LoadLocalState { get; set; } + + public string State { get; set; } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlPipelineListViewModel.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlPipelineListViewModel.cs new file mode 100644 index 00000000000..6ba19c65de5 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlPipelineListViewModel.cs @@ -0,0 +1,10 @@ +using OrchardCore.DataOrchestrator.Models; + +namespace OrchardCore.DataOrchestrator.ViewModels; + +public class EtlPipelineListViewModel +{ + public IList Pipelines { get; set; } = []; + + public string Search { get; set; } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlPipelinePropertiesViewModel.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlPipelinePropertiesViewModel.cs new file mode 100644 index 00000000000..ad47761ad28 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/EtlPipelinePropertiesViewModel.cs @@ -0,0 +1,12 @@ +namespace OrchardCore.DataOrchestrator.ViewModels; + +public class EtlPipelinePropertiesViewModel +{ + public string Name { get; set; } + + public string Description { get; set; } + + public bool IsEnabled { get; set; } + + public string Schedule { get; set; } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/ExcelExportLoadViewModel.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/ExcelExportLoadViewModel.cs new file mode 100644 index 00000000000..8bb0a59b72f --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/ExcelExportLoadViewModel.cs @@ -0,0 +1,8 @@ +namespace OrchardCore.DataOrchestrator.ViewModels; + +public class ExcelExportLoadViewModel +{ + public string FileName { get; set; } + + public string WorksheetName { get; set; } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/ExcelSourceViewModel.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/ExcelSourceViewModel.cs new file mode 100644 index 00000000000..b604e1b777b --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/ExcelSourceViewModel.cs @@ -0,0 +1,10 @@ +namespace OrchardCore.DataOrchestrator.ViewModels; + +public class ExcelSourceViewModel +{ + public string FilePath { get; set; } + + public string WorksheetName { get; set; } + + public bool HasHeaderRow { get; set; } = true; +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/FieldMappingTransformViewModel.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/FieldMappingTransformViewModel.cs new file mode 100644 index 00000000000..4ed02e677d7 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/FieldMappingTransformViewModel.cs @@ -0,0 +1,8 @@ +namespace OrchardCore.DataOrchestrator.ViewModels; + +public class FieldMappingTransformViewModel +{ + public string MappingsJson { get; set; } + + public IList AvailableSourceFields { get; set; } = []; +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/FilterTransformViewModel.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/FilterTransformViewModel.cs new file mode 100644 index 00000000000..71b33bd2e77 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/FilterTransformViewModel.cs @@ -0,0 +1,10 @@ +namespace OrchardCore.DataOrchestrator.ViewModels; + +public class FilterTransformViewModel +{ + public string Field { get; set; } + + public string Operator { get; set; } + + public string Value { get; set; } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/FormatValueTransformViewModel.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/FormatValueTransformViewModel.cs new file mode 100644 index 00000000000..f504d775d18 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/FormatValueTransformViewModel.cs @@ -0,0 +1,16 @@ +namespace OrchardCore.DataOrchestrator.ViewModels; + +public class FormatValueTransformViewModel +{ + public string Field { get; set; } + + public string OutputField { get; set; } + + public string FormatType { get; set; } + + public string FormatString { get; set; } + + public string Culture { get; set; } + + public string TimeZoneId { get; set; } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/JoinDataSetsTransformViewModel.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/JoinDataSetsTransformViewModel.cs new file mode 100644 index 00000000000..c7c0b47f11c --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/JoinDataSetsTransformViewModel.cs @@ -0,0 +1,18 @@ +using Microsoft.AspNetCore.Mvc.Rendering; + +namespace OrchardCore.DataOrchestrator.ViewModels; + +public class JoinDataSetsTransformViewModel +{ + public string JoinSourceActivityId { get; set; } + + public string LeftField { get; set; } + + public string RightField { get; set; } + + public string JoinType { get; set; } + + public string RightPrefix { get; set; } + + public IList AvailableJoinSources { get; set; } = []; +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/JsonExportLoadViewModel.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/JsonExportLoadViewModel.cs new file mode 100644 index 00000000000..7f2ebf6727d --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/JsonExportLoadViewModel.cs @@ -0,0 +1,6 @@ +namespace OrchardCore.DataOrchestrator.ViewModels; + +public class JsonExportLoadViewModel +{ + public string FileName { get; set; } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/JsonSourceViewModel.cs b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/JsonSourceViewModel.cs new file mode 100644 index 00000000000..a2eba7e1844 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/ViewModels/JsonSourceViewModel.cs @@ -0,0 +1,6 @@ +namespace OrchardCore.DataOrchestrator.ViewModels; + +public class JsonSourceViewModel +{ + public string Data { get; set; } +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Admin/Create.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Admin/Create.cshtml new file mode 100644 index 00000000000..3f783b6097c --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Admin/Create.cshtml @@ -0,0 +1,28 @@ +@model EtlPipelinePropertiesViewModel + +

@T["Create Data Pipeline"]

 + +
+
+ + + +
+
+ + +
+
+ + +
+
+ + + @T["Leave empty for manual execution only."] +
+
+ + @T["Cancel"] +
+
diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Admin/CreateActivity.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Admin/CreateActivity.cshtml new file mode 100644 index 00000000000..fae312ba54d --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Admin/CreateActivity.cshtml @@ -0,0 +1,26 @@ +@model EtlActivityEditorViewModel + +

@(string.IsNullOrEmpty(Model.ActivityId) ? T["Add Activity"] : T["Edit Activity"])

 + +
+ + + + + +
+ + @await DisplayAsync(Model.Editor) + +
+ + @if (Url.IsLocalUrl(Model.ReturnUrl)) + { + @T["Cancel"] + } + else + { + @T["Cancel"] + } +
+
diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Admin/Edit.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Admin/Edit.cshtml new file mode 100644 index 00000000000..e87d41117cf --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Admin/Edit.cshtml @@ -0,0 +1,97 @@ +@model EtlPipelineEditorViewModel + +

@RenderTitleSegments(Model.Pipeline.Name)

 + + + + + +
+
+
+
+ + + +
+
+ + @T["Properties"] + @T["Logs"] +
+
+
+
+ +
+ @Html.AntiForgeryToken() + +
+
+
+
+ @foreach (var activityShape in Model.ActivityDesignShapes) + { + @await DisplayAsync(activityShape) + } +
+
+
+ +
+
+ + + @T["Cancel"] +
+
+ +@* Activity Picker Modal *@ + + + + + diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Admin/EditProperties.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Admin/EditProperties.cshtml new file mode 100644 index 00000000000..f13d758565b --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Admin/EditProperties.cshtml @@ -0,0 +1,27 @@ +@model EtlPipelinePropertiesViewModel + +

@T["Pipeline Properties"]

 + +
+
+ + +
+
+ + +
+
+ + +
+
+ + + @T["Leave empty for manual execution only."] +
+
+ + @T["Cancel"] +
+
diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Admin/Index.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Admin/Index.cshtml new file mode 100644 index 00000000000..115a1c68fe3 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Admin/Index.cshtml @@ -0,0 +1,85 @@ +@model EtlPipelineListViewModel + +

@T["Data Pipelines"]

 + +
+
+
+
+
+
+ + +
+
+
+ + @T["Create Pipeline"] + +
+
+
+
+
+ +@if (Model.Pipelines.Count == 0) +{ +
    +
  • + +
    • +
+} +else +{ +
    + @foreach (var pipeline in Model.Pipelines) + { +
  • +
    +
    +
    + @Html.AntiForgeryToken() + +
    + @T["Logs"] + @T["Edit"] +
    + @Html.AntiForgeryToken() + +
    +
    + + @pipeline.Name + @if (!pipeline.IsEnabled) + { + @T["Disabled"] + } + else + { + @T["Enabled"] + } + +
    + @T["Schedule:"] @(pipeline.Schedule ?? T["Manual"].Value) + @T["Last run:"] @(pipeline.LastRunUtc?.ToString("g") ?? T["Never"].Value) +
    + + @if (!string.IsNullOrEmpty(pipeline.Description)) + { +
    @pipeline.Description
    + } +
    +
    • + } +
+} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Admin/Logs.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Admin/Logs.cshtml new file mode 100644 index 00000000000..c3faa086446 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Admin/Logs.cshtml @@ -0,0 +1,90 @@ +@model EtlExecutionLogListViewModel + +

@T["Execution Logs — {0}", Model.Pipeline.Name]

 + + + + + +
+
+
+
+
@T["Recent runs and execution details for this data pipeline."]
+
+
+ @T["Media Library"] + @T["Back to Pipeline"] + @T["All Pipelines"] +
+
+
+
+ +@if (Model.Logs.Count == 0) +{ +
    +
  • +
    @T["No execution logs found."]
    +
    • +
+} +else +{ +
    + @foreach (var log in Model.Logs) + { +
  • +
    +
    + @switch (log.Status) + { + case "Success": + @T["Success"] + break; + case "Failed": + @T["Failed"] + break; + case "Running": + @T["Running"] + break; + case "Cancelled": + @T["Cancelled"] + break; + } +
    + + @T["Run started {0}", log.StartedUtc.ToString("g")] + +
    + @T["Completed:"] @(log.CompletedUtc?.ToString("g") ?? "-") + @T["Duration:"] @(log.CompletedUtc.HasValue ? (log.CompletedUtc.Value - log.StartedUtc).ToString(@"hh\:mm\:ss") : "-") + @T["Records:"] @log.RecordsProcessed / @log.RecordsLoaded + @T["Errors:"] @log.ErrorCount +
    + + @if (log.Errors.Count > 0) + { +
    + @T["Show error details"] +
    +
      + @foreach (var error in log.Errors) + { +
    • @error
      • + } +
    +
    +
    + } +
    +
    • + } +
+} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/EtlActivity.Design.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/EtlActivity.Design.cshtml new file mode 100644 index 00000000000..df943c10cca --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/EtlActivity.Design.cshtml @@ -0,0 +1,40 @@ +@using OrchardCore.DataOrchestrator.Activities +@using OrchardCore.DataOrchestrator.Models +@{ + var activity = (IEtlActivity)Model.Activity; + var activityRecord = (EtlActivityRecord)Model.ActivityRecord; + var isStart = activityRecord.IsStart; + var activityCssClass = activity switch + { + EtlSourceActivity => "activity-source", + EtlTransformActivity => "activity-transform", + EtlLoadActivity => "activity-load", + _ => string.Empty, + }; + var xPosition = activityRecord.X <= 0 ? 0 : activityRecord.X; + var yPosition = activityRecord.Y <= 0 ? 0 : activityRecord.Y; +} +
+ @await DisplayAsync(Model.Content) +
+ @if (activity.HasEditor) + { + + + + } + + + +
+
diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/EtlActivity.Edit.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/EtlActivity.Edit.cshtml new file mode 100644 index 00000000000..ad1bf3a8cc6 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/EtlActivity.Edit.cshtml @@ -0,0 +1 @@ +@await DisplayAsync(Model.Content) diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/EtlActivity.Thumbnail.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/EtlActivity.Thumbnail.cshtml new file mode 100644 index 00000000000..13658d38c5b --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/EtlActivity.Thumbnail.cshtml @@ -0,0 +1,29 @@ +@using OrchardCore.DataOrchestrator.Activities +@model dynamic +@{ + var activity = (IEtlActivity)Model.Activity; + var category = activity.Category; +} +
+
+
+
+ @T[category switch + { + "Transforms" => "Transformation", + _ => category.TrimEnd('s'), + }] +
+
@activity.DisplayText
+ @await DisplayAsync(Model.Content) +
+
+ @T["Add"] +
+
+
diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ContentItemLoad.Fields.Design.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ContentItemLoad.Fields.Design.cshtml new file mode 100644 index 00000000000..fa300b9de81 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ContentItemLoad.Fields.Design.cshtml @@ -0,0 +1,9 @@ +@model EtlActivityViewModel + +
+

@T["Content Item"]

+
+@if (!string.IsNullOrEmpty(Model.Activity.ContentType)) +{ + @T["Type:"] @Model.Activity.ContentType +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ContentItemLoad.Fields.Edit.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ContentItemLoad.Fields.Edit.cshtml new file mode 100644 index 00000000000..6395bf9b9af --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ContentItemLoad.Fields.Edit.cshtml @@ -0,0 +1,7 @@ +@model ContentItemLoadViewModel + +
+ + + @T["The technical name of the content type to create."] +
diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ContentItemLoad.Fields.Thumbnail.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ContentItemLoad.Fields.Thumbnail.cshtml new file mode 100644 index 00000000000..a813f52777e --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ContentItemLoad.Fields.Thumbnail.cshtml @@ -0,0 +1,6 @@ +@model EtlActivityViewModel + +
+
@T["Content Item"]
+
+

@T["Create or update content items from records."]

diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ContentItemSource.Fields.Design.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ContentItemSource.Fields.Design.cshtml new file mode 100644 index 00000000000..2d218a4807f --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ContentItemSource.Fields.Design.cshtml @@ -0,0 +1,17 @@ +@model EtlActivityViewModel + +
+

@T["Content Items"]

+
+@if (!string.IsNullOrEmpty(Model.Activity.ContentType)) +{ +
@T["Type:"] @Model.Activity.ContentType
+} +@if (!string.IsNullOrEmpty(Model.Activity.VersionScope)) +{ +
@T["Versions:"] @Model.Activity.VersionScope
+} +@if (!string.IsNullOrWhiteSpace(Model.Activity.Owner)) +{ +
@T["Owner:"] @Model.Activity.Owner
+} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ContentItemSource.Fields.Edit.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ContentItemSource.Fields.Edit.cshtml new file mode 100644 index 00000000000..4894a89480f --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ContentItemSource.Fields.Edit.cshtml @@ -0,0 +1,35 @@ +@model ContentItemSourceViewModel + +
+ + + @T["Choose which content type to extract."] +
+ +
+ + +
+ +
+ + + @T["Optional owner filter. Leave empty to include all owners."] +
+ +
+
+ + +
+
+ + +
+
diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ContentItemSource.Fields.Thumbnail.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ContentItemSource.Fields.Thumbnail.cshtml new file mode 100644 index 00000000000..ac75b43525f --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ContentItemSource.Fields.Thumbnail.cshtml @@ -0,0 +1,6 @@ +@model EtlActivityViewModel + +
+
@T["Content Items"]
+
+

@T["Extract content items by type."]

diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ExcelExportLoad.Fields.Design.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ExcelExportLoad.Fields.Design.cshtml new file mode 100644 index 00000000000..cf685ffda92 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ExcelExportLoad.Fields.Design.cshtml @@ -0,0 +1,13 @@ +@model EtlActivityViewModel + +
+

@T["Excel Workbook Export"]

+
+@if (!string.IsNullOrWhiteSpace(Model.Activity.FileName)) +{ +
@T["File:"] @Model.Activity.FileName
+} +@if (!string.IsNullOrWhiteSpace(Model.Activity.WorksheetName)) +{ +
@T["Sheet:"] @Model.Activity.WorksheetName
+} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ExcelExportLoad.Fields.Edit.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ExcelExportLoad.Fields.Edit.cshtml new file mode 100644 index 00000000000..b510443c859 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ExcelExportLoad.Fields.Edit.cshtml @@ -0,0 +1,17 @@ +@model ExcelExportLoadViewModel + +
+ + + @T["The workbook is written to Orchard Media using this path, for example `exports/report.xlsx`."] +
+ +
+ + + @T["The worksheet name to create inside the workbook."] +
+ +
+ @T["Exports are currently written directly to the Media Library. Large exports are not queued yet."] +
diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ExcelExportLoad.Fields.Thumbnail.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ExcelExportLoad.Fields.Thumbnail.cshtml new file mode 100644 index 00000000000..ebcb3424a72 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ExcelExportLoad.Fields.Thumbnail.cshtml @@ -0,0 +1,6 @@ +@model EtlActivityViewModel + +
+
@T["Excel Workbook Export"]
+
+

@T["Write the current data set to an Excel workbook."]

diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ExcelSource.Fields.Design.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ExcelSource.Fields.Design.cshtml new file mode 100644 index 00000000000..08f14a0f976 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ExcelSource.Fields.Design.cshtml @@ -0,0 +1,13 @@ +@model EtlActivityViewModel + +
+

@T["Excel Workbook"]

+
+@if (!string.IsNullOrWhiteSpace(Model.Activity.FilePath)) +{ +
@T["File:"] @Model.Activity.FilePath
+} +@if (!string.IsNullOrWhiteSpace(Model.Activity.WorksheetName)) +{ +
@T["Sheet:"] @Model.Activity.WorksheetName
+} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ExcelSource.Fields.Edit.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ExcelSource.Fields.Edit.cshtml new file mode 100644 index 00000000000..c6e831b7bdf --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ExcelSource.Fields.Edit.cshtml @@ -0,0 +1,18 @@ +@model ExcelSourceViewModel + +
+ + + @T["Enter the media library path to an .xlsx file, for example `imports/products.xlsx`."] +
+ +
+ + + @T["Leave empty to read the first worksheet."] +
+ +
+ + +
diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ExcelSource.Fields.Thumbnail.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ExcelSource.Fields.Thumbnail.cshtml new file mode 100644 index 00000000000..7d03ec3ed6d --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/ExcelSource.Fields.Thumbnail.cshtml @@ -0,0 +1,6 @@ +@model EtlActivityViewModel + +
+
@T["Excel Workbook"]
+
+

@T["Extract rows from an Excel file stored in the media library."]

diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FieldMappingTransform.Fields.Design.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FieldMappingTransform.Fields.Design.cshtml new file mode 100644 index 00000000000..43987a2f5d7 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FieldMappingTransform.Fields.Design.cshtml @@ -0,0 +1,9 @@ +@model EtlActivityViewModel + +
+

@T["Field Mapping"]

+
+@if (!string.IsNullOrEmpty(Model.Activity.MappingsJson)) +{ + @T["Mappings configured"] +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FieldMappingTransform.Fields.Edit.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FieldMappingTransform.Fields.Edit.cshtml new file mode 100644 index 00000000000..7973e15631f --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FieldMappingTransform.Fields.Edit.cshtml @@ -0,0 +1,98 @@ +@model FieldMappingTransformViewModel + +
+ + + @if (Model.AvailableSourceFields.Count > 0) + { +
+ @T["Detected source fields:"] +
+ @foreach (var field in Model.AvailableSourceFields) + { + @field + } +
+
+ } +
+ + + + + + + + + +
@T["Source Field"]@T["Output Field"]
+
+ + @T["Use dot-notation such as `DisplayText` or `Author.UserName`."] +
+ + + @foreach (var field in Model.AvailableSourceFields) + { + + } + + + + + diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FieldMappingTransform.Fields.Thumbnail.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FieldMappingTransform.Fields.Thumbnail.cshtml new file mode 100644 index 00000000000..41b973c62dc --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FieldMappingTransform.Fields.Thumbnail.cshtml @@ -0,0 +1,6 @@ +@model EtlActivityViewModel + +
+
@T["Field Mapping"]
+
+

@T["Map and rename fields in the data stream."]

diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FilterTransform.Fields.Design.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FilterTransform.Fields.Design.cshtml new file mode 100644 index 00000000000..ff74e7dba70 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FilterTransform.Fields.Design.cshtml @@ -0,0 +1,9 @@ +@model EtlActivityViewModel + +
+

@T["Filter"]

+
+@if (!string.IsNullOrEmpty(Model.Activity.Field)) +{ + @Model.Activity.Field @Model.Activity.Operator @Model.Activity.Value +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FilterTransform.Fields.Edit.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FilterTransform.Fields.Edit.cshtml new file mode 100644 index 00000000000..652ade14959 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FilterTransform.Fields.Edit.cshtml @@ -0,0 +1,32 @@ +@model FilterTransformViewModel + +
+ + + @T["The name of the field to filter on. Supports dot-notation for nested fields."] +
+ +
+ + + @T["The comparison operator to use. Numeric and date values support greater/less-than comparisons."] +
+ +
+ + + @T["The value to compare the field against. Leave empty for operators like Is Empty."] +
diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FilterTransform.Fields.Thumbnail.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FilterTransform.Fields.Thumbnail.cshtml new file mode 100644 index 00000000000..5eae465c0af --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FilterTransform.Fields.Thumbnail.cshtml @@ -0,0 +1,6 @@ +@model EtlActivityViewModel + +
+
@T["Filter"]
+
+

@T["Filter records by a field condition."]

diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FormatValueTransform.Fields.Design.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FormatValueTransform.Fields.Design.cshtml new file mode 100644 index 00000000000..44eebb4170c --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FormatValueTransform.Fields.Design.cshtml @@ -0,0 +1,13 @@ +@model EtlActivityViewModel + +
+

@T["Format Value"]

+
+@if (!string.IsNullOrWhiteSpace(Model.Activity.Field)) +{ +
@T["Field:"] @Model.Activity.Field
+} +@if (!string.IsNullOrWhiteSpace(Model.Activity.FormatType)) +{ +
@T["Format:"] @Model.Activity.FormatType
+} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FormatValueTransform.Fields.Edit.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FormatValueTransform.Fields.Edit.cshtml new file mode 100644 index 00000000000..5de034f060e --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FormatValueTransform.Fields.Edit.cshtml @@ -0,0 +1,44 @@ +@model FormatValueTransformViewModel + +
+ + + @T["The source field to format. Dot-notation is supported."] +
+ +
+ + + @T["Optional destination field. Leave empty to overwrite the source field."] +
+ +
+ + +
+ +
+ + + @T["Optional .NET format string, such as `C2`, `N0`, `yyyy-MM-dd`, or `G`."] +
+ +
+
+ + + @T["Optional culture name such as `en-US` or `fr-FR`."] +
+
+ + + @T["Used for UTC conversion, for example `Eastern Standard Time`."] +
+
diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FormatValueTransform.Fields.Thumbnail.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FormatValueTransform.Fields.Thumbnail.cshtml new file mode 100644 index 00000000000..57c6a85c5f4 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/FormatValueTransform.Fields.Thumbnail.cshtml @@ -0,0 +1,6 @@ +@model EtlActivityViewModel + +
+
@T["Format Value"]
+
+

@T["Format currency, numbers, casing, dates, and time zones."]

diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JoinDataSetsTransform.Fields.Design.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JoinDataSetsTransform.Fields.Design.cshtml new file mode 100644 index 00000000000..444df2b62da --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JoinDataSetsTransform.Fields.Design.cshtml @@ -0,0 +1,13 @@ +@model EtlActivityViewModel + +
+

@T["Join Data Sets"]

+
+@if (!string.IsNullOrWhiteSpace(Model.Activity.LeftField)) +{ +
@T["Left:"] @Model.Activity.LeftField
+} +@if (!string.IsNullOrWhiteSpace(Model.Activity.RightField)) +{ +
@T["Right:"] @Model.Activity.RightField
+} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JoinDataSetsTransform.Fields.Edit.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JoinDataSetsTransform.Fields.Edit.cshtml new file mode 100644 index 00000000000..40e812b9bdd --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JoinDataSetsTransform.Fields.Edit.cshtml @@ -0,0 +1,37 @@ +@model JoinDataSetsTransformViewModel + +
+ + + @T["Choose the source activity that provides the right-side records for the join."] +
+ +
+
+ + + @T["Field from the current stream, e.g. `ContentItemId`."] +
+
+ + + @T["Field from the selected join source to match against."] +
+
+ +
+
+ + +
+
+ + + @T["Prefix applied to joined fields to avoid collisions, e.g. `Author_`."] +
+
diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JoinDataSetsTransform.Fields.Thumbnail.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JoinDataSetsTransform.Fields.Thumbnail.cshtml new file mode 100644 index 00000000000..d32cbbba9a8 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JoinDataSetsTransform.Fields.Thumbnail.cshtml @@ -0,0 +1,6 @@ +@model EtlActivityViewModel + +
+
@T["Join Data Sets"]
+
+

@T["Join the current stream with another source activity by matching keys."]

diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JsonExportLoad.Fields.Design.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JsonExportLoad.Fields.Design.cshtml new file mode 100644 index 00000000000..be8698481e0 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JsonExportLoad.Fields.Design.cshtml @@ -0,0 +1,9 @@ +@model EtlActivityViewModel + +
+

@T["JSON Export"]

+
+@if (!string.IsNullOrEmpty(Model.Activity.FileName)) +{ + @T["File:"] @Model.Activity.FileName +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JsonExportLoad.Fields.Edit.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JsonExportLoad.Fields.Edit.cshtml new file mode 100644 index 00000000000..2d87a5a68a5 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JsonExportLoad.Fields.Edit.cshtml @@ -0,0 +1,7 @@ +@model JsonExportLoadViewModel + +
+ + + @T["The JSON file is written to Orchard Media using this path, for example `exports/output.json`."] +
diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JsonExportLoad.Fields.Thumbnail.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JsonExportLoad.Fields.Thumbnail.cshtml new file mode 100644 index 00000000000..e1ced4a079c --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JsonExportLoad.Fields.Thumbnail.cshtml @@ -0,0 +1,6 @@ +@model EtlActivityViewModel + +
+
@T["JSON Export"]
+
+

@T["Export records as a JSON file."]

diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JsonSource.Fields.Design.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JsonSource.Fields.Design.cshtml new file mode 100644 index 00000000000..85b9a47a5e0 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JsonSource.Fields.Design.cshtml @@ -0,0 +1,9 @@ +@model EtlActivityViewModel + +
+

@T["JSON Data"]

+
+@if (!string.IsNullOrEmpty(Model.Activity.Data)) +{ + @T["Data provided"] +} diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JsonSource.Fields.Edit.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JsonSource.Fields.Edit.cshtml new file mode 100644 index 00000000000..c4a311dedea --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JsonSource.Fields.Edit.cshtml @@ -0,0 +1,7 @@ +@model JsonSourceViewModel + +
+ + + @T["A JSON array of objects to use as the data source."] +
diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JsonSource.Fields.Thumbnail.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JsonSource.Fields.Thumbnail.cshtml new file mode 100644 index 00000000000..4448f4bcfaf --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/Items/JsonSource.Fields.Thumbnail.cshtml @@ -0,0 +1,6 @@ +@model EtlActivityViewModel + +
+
@T["JSON Data"]
+
+

@T["Extract records from inline JSON."]

diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/_ViewImports.cshtml b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/_ViewImports.cshtml new file mode 100644 index 00000000000..e783640ce80 --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Views/_ViewImports.cshtml @@ -0,0 +1,8 @@ +@inherits OrchardCore.DisplayManagement.Razor.RazorPage +@addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers +@addTagHelper *, OrchardCore.DisplayManagement +@addTagHelper *, OrchardCore.ResourceManagement +@addTagHelper *, OrchardCore.Mvc.Core +@using OrchardCore.DataOrchestrator.Activities +@using OrchardCore.DataOrchestrator.ViewModels +@using OrchardCore.DataOrchestrator.Models diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/wwwroot/Scripts/etl-pipeline-editor.js b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/wwwroot/Scripts/etl-pipeline-editor.js new file mode 100644 index 00000000000..8c6f91e9f7a --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/wwwroot/Scripts/etl-pipeline-editor.js @@ -0,0 +1,2 @@ +!function(t,e,i,n){var o="undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:"undefined"!=typeof window?window:"undefined"!=typeof global?global:{},a="function"==typeof o[n]&&o[n],r=a.cache||{},s="undefined"!=typeof module&&"function"==typeof module.require&&module.require.bind(module);function c(e,i){if(!r[e]){if(!t[e]){var l="function"==typeof o[n]&&o[n];if(!i&&l)return l(e,!0);if(a)return a(e,!0);if(s&&"string"==typeof e)return s(e);var d=new Error("Cannot find module '"+e+"'");throw d.code="MODULE_NOT_FOUND",d}p.resolve=function(i){var n=t[e][1][i];return null!=n?n:i},p.cache={};var u=r[e]=new c.Module(e);t[e][0].call(u.exports,p,u,u.exports,o)}return r[e].exports;function p(t){var e=p.resolve(t);return!1===e?{}:c(e)}}c.isParcelRequire=!0,c.Module=function(t){this.id=t,this.bundle=c,this.exports={}},c.modules=t,c.cache=r,c.parent=a,c.register=function(e,i){t[e]=[function(t,e){e.exports=i},{}]},Object.defineProperty(c,"root",{get:function(){return o[n]}}),o[n]=c;for(var l=0;l{this.init()})}saveCurrentState(){this.saveLocalState()}init(){const t=jsPlumb.getInstance({DragOptions:{cursor:"pointer",zIndex:2e3},ConnectionOverlays:[["Arrow",{width:12,length:12,location:-5}]],Container:this.container});this.jsPlumbInstance=t;this.container.querySelectorAll(".activity").forEach(e=>{this.setupActivity(t,e)}),this.updateConnections(t),this.updateCanvasHeight(),requestAnimationFrame(()=>t.repaintEverything()),t.bind("connection",()=>{this.saveLocalState(),this.updateCanvasHeight()}),t.bind("connectionDetached",()=>{this.saveLocalState()})}setupActivity(t,e){const i=e.getAttribute("data-activity-id"),n=this.pipeline.activities.find(t=>t.id===i);if(!n)return;t.draggable(e,{grid:[10,10],containment:!0,stop:()=>{this.updateCanvasHeight(),this.saveLocalState()}});let o="#7ab02c";n.isTransform&&(o="#3a8acd"),n.isLoad&&(o="#6c757d"),n.outcomes&&n.outcomes.forEach(n=>{t.addEndpoint(e,{endpoint:"Dot",anchor:"Right",paintStyle:{fill:o,radius:7},isSource:!0,connector:["Flowchart",{stub:[40,60],gap:0,cornerRadius:5,alwaysRespectStubs:!0}],connectorStyle:{strokeWidth:2,stroke:"#999999",joinstyle:"round",outlineStroke:"white",outlineWidth:2},hoverPaintStyle:{fill:"#216477",stroke:"#216477"},connectorHoverStyle:{strokeWidth:3,stroke:"#216477"},uuid:`${i}-${n.name}`,parameters:{outcome:n},overlays:[["Label",{location:[.5,1.5],label:n.displayName,cssClass:"outcome-label",visible:!0}]]})}),t.makeTarget(e,{dropOptions:{hoverClass:"hover"},anchor:"Left",endpoint:["Blank",{radius:8}]}),this.setupActivityActions(e,n)}setupActivityActions(t,e){t.addEventListener("click",e=>{if(e.target.closest(".activity-commands"))return;const i=t.querySelector(".activity-commands");i&&i.classList.toggle("d-none")});const i=t.querySelector(".activity-delete-action");i&&i.addEventListener("click",()=>{confirm("Are you sure you want to delete this activity?")&&(this.pipeline.removedActivities=this.pipeline.removedActivities||[],this.pipeline.removedActivities.push(e.id),this.jsPlumbInstance.remove(t),this.saveLocalState())})}updateConnections(t){t.batch(()=>{this.pipeline.transitions.forEach(e=>{const i=t.getEndpoint(`${e.sourceActivityId}-${e.sourceOutcomeName}`),n=this.container.querySelector(`[data-activity-id="${e.destinationActivityId}"]`);i&&n&&t.connect({source:i,target:n,type:"basic"})})})}updateCanvasHeight(){let t=400;this.container.querySelectorAll(".activity").forEach(e=>{const i=e.offsetTop+e.offsetHeight+50;i>t&&(t=i)}),this.container.style.minHeight=t+"px"}getState(){const t=[];this.container.querySelectorAll(".activity").forEach(e=>{const i=e.getAttribute("data-activity-id"),n=this.pipeline.activities.find(t=>t.id===i);n&&t.push({...n,x:e.offsetLeft,y:e.offsetTop,isStart:"true"===e.getAttribute("data-activity-start")})});const e=[];return this.jsPlumbInstance.getConnections().forEach(t=>{const i=t.endpoints[0].getParameter("outcome"),n=t.target;e.push({sourceActivityId:t.source.getAttribute("data-activity-id"),destinationActivityId:n.getAttribute("data-activity-id"),sourceOutcomeName:i?i.name:"Done"})}),{...this.pipeline,activities:t,transitions:e,removedActivities:this.pipeline.removedActivities||[]}}serialize(){return JSON.stringify(this.getState())}saveLocalState(){try{sessionStorage.setItem(`etl-pipeline-${this.localId}`,this.serialize())}catch{}}loadLocalState(){try{const t=sessionStorage.getItem(`etl-pipeline-${this.localId}`);return t?JSON.parse(t):null}catch{return null}}}document.addEventListener("DOMContentLoaded",function(){const t=document.querySelector(".etl-canvas");if(!t)return;const e=t.dataset.pipeline;if(!e)return;const i=JSON.parse(e);i.removedActivities=i.removedActivities||[];const n=t.dataset.deleteActivityPrompt||"Are you sure?",a=t.dataset.localId||"",r="true"===t.dataset.loadLocalState,s=new o(t,i,n,a,r),c=document.getElementById("pipelineEditorForm"),l=document.getElementById("pipelineStateInput");c&&l&&c.addEventListener("submit",()=>{l.value=s.serialize()}),document.querySelectorAll(".activity-add-action, .activity-edit-action").forEach(t=>{t.addEventListener("click",async e=>{e.preventDefault();await(async()=>{if(!c||!l)return!0;l.value=s.serialize(),s.saveCurrentState();const t=new FormData(c);try{return(await fetch(c.action,{method:"POST",body:t,credentials:"same-origin",headers:{"X-Requested-With":"XMLHttpRequest"}})).ok}catch{return!1}})()&&t instanceof HTMLAnchorElement&&(window.location.href=t.href)})});const d=document.getElementById("activity-picker");if(d){let t="all";const e=()=>{const e=d.querySelector(".activity-search"),i=(e?.value||"").toLowerCase();d.querySelectorAll(".activity-card").forEach(e=>{const n=e.querySelector(".card-title"),o=n?n.textContent.toLowerCase():"",a=e.dataset.activityType||"all",r="all"===t||a===t,s=!i||o.indexOf(i)>=0;e.style.display=r&&s?"":"none"})},i=i=>{t=i||"all",e()};d.addEventListener("show.bs.modal",t=>{const e=t.relatedTarget,n=e?.dataset.activityType||"all",o=e?.dataset.pickerTitle||"Available Activities",a=d.querySelector(".modal-title");a&&(a.textContent=o);const r=d.querySelector(".activity-search");r&&(r.value=""),i(n)});const n=d.querySelector(".activity-search");n&&n.addEventListener("input",()=>{e()})}})},{}]},["2W7JX"],"2W7JX","parcelRequire94c2"); +//# sourceMappingURL=etl-pipeline-editor.js.map diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/wwwroot/Scripts/etl-pipeline-editor.js.map b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/wwwroot/Scripts/etl-pipeline-editor.js.map new file mode 100644 index 00000000000..cdd9544961c --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/wwwroot/Scripts/etl-pipeline-editor.js.map @@ -0,0 +1 @@ +{"mappings":"osCAGA,MAAMA,EAMF,WAAAC,CACIC,EACAC,EACAC,EACAC,EACAC,GAMA,GAJAC,KAAKL,UAAYA,EACjBK,KAAKJ,SAAWA,EAChBI,KAAKF,QAAUA,EAEXC,EAAgB,CAChB,MAAME,EAAQD,KAAKD,iBACfE,IACAD,KAAKJ,SAAWK,EAExB,CAEAC,QAAQC,MAAM,KACVH,KAAKI,QAEb,CAEO,gBAAAC,GACHL,KAAKM,gBACT,CAEQ,IAAAF,GACJ,MAAMG,EAAUL,QAAQM,YAAY,CAChCC,YAAa,CAAEC,OAAQ,UAAWC,OAAQ,KAC1CC,mBAAoB,CAChB,CAAC,QAAS,CAAEC,MAAO,GAAIC,OAAQ,GAAIC,UAAU,KAEjDC,UAAWhB,KAAKL,YAGpBK,KAAKiB,gBAAkBV,EAEEP,KAAKL,UAAUuB,iBAA8B,aACrDC,QAASC,IACtBpB,KAAKqB,cAAcd,EAASa,KAGhCpB,KAAKsB,kBAAkBf,GACvBP,KAAKuB,qBACLC,sBAAsB,IAAMjB,EAAQkB,qBAEpClB,EAAQmB,KAAK,aAAc,KACvB1B,KAAKM,iBACLN,KAAKuB,uBAGThB,EAAQmB,KAAK,qBAAsB,KAC/B1B,KAAKM,kBAEb,CAEQ,aAAAe,CAAcd,EAA0Ba,GAC5C,MAAMO,EAAaP,EAAGQ,aAAa,oBAC7BC,EAAW7B,KAAKJ,SAASkC,WAAWC,KAAKC,GAAKA,EAAEC,KAAON,GAE7D,IAAKE,EAAU,OAEftB,EAAQ2B,UAAUd,EAAI,CAClBe,KAAM,CAAC,GAAI,IACXC,aAAa,EACbC,KAAM,KACFrC,KAAKuB,qBACLvB,KAAKM,oBAIb,IAAIgC,EAAQ,UACRT,EAASU,cAAaD,EAAQ,WAC9BT,EAASW,SAAQF,EAAQ,WAEzBT,EAASY,UACTZ,EAASY,SAAStB,QAASuB,IACvBnC,EAAQoC,YAAYvB,EAAI,CACpBwB,SAAU,MACVC,OAAQ,QACRC,WAAY,CAAEC,KAAMT,EAAOU,OAAQ,GACnCC,UAAU,EACVC,UAAW,CAAC,YAAa,CAAEC,KAAM,CAAC,GAAI,IAAKC,IAAK,EAAGC,aAAc,EAAGC,oBAAoB,IACxFC,eAAgB,CAAEC,YAAa,EAAGC,OAAQ,UAAWC,UAAW,QAASC,cAAe,QAASC,aAAc,GAC/GC,gBAAiB,CAAEd,KAAM,UAAWU,OAAQ,WAC5CK,oBAAqB,CAAEN,YAAa,EAAGC,OAAQ,WAC/CM,KAAM,GAAGpC,KAAce,EAAQsB,OAC/BC,WAAY,CAAEvB,QAASA,GACvBwB,SAAU,CACN,CAAC,QAAS,CACNnD,SAAU,CAAC,GAAK,KAChBoD,MAAOzB,EAAQ0B,YACfC,SAAU,gBACVC,SAAS,SAO7B/D,EAAQgE,WAAWnD,EAAI,CACnBoD,YAAa,CAAEC,WAAY,SAC3B5B,OAAQ,OACRD,SAAU,CAAC,QAAS,CAAEI,OAAQ,MAGlChD,KAAK0E,qBAAqBtD,EAAIS,EAClC,CAEQ,oBAAA6C,CAAqBtD,EAAiBS,GAC1CT,EAAGuD,iBAAiB,QAAUC,IAE1B,GADeA,EAAEC,OACNC,QAAQ,sBAAuB,OAE1C,MAAMC,EAAW3D,EAAG4D,cAAc,sBAC9BD,GACAA,EAASE,UAAUC,OAAO,YAIlC,MAAMC,EAAY/D,EAAG4D,cAAc,2BAC/BG,GACAA,EAAUR,iBAAiB,QAAS,KAC3BS,QAAQ,oDAEbpF,KAAKJ,SAASyF,kBAAoBrF,KAAKJ,SAASyF,mBAAqB,GACrErF,KAAKJ,SAASyF,kBAAkBC,KAAKzD,EAASI,IAE9CjC,KAAKiB,gBAAgBsE,OAAOnE,GAC5BpB,KAAKM,mBAGjB,CAEQ,iBAAAgB,CAAkBf,GACtBA,EAAQiF,MAAM,KACVxF,KAAKJ,SAAS6F,YAAYtE,QAASuE,IAC/B,MAAMC,EAAiBpF,EAAQqF,YAAY,GAAGF,EAAEG,oBAAoBH,EAAEI,qBAChEC,EAAW/F,KAAKL,UAAUqF,cAAc,sBAAsBU,EAAEM,2BAElEL,GAAkBI,GAClBxF,EAAQ0F,QAAQ,CACZC,OAAQP,EACRd,OAAQkB,EACRI,KAAM,aAK1B,CAEQ,kBAAA5E,GACJ,IAAI6E,EAAO,IACXpG,KAAKL,UAAUuB,iBAA8B,aAAaC,QAASC,IAC/D,MAAMiF,EAASjF,EAAGkF,UAAYlF,EAAGmF,aAAe,GAC5CF,EAASD,IAAMA,EAAOC,KAE9BrG,KAAKL,UAAU6G,MAAMC,UAAYL,EAAO,IAC5C,CAEO,QAAAM,GACH,MAAM5E,EAAqC,GAE3C9B,KAAKL,UAAUuB,iBAA8B,aAAaC,QAASC,IAC/D,MAAMa,EAAKb,EAAGQ,aAAa,oBACrB+E,EAAW3G,KAAKJ,SAASkC,WAAWC,KAAKC,GAAKA,EAAEC,KAAOA,GACzD0E,GACA7E,EAAWwD,KAAK,IACTqB,EACHC,EAAGxF,EAAGyF,WACNC,EAAG1F,EAAGkF,UACNS,QAAoD,SAA3C3F,EAAGQ,aAAa,2BAKrC,MAAM6D,EAAwC,GAa9C,OAZoBzF,KAAKiB,gBAAgB+F,iBAC7B7F,QAAS8F,IACjB,MACMvE,EADiBuE,EAAKC,UAAU,GACPC,aAAa,WACtCpB,EAAWkB,EAAKpC,OACtBY,EAAYH,KAAK,CACbO,iBAAkBoB,EAAKf,OAAOtE,aAAa,oBAC3CoE,sBAAuBD,EAASnE,aAAa,oBAC7CkE,kBAAmBpD,EAAUA,EAAQsB,KAAO,WAI7C,IACAhE,KAAKJ,SACRkC,WAAYA,EACZ2D,YAAaA,EACbJ,kBAAmBrF,KAAKJ,SAASyF,mBAAqB,GAE9D,CAEO,SAAA+B,GACH,OAAOC,KAAKC,UAAUtH,KAAK0G,WAC/B,CAEQ,cAAApG,GACJ,IACIiH,eAAeC,QAAQ,gBAAgBxH,KAAKF,UAAWE,KAAKoH,YAChE,CAAE,MAEF,CACJ,CAEQ,cAAArH,GACJ,IACI,MAAM0H,EAAOF,eAAeG,QAAQ,gBAAgB1H,KAAKF,WACzD,OAAO2H,EAAOJ,KAAKM,MAAMF,GAAQ,IACrC,CAAE,MACE,OAAO,IACX,CACJ,EA6HJG,SAASjD,iBAAiB,mBA1H1B,WACI,MAAMkD,EAASD,SAAS5C,cAA2B,eACnD,IAAK6C,EAAQ,OAEb,MAAMC,EAAeD,EAAOE,QAAQnI,SACpC,IAAKkI,EAAc,OAEnB,MAAMlI,EAAiCyH,KAAKM,MAAMG,GAClDlI,EAASyF,kBAAoBzF,EAASyF,mBAAqB,GAE3D,MAAMxF,EAAegI,EAAOE,QAAQC,sBAAwB,gBACtDlI,EAAU+H,EAAOE,QAAQjI,SAAW,GACpCC,EAAmD,SAAlC8H,EAAOE,QAAQhI,eAEhCkI,EAAS,IAAIxI,EACfoI,EACAjI,EACAC,EACAC,EACAC,GAIEmI,EAAON,SAASO,eAAe,sBAC/BC,EAAaR,SAASO,eAAe,sBA2BvCD,GAAQE,GACRF,EAAKvD,iBAAiB,SAAU,KAC5ByD,EAAWC,MAAQJ,EAAOb,cAILQ,SAAS1G,iBAA8B,+CAC/CC,QAASmH,IAC1BA,EAAK3D,iBAAiB,QAAS4D,MAAOC,IAClCA,EAAMC,sBAnCUF,WACpB,IAAKL,IAASE,EACV,OAAO,EAGXA,EAAWC,MAAQJ,EAAOb,YAC1Ba,EAAO5H,mBAEP,MAAMqI,EAAW,IAAIC,SAAST,GAE9B,IAUI,aATuBU,MAAMV,EAAKW,OAAQ,CACtCC,OAAQ,OACRC,KAAML,EACNM,YAAa,cACbC,QAAS,CACL,mBAAoB,qBAIZC,EACpB,CAAE,MACE,OAAO,CACX,GAa4BC,IACPb,aAAgBc,oBAC7BC,OAAOtI,SAASuI,KAAOhB,EAAKgB,UAMxC,MAAMC,EAAc3B,SAASO,eAAe,mBAC5C,GAAIoB,EAAa,CACb,IAAIC,EAAqB,MAEzB,MAAMC,EAAuB,KACzB,MAAMC,EAAcH,EAAYvE,cAAgC,oBAC1D2E,GAASD,GAAarB,OAAS,IAAIuB,cAC3BL,EAAYrI,iBAA8B,kBAElDC,QAAS0I,IACX,MAAMC,EAAYD,EAAK7E,cAAc,eAC/B+E,EAAOD,EAAYA,EAAUE,YAAaJ,cAAgB,GAC1DK,EAAWJ,EAAK9B,QAAQmC,cAAgB,MACxCC,EAAqC,QAAvBX,GAAgCS,IAAaT,EAC3DY,GAAgBT,GAASI,EAAKM,QAAQV,IAAU,EAEtDE,EAAKrD,MAAM8D,QAAUH,GAAeC,EAAe,GAAK,UAI1DG,EAAiBL,IACnBV,EAAqBU,GAAgB,MACrCT,KAGJF,EAAY5E,iBAAiB,gBAAkBC,IAC3C,MACM4F,EADa5F,EACO6F,cACpBP,EAAeM,GAAQzC,QAAQmC,cAAgB,MAC/CQ,EAAQF,GAAQzC,QAAQ4C,aAAe,uBAEvCC,EAAarB,EAAYvE,cAAc,gBACzC4F,IACAA,EAAWZ,YAAcU,GAG7B,MAAMhB,EAAcH,EAAYvE,cAAgC,oBAC5D0E,IACAA,EAAYrB,MAAQ,IAGxBkC,EAAcL,KAGlB,MAAMR,EAAcH,EAAYvE,cAAgC,oBAC5D0E,GACAA,EAAY/E,iBAAiB,QAAS,KAClC8E,KAGZ,CACJ,E","sources":["src/OrchardCore.Modules/OrchardCore.DataOrchestrator/Assets/Scripts/etl-pipeline-editor.ts"],"sourcesContent":["/// \n/// \n\nclass EtlPipelineEditor {\n private jsPlumbInstance: jsPlumbInstance;\n private container: HTMLElement;\n private pipeline: EtlPipeline.Pipeline;\n private localId: string;\n\n constructor(\n container: HTMLElement,\n pipeline: EtlPipeline.Pipeline,\n deletePrompt: string,\n localId: string,\n loadLocalState: boolean\n ) {\n this.container = container;\n this.pipeline = pipeline;\n this.localId = localId;\n\n if (loadLocalState) {\n const saved = this.loadLocalState();\n if (saved) {\n this.pipeline = saved;\n }\n }\n\n jsPlumb.ready(() => {\n this.init();\n });\n }\n\n public saveCurrentState(): void {\n this.saveLocalState();\n }\n\n private init() {\n const plumber = jsPlumb.getInstance({\n DragOptions: { cursor: 'pointer', zIndex: 2000 },\n ConnectionOverlays: [\n ['Arrow', { width: 12, length: 12, location: -5 }]\n ],\n Container: this.container\n });\n\n this.jsPlumbInstance = plumber;\n\n const activityElements = this.container.querySelectorAll('.activity');\n activityElements.forEach((el) => {\n this.setupActivity(plumber, el);\n });\n\n this.updateConnections(plumber);\n this.updateCanvasHeight();\n requestAnimationFrame(() => plumber.repaintEverything());\n\n plumber.bind('connection', () => {\n this.saveLocalState();\n this.updateCanvasHeight();\n });\n\n plumber.bind('connectionDetached', () => {\n this.saveLocalState();\n });\n }\n\n private setupActivity(plumber: jsPlumbInstance, el: HTMLElement) {\n const activityId = el.getAttribute('data-activity-id');\n const activity = this.pipeline.activities.find(a => a.id === activityId);\n\n if (!activity) return;\n\n plumber.draggable(el, {\n grid: [10, 10],\n containment: true,\n stop: () => {\n this.updateCanvasHeight();\n this.saveLocalState();\n }\n });\n\n let color = '#7ab02c';\n if (activity.isTransform) color = '#3a8acd';\n if (activity.isLoad) color = '#6c757d';\n\n if (activity.outcomes) {\n activity.outcomes.forEach((outcome: EtlPipeline.Outcome) => {\n plumber.addEndpoint(el, {\n endpoint: 'Dot',\n anchor: 'Right' as any,\n paintStyle: { fill: color, radius: 7 },\n isSource: true,\n connector: ['Flowchart', { stub: [40, 60], gap: 0, cornerRadius: 5, alwaysRespectStubs: true }],\n connectorStyle: { strokeWidth: 2, stroke: '#999999', joinstyle: 'round', outlineStroke: 'white', outlineWidth: 2 },\n hoverPaintStyle: { fill: '#216477', stroke: '#216477' },\n connectorHoverStyle: { strokeWidth: 3, stroke: '#216477' },\n uuid: `${activityId}-${outcome.name}`,\n parameters: { outcome: outcome },\n overlays: [\n ['Label', {\n location: [0.5, 1.5],\n label: outcome.displayName,\n cssClass: 'outcome-label',\n visible: true\n }]\n ]\n } as any);\n });\n }\n\n plumber.makeTarget(el, {\n dropOptions: { hoverClass: 'hover' },\n anchor: 'Left' as any,\n endpoint: ['Blank', { radius: 8 }] as any\n });\n\n this.setupActivityActions(el, activity);\n }\n\n private setupActivityActions(el: HTMLElement, activity: EtlPipeline.Activity) {\n el.addEventListener('click', (e: MouseEvent) => {\n const target = e.target as HTMLElement;\n if (target.closest('.activity-commands')) return;\n\n const commands = el.querySelector('.activity-commands');\n if (commands) {\n commands.classList.toggle('d-none');\n }\n });\n\n const deleteBtn = el.querySelector('.activity-delete-action');\n if (deleteBtn) {\n deleteBtn.addEventListener('click', () => {\n if (!confirm('Are you sure you want to delete this activity?')) return;\n\n this.pipeline.removedActivities = this.pipeline.removedActivities || [];\n this.pipeline.removedActivities.push(activity.id);\n\n this.jsPlumbInstance.remove(el);\n this.saveLocalState();\n });\n }\n }\n\n private updateConnections(plumber: jsPlumbInstance) {\n plumber.batch(() => {\n this.pipeline.transitions.forEach((t: EtlPipeline.Transition) => {\n const sourceEndpoint = plumber.getEndpoint(`${t.sourceActivityId}-${t.sourceOutcomeName}`);\n const targetEl = this.container.querySelector(`[data-activity-id=\"${t.destinationActivityId}\"]`);\n\n if (sourceEndpoint && targetEl) {\n plumber.connect({\n source: sourceEndpoint,\n target: targetEl as any,\n type: 'basic'\n } as any);\n }\n });\n });\n }\n\n private updateCanvasHeight() {\n let maxY = 400;\n this.container.querySelectorAll('.activity').forEach((el) => {\n const bottom = el.offsetTop + el.offsetHeight + 50;\n if (bottom > maxY) maxY = bottom;\n });\n this.container.style.minHeight = maxY + 'px';\n }\n\n public getState(): EtlPipeline.Pipeline {\n const activities: EtlPipeline.Activity[] = [];\n\n this.container.querySelectorAll('.activity').forEach((el) => {\n const id = el.getAttribute('data-activity-id');\n const existing = this.pipeline.activities.find(a => a.id === id);\n if (existing) {\n activities.push({\n ...existing,\n x: el.offsetLeft,\n y: el.offsetTop,\n isStart: el.getAttribute('data-activity-start') === 'true'\n });\n }\n });\n\n const transitions: EtlPipeline.Transition[] = [];\n const connections = this.jsPlumbInstance.getConnections() as any[];\n connections.forEach((conn: any) => {\n const sourceEndpoint = conn.endpoints[0];\n const outcome = sourceEndpoint.getParameter('outcome');\n const targetEl = conn.target;\n transitions.push({\n sourceActivityId: conn.source.getAttribute('data-activity-id'),\n destinationActivityId: targetEl.getAttribute('data-activity-id'),\n sourceOutcomeName: outcome ? outcome.name : 'Done'\n });\n });\n\n return {\n ...this.pipeline,\n activities: activities,\n transitions: transitions,\n removedActivities: this.pipeline.removedActivities || []\n };\n }\n\n public serialize(): string {\n return JSON.stringify(this.getState());\n }\n\n private saveLocalState() {\n try {\n sessionStorage.setItem(`etl-pipeline-${this.localId}`, this.serialize());\n } catch {\n // Storage full or unavailable\n }\n }\n\n private loadLocalState(): EtlPipeline.Pipeline | null {\n try {\n const json = sessionStorage.getItem(`etl-pipeline-${this.localId}`);\n return json ? JSON.parse(json) : null;\n } catch {\n return null;\n }\n }\n}\n\nfunction initEtlPipelineEditor(): void {\n const canvas = document.querySelector('.etl-canvas');\n if (!canvas) return;\n\n const pipelineData = canvas.dataset.pipeline;\n if (!pipelineData) return;\n\n const pipeline: EtlPipeline.Pipeline = JSON.parse(pipelineData);\n pipeline.removedActivities = pipeline.removedActivities || [];\n\n const deletePrompt = canvas.dataset.deleteActivityPrompt || 'Are you sure?';\n const localId = canvas.dataset.localId || '';\n const loadLocalState = canvas.dataset.loadLocalState === 'true';\n\n const editor = new EtlPipelineEditor(\n canvas,\n pipeline,\n deletePrompt,\n localId,\n loadLocalState\n );\n\n // Serialize state into hidden input on form submit\n const form = document.getElementById('pipelineEditorForm') as HTMLFormElement | null;\n const stateInput = document.getElementById('pipelineStateInput') as HTMLInputElement | null;\n const persistPipeline = async (): Promise => {\n if (!form || !stateInput) {\n return true;\n }\n\n stateInput.value = editor.serialize();\n editor.saveCurrentState();\n\n const formData = new FormData(form);\n\n try {\n const response = await fetch(form.action, {\n method: 'POST',\n body: formData,\n credentials: 'same-origin',\n headers: {\n 'X-Requested-With': 'XMLHttpRequest'\n }\n });\n\n return response.ok;\n } catch {\n return false;\n }\n };\n\n if (form && stateInput) {\n form.addEventListener('submit', () => {\n stateInput.value = editor.serialize();\n });\n }\n\n const statePreservingLinks = document.querySelectorAll('.activity-add-action, .activity-edit-action');\n statePreservingLinks.forEach((link) => {\n link.addEventListener('click', async (event: Event) => {\n event.preventDefault();\n const succeeded = await persistPipeline();\n if (succeeded && link instanceof HTMLAnchorElement) {\n window.location.href = link.href;\n }\n });\n });\n\n // Activity picker modal category filtering\n const pickerModal = document.getElementById('activity-picker');\n if (pickerModal) {\n let activeActivityType = 'all';\n\n const applyActivityFilters = (): void => {\n const searchInput = pickerModal.querySelector('.activity-search');\n const query = (searchInput?.value || '').toLowerCase();\n const cards = pickerModal.querySelectorAll('.activity-card');\n\n cards.forEach((card) => {\n const cardTitle = card.querySelector('.card-title');\n const text = cardTitle ? cardTitle.textContent!.toLowerCase() : '';\n const cardType = card.dataset.activityType || 'all';\n const matchesType = activeActivityType === 'all' || cardType === activeActivityType;\n const matchesQuery = !query || text.indexOf(query) >= 0;\n\n card.style.display = matchesType && matchesQuery ? '' : 'none';\n });\n };\n\n const setActiveType = (activityType: string): void => {\n activeActivityType = activityType || 'all';\n applyActivityFilters();\n };\n\n pickerModal.addEventListener('show.bs.modal', (e: Event) => {\n const modalEvent = e as any;\n const button = modalEvent.relatedTarget as HTMLElement;\n const activityType = button?.dataset.activityType || 'all';\n const title = button?.dataset.pickerTitle || 'Available Activities';\n\n const modalTitle = pickerModal.querySelector('.modal-title');\n if (modalTitle) {\n modalTitle.textContent = title;\n }\n\n const searchInput = pickerModal.querySelector('.activity-search');\n if (searchInput) {\n searchInput.value = '';\n }\n\n setActiveType(activityType);\n });\n\n const searchInput = pickerModal.querySelector('.activity-search');\n if (searchInput) {\n searchInput.addEventListener('input', () => {\n applyActivityFilters();\n });\n }\n }\n}\n\ndocument.addEventListener('DOMContentLoaded', initEtlPipelineEditor);\n"],"names":["EtlPipelineEditor","constructor","container","pipeline","deletePrompt","localId","loadLocalState","this","saved","jsPlumb","ready","init","saveCurrentState","saveLocalState","plumber","getInstance","DragOptions","cursor","zIndex","ConnectionOverlays","width","length","location","Container","jsPlumbInstance","querySelectorAll","forEach","el","setupActivity","updateConnections","updateCanvasHeight","requestAnimationFrame","repaintEverything","bind","activityId","getAttribute","activity","activities","find","a","id","draggable","grid","containment","stop","color","isTransform","isLoad","outcomes","outcome","addEndpoint","endpoint","anchor","paintStyle","fill","radius","isSource","connector","stub","gap","cornerRadius","alwaysRespectStubs","connectorStyle","strokeWidth","stroke","joinstyle","outlineStroke","outlineWidth","hoverPaintStyle","connectorHoverStyle","uuid","name","parameters","overlays","label","displayName","cssClass","visible","makeTarget","dropOptions","hoverClass","setupActivityActions","addEventListener","e","target","closest","commands","querySelector","classList","toggle","deleteBtn","confirm","removedActivities","push","remove","batch","transitions","t","sourceEndpoint","getEndpoint","sourceActivityId","sourceOutcomeName","targetEl","destinationActivityId","connect","source","type","maxY","bottom","offsetTop","offsetHeight","style","minHeight","getState","existing","x","offsetLeft","y","isStart","getConnections","conn","endpoints","getParameter","serialize","JSON","stringify","sessionStorage","setItem","json","getItem","parse","document","canvas","pipelineData","dataset","deleteActivityPrompt","editor","form","getElementById","stateInput","value","link","async","event","preventDefault","formData","FormData","fetch","action","method","body","credentials","headers","ok","persistPipeline","HTMLAnchorElement","window","href","pickerModal","activeActivityType","applyActivityFilters","searchInput","query","toLowerCase","card","cardTitle","text","textContent","cardType","activityType","matchesType","matchesQuery","indexOf","display","setActiveType","button","relatedTarget","title","pickerTitle","modalTitle"],"version":3,"file":"etl-pipeline-editor.js.map"} \ No newline at end of file diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/wwwroot/Scripts/etl-pipeline-editor.min.js b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/wwwroot/Scripts/etl-pipeline-editor.min.js new file mode 100644 index 00000000000..af0748d37bb --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/wwwroot/Scripts/etl-pipeline-editor.min.js @@ -0,0 +1 @@ +!function(t,e,i,n){var o="undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:"undefined"!=typeof window?window:"undefined"!=typeof global?global:{},a="function"==typeof o[n]&&o[n],r=a.cache||{},s="undefined"!=typeof module&&"function"==typeof module.require&&module.require.bind(module);function c(e,i){if(!r[e]){if(!t[e]){var l="function"==typeof o[n]&&o[n];if(!i&&l)return l(e,!0);if(a)return a(e,!0);if(s&&"string"==typeof e)return s(e);var d=new Error("Cannot find module '"+e+"'");throw d.code="MODULE_NOT_FOUND",d}p.resolve=function(i){var n=t[e][1][i];return null!=n?n:i},p.cache={};var u=r[e]=new c.Module(e);t[e][0].call(u.exports,p,u,u.exports,o)}return r[e].exports;function p(t){var e=p.resolve(t);return!1===e?{}:c(e)}}c.isParcelRequire=!0,c.Module=function(t){this.id=t,this.bundle=c,this.exports={}},c.modules=t,c.cache=r,c.parent=a,c.register=function(e,i){t[e]=[function(t,e){e.exports=i},{}]},Object.defineProperty(c,"root",{get:function(){return o[n]}}),o[n]=c;for(var l=0;l{this.init()})}saveCurrentState(){this.saveLocalState()}init(){const t=jsPlumb.getInstance({DragOptions:{cursor:"pointer",zIndex:2e3},ConnectionOverlays:[["Arrow",{width:12,length:12,location:-5}]],Container:this.container});this.jsPlumbInstance=t;this.container.querySelectorAll(".activity").forEach(e=>{this.setupActivity(t,e)}),this.updateConnections(t),this.updateCanvasHeight(),requestAnimationFrame(()=>t.repaintEverything()),t.bind("connection",()=>{this.saveLocalState(),this.updateCanvasHeight()}),t.bind("connectionDetached",()=>{this.saveLocalState()})}setupActivity(t,e){const i=e.getAttribute("data-activity-id"),n=this.pipeline.activities.find(t=>t.id===i);if(!n)return;t.draggable(e,{grid:[10,10],containment:!0,stop:()=>{this.updateCanvasHeight(),this.saveLocalState()}});let o="#7ab02c";n.isTransform&&(o="#3a8acd"),n.isLoad&&(o="#6c757d"),n.outcomes&&n.outcomes.forEach(n=>{t.addEndpoint(e,{endpoint:"Dot",anchor:"Right",paintStyle:{fill:o,radius:7},isSource:!0,connector:["Flowchart",{stub:[40,60],gap:0,cornerRadius:5,alwaysRespectStubs:!0}],connectorStyle:{strokeWidth:2,stroke:"#999999",joinstyle:"round",outlineStroke:"white",outlineWidth:2},hoverPaintStyle:{fill:"#216477",stroke:"#216477"},connectorHoverStyle:{strokeWidth:3,stroke:"#216477"},uuid:`${i}-${n.name}`,parameters:{outcome:n},overlays:[["Label",{location:[.5,1.5],label:n.displayName,cssClass:"outcome-label",visible:!0}]]})}),t.makeTarget(e,{dropOptions:{hoverClass:"hover"},anchor:"Left",endpoint:["Blank",{radius:8}]}),this.setupActivityActions(e,n)}setupActivityActions(t,e){t.addEventListener("click",e=>{if(e.target.closest(".activity-commands"))return;const i=t.querySelector(".activity-commands");i&&i.classList.toggle("d-none")});const i=t.querySelector(".activity-delete-action");i&&i.addEventListener("click",()=>{confirm("Are you sure you want to delete this activity?")&&(this.pipeline.removedActivities=this.pipeline.removedActivities||[],this.pipeline.removedActivities.push(e.id),this.jsPlumbInstance.remove(t),this.saveLocalState())})}updateConnections(t){t.batch(()=>{this.pipeline.transitions.forEach(e=>{const i=t.getEndpoint(`${e.sourceActivityId}-${e.sourceOutcomeName}`),n=this.container.querySelector(`[data-activity-id="${e.destinationActivityId}"]`);i&&n&&t.connect({source:i,target:n,type:"basic"})})})}updateCanvasHeight(){let t=400;this.container.querySelectorAll(".activity").forEach(e=>{const i=e.offsetTop+e.offsetHeight+50;i>t&&(t=i)}),this.container.style.minHeight=t+"px"}getState(){const t=[];this.container.querySelectorAll(".activity").forEach(e=>{const i=e.getAttribute("data-activity-id"),n=this.pipeline.activities.find(t=>t.id===i);n&&t.push({...n,x:e.offsetLeft,y:e.offsetTop,isStart:"true"===e.getAttribute("data-activity-start")})});const e=[];return this.jsPlumbInstance.getConnections().forEach(t=>{const i=t.endpoints[0].getParameter("outcome"),n=t.target;e.push({sourceActivityId:t.source.getAttribute("data-activity-id"),destinationActivityId:n.getAttribute("data-activity-id"),sourceOutcomeName:i?i.name:"Done"})}),{...this.pipeline,activities:t,transitions:e,removedActivities:this.pipeline.removedActivities||[]}}serialize(){return JSON.stringify(this.getState())}saveLocalState(){try{sessionStorage.setItem(`etl-pipeline-${this.localId}`,this.serialize())}catch{}}loadLocalState(){try{const t=sessionStorage.getItem(`etl-pipeline-${this.localId}`);return t?JSON.parse(t):null}catch{return null}}}document.addEventListener("DOMContentLoaded",function(){const t=document.querySelector(".etl-canvas");if(!t)return;const e=t.dataset.pipeline;if(!e)return;const i=JSON.parse(e);i.removedActivities=i.removedActivities||[];const n=t.dataset.deleteActivityPrompt||"Are you sure?",a=t.dataset.localId||"",r="true"===t.dataset.loadLocalState,s=new o(t,i,n,a,r),c=document.getElementById("pipelineEditorForm"),l=document.getElementById("pipelineStateInput");c&&l&&c.addEventListener("submit",()=>{l.value=s.serialize()}),document.querySelectorAll(".activity-add-action, .activity-edit-action").forEach(t=>{t.addEventListener("click",async e=>{e.preventDefault();await(async()=>{if(!c||!l)return!0;l.value=s.serialize(),s.saveCurrentState();const t=new FormData(c);try{return(await fetch(c.action,{method:"POST",body:t,credentials:"same-origin",headers:{"X-Requested-With":"XMLHttpRequest"}})).ok}catch{return!1}})()&&t instanceof HTMLAnchorElement&&(window.location.href=t.href)})});const d=document.getElementById("activity-picker");if(d){let t="all";const e=()=>{const e=d.querySelector(".activity-search"),i=(e?.value||"").toLowerCase();d.querySelectorAll(".activity-card").forEach(e=>{const n=e.querySelector(".card-title"),o=n?n.textContent.toLowerCase():"",a=e.dataset.activityType||"all",r="all"===t||a===t,s=!i||o.indexOf(i)>=0;e.style.display=r&&s?"":"none"})},i=i=>{t=i||"all",e()};d.addEventListener("show.bs.modal",t=>{const e=t.relatedTarget,n=e?.dataset.activityType||"all",o=e?.dataset.pickerTitle||"Available Activities",a=d.querySelector(".modal-title");a&&(a.textContent=o);const r=d.querySelector(".activity-search");r&&(r.value=""),i(n)});const n=d.querySelector(".activity-search");n&&n.addEventListener("input",()=>{e()})}})},{}]},["2W7JX"],"2W7JX","parcelRequire94c2"); diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/wwwroot/Styles/etl-pipeline-editor.css b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/wwwroot/Styles/etl-pipeline-editor.css new file mode 100644 index 00000000000..2c317b98a9d --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/wwwroot/Styles/etl-pipeline-editor.css @@ -0,0 +1,128 @@ +.etl-container { + border: 1px solid var(--bs-border-color); + border-radius: 0.25rem; + overflow: hidden; +} + +.etl-canvas-container { + overflow: auto; + max-height: 700px; +} + +.etl-canvas { + position: relative; + min-height: 400px; + overflow: auto; + background-color: var(--bs-body-bg); + background-image: linear-gradient(var(--bs-border-color-translucent) 1px, transparent 1px), linear-gradient(90deg, var(--bs-border-color-translucent) 1px, transparent 1px); + background-size: 20px 20px; +} +.etl-canvas .activity { + position: absolute; + border: 1px solid #346789; + border-radius: 4px; + background-color: var(--bs-secondary-bg); + padding: 1em; + min-width: 120px; + min-height: 60px; + max-width: 250px; + opacity: 0.9; + cursor: move; + z-index: 20; + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.12); + transition: box-shadow 0.2s, opacity 0.2s; +} +.etl-canvas .activity:hover { + box-shadow: 2px 2px 12px rgba(0, 0, 0, 0.3); + opacity: 1; +} +.etl-canvas .activity.activity-source { + border-color: #7ab02c; + border-width: 2px; +} +.etl-canvas .activity.activity-source header h4 i { + color: #7ab02c; +} +.etl-canvas .activity.activity-transform { + border-color: #3a8acd; + border-width: 2px; +} +.etl-canvas .activity.activity-transform header h4 i { + color: #3a8acd; +} +.etl-canvas .activity.activity-load { + border-color: #6c757d; + border-width: 2px; +} +.etl-canvas .activity.activity-load header h4 i { + color: #6c757d; +} +.etl-canvas .activity.activity-start { + border-width: 3px; + font-weight: bold; +} +.etl-canvas .activity header h4 { + font-size: 0.9rem; + margin-bottom: 0.25rem; +} +.etl-canvas .activity header h4 i { + margin-right: 0.3rem; +} +.etl-canvas .activity .activity-commands { + position: absolute; + top: -30px; + right: 0; + white-space: nowrap; +} +.etl-canvas .activity .activity-commands .btn { + border-radius: 50%; + width: 24px; + height: 24px; + padding: 0; + font-size: 0.7rem; + line-height: 24px; +} +.etl-canvas .jtk-endpoint { + z-index: 21; + cursor: pointer; +} +.etl-canvas .connection-label { + z-index: 31; + border: 1px solid var(--bs-border-color); + padding: 0 0.5rem; + border-radius: 2px; + background-color: var(--bs-secondary-bg); + font-size: 0.75rem; +} +.etl-canvas .outcome-label { + z-index: 21; + font-size: 10px; + font-weight: 500; + color: var(--bs-body-color); + background-color: var(--bs-body-bg); + border: 1px solid var(--bs-border-color); + padding: 0.1rem 0.4rem; + border-radius: 2px; + white-space: nowrap; +} + +.modal-activities .activity-picker-categories { + gap: 0.25rem; +} +.modal-activities .activity-card { + margin-bottom: 0.5rem; +} +.modal-activities .activity-card .card { + transition: border-color 0.2s, box-shadow 0.2s, transform 0.2s; +} +.modal-activities .activity-card .card:hover { + border-color: var(--bs-primary); + box-shadow: 0 0.5rem 1rem rgba(0, 0, 0, 0.08); + transform: translateY(-1px); +} +.modal-activities .activity-card .card-title { + font-size: 0.9rem; +} +.modal-activities .activity-card .card-text { + font-size: 0.8rem; +} \ No newline at end of file diff --git a/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/wwwroot/Styles/etl-pipeline-editor.min.css b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/wwwroot/Styles/etl-pipeline-editor.min.css new file mode 100644 index 00000000000..b31ce45260c --- /dev/null +++ b/src/OrchardCore.Modules/OrchardCore.DataOrchestrator/wwwroot/Styles/etl-pipeline-editor.min.css @@ -0,0 +1 @@ +.etl-container{border:1px solid var(--bs-border-color);border-radius:.25rem;overflow:hidden}.etl-canvas-container{max-height:700px;overflow:auto}.etl-canvas{background-color:var(--bs-body-bg);background-image:linear-gradient(var(--bs-border-color-translucent)1px,transparent 1px),linear-gradient(90deg,var(--bs-border-color-translucent)1px,transparent 1px);background-size:20px 20px;min-height:400px;position:relative;overflow:auto}.etl-canvas .activity{background-color:var(--bs-secondary-bg);opacity:.9;cursor:move;z-index:20;border:1px solid #346789;border-radius:4px;min-width:120px;max-width:250px;min-height:60px;padding:1em;transition:box-shadow .2s,opacity .2s;position:absolute;box-shadow:0 1px 3px #0000001f}.etl-canvas .activity:hover{opacity:1;box-shadow:2px 2px 12px #0000004d}.etl-canvas .activity.activity-source{border-width:2px;border-color:#7ab02c}.etl-canvas .activity.activity-source header h4 i{color:#7ab02c}.etl-canvas .activity.activity-transform{border-width:2px;border-color:#3a8acd}.etl-canvas .activity.activity-transform header h4 i{color:#3a8acd}.etl-canvas .activity.activity-load{border-width:2px;border-color:#6c757d}.etl-canvas .activity.activity-load header h4 i{color:#6c757d}.etl-canvas .activity.activity-start{border-width:3px;font-weight:700}.etl-canvas .activity header h4{margin-bottom:.25rem;font-size:.9rem}.etl-canvas .activity header h4 i{margin-right:.3rem}.etl-canvas .activity .activity-commands{white-space:nowrap;position:absolute;top:-30px;right:0}.etl-canvas .activity .activity-commands .btn{border-radius:50%;width:24px;height:24px;padding:0;font-size:.7rem;line-height:24px}.etl-canvas .jtk-endpoint{z-index:21;cursor:pointer}.etl-canvas .connection-label{z-index:31;border:1px solid var(--bs-border-color);background-color:var(--bs-secondary-bg);border-radius:2px;padding:0 .5rem;font-size:.75rem}.etl-canvas .outcome-label{z-index:21;color:var(--bs-body-color);background-color:var(--bs-body-bg);border:1px solid var(--bs-border-color);white-space:nowrap;border-radius:2px;padding:.1rem .4rem;font-size:10px;font-weight:500}.modal-activities .activity-picker-categories{gap:.25rem}.modal-activities .activity-card{margin-bottom:.5rem}.modal-activities .activity-card .card{transition:border-color .2s,box-shadow .2s,transform .2s}.modal-activities .activity-card .card:hover{border-color:var(--bs-primary);transform:translateY(-1px);box-shadow:0 .5rem 1rem #00000014}.modal-activities .activity-card .card-title{font-size:.9rem}.modal-activities .activity-card .card-text{font-size:.8rem} \ No newline at end of file diff --git a/src/OrchardCore/OrchardCore.Application.Cms.Core.Targets/OrchardCore.Application.Cms.Core.Targets.csproj b/src/OrchardCore/OrchardCore.Application.Cms.Core.Targets/OrchardCore.Application.Cms.Core.Targets.csproj index f98984a6726..a3317f14e80 100644 --- a/src/OrchardCore/OrchardCore.Application.Cms.Core.Targets/OrchardCore.Application.Cms.Core.Targets.csproj +++ b/src/OrchardCore/OrchardCore.Application.Cms.Core.Targets/OrchardCore.Application.Cms.Core.Targets.csproj @@ -51,9 +51,11 @@ + + diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentExportContext.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentExportContext.cs new file mode 100644 index 00000000000..1cafe14a1bd --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentExportContext.cs @@ -0,0 +1,8 @@ +using System.Data; + +namespace OrchardCore.ContentTransfer; + +public sealed class ContentExportContext : ImportContentContext +{ + public DataRow Row { get; set; } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentFieldExportMapContext.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentFieldExportMapContext.cs new file mode 100644 index 00000000000..0e70846f4fd --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentFieldExportMapContext.cs @@ -0,0 +1,11 @@ +using System.Data; +using OrchardCore.ContentManagement; + +namespace OrchardCore.ContentTransfer; + +public sealed class ContentFieldExportMapContext : ImportContentFieldContext +{ + public ContentField ContentField { get; set; } + + public DataRow Row { get; set; } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentFieldImportMapContext.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentFieldImportMapContext.cs new file mode 100644 index 00000000000..a8f36cf80c3 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentFieldImportMapContext.cs @@ -0,0 +1,10 @@ +using System.Data; + +namespace OrchardCore.ContentTransfer; + +public sealed class ContentFieldImportMapContext : ImportContentFieldContext +{ + public DataColumnCollection Columns { get; set; } + + public DataRow Row { get; set; } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentImportContext.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentImportContext.cs new file mode 100644 index 00000000000..bbcf7c0f4b5 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentImportContext.cs @@ -0,0 +1,10 @@ +using System.Data; + +namespace OrchardCore.ContentTransfer; + +public sealed class ContentImportContext : ImportContentContext +{ + public DataColumnCollection Columns { get; set; } + + public DataRow Row { get; set; } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentPartExportMapContext.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentPartExportMapContext.cs new file mode 100644 index 00000000000..196af21c6cd --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentPartExportMapContext.cs @@ -0,0 +1,13 @@ +using System.Data; +using OrchardCore.ContentManagement; + +namespace OrchardCore.ContentTransfer; + +public sealed class ContentPartExportMapContext : ImportContentPartContext +{ + public ContentPart ContentPart { get; set; } + + public ContentItem ContentItem { get; set; } + + public DataRow Row { get; set; } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentPartImportMapContext.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentPartImportMapContext.cs new file mode 100644 index 00000000000..303b97fba7d --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentPartImportMapContext.cs @@ -0,0 +1,13 @@ +using System.Data; +using OrchardCore.ContentManagement; + +namespace OrchardCore.ContentTransfer; + +public sealed class ContentPartImportMapContext : ImportContentPartContext +{ + public ContentItem ContentItem { get; set; } + + public DataColumnCollection Columns { get; set; } + + public DataRow Row { get; set; } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentTransferEntry.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentTransferEntry.cs new file mode 100644 index 00000000000..488923cda09 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentTransferEntry.cs @@ -0,0 +1,85 @@ +using OrchardCore.Entities; + +namespace OrchardCore.ContentTransfer; + +public sealed class ContentTransferEntry : Entity +{ + /// + /// The primary key in the database. + /// + public long Id { get; set; } + + /// + /// The logical identifier of the entry. + /// + public string EntryId { get; set; } + + /// + /// The content type being imported. + /// + public string ContentType { get; set; } + + /// + /// When the entry was been created or first created. + /// + public DateTime CreatedUtc { get; set; } + + /// + /// When the entry was last processed. + /// + public DateTime? ProcessSaveUtc { get; set; } + + /// + /// When the entry finished processing. + /// + public DateTime? CompletedUtc { get; set; } + + /// + /// The user id of the user who created this entry. + /// + public string Owner { get; set; } + + /// + /// The name of the user who last modified this entry. + /// + public string Author { get; set; } + + /// + /// The Original file name. + /// + public string UploadedFileName { get; set; } + + /// + /// The stored file name. + /// + public string StoredFileName { get; set; } + + public ContentTransferEntryStatus Status { get; set; } + + /// + /// The direction of the transfer (Import or Export). + /// + public ContentTransferDirection Direction { get; set; } + + /// + /// Error message if the file failed processing. + /// + public string Error { get; set; } +} + +public enum ContentTransferEntryStatus +{ + New, + Processing, + Completed, + CompletedWithErrors, + Canceled, + CanceledWithImportedRecords, + Failed, +} + +public enum ContentTransferDirection +{ + Import, + Export, +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentTransferPermissions.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentTransferPermissions.cs new file mode 100644 index 00000000000..259e99b493a --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ContentTransferPermissions.cs @@ -0,0 +1,44 @@ +using OrchardCore.Security.Permissions; + +namespace OrchardCore.ContentTransfer; + +public sealed class ContentTransferPermissions +{ + public static readonly Permission ListContentTransferEntries = new("ListContentTransferEntries", "List content transfer entries"); + public static readonly Permission DeleteContentTransferEntries = new("DeleteContentTransferEntries", "Delete content transfer entries", new[] { ListContentTransferEntries }); + + public static readonly Permission ImportContentFromFile = new("ImportContentFromFile", "Import content items from file"); + public static readonly Permission ImportContentFromFileOfType = new("ImportContentFromFile_{0}", "Import {0} content items from file", new[] { ImportContentFromFile }); + + public static readonly Permission ExportContentFromFile = new("ExportContentFromFile", "Export content items from file"); + public static readonly Permission ExportContentFromFileOfType = new("ExportContentFromFile_{0}", "Export {0} content items from file", new[] { ExportContentFromFile }); + + private static Dictionary, Permission> _permissionsByType = new(); + + public static Permission CreateDynamicPermission(Permission template, string contentType) + { + ArgumentNullException.ThrowIfNull(template); + + var key = new ValueTuple(template.Name, contentType); + + if (_permissionsByType.TryGetValue(key, out var permission)) + { + return permission; + } + + permission = new Permission( + string.Format(template.Name, contentType), + string.Format(template.Description, contentType), + (template.ImpliedBy ?? []).Select(t => CreateDynamicPermission(t, contentType) + ) + ); + + var localPermissions = new Dictionary, Permission>(_permissionsByType) + { + [key] = permission, + }; + _permissionsByType = localPermissions; + + return permission; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentFieldImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentFieldImportHandler.cs new file mode 100644 index 00000000000..c3e09d986d2 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentFieldImportHandler.cs @@ -0,0 +1,29 @@ +namespace OrchardCore.ContentTransfer; + +/// +/// Handles import and export operations for a specific content field type. +/// Each implementation is resolved by field name via . +/// The StandardFieldImportHandler base class in the Core project provides a convenient implementation for most field types. +/// +public interface IContentFieldImportHandler +{ + /// + /// Returns the column definitions that this field handler supports for import and export. + /// Column names follow the convention: {PartName}_{FieldName}_{PropertyName}. + /// + /// The context containing the content part field definition and part name. + /// A read-only collection of import column definitions for this field. + IReadOnlyCollection GetColumns(ImportContentFieldContext context); + + /// + /// Imports a value from the spreadsheet row and sets it on the content field. + /// + /// The import context containing the data row, content part, and field definition. + Task ImportAsync(ContentFieldImportMapContext context); + + /// + /// Exports a value from the content field into the spreadsheet row. + /// + /// The export context containing the content field, part, and target data row. + Task ExportAsync(ContentFieldExportMapContext context); +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentImportHandler.cs new file mode 100644 index 00000000000..c09136fcde9 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentImportHandler.cs @@ -0,0 +1,31 @@ +namespace OrchardCore.ContentTransfer; + +/// +/// Handles import and export operations at the content item level. +/// Implementations process content item properties that are not specific to any part or field, +/// such as ContentItemId, CreatedUtc, and ModifiedUtc. +/// +public interface IContentImportHandler +{ + /// + /// Returns the column definitions that this handler supports for import and export. + /// Each column describes a data field that can be mapped to/from a spreadsheet column. + /// + /// The context containing the content type definition and content item. + /// A read-only collection of import column definitions. + IReadOnlyCollection GetColumns(ImportContentContext context); + + /// + /// Imports data from a spreadsheet row into the content item. + /// Maps values from to . + /// + /// The import context containing the data row and target content item. + Task ImportAsync(ContentImportContext content); + + /// + /// Exports data from the content item into a spreadsheet row. + /// Maps values from to . + /// + /// The export context containing the content item and target data row. + Task ExportAsync(ContentExportContext content); +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentImportHandlerResolver.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentImportHandlerResolver.cs new file mode 100644 index 00000000000..a09f1cc1c3e --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentImportHandlerResolver.cs @@ -0,0 +1,23 @@ +namespace OrchardCore.ContentTransfer; + +/// +/// Resolves import/export handlers for content parts and fields by their type name. +/// Used by to look up the appropriate handlers +/// when iterating over a content type's parts and fields. +/// +public interface IContentImportHandlerResolver +{ + /// + /// Returns the registered field import handlers for the given content field type name. + /// + /// The content field type name (e.g., "TextField", "BooleanField"). + /// A list of handlers registered for the specified field type. + IList GetFieldHandlers(string fieldName); + + /// + /// Returns the registered part import handlers for the given content part type name. + /// + /// The content part type name (e.g., "TitlePart", "AutoroutePart"). + /// A list of handlers registered for the specified part type. + IList GetPartHandlers(string partName); +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentImportManager.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentImportManager.cs new file mode 100644 index 00000000000..662b1a456d4 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentImportManager.cs @@ -0,0 +1,33 @@ +namespace OrchardCore.ContentTransfer; + +/// +/// The main service that orchestrates the import and export of content items. +/// Coordinates , , +/// and to map data between spreadsheet rows and content items. +/// +public interface IContentImportManager +{ + /// + /// Collects all available column definitions for a content type by invoking all registered + /// content, part, and field handlers. Used to generate spreadsheet headers and UI schemas. + /// + /// The context containing the content type definition. + /// A read-only collection of all import columns supported by the content type. + Task> GetColumnsAsync(ImportContentContext context); + + /// + /// Maps a spreadsheet data row to a content item by invoking all registered content, part, + /// and field import handlers. The handlers read values from the data row and set them + /// on the content item's parts and fields. + /// + /// The import context containing the data row and target content item. + Task ImportAsync(ContentImportContext context); + + /// + /// Maps a content item to a spreadsheet data row by invoking all registered content, part, + /// and field export handlers. The handlers read values from the content item's parts and fields + /// and write them to the data row. + /// + /// The export context containing the content item and target data row. + Task ExportAsync(ContentExportContext context); +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentPartImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentPartImportHandler.cs new file mode 100644 index 00000000000..26da5c2985c --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentPartImportHandler.cs @@ -0,0 +1,27 @@ +namespace OrchardCore.ContentTransfer; + +/// +/// Handles import and export operations for a specific content part type. +/// Each implementation is resolved by part name via . +/// +public interface IContentPartImportHandler +{ + /// + /// Returns the column definitions that this part handler supports for import and export. + /// + /// The context containing the content type part definition. + /// A read-only collection of import column definitions for this part. + IReadOnlyCollection GetColumns(ImportContentPartContext context); + + /// + /// Imports data from a spreadsheet row into the content part. + /// + /// The import context containing the data row and target content item. + Task ImportAsync(ContentPartImportMapContext content); + + /// + /// Exports data from the content part into a spreadsheet row. + /// + /// The export context containing the content part and target data row. + Task ExportAsync(ContentPartExportMapContext content); +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentTransferFileStore.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentTransferFileStore.cs new file mode 100644 index 00000000000..44d511748f0 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentTransferFileStore.cs @@ -0,0 +1,13 @@ +using OrchardCore.FileStorage; + +namespace OrchardCore.ContentTransfer; + +/// +/// A dedicated file store for content transfer import and export files. +/// Extends to provide isolated storage for uploaded Excel files +/// (imports) and generated export files. Files are typically stored under the tenant's +/// App_Data directory in a "ContentTransfer" subfolder. +/// +public interface IContentTransferFileStore : IFileStore +{ +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentTransferNotificationHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentTransferNotificationHandler.cs new file mode 100644 index 00000000000..0f0fe8392d3 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/IContentTransferNotificationHandler.cs @@ -0,0 +1,17 @@ +namespace OrchardCore.ContentTransfer; + +/// +/// Handles notifications for content transfer operations. +/// Resolved optionally by the export background task to notify users when their +/// queued export is ready for download. Implemented when the Notifications module is enabled. +/// +public interface IContentTransferNotificationHandler +{ + /// + /// Sends a notification to the user who requested the export, informing them + /// that the export file is ready for download from the Export Dashboard. + /// + /// The completed export entry containing the owner and file details. + /// The display name of the content type that was exported. + Task NotifyExportCompletedAsync(ContentTransferEntry entry, string contentTypeName); +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ImportColumn.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ImportColumn.cs new file mode 100644 index 00000000000..3177d44e2df --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ImportColumn.cs @@ -0,0 +1,23 @@ +namespace OrchardCore.ContentTransfer; + +public sealed class ImportColumn +{ + public string Name { get; set; } + + public string Description { get; set; } + + public bool IsRequired { get; set; } + + public string[] AdditionalNames { get; set; } = Array.Empty(); + + public string[] ValidValues { get; set; } = Array.Empty(); + + public ImportColumnType Type { get; set; } +} + +public enum ImportColumnType +{ + All, + ImportOnly, + ExportOnly, +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ImportContentContext.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ImportContentContext.cs new file mode 100644 index 00000000000..a09ad9b2314 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ImportContentContext.cs @@ -0,0 +1,11 @@ +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata.Models; + +namespace OrchardCore.ContentTransfer; + +public class ImportContentContext +{ + public ContentItem ContentItem { get; set; } + + public ContentTypeDefinition ContentTypeDefinition { get; set; } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ImportContentFieldContext.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ImportContentFieldContext.cs new file mode 100644 index 00000000000..625cc11a415 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ImportContentFieldContext.cs @@ -0,0 +1,15 @@ +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata.Models; + +namespace OrchardCore.ContentTransfer; + +public class ImportContentFieldContext +{ + public ContentItem ContentItem { get; set; } + + public ContentPartFieldDefinition ContentPartFieldDefinition { get; set; } + + public string PartName { get; set; } + + public ContentPart ContentPart { get; set; } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ImportContentPartContext.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ImportContentPartContext.cs new file mode 100644 index 00000000000..dfd84dc8a36 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ImportContentPartContext.cs @@ -0,0 +1,8 @@ +using OrchardCore.ContentManagement.Metadata.Models; + +namespace OrchardCore.ContentTransfer; + +public class ImportContentPartContext +{ + public ContentTypePartDefinition ContentTypePartDefinition { get; set; } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/OrchardCore.ContentTransfer.Abstractions.csproj b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/OrchardCore.ContentTransfer.Abstractions.csproj new file mode 100644 index 00000000000..f039f0ad4b2 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/OrchardCore.ContentTransfer.Abstractions.csproj @@ -0,0 +1,26 @@ + + + + OrchardCore.ContentTransfer + + OrchardCore ContentTransfer Abstractions + + $(OCCMSDescription) + + Abstractions for ContentTransfer Module + + $(PackageTags) OrchardCoreCMS Abstractions + + + + + + + + + + + + + + diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ValidateFieldImportContext.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ValidateFieldImportContext.cs new file mode 100644 index 00000000000..9c657c57474 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ValidateFieldImportContext.cs @@ -0,0 +1,11 @@ +using System.Data; +using OrchardCore.ContentManagement.Handlers; + +namespace OrchardCore.ContentTransfer; + +public sealed class ValidateFieldImportContext : ImportContentFieldContext +{ + public DataColumnCollection Columns { get; set; } + + public ContentValidateResult ContentValidateResult { get; } = new ContentValidateResult(); +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ValidateImportContext.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ValidateImportContext.cs new file mode 100644 index 00000000000..4a482fb32e2 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ValidateImportContext.cs @@ -0,0 +1,11 @@ +using System.Data; +using OrchardCore.ContentManagement.Handlers; + +namespace OrchardCore.ContentTransfer; + +public sealed class ValidateImportContext : ImportContentContext +{ + public DataColumnCollection Columns { get; set; } + + public ContentValidateResult ContentValidateResult { get; } = new ContentValidateResult(); +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ValidatePartImportContext.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ValidatePartImportContext.cs new file mode 100644 index 00000000000..d923d3a6793 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Abstractions/ValidatePartImportContext.cs @@ -0,0 +1,14 @@ +using System.Data; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Handlers; + +namespace OrchardCore.ContentTransfer; + +public sealed class ValidatePartImportContext : ImportContentPartContext +{ + public ContentItem ContentItem { get; set; } + + public DataColumnCollection Columns { get; set; } + + public ContentValidateResult ContentValidateResult { get; } = new ContentValidateResult(); +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/ContentHandlerOptions.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/ContentHandlerOptions.cs new file mode 100644 index 00000000000..77ffeccafd8 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/ContentHandlerOptions.cs @@ -0,0 +1,66 @@ +using OrchardCore.ContentManagement; + +namespace OrchardCore.ContentTransfer; + +public sealed class ContentHandlerOptions +{ + public readonly Dictionary> ContentParts = []; + + public readonly Dictionary> ContentFields = []; + + internal void AddPartHandler(Type contentPartType, Type handlerType) + { + var option = GetOrAddContentPart(contentPartType); + + if (!typeof(IContentPartImportHandler).IsAssignableFrom(handlerType)) + { + throw new ArgumentException("The type must inherit from " + nameof(IContentPartImportHandler)); + } + + option.Add(handlerType); + } + + internal void AddFieldHandler(Type contentFieldType, Type handlerType) + { + var option = GetOrAddContentField(contentFieldType); + + if (!typeof(IContentFieldImportHandler).IsAssignableFrom(handlerType)) + { + throw new ArgumentException("The type must inherit from " + nameof(IContentFieldImportHandler)); + } + + option.Add(handlerType); + } + + internal List GetOrAddContentPart(Type contentPartType) + { + if (!contentPartType.IsSubclassOf(typeof(ContentPart))) + { + throw new ArgumentException("The type must inherit from " + nameof(ContentPart)); + } + + if (!ContentParts.TryGetValue(contentPartType, out var handlers)) + { + handlers = []; + ContentParts.Add(contentPartType, handlers); + } + + return handlers; + } + + internal List GetOrAddContentField(Type contentFieldType) + { + if (!contentFieldType.IsSubclassOf(typeof(ContentField))) + { + throw new ArgumentException("The type must inherit from " + nameof(ContentField)); + } + + if (!ContentFields.TryGetValue(contentFieldType, out var handlers)) + { + handlers = []; + ContentFields.Add(contentFieldType, handlers); + } + + return handlers; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/ContentHandlerOptionsExtensions.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/ContentHandlerOptionsExtensions.cs new file mode 100644 index 00000000000..240e519f370 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/ContentHandlerOptionsExtensions.cs @@ -0,0 +1,33 @@ +using Microsoft.Extensions.DependencyInjection; +using OrchardCore.ContentManagement; + +namespace OrchardCore.ContentTransfer; + +public static class ContentHandlerOptionsExtensions +{ + public static IServiceCollection AddContentPartImportHandler(this IServiceCollection services) + where TPart : ContentPart + where THandler : IContentPartImportHandler + { + services.AddScoped(typeof(THandler)); + services.Configure(options => + { + options.AddPartHandler(typeof(TPart), typeof(THandler)); + }); + + return services; + } + + public static IServiceCollection AddContentFieldImportHandler(this IServiceCollection services) + where TField : ContentField + where THandler : IContentFieldImportHandler + { + services.AddScoped(typeof(THandler)); + services.Configure(options => + { + options.AddFieldHandler(typeof(TField), typeof(THandler)); + }); + + return services; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/ContentTransferConstants.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/ContentTransferConstants.cs new file mode 100644 index 00000000000..ee48fa4c7c8 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/ContentTransferConstants.cs @@ -0,0 +1,9 @@ +namespace OrchardCore.ContentTransfer; + +public static class ContentTransferConstants +{ + public static class Feature + { + public const string ModuleId = "OrchardCore.ContentTransfer"; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/ContentTransferEntryFilterEngineModelBinder.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/ContentTransferEntryFilterEngineModelBinder.cs new file mode 100644 index 00000000000..082f7388cd4 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/ContentTransferEntryFilterEngineModelBinder.cs @@ -0,0 +1,37 @@ +using Microsoft.AspNetCore.Mvc.ModelBinding; + +namespace OrchardCore.ContentTransfer; + +public sealed class ContentTransferEntryFilterEngineModelBinder : IModelBinder +{ + private readonly IContentTransferEntryAdminListFilterParser _parser; + + public ContentTransferEntryFilterEngineModelBinder(IContentTransferEntryAdminListFilterParser parser) + { + _parser = parser; + } + + public Task BindModelAsync(ModelBindingContext bindingContext) + { + ArgumentNullException.ThrowIfNull(bindingContext, nameof(bindingContext)); + + var modelName = bindingContext.ModelName; + + // Try to fetch the value of the argument by name q= + var valueProviderResult = bindingContext.ValueProvider.GetValue(modelName); + + if (valueProviderResult == ValueProviderResult.None) + { + bindingContext.Result = ModelBindingResult.Success(_parser.Parse(string.Empty)); + + return Task.CompletedTask; + } + + bindingContext.ModelState.SetModelValue(modelName, valueProviderResult); + + // When value is null or empty the parser returns an empty result. + bindingContext.Result = ModelBindingResult.Success(_parser.Parse(valueProviderResult.FirstValue)); + + return Task.CompletedTask; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/ContentTransferEntryQueryContext.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/ContentTransferEntryQueryContext.cs new file mode 100644 index 00000000000..04ae6256957 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/ContentTransferEntryQueryContext.cs @@ -0,0 +1,15 @@ +using YesSql; +using YesSql.Filters.Query.Services; + +namespace OrchardCore.ContentTransfer; + +public sealed class ContentTransferEntryQueryContext : QueryExecutionContext +{ + public ContentTransferEntryQueryContext(IServiceProvider serviceProvider, IQuery query) + : base(query) + { + ServiceProvider = serviceProvider; + } + + public IServiceProvider ServiceProvider { get; } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/ContentTransferEntryQueryResult.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/ContentTransferEntryQueryResult.cs new file mode 100644 index 00000000000..cd465a5bff4 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/ContentTransferEntryQueryResult.cs @@ -0,0 +1,8 @@ +namespace OrchardCore.ContentTransfer; + +public sealed class ContentTransferEntryQueryResult +{ + public IEnumerable Entries { get; set; } + + public int TotalCount { get; set; } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/DefaultContentTransferEntryAdminListFilterProvider.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/DefaultContentTransferEntryAdminListFilterProvider.cs new file mode 100644 index 00000000000..767b1b638f9 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/DefaultContentTransferEntryAdminListFilterProvider.cs @@ -0,0 +1,70 @@ +using OrchardCore.ContentTransfer.Indexes; +using OrchardCore.ContentTransfer.Models; +using YesSql; +using YesSql.Filters.Query; + +namespace OrchardCore.ContentTransfer; + +public sealed class DefaultContentTransferEntryAdminListFilterProvider : IContentTransferEntryAdminListFilterProvider +{ + public void Build(QueryEngineBuilder builder) + { + builder + .WithNamedTerm("status", builder => builder + .OneCondition((val, query, ctx) => + { + if (Enum.TryParse(val, true, out var status)) + { + return new ValueTask>(query.With(x => x.Status == status)); + } + + return new ValueTask>(query); + }) + .MapTo((val, model) => + { + if (Enum.TryParse(val, true, out var status)) + { + model.Status = status; + } + }) + .MapFrom((model) => + { + if (model.Status.HasValue) + { + return (true, model.Status.ToString()); + } + + return (false, string.Empty); + }) + .AlwaysRun() + ) + .WithNamedTerm("sort", builder => builder + .OneCondition((val, query, ctx) => + { + if (Enum.TryParse(val, true, out var sort) && sort == ContentTransferEntryOrder.Oldest) + { + return new ValueTask>(query.With().OrderBy(x => x.CreatedUtc)); + } + + return new ValueTask>(query.With().OrderByDescending(x => x.CreatedUtc)); + }) + .MapTo((val, model) => + { + if (Enum.TryParse(val, true, out var sort)) + { + model.OrderBy = sort; + } + }) + .MapFrom((model) => + { + if (model.OrderBy.HasValue) + { + return (true, model.OrderBy.ToString()); + } + + return (false, string.Empty); + }) + .AlwaysRun() + ); + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/AliasPartContentImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/AliasPartContentImportHandler.cs new file mode 100644 index 00000000000..8c78005071d --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/AliasPartContentImportHandler.cs @@ -0,0 +1,87 @@ +using System.Data; +using Microsoft.Extensions.Localization; +using OrchardCore.Alias.Models; +using OrchardCore.ContentManagement; + +namespace OrchardCore.ContentTransfer.Handlers; + +/// +/// Handles import and export of the content part. +/// Maps the Alias property to a single spreadsheet column. +/// +public sealed class AliasPartContentImportHandler : ContentImportHandlerBase, IContentPartImportHandler +{ + private ImportColumn _column; + + internal readonly IStringLocalizer S; + + public AliasPartContentImportHandler(IStringLocalizer localizer) + { + S = localizer; + } + + public IReadOnlyCollection GetColumns(ImportContentPartContext context) + { + _column ??= new ImportColumn() + { + Name = $"{context.ContentTypePartDefinition.Name}_{nameof(AliasPart.Alias)}", + Description = S["The alias for the {0}", context.ContentTypePartDefinition.ContentTypeDefinition.DisplayName], + }; + + return [_column]; + } + + public Task ImportAsync(ContentPartImportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentItem); + ArgumentNullException.ThrowIfNull(context.Columns); + ArgumentNullException.ThrowIfNull(context.Row); + + var knownColumn = GetColumns(context).FirstOrDefault(); + + if (knownColumn != null) + { + foreach (DataColumn column in context.Columns) + { + if (!Is(column.ColumnName, knownColumn)) + { + continue; + } + + var alias = context.Row[column]?.ToString(); + + if (string.IsNullOrWhiteSpace(alias)) + { + continue; + } + + context.ContentItem.Alter(part => + { + part.Alias = alias; + }); + } + } + + return Task.CompletedTask; + } + + public Task ExportAsync(ContentPartExportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentItem); + ArgumentNullException.ThrowIfNull(context.Row); + + if (_column?.Name != null) + { + var part = context.ContentItem.As(); + + if (part != null) + { + context.Row[_column.Name] = part.Alias ?? string.Empty; + } + } + + return Task.CompletedTask; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/ArchiveLaterPartContentImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/ArchiveLaterPartContentImportHandler.cs new file mode 100644 index 00000000000..2c1cd568742 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/ArchiveLaterPartContentImportHandler.cs @@ -0,0 +1,87 @@ +using System.Data; +using Microsoft.Extensions.Localization; +using OrchardCore.ArchiveLater.Models; +using OrchardCore.ContentManagement; + +namespace OrchardCore.ContentTransfer.Handlers; + +/// +/// Handles import and export of the content part. +/// Maps the ScheduledArchiveUtc property to a single spreadsheet column. +/// +public sealed class ArchiveLaterPartContentImportHandler : ContentImportHandlerBase, IContentPartImportHandler +{ + private ImportColumn _column; + + internal readonly IStringLocalizer S; + + public ArchiveLaterPartContentImportHandler(IStringLocalizer localizer) + { + S = localizer; + } + + public IReadOnlyCollection GetColumns(ImportContentPartContext context) + { + _column ??= new ImportColumn() + { + Name = $"{context.ContentTypePartDefinition.Name}_{nameof(ArchiveLaterPart.ScheduledArchiveUtc)}", + Description = S["The scheduled archive date and time for the {0}", context.ContentTypePartDefinition.ContentTypeDefinition.DisplayName], + }; + + return [_column]; + } + + public Task ImportAsync(ContentPartImportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentItem); + ArgumentNullException.ThrowIfNull(context.Columns); + ArgumentNullException.ThrowIfNull(context.Row); + + var knownColumn = GetColumns(context).FirstOrDefault(); + + if (knownColumn != null) + { + foreach (DataColumn column in context.Columns) + { + if (!Is(column.ColumnName, knownColumn)) + { + continue; + } + + var value = context.Row[column]?.ToString(); + + if (string.IsNullOrWhiteSpace(value) || !DateTime.TryParse(value.Trim(), out var scheduledArchiveUtc)) + { + continue; + } + + context.ContentItem.Alter(part => + { + part.ScheduledArchiveUtc = scheduledArchiveUtc; + }); + } + } + + return Task.CompletedTask; + } + + public Task ExportAsync(ContentPartExportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentItem); + ArgumentNullException.ThrowIfNull(context.Row); + + if (_column?.Name != null) + { + var part = context.ContentItem.As(); + + if (part != null) + { + context.Row[_column.Name] = part.ScheduledArchiveUtc; + } + } + + return Task.CompletedTask; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/AutoroutePartContentImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/AutoroutePartContentImportHandler.cs new file mode 100644 index 00000000000..e09ea51899a --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/AutoroutePartContentImportHandler.cs @@ -0,0 +1,87 @@ +using System.Data; +using Microsoft.Extensions.Localization; +using OrchardCore.Autoroute.Models; +using OrchardCore.ContentManagement; + +namespace OrchardCore.ContentTransfer.Handlers; + +/// +/// Handles import and export of the content part. +/// Maps the Path property to a single spreadsheet column. +/// +public sealed class AutoroutePartContentImportHandler : ContentImportHandlerBase, IContentPartImportHandler +{ + internal readonly IStringLocalizer S; + + private ImportColumn _column; + + public AutoroutePartContentImportHandler(IStringLocalizer localizer) + { + S = localizer; + } + + public IReadOnlyCollection GetColumns(ImportContentPartContext context) + { + _column ??= new ImportColumn() + { + Name = $"{context.ContentTypePartDefinition.Name}_{nameof(AutoroutePart.Path)}", + Description = S["The URL path for the {0}", context.ContentTypePartDefinition.ContentTypeDefinition.DisplayName], + }; + + return [_column]; + } + + public Task ImportAsync(ContentPartImportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentItem); + ArgumentNullException.ThrowIfNull(context.Columns); + ArgumentNullException.ThrowIfNull(context.Row); + + var knownColumn = GetColumns(context).FirstOrDefault(); + + if (knownColumn != null) + { + foreach (DataColumn column in context.Columns) + { + if (!Is(column.ColumnName, knownColumn)) + { + continue; + } + + var path = context.Row[column]?.ToString(); + + if (string.IsNullOrWhiteSpace(path)) + { + continue; + } + + context.ContentItem.Alter(part => + { + part.Path = path; + }); + } + } + + return Task.CompletedTask; + } + + public Task ExportAsync(ContentPartExportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentItem); + ArgumentNullException.ThrowIfNull(context.Row); + + if (_column?.Name != null) + { + var part = context.ContentItem.As(); + + if (part != null) + { + context.Row[_column.Name] = part.Path ?? string.Empty; + } + } + + return Task.CompletedTask; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/CommonContentImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/CommonContentImportHandler.cs new file mode 100644 index 00000000000..fad30a5878d --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/CommonContentImportHandler.cs @@ -0,0 +1,117 @@ +using System.Data; +using Microsoft.Extensions.Localization; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata.Models; + +namespace OrchardCore.ContentTransfer.Handlers; + +public sealed class CommonContentImportHandler : ContentImportHandlerBase, IContentImportHandler +{ + private readonly IContentItemIdGenerator _contentItemIdGenerator; + + internal readonly IStringLocalizer S; + + public CommonContentImportHandler( + IStringLocalizer stringLocalizer, + IContentItemIdGenerator contentItemIdGenerator) + { + _contentItemIdGenerator = contentItemIdGenerator; + S = stringLocalizer; + } + + public IReadOnlyCollection GetColumns(ImportContentContext context) + { + var columns = new List + { + new ImportColumn() + { + Name = nameof(ContentItem.ContentItemId), + Description = S["The id for the {0}", context.ContentTypeDefinition.DisplayName], + }, + new ImportColumn() + { + Name = nameof(ContentItem.CreatedUtc), + Description = S["The UTC created datetime value {0}", context.ContentTypeDefinition.DisplayName], + Type = ImportColumnType.ExportOnly, + }, + new ImportColumn() + { + Name = nameof(ContentItem.ModifiedUtc), + Description = S["The UTC last modified datetime value {0}", context.ContentTypeDefinition.DisplayName], + Type = ImportColumnType.ExportOnly, + }, + }; + + if (context.ContentTypeDefinition.IsVersionable()) + { + columns.Insert(1, new ImportColumn() + { + Name = nameof(ContentItem.ContentItemVersionId), + Description = S["The version id for the {0}", context.ContentTypeDefinition.DisplayName], + }); + } + + return columns; + } + + public Task ImportAsync(ContentImportContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentItem, nameof(context.ContentItem)); + ArgumentNullException.ThrowIfNull(context.Row, nameof(context.Row)); + + foreach (DataColumn column in context.Columns) + { + if (Is(column.ColumnName, nameof(ContentItem.ContentItemId))) + { + var contentItemId = context.Row[column]?.ToString(); + + if (string.IsNullOrWhiteSpace(contentItemId)) + { + continue; + } + + var fakeId = _contentItemIdGenerator.GenerateUniqueId(new ContentItem()); + + if (fakeId.Length == contentItemId.Length) + { + // Just check if the given id matched the fakeId length. + context.ContentItem.ContentItemId = contentItemId; + } + } + else if (context.ContentTypeDefinition.IsVersionable() && Is(column.ColumnName, nameof(ContentItem.ContentItemVersionId))) + { + var contentItemVersionId = context.Row[column]?.ToString(); + + if (!string.IsNullOrWhiteSpace(contentItemVersionId)) + { + var fakeId = _contentItemIdGenerator.GenerateUniqueId(new ContentItem()); + + if (fakeId.Length == contentItemVersionId.Length) + { + context.ContentItem.ContentItemVersionId = contentItemVersionId; + } + } + } + } + + return Task.CompletedTask; + } + + public Task ExportAsync(ContentExportContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentItem, nameof(context.ContentItem)); + ArgumentNullException.ThrowIfNull(context.Row, nameof(context.Row)); + + context.Row[nameof(ContentItem.ContentItemId)] = context.ContentItem.ContentItemId; + if (context.ContentTypeDefinition.IsVersionable()) + { + context.Row[nameof(ContentItem.ContentItemVersionId)] = context.ContentItem.ContentItemVersionId; + } + context.Row[nameof(ContentItem.CreatedUtc)] = context.ContentItem.CreatedUtc; + context.Row[nameof(ContentItem.ModifiedUtc)] = context.ContentItem.ModifiedUtc; + + return Task.CompletedTask; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/ContentImportHandlerBase.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/ContentImportHandlerBase.cs new file mode 100644 index 00000000000..a318a6e953b --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/ContentImportHandlerBase.cs @@ -0,0 +1,40 @@ +using System.Data; + +namespace OrchardCore.ContentTransfer.Handlers; + +public class ContentImportHandlerBase +{ + protected static bool Is(string columnName, params string[] terms) + { + foreach (var term in terms) + { + if (string.Equals(columnName, term, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + protected static bool Is(string columnName, ImportColumn importColumn) + { + if (string.Equals(columnName, importColumn.Name, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + foreach (var term in importColumn.AdditionalNames ?? []) + { + if (string.Equals(columnName, term, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + protected static string[] SplitCellValues(DataRow row, DataColumn column, string separator = ",") + => row[column]?.ToString()?.Split(separator, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) ?? []; +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/BooleanFieldImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/BooleanFieldImportHandler.cs new file mode 100644 index 00000000000..a53400837ba --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/BooleanFieldImportHandler.cs @@ -0,0 +1,58 @@ +using Microsoft.Extensions.Localization; +using OrchardCore.ContentFields.Fields; +using OrchardCore.ContentFields.Settings; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata.Models; + +namespace OrchardCore.ContentTransfer.Handlers.Fields; + +public sealed class BooleanFieldImportHandler : StandardFieldImportHandler +{ + internal readonly IStringLocalizer S; + + public BooleanFieldImportHandler(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + protected override Task SetValueAsync(ContentFieldImportMapContext context, string text) + { + var value = context.ContentPartFieldDefinition.GetSettings()?.DefaultValue ?? false; + + if (!string.IsNullOrEmpty(text) && string.Equals(bool.TrueString, text.Trim(), StringComparison.OrdinalIgnoreCase)) + { + value = true; + } + + context.ContentPart.Alter(context.ContentPartFieldDefinition.Name, (field) => + { + field.Value = value; + }); + + return Task.CompletedTask; + } + + protected override Task GetValueAsync(ContentFieldExportMapContext context) + { + var field = context.ContentPart.Get(context.ContentPartFieldDefinition.Name); + + return Task.FromResult(field?.Value); + } + + protected override string Description(ImportContentFieldContext context) + => S["A numeric value for {0}", context.ContentPartFieldDefinition.DisplayName()]; + + protected override bool IsRequired(ImportContentFieldContext context) + => false; + + protected override string BindingPropertyName => nameof(BooleanField.Value); + + protected override string[] GetValidValues(ImportContentFieldContext context) + { + return new[] + { + bool.TrueString, + bool.FalseString, + }; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/ContentPickerFieldImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/ContentPickerFieldImportHandler.cs new file mode 100644 index 00000000000..71d0062885c --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/ContentPickerFieldImportHandler.cs @@ -0,0 +1,170 @@ +using Microsoft.Extensions.Localization; +using OrchardCore.ContentFields.Fields; +using OrchardCore.ContentFields.Settings; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata; +using OrchardCore.ContentManagement.Metadata.Models; +using OrchardCore.ContentManagement.Records; +using YesSql; +using YesSql.Services; + +namespace OrchardCore.ContentTransfer.Handlers.Fields; + +public sealed class ContentPickerFieldImportHandler : StandardFieldImportHandler +{ + private readonly Dictionary> _data = new(StringComparer.OrdinalIgnoreCase); + private readonly IContentDefinitionManager _contentDefinitionManager; + private readonly ISession _session; + internal readonly IStringLocalizer S; + + public ContentPickerFieldImportHandler( + IStringLocalizer stringLocalizer, + IContentDefinitionManager contentDefinitionManager, + ISession session + ) + { + _contentDefinitionManager = contentDefinitionManager; + _session = session; + S = stringLocalizer; + } + + protected override async Task SetValueAsync(ContentFieldImportMapContext context, string text) + { + if (string.IsNullOrEmpty(text)) + { + return; + } + + var settings = context.ContentPartFieldDefinition.GetSettings(); + var contentTypes = await GetContentTypesAsync(settings); + + var items = await GetItemsAsync(contentTypes, text?.Trim()); + + context.ContentPart.Alter(context.ContentPartFieldDefinition.Name, (field) => + { + var selectedItems = items.Select(x => x.ContentItemId).ToArray(); + + if (settings.Multiple) + { + field.ContentItemIds = selectedItems; + } + else if (selectedItems.Length > 0) + { + field.ContentItemIds = [selectedItems[0]]; + } + else + { + field.ContentItemIds = []; + } + }); + } + + protected override async Task GetValueAsync(ContentFieldExportMapContext context) + { + var settings = context.ContentPartFieldDefinition.GetSettings(); + var contentTypes = await GetContentTypesAsync(settings); + + var field = context.ContentPart.Get(context.ContentPartFieldDefinition.Name); + + if (field?.ContentItemIds == null || field.ContentItemIds.Length == 0) + { + return null; + } + + var items = await GetCachedItems(contentTypes); + + var values = items.Where(x => contentTypes.Contains(x.Key)) + .SelectMany(x => x.Value) + .Where(x => field.ContentItemIds.Contains(x.ContentItemId)) + .Select(x => x.DisplayText); + + return string.Join("|", values); + } + + private async Task> GetContentTypesAsync(ContentPickerFieldSettings settings) + { + List contentTypes; + + if (settings.DisplayAllContentTypes) + { + contentTypes = (await _contentDefinitionManager.ListTypeDefinitionsAsync()) + .Select(x => x.Name) + .ToList(); + } + else if (settings.DisplayedStereotypes != null && settings.DisplayedStereotypes.Length > 0) + { + contentTypes = (await _contentDefinitionManager.ListTypeDefinitionsAsync()) + .Where(x => x.TryGetStereotype(out var stereotype) && settings.DisplayedStereotypes.Contains(stereotype)) + .Select(x => x.Name) + .ToList(); + } + else + { + contentTypes = [.. settings.DisplayedContentTypes]; + } + + return contentTypes; + } + + protected override string Description(ImportContentFieldContext context) + { + var settings = context.ContentPartFieldDefinition.GetSettings(); + + if (settings.Multiple) + { + return S["All values for {0}. Separate each value with bar (i.e., | )", context.ContentPartFieldDefinition.DisplayName()]; + } + + return S["A value for {0}", context.ContentPartFieldDefinition.DisplayName()]; + } + + protected override bool IsRequired(ImportContentFieldContext context) + { + var settings = context.ContentPartFieldDefinition.GetSettings(); + + return settings?.Required ?? false; + } + + protected override string BindingPropertyName + => nameof(ContentPickerField.ContentItemIds); + + private async Task> GetItemsAsync(IEnumerable contentTypes, string displayText) + { + await GetCachedItems(contentTypes); + + return _data.Where(x => contentTypes.Contains(x.Key)) + .SelectMany(x => x.Value) + .Where(x => string.Equals(x.DisplayText, displayText, StringComparison.OrdinalIgnoreCase)); + } + + private async Task>> GetCachedItems(IEnumerable contentTypes) + { + var missingItems = new List(); + + foreach (var contentType in contentTypes) + { + if (!_data.ContainsKey(contentType)) + { + missingItems.Add(contentType); + } + } + + if (missingItems.Count > 0) + { + var records = (await _session.QueryIndex(x => x.Published && x.ContentType.IsIn(missingItems)).ListAsync()) + .GroupBy(x => x.ContentType) + .Select(x => new + { + ContentType = x.Key, + ContentItems = x.Select(y => y), + }).ToList(); + + foreach (var record in records) + { + _data.TryAdd(record.ContentType, record.ContentItems); + } + } + + return _data; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/DateFieldImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/DateFieldImportHandler.cs new file mode 100644 index 00000000000..4a871ddb222 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/DateFieldImportHandler.cs @@ -0,0 +1,45 @@ +using Microsoft.Extensions.Localization; +using OrchardCore.ContentFields.Fields; +using OrchardCore.ContentFields.Settings; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata.Models; + +namespace OrchardCore.ContentTransfer.Handlers.Fields; + +public sealed class DateFieldImportHandler : StandardFieldImportHandler +{ + internal readonly IStringLocalizer S; + + public DateFieldImportHandler(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + protected override Task SetValueAsync(ContentFieldImportMapContext context, string text) + { + if (!string.IsNullOrEmpty(text) && DateTime.TryParse(text.Trim(), out var decimalValue)) + { + context.ContentPart.Alter(context.ContentPartFieldDefinition.Name, (field) => + { + field.Value = decimalValue; + }); + } + + return Task.CompletedTask; + } + + protected override Task GetValueAsync(ContentFieldExportMapContext context) + { + var field = context.ContentPart.Get(context.ContentPartFieldDefinition.Name); + + return Task.FromResult(field?.Value); + } + + protected override string Description(ImportContentFieldContext context) + => S["A date value for {0}", context.ContentPartFieldDefinition.DisplayName()]; + + protected override bool IsRequired(ImportContentFieldContext context) + => context.ContentPartFieldDefinition.GetSettings()?.Required ?? false; + + protected override string BindingPropertyName => nameof(DateField.Value); +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/DateTimeFieldImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/DateTimeFieldImportHandler.cs new file mode 100644 index 00000000000..14ae944b9bb --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/DateTimeFieldImportHandler.cs @@ -0,0 +1,50 @@ +using Microsoft.Extensions.Localization; +using OrchardCore.ContentFields.Fields; +using OrchardCore.ContentFields.Settings; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata.Models; + +namespace OrchardCore.ContentTransfer.Handlers.Fields; + +public sealed class DateTimeFieldImportHandler : StandardFieldImportHandler +{ + internal readonly IStringLocalizer S; + + public DateTimeFieldImportHandler(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + protected override Task SetValueAsync(ContentFieldImportMapContext context, string text) + { + if (!string.IsNullOrEmpty(text) && DateTime.TryParse(text.Trim(), out var decimalValue)) + { + context.ContentPart.Alter(context.ContentPartFieldDefinition.Name, (field) => + { + field.Value = decimalValue; + }); + } + + return Task.CompletedTask; + } + + protected override Task GetValueAsync(ContentFieldExportMapContext context) + { + var field = context.ContentPart.Get(context.ContentPartFieldDefinition.Name); + + return Task.FromResult(field?.Value); + } + + protected override string Description(ImportContentFieldContext context) + => S["A datetime value for {0}", context.ContentPartFieldDefinition.DisplayName()]; + + protected override bool IsRequired(ImportContentFieldContext context) + { + var settings = context.ContentPartFieldDefinition.GetSettings(); + + return settings?.Required ?? false; + } + + protected override string BindingPropertyName + => nameof(DateTimeField.Value); +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/HtmlFieldImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/HtmlFieldImportHandler.cs new file mode 100644 index 00000000000..472b5f99663 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/HtmlFieldImportHandler.cs @@ -0,0 +1,39 @@ +using Microsoft.Extensions.Localization; +using OrchardCore.ContentFields.Fields; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata.Models; + +namespace OrchardCore.ContentTransfer.Handlers.Fields; + +public sealed class HtmlFieldImportHandler : StandardFieldImportHandler +{ + internal readonly IStringLocalizer S; + + public HtmlFieldImportHandler(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + protected override string BindingPropertyName + => nameof(HtmlField.Html); + + protected override Task SetValueAsync(ContentFieldImportMapContext context, string text) + { + context.ContentPart.Alter(context.ContentPartFieldDefinition.Name, field => + { + field.Html = text; + }); + + return Task.CompletedTask; + } + + protected override Task GetValueAsync(ContentFieldExportMapContext context) + { + var field = context.ContentPart.Get(context.ContentPartFieldDefinition.Name); + + return Task.FromResult(field?.Html); + } + + protected override string Description(ImportContentFieldContext context) + => S["An HTML value for {0}", context.ContentPartFieldDefinition.DisplayName()]; +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/LinkFieldImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/LinkFieldImportHandler.cs new file mode 100644 index 00000000000..cc3be434f82 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/LinkFieldImportHandler.cs @@ -0,0 +1,108 @@ +using System.Data; +using Microsoft.Extensions.Localization; +using OrchardCore.ContentFields.Fields; +using OrchardCore.ContentFields.Settings; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata.Models; + +namespace OrchardCore.ContentTransfer.Handlers.Fields; + +public sealed class LinkFieldImportHandler : ContentImportHandlerBase, IContentFieldImportHandler +{ + internal readonly IStringLocalizer S; + + public LinkFieldImportHandler(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + public IReadOnlyCollection GetColumns(ImportContentFieldContext context) + { + var prefix = $"{context.PartName}_{context.ContentPartFieldDefinition.Name}_"; + var settings = context.ContentPartFieldDefinition.GetSettings(); + + return + [ + new ImportColumn() + { + Name = $"{prefix}{nameof(LinkField.Url)}", + Description = S["The URL value for {0}", context.ContentPartFieldDefinition.DisplayName()], + }, + new ImportColumn() + { + Name = $"{prefix}{nameof(LinkField.Text)}", + Description = S["The link text value for {0}", context.ContentPartFieldDefinition.DisplayName()], + IsRequired = settings?.LinkTextMode == LinkTextMode.Required, + }, + new ImportColumn() + { + Name = $"{prefix}{nameof(LinkField.Target)}", + Description = S["The target value for {0}", context.ContentPartFieldDefinition.DisplayName()], + }, + ]; + } + + public Task ImportAsync(ContentFieldImportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentPart); + ArgumentNullException.ThrowIfNull(context.Columns); + ArgumentNullException.ThrowIfNull(context.Row); + + var columns = GetColumns(context).ToArray(); + string url = null; + string text = null; + string target = null; + + foreach (DataColumn column in context.Columns) + { + if (Is(column.ColumnName, columns[0])) + { + url = context.Row[column]?.ToString(); + } + else if (Is(column.ColumnName, columns[1])) + { + text = context.Row[column]?.ToString(); + } + else if (Is(column.ColumnName, columns[2])) + { + target = context.Row[column]?.ToString(); + } + } + + if (url is null && text is null && target is null) + { + return Task.CompletedTask; + } + + context.ContentPart.Alter(context.ContentPartFieldDefinition.Name, field => + { + field.Url = url?.Trim(); + field.Text = text?.Trim(); + field.Target = target?.Trim(); + }); + + return Task.CompletedTask; + } + + public Task ExportAsync(ContentFieldExportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentPart); + ArgumentNullException.ThrowIfNull(context.Row); + + var field = context.ContentPart.Get(context.ContentPartFieldDefinition.Name); + + if (field == null) + { + return Task.CompletedTask; + } + + var columns = GetColumns(context).ToArray(); + context.Row[columns[0].Name] = field.Url ?? string.Empty; + context.Row[columns[1].Name] = field.Text ?? string.Empty; + context.Row[columns[2].Name] = field.Target ?? string.Empty; + + return Task.CompletedTask; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/LocalizationSetContentPickerFieldImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/LocalizationSetContentPickerFieldImportHandler.cs new file mode 100644 index 00000000000..c9ea0f9bfad --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/LocalizationSetContentPickerFieldImportHandler.cs @@ -0,0 +1,56 @@ +using Microsoft.Extensions.Localization; +using OrchardCore.ContentFields.Fields; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata.Models; + +namespace OrchardCore.ContentTransfer.Handlers.Fields; + +public sealed class LocalizationSetContentPickerFieldImportHandler : ContentImportHandlerBase, IContentFieldImportHandler +{ + internal readonly IStringLocalizer S; + + public LocalizationSetContentPickerFieldImportHandler(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + public IReadOnlyCollection GetColumns(ImportContentFieldContext context) + { + return + [ + new ImportColumn() + { + Name = $"{context.PartName}_{context.ContentPartFieldDefinition.Name}_{nameof(LocalizationSetContentPickerField.LocalizationSets)}", + Description = S["A comma-separated list of localization set ids for {0}", context.ContentPartFieldDefinition.DisplayName()], + Type = ImportColumnType.ExportOnly, + }, + ]; + } + + public Task ImportAsync(ContentFieldImportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + return Task.CompletedTask; + } + + public Task ExportAsync(ContentFieldExportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentPart); + ArgumentNullException.ThrowIfNull(context.Row); + + var knownColumn = GetColumns(context).FirstOrDefault(); + + if (knownColumn != null) + { + var field = context.ContentPart.Get(context.ContentPartFieldDefinition.Name); + + if (field?.LocalizationSets?.Length > 0) + { + context.Row[knownColumn.Name] = string.Join(",", field.LocalizationSets); + } + } + + return Task.CompletedTask; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/MarkdownFieldImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/MarkdownFieldImportHandler.cs new file mode 100644 index 00000000000..691b87a4961 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/MarkdownFieldImportHandler.cs @@ -0,0 +1,39 @@ +using Microsoft.Extensions.Localization; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata.Models; +using OrchardCore.Markdown.Fields; + +namespace OrchardCore.ContentTransfer.Handlers.Fields; + +public sealed class MarkdownFieldImportHandler : StandardFieldImportHandler +{ + internal readonly IStringLocalizer S; + + public MarkdownFieldImportHandler(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + protected override string BindingPropertyName + => nameof(MarkdownField.Markdown); + + protected override Task SetValueAsync(ContentFieldImportMapContext context, string text) + { + context.ContentPart.Alter(context.ContentPartFieldDefinition.Name, field => + { + field.Markdown = text; + }); + + return Task.CompletedTask; + } + + protected override Task GetValueAsync(ContentFieldExportMapContext context) + { + var field = context.ContentPart.Get(context.ContentPartFieldDefinition.Name); + + return Task.FromResult(field?.Markdown); + } + + protected override string Description(ImportContentFieldContext context) + => S["A markdown value for {0}", context.ContentPartFieldDefinition.DisplayName()]; +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/MediaFieldImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/MediaFieldImportHandler.cs new file mode 100644 index 00000000000..78343724949 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/MediaFieldImportHandler.cs @@ -0,0 +1,56 @@ +using Microsoft.Extensions.Localization; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata.Models; +using OrchardCore.Media.Fields; + +namespace OrchardCore.ContentTransfer.Handlers.Fields; + +public sealed class MediaFieldImportHandler : ContentImportHandlerBase, IContentFieldImportHandler +{ + internal readonly IStringLocalizer S; + + public MediaFieldImportHandler(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + public IReadOnlyCollection GetColumns(ImportContentFieldContext context) + { + return + [ + new ImportColumn() + { + Name = $"{context.PartName}_{context.ContentPartFieldDefinition.Name}_{nameof(MediaField.Paths)}", + Description = S["A comma-separated list of media paths for {0}", context.ContentPartFieldDefinition.DisplayName()], + Type = ImportColumnType.ExportOnly, + }, + ]; + } + + public Task ImportAsync(ContentFieldImportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + return Task.CompletedTask; + } + + public Task ExportAsync(ContentFieldExportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentPart); + ArgumentNullException.ThrowIfNull(context.Row); + + var knownColumn = GetColumns(context).FirstOrDefault(); + + if (knownColumn != null) + { + var field = context.ContentPart.Get(context.ContentPartFieldDefinition.Name); + + if (field?.Paths?.Length > 0) + { + context.Row[knownColumn.Name] = string.Join(",", field.Paths); + } + } + + return Task.CompletedTask; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/MultiTextFieldImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/MultiTextFieldImportHandler.cs new file mode 100644 index 00000000000..79ebbf21d5f --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/MultiTextFieldImportHandler.cs @@ -0,0 +1,47 @@ +using Microsoft.Extensions.Localization; +using OrchardCore.ContentFields.Fields; +using OrchardCore.ContentFields.Settings; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata.Models; + +namespace OrchardCore.ContentTransfer.Handlers.Fields; + +public sealed class MultiTextFieldImportHandler : StandardFieldImportHandler +{ + internal readonly IStringLocalizer S; + + public MultiTextFieldImportHandler(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + protected override string BindingPropertyName + => nameof(MultiTextField.Values); + + protected override Task SetValueAsync(ContentFieldImportMapContext context, string text) + { + var values = string.IsNullOrWhiteSpace(text) + ? [] + : text.Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + context.ContentPart.Alter(context.ContentPartFieldDefinition.Name, field => + { + field.Values = values; + }); + + return Task.CompletedTask; + } + + protected override Task GetValueAsync(ContentFieldExportMapContext context) + { + var field = context.ContentPart.Get(context.ContentPartFieldDefinition.Name); + + return Task.FromResult(field?.Values == null ? null : string.Join("|", field.Values)); + } + + protected override string Description(ImportContentFieldContext context) + => S["All values for {0}. Separate each value with bar (i.e., | )", context.ContentPartFieldDefinition.DisplayName()]; + + protected override string[] GetValidValues(ImportContentFieldContext context) + => context.ContentPartFieldDefinition.GetSettings()?.Options?.Select(x => x.Value)?.ToArray() ?? []; +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/NumberFieldImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/NumberFieldImportHandler.cs new file mode 100644 index 00000000000..2149ed4ea0d --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/NumberFieldImportHandler.cs @@ -0,0 +1,48 @@ +using Microsoft.Extensions.Localization; +using OrchardCore.ContentFields.Fields; +using OrchardCore.ContentFields.Settings; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata.Models; + +namespace OrchardCore.ContentTransfer.Handlers.Fields; + +public sealed class NumberFieldImportHandler : StandardFieldImportHandler +{ + internal readonly IStringLocalizer S; + + public NumberFieldImportHandler(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + protected override Task SetValueAsync(ContentFieldImportMapContext context, string text) + { + var value = string.IsNullOrEmpty(text) ? context.ContentPartFieldDefinition.GetSettings()?.DefaultValue : text?.Trim(); + + if (decimal.TryParse(value, out var decimalValue)) + { + context.ContentPart.Alter(context.ContentPartFieldDefinition.Name, (field) => + { + field.Value = decimalValue; + }); + } + + return Task.CompletedTask; + } + + protected override Task GetValueAsync(ContentFieldExportMapContext context) + { + var field = context.ContentPart.Get(context.ContentPartFieldDefinition.Name); + + return Task.FromResult(field?.Value); + } + + protected override string Description(ImportContentFieldContext context) + => S["A numeric value for {0}", context.ContentPartFieldDefinition.DisplayName()]; + + protected override bool IsRequired(ImportContentFieldContext context) + => context.ContentPartFieldDefinition.GetSettings()?.Required ?? false; + + protected override string BindingPropertyName + => nameof(NumericField.Value); +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/TaxonomyFieldImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/TaxonomyFieldImportHandler.cs new file mode 100644 index 00000000000..1bc5fc270d4 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/TaxonomyFieldImportHandler.cs @@ -0,0 +1,63 @@ +using Microsoft.Extensions.Localization; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata.Models; +using OrchardCore.Taxonomies.Fields; + +namespace OrchardCore.ContentTransfer.Handlers.Fields; + +public sealed class TaxonomyFieldImportHandler : ContentImportHandlerBase, IContentFieldImportHandler +{ + internal readonly IStringLocalizer S; + + public TaxonomyFieldImportHandler(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + public IReadOnlyCollection GetColumns(ImportContentFieldContext context) + { + var prefix = $"{context.PartName}_{context.ContentPartFieldDefinition.Name}_"; + + return + [ + new ImportColumn() + { + Name = $"{prefix}{nameof(TaxonomyField.TaxonomyContentItemId)}", + Description = S["The taxonomy content item id for {0}", context.ContentPartFieldDefinition.DisplayName()], + Type = ImportColumnType.ExportOnly, + }, + new ImportColumn() + { + Name = $"{prefix}{nameof(TaxonomyField.TermContentItemIds)}", + Description = S["A comma-separated list of taxonomy term content item ids for {0}", context.ContentPartFieldDefinition.DisplayName()], + Type = ImportColumnType.ExportOnly, + }, + ]; + } + + public Task ImportAsync(ContentFieldImportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + return Task.CompletedTask; + } + + public Task ExportAsync(ContentFieldExportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentPart); + ArgumentNullException.ThrowIfNull(context.Row); + + var field = context.ContentPart.Get(context.ContentPartFieldDefinition.Name); + + if (field == null) + { + return Task.CompletedTask; + } + + var columns = GetColumns(context).ToArray(); + context.Row[columns[0].Name] = field.TaxonomyContentItemId ?? string.Empty; + context.Row[columns[1].Name] = field.TermContentItemIds?.Length > 0 ? string.Join(",", field.TermContentItemIds) : string.Empty; + + return Task.CompletedTask; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/TextFieldImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/TextFieldImportHandler.cs new file mode 100644 index 00000000000..4240158d3a7 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/TextFieldImportHandler.cs @@ -0,0 +1,58 @@ +using System.Data; +using Microsoft.Extensions.Localization; +using OrchardCore.ContentFields.Fields; +using OrchardCore.ContentFields.Settings; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata.Models; + +namespace OrchardCore.ContentTransfer.Handlers.Fields; + +public sealed class TextFieldImportHandler : StandardFieldImportHandler +{ + internal readonly IStringLocalizer S; + + public TextFieldImportHandler(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + protected override string BindingPropertyName + => nameof(TextField.Text); + + protected override Task SetValueAsync(ContentFieldImportMapContext context, string text) + { + context.ContentPart.Alter(context.ContentPartFieldDefinition.Name, (field) => + { + field.Text = text?.Trim(); + }); + + return Task.CompletedTask; + } + + protected override Task GetValueAsync(ContentFieldExportMapContext context) + { + var field = context.ContentPart.Get(context.ContentPartFieldDefinition.Name); + + return Task.FromResult(field?.Text); + } + + protected override string Description(ImportContentFieldContext context) + => S["A text value for {0}", context.ContentPartFieldDefinition.DisplayName()]; + + protected override bool IsRequired(ImportContentFieldContext context) + => context.ContentPartFieldDefinition.GetSettings()?.Required ?? false; + + protected override string[] GetValidValues(ImportContentFieldContext context) + { + var predefined = context.ContentPartFieldDefinition.GetSettings(); + + if (predefined == null) + { + var multiText = context.ContentPartFieldDefinition.GetSettings(); + + return multiText?.Options?.Select(x => x.Value)?.ToArray() ?? []; + } + + return predefined?.Options?.Select(x => x.Value)?.ToArray() ?? []; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/TimeFieldImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/TimeFieldImportHandler.cs new file mode 100644 index 00000000000..cca99f181ee --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/TimeFieldImportHandler.cs @@ -0,0 +1,45 @@ +using Microsoft.Extensions.Localization; +using OrchardCore.ContentFields.Fields; +using OrchardCore.ContentFields.Settings; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata.Models; + +namespace OrchardCore.ContentTransfer.Handlers.Fields; + +public sealed class TimeFieldImportHandler : StandardFieldImportHandler +{ + internal readonly IStringLocalizer S; + + public TimeFieldImportHandler(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + protected override Task SetValueAsync(ContentFieldImportMapContext context, string text) + { + if (!string.IsNullOrEmpty(text) && TimeSpan.TryParse(text.Trim(), out var value)) + { + context.ContentPart.Alter(context.ContentPartFieldDefinition.Name, (field) => + { + field.Value = value; + }); + } + return Task.CompletedTask; + } + + protected override Task GetValueAsync(ContentFieldExportMapContext context) + { + var field = context.ContentPart.Get(context.ContentPartFieldDefinition.Name); + + return Task.FromResult(field?.Value); + } + + protected override string Description(ImportContentFieldContext context) + => S["A timespan value for {0}", context.ContentPartFieldDefinition.DisplayName()]; + + protected override bool IsRequired(ImportContentFieldContext context) + => context.ContentPartFieldDefinition.GetSettings()?.Required ?? false; + + protected override string BindingPropertyName + => nameof(TimeField.Value); +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/UserPickerFieldImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/UserPickerFieldImportHandler.cs new file mode 100644 index 00000000000..c2a539fee2c --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/UserPickerFieldImportHandler.cs @@ -0,0 +1,56 @@ +using Microsoft.Extensions.Localization; +using OrchardCore.ContentFields.Fields; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata.Models; + +namespace OrchardCore.ContentTransfer.Handlers.Fields; + +public sealed class UserPickerFieldImportHandler : ContentImportHandlerBase, IContentFieldImportHandler +{ + internal readonly IStringLocalizer S; + + public UserPickerFieldImportHandler(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + public IReadOnlyCollection GetColumns(ImportContentFieldContext context) + { + return + [ + new ImportColumn() + { + Name = $"{context.PartName}_{context.ContentPartFieldDefinition.Name}_{nameof(UserPickerField.UserIds)}", + Description = S["A comma-separated list of user ids for {0}", context.ContentPartFieldDefinition.DisplayName()], + Type = ImportColumnType.ExportOnly, + }, + ]; + } + + public Task ImportAsync(ContentFieldImportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + return Task.CompletedTask; + } + + public Task ExportAsync(ContentFieldExportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentPart); + ArgumentNullException.ThrowIfNull(context.Row); + + var knownColumn = GetColumns(context).FirstOrDefault(); + + if (knownColumn != null) + { + var field = context.ContentPart.Get(context.ContentPartFieldDefinition.Name); + + if (field?.UserIds?.Length > 0) + { + context.Row[knownColumn.Name] = string.Join(",", field.UserIds); + } + } + + return Task.CompletedTask; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/YoutubeFieldImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/YoutubeFieldImportHandler.cs new file mode 100644 index 00000000000..19ba81ee1bb --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/Fields/YoutubeFieldImportHandler.cs @@ -0,0 +1,93 @@ +using System.Data; +using Microsoft.Extensions.Localization; +using OrchardCore.ContentFields.Fields; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata.Models; + +namespace OrchardCore.ContentTransfer.Handlers.Fields; + +public sealed class YoutubeFieldImportHandler : ContentImportHandlerBase, IContentFieldImportHandler +{ + internal readonly IStringLocalizer S; + + public YoutubeFieldImportHandler(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + public IReadOnlyCollection GetColumns(ImportContentFieldContext context) + { + var prefix = $"{context.PartName}_{context.ContentPartFieldDefinition.Name}_"; + + return + [ + new ImportColumn() + { + Name = $"{prefix}{nameof(YoutubeField.RawAddress)}", + Description = S["The raw YouTube URL for {0}", context.ContentPartFieldDefinition.DisplayName()], + }, + new ImportColumn() + { + Name = $"{prefix}{nameof(YoutubeField.EmbeddedAddress)}", + Description = S["The embedded YouTube URL for {0}", context.ContentPartFieldDefinition.DisplayName()], + }, + ]; + } + + public Task ImportAsync(ContentFieldImportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentPart); + ArgumentNullException.ThrowIfNull(context.Columns); + ArgumentNullException.ThrowIfNull(context.Row); + + var columns = GetColumns(context).ToArray(); + string rawAddress = null; + string embeddedAddress = null; + + foreach (DataColumn column in context.Columns) + { + if (Is(column.ColumnName, columns[0])) + { + rawAddress = context.Row[column]?.ToString(); + } + else if (Is(column.ColumnName, columns[1])) + { + embeddedAddress = context.Row[column]?.ToString(); + } + } + + if (rawAddress is null && embeddedAddress is null) + { + return Task.CompletedTask; + } + + context.ContentPart.Alter(context.ContentPartFieldDefinition.Name, field => + { + field.RawAddress = rawAddress?.Trim(); + field.EmbeddedAddress = embeddedAddress?.Trim(); + }); + + return Task.CompletedTask; + } + + public Task ExportAsync(ContentFieldExportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentPart); + ArgumentNullException.ThrowIfNull(context.Row); + + var field = context.ContentPart.Get(context.ContentPartFieldDefinition.Name); + + if (field == null) + { + return Task.CompletedTask; + } + + var columns = GetColumns(context).ToArray(); + context.Row[columns[0].Name] = field.RawAddress ?? string.Empty; + context.Row[columns[1].Name] = field.EmbeddedAddress ?? string.Empty; + + return Task.CompletedTask; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/HtmlBodyPartContentImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/HtmlBodyPartContentImportHandler.cs new file mode 100644 index 00000000000..ad48c119110 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/HtmlBodyPartContentImportHandler.cs @@ -0,0 +1,87 @@ +using System.Data; +using Microsoft.Extensions.Localization; +using OrchardCore.ContentManagement; +using OrchardCore.Html.Models; + +namespace OrchardCore.ContentTransfer.Handlers; + +/// +/// Handles import and export of the content part. +/// Maps the Html property to a single spreadsheet column. +/// +public sealed class HtmlBodyPartContentImportHandler : ContentImportHandlerBase, IContentPartImportHandler +{ + private ImportColumn _column; + + internal readonly IStringLocalizer S; + + public HtmlBodyPartContentImportHandler(IStringLocalizer localizer) + { + S = localizer; + } + + public IReadOnlyCollection GetColumns(ImportContentPartContext context) + { + _column ??= new ImportColumn() + { + Name = $"{context.ContentTypePartDefinition.Name}_{nameof(HtmlBodyPart.Html)}", + Description = S["The HTML body content for the {0}", context.ContentTypePartDefinition.ContentTypeDefinition.DisplayName], + }; + + return [_column]; + } + + public Task ImportAsync(ContentPartImportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentItem); + ArgumentNullException.ThrowIfNull(context.Columns); + ArgumentNullException.ThrowIfNull(context.Row); + + var knownColumn = GetColumns(context).FirstOrDefault(); + + if (knownColumn != null) + { + foreach (DataColumn column in context.Columns) + { + if (!Is(column.ColumnName, knownColumn)) + { + continue; + } + + var html = context.Row[column]?.ToString(); + + if (html == null) + { + continue; + } + + context.ContentItem.Alter(part => + { + part.Html = html; + }); + } + } + + return Task.CompletedTask; + } + + public Task ExportAsync(ContentPartExportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentItem); + ArgumentNullException.ThrowIfNull(context.Row); + + if (_column?.Name != null) + { + var part = context.ContentItem.As(); + + if (part != null) + { + context.Row[_column.Name] = part.Html ?? string.Empty; + } + } + + return Task.CompletedTask; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/LiquidPartContentImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/LiquidPartContentImportHandler.cs new file mode 100644 index 00000000000..7810278697a --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/LiquidPartContentImportHandler.cs @@ -0,0 +1,87 @@ +using System.Data; +using Microsoft.Extensions.Localization; +using OrchardCore.ContentManagement; +using OrchardCore.Liquid.Models; + +namespace OrchardCore.ContentTransfer.Handlers; + +/// +/// Handles import and export of the content part. +/// Maps the Liquid property to a single spreadsheet column. +/// +public sealed class LiquidPartContentImportHandler : ContentImportHandlerBase, IContentPartImportHandler +{ + private ImportColumn _column; + + internal readonly IStringLocalizer S; + + public LiquidPartContentImportHandler(IStringLocalizer localizer) + { + S = localizer; + } + + public IReadOnlyCollection GetColumns(ImportContentPartContext context) + { + _column ??= new ImportColumn() + { + Name = $"{context.ContentTypePartDefinition.Name}_{nameof(LiquidPart.Liquid)}", + Description = S["The liquid template content for the {0}", context.ContentTypePartDefinition.ContentTypeDefinition.DisplayName], + }; + + return [_column]; + } + + public Task ImportAsync(ContentPartImportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentItem); + ArgumentNullException.ThrowIfNull(context.Columns); + ArgumentNullException.ThrowIfNull(context.Row); + + var knownColumn = GetColumns(context).FirstOrDefault(); + + if (knownColumn != null) + { + foreach (DataColumn column in context.Columns) + { + if (!Is(column.ColumnName, knownColumn)) + { + continue; + } + + var liquid = context.Row[column]?.ToString(); + + if (liquid == null) + { + continue; + } + + context.ContentItem.Alter(part => + { + part.Liquid = liquid; + }); + } + } + + return Task.CompletedTask; + } + + public Task ExportAsync(ContentPartExportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentItem); + ArgumentNullException.ThrowIfNull(context.Row); + + if (_column?.Name != null) + { + var part = context.ContentItem.As(); + + if (part != null) + { + context.Row[_column.Name] = part.Liquid ?? string.Empty; + } + } + + return Task.CompletedTask; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/MarkdownBodyPartContentImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/MarkdownBodyPartContentImportHandler.cs new file mode 100644 index 00000000000..d3bb120bd14 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/MarkdownBodyPartContentImportHandler.cs @@ -0,0 +1,87 @@ +using System.Data; +using Microsoft.Extensions.Localization; +using OrchardCore.ContentManagement; +using OrchardCore.Markdown.Models; + +namespace OrchardCore.ContentTransfer.Handlers; + +/// +/// Handles import and export of the content part. +/// Maps the Markdown property to a single spreadsheet column. +/// +public sealed class MarkdownBodyPartContentImportHandler : ContentImportHandlerBase, IContentPartImportHandler +{ + private ImportColumn _column; + + internal readonly IStringLocalizer S; + + public MarkdownBodyPartContentImportHandler(IStringLocalizer localizer) + { + S = localizer; + } + + public IReadOnlyCollection GetColumns(ImportContentPartContext context) + { + _column ??= new ImportColumn() + { + Name = $"{context.ContentTypePartDefinition.Name}_{nameof(MarkdownBodyPart.Markdown)}", + Description = S["The markdown body content for the {0}", context.ContentTypePartDefinition.ContentTypeDefinition.DisplayName], + }; + + return [_column]; + } + + public Task ImportAsync(ContentPartImportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentItem); + ArgumentNullException.ThrowIfNull(context.Columns); + ArgumentNullException.ThrowIfNull(context.Row); + + var knownColumn = GetColumns(context).FirstOrDefault(); + + if (knownColumn != null) + { + foreach (DataColumn column in context.Columns) + { + if (!Is(column.ColumnName, knownColumn)) + { + continue; + } + + var markdown = context.Row[column]?.ToString(); + + if (markdown == null) + { + continue; + } + + context.ContentItem.Alter(part => + { + part.Markdown = markdown; + }); + } + } + + return Task.CompletedTask; + } + + public Task ExportAsync(ContentPartExportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentItem); + ArgumentNullException.ThrowIfNull(context.Row); + + if (_column?.Name != null) + { + var part = context.ContentItem.As(); + + if (part != null) + { + context.Row[_column.Name] = part.Markdown ?? string.Empty; + } + } + + return Task.CompletedTask; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/PublishLaterPartContentImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/PublishLaterPartContentImportHandler.cs new file mode 100644 index 00000000000..c6554a508d1 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/PublishLaterPartContentImportHandler.cs @@ -0,0 +1,87 @@ +using System.Data; +using Microsoft.Extensions.Localization; +using OrchardCore.ContentManagement; +using OrchardCore.PublishLater.Models; + +namespace OrchardCore.ContentTransfer.Handlers; + +/// +/// Handles import and export of the content part. +/// Maps the ScheduledPublishUtc property to a single spreadsheet column. +/// +public sealed class PublishLaterPartContentImportHandler : ContentImportHandlerBase, IContentPartImportHandler +{ + private ImportColumn _column; + + internal readonly IStringLocalizer S; + + public PublishLaterPartContentImportHandler(IStringLocalizer localizer) + { + S = localizer; + } + + public IReadOnlyCollection GetColumns(ImportContentPartContext context) + { + _column ??= new ImportColumn() + { + Name = $"{context.ContentTypePartDefinition.Name}_{nameof(PublishLaterPart.ScheduledPublishUtc)}", + Description = S["The scheduled publish date and time for the {0}", context.ContentTypePartDefinition.ContentTypeDefinition.DisplayName], + }; + + return [_column]; + } + + public Task ImportAsync(ContentPartImportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentItem); + ArgumentNullException.ThrowIfNull(context.Columns); + ArgumentNullException.ThrowIfNull(context.Row); + + var knownColumn = GetColumns(context).FirstOrDefault(); + + if (knownColumn != null) + { + foreach (DataColumn column in context.Columns) + { + if (!Is(column.ColumnName, knownColumn)) + { + continue; + } + + var value = context.Row[column]?.ToString(); + + if (string.IsNullOrWhiteSpace(value) || !DateTime.TryParse(value.Trim(), out var scheduledPublishUtc)) + { + continue; + } + + context.ContentItem.Alter(part => + { + part.ScheduledPublishUtc = scheduledPublishUtc; + }); + } + } + + return Task.CompletedTask; + } + + public Task ExportAsync(ContentPartExportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentItem); + ArgumentNullException.ThrowIfNull(context.Row); + + if (_column?.Name != null) + { + var part = context.ContentItem.As(); + + if (part != null) + { + context.Row[_column.Name] = part.ScheduledPublishUtc; + } + } + + return Task.CompletedTask; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/StandardFieldImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/StandardFieldImportHandler.cs new file mode 100644 index 00000000000..7c56f0e01ff --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/StandardFieldImportHandler.cs @@ -0,0 +1,71 @@ +using System.Data; + +namespace OrchardCore.ContentTransfer.Handlers; + +public abstract class StandardFieldImportHandler : ContentImportHandlerBase, IContentFieldImportHandler +{ + public IReadOnlyCollection GetColumns(ImportContentFieldContext context) + { + return new[] + { + new ImportColumn() + { + Name = $"{context.PartName}_{context.ContentPartFieldDefinition.Name}_{BindingPropertyName}", + Description = Description(context), + IsRequired = IsRequired(context), + ValidValues = GetValidValues(context), + }, + }; + } + + public async Task ImportAsync(ContentFieldImportMapContext context) + { + ArgumentNullException.ThrowIfNull(context, nameof(context)); + ArgumentNullException.ThrowIfNull(context.ContentItem, nameof(context.ContentItem)); + ArgumentNullException.ThrowIfNull(context.Columns, nameof(context.Columns)); + ArgumentNullException.ThrowIfNull(context.Row, nameof(context.Row)); + + var knownColumns = GetColumns(context); + + foreach (DataColumn column in context.Columns) + { + var knownColumn = knownColumns.FirstOrDefault(x => Is(column.ColumnName, x)); + + if (knownColumn == null) + { + continue; + } + + await SetValueAsync(context, context.Row[column]?.ToString()); + } + } + + public async Task ExportAsync(ContentFieldExportMapContext context) + { + ArgumentNullException.ThrowIfNull(context, nameof(context)); + ArgumentNullException.ThrowIfNull(context.ContentItem, nameof(context.ContentItem)); + ArgumentNullException.ThrowIfNull(context.Row, nameof(context.Row)); + + var knownColumn = GetColumns(context).FirstOrDefault(); + + if (knownColumn != null) + { + context.Row[knownColumn.Name] = await GetValueAsync(context); + } + } + + protected virtual string Description(ImportContentFieldContext context) + => string.Empty; + + protected virtual bool IsRequired(ImportContentFieldContext context) + => false; + + protected virtual string[] GetValidValues(ImportContentFieldContext context) + => []; + + protected abstract Task GetValueAsync(ContentFieldExportMapContext context); + + protected abstract Task SetValueAsync(ContentFieldImportMapContext context, string value); + + protected abstract string BindingPropertyName { get; } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/TitlePartContentImportHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/TitlePartContentImportHandler.cs new file mode 100644 index 00000000000..d925c27b0f5 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Handlers/TitlePartContentImportHandler.cs @@ -0,0 +1,94 @@ +using System.Data; +using Microsoft.Extensions.Localization; +using OrchardCore.ContentManagement; +using OrchardCore.Entities; +using OrchardCore.Title.Models; + +namespace OrchardCore.ContentTransfer.Handlers; + +public sealed class TitlePartContentImportHandler : ContentImportHandlerBase, IContentPartImportHandler +{ + internal readonly IStringLocalizer S; + + private ImportColumn _column; + + public TitlePartContentImportHandler(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + public IReadOnlyCollection GetColumns(ImportContentPartContext context) + { + if (_column == null) + { + var settings = context.ContentTypePartDefinition.GetSettings(); + + if (settings.Options == TitlePartOptions.Editable || settings.Options == TitlePartOptions.EditableRequired) + { + _column = new ImportColumn() + { + Name = $"{context.ContentTypePartDefinition.Name}_{nameof(TitlePart.Title)}", + Description = S["The title for the {0}", context.ContentTypePartDefinition.ContentTypeDefinition.DisplayName], + IsRequired = settings.Options == TitlePartOptions.EditableRequired, + }; + } + } + + if (_column == null) + { + return Array.Empty(); + } + + return new[] { _column }; + } + + public Task ImportAsync(ContentPartImportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentItem, nameof(context.ContentItem)); + ArgumentNullException.ThrowIfNull(context.Columns, nameof(context.Columns)); + ArgumentNullException.ThrowIfNull(context.Row, nameof(context.Row)); + + var knownColumn = GetColumns(context).FirstOrDefault(); + + if (knownColumn != null) + { + foreach (DataColumn column in context.Columns) + { + if (!Is(column.ColumnName, knownColumn)) + { + continue; + } + + var title = context.Row[column]?.ToString(); + + if (string.IsNullOrWhiteSpace(title)) + { + continue; + } + + context.ContentItem.DisplayText = title; + context.ContentItem.Alter(part => + { + part.Title = title; + }); + } + } + + return Task.CompletedTask; + } + + public Task ExportAsync(ContentPartExportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentItem, nameof(context.ContentItem)); + ArgumentNullException.ThrowIfNull(context.Row, nameof(context.Row)); + + if (_column?.Name != null) + { + context.Row[_column.Name] = context.ContentItem.DisplayText; + } + + return Task.CompletedTask; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/IContentTransferEntryAdminListFilterParser.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/IContentTransferEntryAdminListFilterParser.cs new file mode 100644 index 00000000000..61031b49f99 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/IContentTransferEntryAdminListFilterParser.cs @@ -0,0 +1,7 @@ +using YesSql.Filters.Query; + +namespace OrchardCore.ContentTransfer; + +public interface IContentTransferEntryAdminListFilterParser : IQueryParser +{ +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/IContentTransferEntryAdminListQueryService.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/IContentTransferEntryAdminListQueryService.cs new file mode 100644 index 00000000000..b34d268fae2 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/IContentTransferEntryAdminListQueryService.cs @@ -0,0 +1,9 @@ +using OrchardCore.ContentTransfer.Models; +using OrchardCore.DisplayManagement.ModelBinding; + +namespace OrchardCore.ContentTransfer; + +public interface IContentTransferEntryAdminListQueryService +{ + Task QueryAsync(int page, int pageSize, ListContentTransferEntryOptions options, IUpdateModel updater); +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/INotificationAdminListFilterProvider.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/INotificationAdminListFilterProvider.cs new file mode 100644 index 00000000000..f68fef668d5 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/INotificationAdminListFilterProvider.cs @@ -0,0 +1,8 @@ +using YesSql.Filters.Query; + +namespace OrchardCore.ContentTransfer; + +public interface IContentTransferEntryAdminListFilterProvider +{ + void Build(QueryEngineBuilder builder); +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Indexes/ContentTransferEntryIndex.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Indexes/ContentTransferEntryIndex.cs new file mode 100644 index 00000000000..36dc38eac26 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Indexes/ContentTransferEntryIndex.cs @@ -0,0 +1,50 @@ +using YesSql.Indexes; + +namespace OrchardCore.ContentTransfer.Indexes; + +public sealed class ContentTransferEntryIndex : MapIndex +{ + /// + /// The logical identifier of the entry. + /// + public string EntryId { get; set; } + + public ContentTransferEntryStatus Status { get; set; } + + /// + /// When the content item has been created or first published. + /// + public DateTime CreatedUtc { get; set; } + + /// + /// The content type being imported. + /// + public string ContentType { get; set; } + + /// + /// The user id of the user who created this entry. + /// + public string Owner { get; set; } + + /// + /// The direction of the transfer (Import or Export). + /// + public ContentTransferDirection Direction { get; set; } +} + +public sealed class ContentTransferEntryIndexProvider : IndexProvider +{ + public override void Describe(DescribeContext context) + { + context.For() + .Map(entry => new ContentTransferEntryIndex() + { + EntryId = entry.EntryId, + ContentType = entry.ContentType, + CreatedUtc = entry.CreatedUtc, + Owner = entry.Owner, + Status = entry.Status, + Direction = entry.Direction, + }); + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Models/ContentImportOptions.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Models/ContentImportOptions.cs new file mode 100644 index 00000000000..03b4612d15e --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Models/ContentImportOptions.cs @@ -0,0 +1,43 @@ +namespace OrchardCore.ContentTransfer.Models; + +public sealed class ContentImportOptions +{ + public const int DefaultImportBatchSize = 100; + + public const int DefaultExportBatchSize = 200; + + public int ImportBatchSize { get; set; } = DefaultImportBatchSize; + + // 100MB is the absolute max. + public const int AbsoluteMaxAllowedFileSizeInBytes = 100 * 1024 * 1024; + + public bool AllowAllContentTypes { get; set; } = true; + + public string[] AllowedContentTypes { get; set; } = []; + + // 20MB is the default. + public long MaxAllowedFileSizeInBytes { get; set; } = 20 * 1024 * 1024; + + public int ExportBatchSize { get; set; } = DefaultExportBatchSize; + + /// + /// The number of records at which exports are queued for background processing + /// instead of being downloaded immediately. + /// + public int ExportQueueThreshold { get; set; } = 500; + + public long GetMaxAllowedSize() + { + if (MaxAllowedFileSizeInBytes < 1) + { + return AbsoluteMaxAllowedFileSizeInBytes; + } + + return Math.Min(MaxAllowedFileSizeInBytes, AbsoluteMaxAllowedFileSizeInBytes); + } + + public double GetMaxAllowedSizeInMb() + { + return GetMaxAllowedSize() / 1000000d; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Models/ContentTransferEntryBulkAction.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Models/ContentTransferEntryBulkAction.cs new file mode 100644 index 00000000000..71430da0996 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Models/ContentTransferEntryBulkAction.cs @@ -0,0 +1,7 @@ +namespace OrchardCore.ContentTransfer.Models; + +public enum ContentTransferEntryBulkAction +{ + None, + Remove, +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Models/ContentTransferEntryOrder.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Models/ContentTransferEntryOrder.cs new file mode 100644 index 00000000000..f0212da58e1 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Models/ContentTransferEntryOrder.cs @@ -0,0 +1,7 @@ +namespace OrchardCore.ContentTransfer.Models; + +public enum ContentTransferEntryOrder +{ + Latest, + Oldest, +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Models/ContentTypeTransferSettings.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Models/ContentTypeTransferSettings.cs new file mode 100644 index 00000000000..302f0631a3a --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Models/ContentTypeTransferSettings.cs @@ -0,0 +1,8 @@ +namespace OrchardCore.ContentTransfer.Models; + +public sealed class ContentTypeTransferSettings +{ + public bool AllowBulkImport { get; set; } + + public bool AllowBulkExport { get; set; } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Models/ImportContent.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Models/ImportContent.cs new file mode 100644 index 00000000000..dba3b9e0d67 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Models/ImportContent.cs @@ -0,0 +1,14 @@ + +using Microsoft.AspNetCore.Http; +using OrchardCore.Entities; + +namespace OrchardCore.ContentTransfer.Models; + +public sealed class ImportContent : Entity +{ + public string ContentTypeName { get; set; } + + public string ContentTypeId { get; set; } + + public IFormFile File { get; set; } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Models/ListContentTransferEntryOptions.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Models/ListContentTransferEntryOptions.cs new file mode 100644 index 00000000000..3149be1461a --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Models/ListContentTransferEntryOptions.cs @@ -0,0 +1,58 @@ +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.ModelBinding; +using Microsoft.AspNetCore.Mvc.Rendering; +using Microsoft.AspNetCore.Routing; +using YesSql.Filters.Query; + +namespace OrchardCore.ContentTransfer.Models; + +public class ListContentTransferEntryOptions +{ + public string OriginalSearchText { get; set; } + + public string SearchText { get; set; } + + public ContentTransferEntryStatus? Status { get; set; } + + public ContentTransferEntryOrder? OrderBy { get; set; } + + public ContentTransferEntryBulkAction? BulkAction { get; set; } + + public int EndIndex { get; set; } + + [BindNever] + public int StartIndex { get; set; } + + [BindNever] + public int EntriesCount { get; set; } + + [BindNever] + public int TotalItemCount { get; set; } + + [ModelBinder(BinderType = typeof(ContentTransferEntryFilterEngineModelBinder), Name = nameof(SearchText))] + public QueryFilterResult FilterResult { get; set; } + + [BindNever] + public List BulkActions { get; set; } + + [BindNever] + public List Statuses { get; set; } + + [BindNever] + public List Sorts { get; set; } + + [BindNever] + public RouteValueDictionary RouteValues { get; set; } = new RouteValueDictionary(); + + [BindNever] + public IList ImportableTypes { get; set; } + + [BindNever] + public IList ExportableTypes { get; set; } + + [BindNever] + public ContentTransferDirection Direction { get; set; } + + [BindNever] + public string Owner { get; set; } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/OrchardCore.ContentTransfer.Core.csproj b/src/OrchardCore/OrchardCore.ContentTransfer.Core/OrchardCore.ContentTransfer.Core.csproj new file mode 100644 index 00000000000..aaf17befe02 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/OrchardCore.ContentTransfer.Core.csproj @@ -0,0 +1,35 @@ + + + + + OrchardCore Contents Core + OrchardCore.ContentsTransfer + + $(OCCMSDescription) + + Core implementation for OrchardCore Contents Transfer + + $(PackageTags) OrchardCoreCMS ContentManagement + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/ContentImportHandlerResolver.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/ContentImportHandlerResolver.cs new file mode 100644 index 00000000000..b3e58883cf7 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/ContentImportHandlerResolver.cs @@ -0,0 +1,53 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +namespace OrchardCore.ContentTransfer.Services; + +public sealed class ContentImportHandlerResolver : IContentImportHandlerResolver +{ + private readonly IServiceProvider _serviceProvider; + private readonly ContentHandlerOptions _contentHandlerOptions; + + public ContentImportHandlerResolver( + IOptions contentHandlerOptions, + IServiceProvider serviceProvider + ) + { + _contentHandlerOptions = contentHandlerOptions.Value; + _serviceProvider = serviceProvider; + } + + public IList GetPartHandlers(string partName) + { + var services = new List(); + + var handlers = _contentHandlerOptions.ContentParts + .Where(x => x.Key.Name == partName) + .Select(x => x.Value) + .FirstOrDefault() ?? []; + + foreach (var handler in handlers) + { + services.Add((IContentPartImportHandler)_serviceProvider.GetRequiredService(handler)); + } + + return services; + } + + public IList GetFieldHandlers(string fieldName) + { + var services = new List(); + + var handlers = _contentHandlerOptions.ContentFields + .Where(x => x.Key.Name == fieldName) + .Select(x => x.Value) + .FirstOrDefault() ?? []; + + foreach (var handler in handlers) + { + services.Add((IContentFieldImportHandler)_serviceProvider.GetRequiredService(handler)); + } + + return services; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/ContentImportManager.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/ContentImportManager.cs new file mode 100644 index 00000000000..8ce3e63237f --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/ContentImportManager.cs @@ -0,0 +1,235 @@ +using Microsoft.Extensions.Logging; +using OrchardCore.ContentManagement; +using OrchardCore.Modules; + +namespace OrchardCore.ContentTransfer.Services; + +public sealed class ContentImportManager : IContentImportManager +{ + private readonly IContentImportHandlerResolver _contentImportHandlerResolver; + private readonly ITypeActivatorFactory _contentPartFactory; + private readonly ITypeActivatorFactory _contentFieldFactory; + private readonly IEnumerable _contentImportHandlers; + private readonly IContentManager _contentManager; + private readonly ILogger _logger; + + public ContentImportManager( + IContentImportHandlerResolver contentImportHandlerResolver, + ITypeActivatorFactory contentPartFactory, + ITypeActivatorFactory contentFieldFactory, + IEnumerable contentImportHandlers, + IContentManager contentManager, + ILogger logger + ) + { + _contentImportHandlerResolver = contentImportHandlerResolver; + _contentPartFactory = contentPartFactory; + _contentFieldFactory = contentFieldFactory; + _contentImportHandlers = contentImportHandlers; + _contentManager = contentManager; + _logger = logger; + } + + public async Task> GetColumnsAsync(ImportContentContext context) + { + ArgumentNullException.ThrowIfNull(context); + + var contentItem = await _contentManager.NewAsync(context.ContentTypeDefinition.Name); + + var columns = new List(); + + columns.AddRange(_contentImportHandlers.Invoke(handler => handler.GetColumns(context), _logger).SelectMany(x => x)); + + foreach (var typePartDefinition in context.ContentTypeDefinition.Parts) + { + var partName = typePartDefinition.PartDefinition.Name; + var partActivator = _contentPartFactory.GetTypeActivator(partName); + var part = contentItem.Get(partActivator.Type, typePartDefinition.Name) as ContentPart; + + if (part == null) + { + part = partActivator.CreateInstance(); + part.Weld(typePartDefinition.Name, part); + } + + var partHandlers = _contentImportHandlerResolver.GetPartHandlers(partName); + + var partContext = new ContentPartImportMapContext() + { + ContentItem = contentItem, + // ContentPart = part, + ContentTypePartDefinition = typePartDefinition, + }; + + var partColumns = partHandlers.Invoke((handler) => handler.GetColumns(partContext), _logger); + + columns.AddRange(partColumns.SelectMany(x => x)); + + if (typePartDefinition.PartDefinition?.Fields == null) + { + continue; + } + + foreach (var partFieldDefinition in typePartDefinition.PartDefinition.Fields) + { + var fieldName = partFieldDefinition.FieldDefinition.Name; + + var fieldActivator = _contentFieldFactory.GetTypeActivator(fieldName); + + var field = part.Get(fieldActivator.Type, partFieldDefinition.Name) as ContentField; + + if (field == null) + { + field = fieldActivator.CreateInstance(); + part.Weld(partFieldDefinition.Name, field); + } + + var fieldContext = new ImportContentFieldContext() + { + ContentPartFieldDefinition = partFieldDefinition, + ContentPart = part, + PartName = typePartDefinition.Name ?? partName, + // ContentField = field, + }; + + var fieldHandlers = _contentImportHandlerResolver.GetFieldHandlers(fieldName); + + var fieldColumns = fieldHandlers.Invoke((handler) => handler.GetColumns(fieldContext), _logger); + + columns.AddRange(fieldColumns.SelectMany(x => x)); + } + } + + return columns; + } + + public async Task ImportAsync(ContentImportContext context) + { + ArgumentNullException.ThrowIfNull(context); + + await _contentImportHandlers.InvokeAsync(handler => handler.ImportAsync(context), _logger); + + foreach (var typePartDefinition in context.ContentTypeDefinition.Parts) + { + var partName = typePartDefinition.PartDefinition.Name; + var partActivator = _contentPartFactory.GetTypeActivator(partName); + + var partContext = new ContentPartImportMapContext() + { + ContentItem = context.ContentItem, + ContentTypePartDefinition = typePartDefinition, + Columns = context.Columns, + Row = context.Row, + }; + + var partHandlers = _contentImportHandlerResolver.GetPartHandlers(partName); + await partHandlers.InvokeAsync((handler) => handler.ImportAsync(partContext), _logger); + + if (typePartDefinition.PartDefinition?.Fields == null) + { + continue; + } + + var part = context.ContentItem.Get(partActivator.Type, typePartDefinition.Name) as ContentPart; + if (part == null) + { + part = partActivator.CreateInstance(); + part.Weld(typePartDefinition.Name, part); + } + + foreach (var partFieldDefinition in typePartDefinition.PartDefinition.Fields) + { + var fieldName = partFieldDefinition.FieldDefinition.Name; + + var fieldActivator = _contentFieldFactory.GetTypeActivator(fieldName); + + var field = part.Get(fieldActivator.Type, partFieldDefinition.Name) as ContentField; + + if (field == null) + { + field = fieldActivator.CreateInstance(); + part.Weld(partFieldDefinition.Name, field); + } + + var fieldContext = new ContentFieldImportMapContext() + { + ContentPartFieldDefinition = partFieldDefinition, + PartName = typePartDefinition.Name ?? partName, + ContentPart = part, + Row = context.Row, + Columns = context.Columns, + ContentItem = context.ContentItem, + }; + + var fieldHandlers = _contentImportHandlerResolver.GetFieldHandlers(fieldName); + + await fieldHandlers.InvokeAsync((handler) => handler.ImportAsync(fieldContext), _logger); + } + } + } + + public async Task ExportAsync(ContentExportContext context) + { + ArgumentNullException.ThrowIfNull(context); + + await _contentImportHandlers.InvokeAsync(handler => handler.ExportAsync(context), _logger); + + foreach (var typePartDefinition in context.ContentTypeDefinition.Parts) + { + var partName = typePartDefinition.PartDefinition.Name; + var partActivator = _contentPartFactory.GetTypeActivator(partName); + var part = context.ContentItem.Get(partActivator.Type, typePartDefinition.Name) as ContentPart; + if (part == null) + { + part = partActivator.CreateInstance(); + part.Weld(typePartDefinition.Name, part); + } + + var partHandlers = _contentImportHandlerResolver.GetPartHandlers(partName); + + var partContext = new ContentPartExportMapContext() + { + ContentItem = context.ContentItem, + ContentPart = part, + ContentTypePartDefinition = typePartDefinition, + Row = context.Row, + }; + + await partHandlers.InvokeAsync((handler) => handler.ExportAsync(partContext), _logger); + + if (typePartDefinition.PartDefinition?.Fields == null) + { + continue; + } + + foreach (var partFieldDefinition in typePartDefinition.PartDefinition.Fields) + { + var fieldName = partFieldDefinition.FieldDefinition.Name; + + var fieldActivator = _contentFieldFactory.GetTypeActivator(fieldName); + + var field = part.Get(fieldActivator.Type, partFieldDefinition.Name) as ContentField; + + if (field == null) + { + field = fieldActivator.CreateInstance(); + part.Weld(partFieldDefinition.Name, field); + } + + var fieldContext = new ContentFieldExportMapContext() + { + ContentPartFieldDefinition = partFieldDefinition, + PartName = typePartDefinition.Name ?? partName, + ContentField = field, + ContentPart = part, + Row = context.Row, + ContentItem = context.ContentItem, + }; + + var fieldHandlers = _contentImportHandlerResolver.GetFieldHandlers(fieldName); + + await fieldHandlers.InvokeAsync((handler) => handler.ExportAsync(fieldContext), _logger); + } + } + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/ContentTransferFileStore.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/ContentTransferFileStore.cs new file mode 100644 index 00000000000..6a0b6c9b60b --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/ContentTransferFileStore.cs @@ -0,0 +1,45 @@ +using OrchardCore.FileStorage; + +namespace OrchardCore.ContentTransfer.Services; + +public sealed class ContentTransferFileStore : IContentTransferFileStore +{ + private readonly IFileStore _fileStore; + + public ContentTransferFileStore(IFileStore fileStore) + { + _fileStore = fileStore; + } + public Task CopyFileAsync(string srcPath, string dstPath) + => _fileStore.CopyFileAsync(srcPath, dstPath); + + public Task CreateFileFromStreamAsync(string path, Stream inputStream, bool overwrite = false) + => _fileStore.CreateFileFromStreamAsync(path, inputStream, overwrite); + + public IAsyncEnumerable GetDirectoryContentAsync(string path = null, bool includeSubDirectories = false) + => _fileStore.GetDirectoryContentAsync(path, includeSubDirectories); + + public Task GetDirectoryInfoAsync(string path) + => _fileStore.GetDirectoryInfoAsync(path); + + public Task GetFileInfoAsync(string path) + => _fileStore.GetFileInfoAsync(path); + + public Task GetFileStreamAsync(string path) + => _fileStore.GetFileStreamAsync(path); + + public Task GetFileStreamAsync(IFileStoreEntry fileStoreEntry) + => _fileStore.GetFileStreamAsync(fileStoreEntry); + + public Task MoveFileAsync(string oldPath, string newPath) + => _fileStore.MoveFileAsync(oldPath, newPath); + + public Task TryCreateDirectoryAsync(string path) + => _fileStore.TryCreateDirectoryAsync(path); + + public Task TryDeleteDirectoryAsync(string path) + => _fileStore.TryDeleteDirectoryAsync(path); + + public Task TryDeleteFileAsync(string path) + => _fileStore.TryDeleteFileAsync(path); +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/ContentTransferSizeLimitAttribute.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/ContentTransferSizeLimitAttribute.cs new file mode 100644 index 00000000000..d68c8c44ee0 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/ContentTransferSizeLimitAttribute.cs @@ -0,0 +1,70 @@ +using Microsoft.AspNetCore.Http.Features; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Filters; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using OrchardCore.ContentTransfer.Models; + +namespace OrchardCore.ContentTransfer.Services; + +[AttributeUsage(AttributeTargets.Class | AttributeTargets.Method, AllowMultiple = false, Inherited = true)] +public sealed class ContentTransferSizeLimitAttribute : Attribute, IFilterFactory, IOrderedFilter +{ + public int Order { get; set; } = 900; + + /// + public bool IsReusable => true; + + /// + public IFilterMetadata CreateInstance(IServiceProvider serviceProvider) + { + var options = serviceProvider.GetRequiredService>(); + + return new InternalContentTransferSizeFilter(options.Value.GetMaxAllowedSize()); + } + + private sealed class InternalContentTransferSizeFilter : IAuthorizationFilter, IRequestFormLimitsPolicy + { + private readonly long _maxFileSize; + + public InternalContentTransferSizeFilter(long maxFileSize) + { + _maxFileSize = maxFileSize; + } + + public void OnAuthorization(AuthorizationFilterContext context) + { + ArgumentNullException.ThrowIfNull(context); + + var effectiveFormPolicy = context.FindEffectivePolicy(); + if (effectiveFormPolicy == null || effectiveFormPolicy == this) + { + var features = context.HttpContext.Features; + var formFeature = features.Get(); + + if (formFeature == null || formFeature.Form == null) + { + // Request form has not been read yet, so set the limits + var formOptions = new FormOptions + { + MultipartBodyLengthLimit = _maxFileSize, + }; + + features.Set(new FormFeature(context.HttpContext.Request, formOptions)); + } + } + + var effectiveRequestSizePolicy = context.FindEffectivePolicy(); + if (effectiveRequestSizePolicy == null) + { + // Will only be available when running OutOfProcess with Kestrel + var maxRequestBodySizeFeature = context.HttpContext.Features.Get(); + + if (maxRequestBodySizeFeature != null && !maxRequestBodySizeFeature.IsReadOnly) + { + maxRequestBodySizeFeature.MaxRequestBodySize = _maxFileSize; + } + } + } + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/ContentTypeAuthorizationHandler.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/ContentTypeAuthorizationHandler.cs new file mode 100644 index 00000000000..4f940fd8d7a --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/ContentTypeAuthorizationHandler.cs @@ -0,0 +1,63 @@ +using Microsoft.AspNetCore.Authorization; +using Microsoft.Extensions.DependencyInjection; +using OrchardCore.ContentManagement; +using OrchardCore.Contents.Security; +using OrchardCore.Security; +using OrchardCore.Security.Permissions; + +namespace OrchardCore.ContentTransfer.Services; + +public sealed class ContentTypeAuthorizationHandler : AuthorizationHandler +{ + private readonly IServiceProvider _serviceProvider; + private IAuthorizationService _authorizationService; + + public ContentTypeAuthorizationHandler(IServiceProvider serviceProvider) + { + _serviceProvider = serviceProvider; + } + + protected override async Task HandleRequirementAsync(AuthorizationHandlerContext context, PermissionRequirement requirement) + { + if (context.HasSucceeded) + { + // This handler is not revoking any pre-existing grants. + return; + } + + // If we are not evaluating a ContentItem then return. + if (context.Resource == null) + { + return; + } + + Permission permission = null; + + // The resource can be a content type name + var contentType = context.Resource is ContentItem contentItem + ? contentItem.ContentType + : context.Resource.ToString(); + + if (requirement.Permission.Name == ContentTransferPermissions.ImportContentFromFile.Name) + { + permission = ContentTypePermissionsHelper.CreateDynamicPermission(ContentTransferPermissions.ImportContentFromFileOfType, contentType); + } + else if (requirement.Permission.Name == ContentTransferPermissions.ExportContentFromFile.Name) + { + permission = ContentTypePermissionsHelper.CreateDynamicPermission(ContentTransferPermissions.ExportContentFromFileOfType, contentType); + } + + if (permission == null) + { + return; + } + + // Lazy load to prevent circular dependencies + _authorizationService ??= _serviceProvider.GetService(); + + if (await _authorizationService.AuthorizeAsync(context.User, permission)) + { + context.Succeed(requirement); + } + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/DefaultContentTransferEntryAdminListQueryService.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/DefaultContentTransferEntryAdminListQueryService.cs new file mode 100644 index 00000000000..34cb96ff244 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/DefaultContentTransferEntryAdminListQueryService.cs @@ -0,0 +1,53 @@ +using OrchardCore.ContentTransfer.Indexes; +using OrchardCore.ContentTransfer.Models; +using OrchardCore.DisplayManagement.ModelBinding; +using YesSql; + +namespace OrchardCore.ContentTransfer.Services; + +public sealed class DefaultContentTransferEntryAdminListQueryService : IContentTransferEntryAdminListQueryService +{ + private readonly ISession _session; + private readonly IServiceProvider _serviceProvider; + + public DefaultContentTransferEntryAdminListQueryService( + ISession session, + IServiceProvider serviceProvider) + { + _session = session; + _serviceProvider = serviceProvider; + } + + public async Task QueryAsync(int page, int pageSize, ListContentTransferEntryOptions options, IUpdateModel updater) + { + var indexedQuery = _session.Query(x => x.Direction == options.Direction); + + if (!string.IsNullOrWhiteSpace(options.Owner)) + { + indexedQuery = indexedQuery.Where(x => x.Owner == options.Owner); + } + + IQuery query = indexedQuery; + + query = await options.FilterResult.ExecuteAsync(new ContentTransferEntryQueryContext(_serviceProvider, query)); + + // Query the count before applying pagination logic. + var totalCount = await query.CountAsync(); + + if (pageSize > 0) + { + if (page > 1) + { + query = query.Skip((page - 1) * pageSize); + } + + query = query.Take(pageSize); + } + + return new ContentTransferEntryQueryResult() + { + Entries = await query.ListAsync(), + TotalCount = totalCount, + }; + } +} diff --git a/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/DefaultContentTypeEntryAdminListFilterParser.cs b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/DefaultContentTypeEntryAdminListFilterParser.cs new file mode 100644 index 00000000000..83bac914124 --- /dev/null +++ b/src/OrchardCore/OrchardCore.ContentTransfer.Core/Services/DefaultContentTypeEntryAdminListFilterParser.cs @@ -0,0 +1,16 @@ +using YesSql.Filters.Query; + +namespace OrchardCore.ContentTransfer.Services; + +public sealed class DefaultContentTypeEntryAdminListFilterParser : IContentTransferEntryAdminListFilterParser +{ + private readonly IQueryParser _parser; + + public DefaultContentTypeEntryAdminListFilterParser(IQueryParser parser) + { + _parser = parser; + } + + public QueryFilterResult Parse(string text) + => _parser.Parse(text); +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/EtlActivity.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/EtlActivity.cs new file mode 100644 index 00000000000..b0b912de320 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/EtlActivity.cs @@ -0,0 +1,55 @@ +using System.Runtime.CompilerServices; +using System.Text.Json.Nodes; +using OrchardCore.DataOrchestrator.Models; + +namespace OrchardCore.DataOrchestrator.Activities; + +/// +/// Base class for ETL activities. +/// +public abstract class EtlActivity : IEtlActivity +{ + public abstract string Name { get; } + + public abstract string DisplayText { get; } + + public abstract string Category { get; } + + public JsonObject Properties { get; set; } = []; + + public virtual bool HasEditor => true; + + public abstract IEnumerable GetPossibleOutcomes(); + + public abstract Task ExecuteAsync(EtlExecutionContext context); + + /// + /// Returns an with the specified outcome names. + /// + protected static EtlActivityResult Outcomes(params string[] names) + { + return EtlActivityResult.Success(names); + } + + /// + /// Reads a property value from the JSON object. + /// + protected virtual T GetProperty(Func defaultValue = null, [CallerMemberName] string name = null) + { + var item = Properties[name]; + + return item != null + ? item.ToObject() + : defaultValue != null + ? defaultValue() + : default; + } + + /// + /// Writes a property value to the JSON object. + /// + protected virtual void SetProperty(object value, [CallerMemberName] string name = null) + { + Properties[name] = value is JsonNode node ? node.DeepClone() : JNode.FromObject(value); + } +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/EtlActivityResult.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/EtlActivityResult.cs new file mode 100644 index 00000000000..12eb638a70a --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/EtlActivityResult.cs @@ -0,0 +1,45 @@ +namespace OrchardCore.DataOrchestrator.Activities; + +/// +/// Represents the result of executing an ETL activity. +/// +public sealed class EtlActivityResult +{ + private EtlActivityResult(IReadOnlyList outcomes, bool isSuccess, string errorMessage) + { + Outcomes = outcomes; + IsSuccess = isSuccess; + ErrorMessage = errorMessage; + } + + /// + /// Gets the outcome names produced by the activity execution. + /// + public IReadOnlyList Outcomes { get; } + + /// + /// Gets whether the activity executed successfully. + /// + public bool IsSuccess { get; } + + /// + /// Gets the error message if the activity failed. + /// + public string ErrorMessage { get; } + + /// + /// Creates a successful result with the specified outcomes. + /// + public static EtlActivityResult Success(params string[] outcomes) + { + return new EtlActivityResult(outcomes, isSuccess: true, errorMessage: null); + } + + /// + /// Creates a failure result with the specified error message. + /// + public static EtlActivityResult Failure(string error) + { + return new EtlActivityResult([], isSuccess: false, errorMessage: error); + } +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/EtlLoadActivity.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/EtlLoadActivity.cs new file mode 100644 index 00000000000..30d62c3a138 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/EtlLoadActivity.cs @@ -0,0 +1,9 @@ +namespace OrchardCore.DataOrchestrator.Activities; + +/// +/// Base class for ETL load activities that write data to a destination. +/// +public abstract class EtlLoadActivity : EtlActivity +{ + public override string Category => "Loads"; +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/EtlOutcome.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/EtlOutcome.cs new file mode 100644 index 00000000000..01e70bd36db --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/EtlOutcome.cs @@ -0,0 +1,28 @@ +namespace OrchardCore.DataOrchestrator.Activities; + +/// +/// Represents a possible outcome of an ETL activity. +/// +public sealed class EtlOutcome +{ + public EtlOutcome(string name) + : this(name, name) + { + } + + public EtlOutcome(string name, string displayName) + { + Name = name; + DisplayName = displayName; + } + + /// + /// Gets the technical name of the outcome. + /// + public string Name { get; } + + /// + /// Gets the display name of the outcome. + /// + public string DisplayName { get; } +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/EtlSourceActivity.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/EtlSourceActivity.cs new file mode 100644 index 00000000000..380a0fca1ec --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/EtlSourceActivity.cs @@ -0,0 +1,9 @@ +namespace OrchardCore.DataOrchestrator.Activities; + +/// +/// Base class for ETL source activities that extract data. +/// +public abstract class EtlSourceActivity : EtlActivity +{ + public override string Category => "Sources"; +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/EtlTransformActivity.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/EtlTransformActivity.cs new file mode 100644 index 00000000000..9e4b6e99781 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/EtlTransformActivity.cs @@ -0,0 +1,9 @@ +namespace OrchardCore.DataOrchestrator.Activities; + +/// +/// Base class for ETL transform activities that modify data. +/// +public abstract class EtlTransformActivity : EtlActivity +{ + public override string Category => "Transforms"; +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/IEtlActivity.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/IEtlActivity.cs new file mode 100644 index 00000000000..f089dfa2e31 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Activities/IEtlActivity.cs @@ -0,0 +1,45 @@ +using System.Text.Json.Nodes; +using OrchardCore.DataOrchestrator.Models; + +namespace OrchardCore.DataOrchestrator.Activities; + +/// +/// Represents an activity in an ETL pipeline. +/// +public interface IEtlActivity +{ + /// + /// Gets the technical name of this activity. + /// + string Name { get; } + + /// + /// Gets the display text for this activity. + /// + string DisplayText { get; } + + /// + /// Gets the category of this activity: "Sources", "Transforms", or "Loads". + /// + string Category { get; } + + /// + /// Gets or sets the activity properties as a JSON object. + /// + JsonObject Properties { get; set; } + + /// + /// Gets whether this activity has an editor. + /// + bool HasEditor { get; } + + /// + /// Returns the possible outcomes of this activity. + /// + IEnumerable GetPossibleOutcomes(); + + /// + /// Executes the activity. + /// + Task ExecuteAsync(EtlExecutionContext context); +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Display/EtlActivityDisplayDriver.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Display/EtlActivityDisplayDriver.cs new file mode 100644 index 00000000000..08cf5cf5b2a --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Display/EtlActivityDisplayDriver.cs @@ -0,0 +1,85 @@ +using OrchardCore.DisplayManagement.Handlers; +using OrchardCore.DisplayManagement.Views; +using OrchardCore.DataOrchestrator.Activities; +using OrchardCore.DataOrchestrator.ViewModels; + +namespace OrchardCore.DataOrchestrator.Display; + +/// +/// Base class for ETL activity display drivers. +/// +public abstract class EtlActivityDisplayDriver : DisplayDriver + where TActivity : class, IEtlActivity +{ + protected static readonly string ActivityName = typeof(TActivity).Name; + + private static readonly string _thumbnailShapeType = $"{ActivityName}_Fields_Thumbnail"; + private static readonly string _designShapeType = $"{ActivityName}_Fields_Design"; + + public override Task DisplayAsync(TActivity activity, BuildDisplayContext context) + { + return CombineAsync( + Shape(_thumbnailShapeType, new EtlActivityViewModel(activity)).Location("Thumbnail", "Content"), + Shape(_designShapeType, new EtlActivityViewModel(activity)).Location("Design", "Content") + ); + } +} + +/// +/// Base class for ETL activity display drivers using a strongly typed view model. +/// +public abstract class EtlActivityDisplayDriver : EtlActivityDisplayDriver + where TActivity : class, IEtlActivity + where TEditViewModel : class, new() +{ + private static readonly string _editShapeType = $"{ActivityName}_Fields_Edit"; + + public override IDisplayResult Edit(TActivity activity, BuildEditorContext context) + { + return Initialize(_editShapeType, viewModel => EditActivityAsync(activity, viewModel)) + .Location("Content"); + } + + public override async Task UpdateAsync(TActivity activity, UpdateEditorContext context) + { + var viewModel = new TEditViewModel(); + await context.Updater.TryUpdateModelAsync(viewModel, Prefix); + await UpdateActivityAsync(viewModel, activity); + + return Edit(activity, context); + } + + /// + /// Populates the view model before it is rendered in the editor. + /// + protected virtual ValueTask EditActivityAsync(TActivity activity, TEditViewModel model) + { + EditActivity(activity, model); + + return ValueTask.CompletedTask; + } + + /// + /// Populates the view model before it is rendered in the editor. + /// + protected virtual void EditActivity(TActivity activity, TEditViewModel model) + { + } + + /// + /// Updates the activity when the view model is validated. + /// + protected virtual Task UpdateActivityAsync(TEditViewModel model, TActivity activity) + { + UpdateActivity(model, activity); + + return Task.CompletedTask; + } + + /// + /// Updates the activity when the view model is validated. + /// + protected virtual void UpdateActivity(TEditViewModel model, TActivity activity) + { + } +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/EtlConstants.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/EtlConstants.cs new file mode 100644 index 00000000000..abf97f237a2 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/EtlConstants.cs @@ -0,0 +1,9 @@ +namespace OrchardCore.DataOrchestrator; + +public static class EtlConstants +{ + public static class Features + { + public const string DataPipelines = "OrchardCore.DataOrchestrator"; + } +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/EtlField.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/EtlField.cs new file mode 100644 index 00000000000..9ee8a652560 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/EtlField.cs @@ -0,0 +1,22 @@ +namespace OrchardCore.DataOrchestrator; + +/// +/// Represents a dynamic field definition for ETL data mapping. +/// +public sealed class EtlField +{ + /// + /// Gets or sets the field name in the output. + /// + public string Name { get; set; } + + /// + /// Gets or sets the path to the field in the source data (JSONPath or Liquid expression). + /// + public string Path { get; set; } + + /// + /// Gets or sets the data type of the field. + /// + public string Type { get; set; } +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/EtlOptions.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/EtlOptions.cs new file mode 100644 index 00000000000..f8081a1eb41 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/EtlOptions.cs @@ -0,0 +1,78 @@ +using OrchardCore.DataOrchestrator.Activities; + +namespace OrchardCore.DataOrchestrator; + +/// +/// Configuration options for registered ETL activities and their display drivers. +/// +public sealed class EtlOptions +{ + private Dictionary ActivityDictionary { get; } = []; + + /// + /// Gets the registered activity types. + /// + public IEnumerable ActivityTypes => ActivityDictionary.Values.Select(x => x.ActivityType).ToList().AsReadOnly(); + + /// + /// Gets the display driver types for all registered activities. + /// + public IEnumerable ActivityDisplayDriverTypes => ActivityDictionary.Values.SelectMany(x => x.DriverTypes).ToList().AsReadOnly(); + + /// + /// Registers an activity type and optionally a display driver type. + /// + public EtlOptions RegisterActivity(Type activityType, Type driverType = null) + { + if (ActivityDictionary.TryGetValue(activityType, out var value)) + { + if (driverType != null) + { + value.DriverTypes.Add(driverType); + } + } + else + { + ActivityDictionary.Add(activityType, new EtlActivityRegistration(activityType, driverType)); + } + + return this; + } + + /// + /// Checks whether an activity type is registered. + /// + public bool IsActivityRegistered(Type activityType) + { + return ActivityDictionary.ContainsKey(activityType); + } +} + +/// +/// Represents a registered ETL activity with its display driver types. +/// +public sealed class EtlActivityRegistration +{ + public EtlActivityRegistration(Type activityType, Type driverType) + { + ActivityType = activityType; + + if (driverType != null) + { + DriverTypes.Add(driverType); + } + } + + public Type ActivityType { get; } + + public IList DriverTypes { get; } = []; +} + +public static class EtlOptionsExtensions +{ + public static EtlOptions RegisterActivity(this EtlOptions options) + where TActivity : IEtlActivity + { + return options.RegisterActivity(typeof(TActivity), typeof(TDriver)); + } +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/EtlPermissions.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/EtlPermissions.cs new file mode 100644 index 00000000000..874d6b2e113 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/EtlPermissions.cs @@ -0,0 +1,15 @@ +using OrchardCore.Security.Permissions; + +namespace OrchardCore.DataOrchestrator; + +public static class EtlPermissions +{ + public static readonly Permission ManageEtlPipelines = + new("ManageEtlPipelines", "Manage ETL pipelines", isSecurityCritical: true); + + public static readonly Permission ExecuteEtlPipelines = + new("ExecuteEtlPipelines", "Execute ETL pipelines", [ManageEtlPipelines]); + + public static readonly Permission ViewEtlPipelines = + new("ViewEtlPipelines", "View ETL pipelines and logs", [ManageEtlPipelines]); +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Helpers/EtlServiceCollectionExtensions.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Helpers/EtlServiceCollectionExtensions.cs new file mode 100644 index 00000000000..82ac3b1b710 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Helpers/EtlServiceCollectionExtensions.cs @@ -0,0 +1,20 @@ +using Microsoft.Extensions.DependencyInjection; +using OrchardCore.DisplayManagement.Handlers; +using OrchardCore.DataOrchestrator.Activities; + +namespace OrchardCore.DataOrchestrator.Helpers; + +public static class EtlServiceCollectionExtensions +{ + /// + /// Registers an ETL activity and its display driver. + /// + public static IServiceCollection AddEtlActivity(this IServiceCollection services) + where TActivity : class, IEtlActivity + where TDriver : class, IDisplayDriver + { + services.Configure(options => options.RegisterActivity(typeof(TActivity), typeof(TDriver))); + + return services; + } +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Models/EtlActivityRecord.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Models/EtlActivityRecord.cs new file mode 100644 index 00000000000..8445c0b94e7 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Models/EtlActivityRecord.cs @@ -0,0 +1,39 @@ +using System.Text.Json.Nodes; + +namespace OrchardCore.DataOrchestrator.Models; + +/// +/// Represents a persisted activity configuration within a pipeline, analogous to ActivityRecord. +/// +public sealed class EtlActivityRecord +{ + /// + /// Gets or sets the unique identifier of this activity instance (UUID). + /// + public string ActivityId { get; set; } + + /// + /// Gets or sets the activity class name. + /// + public string Name { get; set; } + + /// + /// Gets or sets the X coordinate on the designer canvas. + /// + public int X { get; set; } + + /// + /// Gets or sets the Y coordinate on the designer canvas. + /// + public int Y { get; set; } + + /// + /// Gets or sets whether this activity is the starting activity. + /// + public bool IsStart { get; set; } + + /// + /// Gets or sets the activity-specific properties. + /// + public JsonObject Properties { get; set; } = []; +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Models/EtlExecutionContext.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Models/EtlExecutionContext.cs new file mode 100644 index 00000000000..a7e4c0b43c7 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Models/EtlExecutionContext.cs @@ -0,0 +1,83 @@ +using System.Text.Json.Nodes; +using Microsoft.Extensions.Logging; +using OrchardCore.DataOrchestrator.Services; + +namespace OrchardCore.DataOrchestrator.Models; + +/// +/// Provides context for ETL pipeline execution, analogous to WorkflowExecutionContext. +/// +public sealed class EtlExecutionContext +{ + public EtlExecutionContext( + EtlPipelineDefinition pipeline, + IEtlActivityLibrary activityLibrary, + IServiceProvider serviceProvider, + ILogger logger, + CancellationToken cancellationToken) + { + Pipeline = pipeline; + ActivityLibrary = activityLibrary; + ServiceProvider = serviceProvider; + Logger = logger; + CancellationToken = cancellationToken; + } + + /// + /// Gets the pipeline definition being executed. + /// + public EtlPipelineDefinition Pipeline { get; } + + /// + /// Gets the service provider for resolving dependencies. + /// + public IServiceProvider ServiceProvider { get; } + + /// + /// Gets the activity library for instantiating additional activities during execution. + /// + public IEtlActivityLibrary ActivityLibrary { get; } + + /// + /// Gets the cancellation token for the execution. + /// + public CancellationToken CancellationToken { get; } + + /// + /// Gets a dictionary for sharing state between activities. + /// + public IDictionary Properties { get; } = new Dictionary(); + + /// + /// Gets a dictionary of parameter values provided when the pipeline was invoked. + /// + public IDictionary Parameters { get; } = new Dictionary(); + + /// + /// Gets the logger for the execution. + /// + public ILogger Logger { get; } + + /// + /// Gets or sets the data stream flowing between activities. + /// Source activities set this, transforms modify it, and loads consume it. + /// + public IAsyncEnumerable DataStream { get; set; } + + public EtlExecutionContext Clone() + { + var clone = new EtlExecutionContext(Pipeline, ActivityLibrary, ServiceProvider, Logger, CancellationToken); + + foreach (var property in Properties) + { + clone.Properties[property.Key] = property.Value; + } + + foreach (var parameter in Parameters) + { + clone.Parameters[parameter.Key] = parameter.Value; + } + + return clone; + } +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Models/EtlExecutionLog.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Models/EtlExecutionLog.cs new file mode 100644 index 00000000000..8b1c089762f --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Models/EtlExecutionLog.cs @@ -0,0 +1,62 @@ +namespace OrchardCore.DataOrchestrator.Models; + +/// +/// Represents a log entry for an ETL pipeline execution. +/// +public sealed class EtlExecutionLog +{ + /// + /// Gets or sets the document identifier. + /// + public long Id { get; set; } + + /// + /// Gets or sets the pipeline identifier. + /// + public string PipelineId { get; set; } + + /// + /// Gets or sets the pipeline name at the time of execution. + /// + public string PipelineName { get; set; } + + /// + /// Gets or sets the UTC timestamp when execution started. + /// + public DateTime StartedUtc { get; set; } + + /// + /// Gets or sets the UTC timestamp when execution completed. + /// + public DateTime? CompletedUtc { get; set; } + + /// + /// Gets or sets the execution status: "Running", "Success", "Failed", or "Cancelled". + /// + public string Status { get; set; } + + /// + /// Gets or sets the number of records processed. + /// + public int RecordsProcessed { get; set; } + + /// + /// Gets or sets the number of records successfully loaded. + /// + public int RecordsLoaded { get; set; } + + /// + /// Gets or sets the number of errors encountered. + /// + public int ErrorCount { get; set; } + + /// + /// Gets or sets the error messages from the execution. + /// + public IList Errors { get; set; } = []; + + /// + /// Gets or sets the parameter values used for the execution. + /// + public IDictionary Parameters { get; set; } = new Dictionary(); +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Models/EtlPipelineDefinition.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Models/EtlPipelineDefinition.cs new file mode 100644 index 00000000000..0d57758f0bf --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Models/EtlPipelineDefinition.cs @@ -0,0 +1,59 @@ +using System.Text.Json.Nodes; + +namespace OrchardCore.DataOrchestrator.Models; + +/// +/// Represents a persisted ETL pipeline definition, analogous to WorkflowType. +/// +public sealed class EtlPipelineDefinition +{ + /// + /// Gets or sets the document identifier. + /// + public long Id { get; set; } + + /// + /// Gets or sets the unique string identifier for this pipeline. + /// + public string PipelineId { get; set; } + + /// + /// Gets or sets the display name of the pipeline. + /// + public string Name { get; set; } + + /// + /// Gets or sets a description of the pipeline. + /// + public string Description { get; set; } + + /// + /// Gets or sets whether this pipeline is enabled. + /// + public bool IsEnabled { get; set; } + + /// + /// Gets or sets the cron schedule expression. Null for manual-only pipelines. + /// + public string Schedule { get; set; } + + /// + /// Gets or sets the UTC timestamp of the last run. + /// + public DateTime? LastRunUtc { get; set; } + + /// + /// Gets or sets the activity records in this pipeline. + /// + public IList Activities { get; set; } = []; + + /// + /// Gets or sets the transitions between activities. + /// + public IList Transitions { get; set; } = []; + + /// + /// Gets or sets the pipeline parameters. + /// + public IList Parameters { get; set; } = []; +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Models/EtlPipelineParameter.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Models/EtlPipelineParameter.cs new file mode 100644 index 00000000000..912d4139224 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Models/EtlPipelineParameter.cs @@ -0,0 +1,37 @@ +namespace OrchardCore.DataOrchestrator.Models; + +/// +/// Represents a parameter definition for an ETL pipeline. +/// +public sealed class EtlPipelineParameter +{ + /// + /// Gets or sets the parameter name. + /// + public string Name { get; set; } + + /// + /// Gets or sets the display name. + /// + public string DisplayName { get; set; } + + /// + /// Gets or sets the data type: "String", "Number", "Date", or "Boolean". + /// + public string Type { get; set; } + + /// + /// Gets or sets the default value. + /// + public string DefaultValue { get; set; } + + /// + /// Gets or sets whether this parameter is required. + /// + public bool IsRequired { get; set; } + + /// + /// Gets or sets a description of the parameter. + /// + public string Description { get; set; } +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Models/EtlTransition.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Models/EtlTransition.cs new file mode 100644 index 00000000000..92b012fec13 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Models/EtlTransition.cs @@ -0,0 +1,22 @@ +namespace OrchardCore.DataOrchestrator.Models; + +/// +/// Represents a transition between two activities in an ETL pipeline, analogous to Transition. +/// +public sealed class EtlTransition +{ + /// + /// Gets or sets the source activity identifier. + /// + public string SourceActivityId { get; set; } + + /// + /// Gets or sets the outcome name on the source activity. + /// + public string SourceOutcomeName { get; set; } + + /// + /// Gets or sets the destination activity identifier. + /// + public string DestinationActivityId { get; set; } +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/OrchardCore.DataOrchestrator.Abstractions.csproj b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/OrchardCore.DataOrchestrator.Abstractions.csproj new file mode 100644 index 00000000000..12a29e81a22 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/OrchardCore.DataOrchestrator.Abstractions.csproj @@ -0,0 +1,23 @@ + + + + OrchardCore.DataOrchestrator + + OrchardCore ETL Abstractions + $(OCCMSDescription) + + Abstractions for the ETL module. + $(PackageTags) OrchardCoreCMS Abstractions ETL + + + + + + + + + + + + + diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Services/IEtlActivityLibrary.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Services/IEtlActivityLibrary.cs new file mode 100644 index 00000000000..85162e29d9d --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Services/IEtlActivityLibrary.cs @@ -0,0 +1,29 @@ +using OrchardCore.DataOrchestrator.Activities; + +namespace OrchardCore.DataOrchestrator.Services; + +/// +/// Provides access to the registered ETL activities. +/// +public interface IEtlActivityLibrary +{ + /// + /// Returns instances of all registered activities. + /// + IEnumerable ListActivities(); + + /// + /// Returns the distinct categories of all registered activities. + /// + IEnumerable ListCategories(); + + /// + /// Returns the activity instance with the specified name. + /// + IEtlActivity GetActivityByName(string name); + + /// + /// Creates a new instance of the activity with the specified name. + /// + IEtlActivity InstantiateActivity(string name); +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Services/IEtlPipelineExecutor.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Services/IEtlPipelineExecutor.cs new file mode 100644 index 00000000000..3afecdfd86c --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Services/IEtlPipelineExecutor.cs @@ -0,0 +1,21 @@ +using OrchardCore.DataOrchestrator.Models; + +namespace OrchardCore.DataOrchestrator.Services; + +/// +/// Executes ETL pipelines. +/// +public interface IEtlPipelineExecutor +{ + /// + /// Executes the specified pipeline definition. + /// + /// The pipeline definition to execute. + /// Optional parameter values for the execution. + /// A cancellation token. + /// The execution log for the run. + Task ExecuteAsync( + EtlPipelineDefinition pipeline, + IDictionary parameters = null, + CancellationToken cancellationToken = default); +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Services/IEtlPipelineService.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Services/IEtlPipelineService.cs new file mode 100644 index 00000000000..4c920c4cecc --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/Services/IEtlPipelineService.cs @@ -0,0 +1,49 @@ +using OrchardCore.DataOrchestrator.Models; + +namespace OrchardCore.DataOrchestrator.Services; + +/// +/// Manages the persistence of ETL pipeline definitions. +/// +public interface IEtlPipelineService +{ + /// + /// Gets a pipeline definition by its pipeline identifier. + /// + Task GetAsync(string pipelineId); + + /// + /// Gets a pipeline definition by its YesSql document identifier. + /// + Task GetByDocumentIdAsync(long id); + + /// + /// Gets all pipeline definitions. + /// + Task> ListAsync(); + + /// + /// Gets all enabled pipeline definitions. + /// + Task> ListEnabledAsync(); + + /// + /// Saves a pipeline definition (creates or updates). + /// + Task SaveAsync(EtlPipelineDefinition pipeline); + + /// + /// Deletes a pipeline definition by its pipeline identifier. + /// + Task DeleteAsync(string pipelineId); + + /// + /// Saves an execution log entry. + /// + Task SaveLogAsync(EtlExecutionLog log); + + /// + /// Gets execution logs for a specific pipeline. + /// + Task> GetLogsAsync(string pipelineId, int count = 20); +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/ViewModels/EtlActivityViewModel.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/ViewModels/EtlActivityViewModel.cs new file mode 100644 index 00000000000..76160aa70b0 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Abstractions/ViewModels/EtlActivityViewModel.cs @@ -0,0 +1,24 @@ +using Microsoft.AspNetCore.Mvc.ModelBinding; +using OrchardCore.DisplayManagement.Views; +using OrchardCore.DataOrchestrator.Activities; + +namespace OrchardCore.DataOrchestrator.ViewModels; + +/// +/// View model for rendering ETL activity shapes. +/// +public class EtlActivityViewModel : ShapeViewModel + where TActivity : IEtlActivity +{ + public EtlActivityViewModel() + { + } + + public EtlActivityViewModel(TActivity activity) + { + Activity = activity; + } + + [BindNever] + public TActivity Activity { get; set; } +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Indexes/EtlExecutionLogIndex.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Indexes/EtlExecutionLogIndex.cs new file mode 100644 index 00000000000..0ee11ce0bd6 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Indexes/EtlExecutionLogIndex.cs @@ -0,0 +1,33 @@ +using OrchardCore.DataOrchestrator.Models; +using YesSql.Indexes; + +namespace OrchardCore.DataOrchestrator.Indexes; + +/// +/// YesSql index for querying ETL execution logs. +/// +public sealed class EtlExecutionLogIndex : MapIndex +{ + public string PipelineId { get; set; } + + public DateTime StartedUtc { get; set; } + + public string Status { get; set; } +} + +/// +/// YesSql index provider for . +/// +public sealed class EtlExecutionLogIndexProvider : IndexProvider +{ + public override void Describe(DescribeContext context) + { + context.For() + .Map(log => new EtlExecutionLogIndex + { + PipelineId = log.PipelineId, + StartedUtc = log.StartedUtc, + Status = log.Status, + }); + } +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Indexes/EtlPipelineIndex.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Indexes/EtlPipelineIndex.cs new file mode 100644 index 00000000000..8e78ecad68c --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Indexes/EtlPipelineIndex.cs @@ -0,0 +1,33 @@ +using OrchardCore.DataOrchestrator.Models; +using YesSql.Indexes; + +namespace OrchardCore.DataOrchestrator.Indexes; + +/// +/// YesSql index for querying ETL pipeline definitions. +/// +public sealed class EtlPipelineIndex : MapIndex +{ + public string PipelineId { get; set; } + + public string Name { get; set; } + + public bool IsEnabled { get; set; } +} + +/// +/// YesSql index provider for . +/// +public sealed class EtlPipelineIndexProvider : IndexProvider +{ + public override void Describe(DescribeContext context) + { + context.For() + .Map(pipeline => new EtlPipelineIndex + { + PipelineId = pipeline.PipelineId, + Name = pipeline.Name, + IsEnabled = pipeline.IsEnabled, + }); + } +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Core/OrchardCore.DataOrchestrator.Core.csproj b/src/OrchardCore/OrchardCore.DataOrchestrator.Core/OrchardCore.DataOrchestrator.Core.csproj new file mode 100644 index 00000000000..f01ff897bf9 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Core/OrchardCore.DataOrchestrator.Core.csproj @@ -0,0 +1,22 @@ + + + + OrchardCore.DataOrchestrator + + OrchardCore ETL Core + $(OCCMSDescription) + Core services for the ETL module. + $(PackageTags) OrchardCoreCMS ETL + + + + + + + + + + + + + diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Services/EtlActivityDisplayManager.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Services/EtlActivityDisplayManager.cs new file mode 100644 index 00000000000..e95bdaaede3 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Services/EtlActivityDisplayManager.cs @@ -0,0 +1,59 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using OrchardCore.DisplayManagement; +using OrchardCore.DisplayManagement.Descriptors; +using OrchardCore.DisplayManagement.Handlers; +using OrchardCore.DisplayManagement.Layout; +using OrchardCore.DisplayManagement.ModelBinding; +using OrchardCore.Environment.Shell.Builders; +using OrchardCore.DataOrchestrator.Activities; + +namespace OrchardCore.DataOrchestrator.Services; + +/// +/// Default implementation of that builds +/// display and editor shapes for ETL activities. +/// +public sealed class EtlActivityDisplayManager : IEtlActivityDisplayManager +{ + private readonly DisplayManager _displayManager; + + public EtlActivityDisplayManager( + IOptions etlOptions, + IServiceProvider serviceProvider, + IShapeFactory shapeFactory, + IEnumerable placementProviders, + IEnumerable> displayDrivers, + ILogger> displayManagerLogger, + ILayoutAccessor layoutAccessor) + { + var drivers = etlOptions.Value.ActivityDisplayDriverTypes + .Select(x => serviceProvider.CreateInstance>(x)) + .Concat(displayDrivers); + + _displayManager = new DisplayManager( + drivers, + shapeFactory, + placementProviders, + displayManagerLogger, + layoutAccessor); + } + + /// + public Task BuildDisplayAsync(IEtlActivity model, IUpdateModel updater, string displayType = "", string groupId = "") + { + return _displayManager.BuildDisplayAsync(model, updater, displayType, groupId); + } + + /// + public Task BuildEditorAsync(IEtlActivity model, IUpdateModel updater, bool isNew, string groupId = "", string htmlPrefix = "") + { + return _displayManager.BuildEditorAsync(model, updater, isNew, groupId, htmlPrefix); + } + + /// + public Task UpdateEditorAsync(IEtlActivity model, IUpdateModel updater, bool isNew, string groupId = "", string htmlPrefix = "") + { + return _displayManager.UpdateEditorAsync(model, updater, isNew, groupId, htmlPrefix); + } +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Services/EtlActivityLibrary.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Services/EtlActivityLibrary.cs new file mode 100644 index 00000000000..5ca512cdb19 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Services/EtlActivityLibrary.cs @@ -0,0 +1,81 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using OrchardCore.Environment.Shell.Builders; +using OrchardCore.DataOrchestrator.Activities; + +namespace OrchardCore.DataOrchestrator.Services; + +/// +/// Default implementation of that manages +/// the registry of available ETL activities. +/// +public sealed class EtlActivityLibrary : IEtlActivityLibrary +{ + private readonly Lazy> _activityDictionary; + private readonly Lazy> _activityCategories; + private readonly IServiceProvider _serviceProvider; + private readonly ILogger _logger; + + public EtlActivityLibrary( + IOptions etlOptions, + IServiceProvider serviceProvider, + ILogger logger) + { + _serviceProvider = serviceProvider; + _logger = logger; + + _activityDictionary = new Lazy>(() => + etlOptions.Value.ActivityTypes + .Where(x => !x.IsAbstract) + .Select(x => serviceProvider.CreateInstance(x)) + .OrderBy(x => x.Name) + .ToDictionary(x => x.Name)); + + _activityCategories = new Lazy>(() => + _activityDictionary.Value.Values + .OrderBy(x => x.Category) + .Select(x => x.Category) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList()); + } + + private IDictionary ActivityDictionary => _activityDictionary.Value; + + private IList ActivityCategories => _activityCategories.Value; + + /// + public IEnumerable ListActivities() + { + return ActivityDictionary.Values; + } + + /// + public IEnumerable ListCategories() + { + return ActivityCategories; + } + + /// + public IEtlActivity GetActivityByName(string name) + { + return ActivityDictionary.TryGetValue(name, out var activity) ? activity : null; + } + + /// + public IEtlActivity InstantiateActivity(string name) + { + var activityType = GetActivityByName(name)?.GetType(); + + if (activityType == null) + { + if (_logger.IsEnabled(LogLevel.Warning)) + { + _logger.LogWarning("Requested ETL activity '{ActivityName}' does not exist in the library.", name); + } + + return null; + } + + return _serviceProvider.CreateInstance(activityType); + } +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Services/EtlPipelineExecutor.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Services/EtlPipelineExecutor.cs new file mode 100644 index 00000000000..96f7b7d8aa5 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Services/EtlPipelineExecutor.cs @@ -0,0 +1,152 @@ +using System.Text.Json.Nodes; +using Microsoft.Extensions.Logging; +using OrchardCore.DataOrchestrator.Activities; +using OrchardCore.DataOrchestrator.Models; + +namespace OrchardCore.DataOrchestrator.Services; + +/// +/// Default implementation of that uses a stack-based +/// approach following activity transitions. +/// +public sealed class EtlPipelineExecutor : IEtlPipelineExecutor +{ + private readonly IEtlActivityLibrary _activityLibrary; + private readonly IServiceProvider _serviceProvider; + private readonly ILogger _logger; + + public EtlPipelineExecutor( + IEtlActivityLibrary activityLibrary, + IServiceProvider serviceProvider, + ILogger logger) + { + _activityLibrary = activityLibrary; + _serviceProvider = serviceProvider; + _logger = logger; + } + + /// + public async Task ExecuteAsync( + EtlPipelineDefinition pipeline, + IDictionary parameters = null, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(pipeline); + + var log = new EtlExecutionLog + { + PipelineId = pipeline.PipelineId, + PipelineName = pipeline.Name, + StartedUtc = DateTime.UtcNow, + Status = "Running", + }; + + try + { + if (_logger.IsEnabled(LogLevel.Information)) + { + _logger.LogInformation("Starting ETL pipeline '{PipelineName}' ({PipelineId}).", pipeline.Name, pipeline.PipelineId); + } + + var context = new EtlExecutionContext(pipeline, _activityLibrary, _serviceProvider, _logger, cancellationToken); + + if (parameters != null) + { + foreach (var param in parameters) + { + context.Parameters[param.Key] = param.Value; + } + } + + var startActivities = pipeline.Activities.Where(a => a.IsStart).ToList(); + + if (startActivities.Count == 0) + { + throw new InvalidOperationException("Pipeline has no start activity."); + } + + var scheduled = new Stack(); + + foreach (var start in startActivities) + { + scheduled.Push(start); + } + + while (scheduled.Count > 0 && !cancellationToken.IsCancellationRequested) + { + var activityRecord = scheduled.Pop(); + var activity = _activityLibrary.InstantiateActivity(activityRecord.Name); + + if (activity == null) + { + log.Errors.Add($"Activity '{activityRecord.Name}' not found."); + log.ErrorCount++; + continue; + } + + activity.Properties = activityRecord.Properties?.DeepClone() as JsonObject ?? []; + + var result = await activity.ExecuteAsync(context); + + if (!result.IsSuccess) + { + log.Errors.Add($"Activity '{activityRecord.Name}': {result.ErrorMessage}"); + log.ErrorCount++; + continue; + } + + foreach (var outcome in result.Outcomes) + { + var transitions = pipeline.Transitions + .Where(t => t.SourceActivityId == activityRecord.ActivityId + && t.SourceOutcomeName == outcome); + + foreach (var transition in transitions) + { + var next = pipeline.Activities + .FirstOrDefault(a => a.ActivityId == transition.DestinationActivityId); + + if (next != null) + { + scheduled.Push(next); + } + } + } + } + + log.Status = cancellationToken.IsCancellationRequested ? "Cancelled" : + log.ErrorCount > 0 ? "Failed" : "Success"; + + if (_logger.IsEnabled(LogLevel.Information)) + { + _logger.LogInformation( + "ETL pipeline '{PipelineName}' completed with status {Status}.", + pipeline.Name, log.Status); + } + } + catch (OperationCanceledException) + { + log.Status = "Cancelled"; + if (_logger.IsEnabled(LogLevel.Warning)) + { + _logger.LogWarning("ETL pipeline '{PipelineName}' was cancelled.", pipeline.Name); + } + } + catch (Exception ex) + { + log.Status = "Failed"; + log.Errors.Add(ex.Message); + log.ErrorCount++; + if (_logger.IsEnabled(LogLevel.Error)) + { + _logger.LogError(ex, "ETL pipeline '{PipelineName}' failed with an unhandled error.", pipeline.Name); + } + } + finally + { + log.CompletedUtc = DateTime.UtcNow; + } + + return log; + } +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Services/EtlPipelineService.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Services/EtlPipelineService.cs new file mode 100644 index 00000000000..050647a84a8 --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Services/EtlPipelineService.cs @@ -0,0 +1,94 @@ +using Microsoft.Extensions.Logging; +using OrchardCore.DataOrchestrator.Indexes; +using OrchardCore.DataOrchestrator.Models; +using YesSql; + +namespace OrchardCore.DataOrchestrator.Services; + +/// +/// Default implementation of using YesSql. +/// +public sealed class EtlPipelineService : IEtlPipelineService +{ + private readonly ISession _session; + private readonly ILogger _logger; + + public EtlPipelineService(ISession session, ILogger logger) + { + _session = session; + _logger = logger; + } + + /// + public async Task GetAsync(string pipelineId) + { + return await _session.Query( + x => x.PipelineId == pipelineId).FirstOrDefaultAsync(); + } + + /// + public async Task GetByDocumentIdAsync(long id) + { + return await _session.GetAsync(id); + } + + /// + public async Task> ListAsync() + { + return await _session.Query().ListAsync(); + } + + /// + public async Task> ListEnabledAsync() + { + return await _session.Query( + x => x.IsEnabled).ListAsync(); + } + + /// + public async Task SaveAsync(EtlPipelineDefinition pipeline) + { + ArgumentNullException.ThrowIfNull(pipeline); + + await _session.SaveAsync(pipeline); + + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug("Saved ETL pipeline '{PipelineName}' ({PipelineId}).", pipeline.Name, pipeline.PipelineId); + } + } + + /// + public async Task DeleteAsync(string pipelineId) + { + var pipeline = await GetAsync(pipelineId); + + if (pipeline != null) + { + _session.Delete(pipeline); + + if (_logger.IsEnabled(LogLevel.Information)) + { + _logger.LogInformation("Deleted ETL pipeline '{PipelineName}' ({PipelineId}).", pipeline.Name, pipeline.PipelineId); + } + } + } + + /// + public async Task SaveLogAsync(EtlExecutionLog log) + { + ArgumentNullException.ThrowIfNull(log); + + await _session.SaveAsync(log); + } + + /// + public async Task> GetLogsAsync(string pipelineId, int count = 20) + { + return await _session.Query( + x => x.PipelineId == pipelineId) + .OrderByDescending(x => x.StartedUtc) + .Take(count) + .ListAsync(); + } +} diff --git a/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Services/IEtlActivityDisplayManager.cs b/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Services/IEtlActivityDisplayManager.cs new file mode 100644 index 00000000000..86e8f7e8dda --- /dev/null +++ b/src/OrchardCore/OrchardCore.DataOrchestrator.Core/Services/IEtlActivityDisplayManager.cs @@ -0,0 +1,9 @@ +using OrchardCore.DisplayManagement; +using OrchardCore.DataOrchestrator.Activities; + +namespace OrchardCore.DataOrchestrator.Services; + +/// +/// Manages the display of ETL activities using display drivers. +/// +public interface IEtlActivityDisplayManager : IDisplayManager; diff --git a/src/docs/reference/README.md b/src/docs/reference/README.md index 7bb49e7db55..aa003009886 100644 --- a/src/docs/reference/README.md +++ b/src/docs/reference/README.md @@ -63,6 +63,7 @@ Here's a categorized overview of all built-in Orchard Core features at a glance. - [SEO Meta](modules/Seo/README.md) - [Content Fields](modules/ContentFields/README.md) - [Content Preview](modules/ContentPreview/README.md) +- [Content Transfer](modules/ContentTransfer/README.md) - [Taxonomies](modules/Taxonomies/README.md) - [Feeds](modules/Feeds/README.md) - [Forms](modules/Forms/README.md) diff --git a/src/docs/reference/modules/ContentTransfer/README.md b/src/docs/reference/modules/ContentTransfer/README.md new file mode 100644 index 00000000000..ec27cb7fb94 --- /dev/null +++ b/src/docs/reference/modules/ContentTransfer/README.md @@ -0,0 +1,379 @@ +# Content Transfer (`OrchardCore.ContentTransfer`) + +This feature provides a way to bulk import and export content using Excel files (`.xlsx` format). Once the feature is enabled, you can navigate to **Content** > **Bulk Transfers** to import data, and **Content** > **Bulk Export** to export data. + +## Getting Started + +### 1. Enable the Module + +Enable the **Content Transfer** feature from **Admin** > **Configuration** > **Features**. + +### 2. Enable Bulk Import/Export on Content Types + +Before you can import or export content items, you must enable bulk transfer on the desired content types: + +1. Navigate to **Admin** > **Content** > **Content Types**. +2. Edit the content type you want to enable (e.g., `Article`). +3. Check the **Allow Bulk Import** and/or **Allow Bulk Export** options under the content type settings. +4. Save the content type. + +Only content types with these options enabled will appear in the import and export interfaces. + +### 3. Optional: Enable Notifications + +If you enable the **OrchardCore.Notifications** module, users will receive in-app notifications when queued export files are ready for download. + +## Supported File Format + +The Content Transfer feature supports only **Excel Workbook (`.xlsx`)** files. This format is based on the Office Open XML standard and provides: + +- Better compatibility across different systems +- Structured data handling with proper data types +- Support for large datasets + +> **Note:** Older Excel formats (`.xls`) and CSV files are not supported. + +## Bulk Import + +### Importing Content + +1. Navigate to **Content** > **Bulk Transfers**. +2. Click the **Import** button and select the content type you want to import. +3. Upload an Excel file (`.xlsx`). You can download a template for the selected content type to see the expected column format. +4. The file is uploaded and queued for background processing. A background task processes the file in configurable batch sizes, with checkpoint-based resume to ensure records are not re-imported if the process is restarted. +5. Progress is displayed on the **Bulk Transfers** list showing the number of records processed out of the total, along with error counts and status badges. + +### Import Validation + +Imported records are validated using `IContentManager.ValidateAsync()`. Rows that fail validation are tracked. When an import has validation errors: + +- The error count is displayed as a badge on the import entry. +- A **Download errors** option is available in the **Actions** dropdown menu, which generates an Excel file containing only the rejected rows for review and re-import. + +### Template Download + +For each importable content type, you can download a template Excel file that contains the expected column headers. This template is generated based on the registered import handlers and includes column descriptions and required/optional indicators. + +## Bulk Export + +### Exporting Content + +1. Navigate to **Content** > **Bulk Export**. +2. Select the content type you want to export. +3. Choose the export scope: + - **Export all published** (default): Exports all published content items. + - **Partial Export**: Enables additional filters: + - **Created from/to**: Filter by creation date range. + - **Modified from/to**: Filter by modification date range. + - **Owners**: Comma-separated list of usernames to filter by content owner. + - **Published only**: Export only published versions. + - **Latest only**: Export only the latest version of each content item. + - **All versions**: Export all versions of content items. +4. Click **Export**. + +### Immediate vs. Queued Export + +- If the number of matching records is **≤500** (configurable), the export file is generated immediately and downloaded directly. +- If the number of matching records is **>500**, the export is queued for background processing: + - A background task generates the export file using memory-efficient pagination. + - When the `OrchardCore.Notifications` module is enabled, the user is notified when the file is ready. + - The user can visit the **Export Dashboard** (**Content** > **Export Dashboard**) to check the status and download the completed file. + +### Export Dashboard + +The Export Dashboard shows all queued export requests for the current user, including: + +- **Status**: Pending, In Progress, Completed, or Failed. +- **Download**: A download button appears for completed exports. +- **Search**: A client-side search bar to filter entries by content type. + +## Configuration + +The module can be configured via `appsettings.json` using the `OrchardCore_ContentTransfer` section: + +```json +{ + "OrchardCore_ContentTransfer": { + "ImportBatchSize": 100, + "ExportBatchSize": 200, + "ExportQueueThreshold": 500 + } +} +``` + +| Setting | Default | Description | +|---------|---------|-------------| +| `ImportBatchSize` | 100 | Number of records processed per batch during import. | +| `ExportBatchSize` | 200 | Number of records fetched per page during export. | +| `ExportQueueThreshold` | 500 | Maximum number of records for immediate export. Records above this threshold are queued for background processing. | + +## Memory Efficiency + +The Content Transfer module is designed to handle large datasets without excessive memory consumption: + +- **Import**: The uploaded file is read in streaming batches. Only the current batch of rows is held in memory at a time. Processed rows are cleared before the next batch is loaded. +- **Export**: Content items are fetched page-by-page and written directly to a temporary file stream. The file is never fully loaded into memory. + +## Adding a Custom Part Importer + +You can create a custom import/export handler for a content part by implementing the `IContentPartImportHandler` interface. The following illustrates an example of defining the title part importer. + +```csharp +public sealed class TitlePartContentImportHandler : ContentImportHandlerBase, IContentPartImportHandler +{ + private ImportColumn _column; + + internal readonly IStringLocalizer S; + + public TitlePartContentImportHandler(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + public IReadOnlyCollection GetColumns(ImportContentPartContext context) + { + if (_column == null) + { + var settings = context.ContentTypePartDefinition.GetSettings(); + + if (settings.Options == TitlePartOptions.Editable || settings.Options == TitlePartOptions.EditableRequired) + { + _column = new ImportColumn() + { + Name = $"{context.ContentTypePartDefinition.Name}_{nameof(TitlePart.Title)}", + Description = S["The title for the {0}", context.ContentTypePartDefinition.ContentTypeDefinition.DisplayName], + IsRequired = settings.Options == TitlePartOptions.EditableRequired, + }; + } + } + + if (_column == null) + { + return Array.Empty(); + } + + return new[] { _column }; + } + + public Task ImportAsync(ContentPartImportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentItem, nameof(context.ContentItem)); + ArgumentNullException.ThrowIfNull(context.Columns, nameof(context.Columns)); + ArgumentNullException.ThrowIfNull(context.Row, nameof(context.Row)); + + if (_column?.Name != null) + { + foreach (DataColumn column in context.Columns) + { + if (!Is(column.ColumnName, _column)) + { + continue; + } + var title = context.Row[column]?.ToString(); + + if (string.IsNullOrWhiteSpace(title)) + { + continue; + } + + context.ContentItem.DisplayText = title; + context.ContentItem.Alter(part => + { + part.Title = title.Trim(); + }); + } + } + + return Task.CompletedTask; + } + + public Task ExportAsync(ContentPartExportMapContext context) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(context.ContentItem, nameof(context.ContentItem)); + ArgumentNullException.ThrowIfNull(context.Row, nameof(context.Row)); + + if (_column?.Name != null) + { + context.Row[_column.Name] = context.ContentItem.DisplayText; + } + + return Task.CompletedTask; + } +} +``` + +Register the custom implementation in your module's `Startup`: + +```csharp +services.AddContentPartImportHandler(); +``` + +## Adding a Custom Field Importer + +You can create a custom import/export handler for a content field by either implementing the `IContentFieldImportHandler` interface or inheriting from the `StandardFieldImportHandler` class. The following illustrates an example of defining a text field importer. + +```csharp +public sealed class TextFieldImportHandler : StandardFieldImportHandler +{ + private readonly IStringLocalizer S; + + public TextFieldImportHandler(IStringLocalizer stringLocalizer) + { + S = stringLocalizer; + } + + protected override string BindingPropertyName + => nameof(TextField.Text); + + protected override Task SetValueAsync(ContentFieldImportMapContext context, string text) + { + context.ContentPart.Alter(context.ContentPartFieldDefinition.Name, (field) => + { + field.Text = text?.Trim(); + }); + + return Task.CompletedTask; + } + + protected override Task GetValueAsync(ContentFieldExportMapContext context) + { + var field = context.ContentPart.Get(context.ContentPartFieldDefinition.Name); + + return Task.FromResult(field?.Text); + } + + protected override string Description(ImportContentFieldContext context) + => S["A text value for {0}", context.ContentPartFieldDefinition.DisplayName()]; + + protected override bool IsRequired(ImportContentFieldContext context) + => context.ContentPartFieldDefinition.GetSettings()?.Required ?? false; + + protected override string[] GetValidValues(ImportContentFieldContext context) + { + var predefined = context.ContentPartFieldDefinition.GetSettings(); + + if (predefined == null) + { + var multiText = context.ContentPartFieldDefinition.GetSettings(); + + return multiText?.Options?.Select(x => x.Value)?.ToArray() ?? []; + } + + return predefined?.Options?.Select(x => x.Value)?.ToArray() ?? []; + } +} +``` + +Register the custom implementation in your module's `Startup`: + +```csharp +services.AddContentFieldImportHandler(); +``` + +## Adding a Custom Content Item Importer + +You can create a custom import/export handler for common content item properties by implementing the `IContentImportHandler` interface. The following illustrates an example of defining the common properties importer. + +```csharp +public sealed class CommonContentImportHandler : ContentImportHandlerBase, IContentImportHandler +{ + private readonly IContentItemIdGenerator _contentItemIdGenerator; + private readonly IStringLocalizer S; + + public CommonContentImportHandler( + IStringLocalizer stringLocalizer, + IContentItemIdGenerator contentItemIdGenerator) + { + _contentItemIdGenerator = contentItemIdGenerator; + S = stringLocalizer; + } + + public IReadOnlyCollection GetColumns(ImportContentContext context) + { + return new[] + { + new ImportColumn() + { + Name = nameof(ContentItem.ContentItemId), + Description = S["The id for the {0}", context.ContentTypeDefinition.DisplayName], + }, + new ImportColumn() + { + Name = nameof(ContentItem.CreatedUtc), + Description = S["The UTC created datetime value {0}", context.ContentTypeDefinition.DisplayName], + Type = ImportColumnType.ExportOnly, + }, + new ImportColumn() + { + Name = nameof(ContentItem.ModifiedUtc), + Description = S["The UTC last modified datetime value {0}", context.ContentTypeDefinition.DisplayName], + Type = ImportColumnType.ExportOnly, + }, + }; + } + + public Task ImportAsync(ContentImportContext context) + { + ArgumentNullException.ThrowIfNull(context); + + foreach (DataColumn column in context.Columns) + { + if (Is(column.ColumnName, nameof(ContentItem.ContentItemId))) + { + var contentItemId = context.Row[column]?.ToString(); + + if (string.IsNullOrWhiteSpace(contentItemId)) + { + continue; + } + + var fakeId = _contentItemIdGenerator.GenerateUniqueId(new ContentItem()); + + if (fakeId.Length == contentItemId.Length) + { + context.ContentItem.ContentItemId = contentItemId; + } + } + } + + return Task.CompletedTask; + } + + public Task ExportAsync(ContentExportContext context) + { + ArgumentNullException.ThrowIfNull(context); + + context.Row[nameof(ContentItem.ContentItemId)] = context.ContentItem.ContentItemId; + context.Row[nameof(ContentItem.CreatedUtc)] = context.ContentItem.CreatedUtc; + context.Row[nameof(ContentItem.ModifiedUtc)] = context.ContentItem.ModifiedUtc; + + return Task.CompletedTask; + } +} +``` + +Register the custom implementation in your module's `Startup`: + +```csharp +services.AddScoped(); +``` + +## Built-in Import/Export Handlers + +The following handlers are provided out of the box: + +| Handler | Type | Description | +|---------|------|-------------| +| `CommonContentImportHandler` | Content Item | Handles `ContentItemId`, `CreatedUtc`, `ModifiedUtc` properties. | +| `TitlePartContentImportHandler` | Content Part | Handles `TitlePart.Title`. | +| `HtmlBodyPartContentImportHandler` | Content Part | Handles `HtmlBodyPart.Html`. | +| `AutoroutePartContentImportHandler` | Content Part | Handles `AutoroutePart.Path`. | +| `TextFieldImportHandler` | Content Field | Handles `TextField.Text`. | +| `BooleanFieldImportHandler` | Content Field | Handles `BooleanField.Value`. | +| `NumberFieldImportHandler` | Content Field | Handles `NumericField.Value`. | +| `DateFieldImportHandler` | Content Field | Handles `DateField.Value`. | +| `DateTimeFieldImportHandler` | Content Field | Handles `DateTimeField.Value`. | +| `TimeFieldImportHandler` | Content Field | Handles `TimeField.Value`. | +| `ContentPickerFieldImportHandler` | Content Field | Handles `ContentPickerField.ContentItemIds`. | diff --git a/src/docs/reference/modules/DataOrchestrator/README.md b/src/docs/reference/modules/DataOrchestrator/README.md new file mode 100644 index 00000000000..a05d9c724dd --- /dev/null +++ b/src/docs/reference/modules/DataOrchestrator/README.md @@ -0,0 +1,317 @@ +# Data Orchestrator (`OrchardCore.DataOrchestrator`) + +The ETL (Extract, Transform, Load) module provides a visual pipeline editor for building data integration workflows. It allows users to extract data from various sources, transform it using configurable operations, and load it into different destinations. + +## General Concepts + +An ETL pipeline is a collection of **activities** connected by **transitions**. Activities are categorized into three types: + +- **Sources** (Extract): Produce data streams from content items, JSON files, APIs, or custom providers. +- **Transforms**: Modify, filter, map, or reshape data as it flows through the pipeline. +- **Loads** (Destinations): Consume data and write it to files, content items, APIs, or external services. + +Pipelines are defined visually using a drag-and-drop editor (similar to the Workflows editor) and can be executed manually, on a schedule, or on demand with parameters. + +### Pipeline Flow + +``` +Source(s) → Transform(s) → Load(s) +``` + +A pipeline can have: + +- **Multiple sources** that feed data into the pipeline. +- **Multiple transforms** chained together to progressively reshape data. +- **Multiple loads** (fan-out) to send data to different destinations simultaneously. + +## Vocabulary + +### Pipeline Definition + +A document that contains all information about an ETL pipeline: its name, activities, transitions, parameters, schedule, and enabled status. + +### Activity + +A step in a pipeline. Each activity has a **type** (Source, Transform, or Load), configurable **properties**, and one or more **outcomes** that connect to downstream activities via transitions. + +### Transition + +A connection from one activity's outcome to another activity. Transitions define the order of execution and data flow. + +### Pipeline Parameter + +A named input value that can be provided when executing a pipeline on demand. Parameters support types like `String`, `Number`, `Date`, and `Boolean`, with optional default values. + +### Execution Log + +A record of a pipeline run, including start/end times, status (Running, Completed, Failed), records processed, and any errors encountered. + +## Pipeline Editor + +The pipeline editor provides a visual canvas for designing ETL pipelines: + +1. **Activity Picker** — Add sources, transforms, and loads from categorized lists. +2. **Canvas** — Drag and position activities, then connect them via transitions. +3. **Activity Editor** — Click an activity to configure its properties. +4. **Properties Panel** — Set pipeline name, parameters, schedule, and enabled status. + +Activities are color-coded: + +- 🟢 **Green** — Sources +- 🔵 **Blue** — Transforms +- 🔴 **Red** — Loads + +## Built-in Activities + +### Sources + +#### Content Item Source + +Extracts content items from the Orchard Core content store. + +| Setting | Description | +|---------|-------------| +| Content Type | The content type to query (e.g., `Article`, `BlogPost`). | +| Query | Optional Lucene/Elasticsearch query string to filter items. | +| Max Items | Maximum number of items to extract (0 = unlimited). | + +#### JSON Source + +Reads data from a JSON string or file. + +| Setting | Description | +|---------|-------------| +| JSON Data | Raw JSON array or object to use as input. | +| Source Path | JSONPath expression to select data within the JSON structure. | + +### Transforms + +#### Field Mapping Transform + +Maps fields from the source schema to a target schema using JSONPath or Liquid expressions. + +| Setting | Description | +|---------|-------------| +| Mappings | A JSON object mapping target field names to source expressions. | + +Example mappings: + +```json +{ + "Title": "$.ContentItem.DisplayText", + "Author": "$.ContentItem.Owner", + "PublishedDate": "{{ ContentItem.PublishedUtc | date: '%Y-%m-%d' }}" +} +``` + +#### Filter Transform + +Filters records based on a condition expression. + +| Setting | Description | +|---------|-------------| +| Expression | A Liquid expression that evaluates to `true` or `false`. | + +Example expression: + +```liquid +{{ Record.Status == "Published" }} +``` + +### Loads + +#### JSON Export Load + +Exports data as a JSON file. + +| Setting | Description | +|---------|-------------| +| File Path | The output file path (relative to the tenant's data folder). | +| Format | `Array` (default) or `Lines` (JSON Lines format). | + +#### Content Item Load + +Creates or updates content items in the Orchard Core content store. + +| Setting | Description | +|---------|-------------| +| Content Type | The content type to create. | +| Owner | The owner to assign to created items. | +| Published | Whether to publish items immediately. | + +## Pipeline Parameters + +Pipelines can define parameters that are provided at execution time. This is useful for: + +- **Date ranges**: Filter data by a time period. +- **Content types**: Make pipelines reusable across different content types. +- **Output paths**: Specify where to export data. + +Parameters are defined in the pipeline properties and can be referenced in activity expressions: + +```liquid +{{ Parameters.StartDate }} +{{ Parameters.ContentType }} +``` + +### Parameter Types + +| Type | Description | +|------|-------------| +| `String` | Text value | +| `Number` | Numeric value | +| `Date` | Date/time value | +| `Boolean` | True/false value | + +## Execution + +### Manual Execution + +Pipelines can be executed manually from the admin UI by clicking the **Execute** button on the pipeline list or editor page. + +### Scheduled Execution + +Pipelines can run on a schedule using Orchard Core's background task system. Set the **Schedule** property in the pipeline properties to a cron expression: + +| Schedule | Cron Expression | +|----------|----------------| +| Every hour | `0 * * * *` | +| Daily at midnight | `0 0 * * *` | +| Every Monday at 9 AM | `0 9 * * 1` | +| Every 15 minutes | `*/15 * * * *` | + +Only **enabled** pipelines with a schedule are executed automatically. + +### On-Demand with Parameters + +Pipelines with parameters can be executed on demand, prompting the user to provide parameter values before running. + +## Execution Logs + +Each pipeline execution is logged with: + +- **Start/End time** — When the pipeline started and finished. +- **Status** — `Running`, `Completed`, or `Failed`. +- **Records Processed** — Number of records that flowed through the pipeline. +- **Errors** — Any errors encountered during execution, with details. + +View logs from the **Logs** link on the pipeline list page. + +## Extending the ETL Module + +The ETL module is designed to be extensible. Developers can add custom sources, transforms, and loads by implementing activity classes and registering them in their module's `Startup.cs`. + +### Creating a Custom Source + +```csharp +public sealed class SqlServerSource : EtlSourceActivity +{ + public override string Name => nameof(SqlServerSource); + public override LocalizedString DisplayText => S["SQL Server Source"]; + public override LocalizedString Category => S["Sources"]; + + public string ConnectionString + { + get => GetProperty(); + set => SetProperty(value); + } + + public string Query + { + get => GetProperty(); + set => SetProperty(value); + } + + public override IEnumerable GetPossibleOutcomes( + IStringLocalizer localizer) + { + return Outcomes(localizer["Done"]); + } + + public override async Task ExecuteAsync( + EtlExecutionContext context) + { + // Implementation: query SQL Server and yield records + var records = QuerySqlServerAsync(ConnectionString, Query); + context.DataStream = records; + return EtlActivityResult.Success(["Done"]); + } +} +``` + +### Creating a Display Driver + +```csharp +public sealed class SqlServerSourceDisplayDriver + : EtlActivityDisplayDriver +{ + protected override void EditActivity( + SqlServerSource activity, + SqlServerSourceViewModel model) + { + model.ConnectionString = activity.ConnectionString; + model.Query = activity.Query; + } + + protected override void UpdateActivity( + SqlServerSourceViewModel model, + SqlServerSource activity) + { + activity.ConnectionString = model.ConnectionString; + activity.Query = model.Query; + } +} +``` + +### Registering in Startup.cs + +```csharp +public sealed class Startup : StartupBase +{ + public override void ConfigureServices(IServiceCollection services) + { + services.AddEtlActivity(); + } +} +``` + +### Creating View Templates + +Each activity needs three Razor views: + +- `Views/Items/{ActivityName}.Fields.Thumbnail.cshtml` — Card shown in the activity picker. +- `Views/Items/{ActivityName}.Fields.Design.cshtml` — Summary shown on the canvas. +- `Views/Items/{ActivityName}.Fields.Edit.cshtml` — Editor form for configuring the activity. + +## Permissions + +| Permission | Description | +|------------|-------------| +| `ManageEtlPipelines` | Create, edit, and delete ETL pipeline definitions. | +| `ExecuteEtlPipelines` | Execute ETL pipelines manually. | +| `ViewEtlPipelines` | View ETL pipeline definitions and execution logs. | + +## Configuration + +The ETL module can be configured via `appsettings.json`: + +```json +{ + "OrchardCore": { + "OrchardCore_ETL": { + "MaxConcurrentPipelines": 5, + "DefaultTimeout": "00:30:00" + } + } +} +``` + +## Videos + +
+ +## Resources + +- [Workflows Module](../Workflows/README.md) — The ETL pipeline editor shares architectural patterns with the Workflows visual editor. +- [Background Tasks](../BackgroundTasks/README.md) — Scheduled pipeline execution uses the background tasks infrastructure. +- [Contents](../Contents/README.md) — The Content Item Source and Load activities interact with the content management system. diff --git a/src/docs/releases/3.0.0.md b/src/docs/releases/3.0.0.md index b30c8fffa49..43b5ed4b15d 100644 --- a/src/docs/releases/3.0.0.md +++ b/src/docs/releases/3.0.0.md @@ -628,6 +628,20 @@ The Admin → Media → Library screen now includes an "Available Storage" indic If you use a different underlying file store (e.g., Azure Blob storage, Amazon AWS), or if you want to enforce additional storage constraints (i.e., usage quotas), that's also possible. Create a new event handler that implements the `IMediaEventHandler.MediaPermittedStorageAsync(MediaPermittedStorageContext)` method. Note that if you want to use multiple event handlers, you should update the `PermittedStorage` value using the `MediaPermittedStorageContext.Constrain(long)` method rather than manually editing the `PermittedStorage` value. +### Content Transfer Module (New) + +A new **Content Transfer** module has been added that provides bulk import and export of content items using Excel (`.xlsx`) files. Key features include: + +- **Bulk Import**: Upload an Excel file to import content items. Files are processed in the background with configurable batch sizes, checkpoint-based resume, and progress tracking. +- **Bulk Export**: Export content items to an Excel file. Small datasets (≤500 records by default) are exported immediately; larger datasets are queued for background processing and the user is notified when the file is ready. +- **Export Dashboard**: A dedicated dashboard for managing queued export requests and downloading completed export files. +- **Export Filters**: Partial export support with filters for created/modified date ranges, owner usernames, published-only, latest-only, or all versions. +- **Validation**: Imported records are validated via `IContentManager.ValidateAsync()`. Rows that fail validation are tracked, and users can download an Excel file containing only the rejected records for review and re-import. +- **Notifications**: When the `OrchardCore.Notifications` module is enabled, users receive notifications when queued exports complete. +- **Extensible Handlers**: Custom import/export handlers can be added for any content part or content field. Built-in handlers are provided for `TitlePart`, `HtmlBodyPart`, `AutoroutePart`, and standard content fields (Text, Boolean, Number, Date, DateTime, Time, ContentPicker). + +For more information, see the [Content Transfer documentation](../reference/modules/ContentTransfer/README.md). + ## Miscellaneous ### Shapes diff --git a/test/OrchardCore.Tests/Modules/OrchardCore.ContentsTransfer/ContentImportManagerTests.cs b/test/OrchardCore.Tests/Modules/OrchardCore.ContentsTransfer/ContentImportManagerTests.cs new file mode 100644 index 00000000000..9496458e4f8 --- /dev/null +++ b/test/OrchardCore.Tests/Modules/OrchardCore.ContentsTransfer/ContentImportManagerTests.cs @@ -0,0 +1,2023 @@ +using System.Data; +using OrchardCore.Alias.Models; +using OrchardCore.ArchiveLater.Models; +using OrchardCore.Autoroute.Models; +using OrchardCore.ContentFields.Fields; +using OrchardCore.ContentManagement; +using OrchardCore.ContentManagement.Metadata.Models; +using OrchardCore.ContentTransfer; +using OrchardCore.ContentTransfer.Handlers; +using OrchardCore.ContentTransfer.Handlers.Fields; +using OrchardCore.ContentTransfer.Services; +using OrchardCore.Html.Models; +using OrchardCore.Liquid.Models; +using OrchardCore.Markdown.Fields; +using OrchardCore.Markdown.Models; +using OrchardCore.Media.Fields; +using OrchardCore.PublishLater.Models; +using OrchardCore.Taxonomies.Fields; + +namespace OrchardCore.Tests.Modules.OrchardCore.ContentTransfer; + +public class ContentImportManagerTests +{ + private readonly Mock _handlerResolver; + private readonly Mock> _partFactory; + private readonly Mock> _fieldFactory; + private readonly Mock _contentManager; + private readonly Mock> _logger; + private readonly ContentImportManager _manager; + + public ContentImportManagerTests() + { + _handlerResolver = new Mock(); + _partFactory = new Mock>(); + _fieldFactory = new Mock>(); + _contentManager = new Mock(); + _logger = new Mock>(); + + _handlerResolver.Setup(x => x.GetPartHandlers(It.IsAny())) + .Returns([]); + _handlerResolver.Setup(x => x.GetFieldHandlers(It.IsAny())) + .Returns([]); + + _manager = new ContentImportManager( + _handlerResolver.Object, + _partFactory.Object, + _fieldFactory.Object, + [], + _contentManager.Object, + _logger.Object); + } + + [Fact] + public async Task GetColumnsAsync_WithNoParts_ReturnsEmpty() + { + var contentTypeDefinition = new ContentTypeDefinition("TestType", "Test Type"); + + _contentManager.Setup(x => x.NewAsync(It.IsAny())) + .ReturnsAsync(new ContentItem { ContentType = "TestType" }); + + var context = new ImportContentContext + { + ContentItem = new ContentItem { ContentType = "TestType" }, + ContentTypeDefinition = contentTypeDefinition, + }; + + var columns = await _manager.GetColumnsAsync(context); + + Assert.NotNull(columns); + Assert.Empty(columns); + } + + [Fact] + public async Task GetColumnsAsync_WithContentImportHandler_ReturnsHandlerColumns() + { + var handler = new Mock(); + handler.Setup(x => x.GetColumns(It.IsAny())) + .Returns(new[] { new ImportColumn { Name = "TestColumn" } }); + + var manager = new ContentImportManager( + _handlerResolver.Object, + _partFactory.Object, + _fieldFactory.Object, + [handler.Object], + _contentManager.Object, + _logger.Object); + + _contentManager.Setup(x => x.NewAsync(It.IsAny())) + .ReturnsAsync(new ContentItem { ContentType = "TestType" }); + + var context = new ImportContentContext + { + ContentItem = new ContentItem { ContentType = "TestType" }, + ContentTypeDefinition = new ContentTypeDefinition("TestType", "Test Type"), + }; + + var columns = await manager.GetColumnsAsync(context); + + Assert.Single(columns); + Assert.Equal("TestColumn", columns.First().Name); + } + + [Fact] + public async Task ImportAsync_InvokesContentImportHandlers() + { + var handler = new Mock(); + handler.Setup(x => x.ImportAsync(It.IsAny())) + .Returns(Task.CompletedTask); + + var manager = new ContentImportManager( + _handlerResolver.Object, + _partFactory.Object, + _fieldFactory.Object, + [handler.Object], + _contentManager.Object, + _logger.Object); + + var dataTable = new DataTable(); + dataTable.Columns.Add("Title"); + var row = dataTable.NewRow(); + row["Title"] = "Test Title"; + dataTable.Rows.Add(row); + + var context = new ContentImportContext + { + ContentItem = new ContentItem { ContentType = "TestType" }, + ContentTypeDefinition = new ContentTypeDefinition("TestType", "Test Type"), + Columns = dataTable.Columns, + Row = row, + }; + + await manager.ImportAsync(context); + + handler.Verify(x => x.ImportAsync(It.IsAny()), Times.Once); + } + + [Fact] + public async Task ExportAsync_InvokesContentExportHandlers() + { + var handler = new Mock(); + handler.Setup(x => x.ExportAsync(It.IsAny())) + .Returns(Task.CompletedTask); + + var manager = new ContentImportManager( + _handlerResolver.Object, + _partFactory.Object, + _fieldFactory.Object, + [handler.Object], + _contentManager.Object, + _logger.Object); + + var dataTable = new DataTable(); + dataTable.Columns.Add("Title"); + var row = dataTable.NewRow(); + + var context = new ContentExportContext + { + ContentItem = new ContentItem { ContentType = "TestType" }, + ContentTypeDefinition = new ContentTypeDefinition("TestType", "Test Type"), + Row = row, + }; + + await manager.ExportAsync(context); + + handler.Verify(x => x.ExportAsync(It.IsAny()), Times.Once); + } + + [Fact] + public async Task ImportAsync_ThrowsOnNullContext() + { + await Assert.ThrowsAsync(() => _manager.ImportAsync(null)); + } + + [Fact] + public async Task ExportAsync_ThrowsOnNullContext() + { + await Assert.ThrowsAsync(() => _manager.ExportAsync(null)); + } + + [Fact] + public async Task GetColumnsAsync_ThrowsOnNullContext() + { + await Assert.ThrowsAsync(() => _manager.GetColumnsAsync(null)); + } +} + +public class CommonContentImportHandlerTests +{ + private readonly CommonContentImportHandler _handler; + private readonly Mock _idGenerator; + + public CommonContentImportHandlerTests() + { + _idGenerator = new Mock(); + _idGenerator.Setup(x => x.GenerateUniqueId(It.IsAny())) + .Returns("abcdefghijklmnopqrstuvwxyz"); + + _handler = new CommonContentImportHandler( + Mock.Of>(), + _idGenerator.Object); + } + + [Fact] + public void GetColumns_ReturnsContentItemIdColumn() + { + var context = new ImportContentContext + { + ContentItem = new ContentItem(), + ContentTypeDefinition = new ContentTypeDefinition("Test", "Test"), + }; + + var columns = _handler.GetColumns(context); + + Assert.Contains(columns, c => c.Name == nameof(ContentItem.ContentItemId)); + } + + [Fact] + public void GetColumns_ReturnsCreatedUtcAsExportOnly() + { + var context = new ImportContentContext + { + ContentItem = new ContentItem(), + ContentTypeDefinition = new ContentTypeDefinition("Test", "Test"), + }; + + var columns = _handler.GetColumns(context); + + var createdUtcCol = columns.FirstOrDefault(c => c.Name == nameof(ContentItem.CreatedUtc)); + Assert.NotNull(createdUtcCol); + Assert.Equal(ImportColumnType.ExportOnly, createdUtcCol.Type); + } + + [Fact] + public void GetColumns_ReturnsModifiedUtcAsExportOnly() + { + var context = new ImportContentContext + { + ContentItem = new ContentItem(), + ContentTypeDefinition = new ContentTypeDefinition("Test", "Test"), + }; + + var columns = _handler.GetColumns(context); + + var modifiedUtcCol = columns.FirstOrDefault(c => c.Name == nameof(ContentItem.ModifiedUtc)); + Assert.NotNull(modifiedUtcCol); + Assert.Equal(ImportColumnType.ExportOnly, modifiedUtcCol.Type); + } + + [Fact] + public async Task ImportAsync_SetsContentItemId_WhenValidLength() + { + var dataTable = new DataTable(); + dataTable.Columns.Add(nameof(ContentItem.ContentItemId)); + var row = dataTable.NewRow(); + row[nameof(ContentItem.ContentItemId)] = "abcdefghijklmnopqrstuvwxyz"; + dataTable.Rows.Add(row); + + var contentItem = new ContentItem(); + + var context = new ContentImportContext + { + ContentItem = contentItem, + ContentTypeDefinition = new ContentTypeDefinition("Test", "Test"), + Columns = dataTable.Columns, + Row = row, + }; + + await _handler.ImportAsync(context); + + Assert.Equal("abcdefghijklmnopqrstuvwxyz", contentItem.ContentItemId); + } + + [Fact] + public async Task ImportAsync_DoesNotSetContentItemId_WhenEmpty() + { + var dataTable = new DataTable(); + dataTable.Columns.Add(nameof(ContentItem.ContentItemId)); + var row = dataTable.NewRow(); + row[nameof(ContentItem.ContentItemId)] = ""; + dataTable.Rows.Add(row); + + var contentItem = new ContentItem(); + + var context = new ContentImportContext + { + ContentItem = contentItem, + ContentTypeDefinition = new ContentTypeDefinition("Test", "Test"), + Columns = dataTable.Columns, + Row = row, + }; + + await _handler.ImportAsync(context); + + Assert.Null(contentItem.ContentItemId); + } + + [Fact] + public async Task ImportAsync_DoesNotSetContentItemId_WhenWrongLength() + { + var dataTable = new DataTable(); + dataTable.Columns.Add(nameof(ContentItem.ContentItemId)); + var row = dataTable.NewRow(); + row[nameof(ContentItem.ContentItemId)] = "short"; + dataTable.Rows.Add(row); + + var contentItem = new ContentItem(); + + var context = new ContentImportContext + { + ContentItem = contentItem, + ContentTypeDefinition = new ContentTypeDefinition("Test", "Test"), + Columns = dataTable.Columns, + Row = row, + }; + + await _handler.ImportAsync(context); + + Assert.Null(contentItem.ContentItemId); + } + + [Fact] + public async Task ExportAsync_SetsContentItemIdAndDates() + { + var dataTable = new DataTable(); + dataTable.Columns.Add(nameof(ContentItem.ContentItemId)); + dataTable.Columns.Add(nameof(ContentItem.CreatedUtc)); + dataTable.Columns.Add(nameof(ContentItem.ModifiedUtc)); + var row = dataTable.NewRow(); + + var contentItem = new ContentItem + { + ContentItemId = "test-id-123456789012345", + CreatedUtc = new DateTime(2024, 1, 15, 10, 30, 0, DateTimeKind.Utc), + ModifiedUtc = new DateTime(2024, 6, 20, 14, 45, 0, DateTimeKind.Utc), + }; + + var context = new ContentExportContext + { + ContentItem = contentItem, + ContentTypeDefinition = new ContentTypeDefinition("Test", "Test"), + Row = row, + }; + + await _handler.ExportAsync(context); + + Assert.Equal("test-id-123456789012345", row[nameof(ContentItem.ContentItemId)].ToString()); + Assert.Equal(contentItem.CreatedUtc?.ToString(), row[nameof(ContentItem.CreatedUtc)]?.ToString()); + Assert.Equal(contentItem.ModifiedUtc?.ToString(), row[nameof(ContentItem.ModifiedUtc)]?.ToString()); + } + + [Fact] + public async Task ImportAsync_ThrowsOnNullContext() + { + await Assert.ThrowsAsync(() => _handler.ImportAsync(null)); + } + + [Fact] + public async Task ExportAsync_ThrowsOnNullContext() + { + await Assert.ThrowsAsync(() => _handler.ExportAsync(null)); + } +} + +public class TextFieldImportHandlerTests +{ + private readonly TextFieldImportHandler _handler; + + public TextFieldImportHandlerTests() + { + _handler = new TextFieldImportHandler(Mock.Of>()); + } + + [Fact] + public void GetColumns_ReturnsSingleColumn() + { + var context = CreateFieldContext("MyPart", "MyField"); + + var columns = _handler.GetColumns(context); + + Assert.Single(columns); + Assert.Equal("MyPart_MyField_Text", columns.First().Name); + } + + [Fact] + public async Task ImportAsync_SetsTextFieldValue() + { + var part = new ContentPart(); + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_MyField_Text"); + var row = dataTable.NewRow(); + row["MyPart_MyField_Text"] = "Hello World"; + dataTable.Rows.Add(row); + + var context = CreateFieldImportContext("MyPart", "MyField", part, dataTable, row); + + await _handler.ImportAsync(context); + + var field = part.Get("MyField"); + Assert.NotNull(field); + Assert.Equal("Hello World", field.Text); + } + + [Fact] + public async Task ImportAsync_TrimsWhitespace() + { + var part = new ContentPart(); + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_MyField_Text"); + var row = dataTable.NewRow(); + row["MyPart_MyField_Text"] = " Trimmed "; + dataTable.Rows.Add(row); + + var context = CreateFieldImportContext("MyPart", "MyField", part, dataTable, row); + + await _handler.ImportAsync(context); + + var field = part.Get("MyField"); + Assert.Equal("Trimmed", field.Text); + } + + [Fact] + public async Task ExportAsync_ReturnsTextFieldValue() + { + var part = new ContentPart(); + part.Alter("MyField", f => f.Text = "Exported Text"); + + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_MyField_Text"); + var row = dataTable.NewRow(); + + var context = CreateFieldExportContext("MyPart", "MyField", part, row); + + await _handler.ExportAsync(context); + + Assert.Equal("Exported Text", row["MyPart_MyField_Text"].ToString()); + } + + [Fact] + public async Task ExportAsync_ReturnsNull_WhenFieldNotPresent() + { + var part = new ContentPart(); + + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_MyField_Text"); + var row = dataTable.NewRow(); + + var context = CreateFieldExportContext("MyPart", "MyField", part, row); + + await _handler.ExportAsync(context); + + Assert.True(row["MyPart_MyField_Text"] == null || row["MyPart_MyField_Text"] == DBNull.Value); + } + + private static ImportContentFieldContext CreateFieldContext(string partName, string fieldName) + { + return new ImportContentFieldContext + { + PartName = partName, + ContentPartFieldDefinition = CreateFieldDefinition(fieldName, nameof(TextField)), + ContentPart = new ContentPart(), + ContentItem = new ContentItem(), + }; + } + + private static ContentFieldImportMapContext CreateFieldImportContext( + string partName, string fieldName, ContentPart part, DataTable dataTable, DataRow row) + { + return new ContentFieldImportMapContext + { + PartName = partName, + ContentPartFieldDefinition = CreateFieldDefinition(fieldName, nameof(TextField)), + ContentPart = part, + ContentItem = new ContentItem(), + Columns = dataTable.Columns, + Row = row, + }; + } + + private static ContentFieldExportMapContext CreateFieldExportContext( + string partName, string fieldName, ContentPart part, DataRow row) + { + return new ContentFieldExportMapContext + { + PartName = partName, + ContentPartFieldDefinition = CreateFieldDefinition(fieldName, nameof(TextField)), + ContentPart = part, + ContentField = part.Get(fieldName) ?? new TextField(), + ContentItem = new ContentItem(), + Row = row, + }; + } + + private static ContentPartFieldDefinition CreateFieldDefinition(string fieldName, string fieldType) + { + return new ContentPartFieldDefinition(new ContentFieldDefinition(fieldType), fieldName, []); + } +} + +public class BooleanFieldImportHandlerTests +{ + private readonly BooleanFieldImportHandler _handler; + + public BooleanFieldImportHandlerTests() + { + _handler = new BooleanFieldImportHandler(Mock.Of>()); + } + + [Theory] + [InlineData("True", true)] + [InlineData("true", true)] + [InlineData("TRUE", true)] + [InlineData("False", false)] + [InlineData("false", false)] + [InlineData("", false)] + public async Task ImportAsync_ParsesBooleanCorrectly(string input, bool expected) + { + var part = new ContentPart(); + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_IsActive_Value"); + var row = dataTable.NewRow(); + row["MyPart_IsActive_Value"] = input; + dataTable.Rows.Add(row); + + var context = new ContentFieldImportMapContext + { + PartName = "MyPart", + ContentPartFieldDefinition = CreateFieldDefinition("IsActive", nameof(BooleanField)), + ContentPart = part, + ContentItem = new ContentItem(), + Columns = dataTable.Columns, + Row = row, + }; + + await _handler.ImportAsync(context); + + var field = part.Get("IsActive"); + Assert.NotNull(field); + Assert.Equal(expected, field.Value); + } + + [Fact] + public async Task ExportAsync_ReturnsBooleanValue() + { + var part = new ContentPart(); + part.Alter("IsActive", f => f.Value = true); + + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_IsActive_Value"); + var row = dataTable.NewRow(); + + var context = new ContentFieldExportMapContext + { + PartName = "MyPart", + ContentPartFieldDefinition = CreateFieldDefinition("IsActive", nameof(BooleanField)), + ContentPart = part, + ContentField = part.Get("IsActive"), + ContentItem = new ContentItem(), + Row = row, + }; + + await _handler.ExportAsync(context); + + Assert.Equal(true.ToString(), row["MyPart_IsActive_Value"]?.ToString()); + } + + [Fact] + public void GetColumns_ReturnsValidValues() + { + var context = new ImportContentFieldContext + { + PartName = "MyPart", + ContentPartFieldDefinition = CreateFieldDefinition("IsActive", nameof(BooleanField)), + ContentPart = new ContentPart(), + ContentItem = new ContentItem(), + }; + + var columns = _handler.GetColumns(context); + + var column = Assert.Single(columns); + Assert.Contains("True", column.ValidValues); + Assert.Contains("False", column.ValidValues); + } + + private static ContentPartFieldDefinition CreateFieldDefinition(string fieldName, string fieldType) + { + return new ContentPartFieldDefinition(new ContentFieldDefinition(fieldType), fieldName, []); + } +} + +public class NumberFieldImportHandlerTests +{ + private readonly NumberFieldImportHandler _handler; + + public NumberFieldImportHandlerTests() + { + _handler = new NumberFieldImportHandler(Mock.Of>()); + } + + [Theory] + [InlineData("42", 42)] + [InlineData("3.14", 3.14)] + [InlineData("-100", -100)] + [InlineData("0", 0)] + public async Task ImportAsync_ParsesNumberCorrectly(string input, double expected) + { + var part = new ContentPart(); + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_Price_Value"); + var row = dataTable.NewRow(); + row["MyPart_Price_Value"] = input; + dataTable.Rows.Add(row); + + var context = new ContentFieldImportMapContext + { + PartName = "MyPart", + ContentPartFieldDefinition = CreateFieldDefinition("Price", nameof(NumericField)), + ContentPart = part, + ContentItem = new ContentItem(), + Columns = dataTable.Columns, + Row = row, + }; + + await _handler.ImportAsync(context); + + var field = part.Get("Price"); + Assert.NotNull(field); + Assert.Equal((decimal)expected, field.Value); + } + + [Theory] + [InlineData("not-a-number")] + [InlineData("abc")] + public async Task ImportAsync_DoesNotSetValue_ForInvalidInput(string input) + { + var part = new ContentPart(); + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_Price_Value"); + var row = dataTable.NewRow(); + row["MyPart_Price_Value"] = input; + dataTable.Rows.Add(row); + + var context = new ContentFieldImportMapContext + { + PartName = "MyPart", + ContentPartFieldDefinition = CreateFieldDefinition("Price", nameof(NumericField)), + ContentPart = part, + ContentItem = new ContentItem(), + Columns = dataTable.Columns, + Row = row, + }; + + await _handler.ImportAsync(context); + + var field = part.Get("Price"); + Assert.Null(field); + } + + [Fact] + public async Task ExportAsync_ReturnsNumericValue() + { + var part = new ContentPart(); + part.Alter("Price", f => f.Value = 99.99m); + + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_Price_Value"); + var row = dataTable.NewRow(); + + var context = new ContentFieldExportMapContext + { + PartName = "MyPart", + ContentPartFieldDefinition = CreateFieldDefinition("Price", nameof(NumericField)), + ContentPart = part, + ContentField = part.Get("Price"), + ContentItem = new ContentItem(), + Row = row, + }; + + await _handler.ExportAsync(context); + + Assert.Equal(99.99m.ToString(), row["MyPart_Price_Value"]?.ToString()); + } + + private static ContentPartFieldDefinition CreateFieldDefinition(string fieldName, string fieldType) + { + return new ContentPartFieldDefinition(new ContentFieldDefinition(fieldType), fieldName, []); + } +} + +public class DateFieldImportHandlerTests +{ + private readonly DateFieldImportHandler _handler; + + public DateFieldImportHandlerTests() + { + _handler = new DateFieldImportHandler(Mock.Of>()); + } + + [Fact] + public async Task ImportAsync_ParsesValidDate() + { + var part = new ContentPart(); + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_StartDate_Value"); + var row = dataTable.NewRow(); + row["MyPart_StartDate_Value"] = "2024-06-15"; + dataTable.Rows.Add(row); + + var context = new ContentFieldImportMapContext + { + PartName = "MyPart", + ContentPartFieldDefinition = CreateFieldDefinition("StartDate", nameof(DateField)), + ContentPart = part, + ContentItem = new ContentItem(), + Columns = dataTable.Columns, + Row = row, + }; + + await _handler.ImportAsync(context); + + var field = part.Get("StartDate"); + Assert.NotNull(field); + Assert.Equal(new DateTime(2024, 6, 15), field.Value); + } + + [Theory] + [InlineData("")] + [InlineData("not-a-date")] + public async Task ImportAsync_DoesNotSetValue_ForInvalidDate(string input) + { + var part = new ContentPart(); + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_StartDate_Value"); + var row = dataTable.NewRow(); + row["MyPart_StartDate_Value"] = input; + dataTable.Rows.Add(row); + + var context = new ContentFieldImportMapContext + { + PartName = "MyPart", + ContentPartFieldDefinition = CreateFieldDefinition("StartDate", nameof(DateField)), + ContentPart = part, + ContentItem = new ContentItem(), + Columns = dataTable.Columns, + Row = row, + }; + + await _handler.ImportAsync(context); + + var field = part.Get("StartDate"); + Assert.Null(field); + } + + [Fact] + public async Task ExportAsync_ReturnsDateValue() + { + var part = new ContentPart(); + var date = new DateTime(2024, 1, 15); + part.Alter("StartDate", f => f.Value = date); + + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_StartDate_Value"); + var row = dataTable.NewRow(); + + var context = new ContentFieldExportMapContext + { + PartName = "MyPart", + ContentPartFieldDefinition = CreateFieldDefinition("StartDate", nameof(DateField)), + ContentPart = part, + ContentField = part.Get("StartDate"), + ContentItem = new ContentItem(), + Row = row, + }; + + await _handler.ExportAsync(context); + + Assert.Equal(date.ToString(), row["MyPart_StartDate_Value"]?.ToString()); + } + + [Fact] + public async Task ExportAsync_ReturnsNull_WhenFieldNotPresent() + { + var part = new ContentPart(); + + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_StartDate_Value"); + var row = dataTable.NewRow(); + + var context = new ContentFieldExportMapContext + { + PartName = "MyPart", + ContentPartFieldDefinition = CreateFieldDefinition("StartDate", nameof(DateField)), + ContentPart = part, + ContentField = new DateField(), + ContentItem = new ContentItem(), + Row = row, + }; + + await _handler.ExportAsync(context); + + Assert.True(row["MyPart_StartDate_Value"] == null || row["MyPart_StartDate_Value"] == DBNull.Value); + } + + private static ContentPartFieldDefinition CreateFieldDefinition(string fieldName, string fieldType) + { + return new ContentPartFieldDefinition(new ContentFieldDefinition(fieldType), fieldName, []); + } +} + +public class HtmlBodyPartContentImportHandlerTests +{ + private readonly HtmlBodyPartContentImportHandler _handler; + + public HtmlBodyPartContentImportHandlerTests() + { + _handler = new HtmlBodyPartContentImportHandler(Mock.Of>()); + } + + [Fact] + public void GetColumns_ReturnsSingleHtmlColumn() + { + var context = CreatePartContext("HtmlBodyPart"); + + var columns = _handler.GetColumns(context); + + var column = Assert.Single(columns); + Assert.Equal("HtmlBodyPart_Html", column.Name); + } + + [Fact] + public async Task ImportAsync_SetsHtmlOnPart() + { + var contentItem = new ContentItem { ContentType = "Article" }; + var dataTable = new DataTable(); + dataTable.Columns.Add("HtmlBodyPart_Html"); + var row = dataTable.NewRow(); + row["HtmlBodyPart_Html"] = "

Hello World

"; + dataTable.Rows.Add(row); + + var context = new ContentPartImportMapContext + { + ContentTypePartDefinition = CreateTypePartDefinition("HtmlBodyPart"), + ContentItem = contentItem, + Columns = dataTable.Columns, + Row = row, + }; + + await _handler.ImportAsync(context); + + var part = contentItem.As(); + Assert.NotNull(part); + Assert.Equal("

Hello World

", part.Html); + } + + [Fact] + public async Task ImportAsync_SetsEmptyString() + { + var contentItem = new ContentItem { ContentType = "Article" }; + var dataTable = new DataTable(); + dataTable.Columns.Add("HtmlBodyPart_Html"); + var row = dataTable.NewRow(); + row["HtmlBodyPart_Html"] = ""; + dataTable.Rows.Add(row); + + var context = new ContentPartImportMapContext + { + ContentTypePartDefinition = CreateTypePartDefinition("HtmlBodyPart"), + ContentItem = contentItem, + Columns = dataTable.Columns, + Row = row, + }; + + await _handler.ImportAsync(context); + + var part = contentItem.As(); + Assert.NotNull(part); + Assert.Equal(string.Empty, part.Html); + } + + [Fact] + public async Task ImportAsync_IgnoresUnmatchedColumn() + { + var contentItem = new ContentItem { ContentType = "Article" }; + var dataTable = new DataTable(); + dataTable.Columns.Add("WrongColumn"); + var row = dataTable.NewRow(); + row["WrongColumn"] = "

Should not be set

"; + dataTable.Rows.Add(row); + + var context = new ContentPartImportMapContext + { + ContentTypePartDefinition = CreateTypePartDefinition("HtmlBodyPart"), + ContentItem = contentItem, + Columns = dataTable.Columns, + Row = row, + }; + + await _handler.ImportAsync(context); + + var part = contentItem.As(); + Assert.Null(part); + } + + [Fact] + public async Task ExportAsync_WritesHtmlToRow() + { + var contentItem = new ContentItem { ContentType = "Article" }; + contentItem.Alter(p => p.Html = "

Title

"); + + var context = CreatePartContext("HtmlBodyPart"); + _handler.GetColumns(context); + + var dataTable = new DataTable(); + dataTable.Columns.Add("HtmlBodyPart_Html"); + var row = dataTable.NewRow(); + + var exportContext = new ContentPartExportMapContext + { + ContentTypePartDefinition = CreateTypePartDefinition("HtmlBodyPart"), + ContentPart = contentItem.As(), + ContentItem = contentItem, + Row = row, + }; + + await _handler.ExportAsync(exportContext); + + Assert.Equal("

Title

", row["HtmlBodyPart_Html"]?.ToString()); + } + + [Fact] + public async Task ExportAsync_WritesEmptyString_WhenHtmlIsNull() + { + var contentItem = new ContentItem { ContentType = "Article" }; + contentItem.Alter(p => p.Html = null); + + var context = CreatePartContext("HtmlBodyPart"); + _handler.GetColumns(context); + + var dataTable = new DataTable(); + dataTable.Columns.Add("HtmlBodyPart_Html"); + var row = dataTable.NewRow(); + + var exportContext = new ContentPartExportMapContext + { + ContentTypePartDefinition = CreateTypePartDefinition("HtmlBodyPart"), + ContentPart = contentItem.As(), + ContentItem = contentItem, + Row = row, + }; + + await _handler.ExportAsync(exportContext); + + Assert.Equal(string.Empty, row["HtmlBodyPart_Html"]?.ToString()); + } + + private static ImportContentPartContext CreatePartContext(string partName) + { + return new ImportContentPartContext + { + ContentTypePartDefinition = CreateTypePartDefinition(partName), + }; + } + + private static ContentTypePartDefinition CreateTypePartDefinition(string partName) + { + var contentTypeDefinition = new ContentTypeDefinition("Article", "Article"); + var contentPartDefinition = new ContentPartDefinition(partName); + + return new ContentTypePartDefinition(partName, contentPartDefinition, []) + { + ContentTypeDefinition = contentTypeDefinition, + }; + } +} + +public class AutoroutePartContentImportHandlerTests +{ + private readonly AutoroutePartContentImportHandler _handler; + + public AutoroutePartContentImportHandlerTests() + { + _handler = new AutoroutePartContentImportHandler(Mock.Of>()); + } + + [Fact] + public void GetColumns_ReturnsSinglePathColumn() + { + var context = CreatePartContext("AutoroutePart"); + + var columns = _handler.GetColumns(context); + + var column = Assert.Single(columns); + Assert.Equal("AutoroutePart_Path", column.Name); + } + + [Fact] + public async Task ImportAsync_SetsPathOnPart() + { + var contentItem = new ContentItem { ContentType = "Article" }; + var dataTable = new DataTable(); + dataTable.Columns.Add("AutoroutePart_Path"); + var row = dataTable.NewRow(); + row["AutoroutePart_Path"] = "my-article-slug"; + dataTable.Rows.Add(row); + + var context = new ContentPartImportMapContext + { + ContentTypePartDefinition = CreateTypePartDefinition("AutoroutePart"), + ContentItem = contentItem, + Columns = dataTable.Columns, + Row = row, + }; + + await _handler.ImportAsync(context); + + var part = contentItem.As(); + Assert.NotNull(part); + Assert.Equal("my-article-slug", part.Path); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + public async Task ImportAsync_IgnoresEmptyOrWhitespacePath(string input) + { + var contentItem = new ContentItem { ContentType = "Article" }; + var dataTable = new DataTable(); + dataTable.Columns.Add("AutoroutePart_Path"); + var row = dataTable.NewRow(); + row["AutoroutePart_Path"] = input; + dataTable.Rows.Add(row); + + var context = new ContentPartImportMapContext + { + ContentTypePartDefinition = CreateTypePartDefinition("AutoroutePart"), + ContentItem = contentItem, + Columns = dataTable.Columns, + Row = row, + }; + + await _handler.ImportAsync(context); + + var part = contentItem.As(); + Assert.Null(part); + } + + [Fact] + public async Task ImportAsync_IgnoresUnmatchedColumn() + { + var contentItem = new ContentItem { ContentType = "Article" }; + var dataTable = new DataTable(); + dataTable.Columns.Add("WrongColumn"); + var row = dataTable.NewRow(); + row["WrongColumn"] = "some-path"; + dataTable.Rows.Add(row); + + var context = new ContentPartImportMapContext + { + ContentTypePartDefinition = CreateTypePartDefinition("AutoroutePart"), + ContentItem = contentItem, + Columns = dataTable.Columns, + Row = row, + }; + + await _handler.ImportAsync(context); + + var part = contentItem.As(); + Assert.Null(part); + } + + [Fact] + public async Task ExportAsync_WritesPathToRow() + { + var contentItem = new ContentItem { ContentType = "Article" }; + contentItem.Alter(p => p.Path = "exported-article"); + + var context = CreatePartContext("AutoroutePart"); + _handler.GetColumns(context); + + var dataTable = new DataTable(); + dataTable.Columns.Add("AutoroutePart_Path"); + var row = dataTable.NewRow(); + + var exportContext = new ContentPartExportMapContext + { + ContentTypePartDefinition = CreateTypePartDefinition("AutoroutePart"), + ContentPart = contentItem.As(), + ContentItem = contentItem, + Row = row, + }; + + await _handler.ExportAsync(exportContext); + + Assert.Equal("exported-article", row["AutoroutePart_Path"]?.ToString()); + } + + [Fact] + public async Task ExportAsync_WritesEmptyString_WhenPathIsNull() + { + var contentItem = new ContentItem { ContentType = "Article" }; + contentItem.Alter(p => p.Path = null); + + var context = CreatePartContext("AutoroutePart"); + _handler.GetColumns(context); + + var dataTable = new DataTable(); + dataTable.Columns.Add("AutoroutePart_Path"); + var row = dataTable.NewRow(); + + var exportContext = new ContentPartExportMapContext + { + ContentTypePartDefinition = CreateTypePartDefinition("AutoroutePart"), + ContentPart = contentItem.As(), + ContentItem = contentItem, + Row = row, + }; + + await _handler.ExportAsync(exportContext); + + Assert.Equal(string.Empty, row["AutoroutePart_Path"]?.ToString()); + } + + private static ImportContentPartContext CreatePartContext(string partName) + { + return new ImportContentPartContext + { + ContentTypePartDefinition = CreateTypePartDefinition(partName), + }; + } + + private static ContentTypePartDefinition CreateTypePartDefinition(string partName) + { + var contentTypeDefinition = new ContentTypeDefinition("Article", "Article"); + var contentPartDefinition = new ContentPartDefinition(partName); + + return new ContentTypePartDefinition(partName, contentPartDefinition, []) + { + ContentTypeDefinition = contentTypeDefinition, + }; + } +} + +public class MarkdownBodyPartContentImportHandlerTests +{ + private readonly MarkdownBodyPartContentImportHandler _handler; + + public MarkdownBodyPartContentImportHandlerTests() + { + _handler = new MarkdownBodyPartContentImportHandler(Mock.Of>()); + } + + [Fact] + public void GetColumns_ReturnsSingleMarkdownColumn() + { + var context = CreatePartContext("MarkdownBodyPart"); + + var columns = _handler.GetColumns(context); + + var column = Assert.Single(columns); + Assert.Equal("MarkdownBodyPart_Markdown", column.Name); + } + + [Fact] + public async Task ImportAsync_SetsMarkdownOnPart() + { + var contentItem = new ContentItem { ContentType = "Article" }; + var dataTable = new DataTable(); + dataTable.Columns.Add("MarkdownBodyPart_Markdown"); + var row = dataTable.NewRow(); + row["MarkdownBodyPart_Markdown"] = "## Hello World"; + dataTable.Rows.Add(row); + + var context = new ContentPartImportMapContext + { + ContentTypePartDefinition = CreateTypePartDefinition("MarkdownBodyPart"), + ContentItem = contentItem, + Columns = dataTable.Columns, + Row = row, + }; + + await _handler.ImportAsync(context); + + var part = contentItem.As(); + Assert.NotNull(part); + Assert.Equal("## Hello World", part.Markdown); + } + + [Fact] + public async Task ExportAsync_WritesMarkdownToRow() + { + var contentItem = new ContentItem { ContentType = "Article" }; + contentItem.Alter(p => p.Markdown = "## Exported"); + + _handler.GetColumns(CreatePartContext("MarkdownBodyPart")); + + var dataTable = new DataTable(); + dataTable.Columns.Add("MarkdownBodyPart_Markdown"); + var row = dataTable.NewRow(); + + var exportContext = new ContentPartExportMapContext + { + ContentTypePartDefinition = CreateTypePartDefinition("MarkdownBodyPart"), + ContentPart = contentItem.As(), + ContentItem = contentItem, + Row = row, + }; + + await _handler.ExportAsync(exportContext); + + Assert.Equal("## Exported", row["MarkdownBodyPart_Markdown"]?.ToString()); + } + + private static ImportContentPartContext CreatePartContext(string partName) + => new() + { + ContentTypePartDefinition = CreateTypePartDefinition(partName), + }; + + private static ContentTypePartDefinition CreateTypePartDefinition(string partName) + { + var contentTypeDefinition = new ContentTypeDefinition("Article", "Article"); + var contentPartDefinition = new ContentPartDefinition(partName); + + return new ContentTypePartDefinition(partName, contentPartDefinition, []) + { + ContentTypeDefinition = contentTypeDefinition, + }; + } +} + +public class AliasPartContentImportHandlerTests +{ + private readonly AliasPartContentImportHandler _handler; + + public AliasPartContentImportHandlerTests() + { + _handler = new AliasPartContentImportHandler(Mock.Of>()); + } + + [Fact] + public async Task ImportAsync_SetsAliasOnPart() + { + var contentItem = new ContentItem { ContentType = "Article" }; + var dataTable = new DataTable(); + dataTable.Columns.Add("AliasPart_Alias"); + var row = dataTable.NewRow(); + row["AliasPart_Alias"] = "my-alias"; + dataTable.Rows.Add(row); + + var context = new ContentPartImportMapContext + { + ContentTypePartDefinition = CreateTypePartDefinition("AliasPart"), + ContentItem = contentItem, + Columns = dataTable.Columns, + Row = row, + }; + + await _handler.ImportAsync(context); + + var part = contentItem.As(); + Assert.NotNull(part); + Assert.Equal("my-alias", part.Alias); + } + + [Fact] + public async Task ExportAsync_WritesAliasToRow() + { + var contentItem = new ContentItem { ContentType = "Article" }; + contentItem.Alter(p => p.Alias = "exported-alias"); + + _handler.GetColumns(CreatePartContext("AliasPart")); + + var dataTable = new DataTable(); + dataTable.Columns.Add("AliasPart_Alias"); + var row = dataTable.NewRow(); + + var exportContext = new ContentPartExportMapContext + { + ContentTypePartDefinition = CreateTypePartDefinition("AliasPart"), + ContentPart = contentItem.As(), + ContentItem = contentItem, + Row = row, + }; + + await _handler.ExportAsync(exportContext); + + Assert.Equal("exported-alias", row["AliasPart_Alias"]?.ToString()); + } + + private static ImportContentPartContext CreatePartContext(string partName) + => new() + { + ContentTypePartDefinition = CreateTypePartDefinition(partName), + }; + + private static ContentTypePartDefinition CreateTypePartDefinition(string partName) + { + var contentTypeDefinition = new ContentTypeDefinition("Article", "Article"); + var contentPartDefinition = new ContentPartDefinition(partName); + + return new ContentTypePartDefinition(partName, contentPartDefinition, []) + { + ContentTypeDefinition = contentTypeDefinition, + }; + } +} + +public class PublishLaterPartContentImportHandlerTests +{ + private readonly PublishLaterPartContentImportHandler _handler; + + public PublishLaterPartContentImportHandlerTests() + { + _handler = new PublishLaterPartContentImportHandler(Mock.Of>()); + } + + [Fact] + public async Task ImportAsync_SetsScheduledPublishUtcOnPart() + { + var contentItem = new ContentItem { ContentType = "Article" }; + var dataTable = new DataTable(); + dataTable.Columns.Add("PublishLaterPart_ScheduledPublishUtc"); + var row = dataTable.NewRow(); + row["PublishLaterPart_ScheduledPublishUtc"] = "2026-03-23T10:30:00Z"; + dataTable.Rows.Add(row); + + var context = new ContentPartImportMapContext + { + ContentTypePartDefinition = CreateTypePartDefinition("PublishLaterPart"), + ContentItem = contentItem, + Columns = dataTable.Columns, + Row = row, + }; + + await _handler.ImportAsync(context); + + var part = contentItem.As(); + Assert.NotNull(part); + Assert.NotNull(part.ScheduledPublishUtc); + } + + [Fact] + public async Task ExportAsync_WritesScheduledPublishUtcToRow() + { + var contentItem = new ContentItem { ContentType = "Article" }; + var value = new DateTime(2026, 3, 23, 10, 30, 0, DateTimeKind.Utc); + contentItem.Alter(p => p.ScheduledPublishUtc = value); + + _handler.GetColumns(CreatePartContext("PublishLaterPart")); + + var dataTable = new DataTable(); + dataTable.Columns.Add("PublishLaterPart_ScheduledPublishUtc"); + var row = dataTable.NewRow(); + + var exportContext = new ContentPartExportMapContext + { + ContentTypePartDefinition = CreateTypePartDefinition("PublishLaterPart"), + ContentPart = contentItem.As(), + ContentItem = contentItem, + Row = row, + }; + + await _handler.ExportAsync(exportContext); + + Assert.Equal(value.ToString(), row["PublishLaterPart_ScheduledPublishUtc"]?.ToString()); + } + + private static ImportContentPartContext CreatePartContext(string partName) + => new() + { + ContentTypePartDefinition = CreateTypePartDefinition(partName), + }; + + private static ContentTypePartDefinition CreateTypePartDefinition(string partName) + { + var contentTypeDefinition = new ContentTypeDefinition("Article", "Article"); + var contentPartDefinition = new ContentPartDefinition(partName); + + return new ContentTypePartDefinition(partName, contentPartDefinition, []) + { + ContentTypeDefinition = contentTypeDefinition, + }; + } +} + +public class MarkdownFieldImportHandlerTests +{ + private readonly MarkdownFieldImportHandler _handler; + + public MarkdownFieldImportHandlerTests() + { + _handler = new MarkdownFieldImportHandler(Mock.Of>()); + } + + [Fact] + public async Task ImportAsync_SetsMarkdownFieldValue() + { + var part = new ContentPart(); + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_MyField_Markdown"); + var row = dataTable.NewRow(); + row["MyPart_MyField_Markdown"] = "## Markdown"; + dataTable.Rows.Add(row); + + await _handler.ImportAsync(CreateFieldImportContext("MyPart", "MyField", part, dataTable, row)); + + var field = part.Get("MyField"); + Assert.NotNull(field); + Assert.Equal("## Markdown", field.Markdown); + } + + [Fact] + public async Task ExportAsync_ReturnsMarkdownFieldValue() + { + var part = new ContentPart(); + part.Alter("MyField", f => f.Markdown = "## Exported Markdown"); + + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_MyField_Markdown"); + var row = dataTable.NewRow(); + + await _handler.ExportAsync(CreateFieldExportContext("MyPart", "MyField", part, row)); + + Assert.Equal("## Exported Markdown", row["MyPart_MyField_Markdown"]?.ToString()); + } + + private static ContentFieldImportMapContext CreateFieldImportContext(string partName, string fieldName, ContentPart part, DataTable dataTable, DataRow row) + => new() + { + PartName = partName, + ContentPartFieldDefinition = new ContentPartFieldDefinition(new ContentFieldDefinition(nameof(MarkdownField)), fieldName, []), + ContentPart = part, + ContentItem = new ContentItem(), + Columns = dataTable.Columns, + Row = row, + }; + + private static ContentFieldExportMapContext CreateFieldExportContext(string partName, string fieldName, ContentPart part, DataRow row) + => new() + { + PartName = partName, + ContentPartFieldDefinition = new ContentPartFieldDefinition(new ContentFieldDefinition(nameof(MarkdownField)), fieldName, []), + ContentPart = part, + ContentField = part.Get(fieldName) ?? new MarkdownField(), + ContentItem = new ContentItem(), + Row = row, + }; +} + +public class HtmlFieldImportHandlerTests +{ + private readonly HtmlFieldImportHandler _handler; + + public HtmlFieldImportHandlerTests() + { + _handler = new HtmlFieldImportHandler(Mock.Of>()); + } + + [Fact] + public async Task ImportAsync_SetsHtmlFieldValue() + { + var part = new ContentPart(); + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_MyField_Html"); + var row = dataTable.NewRow(); + row["MyPart_MyField_Html"] = "

Hello

"; + dataTable.Rows.Add(row); + + await _handler.ImportAsync(CreateFieldImportContext("MyPart", "MyField", part, dataTable, row)); + + var field = part.Get("MyField"); + Assert.NotNull(field); + Assert.Equal("

Hello

", field.Html); + } + + [Fact] + public async Task ExportAsync_ReturnsHtmlFieldValue() + { + var part = new ContentPart(); + part.Alter("MyField", f => f.Html = "

Exported

"); + + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_MyField_Html"); + var row = dataTable.NewRow(); + + await _handler.ExportAsync(CreateFieldExportContext("MyPart", "MyField", part, row)); + + Assert.Equal("

Exported

", row["MyPart_MyField_Html"]?.ToString()); + } + + private static ContentFieldImportMapContext CreateFieldImportContext(string partName, string fieldName, ContentPart part, DataTable dataTable, DataRow row) + => new() + { + PartName = partName, + ContentPartFieldDefinition = new ContentPartFieldDefinition(new ContentFieldDefinition(nameof(HtmlField)), fieldName, []), + ContentPart = part, + ContentItem = new ContentItem(), + Columns = dataTable.Columns, + Row = row, + }; + + private static ContentFieldExportMapContext CreateFieldExportContext(string partName, string fieldName, ContentPart part, DataRow row) + => new() + { + PartName = partName, + ContentPartFieldDefinition = new ContentPartFieldDefinition(new ContentFieldDefinition(nameof(HtmlField)), fieldName, []), + ContentPart = part, + ContentField = part.Get(fieldName) ?? new HtmlField(), + ContentItem = new ContentItem(), + Row = row, + }; +} + +public class MultiTextFieldImportHandlerTests +{ + private readonly MultiTextFieldImportHandler _handler; + + public MultiTextFieldImportHandlerTests() + { + _handler = new MultiTextFieldImportHandler(Mock.Of>()); + } + + [Fact] + public async Task ImportAsync_SplitsPipeSeparatedValues() + { + var part = new ContentPart(); + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_MyField_Values"); + var row = dataTable.NewRow(); + row["MyPart_MyField_Values"] = "One|Two|Three"; + dataTable.Rows.Add(row); + + await _handler.ImportAsync(CreateFieldImportContext("MyPart", "MyField", part, dataTable, row)); + + var field = part.Get("MyField"); + Assert.NotNull(field); + Assert.Equal(["One", "Two", "Three"], field.Values); + } + + [Fact] + public async Task ExportAsync_JoinsValuesWithPipe() + { + var part = new ContentPart(); + part.Alter("MyField", f => f.Values = ["One", "Two"]); + + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_MyField_Values"); + var row = dataTable.NewRow(); + + await _handler.ExportAsync(CreateFieldExportContext("MyPart", "MyField", part, row)); + + Assert.Equal("One|Two", row["MyPart_MyField_Values"]?.ToString()); + } + + private static ContentFieldImportMapContext CreateFieldImportContext(string partName, string fieldName, ContentPart part, DataTable dataTable, DataRow row) + => new() + { + PartName = partName, + ContentPartFieldDefinition = new ContentPartFieldDefinition(new ContentFieldDefinition(nameof(MultiTextField)), fieldName, []), + ContentPart = part, + ContentItem = new ContentItem(), + Columns = dataTable.Columns, + Row = row, + }; + + private static ContentFieldExportMapContext CreateFieldExportContext(string partName, string fieldName, ContentPart part, DataRow row) + => new() + { + PartName = partName, + ContentPartFieldDefinition = new ContentPartFieldDefinition(new ContentFieldDefinition(nameof(MultiTextField)), fieldName, []), + ContentPart = part, + ContentField = part.Get(fieldName) ?? new MultiTextField(), + ContentItem = new ContentItem(), + Row = row, + }; +} + +public class MediaFieldImportHandlerTests +{ + private readonly MediaFieldImportHandler _handler; + + public MediaFieldImportHandlerTests() + { + _handler = new MediaFieldImportHandler(Mock.Of>()); + } + + [Fact] + public void GetColumns_ReturnsExportOnlyPathsColumn() + { + var context = new ImportContentFieldContext + { + PartName = "GalleryPart", + ContentPartFieldDefinition = new ContentPartFieldDefinition(new ContentFieldDefinition(nameof(MediaField)), "Images", []), + ContentPart = new ContentPart(), + ContentItem = new ContentItem(), + }; + + var column = Assert.Single(_handler.GetColumns(context)); + Assert.Equal("GalleryPart_Images_Paths", column.Name); + Assert.Equal(ImportColumnType.ExportOnly, column.Type); + } + + [Fact] + public async Task ImportAsync_DoesNothing() + { + var part = new ContentPart(); + var dataTable = new DataTable(); + dataTable.Columns.Add("GalleryPart_Images_Paths"); + var row = dataTable.NewRow(); + row["GalleryPart_Images_Paths"] = "media/a.jpg,media/b.jpg"; + dataTable.Rows.Add(row); + + await _handler.ImportAsync(new ContentFieldImportMapContext + { + PartName = "GalleryPart", + ContentPartFieldDefinition = new ContentPartFieldDefinition(new ContentFieldDefinition(nameof(MediaField)), "Images", []), + ContentPart = part, + ContentItem = new ContentItem(), + Columns = dataTable.Columns, + Row = row, + }); + + Assert.Null(part.Get("Images")); + } + + [Fact] + public async Task ExportAsync_JoinsMediaPathsWithComma() + { + var part = new ContentPart(); + part.Alter("Images", field => field.Paths = ["media/a.jpg", "media/b.jpg"]); + + var dataTable = new DataTable(); + dataTable.Columns.Add("GalleryPart_Images_Paths"); + var row = dataTable.NewRow(); + + await _handler.ExportAsync(new ContentFieldExportMapContext + { + PartName = "GalleryPart", + ContentPartFieldDefinition = new ContentPartFieldDefinition(new ContentFieldDefinition(nameof(MediaField)), "Images", []), + ContentPart = part, + ContentField = part.Get("Images"), + ContentItem = new ContentItem(), + Row = row, + }); + + Assert.Equal("media/a.jpg,media/b.jpg", row["GalleryPart_Images_Paths"]?.ToString()); + } +} + +public class ArchiveLaterPartContentImportHandlerTests +{ + private readonly ArchiveLaterPartContentImportHandler _handler; + + public ArchiveLaterPartContentImportHandlerTests() + { + _handler = new ArchiveLaterPartContentImportHandler(Mock.Of>()); + } + + [Fact] + public async Task ImportAsync_SetsScheduledArchiveUtcOnPart() + { + var contentItem = new ContentItem { ContentType = "Article" }; + var dataTable = new DataTable(); + dataTable.Columns.Add("ArchiveLaterPart_ScheduledArchiveUtc"); + var row = dataTable.NewRow(); + row["ArchiveLaterPart_ScheduledArchiveUtc"] = "2026-03-24T10:30:00Z"; + dataTable.Rows.Add(row); + + await _handler.ImportAsync(new ContentPartImportMapContext + { + ContentTypePartDefinition = CreateTypePartDefinition("ArchiveLaterPart"), + ContentItem = contentItem, + Columns = dataTable.Columns, + Row = row, + }); + + Assert.NotNull(contentItem.As()?.ScheduledArchiveUtc); + } + + [Fact] + public async Task ExportAsync_WritesScheduledArchiveUtcToRow() + { + var value = new DateTime(2026, 3, 24, 10, 30, 0, DateTimeKind.Utc); + var contentItem = new ContentItem { ContentType = "Article" }; + contentItem.Alter(p => p.ScheduledArchiveUtc = value); + + _handler.GetColumns(new ImportContentPartContext { ContentTypePartDefinition = CreateTypePartDefinition("ArchiveLaterPart") }); + + var dataTable = new DataTable(); + dataTable.Columns.Add("ArchiveLaterPart_ScheduledArchiveUtc"); + var row = dataTable.NewRow(); + + await _handler.ExportAsync(new ContentPartExportMapContext + { + ContentTypePartDefinition = CreateTypePartDefinition("ArchiveLaterPart"), + ContentPart = contentItem.As(), + ContentItem = contentItem, + Row = row, + }); + + Assert.Equal(value.ToString(), row["ArchiveLaterPart_ScheduledArchiveUtc"]?.ToString()); + } + + private static ContentTypePartDefinition CreateTypePartDefinition(string partName) + { + var contentTypeDefinition = new ContentTypeDefinition("Article", "Article"); + var contentPartDefinition = new ContentPartDefinition(partName); + + return new ContentTypePartDefinition(partName, contentPartDefinition, []) + { + ContentTypeDefinition = contentTypeDefinition, + }; + } +} + +public class LiquidPartContentImportHandlerTests +{ + private readonly LiquidPartContentImportHandler _handler; + + public LiquidPartContentImportHandlerTests() + { + _handler = new LiquidPartContentImportHandler(Mock.Of>()); + } + + [Fact] + public async Task ImportAsync_SetsLiquidOnPart() + { + var contentItem = new ContentItem { ContentType = "Widget" }; + var dataTable = new DataTable(); + dataTable.Columns.Add("LiquidPart_Liquid"); + var row = dataTable.NewRow(); + row["LiquidPart_Liquid"] = "{{ Model.ContentItem.DisplayText }}"; + dataTable.Rows.Add(row); + + await _handler.ImportAsync(new ContentPartImportMapContext + { + ContentTypePartDefinition = CreateTypePartDefinition("LiquidPart"), + ContentItem = contentItem, + Columns = dataTable.Columns, + Row = row, + }); + + Assert.Equal("{{ Model.ContentItem.DisplayText }}", contentItem.As()?.Liquid); + } + + [Fact] + public async Task ExportAsync_WritesLiquidToRow() + { + var contentItem = new ContentItem { ContentType = "Widget" }; + contentItem.Alter(p => p.Liquid = "{{ Model.ContentItem.DisplayText }}"); + + _handler.GetColumns(new ImportContentPartContext { ContentTypePartDefinition = CreateTypePartDefinition("LiquidPart") }); + + var dataTable = new DataTable(); + dataTable.Columns.Add("LiquidPart_Liquid"); + var row = dataTable.NewRow(); + + await _handler.ExportAsync(new ContentPartExportMapContext + { + ContentTypePartDefinition = CreateTypePartDefinition("LiquidPart"), + ContentPart = contentItem.As(), + ContentItem = contentItem, + Row = row, + }); + + Assert.Equal("{{ Model.ContentItem.DisplayText }}", row["LiquidPart_Liquid"]?.ToString()); + } + + private static ContentTypePartDefinition CreateTypePartDefinition(string partName) + { + var contentTypeDefinition = new ContentTypeDefinition("Widget", "Widget"); + var contentPartDefinition = new ContentPartDefinition(partName); + + return new ContentTypePartDefinition(partName, contentPartDefinition, []) + { + ContentTypeDefinition = contentTypeDefinition, + }; + } +} + +public class LinkFieldImportHandlerTests +{ + private readonly LinkFieldImportHandler _handler; + + public LinkFieldImportHandlerTests() + { + _handler = new LinkFieldImportHandler(Mock.Of>()); + } + + [Fact] + public async Task ImportAsync_SetsAllLinkProperties() + { + var part = new ContentPart(); + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_MyField_Url"); + dataTable.Columns.Add("MyPart_MyField_Text"); + dataTable.Columns.Add("MyPart_MyField_Target"); + var row = dataTable.NewRow(); + row["MyPart_MyField_Url"] = "https://example.com"; + row["MyPart_MyField_Text"] = "Example"; + row["MyPart_MyField_Target"] = "_blank"; + dataTable.Rows.Add(row); + + await _handler.ImportAsync(new ContentFieldImportMapContext + { + PartName = "MyPart", + ContentPartFieldDefinition = new ContentPartFieldDefinition(new ContentFieldDefinition(nameof(LinkField)), "MyField", []), + ContentPart = part, + ContentItem = new ContentItem(), + Columns = dataTable.Columns, + Row = row, + }); + + var field = part.Get("MyField"); + Assert.NotNull(field); + Assert.Equal("https://example.com", field.Url); + Assert.Equal("Example", field.Text); + Assert.Equal("_blank", field.Target); + } + + [Fact] + public async Task ExportAsync_WritesAllLinkProperties() + { + var part = new ContentPart(); + part.Alter("MyField", field => + { + field.Url = "https://example.com"; + field.Text = "Example"; + field.Target = "_blank"; + }); + + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_MyField_Url"); + dataTable.Columns.Add("MyPart_MyField_Text"); + dataTable.Columns.Add("MyPart_MyField_Target"); + var row = dataTable.NewRow(); + + await _handler.ExportAsync(new ContentFieldExportMapContext + { + PartName = "MyPart", + ContentPartFieldDefinition = new ContentPartFieldDefinition(new ContentFieldDefinition(nameof(LinkField)), "MyField", []), + ContentPart = part, + ContentField = part.Get("MyField"), + ContentItem = new ContentItem(), + Row = row, + }); + + Assert.Equal("https://example.com", row["MyPart_MyField_Url"]?.ToString()); + Assert.Equal("Example", row["MyPart_MyField_Text"]?.ToString()); + Assert.Equal("_blank", row["MyPart_MyField_Target"]?.ToString()); + } +} + +public class YoutubeFieldImportHandlerTests +{ + private readonly YoutubeFieldImportHandler _handler; + + public YoutubeFieldImportHandlerTests() + { + _handler = new YoutubeFieldImportHandler(Mock.Of>()); + } + + [Fact] + public async Task ImportAsync_SetsYoutubeAddresses() + { + var part = new ContentPart(); + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_Video_RawAddress"); + dataTable.Columns.Add("MyPart_Video_EmbeddedAddress"); + var row = dataTable.NewRow(); + row["MyPart_Video_RawAddress"] = "https://youtu.be/test"; + row["MyPart_Video_EmbeddedAddress"] = "https://www.youtube.com/embed/test"; + dataTable.Rows.Add(row); + + await _handler.ImportAsync(new ContentFieldImportMapContext + { + PartName = "MyPart", + ContentPartFieldDefinition = new ContentPartFieldDefinition(new ContentFieldDefinition(nameof(YoutubeField)), "Video", []), + ContentPart = part, + ContentItem = new ContentItem(), + Columns = dataTable.Columns, + Row = row, + }); + + var field = part.Get("Video"); + Assert.NotNull(field); + Assert.Equal("https://youtu.be/test", field.RawAddress); + Assert.Equal("https://www.youtube.com/embed/test", field.EmbeddedAddress); + } + + [Fact] + public async Task ExportAsync_WritesYoutubeAddresses() + { + var part = new ContentPart(); + part.Alter("Video", field => + { + field.RawAddress = "https://youtu.be/test"; + field.EmbeddedAddress = "https://www.youtube.com/embed/test"; + }); + + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_Video_RawAddress"); + dataTable.Columns.Add("MyPart_Video_EmbeddedAddress"); + var row = dataTable.NewRow(); + + await _handler.ExportAsync(new ContentFieldExportMapContext + { + PartName = "MyPart", + ContentPartFieldDefinition = new ContentPartFieldDefinition(new ContentFieldDefinition(nameof(YoutubeField)), "Video", []), + ContentPart = part, + ContentField = part.Get("Video"), + ContentItem = new ContentItem(), + Row = row, + }); + + Assert.Equal("https://youtu.be/test", row["MyPart_Video_RawAddress"]?.ToString()); + Assert.Equal("https://www.youtube.com/embed/test", row["MyPart_Video_EmbeddedAddress"]?.ToString()); + } +} + +public class UserPickerFieldImportHandlerTests +{ + private readonly UserPickerFieldImportHandler _handler; + + public UserPickerFieldImportHandlerTests() + { + _handler = new UserPickerFieldImportHandler(Mock.Of>()); + } + + [Fact] + public async Task ExportAsync_JoinsUserIdsWithComma() + { + var part = new ContentPart(); + part.Alter("Authors", field => field.UserIds = ["user1", "user2"]); + + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_Authors_UserIds"); + var row = dataTable.NewRow(); + + await _handler.ExportAsync(new ContentFieldExportMapContext + { + PartName = "MyPart", + ContentPartFieldDefinition = new ContentPartFieldDefinition(new ContentFieldDefinition(nameof(UserPickerField)), "Authors", []), + ContentPart = part, + ContentField = part.Get("Authors"), + ContentItem = new ContentItem(), + Row = row, + }); + + Assert.Equal("user1,user2", row["MyPart_Authors_UserIds"]?.ToString()); + } +} + +public class TaxonomyFieldImportHandlerTests +{ + private readonly TaxonomyFieldImportHandler _handler; + + public TaxonomyFieldImportHandlerTests() + { + _handler = new TaxonomyFieldImportHandler(Mock.Of>()); + } + + [Fact] + public async Task ExportAsync_WritesTaxonomyAndTermIds() + { + var part = new ContentPart(); + part.Alter("Categories", field => + { + field.TaxonomyContentItemId = "taxonomy-id"; + field.TermContentItemIds = ["term1", "term2"]; + }); + + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_Categories_TaxonomyContentItemId"); + dataTable.Columns.Add("MyPart_Categories_TermContentItemIds"); + var row = dataTable.NewRow(); + + await _handler.ExportAsync(new ContentFieldExportMapContext + { + PartName = "MyPart", + ContentPartFieldDefinition = new ContentPartFieldDefinition(new ContentFieldDefinition(nameof(TaxonomyField)), "Categories", []), + ContentPart = part, + ContentField = part.Get("Categories"), + ContentItem = new ContentItem(), + Row = row, + }); + + Assert.Equal("taxonomy-id", row["MyPart_Categories_TaxonomyContentItemId"]?.ToString()); + Assert.Equal("term1,term2", row["MyPart_Categories_TermContentItemIds"]?.ToString()); + } +} + +public class LocalizationSetContentPickerFieldImportHandlerTests +{ + private readonly LocalizationSetContentPickerFieldImportHandler _handler; + + public LocalizationSetContentPickerFieldImportHandlerTests() + { + _handler = new LocalizationSetContentPickerFieldImportHandler(Mock.Of>()); + } + + [Fact] + public async Task ExportAsync_JoinsLocalizationSetsWithComma() + { + var part = new ContentPart(); + part.Alter("LocalizedItems", field => field.LocalizationSets = ["set1", "set2"]); + + var dataTable = new DataTable(); + dataTable.Columns.Add("MyPart_LocalizedItems_LocalizationSets"); + var row = dataTable.NewRow(); + + await _handler.ExportAsync(new ContentFieldExportMapContext + { + PartName = "MyPart", + ContentPartFieldDefinition = new ContentPartFieldDefinition(new ContentFieldDefinition(nameof(LocalizationSetContentPickerField)), "LocalizedItems", []), + ContentPart = part, + ContentField = part.Get("LocalizedItems"), + ContentItem = new ContentItem(), + Row = row, + }); + + Assert.Equal("set1,set2", row["MyPart_LocalizedItems_LocalizationSets"]?.ToString()); + } +} diff --git a/test/OrchardCore.Tests/Modules/OrchardCore.ContentsTransfer/ImportContentDisplayDriverTests.cs b/test/OrchardCore.Tests/Modules/OrchardCore.ContentsTransfer/ImportContentDisplayDriverTests.cs new file mode 100644 index 00000000000..9f462e89658 --- /dev/null +++ b/test/OrchardCore.Tests/Modules/OrchardCore.ContentsTransfer/ImportContentDisplayDriverTests.cs @@ -0,0 +1,277 @@ +using DocumentFormat.OpenXml; +using DocumentFormat.OpenXml.Packaging; +using DocumentFormat.OpenXml.Spreadsheet; + +namespace OrchardCore.Tests.Modules.OrchardCore.ContentTransfer; + +public class ExcelFileValidationTests +{ + private static readonly HashSet _allowedExtensions = new(StringComparer.OrdinalIgnoreCase) + { + ".xlsx", + }; + + [Theory] + [InlineData("test.xlsx", true)] + [InlineData("TEST.XLSX", true)] + [InlineData("file.Xlsx", true)] + public void ValidateExtension_WithXlsxFile_ShouldBeAllowed(string fileName, bool expected) + { + // Arrange + var extension = Path.GetExtension(fileName); + + // Act + var isAllowed = _allowedExtensions.Contains(extension); + + // Assert + Assert.Equal(expected, isAllowed); + } + + [Theory] + [InlineData(".csv")] + [InlineData(".xls")] + [InlineData(".txt")] + [InlineData(".pdf")] + [InlineData(".xml")] + public void ValidateExtension_WithNonXlsxFile_ShouldNotBeAllowed(string extension) + { + // Act + var isAllowed = _allowedExtensions.Contains(extension); + + // Assert + Assert.False(isAllowed); + } +} + +public class ExcelFileReaderTests +{ + [Fact] + public void CreateValidXlsxFile_ShouldBeReadable() + { + // Arrange + var memoryStream = CreateValidXlsxFile(["Column1", "Column2"], + [ + ["Value1A", "Value1B"], + ["Value2A", "Value2B"], + ]); + + // Act + using var document = SpreadsheetDocument.Open(memoryStream, false); + var workbookPart = document.WorkbookPart; + var sheets = workbookPart.Workbook.Descendants().ToList(); + + // Assert + Assert.Single(sheets); + Assert.NotNull(document); + } + + [Fact] + public void CreateValidXlsxFile_ShouldContainExpectedData() + { + // Arrange + var headers = new[] { "Name", "Description" }; + var data = new[] + { + new[] { "Item1", "Description1" }, + new[] { "Item2", "Description2" }, + }; + + var memoryStream = CreateValidXlsxFile(headers, data); + + // Act + var result = ReadXlsxFile(memoryStream); + + // Assert + Assert.Equal(2, result.Columns.Count); + Assert.Equal("Name", result.Columns[0]); + Assert.Equal("Description", result.Columns[1]); + Assert.Equal(2, result.Rows.Count); + Assert.Equal("Item1", result.Rows[0][0]); + Assert.Equal("Description1", result.Rows[0][1]); + } + + [Fact] + public void ReadXlsxFile_WithEmptyFile_ShouldReturnEmptyResult() + { + // Arrange + var memoryStream = CreateValidXlsxFile([], []); + + // Act + var result = ReadXlsxFile(memoryStream); + + // Assert + Assert.Empty(result.Columns); + Assert.Empty(result.Rows); + } + + [Fact] + public void ReadXlsxFile_WithHeadersOnly_ShouldReturnHeadersAndNoRows() + { + // Arrange + var memoryStream = CreateValidXlsxFile(["Header1", "Header2", "Header3"], []); + + // Act + var result = ReadXlsxFile(memoryStream); + + // Assert + Assert.Equal(3, result.Columns.Count); + Assert.Empty(result.Rows); + } + + [Fact] + public void GetCellReference_ShouldReturnCorrectReference() + { + // Assert + Assert.Equal("A1", GetCellReference(1, 1)); + Assert.Equal("B2", GetCellReference(2, 2)); + Assert.Equal("Z1", GetCellReference(26, 1)); + Assert.Equal("AA1", GetCellReference(27, 1)); + } + + private static MemoryStream CreateValidXlsxFile(string[] headers, string[][] rows) + { + var memoryStream = new MemoryStream(); + using (var document = SpreadsheetDocument.Create(memoryStream, SpreadsheetDocumentType.Workbook)) + { + var workbookPart = document.AddWorkbookPart(); + workbookPart.Workbook = new Workbook(); + + var worksheetPart = workbookPart.AddNewPart(); + worksheetPart.Worksheet = new Worksheet(new SheetData()); + + var sheets = workbookPart.Workbook.AppendChild(new Sheets()); + var sheet = new Sheet + { + Id = workbookPart.GetIdOfPart(worksheetPart), + SheetId = 1, + Name = "Sheet1", + }; + sheets.Append(sheet); + + var sheetData = worksheetPart.Worksheet.GetFirstChild(); + + // Add header row + if (headers.Length > 0) + { + var headerRow = new Row { RowIndex = 1 }; + sheetData.Append(headerRow); + + for (var i = 0; i < headers.Length; i++) + { + var cell = new Cell + { + CellReference = GetCellReference((uint)i + 1, 1), + DataType = CellValues.String, + CellValue = new CellValue(headers[i]), + }; + headerRow.Append(cell); + } + } + + // Add data rows + for (var rowIndex = 0; rowIndex < rows.Length; rowIndex++) + { + var dataRow = new Row { RowIndex = (uint)(rowIndex + 2) }; + sheetData.Append(dataRow); + + for (var colIndex = 0; colIndex < rows[rowIndex].Length; colIndex++) + { + var cell = new Cell + { + CellReference = GetCellReference((uint)colIndex + 1, (uint)(rowIndex + 2)), + DataType = CellValues.String, + CellValue = new CellValue(rows[rowIndex][colIndex]), + }; + dataRow.Append(cell); + } + } + + workbookPart.Workbook.Save(); + } + + memoryStream.Seek(0, SeekOrigin.Begin); + return memoryStream; + } + + private static (List Columns, List Rows) ReadXlsxFile(Stream stream) + { + var columns = new List(); + var rows = new List(); + + using var document = SpreadsheetDocument.Open(stream, false); + var workbookPart = document.WorkbookPart; + var sheets = workbookPart.Workbook.Descendants().ToList(); + + if (sheets.Count == 0) + { + return (columns, rows); + } + + var worksheetPart = (WorksheetPart)workbookPart.GetPartById(sheets[0].Id); + var sheetData = worksheetPart.Worksheet.GetFirstChild(); + var allRows = sheetData.Descendants().ToList(); + + if (allRows.Count == 0) + { + return (columns, rows); + } + + // Read header row + var headerRow = allRows[0]; + foreach (var cell in headerRow.Descendants()) + { + columns.Add(GetCellValue(cell, workbookPart)); + } + + // Read data rows + for (var i = 1; i < allRows.Count; i++) + { + var rowData = new List(); + foreach (var cell in allRows[i].Descendants()) + { + rowData.Add(GetCellValue(cell, workbookPart)); + } + rows.Add([.. rowData]); + } + + return (columns, rows); + } + + private static string GetCellValue(Cell cell, WorkbookPart workbookPart) + { + if (cell?.CellValue == null) + { + return string.Empty; + } + + var value = cell.CellValue.Text; + + if (cell.DataType?.Value == CellValues.SharedString) + { + var sharedStringTable = workbookPart.GetPartsOfType() + .FirstOrDefault()?.SharedStringTable; + + if (sharedStringTable != null && int.TryParse(value, out var index)) + { + return sharedStringTable.ElementAt(index).InnerText; + } + } + + return value ?? string.Empty; + } + + private static string GetCellReference(uint columnIndex, uint rowIndex) + { + var columnName = string.Empty; + var dividend = columnIndex; + + while (dividend > 0) + { + var modulo = (dividend - 1) % 26; + columnName = Convert.ToChar(65 + modulo) + columnName; + dividend = (dividend - modulo) / 26; + } + + return columnName + rowIndex; + } +}