Initial commit adding the ability to use TextServices

src/IIIFPresentation/API.Tests/Converters/ManifestConverterTests.cs

@@ -310,7 +310,91 @@ public void SetGeneratedFields_SetsSpace()
         // Assert
         result.Space.Should().Be("https://dlcs.test/customers/123/spaces/321");
     }
-
+
+    [Fact]
+    public void SetGeneratedFields_SetsPipelineFromJobs_WhenPipelineJobsPresent()
+    {
+        // Arrange
+        var iiifManifest = new PresentationManifest();
+        var dbManifest = new DBManifest
+        {
+            CustomerId = 1, Id = "id",
+            Hierarchy = [new Hierarchy { Slug = "slug" }],
+            PipelineJobs =
+            [
+                new PipelineJob
+                {
+                    ManifestId = "id", CustomerId = 1,
+                    JobType = PipelineJobType.TextService,
+                    Status = PipelineJobStatus.Waiting,
+                    Created = DateTime.UtcNow
+                }
+            ]
+        };
+
+        // Act
+        var result = iiifManifest.SetGeneratedFields(dbManifest, pathGenerator, settingsBasedPathGenerator);
+
+        // Assert
+        result.Pipeline.Should().ContainSingle(p => p.Name == PipelineHelper.TextPipelineName && p.Status == "Waiting");
+    }
+
+    [Fact]
+    public void SetGeneratedFields_ReturnsLatestJobPerType_WhenMultipleJobsOfSameType()
+    {
+        // Arrange
+        var iiifManifest = new PresentationManifest();
+        var older = DateTime.UtcNow.AddHours(-1);
+        var newer = DateTime.UtcNow;
+        var dbManifest = new DBManifest
+        {
+            CustomerId = 1, Id = "id",
+            Hierarchy = [new Hierarchy { Slug = "slug" }],
+            PipelineJobs =
+            [
+                new PipelineJob
+                {
+                    ManifestId = "id", CustomerId = 1,
+                    JobType = PipelineJobType.TextService,
+                    Status = PipelineJobStatus.Completed,
+                    Created = older
+                },
+                new PipelineJob
+                {
+                    ManifestId = "id", CustomerId = 1,
+                    JobType = PipelineJobType.TextService,
+                    Status = PipelineJobStatus.Waiting,
+                    Created = newer
+                }
+            ]
+        };
+
+        // Act
+        var result = iiifManifest.SetGeneratedFields(dbManifest, pathGenerator, settingsBasedPathGenerator);
+
+        // Assert
+        result.Pipeline.Should().ContainSingle(p => p.Status == "Waiting", "only the most recently created job per type should appear");
+    }
+
+    [Fact]
+    public void SetGeneratedFields_DoesNotSetPipeline_WhenNoPipelineJobs()
+    {
+        // Arrange
+        var iiifManifest = new PresentationManifest { Pipeline = null };
+        var dbManifest = new DBManifest
+        {
+            CustomerId = 1, Id = "id",
+            Hierarchy = [new Hierarchy { Slug = "slug" }],
+            PipelineJobs = null
+        };
+
+        // Act
+        var result = iiifManifest.SetGeneratedFields(dbManifest, pathGenerator, settingsBasedPathGenerator);
+
+        // Assert
+        result.Pipeline.Should().BeNull();
+    }
+
     [Fact]
     public void GenerateProvisionalCanvases_SetsItems_IfNotSet()
     {

src/IIIFPresentation/API.Tests/Integration/GetManifestTests.cs

@@ -442,6 +442,81 @@ await amazonS3.PutObjectAsync(new()
         });
     }
 
+    [Fact]
+    public async Task Get_IiifManifest_Flat_ReturnsAccepted_WhenPipelineJobQueued()
+    {
+        var id = TestIdentifiers.IdWithSuffix(suffix: "_pipelineQueued");
+
+        // Arrange
+        var dbManifest = await dbContext.Manifests.AddTestManifest(id);
+        await dbContext.PipelineJobs.AddAsync(new PipelineJob
+        {
+            ManifestId = id,
+            JobType = PipelineJobType.TextService,
+            CustomerId = 1,
+
+            Status = PipelineJobStatus.Waiting,
+            Created = DateTime.UtcNow
+        });
+        await dbContext.SaveChangesAsync();
+
+        await amazonS3.PutObjectAsync(new()
+        {
+            BucketName = LocalStackFixture.StorageBucketName,
+            Key = $"staging/1/manifests/{id}",
+            ContentBody = TestContent.ManifestJson
+        });
+
+        var requestMessage =
+            HttpRequestMessageBuilder.GetPrivateRequest(HttpMethod.Get, $"1/manifests/{id}");
+        var response = await httpClient.AsCustomer().SendAsync(requestMessage);
+
+        // Assert
+        response.StatusCode.Should().Be(HttpStatusCode.Accepted);
+        response.Headers.Should().NotContainKey(HeaderNames.ETag, "manifest is not yet finalised");
+        var manifest = await response.ReadAsPresentationJsonAsync<PresentationManifest>();
+        manifest.Should().NotBeNull();
+        manifest!.Id.Should().Be($"http://localhost/1/manifests/{id}");
+        manifest.Pipeline.Should().ContainSingle(p => p.Name == PipelineHelper.TextPipelineName && p.Status == "Waiting");
+    }
+
+    [Fact]
+    public async Task Get_IiifManifest_Flat_ReturnsOK_WhenPipelineJobCompleted()
+    {
+        var id = TestIdentifiers.IdWithSuffix(suffix: "_pipelineCompleted");
+
+        // Arrange
+        var dbManifest = await dbContext.Manifests.AddTestManifest(id);
+        await dbContext.PipelineJobs.AddAsync(new PipelineJob
+        {
+            ManifestId = id,
+            JobType = PipelineJobType.TextService,
+            CustomerId = 1,
+
+            Status = PipelineJobStatus.Completed,
+            Created = DateTime.UtcNow,
+            Finished = DateTime.UtcNow
+        });
+        await dbContext.SaveChangesAsync();
+
+        await amazonS3.PutObjectAsync(new()
+        {
+            BucketName = LocalStackFixture.StorageBucketName,
+            Key = $"1/manifests/{id}",
+            ContentBody = TestContent.ManifestJson
+        });
+
+        var requestMessage =
+            HttpRequestMessageBuilder.GetPrivateRequest(HttpMethod.Get, $"1/manifests/{id}");
+        var response = await httpClient.AsCustomer().SendAsync(requestMessage);
+
+        // Assert
+        response.StatusCode.Should().Be(HttpStatusCode.OK);
+        response.Headers.Should().ContainKey(HeaderNames.ETag, "pipeline is complete so manifest is final");
+        var manifest = await response.ReadAsPresentationJsonAsync<PresentationManifest>();
+        manifest!.Pipeline.Should().ContainSingle(p => p.Name == PipelineHelper.TextPipelineName && p.Status == "Completed");
+    }
+
     [Fact]
     public async Task Get_IiifManifest_Hierarchical_ReturnsNotFoundWhenIngesting()
     {

src/IIIFPresentation/API.Tests/Integration/ModifyManifestCreateTests.cs

@@ -22,6 +22,7 @@
 using Test.Helpers;
 using Test.Helpers.Helpers;
 using Test.Helpers.Integration;
+using Services.TextServices;
 using Batch = DLCS.Models.Batch;
 using Collection = Models.Database.Collections.Collection;
 using Manifest = Models.Database.Collections.Manifest;
@@ -36,6 +37,7 @@ public class ModifyManifestCreateTests : IClassFixture<PresentationAppFactory<Pr
     private readonly PresentationContext dbContext;
     private readonly IAmazonS3 amazonS3;
     private static readonly IDlcsApiClient DLCSApiClient = A.Fake<IDlcsApiClient>();
+    private static readonly ITextServicesClient TextServicesClient = A.Fake<ITextServicesClient>();
     private const int Customer = 1;
     private const int ExampleCustomer = 601;
     private const int InvalidSpaceCustomer = 34512;
@@ -49,10 +51,12 @@ public ModifyManifestCreateTests(StorageFixture storageFixture, PresentationAppF
             .Returns(new Space { Id = NewlyCreatedSpace, Name = "test" });
         A.CallTo(() => DLCSApiClient.CreateSpace(InvalidSpaceCustomer, A<string>._, A<CancellationToken>._))
             .ThrowsAsync(new DlcsException("Error creating DLCS space", HttpStatusCode.BadRequest));
+        A.CallTo(() => TextServicesClient.CreateOrUpdateJob(A<PipelineJob>._, A<string>._, A<string>._, A<CancellationToken>._))
+            .Returns(true);
         httpClient = factory
             .ConfigureBasicIntegrationTestHttpClient(storageFixture.DbFixture,
                 appFactory => appFactory.WithLocalStack(storageFixture.LocalStackFixture),
-                services => services.AddSingleton(DLCSApiClient)
+                services => services.AddSingleton(DLCSApiClient).AddSingleton(TextServicesClient)
             );
 
         storageFixture.DbFixture.CleanUp();
@@ -1885,4 +1889,26 @@ private static void SetupApiClientWithBatchReturn(params string[] assetIds)
                 A<CancellationToken>._))
             .Returns([new Batch { Finished = null, ResourceId = TestIdentifiers.BatchId().ToString() }]);
     }
+
+    [Fact]
+    public async Task CreateManifest_ReturnsAccepted_WithNoETag_WhenManifestHasPipeline()
+    {
+        // Arrange
+        var manifest = new PresentationManifest
+        {
+            Parent = $"http://localhost/{Customer}/collections/{RootCollection.Id}",
+            Slug = TestIdentifiers.Slug(),
+            Pipeline = [new PipelineItem { Name = "text", Config = new PipelineConfig { Action = "Index" } }]
+        };
+        var requestMessage =
+            HttpRequestMessageBuilder.GetPrivateRequest(HttpMethod.Post, $"{Customer}/manifests", manifest.AsJson());
+
+        // Act
+        var response = await httpClient.AsCustomer().SendAsync(requestMessage);
+
+        // Assert
+        response.StatusCode.Should().Be(HttpStatusCode.Accepted);
+        response.Headers.Should().NotContainKey(Microsoft.Net.Http.Headers.HeaderNames.ETag,
+            "202 responses must not include an ETag as the manifest is not yet finalised");
+    }
 }

src/IIIFPresentation/API/API.csproj

@@ -10,7 +10,7 @@
 
     <ItemGroup>
         <PackageReference Include="FluentValidation.DependencyInjectionExtensions" Version="11.10.0" />
-        <PackageReference Include="iiif-net" Version="0.4.2" />
+        <PackageReference Include="iiif-net" Version="0.4.3" />
         <PackageReference Include="JsonDiffPatch.Net" Version="2.5.0" />
         <PackageReference Include="LazyCache.AspNetCore" Version="2.4.0" />
         <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.8" />

src/IIIFPresentation/API/Converters/ManifestConverter.cs

@@ -67,6 +67,16 @@ public static PresentationManifest SetGeneratedFields(this PresentationManifest
             foreach (var adjunct in iiifManifest.Adjuncts!) adjunct.Remove(AssetProperties.Asset);
         }
 
+        if (!dbManifest.PipelineJobs.IsNullOrEmpty())
+        {
+            iiifManifest.Pipeline = dbManifest.PipelineJobs!
+                .GroupBy(j => j.JobType)
+                // get the last created version of each pipeline item - we don't care about history
+                .Select(g => g.OrderByDescending(j => j.Created).First()
+                    .ToPipelineItem())
+                .ToList();
+        }
+
         iiifManifest.EnsurePresentation3Context();
         iiifManifest.EnsureContext(PresentationJsonLdContext.Context);
 

src/IIIFPresentation/API/Features/Manifest/DlcsManifestCoordinator.cs

@@ -307,7 +307,7 @@ private async Task DeleteUnusedAdjuncts(int customerId, List<AdjunctInteraction>
 
         foreach (var adjunctInteraction in adjuncts)
         {
-            if (adjunctInteraction.ExistingAdjunctIds is not { Count: > 0 }) continue;
+            if (adjunctInteraction.ExistingAdjunctIds.IsNullOrEmpty()) continue;
 
             var currentIds = adjunctInteraction.Adjuncts
                 .Select(a => a[AdjunctProperties.Id]?.Value<string>())

src/IIIFPresentation/API/Features/Manifest/ManifestReadService.cs

@@ -37,7 +37,7 @@ public async Task<FetchEntityResult<PresentationManifest>> GetManifest(int custo
         IImmutableSet<Guid> ifNoneMatch, bool pathOnly, CancellationToken cancellationToken)
     {
         var dbManifest = await dbContext.RetrieveManifestAsync(manifestId, withBatches: true,
-            cancellationToken: cancellationToken);
+            withPipelineJobs: true, cancellationToken: cancellationToken);
 
         if (dbManifest == null) return FetchEntityResult<PresentationManifest>.NotFound();
 
@@ -57,11 +57,11 @@ public async Task<FetchEntityResult<PresentationManifest>> GetManifest(int custo
 
         var getAssets = dlcsManifestCoordinator.GetAssets(customerId, dbManifest, cancellationToken);
         PresentationManifest? manifest = null;
-        if (dbManifest.IsIngesting())
+        if (dbManifest.HasFurtherWork())
         {
             manifest = await iiifS3.ReadIIIFFromS3<PresentationManifest>(dbManifest, BucketLocationType.Staging, cancellationToken);
             if (manifest == null)
-                logger.LogError("Manifest {DbManifestId} IsIngesting but can't read from staging", dbManifest.Id);
+                logger.LogError("Manifest {DbManifestId} has further work pending but can't read from staging", dbManifest.Id);
         }
 
         // if is not ingesting read from "real" location
@@ -84,7 +84,7 @@ public async Task<FetchEntityResult<PresentationManifest>> GetManifest(int custo
             m => Enumerable.Single(m.Hierarchy!, h => h.Canonical));
 
         Guid? etag = dbManifest.Etag;
-        if (dbManifest.IsIngesting())
+        if (dbManifest.HasFurtherWork())
         {
             manifest.CurrentlyIngesting = true;
             etag = null;