Non zip artifacts

.github/workflows/nuget.yml

@@ -170,6 +170,7 @@ jobs:
         Runner.Client -C testworkflows/inherit_vars --var ACTIONS_STEP_DEBUG=true
         Runner.Client -C testworkflows/actions_artifacts_v4 -s ACTIONS_STEP_DEBUG=true -s ACTIONS_RUNNER_DEBUG=true --runner-version v2.311.0
         Runner.Client -C testworkflows/actions_artifacts_v4 -s ACTIONS_STEP_DEBUG=true -s ACTIONS_RUNNER_DEBUG=true
+        Runner.Client -C testworkflows/actions_artifacts_v7_v8 -s ACTIONS_STEP_DEBUG=true -s ACTIONS_RUNNER_DEBUG=true
         Runner.Client -C testworkflows/cache-save-restore-order-tests -s ACTIONS_STEP_DEBUG=true -s ACTIONS_RUNNER_DEBUG=true
         Runner.Client --event azpipelines -C testworkflows/azpipelines/cross-repo-checkout -W testworkflows/azpipelines/cross-repo-checkout/pipeline.yml --local-repository az/containermatrix@main=testworkflows/azpipelines/containermatrix
         Runner.Client --event azpipelines -C testworkflows/azpipelines/typedtemplates -W testworkflows/azpipelines/typedtemplates/pipeline.yml

src/Runner.Server/Controllers/ArtifactCacheManagementController.cs

@@ -26,23 +26,21 @@ public ArtifactCacheManagementController(IConfiguration conf, SqLiteDb db) : bas
 
         [HttpDelete("artifacts")]
         public void DeleteArtifacts() {
-            var artifactspath = Path.Combine(GitHub.Runner.Sdk.GharunUtil.GetLocalStorage(), "artifacts");
             var records = db.ArtifactRecords.ToArray();
             db.ArtifactRecords.RemoveRange(records);
             db.SaveChanges();
             foreach(var rec in records) {
-                System.IO.File.Delete(Path.Combine(artifactspath, rec.StoreName));
+                AzureBlobStorageContoller.DeleteBlobFilePath("artifacts/" + rec.StoreName);
             }
         }
 
         [HttpDelete("cache")]
         public void DeleteCache() {
-            var cachepath = Path.Combine(GitHub.Runner.Sdk.GharunUtil.GetLocalStorage(), "cache");
             var caches = db.Caches.ToArray();
             db.Caches.RemoveRange(caches);
             db.SaveChanges();
             foreach(var cache in caches) {
-                System.IO.File.Delete(Path.Combine(cachepath, cache.Storage));
+                AzureBlobStorageContoller.DeleteBlobFilePath("cache/" + cache.Storage);
             }
         }
     }

src/Runner.Server/Controllers/ArtifactController.cs

@@ -70,12 +70,13 @@
                         var ofile = await (from f in _context.Entry(filecontainer).Collection(f => f.Files).Query() where f.FileName.ToLower() == file.FileName.ToLower() select f).FirstOrDefaultAsync();
                         if(ofile != null) {
                             try {
-                                System.IO.File.Delete(Path.Combine(_targetFilePath, ofile.StoreName));
+                                AzureBlobStorageContoller.DeleteBlobFilePath("artifacts/" + ofile.StoreName);
                             } catch {
 
                             }
                             ofile.StoreName = file.StoreName;
                             ofile.GZip = file.GZip;
+                            ofile.ContentType = file.ContentType;
                             _context.Remove(file);
                         } else {
                             file.FileContainer = filecontainer;
@@ -207,10 +208,10 @@
                 var content = new ContentDisposition();
                 content.DispositionType = DispositionTypeNames.Attachment;
                 content.FileName = container.FileName.Substring(lastPathSep);
                 Response.Headers.Add("Content-Disposition", content.ToString());
             }
             if(container.GZip) {
                 Response.Headers.Add("Content-Encoding", "gzip");
             }
             return new FileStreamResult(System.IO.File.OpenRead(Path.Combine(_targetFilePath, container.StoreName)), "application/octet-stream") { EnableRangeProcessing = true };
         }

src/Runner.Server/Controllers/ArtifactControllerV2.cs

@@ -18,6 +18,7 @@
 using Microsoft.IdentityModel.Tokens;
 using System.Text;
 using System.Xml.Linq;
+using System.Net.Mime;
 
 namespace Runner.Server.Controllers
 {
@@ -35,22 +36,16 @@ public ArtifactControllerV2(SqLiteDb _context, IConfiguration configuration) : b
             formatter = new JsonFormatter(JsonFormatter.Settings.Default.WithIndentation().WithPreserveProtoFieldNames(true).WithFormatDefaultValues(false));
         }
 
-        private string CreateSignature(int id) {
-            using var rsa = RSA.Create(Startup.AccessTokenParameter);
-            return Base64UrlEncoder.Encode(rsa.SignData(Encoding.UTF8.GetBytes(id.ToString()), HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1));
-        }
-
-        private bool VerifySignature(int id, string sig) {
-            using var rsa = RSA.Create(Startup.AccessTokenParameter);
-            return rsa.VerifyData(Encoding.UTF8.GetBytes(id.ToString()), Base64UrlEncoder.DecodeBytes(sig), HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
-        }
-
         [HttpPost("CreateArtifact")]
         public async Task<string> CreateArtifact([FromBody, Protobuf] Github.Actions.Results.Api.V1.CreateArtifactRequest body) {
             var guid = Guid.Parse(body.WorkflowJobRunBackendId);
             var jobInfo = (from j in _context.Jobs where j.JobId == guid select new { j.runid, j.WorkflowRunAttempt.Attempt }).FirstOrDefault();
             var artifacts = new ArtifactController(_context, Configuration);
             var fname = $"{body.Name}.zip";
+            if(!string.IsNullOrWhiteSpace(body.MimeType) && body.MimeType != "application/zip")
+            {
+                fname = body.Name;
+            }
             var container = await artifacts.CreateContainer(jobInfo.runid, jobInfo.Attempt, new CreateActionsStorageArtifactParameters() { Name = body.Name }, jobInfo.Attempt);
             if(_context.Entry(container).Collection(c => c.Files).Query().Any()) {
                 //var files = _context.Entry(container).Collection(c => c.Files).Query().ToList();
@@ -59,51 +54,18 @@ public async Task<string> CreateArtifact([FromBody, Protobuf] Github.Actions.Res
                     Ok = false
                 });
             }
-            var record = new ArtifactRecord() {FileName = fname, StoreName = Path.GetRandomFileName(), GZip = false, FileContainer = container} ;
+            var record = new ArtifactRecord() {FileName = fname, StoreName = Path.GetRandomFileName(), GZip = false, FileContainer = container, ContentType = body.MimeType} ;
             _context.ArtifactRecords.Add(record);
             await _context.SaveChangesAsync();
 
             var resp = new Github.Actions.Results.Api.V1.CreateArtifactResponse
             {
                 Ok = true,
-                SignedUploadUrl = new Uri(new Uri(ServerUrl), $"twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact?id={record.Id}&sig={CreateSignature(record.Id)}").ToString()
+                SignedUploadUrl = AzureBlobStorageContoller.CreateSignedUrl(ServerUrl, "artifacts/" + record.StoreName, write: true)
             };
             return formatter.Format(resp);
         }
 
-        [HttpPut("UploadArtifact")]
-        [AllowAnonymous]
-        public async Task<IActionResult> UploadArtifact(int id, string sig, string comp = null, bool seal = false, string blockid = null) {
-            if(string.IsNullOrEmpty(sig) || !VerifySignature(id, sig)) {
-                return NotFound();
-            }
-            if(comp == "block" || comp == "appendBlock" || comp == null) {
-                var record = await _context.ArtifactRecords.FindAsync(id);
-                var _targetFilePath = Path.Combine(GitHub.Runner.Sdk.GharunUtil.GetLocalStorage(), "artifacts");
-                Directory.CreateDirectory(_targetFilePath);
-                using(var targetStream = new FileStream(Path.Combine(_targetFilePath, string.IsNullOrWhiteSpace(blockid) ? record.StoreName : $"{record.StoreName}-{System.Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(blockid))}"), FileMode.OpenOrCreate | FileMode.Append, FileAccess.Write, FileShare.Write)) {
-                    await Request.Body.CopyToAsync(targetStream);
-                }
-                return Created(HttpContext.Request.GetEncodedUrl(), null);
-            }
-            if(comp == "blocklist") {
-                XElement blockList = await XElement.LoadAsync(Request.Body, LoadOptions.None, Request.HttpContext.RequestAborted);
-                var record = await _context.ArtifactRecords.FindAsync(id);
-                var _targetFilePath = Path.Combine(GitHub.Runner.Sdk.GharunUtil.GetLocalStorage(), "artifacts");
-                Directory.CreateDirectory(_targetFilePath);
-                using(var targetStream = new FileStream(Path.Combine(_targetFilePath, record.StoreName), FileMode.Create, FileAccess.Write, FileShare.Write))
-                foreach(var block in from item in blockList.Descendants("Latest") select item.Value) {
-                    var filename = $"{record.StoreName}-{System.Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(block))}";
-                    using(var sourceStream = new FileStream(Path.Combine(_targetFilePath, filename), FileMode.Open, FileAccess.Read, FileShare.Read)) {
-                        await sourceStream.CopyToAsync(targetStream);
-                    }
-                    System.IO.File.Delete(Path.Combine(_targetFilePath, filename));
-                }
-                return Created(HttpContext.Request.GetEncodedUrl(), null);
-            }
-            return Ok();
-        }
-
         [HttpPost("FinalizeArtifact")]
         public string FinalizeArtifact([FromBody, Protobuf] Github.Actions.Results.Api.V1.FinalizeArtifactRequest body) {
             var attempt = long.Parse(User.FindFirst("attempt")?.Value ?? "-1");
@@ -127,7 +89,7 @@ public string ListArtifacts([FromBody, Protobuf] Github.Actions.Results.Api.V1.L
             var attempt = long.Parse(User.FindFirst("attempt")?.Value ?? "-1");
             var artifactsMinAttempt = long.Parse(User.FindFirst("artifactsMinAttempt")?.Value ?? "-1");
             var runid = long.Parse(body.WorkflowRunBackendId);
-            resp.Artifacts.AddRange(from fileContainer in _context.ArtifactFileContainer where (fileContainer.Container.Attempt.Attempt >= artifactsMinAttempt || artifactsMinAttempt == -1) && (fileContainer.Container.Attempt.Attempt <= attempt || attempt == -1) && fileContainer.Container.Attempt.WorkflowRun.Id == runid && fileContainer.Files.Count == 1 && !fileContainer.Files.FirstOrDefault().FileName.Contains('/') && fileContainer.Files.FirstOrDefault().FileName.EndsWith(".zip") && (body.IdFilter == null || body.IdFilter == fileContainer.Id)  && (body.NameFilter == null || body.NameFilter.ToLower() == fileContainer.Name.ToLower()) orderby fileContainer.Container.Attempt.Attempt descending select new Github.Actions.Results.Api.V1.ListArtifactsResponse_MonolithArtifact
+            resp.Artifacts.AddRange(from fileContainer in _context.ArtifactFileContainer where (fileContainer.Container.Attempt.Attempt >= artifactsMinAttempt || artifactsMinAttempt == -1) && (fileContainer.Container.Attempt.Attempt <= attempt || attempt == -1) && fileContainer.Container.Attempt.WorkflowRun.Id == runid && fileContainer.Files.Count == 1 && !fileContainer.Files.FirstOrDefault().FileName.Contains('/') && (fileContainer.Files.FirstOrDefault().FileName.EndsWith(".zip") || !string.IsNullOrWhiteSpace(fileContainer.Files.FirstOrDefault().ContentType)) && (body.IdFilter == null || body.IdFilter == fileContainer.Id) && (body.NameFilter == null || body.NameFilter.ToLower() == fileContainer.Name.ToLower()) orderby fileContainer.Container.Attempt.Attempt descending select new Github.Actions.Results.Api.V1.ListArtifactsResponse_MonolithArtifact
             {
                 CreatedAt = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTimeOffset(System.DateTimeOffset.UtcNow),
                 DatabaseId = fileContainer.Id,
@@ -144,25 +106,14 @@ public async Task<string> GetSignedArtifactURL([FromBody, Protobuf] Github.Actio
             var attempt = long.Parse(User.FindFirst("attempt")?.Value ?? "-1");
             var artifactsMinAttempt = long.Parse(User.FindFirst("artifactsMinAttempt")?.Value ?? "-1");
             var runid = long.Parse(body.WorkflowRunBackendId);
-            var file = await (from fileContainer in _context.ArtifactFileContainer where (fileContainer.Container.Attempt.Attempt >= artifactsMinAttempt || artifactsMinAttempt == -1) && (fileContainer.Container.Attempt.Attempt <= attempt || attempt == -1) && fileContainer.Container.Attempt.WorkflowRun.Id == runid && fileContainer.Files.Count == 1 && fileContainer.Files.FirstOrDefault().FileName.ToLower() == $"{body.Name}.zip".ToLower() orderby fileContainer.Container.Attempt.Attempt descending select fileContainer.Files.FirstOrDefault()).FirstAsync();
+            var file = await (from fileContainer in _context.ArtifactFileContainer where (fileContainer.Container.Attempt.Attempt >= artifactsMinAttempt || artifactsMinAttempt == -1) && (fileContainer.Container.Attempt.Attempt <= attempt || attempt == -1) && fileContainer.Container.Attempt.WorkflowRun.Id == runid && fileContainer.Files.Count == 1 && (fileContainer.Files.FirstOrDefault().FileName.ToLower() == $"{body.Name}.zip".ToLower() || fileContainer.Files.FirstOrDefault().ContentType != null) orderby fileContainer.Container.Attempt.Attempt descending select fileContainer.Files.FirstOrDefault()).FirstAsync();
             var resp = new Github.Actions.Results.Api.V1.GetSignedArtifactURLResponse
             {
-                SignedUrl = new Uri(new Uri(ServerUrl), $"twirp/github.actions.results.api.v1.ArtifactService/DownloadArtifact?id={file.Id}&sig={CreateSignature(file.Id)}").ToString()
+                SignedUrl = AzureBlobStorageContoller.CreateSignedUrl(ServerUrl, "artifacts/" + file.StoreName, contentType: file.ContentType, contentDisposition: new ContentDisposition(DispositionTypeNames.Inline) { FileName = file.FileName }.ToString())
             };
             return formatter.Format(resp);
         }
 
-        [AllowAnonymous]
-        [HttpGet("DownloadArtifact")]
-        public IActionResult DownloadArtifact(int id, string sig) {
-            if(string.IsNullOrEmpty(sig) || !VerifySignature(id, sig)) {
-                return NotFound();
-            }
-            var container = _context.ArtifactRecords.Find(id);
-            var _targetFilePath = Path.Combine(GitHub.Runner.Sdk.GharunUtil.GetLocalStorage(), "artifacts");
-            return new FileStreamResult(System.IO.File.OpenRead(Path.Combine(_targetFilePath, container.StoreName)), "application/octet-stream") { EnableRangeProcessing = true };
-        }
-
         [AllowAnonymous]
         [HttpPost("DeleteArtifact")]
         public async Task<IActionResult> DeleteArtifact([FromBody, Protobuf] Github.Actions.Results.Api.V1.DeleteArtifactRequest body) {
@@ -175,8 +126,7 @@ public async Task<IActionResult> DeleteArtifact([FromBody, Protobuf] Github.Acti
             }
             _context.ArtifactRecords.Remove(res.file);
             _context.ArtifactFileContainer.Remove(res.fileContainer);
-            var _targetFilePath = Path.Combine(GitHub.Runner.Sdk.GharunUtil.GetLocalStorage(), "artifacts");
-            System.IO.File.Delete(Path.Combine(_targetFilePath, res.file.StoreName));
+            AzureBlobStorageContoller.DeleteBlobFilePath("artifacts/" + res.file.StoreName);
             await _context.SaveChangesAsync();
 
             var resp = new Github.Actions.Results.Api.V1.DeleteArtifactResponse {