From 38477ab19377feb18b1e6aeeb043377261cc6e75 Mon Sep 17 00:00:00 2001 From: Kharkunov Eugene Date: Wed, 26 Aug 2026 23:21:43 +0300 Subject: [PATCH 1/3] Initial SPM support --- server/README_METRICS.md | 2 + server/build.gradle | 4 +- server/configs/application-standalone-dev.yml | 7 + .../standalone/setup-standalone-env.sh | 5 +- .../com/defold/extender/AsyncBuilder.java | 10 +- .../java/com/defold/extender/Extender.java | 269 +++++--- .../extender/metrics/MetricsWriter.java | 4 + .../extender/services/ResolvedNativeDeps.java | 50 ++ .../services/cocoapods/ResolvedPods.java | 26 +- .../services/spm/ResolvedPackages.java | 258 ++++++++ .../services/spm/SpmBuildOutputParser.java | 334 ++++++++++ .../services/spm/SpmManifestParser.java | 473 ++++++++++++++ .../spm/SpmManifestParsingException.java | 13 + .../services/spm/SpmServiceBuildState.java | 149 +++++ .../services/spm/SpmServiceConfiguration.java | 27 + .../spm/SwiftPackageManagerService.java | 582 ++++++++++++++++++ .../defold/extender/utils/VersionUtil.java | 44 ++ server/src/main/resources/application.yml | 13 + .../src/main/resources/template.package-swift | 33 + .../src/main/resources/template.project-yml | 26 + .../services/spm/ResolvedPackagesTest.java | 209 +++++++ .../spm/SpmBuildOutputParserTest.java | 189 ++++++ .../services/spm/SpmManifestParserTest.java | 226 +++++++ .../spm/SwiftPackageManagerServiceTest.java | 166 +++++ .../test-data/spm-buildlogs/dynamic-ios.log | 31 + server/test-data/spm-buildlogs/static-ios.log | 20 + .../test-data/spm-project/spmext/ext.manifest | 1 + .../spm-project/spmext/ios/SwiftPackages.json | 11 + .../spm-project/spmext/osx/SwiftPackages.json | 11 + .../spm-project/spmext/src/spmext.mm | 42 ++ server/test-data/swiftpackages/bad_json.json | 1 + .../both_version_and_branch.json | 6 + server/test-data/swiftpackages/branch.json | 10 + .../swiftpackages/branch_injection.json | 6 + .../swiftpackages/empty_products.json | 6 + .../swiftpackages/merge_conflict.json | 10 + .../test-data/swiftpackages/merge_extra.json | 16 + .../swiftpackages/no_requirement.json | 6 + .../swiftpackages/oversized_packages.json | 236 +++++++ .../product_swift_injection.json | 6 + .../test-data/swiftpackages/regular_ios.json | 16 + .../test-data/swiftpackages/regular_osx.json | 11 + server/test-data/swiftpackages/revision.json | 10 + .../swiftpackages/revision_short.json | 6 + .../test-data/swiftpackages/url_backtick.json | 6 + .../swiftpackages/url_custom_port.json | 6 + .../swiftpackages/url_dollar_paren.json | 6 + .../swiftpackages/url_file_scheme.json | 6 + .../test-data/swiftpackages/url_newline.json | 6 + server/test-data/swiftpackages/url_query.json | 6 + .../swiftpackages/url_quote_breakout.json | 6 + .../test-data/swiftpackages/url_ssh_scp.json | 6 + .../swiftpackages/url_traversal.json | 6 + .../test-data/swiftpackages/url_userinfo.json | 6 + .../swiftpackages/version_injection.json | 6 + .../swiftpackages/wrapper_type_bad.json | 7 + .../swiftpackages/wrapper_type_dynamic.json | 7 + .../swiftpackages/wrong_platform_value.json | 6 + 58 files changed, 3593 insertions(+), 74 deletions(-) create mode 100644 server/src/main/java/com/defold/extender/services/ResolvedNativeDeps.java create mode 100644 server/src/main/java/com/defold/extender/services/spm/ResolvedPackages.java create mode 100644 server/src/main/java/com/defold/extender/services/spm/SpmBuildOutputParser.java create mode 100644 server/src/main/java/com/defold/extender/services/spm/SpmManifestParser.java create mode 100644 server/src/main/java/com/defold/extender/services/spm/SpmManifestParsingException.java create mode 100644 server/src/main/java/com/defold/extender/services/spm/SpmServiceBuildState.java create mode 100644 server/src/main/java/com/defold/extender/services/spm/SpmServiceConfiguration.java create mode 100644 server/src/main/java/com/defold/extender/services/spm/SwiftPackageManagerService.java create mode 100644 server/src/main/java/com/defold/extender/utils/VersionUtil.java create mode 100644 server/src/main/resources/template.package-swift create mode 100644 server/src/main/resources/template.project-yml create mode 100644 server/src/test/java/com/defold/extender/services/spm/ResolvedPackagesTest.java create mode 100644 server/src/test/java/com/defold/extender/services/spm/SpmBuildOutputParserTest.java create mode 100644 server/src/test/java/com/defold/extender/services/spm/SpmManifestParserTest.java create mode 100644 server/src/test/java/com/defold/extender/services/spm/SwiftPackageManagerServiceTest.java create mode 100644 server/test-data/spm-buildlogs/dynamic-ios.log create mode 100644 server/test-data/spm-buildlogs/static-ios.log create mode 100644 server/test-data/spm-project/spmext/ext.manifest create mode 100644 server/test-data/spm-project/spmext/ios/SwiftPackages.json create mode 100644 server/test-data/spm-project/spmext/osx/SwiftPackages.json create mode 100644 server/test-data/spm-project/spmext/src/spmext.mm create mode 100644 server/test-data/swiftpackages/bad_json.json create mode 100644 server/test-data/swiftpackages/both_version_and_branch.json create mode 100644 server/test-data/swiftpackages/branch.json create mode 100644 server/test-data/swiftpackages/branch_injection.json create mode 100644 server/test-data/swiftpackages/empty_products.json create mode 100644 server/test-data/swiftpackages/merge_conflict.json create mode 100644 server/test-data/swiftpackages/merge_extra.json create mode 100644 server/test-data/swiftpackages/no_requirement.json create mode 100644 server/test-data/swiftpackages/oversized_packages.json create mode 100644 server/test-data/swiftpackages/product_swift_injection.json create mode 100644 server/test-data/swiftpackages/regular_ios.json create mode 100644 server/test-data/swiftpackages/regular_osx.json create mode 100644 server/test-data/swiftpackages/revision.json create mode 100644 server/test-data/swiftpackages/revision_short.json create mode 100644 server/test-data/swiftpackages/url_backtick.json create mode 100644 server/test-data/swiftpackages/url_custom_port.json create mode 100644 server/test-data/swiftpackages/url_dollar_paren.json create mode 100644 server/test-data/swiftpackages/url_file_scheme.json create mode 100644 server/test-data/swiftpackages/url_newline.json create mode 100644 server/test-data/swiftpackages/url_query.json create mode 100644 server/test-data/swiftpackages/url_quote_breakout.json create mode 100644 server/test-data/swiftpackages/url_ssh_scp.json create mode 100644 server/test-data/swiftpackages/url_traversal.json create mode 100644 server/test-data/swiftpackages/url_userinfo.json create mode 100644 server/test-data/swiftpackages/version_injection.json create mode 100644 server/test-data/swiftpackages/wrapper_type_bad.json create mode 100644 server/test-data/swiftpackages/wrapper_type_dynamic.json create mode 100644 server/test-data/swiftpackages/wrong_platform_value.json diff --git a/server/README_METRICS.md b/server/README_METRICS.md index 29d1a2618..6674bc1ee 100644 --- a/server/README_METRICS.md +++ b/server/README_METRICS.md @@ -14,6 +14,7 @@ Below is the table with metrics description that collected inside application. T |extender.job.sdk |Counter |Unit |How many times exact Defold sdk was used for building | |extender.job.gradle.download |Timer |Milliseconds |How long Gradle was downloading dependencies | |extender.job.cocoapods.install |Timer |Milliseconds |How long Cocoapods was installing dependencies | +|extender.job.spm.resolve |Timer |Milliseconds |How long Swift package dependencies were resolving and building | |extender.job.build |Timer |Milliseconds |How long build was | |extender.job.remoteBuild |Timer |Milliseconds |How long the remote build was | |extender.job.zip |Timer |Milliseconds |How long result was zipping | @@ -22,6 +23,7 @@ Below is the table with metrics description that collected inside application. T |extender.job.cache.download |Timer |Milliseconds |How long cache downloading operation was | |extender.build.task |Counter |Unit |How many builds were handled | |extender.service.cocoapods.get |Timer |Milliseconds |How long Cocoapods dependecies downloading was | +|extender.service.spm.get |Timer |Milliseconds |How long Swift package resolution inside the SPM service was | |extender.service.sdk.get.download |Counter |Unit |How many times Defold sdk was downloaded | |extender.service.sdk.get.duration |Timer |Milliseconds |How long Defold sdk was downloading | |extender.service.gradle.unpack |Timer |Milliseconds |How long Gradle was unpacking dependencies | diff --git a/server/build.gradle b/server/build.gradle index 51d5915b4..bfbc2418a 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -236,7 +236,9 @@ test { // builders they need (see docker-compose.yml profiles) // -PdefoldVersions=all|ci|latest|1.12.3 which Defold SDK versions to test against // -PreuseStack=true leave the docker stack running between runs - ["targetPlatforms", "defoldVersions", "reuseStack"].each { name -> + // -PspmE2e=true run the network-touching SPM service test on a Mac + // with Xcode + xcodegen installed + ["targetPlatforms", "defoldVersions", "reuseStack", "spmE2e"].each { name -> if (project.hasProperty(name)) { systemProperty "extender.test.${name}", project.getProperty(name) } diff --git a/server/configs/application-standalone-dev.yml b/server/configs/application-standalone-dev.yml index 0274c2c4b..00364e11e 100644 --- a/server/configs/application-standalone-dev.yml +++ b/server/configs/application-standalone-dev.yml @@ -16,3 +16,10 @@ extender: repo-update-cron: "0 0 * * * *" # update spec repo every 1 h cache-dir-rotate-cron: "0 10 2 * * *" # once per day old-cache-clean-cron: "0 10 6 * * *" # once per day after directory rotation + spm: + enabled: true + home-dir-prefix: /tmp/extender/.swiftpm + default-developer-dir: /Applications/Xcode.app/Contents/Developer + xcodegen-path: /opt/homebrew/bin/xcodegen + cache-dir-rotate-cron: "0 20 2 * * *" # once per day + old-cache-clean-cron: "0 20 6 * * *" # once per day after directory rotation diff --git a/server/scripts/standalone/setup-standalone-env.sh b/server/scripts/standalone/setup-standalone-env.sh index 0c3d6b21b..ea8356994 100755 --- a/server/scripts/standalone/setup-standalone-env.sh +++ b/server/scripts/standalone/setup-standalone-env.sh @@ -218,4 +218,7 @@ echo "[setup] Installing dotnet" install_dotnet echo "[setup] Install hmap utility" -brew install milend/taps/hmap \ No newline at end of file +brew install milend/taps/hmap + +echo "[setup] Install xcodegen (Swift Package Manager support)" +brew install xcodegen \ No newline at end of file diff --git a/server/src/main/java/com/defold/extender/AsyncBuilder.java b/server/src/main/java/com/defold/extender/AsyncBuilder.java index fcdd459de..82c56abd4 100644 --- a/server/src/main/java/com/defold/extender/AsyncBuilder.java +++ b/server/src/main/java/com/defold/extender/AsyncBuilder.java @@ -20,6 +20,7 @@ import com.defold.extender.services.GradleService; import com.defold.extender.services.cocoapods.CocoaPodsService; import com.defold.extender.services.data.DefoldSdk; +import com.defold.extender.services.spm.SwiftPackageManagerService; import org.apache.commons.io.FileUtils; import org.eclipse.jetty.io.EofException; @@ -38,6 +39,7 @@ public class AsyncBuilder { private DefoldSdkService defoldSdkService; private GradleService gradleService; private CocoaPodsService cocoaPodsService; + private SwiftPackageManagerService swiftPackageManagerService; private BuildProgressService buildProgressService; private File jobResultLocation; private long resultLifetime; @@ -46,12 +48,14 @@ public class AsyncBuilder { public AsyncBuilder(DefoldSdkService defoldSdkService, GradleService gradleService, Optional cocoaPodsService, + Optional swiftPackageManagerService, BuildProgressService buildProgressService, @Value("${extender.job-result.location}") String jobResultLocation, @Value("${extender.job-result.lifetime:1200000}") long jobResultLifetime) { this.defoldSdkService = defoldSdkService; this.gradleService = gradleService; cocoaPodsService.ifPresent(val -> { this.cocoaPodsService = val; }); + swiftPackageManagerService.ifPresent(val -> { this.swiftPackageManagerService = val; }); this.buildProgressService = buildProgressService; this.jobResultLocation = new File(jobResultLocation); this.keepJobDirectory = System.getenv("DM_DEBUG_KEEP_JOB_FOLDER") != null || System.getenv("DM_DEBUG_JOB_FOLDER") != null; @@ -124,11 +128,15 @@ public void asyncBuildEngine(MetricsWriter metricsWriter, String platform, Strin metricsWriter.measureGradleDownload(); } - // Resolve CocoaPods dependencies + // Resolve CocoaPods and Swift package dependencies if (ExtenderUtil.isAppleTarget(platform)) { progressReporter.stage(BuildStage.DEPENDENCIES, "Resolving CocoaPods dependencies"); extender.resolve(cocoaPodsService); metricsWriter.measureCocoaPodsInstallation(); + + progressReporter.stage(BuildStage.DEPENDENCIES, "Resolving Swift package dependencies"); + extender.resolve(swiftPackageManagerService); + metricsWriter.measureSpmResolution(); } // Build engine diff --git a/server/src/main/java/com/defold/extender/Extender.java b/server/src/main/java/com/defold/extender/Extender.java index ba72c1395..7250bdf75 100644 --- a/server/src/main/java/com/defold/extender/Extender.java +++ b/server/src/main/java/com/defold/extender/Extender.java @@ -43,11 +43,15 @@ import java.util.zip.ZipFile; import com.defold.extender.services.GradleService; +import com.defold.extender.services.ResolvedNativeDeps; +import com.defold.extender.services.spm.ResolvedPackages; +import com.defold.extender.services.spm.SwiftPackageManagerService; import com.defold.extender.services.cocoapods.CocoaPodsService; import com.defold.extender.services.cocoapods.PodBuildSpec; import com.defold.extender.services.cocoapods.PodUtils; import com.defold.extender.services.cocoapods.ResolvedPods; import com.defold.extender.utils.PodBuildUtil; +import com.defold.extender.utils.VersionUtil; import com.defold.extender.builders.CSharpBuilder; import com.defold.extender.log.Markers; import com.defold.extender.metrics.MetricsWriter; @@ -82,6 +86,8 @@ class Extender { private List androidPackages; private List outputFiles; private ResolvedPods resolvedPods; + // every resolved dependency manager (CocoaPods, SPM); consumed uniformly + private final List resolvedNativeDeps = new ArrayList<>(); private int nameCounter = 0; @@ -564,10 +570,10 @@ private void emitSwiftHeader(PodBuildSpec pod, Map manifestConte includes.addAll(getPodIncludeDir(pod)); List frameworks = new ArrayList<>(); - frameworks.addAll(resolvedPods.getFrameworks()); + frameworks.addAll(collectDepFrameworks()); List frameworkPaths = new ArrayList<>(); frameworkPaths.addAll(getFrameworkPaths(pod.dir)); - frameworkPaths.addAll(resolvedPods.getFrameworksSearchPaths()); + frameworkPaths.addAll(collectDepFrameworkSearchPaths()); File sourceListFile = ExtenderUtil.writeSourceFilesListToTmpFile(pod.intermediatedDir, pod.swiftSourceFilePaths); @@ -588,10 +594,10 @@ private void emitSwiftModule(PodBuildSpec pod, Map manifestConte includes.addAll(getPodIncludeDir(pod)); List frameworks = new ArrayList<>(); - frameworks.addAll(resolvedPods.getFrameworks()); + frameworks.addAll(collectDepFrameworks()); List frameworkPaths = new ArrayList<>(); frameworkPaths.addAll(getFrameworkPaths(pod.dir)); - frameworkPaths.addAll(resolvedPods.getFrameworksSearchPaths()); + frameworkPaths.addAll(collectDepFrameworkSearchPaths()); File sourceListFile = ExtenderUtil.writeSourceFilesListToTmpFile(pod.intermediatedDir, pod.swiftSourceFilePaths); @@ -638,9 +644,9 @@ private File addCompileFileSwift(PodBuildSpec pod, int index, File src, Map frameworks = new ArrayList<>(); - frameworks.addAll(resolvedPods.getFrameworks()); + frameworks.addAll(collectDepFrameworks()); List frameworkPaths = new ArrayList<>(); - frameworkPaths.addAll(resolvedPods.getFrameworksSearchPaths()); + frameworkPaths.addAll(collectDepFrameworkSearchPaths()); File sourceFileList = ExtenderUtil.writeSourceFilesListToTmpFile(pod.intermediatedDir, swiftSourceFilePaths); File primarySourceFile = ExtenderUtil.writeSourceFilesListToTmpFile(pod.intermediatedDir, Set.of(swiftPrimarySourceFile)); @@ -668,10 +674,8 @@ private File addCompileFileCpp_Internal(int index, File extDir, File src, Map frameworkPaths = new ArrayList<>(); frameworkPaths.addAll(getFrameworkPaths(extDir)); - if (resolvedPods != null) { - frameworks.addAll(resolvedPods.getFrameworks()); - frameworkPaths.addAll(resolvedPods.getFrameworksSearchPaths()); - } + frameworks.addAll(collectDepFrameworks()); + frameworkPaths.addAll(collectDepFrameworkSearchPaths()); Map context = createContext(manifestContext); context.put("src", ExtenderUtil.getRelativePath(buildState.jobDir, src)); @@ -817,7 +821,7 @@ private List compileExtensionSourceFiles(File extDir, Map objs = new ArrayList<>(); List commands = new ArrayList<>(); - List additionalIncludes = resolvedPods != null ? resolvedPods.getAdditionalIncludePaths() : List.of(); + List additionalIncludes = collectDepAdditionalIncludePaths(); for (File src : srcFiles) { final int i = getAndIncreaseNameCount(); @@ -1110,54 +1114,70 @@ private List buildPods() throws IOException, InterruptedException, Extende } } - LOGGER.info("buildPods - adding framework resource to build output"); - File resourcesBuildDir = new File(buildState.buildDir, "resources"); - resourcesBuildDir.mkdir(); + return outputFiles; + } - List resources = resolvedPods.getAllPodResources(); - for (File resourceFile : resources) { - if (resourceFile.isFile()) { - File resourceDestFile = new File(resourcesBuildDir, resourceFile.getName()); - resourceDestFile.getParentFile().mkdirs(); - Files.copy(resourceFile.toPath(), resourceDestFile.toPath()); - outputFiles.add(resourceDestFile); - } else { - File resourceDestDir = new File(resourcesBuildDir, resourceFile.getName()); - resourceDestDir.mkdirs(); - FileUtils.copyDirectory(resourceFile, resourceDestDir); - outputFiles.add(resourceDestDir); - } + private List buildNativeDepsArtifacts() throws IOException, InterruptedException, ExtenderException { + List outputFiles = new ArrayList<>(); + if (resolvedNativeDeps.isEmpty()) { + return outputFiles; } - LOGGER.info("buildPods - creating and adding resource bundles"); - outputFiles.addAll(resolvedPods.createResourceBundles(resourcesBuildDir, buildState.fullPlatform)); - - LOGGER.info("buildPods - adding dynamic frameworks to build output"); + LOGGER.info("buildNativeDepsArtifacts - adding resources to build output"); + File resourcesBuildDir = new File(buildState.buildDir, "resources"); + resourcesBuildDir.mkdir(); File frameworksBuildDir = new File(buildState.buildDir, "frameworks"); frameworksBuildDir.mkdir(); - List dynamicFrameworks = resolvedPods.getDynamicFrameworks(); - for (File framework : dynamicFrameworks) { - // copy framework and filter out certain files and folders - LOGGER.info("buildPods - adding {}", framework.getName()); - File frameworkDestDir = new File(frameworksBuildDir, framework.getName()); - FileUtils.copyDirectory(framework, frameworkDestDir, new FileFilter() { - @Override - public boolean accept(File pathname) { - String name = pathname.getName(); - return !name.equals("Headers") - && !name.equals("Modules"); + for (ResolvedNativeDeps deps : resolvedNativeDeps) { + for (File resourceFile : deps.getResources()) { + // the resources of every dependency manager are flattened into one directory, + // so a pod and a Swift package can vendor the same name - overwrite and report + // it instead of failing the build on the second copy + File resourceDest = new File(resourcesBuildDir, resourceFile.getName()); + boolean collision = resourceDest.exists(); + if (collision) { + LOGGER.warn("buildNativeDepsArtifacts - duplicate resource {}, overwriting", resourceFile.getName()); + if (resourceDest.isDirectory() != resourceFile.isDirectory()) { + FileUtils.forceDelete(resourceDest); + } } - }); - outputFiles.add(frameworkDestDir); - } + if (resourceFile.isFile()) { + resourceDest.getParentFile().mkdirs(); + Files.copy(resourceFile.toPath(), resourceDest.toPath(), StandardCopyOption.REPLACE_EXISTING); + } else { + resourceDest.mkdirs(); + FileUtils.copyDirectory(resourceFile, resourceDest); + } + if (!collision) { + outputFiles.add(resourceDest); + } + } - File podfileLock = resolvedPods.getPodfileLock(); - if (podfileLock != null) { - LOGGER.info("buildPods - adding Podfile.lock to build output"); - File destPodFileLock = new File(buildState.buildDir, "Podfile.lock"); - FileUtils.copyFile(podfileLock, destPodFileLock); - outputFiles.add(destPodFileLock); + outputFiles.addAll(deps.createResourceBundles(resourcesBuildDir, buildState.fullPlatform)); + + for (File framework : deps.getDynamicFrameworks()) { + // copy framework and filter out certain files and folders + LOGGER.info("buildNativeDepsArtifacts - adding {}", framework.getName()); + File frameworkDestDir = new File(frameworksBuildDir, framework.getName()); + FileUtils.copyDirectory(framework, frameworkDestDir, new FileFilter() { + @Override + public boolean accept(File pathname) { + String name = pathname.getName(); + return !name.equals("Headers") + && !name.equals("Modules"); + } + }); + outputFiles.add(frameworkDestDir); + } + + File lockFile = deps.getLockFile(); + if (lockFile != null) { + LOGGER.info("buildNativeDepsArtifacts - adding {} to build output", lockFile.getName()); + File destLockFile = new File(buildState.buildDir, lockFile.getName()); + FileUtils.copyFile(lockFile, destLockFile); + outputFiles.add(destLockFile); + } } return outputFiles; @@ -1428,13 +1448,11 @@ private void getProjectPaths(Map mainContext, Map buildEngine() throws ExtenderException { try { progressReporter.stage(BuildStage.COMPILING, "Building pods"); outputFiles.addAll(buildPods()); + outputFiles.addAll(buildNativeDepsArtifacts()); // An easy way to disable building an extension, is if the symbol name is // disabled at the .appmanifest level @@ -2444,16 +2463,19 @@ private List buildEngine() throws ExtenderException { resourceFile = buildWin32Resources(mergedAppContext); } - Map podAppContext = new HashMap<>(); - if (resolvedPods != null) { - podAppContext.put("frameworks", resolvedPods.getFrameworks()); - podAppContext.put("weakFrameworks", resolvedPods.getWeakFrameworks()); - podAppContext.put("libs", resolvedPods.getStaticLibraries()); - podAppContext.put("linkFlags", resolvedPods.getAllPodLinkFlags()); - podAppContext.put("osMinVersion", resolvedPods.getPlatformMinVersion()); - podAppContext.put("env.IOS_VERSION_MIN", resolvedPods.getPlatformMinVersion()); + Map nativeDepsContext = new HashMap<>(); + if (!resolvedNativeDeps.isEmpty()) { + nativeDepsContext.put("frameworks", collectDepFrameworks()); + nativeDepsContext.put("weakFrameworks", collectDepWeakFrameworks()); + nativeDepsContext.put("libs", collectDepStaticLibraries()); + nativeDepsContext.put("linkFlags", collectDepLinkFlags()); + String minVersion = maxDepMinVersion(null); + if (minVersion != null) { + nativeDepsContext.put("osMinVersion", minVersion); + nativeDepsContext.put("env.IOS_VERSION_MIN", minVersion); + } } - Map mergedAppContextWithPods = ExtenderUtil.mergeContexts(mergedAppContext, podAppContext); + Map mergedAppContextWithPods = ExtenderUtil.mergeContexts(mergedAppContext, nativeDepsContext); progressReporter.stage(BuildStage.LINKING, "Linking engine"); outputFiles.addAll(linkEngine(symbols, mergedAppContextWithPods, resourceFile)); @@ -2704,9 +2726,7 @@ private List buildApple(String platform) throws ExtenderException { privacyManifests.addAll(ExtenderUtil.listFilesMatchingRecursive(buildState.uploadDir, "PrivacyInfo.xcprivacy")); // no need to deal with PrivacyInfo manifests from pods because they will be packed into resource bundle // but that functionality saved for the backward compatability with older engine's versions (before 1.10.12) - if (resolvedPods != null) { - privacyManifests.addAll(resolvedPods.getPodsPrivacyManifests()); - } + privacyManifests.addAll(collectDepPrivacyManifests()); // do nothing if there are no privacy manifests if (privacyManifests.isEmpty()) { @@ -2871,13 +2891,122 @@ void resolveLocalAars() throws ExtenderException { } } + private List collectDepFrameworks() { + List result = new ArrayList<>(); + for (ResolvedNativeDeps deps : resolvedNativeDeps) { + result.addAll(deps.getFrameworks()); + } + return result; + } + + private Set collectDepBuiltFrameworks() { + Set result = new HashSet<>(); + for (ResolvedNativeDeps deps : resolvedNativeDeps) { + result.addAll(deps.getBuiltFrameworks()); + } + return result; + } + + private List collectDepFrameworkSearchPaths() { + List result = new ArrayList<>(); + for (ResolvedNativeDeps deps : resolvedNativeDeps) { + result.addAll(deps.getFrameworksSearchPaths()); + } + return result; + } + + private List collectDepStaticLibraries() { + List result = new ArrayList<>(); + for (ResolvedNativeDeps deps : resolvedNativeDeps) { + result.addAll(deps.getStaticLibraries()); + } + return result; + } + + private List collectDepLibrarySearchPaths() { + List result = new ArrayList<>(); + for (ResolvedNativeDeps deps : resolvedNativeDeps) { + result.addAll(deps.getLibrarySearchPaths()); + } + return result; + } + + private List collectDepWeakFrameworks() { + List result = new ArrayList<>(); + for (ResolvedNativeDeps deps : resolvedNativeDeps) { + result.addAll(deps.getWeakFrameworks()); + } + return result; + } + + private List collectDepLinkFlags() { + List result = new ArrayList<>(); + for (ResolvedNativeDeps deps : resolvedNativeDeps) { + result.addAll(deps.getLinkFlags()); + } + return result; + } + + private List collectDepAdditionalIncludePaths() { + List result = new ArrayList<>(); + for (ResolvedNativeDeps deps : resolvedNativeDeps) { + result.addAll(deps.getAdditionalIncludePaths()); + } + return result; + } + + private List collectDepPrivacyManifests() { + List result = new ArrayList<>(); + for (ResolvedNativeDeps deps : resolvedNativeDeps) { + result.addAll(deps.getPrivacyManifests()); + } + return result; + } + + private String maxDepMinVersion(String fallback) throws ExtenderException { + String result = fallback; + for (ResolvedNativeDeps deps : resolvedNativeDeps) { + String minVersion = deps.getPlatformMinVersion(); + if (minVersion == null) { + continue; + } + if (result == null || VersionUtil.compareVersions(minVersion, result) > 0) { + result = minVersion; + } + } + return result; + } + void resolve(CocoaPodsService cocoaPodsService) throws ExtenderException { + if (cocoaPodsService == null) { + LOGGER.warn("CocoaPods service is not available, skipping CocoaPods dependency resolution"); + return; + } try { resolvedPods = cocoaPodsService.resolveDependencies(platformConfig, buildState); } catch (IOException e) { throw new ExtenderException(e, "Failed to resolve CocoaPod dependencies. " + e.getMessage()); } + if (resolvedPods != null) { + resolvedNativeDeps.add(resolvedPods); + } + } + + void resolve(SwiftPackageManagerService swiftPackageManagerService) throws ExtenderException { + if (swiftPackageManagerService == null) { + LOGGER.info("SPM service is not available, skipping Swift package dependency resolution"); + return; + } + try { + ResolvedPackages resolvedPackages = swiftPackageManagerService.resolveDependencies(platformConfig, buildState); + if (resolvedPackages != null) { + resolvedNativeDeps.add(resolvedPackages); + } + } + catch (IOException e) { + throw new ExtenderException(e, "Failed to resolve Swift package dependencies. " + e.getMessage()); + } } void build() throws ExtenderException { diff --git a/server/src/main/java/com/defold/extender/metrics/MetricsWriter.java b/server/src/main/java/com/defold/extender/metrics/MetricsWriter.java index bc4d3130b..ebad18cc5 100644 --- a/server/src/main/java/com/defold/extender/metrics/MetricsWriter.java +++ b/server/src/main/java/com/defold/extender/metrics/MetricsWriter.java @@ -51,6 +51,10 @@ public void measureCocoaPodsInstallation() { metricsTimer(this.registry, "extender.job.cocoapods.install", timer.start()); } + public void measureSpmResolution() { + metricsTimer(this.registry, "extender.job.spm.resolve", timer.start()); + } + public void measureEngineBuild(final String platform) { metricsTimer(this.registry, "extender.job.build", timer.start(), "platform", platform); } diff --git a/server/src/main/java/com/defold/extender/services/ResolvedNativeDeps.java b/server/src/main/java/com/defold/extender/services/ResolvedNativeDeps.java new file mode 100644 index 000000000..5bd788081 --- /dev/null +++ b/server/src/main/java/com/defold/extender/services/ResolvedNativeDeps.java @@ -0,0 +1,50 @@ +package com.defold.extender.services; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Set; + +import com.defold.extender.ExtenderException; + +/** + * Native dependencies resolved by a dependency manager (CocoaPods, Swift Package Manager) + * for an Apple target. Extender consumes every implementation uniformly: the values feed + * the ext.* template variables of the compile/link commands and the build output. + */ +public interface ResolvedNativeDeps { + + List getFrameworks(); + + default Set getBuiltFrameworks() { return Set.of(); } + + default List getWeakFrameworks() { return List.of(); } + + List getFrameworksSearchPaths(); + + default List getStaticLibraries() { return List.of(); } + + default List getLibrarySearchPaths() { return List.of(); } + + /** Include paths added when compiling the user's extension sources. */ + default List getAdditionalIncludePaths() { return List.of(); } + + /** Raw additional flags for the final engine link. */ + default List getLinkFlags() { return List.of(); } + + /** Minimum OS version required by the resolved dependencies, or null if not constrained. */ + String getPlatformMinVersion(); + + default List getResources() { return List.of(); } + + default List createResourceBundles(File targetDir, String platform) throws IOException, ExtenderException { return List.of(); } + + /** Dynamically linked frameworks that must be embedded into the application bundle. */ + default List getDynamicFrameworks() { return List.of(); } + + /** Podfile.lock / Package.resolved, emitted with the build output; null when absent. */ + default File getLockFile() { return null; } + + /** Privacy manifests to merge for engines that predate resource-bundle packaging. */ + default List getPrivacyManifests() { return List.of(); } +} diff --git a/server/src/main/java/com/defold/extender/services/cocoapods/ResolvedPods.java b/server/src/main/java/com/defold/extender/services/cocoapods/ResolvedPods.java index 1461bf3b6..121006a99 100644 --- a/server/src/main/java/com/defold/extender/services/cocoapods/ResolvedPods.java +++ b/server/src/main/java/com/defold/extender/services/cocoapods/ResolvedPods.java @@ -19,10 +19,11 @@ import com.defold.extender.ExtenderConst; import com.defold.extender.ExtenderException; import com.defold.extender.ExtenderUtil; +import com.defold.extender.services.ResolvedNativeDeps; import com.defold.extender.services.cocoapods.PlistBuddyWrapper.CreateBundlePlistArgs; import com.defold.extender.utils.FrameworkUtil; -public class ResolvedPods { +public class ResolvedPods implements ResolvedNativeDeps { private static final Logger LOGGER = LoggerFactory.getLogger(ResolvedPods.class); private List pods = new ArrayList<>(); private File podsDir; @@ -308,6 +309,29 @@ public File getTargetSupportFilesDir() { return targetSupportFilesDir; } + // ResolvedNativeDeps aliases for the pod-named getters + + @Override + public List getLinkFlags() { + return getAllPodLinkFlags(); + } + + @Override + public List getResources() { + return getAllPodResources(); + } + + @Override + public File getLockFile() { + return getPodfileLock(); + } + + @Override + @SuppressWarnings("deprecation") + public List getPrivacyManifests() { + return getPodsPrivacyManifests(); + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/server/src/main/java/com/defold/extender/services/spm/ResolvedPackages.java b/server/src/main/java/com/defold/extender/services/spm/ResolvedPackages.java new file mode 100644 index 000000000..3c02bdc31 --- /dev/null +++ b/server/src/main/java/com/defold/extender/services/spm/ResolvedPackages.java @@ -0,0 +1,258 @@ +package com.defold.extender.services.spm; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import com.defold.extender.ExtenderException; +import com.defold.extender.ExtenderUtil; +import com.defold.extender.services.ResolvedNativeDeps; +import com.defold.extender.services.spm.SpmBuildOutputParser.LinkInfo; +import com.defold.extender.utils.FrameworkUtil; + +/** + * The harvested result of an SPM wrapper build: everything Extender needs to compile the + * extension sources against the packages and link the engine with them. A static wrapper + * is linked into the engine; a dylib wrapper is additionally embedded — both fall out of + * the dynamic-frameworks classification. + */ +public class ResolvedPackages implements ResolvedNativeDeps { + + private final List frameworks = new ArrayList<>(); + private final List frameworksSearchPaths = new ArrayList<>(); + private final List staticLibraries = new ArrayList<>(); + private final List librarySearchPaths = new ArrayList<>(); + private final List additionalIncludePaths = new ArrayList<>(); + private final List linkFlags = new ArrayList<>(); + private final List resources = new ArrayList<>(); + private final List dynamicFrameworks = new ArrayList<>(); + private final List privacyManifests = new ArrayList<>(); + private final File lockFile; + private final String platformMinVersion; + + private ResolvedPackages(File lockFile, String platformMinVersion) { + this.lockFile = lockFile; + this.platformMinVersion = platformMinVersion; + } + + static ResolvedPackages harvest(SpmServiceBuildState buildState, String platformMinVersion, LinkInfo linkInfo, + File swiftRuntimeLibDir) throws IOException, ExtenderException { + File productsDir = buildState.getProductsDir(); + if (!productsDir.isDirectory()) { + throw new ExtenderException("Swift package build products directory does not exist: " + productsDir); + } + + File lockFile = buildState.getLockFile().isFile() ? buildState.getLockFile() : null; + ResolvedPackages resolved = new ResolvedPackages(lockFile, platformMinVersion); + + File[] entries = productsDir.listFiles(); + if (entries == null) { + throw new ExtenderException("Cannot list the Swift package build products directory: " + productsDir); + } + Arrays.sort(entries); + // product framework name -> is a dylib + java.util.Map productFrameworks = new java.util.LinkedHashMap<>(); + for (File entry : entries) { + String name = entry.getName(); + if (name.endsWith(".framework") && entry.isDirectory()) { + boolean dynamic = FrameworkUtil.isDynamicallyLinked(entry); + productFrameworks.put(name.substring(0, name.length() - ".framework".length()), dynamic); + if (dynamic) { + resolved.dynamicFrameworks.add(entry); + } + } + else if (name.endsWith(".bundle") && entry.isDirectory()) { + resolved.resources.add(entry); + } + } + + // every products-dir framework stays on the engine link: a wrapper pulls static + // framework members only on demand, so the framework remains the authoritative + // home of ObjC classes the extension code references itself + Set frameworkNames = new LinkedHashSet<>(productFrameworks.keySet()); + + Set librarySearchPaths = new LinkedHashSet<>(); + if (linkInfo != null) { + frameworkNames.addAll(linkInfo.frameworkNames()); + // a -l naming a products-dir library is a wrapper input the wrapper already + // resolved; system libs (c++, z, ...) have no products-dir counterpart and stay + for (String lib : linkInfo.systemLibs()) { + if (!new File(productsDir, "lib" + lib + ".a").exists()) { + resolved.staticLibraries.add(lib); + } + } + librarySearchPaths.addAll(linkInfo.librarySearchPaths()); + } + // package objects auto-link the Swift compatibility archives from the building + // Xcode's toolchain; a static wrapper's libtool line carries no -L for them + if (swiftRuntimeLibDir != null && swiftRuntimeLibDir.isDirectory()) { + librarySearchPaths.add(swiftRuntimeLibDir.getAbsolutePath()); + } + resolved.librarySearchPaths.addAll(librarySearchPaths); + resolved.frameworks.addAll(frameworkNames); + + resolved.frameworksSearchPaths.add(productsDir.getAbsolutePath()); + File packageFrameworksDir = new File(productsDir, "PackageFrameworks"); + String[] packageFrameworks = packageFrameworksDir.list(); + if (packageFrameworks != null && packageFrameworks.length > 0) { + resolved.frameworksSearchPaths.add(packageFrameworksDir.getAbsolutePath()); + } + + Set includePaths = new LinkedHashSet<>(); + File generatedModuleMapsDir = buildState.getGeneratedModuleMapsDir(); + if (generatedModuleMapsDir.isDirectory()) { + includePaths.add(generatedModuleMapsDir.getAbsolutePath()); + } + includePaths.add(productsDir.getAbsolutePath()); + File productsIncludeDir = new File(productsDir, "include"); + if (productsIncludeDir.isDirectory()) { + includePaths.add(productsIncludeDir.getAbsolutePath()); + } + collectModuleMapIncludePaths(generatedModuleMapsDir, new File(buildState.getWorkingDir(), "include"), includePaths); + resolved.additionalIncludePaths.addAll(includePaths); + + // an embedded framework resolves the Swift runtime and its own location at run time + if (!resolved.dynamicFrameworks.isEmpty()) { + resolved.linkFlags.add("-Wl,-rpath,/usr/lib/swift"); + resolved.linkFlags.add("-Wl,-rpath,@executable_path/Frameworks"); + } + + // versioned (macOS) framework bundles expose the same manifest through the + // Resources and Versions/A + Versions/Current symlinks — keep one per file + Set seenManifests = new LinkedHashSet<>(); + for (File manifest : ExtenderUtil.listFilesMatchingRecursive(productsDir, "PrivacyInfo.xcprivacy")) { + if (seenManifests.add(manifest.getCanonicalPath())) { + resolved.privacyManifests.add(manifest); + } + } + + return resolved; + } + + private static final java.util.regex.Pattern MODULE_NAME_PATTERN = + java.util.regex.Pattern.compile("module\\s+([A-Za-z0-9_]+)"); + private static final java.util.regex.Pattern UMBRELLA_PATTERN = + java.util.regex.Pattern.compile("umbrella\\s+(?:header\\s+)?\"([^\"]+)\""); + + /** + * Source-built packages keep their public headers in the package checkout; the umbrella + * named by each generated module map is the only reliable pointer to them. The derived + * -I paths (plus a symlinked module-name dir for flat layouts) are what lets extension + * code use framework-style imports of source-built modules. + */ + static void collectModuleMapIncludePaths(File generatedModuleMapsDir, File syntheticIncludeDir, Set includePaths) + throws IOException { + if (!generatedModuleMapsDir.isDirectory()) { + return; + } + File[] moduleMaps = generatedModuleMapsDir.listFiles((dir, name) -> name.endsWith(".modulemap")); + if (moduleMaps == null) { + return; + } + Arrays.sort(moduleMaps); + for (File moduleMap : moduleMaps) { + String content = new String(java.nio.file.Files.readAllBytes(moduleMap.toPath())); + java.util.regex.Matcher nameMatcher = MODULE_NAME_PATTERN.matcher(content); + java.util.regex.Matcher umbrellaMatcher = UMBRELLA_PATTERN.matcher(content); + if (!nameMatcher.find() || !umbrellaMatcher.find()) { + continue; + } + String moduleName = nameMatcher.group(1); + File umbrella = new File(umbrellaMatcher.group(1)); + // umbrella can name a header file or a headers directory + File headersDir = umbrella.isDirectory() ? umbrella : umbrella.getParentFile(); + if (headersDir == null || !headersDir.isDirectory()) { + continue; + } + if (headersDir.getName().equals(moduleName) && headersDir.getParentFile() != null) { + includePaths.add(headersDir.getParentFile().getAbsolutePath()); + } + else { + // the dir itself serves imports whose name differs from + // the module; the module-name symlink serves flat layouts + includePaths.add(headersDir.getAbsolutePath()); + File link = new File(syntheticIncludeDir, moduleName); + if (!java.nio.file.Files.isSymbolicLink(link.toPath())) { + syntheticIncludeDir.mkdirs(); + java.nio.file.Files.createSymbolicLink(link.toPath(), headersDir.toPath()); + } + includePaths.add(syntheticIncludeDir.getAbsolutePath()); + } + } + } + + @Override + public List getFrameworks() { + return frameworks; + } + + @Override + public List getFrameworksSearchPaths() { + return frameworksSearchPaths; + } + + @Override + public List getStaticLibraries() { + return staticLibraries; + } + + @Override + public List getLibrarySearchPaths() { + return librarySearchPaths; + } + + @Override + public List getAdditionalIncludePaths() { + return additionalIncludePaths; + } + + @Override + public List getLinkFlags() { + return linkFlags; + } + + @Override + public String getPlatformMinVersion() { + return platformMinVersion; + } + + @Override + public List getResources() { + return resources; + } + + @Override + public List getDynamicFrameworks() { + return dynamicFrameworks; + } + + @Override + public File getLockFile() { + return lockFile; + } + + @Override + public List getPrivacyManifests() { + return privacyManifests; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("frameworks: " + frameworks + "\n"); + sb.append("framework search paths: " + frameworksSearchPaths + "\n"); + sb.append("system libraries: " + staticLibraries + "\n"); + sb.append("library search paths: " + librarySearchPaths + "\n"); + sb.append("include paths: " + additionalIncludePaths + "\n"); + sb.append("resource bundles: " + resources.size() + "\n"); + sb.append("dynamic frameworks: " + dynamicFrameworks.size() + "\n"); + sb.append("privacy manifests: " + privacyManifests.size() + "\n"); + sb.append("platform min version: " + platformMinVersion + "\n"); + sb.append("Package.resolved: " + lockFile); + return sb.toString(); + } +} diff --git a/server/src/main/java/com/defold/extender/services/spm/SpmBuildOutputParser.java b/server/src/main/java/com/defold/extender/services/spm/SpmBuildOutputParser.java new file mode 100644 index 000000000..23bc2743c --- /dev/null +++ b/server/src/main/java/com/defold/extender/services/spm/SpmBuildOutputParser.java @@ -0,0 +1,334 @@ +package com.defold.extender.services.spm; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Extracts the wrapper's final link line and the generated module maps from the captured + * xcodebuild log. A dynamic wrapper links via clang -dynamiclib, a static one archives via + * libtool -static; package targets are prelinked with clang -r into loose .o files + * whose Ld lines must not be mistaken for the wrapper link. + */ +public class SpmBuildOutputParser { + + private static final Logger LOGGER = LoggerFactory.getLogger(SpmBuildOutputParser.class); + + public record LinkInfo(List systemLibs, + List frameworkNames, + List librarySearchPaths, + List rpaths, + List forceLoad) {} + + private static final Pattern MODULE_MAP_PATTERN = Pattern.compile("-fmodule-map-file=([^\\s']+)"); + // an exact -l token. This on its own does not separate libraries from the ld + // flags that also begin with -l ("-lto_library" matches it too) — those are listed + // in LD_FLAGS_WITH_ARG and consumed with their argument before this is tried. + private static final Pattern LIB_TOKEN_PATTERN = Pattern.compile("-l[A-Za-z0-9_+.]+"); + // ld flags that begin with -l but name no library; each takes a following argument + private static final Set LD_FLAGS_WITH_ARG = Set.of("-lto_library", "-lazy_library", "-lazy_framework"); + + // Xcode moves long link commands into response files referenced as @. The flags + // inside are part of the link line and are expanded here — an unexpanded token silently + // drops every -framework/-l/-L the wrapper's consumers need. + private static final int MAX_RESPONSE_FILE_DEPTH = 4; + private static final long MAX_RESPONSE_FILE_SIZE = 4L * 1024 * 1024; + // dyld run-path placeholders: an "@" token that is a path value, not a response file + private static final List DYLD_PATH_PREFIXES = List.of("@rpath/", "@executable_path/", "@loader_path/"); + + /** + * Single streaming pass — a large package graph produces a log too big to hold whole. + * The clang -dynamiclib line wins over the libtool -static fallback. + */ + public static String findWrapperLinkLine(File logFile, String wrapperName) throws IOException { + String wrapperBinary = wrapperBinaryPath(wrapperName); + String libtoolLine = null; + // an xcodebuild log is not guaranteed to be valid UTF-8; InputStreamReader replaces + // malformed input where Files.newBufferedReader would throw + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(new FileInputStream(logFile), StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + // cheap pre-filter: both link steps name the wrapper in their -o argument + if (!line.contains(wrapperName)) { + continue; + } + if (isDynamicLinkLine(line, wrapperBinary)) { + return line; + } + if (libtoolLine == null && isLibtoolLine(line, wrapperBinary)) { + libtoolLine = line; + } + } + } + return libtoolLine; + } + + /** The clang -dynamiclib invocation that links .framework/, or null. */ + public static String findWrapperLinkLine(String log, String wrapperName) { + String wrapperBinary = wrapperBinaryPath(wrapperName); + for (String line : log.split("\n")) { + if (isDynamicLinkLine(line, wrapperBinary)) { + return line; + } + } + return null; + } + + /** The libtool -static invocation that archives .framework/, or null. */ + public static String findWrapperLibtoolLine(String log, String wrapperName) { + String wrapperBinary = wrapperBinaryPath(wrapperName); + for (String line : log.split("\n")) { + if (isLibtoolLine(line, wrapperBinary)) { + return line; + } + } + return null; + } + + private static String wrapperBinaryPath(String wrapperName) { + return wrapperName + ".framework/" + wrapperName; + } + + private static boolean isDynamicLinkLine(String line, String wrapperBinary) { + return line.contains("-dynamiclib") && outputsWrapperBinary(line, wrapperBinary); + } + + private static boolean isLibtoolLine(String line, String wrapperBinary) { + return line.contains("libtool") && line.contains("-static") && outputsWrapperBinary(line, wrapperBinary); + } + + private static boolean outputsWrapperBinary(String line, String wrapperBinary) { + // macOS frameworks are versioned bundles: the binary is .framework/Versions// + int slash = wrapperBinary.indexOf('/'); + String bundleName = wrapperBinary.substring(0, slash); + String binaryName = wrapperBinary.substring(slash); + List tokens = tokenize(line); + for (int i = 0; i < tokens.size() - 1; i++) { + if (!tokens.get(i).equals("-o")) { + continue; + } + String output = tokens.get(i + 1); + if (output.endsWith(binaryName) && output.contains(bundleName + "/")) { + return true; + } + } + return false; + } + + /** + * Tokenizes one link line and collects the flags a consumer of the wrapper needs. + * Search paths under excludePathPrefix (the job's DerivedData) are dropped — they + * are job-local and re-expressed from the harvested products directory instead. + */ + public static LinkInfo parseLinkLine(String linkLine, String excludePathPrefix) { + return parseLinkLine(linkLine, excludePathPrefix, null); + } + + /** + * @param responseFileBaseDir directory relative @ response-file tokens resolve + * against (the working directory of the build step) + */ + public static LinkInfo parseLinkLine(String linkLine, String excludePathPrefix, File responseFileBaseDir) { + Set systemLibs = new LinkedHashSet<>(); + Set frameworks = new LinkedHashSet<>(); + Set librarySearchPaths = new LinkedHashSet<>(); + Set rpaths = new LinkedHashSet<>(); + Set forceLoad = new LinkedHashSet<>(); + + List tokens = expandResponseFiles(tokenize(linkLine), responseFileBaseDir, 0); + for (int i = 0; i < tokens.size(); i++) { + String token = tokens.get(i); + if (token.equals("-framework") && i + 1 < tokens.size()) { + frameworks.add(tokens.get(++i)); + } + else if (LD_FLAGS_WITH_ARG.contains(token)) { + // consume the flag's argument so it is not mistaken for an input file + i++; + } + else if (LIB_TOKEN_PATTERN.matcher(token).matches()) { + systemLibs.add(token.substring(2)); + } + else if (token.startsWith("-L") && token.length() > 2) { + String path = token.substring(2); + if (!path.startsWith(excludePathPrefix)) { + librarySearchPaths.add(path); + } + } + else if (token.equals("-force_load") && i + 1 < tokens.size()) { + forceLoad.add(tokens.get(++i)); + } + else if (token.equals("-Xlinker") && i + 1 < tokens.size()) { + String linkerArg = tokens.get(++i); + if (linkerArg.equals("-rpath") && i + 2 < tokens.size() && tokens.get(i + 1).equals("-Xlinker")) { + rpaths.add(tokens.get(i + 2)); + i += 2; + } + else if (linkerArg.equals("-force_load") && i + 2 < tokens.size() && tokens.get(i + 1).equals("-Xlinker")) { + forceLoad.add(tokens.get(i + 2)); + i += 2; + } + } + else if (token.startsWith("-Wl,")) { + String[] parts = token.substring(4).split(","); + if (parts.length > 1 && parts[0].equals("-rpath")) { + rpaths.add(parts[1]); + } + else if (parts.length > 1 && parts[0].equals("-force_load")) { + forceLoad.add(parts[1]); + } + } + } + + return new LinkInfo( + new ArrayList<>(systemLibs), + new ArrayList<>(frameworks), + new ArrayList<>(librarySearchPaths), + new ArrayList<>(rpaths), + new ArrayList<>(forceLoad)); + } + + /** All -fmodule-map-file= values in the log, deduplicated, quotes stripped. */ + public static List parseModuleMaps(String log) { + Set moduleMaps = new LinkedHashSet<>(); + Matcher matcher = MODULE_MAP_PATTERN.matcher(log); + while (matcher.find()) { + moduleMaps.add(matcher.group(1)); + } + return new ArrayList<>(moduleMaps); + } + + /** The object files fed to the wrapper link, from .LinkFileList. */ + public static List readLinkFileList(File linkFileList) throws IOException { + List result = new ArrayList<>(); + for (String line : Files.readAllLines(linkFileList.toPath())) { + String trimmed = line.trim(); + if (!trimmed.isEmpty()) { + result.add(trimmed); + } + } + return result; + } + + /** + * Replaces every @ response-file token with the tokens the file holds. A file + * that cannot be expanded is reported and dropped: silently keeping the token would + * leave the harvested link surface incomplete with no trace of why. + */ + private static List expandResponseFiles(List tokens, File baseDir, int depth) { + boolean hasResponseFile = false; + for (String token : tokens) { + if (isResponseFileToken(token)) { + hasResponseFile = true; + break; + } + } + if (!hasResponseFile) { + return tokens; + } + + List expanded = new ArrayList<>(tokens.size()); + for (String token : tokens) { + if (!isResponseFileToken(token)) { + expanded.add(token); + continue; + } + String path = token.substring(1); + File file = new File(path); + if (!file.isAbsolute() && baseDir != null) { + file = new File(baseDir, path); + } + if (depth >= MAX_RESPONSE_FILE_DEPTH) { + LOGGER.warn("Link response file {} nested deeper than {} levels, its link flags are not harvested", + file, MAX_RESPONSE_FILE_DEPTH); + continue; + } + if (!file.isFile()) { + LOGGER.warn("Link response file {} does not exist, its link flags are not harvested", file); + continue; + } + if (file.length() > MAX_RESPONSE_FILE_SIZE) { + LOGGER.warn("Link response file {} is larger than {} bytes, its link flags are not harvested", + file, MAX_RESPONSE_FILE_SIZE); + continue; + } + try { + String contents = new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8); + expanded.addAll(expandResponseFiles(tokenize(contents), file.getParentFile(), depth + 1)); + } catch (IOException e) { + LOGGER.warn("Failed to read link response file {}, its link flags are not harvested", file, e); + } + } + return expanded; + } + + private static boolean isResponseFileToken(String token) { + if (token.length() < 2 || token.charAt(0) != '@') { + return false; + } + for (String prefix : DYLD_PATH_PREFIXES) { + if (token.startsWith(prefix)) { + return false; + } + } + return true; + } + + /** + * Splits a command line into tokens the way a shell would: quoted runs and + * backslash-escaped characters keep their whitespace, quotes are not part of the value. + */ + private static List tokenize(String text) { + List tokens = new ArrayList<>(); + StringBuilder current = new StringBuilder(); + boolean inToken = false; + char quote = 0; + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + if (c == '\\' && quote != '\'' && i + 1 < text.length()) { + current.append(text.charAt(++i)); + inToken = true; + } + else if (quote != 0) { + if (c == quote) { + quote = 0; + } + else { + current.append(c); + } + } + else if (c == '\'' || c == '"') { + quote = c; + inToken = true; + } + else if (Character.isWhitespace(c)) { + if (inToken) { + tokens.add(current.toString()); + current.setLength(0); + inToken = false; + } + } + else { + current.append(c); + inToken = true; + } + } + if (inToken) { + tokens.add(current.toString()); + } + return tokens; + } +} diff --git a/server/src/main/java/com/defold/extender/services/spm/SpmManifestParser.java b/server/src/main/java/com/defold/extender/services/spm/SpmManifestParser.java new file mode 100644 index 000000000..4b1db6de9 --- /dev/null +++ b/server/src/main/java/com/defold/extender/services/spm/SpmManifestParser.java @@ -0,0 +1,473 @@ +package com.defold.extender.services.spm; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Pattern; + +import com.defold.extender.ExtenderException; +import com.defold.extender.utils.VersionUtil; + +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; + +/** + * Parses and sanitizes SwiftPackages.json manifests found in extension uploads. Every value + * that ends up in a generated Package.swift or project.yml is reconstructed from validated + * parts — raw manifest input is never echoed into a template. + * + * Manifest schema: + * { + * "platform": "ios", + * "minVersion": "13.0", + * "packages": [ + * { "url": "https://github.com/firebase/firebase-ios-sdk.git", + * "version": "12.0.0", // or "from" / "branch" / "revision" + * "products": ["FirebaseAnalytics"] } + * ] + * } + */ +public class SpmManifestParser { + + static final int MAX_MANIFEST_FILE_SIZE = 64 * 1024; + static final int MAX_MANIFESTS = 32; + static final int MAX_PACKAGES = 32; + static final int MAX_PRODUCTS = 64; + static final int MAX_URL_LENGTH = 2048; + + private static final Pattern HOST_PATTERN = Pattern.compile("[A-Za-z0-9.-]+"); + private static final Pattern PATH_PATTERN = Pattern.compile("[A-Za-z0-9._/-]+"); + private static final Pattern VERSION_PATTERN = Pattern.compile("\\d+(\\.\\d+){0,2}([-+][A-Za-z0-9.-]{1,40})?"); + private static final Pattern MIN_VERSION_PATTERN = Pattern.compile("\\d+(\\.\\d+){0,2}"); + // SwiftPM's Version literal fatalErrors on fewer than three components and a platform + // version needs at least two; unambiguous short forms are padded rather than rejected + private static final int VERSION_COMPONENTS = 3; + private static final int MIN_VERSION_COMPONENTS = 2; + private static final Pattern BRANCH_PATTERN = Pattern.compile("[A-Za-z0-9][A-Za-z0-9._/-]{0,99}"); + private static final Pattern REVISION_PATTERN = Pattern.compile("[0-9a-f]{40}"); + private static final Pattern PRODUCT_PATTERN = Pattern.compile("[A-Za-z0-9_.-]{1,100}"); + + public enum RequirementKind { EXACT, FROM, BRANCH, REVISION } + + public static final class Requirement { + public final RequirementKind kind; + public final String value; + + Requirement(RequirementKind kind, String value) { + this.kind = kind; + this.value = value; + } + + /** The dependency requirement as a Package.swift argument, built from validated parts only. */ + public String toSwiftArgument() { + switch (kind) { + case EXACT: return String.format("exact: \"%s\"", value); + case FROM: return String.format("from: \"%s\"", value); + case BRANCH: return String.format("branch: \"%s\"", value); + case REVISION: return String.format("revision: \"%s\"", value); + default: throw new IllegalStateException("Unknown requirement kind " + kind); + } + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Requirement)) { + return false; + } + Requirement req = (Requirement) other; + return kind == req.kind && value.equals(req.value); + } + + @Override + public int hashCode() { + return Objects.hash(kind, value); + } + + @Override + public String toString() { + return toSwiftArgument(); + } + } + + public static final class PackageRef { + public final String url; // sanitized, reconstructed + public final Requirement requirement; + public final Set products = new LinkedHashSet<>(); + + PackageRef(String url, Requirement requirement) { + this.url = url; + this.requirement = requirement; + } + + /** SPM package identity: the repository basename without the ".git" suffix. */ + public String label() { + String basename = url.substring(url.lastIndexOf('/') + 1); + if (basename.endsWith(".git")) { + basename = basename.substring(0, basename.length() - 4); + } + return basename; + } + } + + public static final class ParseResult { + public String platform; + public String minVersion; + // optional per-graph override of the wrapper linkage: "static" | "dynamic" | null + public String wrapperType; + // keyed by canonical URL (lowercased host, ".git" suffix stripped) + public final Map packages = new LinkedHashMap<>(); + + public ParseResult mergeWith(ParseResult other) throws SpmManifestParsingException { + if (platform != null && other.platform != null && !platform.equals(other.platform)) { + throw new SpmManifestParsingException( + String.format("Mismatch 'platform' between Swift package manifests: %s != %s", platform, other.platform)); + } + if (wrapperType != null && other.wrapperType != null && !wrapperType.equals(other.wrapperType)) { + throw new SpmManifestParsingException( + String.format("Mismatch 'wrapperType' between Swift package manifests: %s != %s", wrapperType, other.wrapperType)); + } + if (wrapperType == null) { + wrapperType = other.wrapperType; + } + if (platform == null) { + platform = other.platform; + } + + if (minVersion == null) { + minVersion = other.minVersion; + } + else if (other.minVersion != null) { + try { + if (VersionUtil.compareVersions(other.minVersion, minVersion) > 0) { + minVersion = other.minVersion; + } + } catch (ExtenderException e) { + throw new SpmManifestParsingException( + String.format("Failed to compare platform min versions '%s' and '%s'", minVersion, other.minVersion), e); + } + } + + for (Map.Entry entry : other.packages.entrySet()) { + PackageRef existing = packages.get(entry.getKey()); + if (existing == null) { + packages.put(entry.getKey(), entry.getValue()); + } + else { + if (!existing.requirement.equals(entry.getValue().requirement)) { + throw new SpmManifestParsingException( + String.format("Conflicting requirements for Swift package '%s': %s != %s", + existing.url, existing.requirement, entry.getValue().requirement)); + } + existing.products.addAll(entry.getValue().products); + } + } + checkPackageCount(packages); + checkProductCount(packages.values()); + return this; + } + } + + /** + * The result is seeded with the build's platform family and default min version so + * every manifest is validated against the platform actually being built. + */ + public static ParseResult parseManifests(List manifestFiles, String platformFamily, String defaultMinVersion) + throws IOException, SpmManifestParsingException { + if (manifestFiles.size() > MAX_MANIFESTS) { + throw new SpmManifestParsingException( + String.format("Upload declares more than %d Swift package manifests", MAX_MANIFESTS)); + } + ParseResult result = new ParseResult(); + result.platform = platformFamily; + if (defaultMinVersion != null) { + if (!MIN_VERSION_PATTERN.matcher(defaultMinVersion).matches()) { + throw new SpmManifestParsingException( + String.format("Invalid platform min version in the job environment: '%s'", defaultMinVersion)); + } + result.minVersion = padVersion(defaultMinVersion, MIN_VERSION_COMPONENTS); + } + for (File manifestFile : manifestFiles) { + result.mergeWith(parseManifest(manifestFile)); + } + return result; + } + + public static ParseResult parseManifest(File manifestFile) throws IOException, SpmManifestParsingException { + if (manifestFile.length() > MAX_MANIFEST_FILE_SIZE) { + throw new SpmManifestParsingException( + String.format("Swift package manifest %s is larger than %d bytes", manifestFile.getName(), MAX_MANIFEST_FILE_SIZE)); + } + + JsonNode root; + try { + root = new ObjectMapper().readTree(manifestFile); + } catch (RuntimeException e) { + throw new SpmManifestParsingException( + String.format("Swift package manifest %s is not valid JSON", manifestFile.getName()), e); + } + if (root == null || !root.isObject()) { + throw new SpmManifestParsingException( + String.format("Swift package manifest %s must contain a JSON object", manifestFile.getName())); + } + + ParseResult result = new ParseResult(); + result.platform = parsePlatform(root); + result.minVersion = parseMinVersion(root); + result.wrapperType = parseWrapperType(root); + + JsonNode packagesNode = root.get("packages"); + if (packagesNode == null || !packagesNode.isArray() || packagesNode.isEmpty()) { + throw new SpmManifestParsingException("Swift package manifest must declare a non-empty 'packages' list"); + } + if (packagesNode.size() > MAX_PACKAGES) { + throw new SpmManifestParsingException( + String.format("Swift package manifest declares more than %d packages", MAX_PACKAGES)); + } + + for (JsonNode packageNode : packagesNode) { + if (!packageNode.isObject()) { + throw new SpmManifestParsingException("Each entry in 'packages' must be a JSON object"); + } + String url = sanitizeUrl(textValue(packageNode, "url")); + Requirement requirement = parseRequirement(packageNode, url); + PackageRef ref = new PackageRef(url, requirement); + ref.products.addAll(parseProducts(packageNode, url)); + + String key = canonicalKey(url); + PackageRef existing = result.packages.get(key); + if (existing == null) { + result.packages.put(key, ref); + } + else { + if (!existing.requirement.equals(ref.requirement)) { + throw new SpmManifestParsingException( + String.format("Conflicting requirements for Swift package '%s': %s != %s", + url, existing.requirement, ref.requirement)); + } + existing.products.addAll(ref.products); + } + } + checkProductCount(result.packages.values()); + + return result; + } + + // the static wrapper breaks when a graph feeds the same binary library to libtool + // twice; a dynamic wrapper links such graphs cleanly + private static String parseWrapperType(JsonNode root) throws SpmManifestParsingException { + JsonNode node = root.get("wrapperType"); + if (node == null || node.isNull()) { + return null; + } + String wrapperType = node.isTextual() ? node.asString() : null; + if (!"static".equals(wrapperType) && !"dynamic".equals(wrapperType)) { + throw new SpmManifestParsingException( + String.format("Invalid 'wrapperType' in Swift package manifest: '%s' (expected 'static' or 'dynamic')", node.asString())); + } + return wrapperType; + } + + private static String parsePlatform(JsonNode root) throws SpmManifestParsingException { + String platform = textValue(root, "platform"); + if (!"ios".equals(platform) && !"osx".equals(platform)) { + throw new SpmManifestParsingException( + String.format("Unsupported 'platform' in Swift package manifest: '%s' (expected 'ios' or 'osx')", platform)); + } + return platform; + } + + private static String parseMinVersion(JsonNode root) throws SpmManifestParsingException { + JsonNode node = root.get("minVersion"); + if (node == null || node.isNull()) { + return null; + } + String minVersion = node.isTextual() ? node.asString() : null; + if (minVersion == null || !MIN_VERSION_PATTERN.matcher(minVersion).matches()) { + throw new SpmManifestParsingException( + String.format("Invalid 'minVersion' in Swift package manifest: '%s'", node.asString())); + } + return padVersion(minVersion, MIN_VERSION_COMPONENTS); + } + + /** Pads a validated version to the number of components its SwiftPM grammar requires. */ + private static String padVersion(String version, int components) { + int cut = version.length(); + for (int i = 0; i < version.length(); i++) { + char c = version.charAt(i); + if (c == '-' || c == '+') { + cut = i; + break; + } + } + StringBuilder core = new StringBuilder(version.substring(0, cut)); + for (int i = core.toString().split("\\.", -1).length; i < components; i++) { + core.append(".0"); + } + return core + version.substring(cut); + } + + private static Requirement parseRequirement(JsonNode packageNode, String url) throws SpmManifestParsingException { + Requirement requirement = null; + for (RequirementKind kind : RequirementKind.values()) { + String field = fieldNameFor(kind); + JsonNode node = packageNode.get(field); + if (node == null) { + continue; + } + if (requirement != null) { + throw new SpmManifestParsingException( + String.format("Swift package '%s' declares more than one of version/from/branch/revision", url)); + } + String value = node.isTextual() ? node.asString() : null; + if (value == null || !patternFor(kind).matcher(value).matches()) { + throw new SpmManifestParsingException( + String.format("Invalid '%s' for Swift package '%s': '%s'", field, url, node.asString())); + } + if (kind == RequirementKind.EXACT || kind == RequirementKind.FROM) { + value = padVersion(value, VERSION_COMPONENTS); + } + requirement = new Requirement(kind, value); + } + if (requirement == null) { + throw new SpmManifestParsingException( + String.format("Swift package '%s' must declare exactly one of version/from/branch/revision", url)); + } + return requirement; + } + + private static List parseProducts(JsonNode packageNode, String url) throws SpmManifestParsingException { + JsonNode productsNode = packageNode.get("products"); + if (productsNode == null || !productsNode.isArray() || productsNode.isEmpty()) { + throw new SpmManifestParsingException( + String.format("Swift package '%s' must declare a non-empty 'products' list", url)); + } + List products = new java.util.ArrayList<>(); + for (JsonNode productNode : productsNode) { + String product = productNode.isTextual() ? productNode.asString() : null; + if (product == null || !PRODUCT_PATTERN.matcher(product).matches()) { + throw new SpmManifestParsingException( + String.format("Invalid product name for Swift package '%s': '%s'", url, productNode.asString())); + } + products.add(product); + } + return products; + } + + private static void checkPackageCount(Map packages) throws SpmManifestParsingException { + if (packages.size() > MAX_PACKAGES) { + throw new SpmManifestParsingException( + String.format("Swift package manifests declare more than %d packages in total", MAX_PACKAGES)); + } + } + + private static void checkProductCount(Iterable packages) throws SpmManifestParsingException { + int count = 0; + for (PackageRef ref : packages) { + count += ref.products.size(); + } + if (count > MAX_PRODUCTS) { + throw new SpmManifestParsingException( + String.format("Swift package manifests declare more than %d products in total", MAX_PRODUCTS)); + } + } + + private static String fieldNameFor(RequirementKind kind) { + switch (kind) { + case EXACT: return "version"; + case FROM: return "from"; + case BRANCH: return "branch"; + case REVISION: return "revision"; + default: throw new IllegalStateException("Unknown requirement kind " + kind); + } + } + + private static Pattern patternFor(RequirementKind kind) { + switch (kind) { + case EXACT: + case FROM: return VERSION_PATTERN; + case BRANCH: return BRANCH_PATTERN; + case REVISION: return REVISION_PATTERN; + default: throw new IllegalStateException("Unknown requirement kind " + kind); + } + } + + private static String textValue(JsonNode node, String field) throws SpmManifestParsingException { + JsonNode value = node.get(field); + if (value == null || !value.isTextual() || value.asString().isEmpty()) { + throw new SpmManifestParsingException( + String.format("Missing or invalid '%s' in Swift package manifest", field)); + } + return value.asString(); + } + + /** + * Validates a package URL and reconstructs it from its parsed parts. + * Only plain https URLs pointing at a repository path are accepted. + */ + static String sanitizeUrl(String rawUrl) throws SpmManifestParsingException { + if (rawUrl.length() > MAX_URL_LENGTH) { + throw new SpmManifestParsingException("Swift package URL is too long"); + } + URI uri; + try { + uri = new URI(rawUrl); + } catch (URISyntaxException e) { + throw new SpmManifestParsingException(String.format("Invalid Swift package URL: '%s'", rawUrl), e); + } + if (!"https".equals(uri.getScheme())) { + throw new SpmManifestParsingException( + String.format("Swift package URL must use the https scheme: '%s'", rawUrl)); + } + if (uri.getUserInfo() != null) { + throw new SpmManifestParsingException( + String.format("Swift package URL must not contain credentials: '%s'", rawUrl)); + } + if (uri.getPort() != -1 && uri.getPort() != 443) { + throw new SpmManifestParsingException( + String.format("Swift package URL must not use a custom port: '%s'", rawUrl)); + } + if (uri.getQuery() != null || uri.getFragment() != null) { + throw new SpmManifestParsingException( + String.format("Swift package URL must not contain a query or fragment: '%s'", rawUrl)); + } + String host = uri.getHost(); + if (host == null || !HOST_PATTERN.matcher(host).matches()) { + throw new SpmManifestParsingException( + String.format("Invalid host in Swift package URL: '%s'", rawUrl)); + } + String path = uri.getPath(); + if (path == null || path.length() < 2 || !PATH_PATTERN.matcher(path).matches()) { + throw new SpmManifestParsingException( + String.format("Invalid repository path in Swift package URL: '%s'", rawUrl)); + } + for (String segment : path.split("/")) { + if (segment.equals(".") || segment.equals("..")) { + throw new SpmManifestParsingException( + String.format("Invalid repository path in Swift package URL: '%s'", rawUrl)); + } + } + return "https://" + host + path; + } + + /** Canonical map key for a sanitized URL: lowercased host, ".git" suffix and trailing "/" stripped. */ + static String canonicalKey(String sanitizedUrl) { + String withoutScheme = sanitizedUrl.substring("https://".length()); + int slash = withoutScheme.indexOf('/'); + String host = withoutScheme.substring(0, slash).toLowerCase(); + String path = withoutScheme.substring(slash); + if (path.endsWith("/")) { + path = path.substring(0, path.length() - 1); + } + if (path.endsWith(".git")) { + path = path.substring(0, path.length() - 4); + } + return host + path; + } +} diff --git a/server/src/main/java/com/defold/extender/services/spm/SpmManifestParsingException.java b/server/src/main/java/com/defold/extender/services/spm/SpmManifestParsingException.java new file mode 100644 index 000000000..acfd81e70 --- /dev/null +++ b/server/src/main/java/com/defold/extender/services/spm/SpmManifestParsingException.java @@ -0,0 +1,13 @@ +package com.defold.extender.services.spm; + +import com.defold.extender.ExtenderException; + +public class SpmManifestParsingException extends ExtenderException { + public SpmManifestParsingException(String reason) { + super(reason); + } + + public SpmManifestParsingException(String reason, Exception cause) { + super(cause, reason); + } +} diff --git a/server/src/main/java/com/defold/extender/services/spm/SpmServiceBuildState.java b/server/src/main/java/com/defold/extender/services/spm/SpmServiceBuildState.java new file mode 100644 index 000000000..90031581b --- /dev/null +++ b/server/src/main/java/com/defold/extender/services/spm/SpmServiceBuildState.java @@ -0,0 +1,149 @@ +package com.defold.extender.services.spm; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import com.defold.extender.ExtenderBuildState; +import com.defold.extender.ExtenderUtil; +import com.defold.extender.services.cocoapods.PodUtils; + +/** + * Per-job working directory layout and platform mapping for the SPM wrapper build: + * + * /SwiftPackageManagerService/ + * Package/{Package.swift, Sources/SpmDeps/Empty.swift} # generated aggregator package + * Wrapper/{project.yml, Sources/Dummy.swift} # XcodeGen spec; xcodegen emits SpmWrapper.xcodeproj here + * DerivedData/ # per-job xcodebuild -derivedDataPath + * ModuleCache/ # per-job CLANG_MODULE_CACHE_PATH + * build.log # captured xcodebuild output + */ +public class SpmServiceBuildState { + + public static final String WRAPPER_NAME = "SpmWrapper"; + public static final String AGGREGATOR_NAME = "SpmDeps"; + static final String BUILD_CONFIGURATION = "Release"; + + File workingDir; + File packageDir; + File wrapperDir; + File derivedDataDir; + File moduleCacheDir; + File clonedSourcePackagesDir; + File buildLogFile; + PodUtils.Platform selectedPlatform; + String buildArch; + + SpmServiceBuildState() { } + + SpmServiceBuildState(ExtenderBuildState extenderBuildState) { + this.workingDir = new File(extenderBuildState.getJobDir(), "SwiftPackageManagerService"); + this.packageDir = new File(workingDir, "Package"); + this.wrapperDir = new File(workingDir, "Wrapper"); + this.derivedDataDir = new File(workingDir, "DerivedData"); + this.moduleCacheDir = new File(workingDir, "ModuleCache"); + this.clonedSourcePackagesDir = new File(workingDir, "clonedSourcePackages"); + this.buildLogFile = new File(workingDir, "build.log"); + new File(packageDir, "Sources/" + AGGREGATOR_NAME).mkdirs(); + new File(wrapperDir, "Sources").mkdirs(); + this.derivedDataDir.mkdirs(); + this.moduleCacheDir.mkdirs(); + this.clonedSourcePackagesDir.mkdirs(); + + this.buildArch = extenderBuildState.getBuildArch(); + this.selectedPlatform = PodUtils.Platform.UNKNOWN; + String platform = extenderBuildState.getBuildPlatform(); + if (ExtenderUtil.isIOSTarget(platform)) { + this.selectedPlatform = this.buildArch.equals("arm64") ? PodUtils.Platform.IPHONEOS : PodUtils.Platform.IPHONESIMULATOR; + } else if (ExtenderUtil.isMacOSTarget(platform)) { + this.selectedPlatform = PodUtils.Platform.MACOSX; + } + } + + public File getWorkingDir() { + return workingDir; + } + + public File getPackageDir() { + return packageDir; + } + + public File getWrapperDir() { + return wrapperDir; + } + + public File getDerivedDataDir() { + return derivedDataDir; + } + + public File getModuleCacheDir() { + return moduleCacheDir; + } + + public File getClonedSourcePackagesDir() { + return clonedSourcePackagesDir; + } + + public File getBuildLogFile() { + return buildLogFile; + } + + public PodUtils.Platform getSelectedPlatform() { + return selectedPlatform; + } + + public File getXcodeProjDir() { + return new File(wrapperDir, WRAPPER_NAME + ".xcodeproj"); + } + + /** Xcode's expected location of the wrapper project's Package.resolved lockfile. */ + public File getLockFile() { + return new File(getXcodeProjDir(), "project.xcworkspace/xcshareddata/swiftpm/Package.resolved"); + } + + public String getDestination() { + switch (selectedPlatform) { + case IPHONEOS: return "generic/platform=iOS"; + case IPHONESIMULATOR: return "generic/platform=iOS Simulator"; + case MACOSX: return "generic/platform=macOS"; + default: throw new IllegalStateException("Unsupported platform " + selectedPlatform); + } + } + + /** + * Explicit build settings pinning the build to the single architecture of this job. + * A generic iOS device build is arm64-only by default; simulator and macOS builds + * would otherwise emit every architecture of the host toolchain. + */ + public List getExtraBuildSettings() { + List settings = new ArrayList<>(); + if (selectedPlatform != PodUtils.Platform.IPHONEOS) { + settings.add("ARCHS=" + buildArch); + settings.add("ONLY_ACTIVE_ARCH=NO"); + } + return settings; + } + + String getProductsDirName() { + switch (selectedPlatform) { + case IPHONEOS: return BUILD_CONFIGURATION + "-iphoneos"; + case IPHONESIMULATOR: return BUILD_CONFIGURATION + "-iphonesimulator"; + case MACOSX: return BUILD_CONFIGURATION; + default: throw new IllegalStateException("Unsupported platform " + selectedPlatform); + } + } + + public File getProductsDir() { + return new File(derivedDataDir, "Build/Products/" + getProductsDirName()); + } + + /** The directory holding the generated .modulemap and -Swift.h files. */ + public File getGeneratedModuleMapsDir() { + File intermediates = new File(derivedDataDir, "Build/Intermediates.noindex"); + File withSuffix = new File(intermediates, "GeneratedModuleMaps-" + selectedPlatform.toString().toLowerCase()); + if (withSuffix.isDirectory()) { + return withSuffix; + } + return new File(intermediates, "GeneratedModuleMaps"); + } +} diff --git a/server/src/main/java/com/defold/extender/services/spm/SpmServiceConfiguration.java b/server/src/main/java/com/defold/extender/services/spm/SpmServiceConfiguration.java new file mode 100644 index 000000000..1ecc67bc6 --- /dev/null +++ b/server/src/main/java/com/defold/extender/services/spm/SpmServiceConfiguration.java @@ -0,0 +1,27 @@ +package com.defold.extender.services.spm; + +import java.util.HashMap; +import java.util.Map; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * The per-build Xcode routing table, a map @Value cannot bind. One instance serves builds + * pinned to different Xcode versions, so DEVELOPER_DIR is resolved per build from this map. + */ +@Component +@ConfigurationProperties(prefix = "extender.spm") +public class SpmServiceConfiguration { + + // Xcode version (e.g. "26.2") -> developer dir (e.g. /Applications/Xcode_26.2.app/Contents/Developer) + private Map xcodeDeveloperDirs = new HashMap<>(); + + public Map getXcodeDeveloperDirs() { + return xcodeDeveloperDirs; + } + + public void setXcodeDeveloperDirs(Map xcodeDeveloperDirs) { + this.xcodeDeveloperDirs = xcodeDeveloperDirs; + } +} diff --git a/server/src/main/java/com/defold/extender/services/spm/SwiftPackageManagerService.java b/server/src/main/java/com/defold/extender/services/spm/SwiftPackageManagerService.java new file mode 100644 index 000000000..de43a2e4b --- /dev/null +++ b/server/src/main/java/com/defold/extender/services/spm/SwiftPackageManagerService.java @@ -0,0 +1,582 @@ +package com.defold.extender.services.spm; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.InvalidPathException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.regex.Pattern; + +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.core.io.Resource; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import com.defold.extender.ExtenderBuildState; +import com.defold.extender.ExtenderException; +import com.defold.extender.ExtenderUtil; +import com.defold.extender.PlatformConfig; +import com.defold.extender.TemplateExecutor; +import com.defold.extender.metrics.MetricsWriter; +import com.defold.extender.process.ProcessExecutor; +import com.defold.extender.services.spm.SpmBuildOutputParser.LinkInfo; +import com.defold.extender.services.spm.SpmManifestParser.PackageRef; + +import io.micrometer.core.instrument.MeterRegistry; + +/** + * Resolves Swift Package Manager dependencies declared in SwiftPackages.json manifests: + * a generated aggregator Package.swift plus an XcodeGen wrapper project are built with + * xcodebuild, and the products directory and the wrapper's final link line are harvested + * into a {@link ResolvedPackages}. + * + * Xcode is selected per job via the child processes' DEVELOPER_DIR (never xcode-select — + * concurrent jobs may need different Xcode versions). + */ +@Service +@ConditionalOnProperty(prefix = "extender", name = "spm.enabled", havingValue = "true") +public class SwiftPackageManagerService { + + private static final Logger LOGGER = LoggerFactory.getLogger(SwiftPackageManagerService.class); + + static final String MANIFEST_FILENAME = "SwiftPackages.json"; + static final String LOCK_FILENAME = "Package.resolved"; + private static final String CURRENT_CACHE_DIR_FILE = "current_spm_cache.txt"; + private static final String OLD_CACHE_DIR_FILE = "old_spm_caches.txt"; + private static final String DEFAULT_CACHE_SUBDIR = "default"; + private static final Pattern UNSAFE_CACHE_SUBDIR_CHARS = Pattern.compile("[^A-Za-z0-9._-]"); + private static final int MAX_CACHE_SUBDIR_LENGTH = 64; + private static final String FALLBACK_IOS_MIN_VERSION = "12.0"; + private static final String FALLBACK_MACOS_MIN_VERSION = "10.15"; + + private final Object syncLock = new Object(); + // builds take the read lock (SwiftPM locks concurrent resolutions inside the cache + // itself), rotation and cleanup take the write lock + private final ReentrantReadWriteLock cacheLock = new ReentrantReadWriteLock(true); + private Path currentCacheDir = Path.of(""); + + private final String packageSwiftTemplateContents; + private final String projectYmlTemplateContents; + private final TemplateExecutor templateExecutor = new TemplateExecutor(); + private final MeterRegistry meterRegistry; + private final SpmServiceConfiguration spmConfiguration; + + // package-private so service-level tests can construct the service directly + @Value("${extender.spm.home-dir-prefix}") String homeDirPrefix; + @Value("${extender.spm.default-developer-dir}") String defaultDeveloperDir; + @Value("${extender.spm.xcodegen-path:xcodegen}") String xcodegenPath; + @Value("${extender.spm.wrapper-mach-o-type:staticlib}") String wrapperMachOType; + @Value("${extender.spm.swift-version:6.0}") String swiftVersion; + + SwiftPackageManagerService(@Value("classpath:template.package-swift") Resource packageSwiftTemplate, + @Value("classpath:template.project-yml") Resource projectYmlTemplate, + SpmServiceConfiguration spmConfiguration, + MeterRegistry meterRegistry) throws IOException { + this.meterRegistry = meterRegistry; + this.spmConfiguration = spmConfiguration; + this.packageSwiftTemplateContents = ExtenderUtil.readContentFromResource(packageSwiftTemplate); + this.projectYmlTemplateContents = ExtenderUtil.readContentFromResource(projectYmlTemplate); + } + + @EventListener(ApplicationReadyEvent.class) + public void runAfterStartup() { + ensureCacheDirInitialized(); + cleanupOldCacheDirectories(); + + if (!new File(defaultDeveloperDir).isDirectory()) { + LOGGER.warn("SPM default developer dir does not exist: {}", defaultDeveloperDir); + } + LOGGER.info("SPM startup task completed"); + } + + /** + * A build can reach the service before the ApplicationReadyEvent listener has run + * (listeners execute sequentially and earlier ones take seconds), so the cache dir + * must never be read from the raw field. + */ + private Path ensureCacheDirInitialized() { + synchronized (this.syncLock) { + if (!this.currentCacheDir.toString().isEmpty()) { + return this.currentCacheDir; + } + Path storedCacheDir = readCurrentCacheDir(); + if (storedCacheDir != null && storedCacheDir.startsWith(this.homeDirPrefix)) { + this.currentCacheDir = storedCacheDir; + } else { + LOGGER.info("SPM has no current cache dir or prefix is changed. Created..."); + Path newCacheDir = generateCacheDirPath(); + try { + Files.createDirectories(newCacheDir); + } catch (IOException | UnsupportedOperationException | SecurityException exc) { + LOGGER.warn("Cannot create SPM cache directory {}", newCacheDir, exc); + } + this.currentCacheDir = newCacheDir; + storeCurrentCacheDir(newCacheDir); + } + return this.currentCacheDir; + } + } + + private Map createJobEnvContext(Map env) { + Map context = new HashMap<>(env); + context.putIfAbsent("env.IOS_VERSION_MIN", System.getenv("IOS_VERSION_MIN")); + context.putIfAbsent("env.MACOS_VERSION_MIN", System.getenv("MACOS_VERSION_MIN")); + context.putIfAbsent("env.XCODE_VERSION", System.getenv("XCODE_VERSION")); + return context; + } + + /** Returns null when the upload declares no Swift package manifests. */ + public ResolvedPackages resolveDependencies(PlatformConfig config, ExtenderBuildState buildState) throws IOException, ExtenderException { + String platform = buildState.getBuildPlatform(); + if (!ExtenderUtil.isAppleTarget(platform)) { + throw new ExtenderException("Unsupported platform " + platform); + } + + Map jobEnvContext = createJobEnvContext(config.context); + File jobDir = buildState.getJobDir(); + + List allManifests = ExtenderUtil.listFilesMatchingRecursive(jobDir, MANIFEST_FILENAME); + List platformManifests = new ArrayList<>(); + for (File manifest : allManifests) { + String parentFolder = manifest.getParentFile().getName(); + if ((platform.contains("ios") && parentFolder.contains("ios")) || + (platform.contains("osx") && parentFolder.contains("osx"))) { + platformManifests.add(manifest); + } + else { + LOGGER.warn("Unexpected {} found in {}", MANIFEST_FILENAME, manifest); + } + } + if (platformManifests.isEmpty()) { + LOGGER.info("Project has no Swift package dependencies"); + return null; + } + + SpmServiceBuildState spmBuildState = new SpmServiceBuildState(buildState); + return resolveDependencies(platformManifests, spmBuildState, jobEnvContext, ExtenderUtil.isIOSTarget(platform)); + } + + ResolvedPackages resolveDependencies(List platformManifests, SpmServiceBuildState spmBuildState, + Map jobEnvContext, boolean isIOS) throws IOException, ExtenderException { + long methodStart = System.currentTimeMillis(); + LOGGER.info("Resolving Swift package dependencies"); + + String platformFamily = isIOS ? "ios" : "osx"; + String defaultMinVersion = defaultMinVersion(jobEnvContext, isIOS); + SpmManifestParser.ParseResult manifest = SpmManifestParser.parseManifests(platformManifests, platformFamily, defaultMinVersion); + + generateProjectFiles(spmBuildState, manifest, isIOS); + + String xcodeVersion = resolveXcodeVersion(jobEnvContext); + String developerDir = resolveDeveloperDir(xcodeVersion); + LOGGER.info("Building Swift packages with DEVELOPER_DIR={} (XCODE_VERSION={})", developerDir, xcodeVersion); + Map processEnv = hardenedProcessEnv(developerDir); + + cacheLock.readLock().lock(); + try { + // checkouts must be per job: SwiftPM prunes -clonedSourcePackagesDirPath to the + // current graph, so sharing it between jobs with different dependencies thrashes. + // The shared warmth is -packageCachePath (repo mirrors + binary artifacts). + File packageCacheDir = new File(sharedCacheDirFor(xcodeVersion), "packageCache"); + packageCacheDir.mkdirs(); + File clonedSourcesDir = spmBuildState.getClonedSourcePackagesDir(); + + generateXcodeProject(spmBuildState, processEnv); + copyUserLockFile(platformManifests, spmBuildState); + buildWrapperProject(spmBuildState, clonedSourcesDir, packageCacheDir, processEnv); + } finally { + cacheLock.readLock().unlock(); + } + + LinkInfo linkInfo = parseLinkInfo(spmBuildState); + File swiftRuntimeLibDir = new File(developerDir, "Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/" + + spmBuildState.getSelectedPlatform().toString().toLowerCase()); + ResolvedPackages resolvedPackages = ResolvedPackages.harvest(spmBuildState, manifest.minVersion, linkInfo, swiftRuntimeLibDir); + + MetricsWriter.metricsTimer(meterRegistry, "extender.service.spm.get", System.currentTimeMillis() - methodStart); + LOGGER.info("Resolved Swift package dependencies"); + LOGGER.info(resolvedPackages.toString()); + + return resolvedPackages; + } + + private String defaultMinVersion(Map jobEnvContext, boolean isIOS) { + Object minVersion = jobEnvContext.get(isIOS ? "env.IOS_VERSION_MIN" : "env.MACOS_VERSION_MIN"); + if (minVersion != null) { + return minVersion.toString(); + } + String fallback = isIOS ? FALLBACK_IOS_MIN_VERSION : FALLBACK_MACOS_MIN_VERSION; + LOGGER.warn("No platform min version in the job environment, using {}", fallback); + return fallback; + } + + private String resolveXcodeVersion(Map jobEnvContext) { + Object version = jobEnvContext.get("env.XCODE_VERSION"); + return version != null ? version.toString() : null; + } + + // resolved per build: one instance serves builds pinned to different Xcode versions + private String resolveDeveloperDir(String xcodeVersion) { + if (xcodeVersion != null) { + String developerDir = spmConfiguration.getXcodeDeveloperDirs().get(xcodeVersion); + if (developerDir != null) { + return developerDir; + } + if (!spmConfiguration.getXcodeDeveloperDirs().isEmpty()) { + LOGGER.warn("No Xcode developer dir configured for XCODE_VERSION={}, using default", xcodeVersion); + } + } + return defaultDeveloperDir; + } + + private Map hardenedProcessEnv(String developerDir) { + Map env = new HashMap<>(); + env.put("DEVELOPER_DIR", developerDir); + // strips every git credential helper so public packages clone anonymously with no + // keychain access; only effective together with xcodebuild -scmProvider system + env.put("GIT_CONFIG_NOSYSTEM", "1"); + env.put("GIT_CONFIG_GLOBAL", "/dev/null"); + env.put("GIT_TERMINAL_PROMPT", "0"); + env.put("GIT_ASKPASS", "/usr/bin/true"); + return env; + } + + void generateProjectFiles(SpmServiceBuildState buildState, SpmManifestParser.ParseResult manifest, boolean isIOS) throws IOException { + Files.writeString(new File(buildState.getPackageDir(), "Package.swift").toPath(), generatePackageSwift(manifest, isIOS)); + Files.writeString(new File(buildState.getPackageDir(), "Sources/" + SpmServiceBuildState.AGGREGATOR_NAME + "/Empty.swift").toPath(), + "// Intentionally empty. The aggregator target only re-exports the requested package products.\n"); + Files.writeString(new File(buildState.getWrapperDir(), "project.yml").toPath(), generateProjectYml(manifest, isIOS)); + Files.writeString(new File(buildState.getWrapperDir(), "Sources/Dummy.swift").toPath(), + "// Dummy source so the wrapper framework target is well-formed.\n" + + "public enum SpmWrapperMarker {}\n"); + } + + String generatePackageSwift(SpmManifestParser.ParseResult manifest, boolean isIOS) { + List> packages = new ArrayList<>(); + List> productDeps = new ArrayList<>(); + for (PackageRef ref : manifest.packages.values()) { + packages.add(Map.of("URL", ref.url, "REQUIREMENT", ref.requirement.toSwiftArgument())); + for (String product : ref.products) { + productDeps.add(Map.of("PRODUCT", product, "PACKAGE_LABEL", ref.label())); + } + } + Map context = new HashMap<>(); + context.put("AGGREGATOR_NAME", SpmServiceBuildState.AGGREGATOR_NAME); + context.put("SPM_PLATFORM", isIOS ? "iOS" : "macOS"); + context.put("PLATFORM_MIN_VERSION", manifest.minVersion); + context.put("PACKAGES", packages); + context.put("PRODUCT_DEPS", productDeps); + return templateExecutor.execute(packageSwiftTemplateContents, context); + } + + String generateProjectYml(SpmManifestParser.ParseResult manifest, boolean isIOS) { + Map context = new HashMap<>(); + context.put("WRAPPER_NAME", SpmServiceBuildState.WRAPPER_NAME); + context.put("AGGREGATOR_NAME", SpmServiceBuildState.AGGREGATOR_NAME); + context.put("XCODEGEN_PLATFORM", isIOS ? "iOS" : "macOS"); + context.put("PLATFORM_MIN_VERSION", manifest.minVersion); + context.put("SWIFT_VERSION", swiftVersion); + context.put("MACH_O_TYPE", machOTypeFor(manifest)); + return templateExecutor.execute(projectYmlTemplateContents, context); + } + + String machOTypeFor(SpmManifestParser.ParseResult manifest) { + if ("static".equals(manifest.wrapperType)) { + return "staticlib"; + } + if ("dynamic".equals(manifest.wrapperType)) { + return "mh_dylib"; + } + return wrapperMachOType; + } + + private void generateXcodeProject(SpmServiceBuildState buildState, Map processEnv) throws ExtenderException { + ProcessExecutor processExecutor = new ProcessExecutor(); + processExecutor.setCwd(buildState.getWrapperDir()); + processExecutor.putEnv(processEnv); + try { + processExecutor.execute(List.of(xcodegenPath, "generate", "--spec", "project.yml")); + } catch (IOException | InterruptedException e) { + throw new ExtenderException(e, "xcodegen generate failed:\n" + processExecutor.getOutput()); + } + } + + // an uploaded Package.resolved pins the graph; Xcode only reads it from inside the + // generated project's workspace + private void copyUserLockFile(List platformManifests, SpmServiceBuildState buildState) throws IOException { + List lockFiles = new ArrayList<>(); + for (File manifest : platformManifests) { + File candidate = new File(manifest.getParentFile(), LOCK_FILENAME); + if (candidate.isFile()) { + lockFiles.add(candidate); + } + } + if (lockFiles.isEmpty()) { + return; + } + // the directory-walk order is filesystem-dependent; the picked lock file must be + // the same on every build of the same upload + lockFiles.sort(Comparator.comparing(File::getAbsolutePath)); + File userLockFile = lockFiles.get(0); + if (lockFiles.size() > 1) { + LOGGER.warn("Multiple {} files uploaded {}, using {}", LOCK_FILENAME, lockFiles, userLockFile); + } + + File target = buildState.getLockFile(); + target.getParentFile().mkdirs(); + FileUtils.copyFile(userLockFile, target); + LOGGER.info("Using uploaded {} to pin Swift package versions", LOCK_FILENAME); + } + + private void buildWrapperProject(SpmServiceBuildState buildState, File clonedSourcesDir, File packageCacheDir, + Map processEnv) throws IOException, ExtenderException { + List args = new ArrayList<>(List.of( + "xcodebuild", + "-project", buildState.getXcodeProjDir().getAbsolutePath(), + "-scheme", SpmServiceBuildState.WRAPPER_NAME, + "-destination", buildState.getDestination(), + "-configuration", SpmServiceBuildState.BUILD_CONFIGURATION, + "-scmProvider", "system", + "-derivedDataPath", buildState.getDerivedDataDir().getAbsolutePath(), + "-clonedSourcePackagesDirPath", clonedSourcesDir.getAbsolutePath(), + "-packageCachePath", packageCacheDir.getAbsolutePath(), + "-skipPackagePluginValidation", + "CLANG_MODULE_CACHE_PATH=" + buildState.getModuleCacheDir().getAbsolutePath(), + "CODE_SIGNING_ALLOWED=NO")); + args.addAll(buildState.getExtraBuildSettings()); + args.add("build"); + + ProcessExecutor processExecutor = new ProcessExecutor(); + processExecutor.setCwd(buildState.getWrapperDir()); + processExecutor.putEnv(processEnv); + try { + processExecutor.execute(args); + } catch (IOException | InterruptedException e) { + writeBuildLog(buildState, processExecutor); + throw new ExtenderException(e, "Swift package build failed:\n" + processExecutor.getOutput()); + } + writeBuildLog(buildState, processExecutor); + } + + private void writeBuildLog(SpmServiceBuildState buildState, ProcessExecutor processExecutor) { + try { + processExecutor.writeLog(buildState.getBuildLogFile()); + } catch (IOException e) { + LOGGER.warn("Failed to write SPM build log", e); + } + } + + private LinkInfo parseLinkInfo(SpmServiceBuildState buildState) throws IOException { + String linkLine = SpmBuildOutputParser.findWrapperLinkLine(buildState.getBuildLogFile(), SpmServiceBuildState.WRAPPER_NAME); + if (linkLine == null) { + LOGGER.warn("No wrapper link line found in the SPM build log; system link dependencies are not harvested"); + return null; + } + // everything on a libtool -static line is an input merged INTO the wrapper archive, + // not an external link dependency — harvesting it would double-link or dangle + if (linkLine.contains("libtool") && linkLine.contains("-static")) { + return null; + } + // xcodebuild runs the link step from the wrapper dir; @ response files on the + // line resolve against it + return SpmBuildOutputParser.parseLinkLine(linkLine, buildState.getDerivedDataDir().getAbsolutePath(), + buildState.getWrapperDir()); + } + + // Shared cache layout: ///packageCache, keyed by + // Xcode version because SwiftPM state is not guaranteed compatible across Swift versions. + + private File sharedCacheDirFor(String xcodeVersion) { + return new File(ensureCacheDirInitialized().toFile(), cacheSubDirFor(xcodeVersion)); + } + + // XCODE_VERSION comes from the job's build.yml: a value carrying a separator would place + // the cache outside home-dir-prefix, where cleanup never reclaims it + static String cacheSubDirFor(String xcodeVersion) { + if (xcodeVersion == null) { + return DEFAULT_CACHE_SUBDIR; + } + String sanitized = UNSAFE_CACHE_SUBDIR_CHARS.matcher(xcodeVersion).replaceAll("_"); + if (sanitized.length() > MAX_CACHE_SUBDIR_LENGTH) { + sanitized = sanitized.substring(0, MAX_CACHE_SUBDIR_LENGTH); + } + // "." and ".." resolve to the cache root and its parent + boolean allDots = !sanitized.isEmpty() && sanitized.chars().allMatch(c -> c == '.'); + if (sanitized.isEmpty() || allDots) { + LOGGER.warn("XCODE_VERSION '{}' is not usable as a cache directory name, using '{}'", + xcodeVersion, DEFAULT_CACHE_SUBDIR); + return DEFAULT_CACHE_SUBDIR; + } + return sanitized; + } + + private Path generateCacheDirPath() { + return Path.of(this.homeDirPrefix, UUID.randomUUID().toString()); + } + + private Path readCurrentCacheDir() { + File currentCacheDirFile = Path.of(this.homeDirPrefix, CURRENT_CACHE_DIR_FILE).toFile(); + if (!currentCacheDirFile.exists()) { + return null; + } + try (BufferedReader reader = new BufferedReader(new FileReader(currentCacheDirFile))) { + String strPath = reader.readLine(); + if (strPath != null) { + Path result = Path.of(strPath); + return result.toFile().exists() ? result : null; + } + } catch (IOException io) { + LOGGER.warn("Exception while read current SPM cache path file", io); + } + return null; + } + + private void storeCurrentCacheDir(Path cacheDir) { + try { + Files.createDirectories(Path.of(this.homeDirPrefix)); + } catch (IOException exc) { + LOGGER.warn("Can't create directories to store SPM cache path", exc); + return; + } + try (FileWriter writer = new FileWriter(new File(this.homeDirPrefix, CURRENT_CACHE_DIR_FILE))) { + writer.append(cacheDir.toAbsolutePath().toString()); + } catch (IOException exc) { + LOGGER.warn("Error while writing to current SPM cache path file", exc); + } + } + + @Scheduled(cron = "${extender.spm.cache-dir-rotate-cron:0 20 2 * * *}") + public void rotateCacheDirectory() { + cacheLock.writeLock().lock(); + try { + rotateCacheDirectoryLocked(); + } finally { + cacheLock.writeLock().unlock(); + } + } + + private void rotateCacheDirectoryLocked() { + LOGGER.info("Rotate SPM cache directory"); + Path newCacheDir = generateCacheDirPath(); + Path cacheDir = ensureCacheDirInitialized(); + try { + Files.createDirectories(newCacheDir); + } catch (IOException | UnsupportedOperationException | SecurityException exc) { + LOGGER.warn("Cannot create new SPM cache directory", exc); + return; + } + + try (FileWriter writer = new FileWriter(new File(this.homeDirPrefix, OLD_CACHE_DIR_FILE), true)) { + writer.append(cacheDir.toAbsolutePath().toString()); + writer.append("\n"); + } catch (IOException exc) { + LOGGER.warn("Error while writing to old SPM cache paths file", exc); + } + synchronized (this.syncLock) { + this.currentCacheDir = newCacheDir; + storeCurrentCacheDir(newCacheDir); + } + } + + @Scheduled(cron = "${extender.spm.old-cache-clean-cron:0 20 6 * * *}") + public void cleanupOldCacheDirectories() { + cacheLock.writeLock().lock(); + try { + cleanupOldCacheDirectoriesLocked(); + } finally { + cacheLock.writeLock().unlock(); + } + } + + private void cleanupOldCacheDirectoriesLocked() { + LOGGER.info("Cleanup old SPM cache directories"); + Path activeCacheDir = ensureCacheDirInitialized().toAbsolutePath().normalize(); + File oldDirFile = Path.of(this.homeDirPrefix, OLD_CACHE_DIR_FILE).toFile(); + if (!oldDirFile.exists()) { + return; + } + + List retained = new ArrayList<>(); + try (BufferedReader reader = new BufferedReader(new FileReader(oldDirFile))) { + String strPath = reader.readLine(); + while (strPath != null) { + String path = strPath.trim(); + if (!path.isEmpty() && !removeOldCacheDirectory(path, activeCacheDir)) { + retained.add(path); + } + strPath = reader.readLine(); + } + } catch (IOException io) { + // leave the file untouched so that nothing is forgotten + LOGGER.warn("Exception while read old SPM cache paths file", io); + return; + } + + if (retained.isEmpty()) { + oldDirFile.delete(); + } else { + // keep the directories we failed to remove so the next run retries them, instead of + // dropping the whole file and leaking them for good + LOGGER.warn("{} old SPM cache directories could not be removed and will be retried", retained.size()); + try (FileWriter writer = new FileWriter(oldDirFile, false)) { + for (String path : retained) { + writer.append(path); + writer.append("\n"); + } + } catch (IOException exc) { + LOGGER.warn("Error while rewriting old SPM cache paths file", exc); + } + } + } + + /** + * Remove a single old SPM cache directory. + * @param strPath Path of the directory to remove + * @param activeCacheDir The cache directory currently in use, which must never be removed + * @return true if the directory is gone or should not be retried, false if removal failed + */ + private boolean removeOldCacheDirectory(String strPath, Path activeCacheDir) { + Path dirPath; + try { + dirPath = Path.of(strPath); + } catch (InvalidPathException exc) { + LOGGER.warn("Ignoring malformed old SPM cache path {}", strPath, exc); + return true; + } + // never delete the directory builds are currently using. It gets appended to the file + // again by the next rotation, so it is safe to drop it from the list here. + if (dirPath.toAbsolutePath().normalize().equals(activeCacheDir)) { + LOGGER.warn("Skip removal of SPM cache directory {}, it is the current one", strPath); + return true; + } + File path = dirPath.toFile(); + if (!path.exists()) { + return true; + } + LOGGER.info("Remove old SPM cache directory: {}", strPath); + try { + FileUtils.deleteDirectory(path); + return true; + } catch (IOException exc) { + // a single failure must not abort the cleanup of the remaining directories + LOGGER.warn("Unable to remove old SPM cache directory {}", strPath, exc); + return false; + } + } +} diff --git a/server/src/main/java/com/defold/extender/utils/VersionUtil.java b/server/src/main/java/com/defold/extender/utils/VersionUtil.java new file mode 100644 index 000000000..ad036ec73 --- /dev/null +++ b/server/src/main/java/com/defold/extender/utils/VersionUtil.java @@ -0,0 +1,44 @@ +package com.defold.extender.utils; + +import com.defold.extender.ExtenderException; + +public class VersionUtil { + + // Strip semver pre-release and build metadata (everything from the first '-' or '+'), + // e.g. "1.0.0-beta" -> "1.0.0", "2.0.0+build.7" -> "2.0.0". Comparison ignores these segments. + private static String stripVersionSuffix(String version) { + int cut = version.length(); + for (int i = 0; i < version.length(); i++) { + char c = version.charAt(i); + if (c == '-' || c == '+') { + cut = i; + break; + } + } + return version.substring(0, cut); + } + + public static int compareVersions(String version1, String version2) throws ExtenderException { + int result = 0; + String[] parts1 = stripVersionSuffix(version1).split("\\."); + String[] parts2 = stripVersionSuffix(version2).split("\\."); + int length = Math.max(parts1.length, parts2.length); + for (int i = 0; i < length; i++) { + try { + int v1 = i < parts1.length && !parts1[i].isEmpty() ? Integer.parseInt(parts1[i]) : 0; + int v2 = i < parts2.length && !parts2[i].isEmpty() ? Integer.parseInt(parts2[i]) : 0; + if (v1 < v2) { + result = -1; + break; + } else if (v1 > v2) { + result = 1; + break; + } + } catch (NumberFormatException exc) { + throw new ExtenderException(exc, + String.format("Failed to compare versions '%s' and '%s'", version1, version2)); + } + } + return result; + } +} diff --git a/server/src/main/resources/application.yml b/server/src/main/resources/application.yml index 871666d49..515865ff3 100644 --- a/server/src/main/resources/application.yml +++ b/server/src/main/resources/application.yml @@ -45,6 +45,19 @@ extender: repo-update-cron: "0 0 * * * *" # update spec repo every 1 h cache-dir-rotate-cron: "0 10 2 * * *" # once per day old-cache-clean-cron: "0 10 6 * * *" # once per day after directory rotation + spm: + enabled: false + home-dir-prefix: /tmp/.swiftpm + # DEVELOPER_DIR is selected per build: the job's XCODE_VERSION is looked up in + # the (optional) xcode-developer-dirs map, falling back to default-developer-dir: + # xcode-developer-dirs: + # "26.2": /Applications/Xcode_26.2.app/Contents/Developer + default-developer-dir: /Applications/Xcode.app/Contents/Developer + xcodegen-path: xcodegen + wrapper-mach-o-type: staticlib # staticlib | mh_dylib + swift-version: "6.0" + cache-dir-rotate-cron: "0 20 2 * * *" # once per day + old-cache-clean-cron: "0 20 6 * * *" # once per day after directory rotation # refer to README_SECURITY.md for information on securing your server authentication: # empty string, all platforms allowed without authentication diff --git a/server/src/main/resources/template.package-swift b/server/src/main/resources/template.package-swift new file mode 100644 index 000000000..476240832 --- /dev/null +++ b/server/src/main/resources/template.package-swift @@ -0,0 +1,33 @@ +// swift-tools-version: 5.9 +// Generated by Extender from the uploaded SwiftPackages.json manifests. +import PackageDescription + +let package = Package( + name: "{{AGGREGATOR_NAME}}", + platforms: [ + .{{SPM_PLATFORM}}("{{PLATFORM_MIN_VERSION}}") + ], + products: [ + .library( + name: "{{AGGREGATOR_NAME}}", + type: .static, + targets: ["{{AGGREGATOR_NAME}}"] + ) + ], + dependencies: [ +{{#PACKAGES}} + .package(url: "{{URL}}", {{{REQUIREMENT}}}), +{{/PACKAGES}} + ], + targets: [ + .target( + name: "{{AGGREGATOR_NAME}}", + dependencies: [ +{{#PRODUCT_DEPS}} + .product(name: "{{PRODUCT}}", package: "{{PACKAGE_LABEL}}"), +{{/PRODUCT_DEPS}} + ], + path: "Sources/{{AGGREGATOR_NAME}}" + ) + ] +) diff --git a/server/src/main/resources/template.project-yml b/server/src/main/resources/template.project-yml new file mode 100644 index 000000000..8c0ef01d0 --- /dev/null +++ b/server/src/main/resources/template.project-yml @@ -0,0 +1,26 @@ +# Generated by Extender. XcodeGen spec for the wrapper project whose single framework +# target links the aggregator package product. +name: {{WRAPPER_NAME}} +options: + bundleIdPrefix: com.defold.extender + deploymentTarget: + {{XCODEGEN_PLATFORM}}: "{{PLATFORM_MIN_VERSION}}" + createIntermediateGroups: true +packages: + {{AGGREGATOR_NAME}}: + path: ../Package +targets: + {{WRAPPER_NAME}}: + type: framework + platform: {{XCODEGEN_PLATFORM}} + sources: + - path: Sources + settings: + base: + CODE_SIGNING_ALLOWED: "NO" + SWIFT_VERSION: "{{SWIFT_VERSION}}" + MACH_O_TYPE: {{MACH_O_TYPE}} + GENERATE_INFOPLIST_FILE: "YES" + dependencies: + - package: {{AGGREGATOR_NAME}} + product: {{AGGREGATOR_NAME}} diff --git a/server/src/test/java/com/defold/extender/services/spm/ResolvedPackagesTest.java b/server/src/test/java/com/defold/extender/services/spm/ResolvedPackagesTest.java new file mode 100644 index 000000000..631ca24a3 --- /dev/null +++ b/server/src/test/java/com/defold/extender/services/spm/ResolvedPackagesTest.java @@ -0,0 +1,209 @@ +package com.defold.extender.services.spm; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.file.Files; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.condition.EnabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.api.Test; + +import com.defold.extender.ExtenderException; +import com.defold.extender.services.cocoapods.PodUtils; +import com.defold.extender.services.spm.SpmBuildOutputParser.LinkInfo; + +// classification of framework binaries shells out to file(1) with Mach-O magic bytes, +// which is only reliable on a Mac +@EnabledOnOs({OS.MAC}) +public class ResolvedPackagesTest { + + private SpmServiceBuildState createBuildState(File rootDir) { + SpmServiceBuildState buildState = new SpmServiceBuildState(); + buildState.workingDir = rootDir; + buildState.packageDir = new File(rootDir, "Package"); + buildState.wrapperDir = new File(rootDir, "Wrapper"); + buildState.derivedDataDir = new File(rootDir, "DerivedData"); + buildState.moduleCacheDir = new File(rootDir, "ModuleCache"); + buildState.buildLogFile = new File(rootDir, "build.log"); + buildState.selectedPlatform = PodUtils.Platform.IPHONEOS; + buildState.buildArch = "arm64"; + return buildState; + } + + // minimal Mach-O 64-bit dylib header; file(1) reports it as a dynamically linked shared library + private static void writeDylibBinary(File file) throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(32).order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt(0xFEEDFACF); // MH_MAGIC_64 + buffer.putInt(0x0100000C); // CPU_TYPE_ARM64 + buffer.putInt(0); // cpusubtype + buffer.putInt(6); // MH_DYLIB + buffer.putInt(0).putInt(0).putInt(0).putInt(0); + try (FileOutputStream out = new FileOutputStream(file)) { + out.write(buffer.array()); + } + } + + private static void writeStaticArchiveBinary(File file) throws IOException { + Files.writeString(file.toPath(), "!\n"); + } + + private static File createFramework(File productsDir, String name, boolean dynamic) throws IOException { + File framework = new File(productsDir, name + ".framework"); + framework.mkdirs(); + File binary = new File(framework, name); + if (dynamic) { + writeDylibBinary(binary); + } else { + writeStaticArchiveBinary(binary); + } + return framework; + } + + private void createCommonProducts(SpmServiceBuildState buildState) throws IOException { + File productsDir = buildState.getProductsDir(); + productsDir.mkdirs(); + + createFramework(productsDir, "FirebaseAnalytics", false); + File sentry = createFramework(productsDir, "Sentry", true); + Files.writeString(new File(sentry, "PrivacyInfo.xcprivacy").toPath(), ""); + + File bundle = new File(productsDir, "Firebase_FirebaseCore.bundle"); + bundle.mkdirs(); + Files.writeString(new File(bundle, "PrivacyInfo.xcprivacy").toPath(), ""); + + File moduleMapsDir = new File(buildState.getDerivedDataDir(), "Build/Intermediates.noindex/GeneratedModuleMaps-iphoneos"); + moduleMapsDir.mkdirs(); + + File lockFile = buildState.getLockFile(); + lockFile.getParentFile().mkdirs(); + Files.writeString(lockFile.toPath(), "{}"); + } + + private static LinkInfo linkInfo() { + return new LinkInfo( + List.of("c++", "z", "sqlite3"), + List.of("UIKit", "Security", "Sentry"), + List.of("/usr/lib/swift"), + List.of("/usr/lib/swift", "@executable_path/Frameworks"), + List.of()); + } + + @Test + public void testHarvestDynamicWrapper(@TempDir File rootDir) throws IOException, ExtenderException { + SpmServiceBuildState buildState = createBuildState(rootDir); + createCommonProducts(buildState); + createFramework(buildState.getProductsDir(), "SpmWrapper", true); + File swiftLibDir = new File(rootDir, "swift-runtime/iphoneos"); + swiftLibDir.mkdirs(); + + ResolvedPackages resolved = ResolvedPackages.harvest(buildState, "15.0", linkInfo(), swiftLibDir); + + // products-dir frameworks first (sorted), then link-line names deduplicated + assertEquals(List.of("FirebaseAnalytics", "Sentry", "SpmWrapper", "UIKit", "Security"), resolved.getFrameworks()); + assertEquals(List.of(buildState.getProductsDir().getAbsolutePath()), resolved.getFrameworksSearchPaths()); + assertEquals(List.of("c++", "z", "sqlite3"), resolved.getStaticLibraries()); + // link-line paths first, then the derived toolchain swift runtime dir + assertEquals(List.of("/usr/lib/swift", swiftLibDir.getAbsolutePath()), resolved.getLibrarySearchPaths()); + + // the dylib wrapper and the dylib dependency get embedded; the static framework does not + assertEquals(2, resolved.getDynamicFrameworks().size()); + assertTrue(resolved.getDynamicFrameworks().stream().anyMatch(f -> f.getName().equals("SpmWrapper.framework"))); + assertTrue(resolved.getDynamicFrameworks().stream().anyMatch(f -> f.getName().equals("Sentry.framework"))); + assertEquals(List.of("-Wl,-rpath,/usr/lib/swift", "-Wl,-rpath,@executable_path/Frameworks"), resolved.getLinkFlags()); + + assertEquals(1, resolved.getResources().size()); + assertEquals("Firebase_FirebaseCore.bundle", resolved.getResources().get(0).getName()); + assertEquals(2, resolved.getPrivacyManifests().size()); + + assertEquals(List.of( + new File(buildState.getDerivedDataDir(), "Build/Intermediates.noindex/GeneratedModuleMaps-iphoneos").getAbsolutePath(), + buildState.getProductsDir().getAbsolutePath()), + resolved.getAdditionalIncludePaths()); + + assertNotNull(resolved.getLockFile()); + assertEquals("15.0", resolved.getPlatformMinVersion()); + assertTrue(resolved.getBuiltFrameworks().isEmpty()); + assertTrue(resolved.getWeakFrameworks().isEmpty()); + } + + @Test + public void testHarvestStaticWrapper(@TempDir File rootDir) throws IOException, ExtenderException { + SpmServiceBuildState buildState = createBuildState(rootDir); + createCommonProducts(buildState); + createFramework(buildState.getProductsDir(), "SpmWrapper", false); + + ResolvedPackages resolved = ResolvedPackages.harvest(buildState, "15.0", linkInfo(), null); + + // static wrapper is linked, not embedded; only the dylib dependency gets embedded + assertTrue(resolved.getFrameworks().contains("SpmWrapper")); + assertEquals(1, resolved.getDynamicFrameworks().size()); + assertEquals("Sentry.framework", resolved.getDynamicFrameworks().get(0).getName()); + } + + @Test + public void testCollectModuleMapIncludePaths(@TempDir File rootDir) throws IOException { + File maps = new File(rootDir, "GeneratedModuleMaps-iphoneos"); + maps.mkdirs(); + + // Firebase-style layout: umbrella at .../Public/FirebaseCore/FirebaseCore.h -> + // the Public dir serves #import directly + File nestedHeaders = new File(rootDir, "checkouts/firebase/Sources/Public/FirebaseCore"); + nestedHeaders.mkdirs(); + Files.writeString(new File(nestedHeaders, "FirebaseCore.h").toPath(), "//"); + Files.writeString(new File(maps, "FirebaseCore.modulemap").toPath(), + "module FirebaseCore {\numbrella header \"" + new File(nestedHeaders, "FirebaseCore.h").getAbsolutePath() + "\"\nexport *\n}"); + + // flat layout: headers directly in Public/ -> synthetic symlink dir + File flatHeaders = new File(rootDir, "checkouts/sentry/Public"); + flatHeaders.mkdirs(); + Files.writeString(new File(flatHeaders, "Sentry.h").toPath(), "//"); + Files.writeString(new File(maps, "Sentry.modulemap").toPath(), + "module Sentry {\numbrella header \"" + new File(flatHeaders, "Sentry.h").getAbsolutePath() + "\"\n}"); + + // no umbrella -> ignored + Files.writeString(new File(maps, "Odd.modulemap").toPath(), "module Odd { header \"X.h\" }"); + + Set includePaths = new LinkedHashSet<>(); + File syntheticDir = new File(rootDir, "include"); + ResolvedPackages.collectModuleMapIncludePaths(maps, syntheticDir, includePaths); + + assertTrue(includePaths.contains(nestedHeaders.getParentFile().getAbsolutePath())); + // the mismatch case exposes both the headers dir itself and the module-name symlink + assertTrue(includePaths.contains(flatHeaders.getAbsolutePath())); + assertTrue(includePaths.contains(syntheticDir.getAbsolutePath())); + assertTrue(Files.isSymbolicLink(new File(syntheticDir, "Sentry").toPath())); + assertTrue(new File(syntheticDir, "Sentry/Sentry.h").isFile()); + assertEquals(3, includePaths.size()); + } + + @Test + public void testHarvestWithoutLinkInfoAndLockFile(@TempDir File rootDir) throws IOException, ExtenderException { + SpmServiceBuildState buildState = createBuildState(rootDir); + File productsDir = buildState.getProductsDir(); + productsDir.mkdirs(); + createFramework(productsDir, "SpmWrapper", false); + + ResolvedPackages resolved = ResolvedPackages.harvest(buildState, "15.0", null, null); + + assertEquals(List.of("SpmWrapper"), resolved.getFrameworks()); + assertTrue(resolved.getStaticLibraries().isEmpty()); + assertTrue(resolved.getLibrarySearchPaths().isEmpty()); + assertTrue(resolved.getDynamicFrameworks().isEmpty()); + assertTrue(resolved.getLinkFlags().isEmpty()); + assertNull(resolved.getLockFile()); + // no GeneratedModuleMaps dir was created; only the products dir remains + assertEquals(List.of(productsDir.getAbsolutePath()), resolved.getAdditionalIncludePaths()); + } +} diff --git a/server/src/test/java/com/defold/extender/services/spm/SpmBuildOutputParserTest.java b/server/src/test/java/com/defold/extender/services/spm/SpmBuildOutputParserTest.java new file mode 100644 index 000000000..17e588b6d --- /dev/null +++ b/server/src/test/java/com/defold/extender/services/spm/SpmBuildOutputParserTest.java @@ -0,0 +1,189 @@ +package com.defold.extender.services.spm; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.List; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.defold.extender.services.spm.SpmBuildOutputParser.LinkInfo; + +public class SpmBuildOutputParserTest { + + private static final String DERIVED_DATA = "/tmp/job123/SwiftPackageManagerService/DerivedData"; + + private String dynamicLog; + + @BeforeEach + public void setUp() throws IOException { + this.dynamicLog = Files.readString(new File("test-data/spm-buildlogs/dynamic-ios.log").toPath()); + } + + @Test + public void testFindWrapperLinkLine() { + String line = SpmBuildOutputParser.findWrapperLinkLine(dynamicLog, "SpmWrapper"); + assertNotNull(line); + assertTrue(line.contains("-dynamiclib")); + // the relocatable package prelink (clang -r -o .../Firebase.o) must not match + assertTrue(line.contains("SpmWrapper.framework/SpmWrapper")); + assertNull(SpmBuildOutputParser.findWrapperLinkLine(dynamicLog, "OtherWrapper")); + // the dynamic log has no libtool step + assertNull(SpmBuildOutputParser.findWrapperLibtoolLine(dynamicLog, "SpmWrapper")); + } + + @Test + public void testParseLinkLine() { + String line = SpmBuildOutputParser.findWrapperLinkLine(dynamicLog, "SpmWrapper"); + LinkInfo info = SpmBuildOutputParser.parseLinkLine(line, DERIVED_DATA); + + // duplicated -l flags collapse, order of first appearance preserved + assertEquals(List.of("c++", "z", "sqlite3"), info.systemLibs()); + + // duplicated -framework flags collapse; system + binary dependency frameworks + assertEquals(List.of( + "Accelerate", "UIKit", "Security", "StoreKit", + "FirebaseAnalytics", "GoogleAppMeasurementIdentitySupport", "GoogleAppMeasurement", + "GoogleAdsOnDeviceConversion", "FBSDKCoreKit", "FBAEMKit", "FBSDKCoreKit_Basics", "Sentry"), + info.frameworkNames()); + + // job-local DerivedData search paths dropped, Swift runtime paths kept + assertEquals(List.of( + "/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/iphoneos", + "/usr/lib/swift"), + info.librarySearchPaths()); + + // -Xlinker -rpath -Xlinker pairs collapse to their values + assertEquals(List.of("/usr/lib/swift", "@executable_path/Frameworks"), info.rpaths()); + + assertTrue(info.forceLoad().isEmpty()); + } + + @Test + public void testParseLinkLineGuards() { + // -lto_library is an ld flag, not a library, whether or not it is wrapped in -Xlinker; + // a missing response file is reported and skipped, never parsed as a library + String line = "clang -dynamiclib -Xlinker -lto_library -Xlinker /x/libLTO.dylib " + + "-lto_library /x/libLTO.dylib @/x/does-not-exist-linker-args.resp " + + "-lc++ -Wl,-rpath,/usr/lib/swift " + + "-force_load /x/libFoo.a -o /x/SpmWrapper.framework/SpmWrapper"; + LinkInfo info = SpmBuildOutputParser.parseLinkLine(line, DERIVED_DATA); + assertEquals(List.of("c++"), info.systemLibs()); + assertEquals(List.of("/usr/lib/swift"), info.rpaths()); + assertEquals(List.of("/x/libFoo.a"), info.forceLoad()); + } + + @Test + public void testParseLinkLineExpandsResponseFile(@TempDir File tmpDir) throws IOException { + // Xcode spills long link commands into a response file; the flags inside are part of + // the link line and are dropped from the engine link unless the @ is expanded + File nested = new File(tmpDir, "nested.resp"); + Files.writeString(nested.toPath(), "-framework Security\n-lz\n"); + File responseFile = new File(tmpDir, "SpmWrapper-linker-args.resp"); + Files.writeString(responseFile.toPath(), + "-framework FirebaseAnalytics\n" + + "'-framework' 'FBSDKCoreKit'\n" + + "-lsqlite3 -L/usr/lib/swift -L" + DERIVED_DATA + "/Build/Products\n" + + "@nested.resp\n"); + + String line = "clang -dynamiclib -lc++ @" + responseFile.getAbsolutePath() + + " -o /x/SpmWrapper.framework/SpmWrapper"; + LinkInfo info = SpmBuildOutputParser.parseLinkLine(line, DERIVED_DATA); + + assertEquals(List.of("c++", "sqlite3", "z"), info.systemLibs()); + assertEquals(List.of("FirebaseAnalytics", "FBSDKCoreKit", "Security"), info.frameworkNames()); + // DerivedData search paths are dropped inside a response file too + assertEquals(List.of("/usr/lib/swift"), info.librarySearchPaths()); + } + + @Test + public void testParseLinkLineResolvesRelativeResponseFile(@TempDir File tmpDir) throws IOException { + File responseFile = new File(tmpDir, "args.resp"); + Files.writeString(responseFile.toPath(), "-framework StoreKit\n"); + LinkInfo info = SpmBuildOutputParser.parseLinkLine( + "clang -dynamiclib @args.resp -o /x/SpmWrapper.framework/SpmWrapper", DERIVED_DATA, tmpDir); + assertEquals(List.of("StoreKit"), info.frameworkNames()); + } + + @Test + public void testParseLinkLineKeepsDyldPlaceholders() { + // @rpath/@executable_path/@loader_path are path values, not response files + LinkInfo info = SpmBuildOutputParser.parseLinkLine( + "clang -dynamiclib -install_name @rpath/SpmWrapper.framework/SpmWrapper " + + "-Xlinker -rpath -Xlinker @executable_path/Frameworks -o /x/SpmWrapper.framework/SpmWrapper", + DERIVED_DATA); + assertEquals(List.of("@executable_path/Frameworks"), info.rpaths()); + } + + @Test + public void testFindWrapperLinkLineStreamed() throws IOException { + // production reads the log from disk one line at a time; a dynamic wrapper's + // clang -dynamiclib line wins, a static wrapper falls back to its libtool line + File dynamicLogFile = new File("test-data/spm-buildlogs/dynamic-ios.log"); + String line = SpmBuildOutputParser.findWrapperLinkLine(dynamicLogFile, "SpmWrapper"); + assertNotNull(line); + assertTrue(line.contains("-dynamiclib")); + assertNull(SpmBuildOutputParser.findWrapperLinkLine(dynamicLogFile, "OtherWrapper")); + + File staticLogFile = new File("test-data/spm-buildlogs/static-ios.log"); + String libtoolLine = SpmBuildOutputParser.findWrapperLinkLine(staticLogFile, "SpmWrapper"); + assertNotNull(libtoolLine); + assertTrue(libtoolLine.contains("libtool")); + assertTrue(libtoolLine.contains("SpmWrapper.framework/SpmWrapper")); + } + + @Test + public void testParseModuleMaps() { + List moduleMaps = SpmBuildOutputParser.parseModuleMaps(dynamicLog); + // values appear single-quoted in the log; quotes must not leak into the paths + assertEquals(List.of( + DERIVED_DATA + "/Build/Intermediates.noindex/GeneratedModuleMaps-iphoneos/GoogleUtilities-Logger.modulemap", + DERIVED_DATA + "/Build/Intermediates.noindex/GeneratedModuleMaps-iphoneos/GoogleUtilities-Environment.modulemap", + DERIVED_DATA + "/Build/Intermediates.noindex/GeneratedModuleMaps-iphoneos/third-party-IsAppEncrypted.modulemap"), + moduleMaps); + } + + @Test + public void testFindWrapperLibtoolLine() throws IOException { + // real log of a MACH_O_TYPE=staticlib build: package targets are archived with + // libtool too (-o .o), only the wrapper's line may match + String staticLog = Files.readString(new File("test-data/spm-buildlogs/static-ios.log").toPath()); + assertNull(SpmBuildOutputParser.findWrapperLinkLine(staticLog, "SpmWrapper")); + + String line = SpmBuildOutputParser.findWrapperLibtoolLine(staticLog, "SpmWrapper"); + assertNotNull(line); + assertTrue(line.contains("SpmWrapper.framework/SpmWrapper")); + + // a libtool archive line carries no system link dependencies, and its DerivedData + // search paths are dropped + LinkInfo info = SpmBuildOutputParser.parseLinkLine(line, DERIVED_DATA); + assertTrue(info.systemLibs().isEmpty()); + assertTrue(info.frameworkNames().isEmpty()); + assertTrue(info.librarySearchPaths().isEmpty()); + assertTrue(info.rpaths().isEmpty()); + } + + @Test + public void testFindWrapperLibtoolLineVersionedFramework() { + // macOS framework bundles are versioned: the binary is .framework/Versions/A/ + String log = " /x/usr/bin/libtool -static -arch_only arm64 -D -syslibroot /sdk" + + " -filelist /x/SpmWrapper.LinkFileList" + + " -o /x/Build/Products/Release/SpmWrapper.framework/Versions/A/SpmWrapper\n"; + assertNotNull(SpmBuildOutputParser.findWrapperLibtoolLine(log, "SpmWrapper")); + assertNull(SpmBuildOutputParser.findWrapperLibtoolLine(log, "OtherWrapper")); + } + + @Test + public void testReadLinkFileList(@TempDir File tmpDir) throws IOException { + File listFile = new File(tmpDir, "SpmWrapper.LinkFileList"); + Files.writeString(listFile.toPath(), "/a/Dummy.o\n\n/b/Firebase.o\n"); + assertEquals(List.of("/a/Dummy.o", "/b/Firebase.o"), SpmBuildOutputParser.readLinkFileList(listFile)); + } +} diff --git a/server/src/test/java/com/defold/extender/services/spm/SpmManifestParserTest.java b/server/src/test/java/com/defold/extender/services/spm/SpmManifestParserTest.java new file mode 100644 index 000000000..abe67b6b3 --- /dev/null +++ b/server/src/test/java/com/defold/extender/services/spm/SpmManifestParserTest.java @@ -0,0 +1,226 @@ +package com.defold.extender.services.spm; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import com.defold.extender.services.spm.SpmManifestParser.PackageRef; +import com.defold.extender.services.spm.SpmManifestParser.ParseResult; +import com.defold.extender.services.spm.SpmManifestParser.RequirementKind; + +public class SpmManifestParserTest { + + private static File fixture(String name) { + return new File("test-data/swiftpackages/" + name); + } + + @Test + public void testRegularIosManifest() throws IOException, SpmManifestParsingException { + ParseResult result = SpmManifestParser.parseManifests(List.of(fixture("regular_ios.json")), "ios", "11.0"); + assertEquals("ios", result.platform); + assertEquals("13.0", result.minVersion); + assertEquals(2, result.packages.size()); + + PackageRef firebase = result.packages.get("github.com/firebase/firebase-ios-sdk"); + assertNotNull(firebase); + assertEquals("https://github.com/firebase/firebase-ios-sdk.git", firebase.url); + assertEquals("firebase-ios-sdk", firebase.label()); + assertEquals(RequirementKind.EXACT, firebase.requirement.kind); + assertEquals("exact: \"12.0.0\"", firebase.requirement.toSwiftArgument()); + assertEquals(Set.of("FirebaseAnalytics", "FirebaseRemoteConfig"), firebase.products); + + PackageRef sentry = result.packages.get("github.com/getsentry/sentry-cocoa"); + assertNotNull(sentry); + assertEquals(RequirementKind.FROM, sentry.requirement.kind); + assertEquals("from: \"9.0.0\"", sentry.requirement.toSwiftArgument()); + } + + @Test + public void testRegularOsxManifest() throws IOException, SpmManifestParsingException { + ParseResult result = SpmManifestParser.parseManifests(List.of(fixture("regular_osx.json")), "osx", "10.15"); + assertEquals("osx", result.platform); + assertEquals("11.0", result.minVersion); + PackageRef ref = result.packages.values().iterator().next(); + assertEquals("swift-argument-parser", ref.label()); + } + + @Test + public void testBranchAndRevisionRequirements() throws IOException, SpmManifestParsingException { + ParseResult branch = SpmManifestParser.parseManifest(fixture("branch.json")); + assertEquals("branch: \"release/1.1\"", branch.packages.values().iterator().next().requirement.toSwiftArgument()); + + ParseResult revision = SpmManifestParser.parseManifest(fixture("revision.json")); + assertEquals("revision: \"94cf62b3ba8d4bed62680341ce53483da1d0c1a7\"", + revision.packages.values().iterator().next().requirement.toSwiftArgument()); + } + + @Test + public void testDefaultMinVersionKeptWhenManifestHasNone() throws IOException, SpmManifestParsingException { + ParseResult result = SpmManifestParser.parseManifests(List.of(fixture("branch.json")), "ios", "11.0"); + assertEquals("11.0", result.minVersion); + } + + @Test + public void testMergeManifests() throws IOException, SpmManifestParsingException { + // merge_extra declares firebase without ".git" and with different host casing plus one extra + // product; the canonical key must fold it into the same package. minVersion is the max. + ParseResult result = SpmManifestParser.parseManifests( + List.of(fixture("regular_ios.json"), fixture("merge_extra.json")), "ios", "11.0"); + assertEquals("15.0", result.minVersion); + assertEquals(3, result.packages.size()); + PackageRef firebase = result.packages.get("github.com/firebase/firebase-ios-sdk"); + assertEquals(Set.of("FirebaseAnalytics", "FirebaseRemoteConfig", "FirebaseCrashlytics"), firebase.products); + } + + @Test + public void testMergeConflictingRequirements() { + assertThrows(SpmManifestParsingException.class, () -> SpmManifestParser.parseManifests( + List.of(fixture("regular_ios.json"), fixture("merge_conflict.json")), "ios", "11.0")); + } + + @Test + public void testPlatformMismatchWithBuildPlatform() { + assertThrows(SpmManifestParsingException.class, () -> SpmManifestParser.parseManifests( + List.of(fixture("regular_ios.json")), "osx", "11.0")); + } + + private static Stream rejectedManifests() { + return Stream.of( + Arguments.of("url_file_scheme.json"), + Arguments.of("url_ssh_scp.json"), + Arguments.of("url_userinfo.json"), + Arguments.of("url_backtick.json"), + Arguments.of("url_dollar_paren.json"), + Arguments.of("url_quote_breakout.json"), + Arguments.of("url_newline.json"), + Arguments.of("url_traversal.json"), + Arguments.of("url_custom_port.json"), + Arguments.of("url_query.json"), + Arguments.of("product_swift_injection.json"), + Arguments.of("version_injection.json"), + Arguments.of("branch_injection.json"), + Arguments.of("revision_short.json"), + Arguments.of("both_version_and_branch.json"), + Arguments.of("no_requirement.json"), + Arguments.of("empty_products.json"), + Arguments.of("oversized_packages.json"), + Arguments.of("bad_json.json"), + Arguments.of("wrong_platform_value.json"), + Arguments.of("wrapper_type_bad.json") + ); + } + + @Test + public void testWrapperTypeParsed() throws IOException, SpmManifestParsingException { + ParseResult result = SpmManifestParser.parseManifest(fixture("wrapper_type_dynamic.json")); + assertEquals("dynamic", result.wrapperType); + // absent by default, and survives a merge with a manifest that has none + ParseResult merged = SpmManifestParser.parseManifests( + List.of(fixture("regular_ios.json"), fixture("wrapper_type_dynamic.json")), "ios", "11.0"); + assertEquals("dynamic", merged.wrapperType); + } + + @Tag("security") + @ParameterizedTest + @MethodSource("rejectedManifests") + public void testManifestRejected(String fixtureName) { + assertThrows(SpmManifestParsingException.class, () -> SpmManifestParser.parseManifest(fixture(fixtureName))); + } + + @Tag("security") + @Test + public void testSanitizeUrlReconstructs() throws SpmManifestParsingException { + assertEquals("https://github.com/foo/bar.git", SpmManifestParser.sanitizeUrl("https://github.com/foo/bar.git")); + assertEquals("https://github.com/foo/bar", SpmManifestParser.sanitizeUrl("https://github.com/foo/bar")); + } + + private static File writeManifest(File dir, String name, String minVersion, int firstPackage, int count) + throws IOException { + StringBuilder json = new StringBuilder("{ \"platform\": \"ios\", \"minVersion\": \"" + minVersion + "\", \"packages\": ["); + for (int i = 0; i < count; i++) { + json.append(i > 0 ? ", " : ""); + json.append(String.format( + "{ \"url\": \"https://github.com/foo/repo%d.git\", \"version\": \"1.0.0\", \"products\": [\"P%d\"] }", + firstPackage + i, firstPackage + i)); + } + json.append("] }"); + File manifest = new File(dir, name); + Files.writeString(manifest.toPath(), json.toString()); + return manifest; + } + + @Test + public void testShortVersionsPaddedToSwiftPmGrammar(@TempDir File tmpDir) throws IOException, SpmManifestParsingException { + // SwiftPM's Version literal fatalErrors on "1.0" and a platform rejects .iOS("13"); + // both are unambiguous, so they are padded here rather than failing inside xcodebuild + File manifest = new File(tmpDir, "SwiftPackages.json"); + Files.writeString(manifest.toPath(), + "{ \"platform\": \"ios\", \"minVersion\": \"13\", \"packages\": [" + + " { \"url\": \"https://github.com/foo/bar.git\", \"version\": \"1.0\", \"products\": [\"X\"] }," + + " { \"url\": \"https://github.com/foo/baz.git\", \"from\": \"9\", \"products\": [\"Y\"] } ] }"); + + ParseResult result = SpmManifestParser.parseManifest(manifest); + assertEquals("13.0", result.minVersion); + assertEquals("exact: \"1.0.0\"", result.packages.get("github.com/foo/bar").requirement.toSwiftArgument()); + assertEquals("from: \"9.0.0\"", result.packages.get("github.com/foo/baz").requirement.toSwiftArgument()); + } + + @Test + public void testPrereleaseVersionKeepsSuffixWhenPadded(@TempDir File tmpDir) throws IOException, SpmManifestParsingException { + File manifest = new File(tmpDir, "SwiftPackages.json"); + Files.writeString(manifest.toPath(), + "{ \"platform\": \"ios\", \"packages\": [" + + " { \"url\": \"https://github.com/foo/bar.git\", \"version\": \"2.0-beta.1\", \"products\": [\"X\"] } ] }"); + assertEquals("exact: \"2.0.0-beta.1\"", + SpmManifestParser.parseManifest(manifest).packages.get("github.com/foo/bar").requirement.toSwiftArgument()); + } + + @Test + public void testMergedPackageCountCapped(@TempDir File tmpDir) throws IOException { + // the per-file cap says nothing about the total: two legal manifests must not + // add up to more packages than a single one is allowed to declare + File first = writeManifest(tmpDir, "first.json", "13.0", 0, SpmManifestParser.MAX_PACKAGES); + File second = writeManifest(tmpDir, "second.json", "13.0", SpmManifestParser.MAX_PACKAGES, 1); + assertThrows(SpmManifestParsingException.class, + () -> SpmManifestParser.parseManifests(List.of(first, second), "ios", "11.0")); + } + + @Test + public void testManifestCountCapped(@TempDir File tmpDir) throws IOException { + List manifests = new ArrayList<>(); + for (int i = 0; i <= SpmManifestParser.MAX_MANIFESTS; i++) { + manifests.add(writeManifest(tmpDir, "m" + i + ".json", "13.0", 0, 1)); + } + assertThrows(SpmManifestParsingException.class, + () -> SpmManifestParser.parseManifests(manifests, "ios", "11.0")); + } + + @Test + public void testInvalidDefaultMinVersionRejected() { + assertThrows(SpmManifestParsingException.class, () -> SpmManifestParser.parseManifests( + List.of(fixture("regular_ios.json")), "ios", "11.0\"), .iOS(\"9")); + } + + @Test + public void testCanonicalKey() { + assertEquals("github.com/foo/bar", SpmManifestParser.canonicalKey("https://github.com/foo/bar.git")); + assertEquals("github.com/foo/bar", SpmManifestParser.canonicalKey("https://GitHub.com/foo/bar")); + assertEquals("github.com/foo/bar", SpmManifestParser.canonicalKey("https://github.com/foo/bar/")); + } +} diff --git a/server/src/test/java/com/defold/extender/services/spm/SwiftPackageManagerServiceTest.java b/server/src/test/java/com/defold/extender/services/spm/SwiftPackageManagerServiceTest.java new file mode 100644 index 000000000..d8132334a --- /dev/null +++ b/server/src/test/java/com/defold/extender/services/spm/SwiftPackageManagerServiceTest.java @@ -0,0 +1,166 @@ +package com.defold.extender.services.spm; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.junit.jupiter.api.condition.EnabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.io.TempDir; + +import org.springframework.core.io.FileSystemResource; + +import com.defold.extender.ExtenderException; +import com.defold.extender.services.cocoapods.PodUtils; + +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; + +@EnabledOnOs({OS.MAC}) +public class SwiftPackageManagerServiceTest { + + private SwiftPackageManagerService createService() throws IOException { + SwiftPackageManagerService service = new SwiftPackageManagerService( + new FileSystemResource("src/main/resources/template.package-swift"), + new FileSystemResource("src/main/resources/template.project-yml"), + new SpmServiceConfiguration(), + new SimpleMeterRegistry()); + service.swiftVersion = "6.0"; + service.wrapperMachOType = "staticlib"; + service.defaultDeveloperDir = "/Applications/Xcode.app/Contents/Developer"; + service.xcodegenPath = new File("/opt/homebrew/bin/xcodegen").exists() ? "/opt/homebrew/bin/xcodegen" : "xcodegen"; + return service; + } + + @Test + public void testGeneratePackageSwift() throws IOException, SpmManifestParsingException { + SwiftPackageManagerService service = createService(); + SpmManifestParser.ParseResult manifest = SpmManifestParser.parseManifests( + List.of(new File("test-data/swiftpackages/regular_ios.json")), "ios", "11.0"); + + String packageSwift = service.generatePackageSwift(manifest, true); + + assertTrue(packageSwift.contains("name: \"SpmDeps\"")); + assertTrue(packageSwift.contains(".iOS(\"13.0\")")); + assertTrue(packageSwift.contains(".library(")); + assertTrue(packageSwift.contains("type: .static")); + assertTrue(packageSwift.contains(".package(url: \"https://github.com/firebase/firebase-ios-sdk.git\", exact: \"12.0.0\"),")); + assertTrue(packageSwift.contains(".package(url: \"https://github.com/getsentry/sentry-cocoa.git\", from: \"9.0.0\"),")); + assertTrue(packageSwift.contains(".product(name: \"FirebaseAnalytics\", package: \"firebase-ios-sdk\"),")); + assertTrue(packageSwift.contains(".product(name: \"FirebaseRemoteConfig\", package: \"firebase-ios-sdk\"),")); + assertTrue(packageSwift.contains(".product(name: \"Sentry\", package: \"sentry-cocoa\"),")); + } + + @Test + public void testGenerateProjectYml() throws IOException, SpmManifestParsingException { + SwiftPackageManagerService service = createService(); + SpmManifestParser.ParseResult manifest = SpmManifestParser.parseManifests( + List.of(new File("test-data/swiftpackages/regular_ios.json")), "ios", "11.0"); + + String projectYml = service.generateProjectYml(manifest, true); + + assertTrue(projectYml.contains("name: SpmWrapper")); + assertTrue(projectYml.contains("iOS: \"13.0\"")); + assertTrue(projectYml.contains("platform: iOS")); + assertTrue(projectYml.contains("SWIFT_VERSION: \"6.0\"")); + assertTrue(projectYml.contains("MACH_O_TYPE: staticlib")); + assertTrue(projectYml.contains("GENERATE_INFOPLIST_FILE: \"YES\"")); + assertTrue(projectYml.contains("path: ../Package")); + assertTrue(projectYml.contains("product: SpmDeps")); + } + + @Test + public void testGenerateProjectYmlMacOs() throws IOException, SpmManifestParsingException { + SwiftPackageManagerService service = createService(); + service.wrapperMachOType = "mh_dylib"; + SpmManifestParser.ParseResult manifest = SpmManifestParser.parseManifests( + List.of(new File("test-data/swiftpackages/regular_osx.json")), "osx", "10.15"); + + String projectYml = service.generateProjectYml(manifest, false); + + assertTrue(projectYml.contains("macOS: \"11.0\"")); + assertTrue(projectYml.contains("platform: macOS")); + assertTrue(projectYml.contains("MACH_O_TYPE: mh_dylib")); + } + + @Test + public void testWrapperTypeOverride() throws IOException, SpmManifestParsingException { + SwiftPackageManagerService service = createService(); + SpmManifestParser.ParseResult manifest = SpmManifestParser.parseManifests( + List.of(new File("test-data/swiftpackages/regular_ios.json")), "ios", "11.0"); + + assertEquals("staticlib", service.machOTypeFor(manifest)); + manifest.wrapperType = "dynamic"; + assertEquals("mh_dylib", service.machOTypeFor(manifest)); + assertTrue(service.generateProjectYml(manifest, true).contains("MACH_O_TYPE: mh_dylib")); + manifest.wrapperType = "static"; + assertEquals("staticlib", service.machOTypeFor(manifest)); + } + + @Test + public void testCacheSubDirForSanitizesJobValue() { + // XCODE_VERSION comes from the job's build.yml and must never steer the shared + // cache out of home-dir-prefix, where the cleanup task would never reclaim it + assertEquals("16.2", SwiftPackageManagerService.cacheSubDirFor("16.2")); + assertEquals("16.0__16A242d_", SwiftPackageManagerService.cacheSubDirFor("16.0 (16A242d)")); + assertEquals("default", SwiftPackageManagerService.cacheSubDirFor(null)); + assertEquals("default", SwiftPackageManagerService.cacheSubDirFor("..")); + assertEquals("_", SwiftPackageManagerService.cacheSubDirFor("/")); + assertEquals(".._.._etc", SwiftPackageManagerService.cacheSubDirFor("../../etc")); + assertEquals(64, SwiftPackageManagerService.cacheSubDirFor("9".repeat(200)).length()); + } + + /** + * Real xcodegen + xcodebuild run against a small public package. Needs a Mac with + * Xcode (license accepted), xcodegen, and network access: + * ./gradlew :server:test -PexcludeTags=integration -PspmE2e=true --tests SwiftPackageManagerServiceTest + */ + @Test + @EnabledIfSystemProperty(named = "extender.test.spmE2e", matches = "true") + public void testResolveDependenciesEndToEnd(@TempDir File rootDir) throws IOException, ExtenderException { + SwiftPackageManagerService service = createService(); + service.homeDirPrefix = new File(rootDir, "spm-cache").getAbsolutePath(); + service.runAfterStartup(); + + File manifestDir = new File(rootDir, "job/upload/spmext/ios"); + manifestDir.mkdirs(); + File manifest = new File(manifestDir, "SwiftPackages.json"); + Files.writeString(manifest.toPath(), + "{ \"platform\": \"ios\", \"minVersion\": \"15.0\", \"packages\": [" + + " { \"url\": \"https://github.com/apple/swift-argument-parser.git\", \"from\": \"1.3.0\", \"products\": [\"ArgumentParser\"] } ] }"); + + SpmServiceBuildState buildState = new SpmServiceBuildState(); + buildState.workingDir = new File(rootDir, "job/SwiftPackageManagerService"); + buildState.packageDir = new File(buildState.workingDir, "Package"); + buildState.wrapperDir = new File(buildState.workingDir, "Wrapper"); + buildState.derivedDataDir = new File(buildState.workingDir, "DerivedData"); + buildState.moduleCacheDir = new File(buildState.workingDir, "ModuleCache"); + buildState.clonedSourcePackagesDir = new File(buildState.workingDir, "clonedSourcePackages"); + buildState.buildLogFile = new File(buildState.workingDir, "build.log"); + buildState.selectedPlatform = PodUtils.Platform.IPHONEOS; + buildState.buildArch = "arm64"; + new File(buildState.packageDir, "Sources/" + SpmServiceBuildState.AGGREGATOR_NAME).mkdirs(); + new File(buildState.wrapperDir, "Sources").mkdirs(); + buildState.derivedDataDir.mkdirs(); + buildState.moduleCacheDir.mkdirs(); + buildState.clonedSourcePackagesDir.mkdirs(); + + ResolvedPackages resolved = service.resolveDependencies( + List.of(manifest), buildState, Map.of("env.IOS_VERSION_MIN", "15.0"), true); + + assertNotNull(resolved); + assertTrue(new File(buildState.getProductsDir(), "SpmWrapper.framework/SpmWrapper").isFile()); + assertTrue(resolved.getFrameworks().contains("SpmWrapper")); + assertNotNull(resolved.getLockFile()); + assertTrue(resolved.getLockFile().isFile()); + assertEquals("15.0", resolved.getPlatformMinVersion()); + assertTrue(buildState.getBuildLogFile().isFile()); + } +} diff --git a/server/test-data/spm-buildlogs/dynamic-ios.log b/server/test-data/spm-buildlogs/dynamic-ios.log new file mode 100644 index 000000000..4fe21055d --- /dev/null +++ b/server/test-data/spm-buildlogs/dynamic-ios.log @@ -0,0 +1,31 @@ +Command line invocation: + /Applications/Xcode.app/Contents/Developer/usr/bin/xcodebuild -project /tmp/job123/SwiftPackageManagerService/Wrapper/SpmWrapper.xcodeproj -scheme SpmWrapper -destination generic/platform=iOS -configuration Release -derivedDataPath /tmp/job123/SwiftPackageManagerService/DerivedData -clonedSourcePackagesDirPath /tmp/extender/.swiftpm/cache/clonedSourcePackages -packageCachePath /tmp/extender/.swiftpm/cache/packageCache -skipPackagePluginValidation CLANG_MODULE_CACHE_PATH=/tmp/job123/SwiftPackageManagerService/ModuleCache CODE_SIGNING_ALLOWED=NO build + +Build settings from command line: + CLANG_MODULE_CACHE_PATH = /tmp/job123/SwiftPackageManagerService/ModuleCache + CODE_SIGNING_ALLOWED = NO + + cd /tmp/extender/.swiftpm/cache/clonedSourcePackages/checkouts/GoogleUtilities + write-file /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/GoogleUtilities.build/Release-iphoneos/GoogleUtilities-UserDefaults.build/Objects-normal/arm64/GoogleUtilities-UserDefaults.LinkFileList + +WriteAuxiliaryFile /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/GoogleUtilities.build/Release-iphoneos/GoogleUtilities-UserDefaults.build/Objects-normal/arm64/e6072d4f65d7061329687fe24e3d63a7-common-args.resp (in target 'GoogleUtilities-UserDefaults' from project 'GoogleUtilities') + cd /tmp/extender/.swiftpm/cache/clonedSourcePackages/checkouts/GoogleUtilities + write-file /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/GoogleUtilities.build/Release-iphoneos/GoogleUtilities-UserDefaults.build/Objects-normal/arm64/e6072d4f65d7061329687fe24e3d63a7-common-args.resp +'-std=c99' -fobjc-arc -fmodules -gmodules '-fmodules-cache-path=/tmp/job123/SwiftPackageManagerService/ModuleCache' '-fmodule-name=GoogleUtilities_UserDefaults' -fpascal-strings -Os -DSWIFT_PACKAGE '-DOBJC_OLD_DISPATCH_PROTOTYPES=1' -g -I/tmp/extender/.swiftpm/cache/clonedSourcePackages/checkouts/GoogleUtilities/GoogleUtilities/Logger/Public -I/tmp/extender/.swiftpm/cache/clonedSourcePackages/checkouts/GoogleUtilities/GoogleUtilities/Environment/Public -I/tmp/extender/.swiftpm/cache/clonedSourcePackages/checkouts/GoogleUtilities/third_party/IsAppEncrypted/Public -I/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/include -I/tmp/extender/.swiftpm/cache/clonedSourcePackages/checkouts/GoogleUtilities/GoogleUtilities/UserDefaults/Public -I/tmp/extender/.swiftpm/cache/clonedSourcePackages/checkouts/GoogleUtilities -I/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/GoogleUtilities.build/Release-iphoneos/GoogleUtilities-UserDefaults.build/DerivedSources-normal/arm64 -I/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/GoogleUtilities.build/Release-iphoneos/GoogleUtilities-UserDefaults.build/DerivedSources/arm64 -I/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/GoogleUtilities.build/Release-iphoneos/GoogleUtilities-UserDefaults.build/DerivedSources -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos -iframework /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/Library/Frameworks -iframework /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS26.5.sdk/Developer/Library/Frameworks '-fmodule-map-file=/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/GeneratedModuleMaps-iphoneos/GoogleUtilities-Logger.modulemap' '-fmodule-map-file=/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/GeneratedModuleMaps-iphoneos/GoogleUtilities-Environment.modulemap' '-fmodule-map-file=/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/GeneratedModuleMaps-iphoneos/third-party-IsAppEncrypted.modulemap' -DXcode + + +Ld /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/Firebase.o normal (in target 'Firebase' from project 'Firebase') + cd /tmp/extender/.swiftpm/cache/clonedSourcePackages/checkouts/firebase-ios-sdk + /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang -Xlinker -reproducible -target arm64-apple-ios15.0 -r -isysroot /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS26.5.sdk -Os -w -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/EagerLinkingTBDs/Release-iphoneos -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos -iframework /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/Library/Frameworks -iframework /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS26.5.sdk/Developer/Library/Frameworks -filelist /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Firebase.build/Release-iphoneos/Firebase.build/Objects-normal/arm64/Firebase.LinkFileList -nostdlib -Xlinker -object_path_lto -Xlinker /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Firebase.build/Release-iphoneos/Firebase.build/Objects-normal/arm64/Firebase_lto.o -Xlinker -dependency_info -Xlinker /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Firebase.build/Release-iphoneos/Firebase.build/Objects-normal/arm64/Firebase_dependency_info.dat -fobjc-arc -fobjc-link-runtime -o /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/Firebase.o + + + +Ld /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/SpmWrapper.framework/SpmWrapper normal (in target 'SpmWrapper' from project 'SpmWrapper') + cd /tmp/job123/SwiftPackageManagerService/Wrapper + /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang -Xlinker -reproducible -target arm64-apple-ios15.0 -dynamiclib -isysroot /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS26.5.sdk -Os -L/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/EagerLinkingTBDs/Release-iphoneos -L/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/EagerLinkingTBDs/Release-iphoneos -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/PackageFrameworks -F/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos -filelist /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/SpmWrapper.build/Release-iphoneos/SpmWrapper.build/Objects-normal/arm64/SpmWrapper.LinkFileList -install_name @rpath/SpmWrapper.framework/SpmWrapper -Xlinker -rpath -Xlinker /usr/lib/swift -Xlinker -rpath -Xlinker @executable_path/Frameworks -Xlinker -dead_strip -Xlinker -object_path_lto -Xlinker /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/SpmWrapper.build/Release-iphoneos/SpmWrapper.build/Objects-normal/arm64/SpmWrapper_lto.o -Xlinker -dependency_info -Xlinker /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/SpmWrapper.build/Release-iphoneos/SpmWrapper.build/Objects-normal/arm64/SpmWrapper_dependency_info.dat -fobjc-link-runtime -L/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/iphoneos -L/usr/lib/swift -Xlinker -add_ast_path -Xlinker /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/SpmWrapper.build/Release-iphoneos/SpmWrapper.build/Objects-normal/arm64/SpmWrapper.swiftmodule @/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/SpmWrapper.build/Release-iphoneos/SpmWrapper.build/Objects-normal/arm64/SpmWrapper-linker-args.resp -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -Wl,-no_warn_duplicate_libraries -lc++ -framework Accelerate -lc++ -lz -framework UIKit -framework Security -lz -framework StoreKit -lsqlite3 -lc++ -lz -lsqlite3 -lc++ -lz -framework StoreKit -lc++ -framework FirebaseAnalytics -framework GoogleAppMeasurementIdentitySupport -framework GoogleAppMeasurement -framework GoogleAdsOnDeviceConversion -framework FBSDKCoreKit -framework FBAEMKit -framework FBSDKCoreKit_Basics -framework FBSDKCoreKit -framework FBSDKCoreKit -framework FBAEMKit -framework FBAEMKit -framework FBSDKCoreKit_Basics -framework FBAEMKit -framework FBSDKCoreKit_Basics -framework FBSDKCoreKit_Basics -framework FBSDKCoreKit_Basics -framework Sentry -framework Sentry -compatibility_version 1 -current_version 1 -o /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/SpmWrapper.framework/SpmWrapper -Xlinker -add_ast_path -Xlinker /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/MyAwesomeLib.build/Release-iphoneos/MyAwesomeLib.build/Objects-normal/arm64/MyAwesomeLib.swiftmodule @/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/MyAwesomeLib.build/Release-iphoneos/MyAwesomeLib.build/Objects-normal/arm64/MyAwesomeLib-linker-args.resp -Xlinker -add_ast_path -Xlinker /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Firebase.build/Release-iphoneos/FirebaseCoreInternal.build/Objects-normal/arm64/FirebaseCoreInternal.swiftmodule @/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Firebase.build/Release-iphoneos/FirebaseCoreInternal.build/Objects-normal/arm64/FirebaseCoreInternal-linker-args.resp -Xlinker -add_ast_path -Xlinker /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Firebase.build/Release-iphoneos/FirebaseRemoteConfig.build/Objects-normal/arm64/FirebaseRemoteConfig.swiftmodule @/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Firebase.build/Release-iphoneos/FirebaseRemoteConfig.build/Objects-normal/arm64/FirebaseRemoteConfig-linker-args.resp -Xlinker -add_ast_path -Xlinker /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Firebase.build/Release-iphoneos/FirebaseRemoteConfigInterop.build/Objects-normal/arm64/FirebaseRemoteConfigInterop.swiftmodule @/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Firebase.build/Release-iphoneos/FirebaseRemoteConfigInterop.build/Objects-normal/arm64/FirebaseRemoteConfigInterop-linker-args.resp -Xlinker -add_ast_path -Xlinker /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Firebase.build/Release-iphoneos/FirebaseSharedSwift.build/Objects-normal/arm64/FirebaseSharedSwift.swiftmodule @/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Firebase.build/Release-iphoneos/FirebaseSharedSwift.build/Objects-normal/arm64/FirebaseSharedSwift-linker-args.resp -Xlinker -add_ast_path -Xlinker /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Facebook.build/Release-iphoneos/FacebookCore.build/Objects-normal/arm64/FacebookCore.swiftmodule @/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Facebook.build/Release-iphoneos/FacebookCore.build/Objects-normal/arm64/FacebookCore-linker-args.resp -Xlinker -add_ast_path -Xlinker /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Facebook.build/Release-iphoneos/FacebookAEM.build/Objects-normal/arm64/FacebookAEM.swiftmodule @/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Facebook.build/Release-iphoneos/FacebookAEM.build/Objects-normal/arm64/FacebookAEM-linker-args.resp -Xlinker -add_ast_path -Xlinker /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Facebook.build/Release-iphoneos/FacebookBasics.build/Objects-normal/arm64/FacebookBasics.swiftmodule @/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Facebook.build/Release-iphoneos/FacebookBasics.build/Objects-normal/arm64/FacebookBasics-linker-args.resp -Xlinker -add_ast_path -Xlinker /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Sentry.build/Release-iphoneos/SentryCppHelper.build/Objects-normal/arm64/SentryCppHelper.swiftmodule @/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Sentry.build/Release-iphoneos/SentryCppHelper.build/Objects-normal/arm64/SentryCppHelper-linker-args.resp + + + /usr/bin/touch -c /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/SpmWrapper.framework + +** BUILD SUCCEEDED ** + diff --git a/server/test-data/spm-buildlogs/static-ios.log b/server/test-data/spm-buildlogs/static-ios.log new file mode 100644 index 000000000..95d59aff9 --- /dev/null +++ b/server/test-data/spm-buildlogs/static-ios.log @@ -0,0 +1,20 @@ +Command line invocation: + /Applications/Xcode.app/Contents/Developer/usr/bin/xcodebuild -project /tmp/job123/SwiftPackageManagerService/Wrapper/SpmWrapper.xcodeproj -scheme SpmWrapper -destination generic/platform=iOS -configuration Release -scmProvider system -derivedDataPath /tmp/job123/SwiftPackageManagerService/DerivedData -clonedSourcePackagesDirPath /tmp/extender/.swiftpm/cache/clonedSourcePackages -packageCachePath /tmp/extender/.swiftpm/cache/packageCache -skipPackagePluginValidation CLANG_MODULE_CACHE_PATH=/tmp/job123/SwiftPackageManagerService/ModuleCache CODE_SIGNING_ALLOWED=NO MACH_O_TYPE=staticlib build + +Build settings from command line: + CLANG_MODULE_CACHE_PATH = /tmp/job123/SwiftPackageManagerService/ModuleCache + CODE_SIGNING_ALLOWED = NO + +Libtool /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/Firebase.o normal (in target 'Firebase' from project 'Firebase') + cd /tmp/extender/.swiftpm/cache/clonedSourcePackages/checkouts/firebase-ios-sdk + /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/libtool -static -arch_only arm64 -D -syslibroot /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS26.5.sdk -L/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos -L/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/usr/lib -filelist /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Firebase.build/Release-iphoneos/Firebase.build/Objects-normal/arm64/Firebase.LinkFileList -dependency_info /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/Firebase.build/Release-iphoneos/Firebase.build/Objects-normal/arm64/Firebase_libtool_dependency_info.dat -o /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/Firebase.o + + +Libtool /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/SpmWrapper.framework/SpmWrapper normal (in target 'SpmWrapper' from project 'SpmWrapper') + cd /tmp/job123/SwiftPackageManagerService/Wrapper + /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/libtool -static -arch_only arm64 -D -syslibroot /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS26.5.sdk -L/tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos -filelist /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/SpmWrapper.build/Release-iphoneos/SpmWrapper.build/Objects-normal/arm64/SpmWrapper.LinkFileList -dependency_info /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Intermediates.noindex/SpmWrapper.build/Release-iphoneos/SpmWrapper.build/Objects-normal/arm64/SpmWrapper_libtool_dependency_info.dat -o /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/SpmWrapper.framework/SpmWrapper + + /usr/bin/touch -c /tmp/job123/SwiftPackageManagerService/DerivedData/Build/Products/Release-iphoneos/SpmWrapper.framework + +** BUILD SUCCEEDED ** + diff --git a/server/test-data/spm-project/spmext/ext.manifest b/server/test-data/spm-project/spmext/ext.manifest new file mode 100644 index 000000000..db639e7c9 --- /dev/null +++ b/server/test-data/spm-project/spmext/ext.manifest @@ -0,0 +1 @@ +name: SpmExt diff --git a/server/test-data/spm-project/spmext/ios/SwiftPackages.json b/server/test-data/spm-project/spmext/ios/SwiftPackages.json new file mode 100644 index 000000000..b16c548da --- /dev/null +++ b/server/test-data/spm-project/spmext/ios/SwiftPackages.json @@ -0,0 +1,11 @@ +{ + "platform": "ios", + "minVersion": "15.0", + "packages": [ + { + "url": "https://github.com/getsentry/sentry-cocoa.git", + "from": "9.0.0", + "products": ["Sentry"] + } + ] +} diff --git a/server/test-data/spm-project/spmext/osx/SwiftPackages.json b/server/test-data/spm-project/spmext/osx/SwiftPackages.json new file mode 100644 index 000000000..113e21b71 --- /dev/null +++ b/server/test-data/spm-project/spmext/osx/SwiftPackages.json @@ -0,0 +1,11 @@ +{ + "platform": "osx", + "minVersion": "12.0", + "packages": [ + { + "url": "https://github.com/getsentry/sentry-cocoa.git", + "from": "9.0.0", + "products": ["Sentry"] + } + ] +} diff --git a/server/test-data/spm-project/spmext/src/spmext.mm b/server/test-data/spm-project/spmext/src/spmext.mm new file mode 100644 index 000000000..0a2bbba67 --- /dev/null +++ b/server/test-data/spm-project/spmext/src/spmext.mm @@ -0,0 +1,42 @@ +#include + +#if defined(DM_PLATFORM_IOS) || defined(DM_PLATFORM_OSX) +// The generated Swift interface header needs the platform UI types declared up front +// when compiled without clang modules +#if defined(DM_PLATFORM_IOS) +#import +#else +#import +#endif +#import +// SentrySDK is implemented in Swift; its ObjC interface lives in the generated header +#import +#endif + +static dmExtension::Result AppInitializeSpmExt(dmExtension::AppParams* params) +{ +#if defined(DM_PLATFORM_IOS) || defined(DM_PLATFORM_OSX) + [SentrySDK startWithConfigureOptions:^(SentryOptions *options) { + options.dsn = @"https://examplePublicKey@o0.ingest.sentry.io/0"; + }]; + dmLogInfo("SpmExt: Sentry started via Swift Package Manager dependency"); +#endif + return dmExtension::RESULT_OK; +} + +static dmExtension::Result AppFinalizeSpmExt(dmExtension::AppParams* params) +{ + return dmExtension::RESULT_OK; +} + +static dmExtension::Result InitializeSpmExt(dmExtension::Params* params) +{ + return dmExtension::RESULT_OK; +} + +static dmExtension::Result FinalizeSpmExt(dmExtension::Params* params) +{ + return dmExtension::RESULT_OK; +} + +DM_DECLARE_EXTENSION(SpmExt, "SpmExt", AppInitializeSpmExt, AppFinalizeSpmExt, InitializeSpmExt, 0, 0, FinalizeSpmExt) diff --git a/server/test-data/swiftpackages/bad_json.json b/server/test-data/swiftpackages/bad_json.json new file mode 100644 index 000000000..4c687365a --- /dev/null +++ b/server/test-data/swiftpackages/bad_json.json @@ -0,0 +1 @@ +{ this is not json diff --git a/server/test-data/swiftpackages/both_version_and_branch.json b/server/test-data/swiftpackages/both_version_and_branch.json new file mode 100644 index 000000000..9eb58c19d --- /dev/null +++ b/server/test-data/swiftpackages/both_version_and_branch.json @@ -0,0 +1,6 @@ +{ + "platform": "ios", + "packages": [ + { "url": "https://github.com/foo/bar.git", "version": "1.0.0", "branch": "main", "products": ["X"] } + ] +} diff --git a/server/test-data/swiftpackages/branch.json b/server/test-data/swiftpackages/branch.json new file mode 100644 index 000000000..e35d423d4 --- /dev/null +++ b/server/test-data/swiftpackages/branch.json @@ -0,0 +1,10 @@ +{ + "platform": "ios", + "packages": [ + { + "url": "https://github.com/apple/swift-collections.git", + "branch": "release/1.1", + "products": ["Collections"] + } + ] +} diff --git a/server/test-data/swiftpackages/branch_injection.json b/server/test-data/swiftpackages/branch_injection.json new file mode 100644 index 000000000..91813f485 --- /dev/null +++ b/server/test-data/swiftpackages/branch_injection.json @@ -0,0 +1,6 @@ +{ + "platform": "ios", + "packages": [ + { "url": "https://github.com/foo/bar.git", "branch": "main\"); import Foundation//", "products": ["X"] } + ] +} diff --git a/server/test-data/swiftpackages/empty_products.json b/server/test-data/swiftpackages/empty_products.json new file mode 100644 index 000000000..3efbe208e --- /dev/null +++ b/server/test-data/swiftpackages/empty_products.json @@ -0,0 +1,6 @@ +{ + "platform": "ios", + "packages": [ + { "url": "https://github.com/foo/bar.git", "version": "1.0.0", "products": [] } + ] +} diff --git a/server/test-data/swiftpackages/merge_conflict.json b/server/test-data/swiftpackages/merge_conflict.json new file mode 100644 index 000000000..38262e187 --- /dev/null +++ b/server/test-data/swiftpackages/merge_conflict.json @@ -0,0 +1,10 @@ +{ + "platform": "ios", + "packages": [ + { + "url": "https://github.com/firebase/firebase-ios-sdk.git", + "version": "11.0.0", + "products": ["FirebaseAnalytics"] + } + ] +} diff --git a/server/test-data/swiftpackages/merge_extra.json b/server/test-data/swiftpackages/merge_extra.json new file mode 100644 index 000000000..5f43f15ec --- /dev/null +++ b/server/test-data/swiftpackages/merge_extra.json @@ -0,0 +1,16 @@ +{ + "platform": "ios", + "minVersion": "15.0", + "packages": [ + { + "url": "https://GitHub.com/firebase/firebase-ios-sdk", + "version": "12.0.0", + "products": ["FirebaseCrashlytics"] + }, + { + "url": "https://github.com/facebook/facebook-ios-sdk.git", + "from": "18.0.0", + "products": ["FacebookCore"] + } + ] +} diff --git a/server/test-data/swiftpackages/no_requirement.json b/server/test-data/swiftpackages/no_requirement.json new file mode 100644 index 000000000..f18f6bdab --- /dev/null +++ b/server/test-data/swiftpackages/no_requirement.json @@ -0,0 +1,6 @@ +{ + "platform": "ios", + "packages": [ + { "url": "https://github.com/foo/bar.git", "products": ["X"] } + ] +} diff --git a/server/test-data/swiftpackages/oversized_packages.json b/server/test-data/swiftpackages/oversized_packages.json new file mode 100644 index 000000000..1c878f238 --- /dev/null +++ b/server/test-data/swiftpackages/oversized_packages.json @@ -0,0 +1,236 @@ +{ + "platform": "ios", + "packages": [ + { + "url": "https://github.com/foo/repo0.git", + "version": "1.0.0", + "products": [ + "P0" + ] + }, + { + "url": "https://github.com/foo/repo1.git", + "version": "1.0.0", + "products": [ + "P1" + ] + }, + { + "url": "https://github.com/foo/repo2.git", + "version": "1.0.0", + "products": [ + "P2" + ] + }, + { + "url": "https://github.com/foo/repo3.git", + "version": "1.0.0", + "products": [ + "P3" + ] + }, + { + "url": "https://github.com/foo/repo4.git", + "version": "1.0.0", + "products": [ + "P4" + ] + }, + { + "url": "https://github.com/foo/repo5.git", + "version": "1.0.0", + "products": [ + "P5" + ] + }, + { + "url": "https://github.com/foo/repo6.git", + "version": "1.0.0", + "products": [ + "P6" + ] + }, + { + "url": "https://github.com/foo/repo7.git", + "version": "1.0.0", + "products": [ + "P7" + ] + }, + { + "url": "https://github.com/foo/repo8.git", + "version": "1.0.0", + "products": [ + "P8" + ] + }, + { + "url": "https://github.com/foo/repo9.git", + "version": "1.0.0", + "products": [ + "P9" + ] + }, + { + "url": "https://github.com/foo/repo10.git", + "version": "1.0.0", + "products": [ + "P10" + ] + }, + { + "url": "https://github.com/foo/repo11.git", + "version": "1.0.0", + "products": [ + "P11" + ] + }, + { + "url": "https://github.com/foo/repo12.git", + "version": "1.0.0", + "products": [ + "P12" + ] + }, + { + "url": "https://github.com/foo/repo13.git", + "version": "1.0.0", + "products": [ + "P13" + ] + }, + { + "url": "https://github.com/foo/repo14.git", + "version": "1.0.0", + "products": [ + "P14" + ] + }, + { + "url": "https://github.com/foo/repo15.git", + "version": "1.0.0", + "products": [ + "P15" + ] + }, + { + "url": "https://github.com/foo/repo16.git", + "version": "1.0.0", + "products": [ + "P16" + ] + }, + { + "url": "https://github.com/foo/repo17.git", + "version": "1.0.0", + "products": [ + "P17" + ] + }, + { + "url": "https://github.com/foo/repo18.git", + "version": "1.0.0", + "products": [ + "P18" + ] + }, + { + "url": "https://github.com/foo/repo19.git", + "version": "1.0.0", + "products": [ + "P19" + ] + }, + { + "url": "https://github.com/foo/repo20.git", + "version": "1.0.0", + "products": [ + "P20" + ] + }, + { + "url": "https://github.com/foo/repo21.git", + "version": "1.0.0", + "products": [ + "P21" + ] + }, + { + "url": "https://github.com/foo/repo22.git", + "version": "1.0.0", + "products": [ + "P22" + ] + }, + { + "url": "https://github.com/foo/repo23.git", + "version": "1.0.0", + "products": [ + "P23" + ] + }, + { + "url": "https://github.com/foo/repo24.git", + "version": "1.0.0", + "products": [ + "P24" + ] + }, + { + "url": "https://github.com/foo/repo25.git", + "version": "1.0.0", + "products": [ + "P25" + ] + }, + { + "url": "https://github.com/foo/repo26.git", + "version": "1.0.0", + "products": [ + "P26" + ] + }, + { + "url": "https://github.com/foo/repo27.git", + "version": "1.0.0", + "products": [ + "P27" + ] + }, + { + "url": "https://github.com/foo/repo28.git", + "version": "1.0.0", + "products": [ + "P28" + ] + }, + { + "url": "https://github.com/foo/repo29.git", + "version": "1.0.0", + "products": [ + "P29" + ] + }, + { + "url": "https://github.com/foo/repo30.git", + "version": "1.0.0", + "products": [ + "P30" + ] + }, + { + "url": "https://github.com/foo/repo31.git", + "version": "1.0.0", + "products": [ + "P31" + ] + }, + { + "url": "https://github.com/foo/repo32.git", + "version": "1.0.0", + "products": [ + "P32" + ] + } + ] +} \ No newline at end of file diff --git a/server/test-data/swiftpackages/product_swift_injection.json b/server/test-data/swiftpackages/product_swift_injection.json new file mode 100644 index 000000000..a5adbbd44 --- /dev/null +++ b/server/test-data/swiftpackages/product_swift_injection.json @@ -0,0 +1,6 @@ +{ + "platform": "ios", + "packages": [ + { "url": "https://github.com/foo/bar.git", "version": "1.0.0", "products": ["X\"), .product(name: \"Y"] } + ] +} diff --git a/server/test-data/swiftpackages/regular_ios.json b/server/test-data/swiftpackages/regular_ios.json new file mode 100644 index 000000000..efb9efa5e --- /dev/null +++ b/server/test-data/swiftpackages/regular_ios.json @@ -0,0 +1,16 @@ +{ + "platform": "ios", + "minVersion": "13.0", + "packages": [ + { + "url": "https://github.com/firebase/firebase-ios-sdk.git", + "version": "12.0.0", + "products": ["FirebaseAnalytics", "FirebaseRemoteConfig"] + }, + { + "url": "https://github.com/getsentry/sentry-cocoa.git", + "from": "9.0.0", + "products": ["Sentry"] + } + ] +} diff --git a/server/test-data/swiftpackages/regular_osx.json b/server/test-data/swiftpackages/regular_osx.json new file mode 100644 index 000000000..ed5a56cc7 --- /dev/null +++ b/server/test-data/swiftpackages/regular_osx.json @@ -0,0 +1,11 @@ +{ + "platform": "osx", + "minVersion": "11.0", + "packages": [ + { + "url": "https://github.com/apple/swift-argument-parser.git", + "version": "1.5.0", + "products": ["ArgumentParser"] + } + ] +} diff --git a/server/test-data/swiftpackages/revision.json b/server/test-data/swiftpackages/revision.json new file mode 100644 index 000000000..c5606e7f9 --- /dev/null +++ b/server/test-data/swiftpackages/revision.json @@ -0,0 +1,10 @@ +{ + "platform": "ios", + "packages": [ + { + "url": "https://github.com/apple/swift-collections.git", + "revision": "94cf62b3ba8d4bed62680341ce53483da1d0c1a7", + "products": ["Collections"] + } + ] +} diff --git a/server/test-data/swiftpackages/revision_short.json b/server/test-data/swiftpackages/revision_short.json new file mode 100644 index 000000000..fa1ce42c7 --- /dev/null +++ b/server/test-data/swiftpackages/revision_short.json @@ -0,0 +1,6 @@ +{ + "platform": "ios", + "packages": [ + { "url": "https://github.com/foo/bar.git", "revision": "abc123", "products": ["X"] } + ] +} diff --git a/server/test-data/swiftpackages/url_backtick.json b/server/test-data/swiftpackages/url_backtick.json new file mode 100644 index 000000000..e30ba60f5 --- /dev/null +++ b/server/test-data/swiftpackages/url_backtick.json @@ -0,0 +1,6 @@ +{ + "platform": "ios", + "packages": [ + { "url": "https://github.com/foo/`touch pwned`.git", "version": "1.0.0", "products": ["X"] } + ] +} diff --git a/server/test-data/swiftpackages/url_custom_port.json b/server/test-data/swiftpackages/url_custom_port.json new file mode 100644 index 000000000..6b9535987 --- /dev/null +++ b/server/test-data/swiftpackages/url_custom_port.json @@ -0,0 +1,6 @@ +{ + "platform": "ios", + "packages": [ + { "url": "https://github.com:8443/foo/bar.git", "version": "1.0.0", "products": ["X"] } + ] +} diff --git a/server/test-data/swiftpackages/url_dollar_paren.json b/server/test-data/swiftpackages/url_dollar_paren.json new file mode 100644 index 000000000..e3f4a7f90 --- /dev/null +++ b/server/test-data/swiftpackages/url_dollar_paren.json @@ -0,0 +1,6 @@ +{ + "platform": "ios", + "packages": [ + { "url": "https://github.com/foo/$(whoami).git", "version": "1.0.0", "products": ["X"] } + ] +} diff --git a/server/test-data/swiftpackages/url_file_scheme.json b/server/test-data/swiftpackages/url_file_scheme.json new file mode 100644 index 000000000..bd15e60b9 --- /dev/null +++ b/server/test-data/swiftpackages/url_file_scheme.json @@ -0,0 +1,6 @@ +{ + "platform": "ios", + "packages": [ + { "url": "file:///etc/passwd", "version": "1.0.0", "products": ["X"] } + ] +} diff --git a/server/test-data/swiftpackages/url_newline.json b/server/test-data/swiftpackages/url_newline.json new file mode 100644 index 000000000..d9c4df8d6 --- /dev/null +++ b/server/test-data/swiftpackages/url_newline.json @@ -0,0 +1,6 @@ +{ + "platform": "ios", + "packages": [ + { "url": "https://github.com/foo\nbar.git", "version": "1.0.0", "products": ["X"] } + ] +} diff --git a/server/test-data/swiftpackages/url_query.json b/server/test-data/swiftpackages/url_query.json new file mode 100644 index 000000000..691050005 --- /dev/null +++ b/server/test-data/swiftpackages/url_query.json @@ -0,0 +1,6 @@ +{ + "platform": "ios", + "packages": [ + { "url": "https://github.com/foo/bar.git?evil=1", "version": "1.0.0", "products": ["X"] } + ] +} diff --git a/server/test-data/swiftpackages/url_quote_breakout.json b/server/test-data/swiftpackages/url_quote_breakout.json new file mode 100644 index 000000000..4e6aa91aa --- /dev/null +++ b/server/test-data/swiftpackages/url_quote_breakout.json @@ -0,0 +1,6 @@ +{ + "platform": "ios", + "packages": [ + { "url": "https://github.com/foo/bar.git\", .branch(\"x", "version": "1.0.0", "products": ["X"] } + ] +} diff --git a/server/test-data/swiftpackages/url_ssh_scp.json b/server/test-data/swiftpackages/url_ssh_scp.json new file mode 100644 index 000000000..43dfc622a --- /dev/null +++ b/server/test-data/swiftpackages/url_ssh_scp.json @@ -0,0 +1,6 @@ +{ + "platform": "ios", + "packages": [ + { "url": "git@github.com:foo/bar.git", "version": "1.0.0", "products": ["X"] } + ] +} diff --git a/server/test-data/swiftpackages/url_traversal.json b/server/test-data/swiftpackages/url_traversal.json new file mode 100644 index 000000000..e67f5cc26 --- /dev/null +++ b/server/test-data/swiftpackages/url_traversal.json @@ -0,0 +1,6 @@ +{ + "platform": "ios", + "packages": [ + { "url": "https://github.com/foo/../../internal/repo.git", "version": "1.0.0", "products": ["X"] } + ] +} diff --git a/server/test-data/swiftpackages/url_userinfo.json b/server/test-data/swiftpackages/url_userinfo.json new file mode 100644 index 000000000..0b87ad630 --- /dev/null +++ b/server/test-data/swiftpackages/url_userinfo.json @@ -0,0 +1,6 @@ +{ + "platform": "ios", + "packages": [ + { "url": "https://user:token@github.com/foo/bar.git", "version": "1.0.0", "products": ["X"] } + ] +} diff --git a/server/test-data/swiftpackages/version_injection.json b/server/test-data/swiftpackages/version_injection.json new file mode 100644 index 000000000..94af45680 --- /dev/null +++ b/server/test-data/swiftpackages/version_injection.json @@ -0,0 +1,6 @@ +{ + "platform": "ios", + "packages": [ + { "url": "https://github.com/foo/bar.git", "version": "1.0.0\"), .branch(\"x", "products": ["X"] } + ] +} diff --git a/server/test-data/swiftpackages/wrapper_type_bad.json b/server/test-data/swiftpackages/wrapper_type_bad.json new file mode 100644 index 000000000..45dbeaf83 --- /dev/null +++ b/server/test-data/swiftpackages/wrapper_type_bad.json @@ -0,0 +1,7 @@ +{ + "platform": "ios", + "wrapperType": "framework; rm -rf /", + "packages": [ + { "url": "https://github.com/foo/bar.git", "version": "1.0.0", "products": ["X"] } + ] +} diff --git a/server/test-data/swiftpackages/wrapper_type_dynamic.json b/server/test-data/swiftpackages/wrapper_type_dynamic.json new file mode 100644 index 000000000..3040ba9fe --- /dev/null +++ b/server/test-data/swiftpackages/wrapper_type_dynamic.json @@ -0,0 +1,7 @@ +{ + "platform": "ios", + "wrapperType": "dynamic", + "packages": [ + { "url": "https://github.com/foo/bar.git", "version": "1.0.0", "products": ["X"] } + ] +} diff --git a/server/test-data/swiftpackages/wrong_platform_value.json b/server/test-data/swiftpackages/wrong_platform_value.json new file mode 100644 index 000000000..078d70734 --- /dev/null +++ b/server/test-data/swiftpackages/wrong_platform_value.json @@ -0,0 +1,6 @@ +{ + "platform": "android", + "packages": [ + { "url": "https://github.com/foo/bar.git", "version": "1.0.0", "products": ["X"] } + ] +} From 1c2573fe72754083420335c8451d1f818217545a Mon Sep 17 00:00:00 2001 From: Kharkunov Eugene Date: Sun, 30 Aug 2026 11:41:11 +0300 Subject: [PATCH 2/3] Address CodeQL alerts: isString(), flipped privacy-manifest delegation, createDirectories --- server/src/main/java/com/defold/extender/Extender.java | 4 ++-- .../extender/services/cocoapods/ResolvedPods.java | 5 ++--- .../extender/services/spm/SpmManifestParser.java | 10 +++++----- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/com/defold/extender/Extender.java b/server/src/main/java/com/defold/extender/Extender.java index 7250bdf75..f7ae673ae 100644 --- a/server/src/main/java/com/defold/extender/Extender.java +++ b/server/src/main/java/com/defold/extender/Extender.java @@ -1125,9 +1125,9 @@ private List buildNativeDepsArtifacts() throws IOException, InterruptedExc LOGGER.info("buildNativeDepsArtifacts - adding resources to build output"); File resourcesBuildDir = new File(buildState.buildDir, "resources"); - resourcesBuildDir.mkdir(); + Files.createDirectories(resourcesBuildDir.toPath()); File frameworksBuildDir = new File(buildState.buildDir, "frameworks"); - frameworksBuildDir.mkdir(); + Files.createDirectories(frameworksBuildDir.toPath()); for (ResolvedNativeDeps deps : resolvedNativeDeps) { for (File resourceFile : deps.getResources()) { diff --git a/server/src/main/java/com/defold/extender/services/cocoapods/ResolvedPods.java b/server/src/main/java/com/defold/extender/services/cocoapods/ResolvedPods.java index 121006a99..88727d9c3 100644 --- a/server/src/main/java/com/defold/extender/services/cocoapods/ResolvedPods.java +++ b/server/src/main/java/com/defold/extender/services/cocoapods/ResolvedPods.java @@ -302,7 +302,7 @@ public boolean useFrameworks() { @Deprecated public List getPodsPrivacyManifests() { - return ExtenderUtil.listFilesMatchingRecursive(podsDir, "PrivacyInfo.xcprivacy"); + return getPrivacyManifests(); } public File getTargetSupportFilesDir() { @@ -327,9 +327,8 @@ public File getLockFile() { } @Override - @SuppressWarnings("deprecation") public List getPrivacyManifests() { - return getPodsPrivacyManifests(); + return ExtenderUtil.listFilesMatchingRecursive(podsDir, "PrivacyInfo.xcprivacy"); } @Override diff --git a/server/src/main/java/com/defold/extender/services/spm/SpmManifestParser.java b/server/src/main/java/com/defold/extender/services/spm/SpmManifestParser.java index 4b1db6de9..061a57f92 100644 --- a/server/src/main/java/com/defold/extender/services/spm/SpmManifestParser.java +++ b/server/src/main/java/com/defold/extender/services/spm/SpmManifestParser.java @@ -266,7 +266,7 @@ private static String parseWrapperType(JsonNode root) throws SpmManifestParsingE if (node == null || node.isNull()) { return null; } - String wrapperType = node.isTextual() ? node.asString() : null; + String wrapperType = node.isString() ? node.asString() : null; if (!"static".equals(wrapperType) && !"dynamic".equals(wrapperType)) { throw new SpmManifestParsingException( String.format("Invalid 'wrapperType' in Swift package manifest: '%s' (expected 'static' or 'dynamic')", node.asString())); @@ -288,7 +288,7 @@ private static String parseMinVersion(JsonNode root) throws SpmManifestParsingEx if (node == null || node.isNull()) { return null; } - String minVersion = node.isTextual() ? node.asString() : null; + String minVersion = node.isString() ? node.asString() : null; if (minVersion == null || !MIN_VERSION_PATTERN.matcher(minVersion).matches()) { throw new SpmManifestParsingException( String.format("Invalid 'minVersion' in Swift package manifest: '%s'", node.asString())); @@ -325,7 +325,7 @@ private static Requirement parseRequirement(JsonNode packageNode, String url) th throw new SpmManifestParsingException( String.format("Swift package '%s' declares more than one of version/from/branch/revision", url)); } - String value = node.isTextual() ? node.asString() : null; + String value = node.isString() ? node.asString() : null; if (value == null || !patternFor(kind).matcher(value).matches()) { throw new SpmManifestParsingException( String.format("Invalid '%s' for Swift package '%s': '%s'", field, url, node.asString())); @@ -350,7 +350,7 @@ private static List parseProducts(JsonNode packageNode, String url) thro } List products = new java.util.ArrayList<>(); for (JsonNode productNode : productsNode) { - String product = productNode.isTextual() ? productNode.asString() : null; + String product = productNode.isString() ? productNode.asString() : null; if (product == null || !PRODUCT_PATTERN.matcher(product).matches()) { throw new SpmManifestParsingException( String.format("Invalid product name for Swift package '%s': '%s'", url, productNode.asString())); @@ -400,7 +400,7 @@ private static Pattern patternFor(RequirementKind kind) { private static String textValue(JsonNode node, String field) throws SpmManifestParsingException { JsonNode value = node.get(field); - if (value == null || !value.isTextual() || value.asString().isEmpty()) { + if (value == null || !value.isString() || value.asString().isEmpty()) { throw new SpmManifestParsingException( String.format("Missing or invalid '%s' in Swift package manifest", field)); } From 27b845989dde06bb9a769ebddc1764d17bf2201f Mon Sep 17 00:00:00 2001 From: Kharkunov Eugene Date: Sun, 30 Aug 2026 13:13:13 +0300 Subject: [PATCH 3/3] Harvest dynamic package products from PackageFrameworks --- .../services/spm/ResolvedPackages.java | 28 +++++++++++++++++-- .../services/spm/ResolvedPackagesTest.java | 25 +++++++++++++++++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/defold/extender/services/spm/ResolvedPackages.java b/server/src/main/java/com/defold/extender/services/spm/ResolvedPackages.java index 3c02bdc31..7aba9f746 100644 --- a/server/src/main/java/com/defold/extender/services/spm/ResolvedPackages.java +++ b/server/src/main/java/com/defold/extender/services/spm/ResolvedPackages.java @@ -70,6 +70,30 @@ else if (name.endsWith(".bundle") && entry.isDirectory()) { } } + // a package library product declared `type: .dynamic` does not merge into the + // wrapper; it builds as a dylib framework under PackageFrameworks and must be + // linked and embedded like any other dynamic framework + File packageFrameworksDir = new File(productsDir, "PackageFrameworks"); + File[] packageFrameworkEntries = packageFrameworksDir.listFiles(); + if (packageFrameworkEntries != null) { + Arrays.sort(packageFrameworkEntries); + for (File entry : packageFrameworkEntries) { + String name = entry.getName(); + if (!name.endsWith(".framework") || !entry.isDirectory()) { + continue; + } + String frameworkName = name.substring(0, name.length() - ".framework".length()); + if (productFrameworks.containsKey(frameworkName)) { + continue; + } + boolean dynamic = FrameworkUtil.isDynamicallyLinked(entry); + productFrameworks.put(frameworkName, dynamic); + if (dynamic) { + resolved.dynamicFrameworks.add(entry); + } + } + } + // every products-dir framework stays on the engine link: a wrapper pulls static // framework members only on demand, so the framework remains the authoritative // home of ObjC classes the extension code references itself @@ -96,9 +120,7 @@ else if (name.endsWith(".bundle") && entry.isDirectory()) { resolved.frameworks.addAll(frameworkNames); resolved.frameworksSearchPaths.add(productsDir.getAbsolutePath()); - File packageFrameworksDir = new File(productsDir, "PackageFrameworks"); - String[] packageFrameworks = packageFrameworksDir.list(); - if (packageFrameworks != null && packageFrameworks.length > 0) { + if (packageFrameworkEntries != null && packageFrameworkEntries.length > 0) { resolved.frameworksSearchPaths.add(packageFrameworksDir.getAbsolutePath()); } diff --git a/server/src/test/java/com/defold/extender/services/spm/ResolvedPackagesTest.java b/server/src/test/java/com/defold/extender/services/spm/ResolvedPackagesTest.java index 631ca24a3..a97ebffc4 100644 --- a/server/src/test/java/com/defold/extender/services/spm/ResolvedPackagesTest.java +++ b/server/src/test/java/com/defold/extender/services/spm/ResolvedPackagesTest.java @@ -188,6 +188,31 @@ public void testCollectModuleMapIncludePaths(@TempDir File rootDir) throws IOExc assertEquals(3, includePaths.size()); } + @Test + public void testHarvestPackageFrameworks(@TempDir File rootDir) throws IOException, ExtenderException { + SpmServiceBuildState buildState = createBuildState(rootDir); + createCommonProducts(buildState); + createFramework(buildState.getProductsDir(), "SpmWrapper", false); + File packageFrameworksDir = new File(buildState.getProductsDir(), "PackageFrameworks"); + createFramework(packageFrameworksDir, "SentryDynamic", true); + // a name that also exists at the products root is classified once, root wins + createFramework(packageFrameworksDir, "Sentry", true); + + ResolvedPackages resolved = ResolvedPackages.harvest(buildState, "15.0", null, null); + + assertEquals(List.of("FirebaseAnalytics", "Sentry", "SpmWrapper", "SentryDynamic"), resolved.getFrameworks()); + assertEquals(List.of( + buildState.getProductsDir().getAbsolutePath(), + packageFrameworksDir.getAbsolutePath()), + resolved.getFrameworksSearchPaths()); + + assertEquals(2, resolved.getDynamicFrameworks().size()); + assertTrue(resolved.getDynamicFrameworks().stream().anyMatch(f -> f.getName().equals("SentryDynamic.framework") + && f.getParentFile().getName().equals("PackageFrameworks"))); + assertTrue(resolved.getDynamicFrameworks().stream().anyMatch(f -> f.getName().equals("Sentry.framework") + && !f.getParentFile().getName().equals("PackageFrameworks"))); + } + @Test public void testHarvestWithoutLinkInfoAndLockFile(@TempDir File rootDir) throws IOException, ExtenderException { SpmServiceBuildState buildState = createBuildState(rootDir);